GUI bugfixes, code cleanup

USAGE.md

@@ -9,7 +9,6 @@ Main endpoints are:
 - `/metrics` or `/f5tt/metrics`
 
 For `/instances` and `/f5tt/instances` endpoints if the request includes the `Accept-Encoding: gzip` header the response will be compressed
-The `/` endpoint displays a basic HTML view of generated JSON files
 
 A sample Postman collection is [available here](/contrib/postman)
 
@@ -21,8 +20,6 @@ the `type` query string parameter can be used to retrieve a logical view of the
 
 | Output type | URI | Description |
 |---|:---|:---|
-| HTML | / | Full data visualization |
-| HTML | /?type=CVE | CVE data visualization |
 | JSON | /instances | NGINX instances inventory and CVE details |
 | JSON | /instances?type=CVE | NGINX instances CVE details |
 | JSON | /instances?type=timebased&month=M&slot=N |M = 0 to get time-based usage for the current month, -1 for previous month (defaults to -1 if not specified) - N = Aggregation based on N-hours timeslot (defaults to 4 if not specified) |
@@ -145,8 +142,3 @@ Authorization: Basic YWE6YmI=
 ## Visualization
 
 See [Grafana](/contrib/grafana)
-
-
-## Sample report e-mail
-
-<img src="./images/reporting-mail.png"/>

contrib/GUI/README.md

@@ -36,7 +36,6 @@ $ ./secondsight-gui.sh -c start
 -> Deploying Second Sight GUI
 Creating network "secondsight-gui_default" with the default driver
 Creating volume "secondsight-gui_postgres_data" with default driver
-Creating bigiq-collector ... done
 Creating postgres        ... done
 Creating f5tt            ... done
 Creating init-db         ... done
@@ -61,13 +60,11 @@ Stopping nginx           ... done
 Stopping secondsight-gui ... done
 Stopping f5tt            ... done
 Stopping postgres        ... done
-Stopping bigiq-collector ... done
 Removing nginx           ... done
 Removing secondsight-gui ... done
 Removing init-db         ... done
 Removing f5tt            ... done
 Removing postgres        ... done
-Removing bigiq-collector ... done
 Removing network secondsight-gui_default
 $
 ```

contrib/GUI/psql/psql-init.sql

@@ -146,7 +146,7 @@ where
 insert into config (name,value) values
 ('dataplane.address','http://nim.test.lab'),
 ('dataplane.type','NGINX Management Suite'),
-('f5tt.helper.address','http://bigiq-collector:5001'),
+('f5tt.helper.address','http://f5tt:5001'),
 ('f5tt.address','http://f5tt:5000');
 
 insert into audit_types (id,tag,description) values

contrib/GUI/secondsight-gui.yaml

@@ -32,13 +32,6 @@ services:
       postgres:
         condition: service_healthy
 
-  bigiq-collector:
-    image: fiorucci/bigiq-collector:latest
-    container_name: "bigiq-collector"
-    restart: always
-    ports:
-      - "5001:5001"
-
   f5tt:
     image: fiorucci/f5-telemetry-tracker:latest
     container_name: "f5tt"
@@ -47,13 +40,11 @@ services:
       - "5000:5000"
     environment:
       - DATAPLANE_TYPE=BIG_IQ
-      - DATAPLANE_FQDN=http://bigiq-collector:5001
+      - DATAPLANE_FQDN=http://127.0.0.1:5001
       - DATAPLANE_USERNAME="notused"
       - DATAPLANE_PASSWORD="notused"
       ### Optional NIST API Key for CVE tracking (https://nvd.nist.gov/developers/request-an-api-key)
       #- NIST_API_KEY=xxxxxxx
-    depends_on:
-      - bigiq-collector
 
   secondsight-gui:
     image: fiorucci/secondsight-gui:latest

contrib/bigiq-collect/README.md

@@ -7,14 +7,6 @@ the .tgz file can then be processed offline by Second Sight to build all target
 
 `sampledata.tgz` is provided for testing purposes
 
-## Building the docker image
-
-The docker image can currently be used in REST API-based data postprocessing mode only, and can be built using:
-
-```
-$ docker build --no-cache -t bigiq-collector:latest .
-```
-
 ## Usage - Data collection
 
 - Copy (scp) `bigIQCollect.sh` from your local host to your BIG-IQ CM instance, under `/tmp/`

contrib/bigiq-collect/f5ttfs.py

@@ -111,7 +111,6 @@ def getCheckBillingReport(reportId):
 def getDownloadBillingReport(reportFile):
   return getFileContent("utilitybilling-billingreport-"+reportFile)
 
-
 @app.route('/mgmt/ap/query/v1/tenants/default/products/device/metric-query', methods=['POST'])
 def getDeviceMetric():
   content = request.get_json(silent=True)

f5tt/Dockerfile

@@ -1,20 +1,26 @@
-FROM ubuntu:latest
-RUN apt-get update
+FROM ubuntu:20.04
 
-RUN apt-get install -y -q build-essential python3-pip python3-dev python3-simplejson git nano curl
-RUN pip3 install --upgrade virtualenv
+ENV TZ=Europe/Rome
+ENV DEBIAN_FRONTEND=noninteractive
 
-RUN mkdir deployment
-RUN virtualenv /deployment/env/
-RUN /deployment/env/bin/python -m pip install --upgrade pip
-RUN /deployment/env/bin/pip3 install fastapi uvicorn requests jinja2 json2html clickhouse-driver
+RUN apt-get update && \
+        apt-get install -y -q build-essential python3-pip python3-dev python3-simplejson git nano curl && \
+        pip3 install fastapi uvicorn requests clickhouse-driver python-dateutil && \
+        # f5ttfs.py
+        pip3 install flask && \
+        mkdir deployment
 
 WORKDIR /deployment
+
+COPY f5ttfs.py .
 COPY app.py .
 COPY nms.py .
 COPY bigiq.py .
 COPY cveDB.py .
 COPY f5ttCH.py .
 COPY utils.py .
+COPY startup.sh .
+
+RUN chmod +x startup.sh
 
-CMD env/bin/python3 app.py
+CMD ./startup.sh

f5tt/app.py

@@ -3,7 +3,6 @@
 from fastapi import FastAPI, Response, Request
 from fastapi.responses import JSONResponse,StreamingResponse
 from typing import Optional
-from json2html import *
 import os
 import sys
 import ssl
@@ -139,7 +138,6 @@ def scheduledEmail(email_server, email_server_port, email_server_type, email_aut
 # Returns stats in json format
 @app.get("/instances")
 @app.get("/f5tt/instances")
-@app.get("/")
 def getInstances(request: Request,type: Optional[str] = None,month: Optional[int] = -1,slot: Optional[int] = 4):
     if nc_mode == 'NGINX_MANAGEMENT_SYSTEM':
         if type == None:
@@ -171,13 +169,8 @@ def getInstances(request: Request,type: Optional[str] = None,month: Optional[int
           code = 404
 
     # Web UI
-    if request.url.path == '/':
-      attributes = 'id=\"info-table\" class=\"table table-bordered table-hover\"'
-      f5tt_output = json2html.convert(json=reply,table_attributes=attributes)
-      f5tt_output_media_type = 'text/html'
-    else:
-      f5tt_output = reply
-      f5tt_output_media_type = 'application/json'
+    f5tt_output = reply
+    f5tt_output_media_type = 'application/json'
 
     # gzip responses supported if the client sends header "Accept-Encoding: gzip"
     responseSent = False
@@ -193,10 +186,7 @@ def getInstances(request: Request,type: Optional[str] = None,month: Optional[int
           return Response(content=deflatedReply,media_type=f5tt_output_media_type,headers={ 'Content-Encoding': 'gzip' })
 
     if responseSent == False:
-        if request.url.path == '/':
-          return Response(content=f5tt_output,media_type=f5tt_output_media_type,headers={ 'Content-Type': 'text/html' })
-        else:
-          return JSONResponse(content=f5tt_output,status_code=code)
+        return JSONResponse(content=f5tt_output,status_code=code)
 
 
 # Returns stats in prometheus format

f5tt/bigiq.py

@@ -9,7 +9,6 @@
 import smtplib
 import urllib3.exceptions
 import base64
-import pandas as pd
 from io import BytesIO
 from requests import Request, Session
 from requests.packages.urllib3.exceptions import InsecureRequestWarning

f5tt/cveDB.py

@@ -110,9 +110,9 @@ def getF5(product="*",version="*"):
               cveId=cve['cve']['CVE_data_meta']['ID']
               cveUrl=cve['cve']['references']['reference_data'][0]['url']
               cveDesc=cve['cve']['description']['description_data'][0]['value']
-              cveBaseSeverity=cve['impact']['baseMetricV3']['cvssV3']['baseSeverity']
-              cveBaseScore=cve['impact']['baseMetricV3']['cvssV3']['baseScore']
-              cveExplScore=cve['impact']['baseMetricV2']['exploitabilityScore']
+              cveBaseSeverity=cve['impact']['baseMetricV3']['cvssV3']['baseSeverity'] if 'baseMetricV3' in cve['impact'] else ''
+              cveBaseScore=cve['impact']['baseMetricV3']['cvssV3']['baseScore'] if 'baseMetricV3' in cve['impact'] else ''
+              cveExplScore=cve['impact']['baseMetricV2']['exploitabilityScore'] if 'baseMetricV2' in cve['impact'] else ''
 
               if cveId not in matchingCVE:
                 matchingCVE[cveId]={"id":cveId,"url":cveUrl,"description":cveDesc,"baseSeverity":cveBaseSeverity,"baseScore":cveBaseScore,"exploitabilityScore":cveExplScore}