Merge pull request #34 from se4ai2324-uniba/tools

Tools
se4ai2324-uniba · Dec 16, 2023 · 3d029fc · 3d029fc
2 parents e93adb5 + f9cb0e0
commit 3d029fc
Show file tree

Hide file tree

Showing 33 changed files with 157 additions and 9 deletions.
diff --git a/.gitignore b/.gitignore
@@ -98,6 +98,8 @@ mlruns/0/meta.yaml
 # python virtual environment config
 *.cfg
 
-api_env/
+tools/prometheus-2.48.1/*
+!/tools/prometheus-2.48.1/Dockerfile
+!/tools/prometheus-2.48.1/prometheus.yml
 
-performance_env/
+performance_env/
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -27,3 +27,13 @@ services:
     networks:
       - local
 
+  prometheus:
+    image: simonegramegna/prometheus:latest
+    build:
+      context: ./tools/prometheus-2.48.1
+      dockerfile: Dockerfile
+    ports:
+      - "9090:9090"
+    networks:
+      - local
+
diff --git a/dvc.lock b/dvc.lock
@@ -130,8 +130,8 @@ stages:
     outs:
     - path: models/tuned_rf_model.pkl
       hash: md5
-      md5: f55f87c8ac882a9a661d855f82ca663a
-      size: 703244753
+      md5: 023221cdcda215232b57aec9e91d8f72
+      size: 780659602
   predict:
     cmd:
     - python src/models/predict.py
@@ -146,6 +146,8 @@ stages:
       size: 30782445
     - path: models/tuned_rf_model.pkl
       hash: md5
+      md5: 023221cdcda215232b57aec9e91d8f72
+      size: 780659602
       md5: f55f87c8ac882a9a661d855f82ca663a
       size: 703244753
     - path: src/models/predict.py
@@ -171,5 +173,5 @@ stages:
       size: 155
     - path: reports/confusion_matrix_tuned_rf.json
       hash: md5
-      md5: 5ffa40c22ff9bd5dc8efcc12520b2f29
+      md5: 5e323a96da9771f5495812f7580a1bb6
       size: 155
diff --git a/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/meta.yaml b/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/meta.yaml
@@ -0,0 +1,15 @@
+artifact_uri: file:///D:\Ivan\Desktop\lanubile\MalURLs\mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/artifacts
+end_time: 1702399159339
+entry_point_name: ''
+experiment_id: '573845569101247538'
+lifecycle_stage: active
+run_id: 5d925ffaa68d403b94dc614c913fa5fb
+run_name: delightful-mule-19
+run_uuid: 5d925ffaa68d403b94dc614c913fa5fb
+source_name: ''
+source_type: 4
+source_version: ''
+start_time: 1702399155566
+status: 3
+tags: []
+user_id: 75897e6620fc552361f424ffc621ee0c6c55322f
diff --git a/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/metrics/Weighted_avg_f1 b/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/metrics/Weighted_avg_f1
@@ -0,0 +1 @@
+1702399159334 0.8967420943165961 0
diff --git a/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/metrics/Weighted_avg_precision b/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/metrics/Weighted_avg_precision
@@ -0,0 +1 @@
+1702399159334 0.8968715419097099 0
diff --git a/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/metrics/Weighted_avg_recall b/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/metrics/Weighted_avg_recall
@@ -0,0 +1 @@
+1702399159334 0.89675 0
diff --git a/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/metrics/accuracy b/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/metrics/accuracy
@@ -0,0 +1 @@
+1702399159334 0.89675 0
diff --git a/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/tags/mlflow.parentRunId b/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/tags/mlflow.parentRunId
@@ -0,0 +1 @@
+be57f7deba8f4861a51c101064c1520c
diff --git a/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/tags/mlflow.runName b/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/tags/mlflow.runName
@@ -0,0 +1 @@
+delightful-mule-19
diff --git a/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/tags/mlflow.source.git.commit b/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/tags/mlflow.source.git.commit
@@ -0,0 +1 @@
+972145f000b501ad23b8076920c6f87453208c71
diff --git a/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/tags/mlflow.source.name b/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/tags/mlflow.source.name
@@ -0,0 +1 @@
+src/models/predict.py
diff --git a/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/tags/mlflow.source.type b/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/tags/mlflow.source.type
@@ -0,0 +1 @@
+LOCAL
diff --git a/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/tags/mlflow.user b/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/tags/mlflow.user
@@ -0,0 +1 @@
+75897e6620fc552361f424ffc621ee0c6c55322f
diff --git a/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/tags/model_type b/mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/tags/model_type
@@ -0,0 +1 @@
+random_forest_tuned_model
diff --git a/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/meta.yaml b/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/meta.yaml
@@ -0,0 +1,15 @@
+artifact_uri: file:///D:\Ivan\Desktop\lanubile\MalURLs\mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/artifacts
+end_time: 1702399159341
+entry_point_name: ''
+experiment_id: '573845569101247538'
+lifecycle_stage: active
+run_id: be57f7deba8f4861a51c101064c1520c
+run_name: stylish-hog-713
+run_uuid: be57f7deba8f4861a51c101064c1520c
+source_name: ''
+source_type: 4
+source_version: ''
+start_time: 1702399155271
+status: 3
+tags: []
+user_id: 75897e6620fc552361f424ffc621ee0c6c55322f
diff --git a/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/metrics/Weighted_avg_f1 b/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/metrics/Weighted_avg_f1
@@ -0,0 +1 @@
+1702399155481 0.8929922686914129 0
diff --git a/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/metrics/Weighted_avg_precision b/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/metrics/Weighted_avg_precision
@@ -0,0 +1 @@
+1702399155481 0.8931136098332417 0
diff --git a/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/metrics/Weighted_avg_recall b/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/metrics/Weighted_avg_recall
@@ -0,0 +1 @@
+1702399155481 0.893 0
diff --git a/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/metrics/accuracy b/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/metrics/accuracy
@@ -0,0 +1 @@
+1702399155481 0.893 0
diff --git a/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/tags/mlflow.runName b/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/tags/mlflow.runName
@@ -0,0 +1 @@
+stylish-hog-713
diff --git a/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/tags/mlflow.source.git.commit b/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/tags/mlflow.source.git.commit
@@ -0,0 +1 @@
+972145f000b501ad23b8076920c6f87453208c71
diff --git a/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/tags/mlflow.source.name b/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/tags/mlflow.source.name
@@ -0,0 +1 @@
+src/models/predict.py
diff --git a/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/tags/mlflow.source.type b/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/tags/mlflow.source.type
@@ -0,0 +1 @@
+LOCAL
diff --git a/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/tags/mlflow.user b/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/tags/mlflow.user
@@ -0,0 +1 @@
+75897e6620fc552361f424ffc621ee0c6c55322f
diff --git a/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/tags/model_type b/mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/tags/model_type
@@ -0,0 +1 @@
+random_forest_base_model
diff --git a/mlruns/models/base_rf_model/version-19/meta.yaml b/mlruns/models/base_rf_model/version-19/meta.yaml
@@ -0,0 +1,14 @@
+aliases: []
+creation_timestamp: 1702399155522
+current_stage: None
+description: null
+last_updated_timestamp: 1702399155522
+name: base_rf_model
+run_id: be57f7deba8f4861a51c101064c1520c
+run_link: null
+source: file:///D:\Ivan\Desktop\lanubile\MalURLs\mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/artifacts/models
+status: READY
+status_message: null
+storage_location: file:///D:\Ivan\Desktop\lanubile\MalURLs\mlruns/573845569101247538/be57f7deba8f4861a51c101064c1520c/artifacts/models
+user_id: null
+version: 19
diff --git a/mlruns/models/tuned_rf_model/meta.yaml b/mlruns/models/tuned_rf_model/meta.yaml
@@ -1,5 +1,4 @@
 aliases: {}
 creation_timestamp: 1702729804102
-description: null
-last_updated_timestamp: 1702729804118
+last_updated_timestamp: 1702399159298
 name: tuned_rf_model
diff --git a/mlruns/models/tuned_rf_model/version-19/meta.yaml b/mlruns/models/tuned_rf_model/version-19/meta.yaml
@@ -0,0 +1,14 @@
+aliases: []
+creation_timestamp: 1702399159298
+current_stage: None
+description: null
+last_updated_timestamp: 1702399159298
+name: tuned_rf_model
+run_id: 5d925ffaa68d403b94dc614c913fa5fb
+run_link: null
+source: file:///D:\Ivan\Desktop\lanubile\MalURLs\mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/artifacts/models
+status: READY
+status_message: null
+storage_location: file:///D:\Ivan\Desktop\lanubile\MalURLs\mlruns/573845569101247538/5d925ffaa68d403b94dc614c913fa5fb/artifacts/models
+user_id: null
+version: 19
diff --git a/reports/confusion_matrix_tuned_rf.json b/reports/confusion_matrix_tuned_rf.json
@@ -5,8 +5,8 @@
             189
         ],
         [
-            223,
-            1777
+            224,
+            1776
         ]
     ]
 }
diff --git a/tools/locustfile.py b/tools/locustfile.py
@@ -0,0 +1,22 @@
+import time
+from locust import HttpUser, task, between
+
+class TestUser(HttpUser):
+    wait_time = between(1, 5)
+
+    @task
+    def get_features(self):
+        self.client.post('/get_features', json={"url":"www.google.it"}, name="Get Features")
+        '''x = self.client.post('/get_features', json={"url":"www.google.it"})
+        y = x.json()
+        print(y)'''
+
+    @task
+    def get_models(self):
+        self.client.get('/models', name="Get Models")
+
+    @task
+    def scan(self):
+        self.client.post('/scan', json={"url":"www.google.it","model":"base_rf"}, name="Scan")
+
+
diff --git a/tools/prometheus-2.48.1/Dockerfile b/tools/prometheus-2.48.1/Dockerfile
@@ -0,0 +1,2 @@
+FROM prom/prometheus
+ADD prometheus.yml /etc/prometheus/
diff --git a/tools/prometheus-2.48.1/prometheus.yml b/tools/prometheus-2.48.1/prometheus.yml
@@ -0,0 +1,32 @@
+# my global config
+global:
+  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+# Alertmanager configuration
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+          # - alertmanager:9093
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+  # - "first_rules.yml"
+  # - "second_rules.yml"
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: "prometheus"
+
+    # metrics_path defaults to '/metrics'
+    # scheme defaults to 'http'.
+
+    static_configs:
+      - targets: ["localhost:9090"]
+  - job_name: "docker"
+    static_configs:
+      - targets: ["host.docker.internal:9323"]
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,8 +5,8 @@ @@
             ],
             [
-,
+,
             ]
         ]
     }
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		FROM prom/prometheus
		ADD prometheus.yml /etc/prometheus/