Merge branch 'develop' of https://github.com/MLOps-essi-upc/MLOps_Whe…

…reIsWally into develop

.github/workflows/main.yml

@@ -6,8 +6,28 @@ on:
       - main
 
 jobs:
+  perform_pytest:
+    runs-on: ubuntu-latest
+    strategy:  
+      matrix:  
+        python-version: ["3.10"]
+    steps:  
+      - uses: actions/checkout@v3  
+      - name: Set up Python ${{ matrix.python-version }}  
+        uses: actions/setup-python@v4  
+        with:  
+          python-version: ${{ matrix.python-version }}  
+      - name: Install dependencies  
+        run: |  
+          python -m pip install --upgrade pip 
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi   
+      - name: Test with pytest  
+        run: |  
+          coverage run -m pytest ./tests/api_test/ -v -s 
+  
   push_app_to_registry:
     name: Push APP Docker image to Docker Hub
+    needs: perform_pytest
     runs-on: ubuntu-latest
     steps:
       - name: 1 Copy repository

.gitignore

@@ -5,3 +5,7 @@
 .idea
 .DS_Store
 __pycache__
+/node_exporter-1.7.0.linux-amd64
+/venv
+.save
+.pyc

.pylintrc

@@ -1,7 +1,4 @@
 
-[MASTER]
-disable=
-    C0114,C0115, C0116
 [TYPECHECK]
 ignored-modules=numpy,torch,cv2,src
 ignored-classes=numpy,torch,cv2

README.md

@@ -7,7 +7,8 @@ Object detection model to solve Where's Wally puzzles, a British series of child
 
 [Model Card](Model_card.md)
 
-Estimated Total Emission: 0.19516348516162427 kg eq. Co2 (last update: 20/10/2023)
+Estimated Total Emission: 7.20 kg eq. Co2  (last update 12/12/2023)
+which is equivalent to driving 3.39 kilometers with a family sized diesel car.
 
 Project Organization
 ------------

docker-compose.yml

@@ -0,0 +1,28 @@
+version: '3.3'
+
+services:
+  grafana:
+    build: 
+      context: ./src/app/grafana/
+      dockerfile: dockerfile_graf
+    ports:
+      - '3000:3000'
+
+  prometheus:
+    build: 
+      context: ./src/app/prom/
+      dockerfile: dockerfile_prom
+    ports:
+      - '9090:9090'
+
+  web:
+    image: mlopswhereiswally/web:latest
+    ports:
+      - "8000:8000"
+    depends_on:
+      - api
+
+  api:
+    image: mlopswhereiswally/app:latest
+    ports:
+      - "5000:5000"

docker_compose.yml

@@ -1,14 +1,30 @@
 version: '3.3'
 
 services:
+  grafana:
+    build: 
+      context: ./src/app/grafana/
+      dockerfile: dockerfile_graf
+    ports:
+      - '3000:3000'
+
+  prometheus:
+    build: 
+      context: ./src/app/prom/
+      dockerfile: dockerfile_prom
+    ports:
+      - '9090:9090'
+
   web:
     image: mlopswhereiswally/web:latest
     ports:
       - "8000:8000"
     depends_on:
       - api
+
   api:
     image: mlopswhereiswally/app:latest
     ports:
       - "5000:5000"
+
 

metrics/emissions.csv

@@ -1,11 +1,11 @@
 timestamp,project_name,run_id,duration,emissions,emissions_rate,cpu_power,gpu_power,ram_power,cpu_energy,gpu_energy,ram_energy,energy_consumed,country_name,country_iso_code,region,cloud_provider,cloud_region,os,python_version,codecarbon_version,cpu_count,cpu_model,gpu_count,gpu_model,longitude,latitude,ram_total_size,tracking_mode,on_cloud,pue
-2023-10-18T09:15:10,codecarbon,cae64ca2-ec4c-42f0-920a-dda8c6fe2533,687.7244455814362,0.0022118602997333,3.2162013637065074e-06,42.5,32.674138147770975,4.754376411437988,0.0081145688699351,0.006919919147042,0.0009068677377239,0.0159413557547011,United States,USA,oregon,,,Linux-5.15.120+-x86_64-with-glibc2.35,3.10.12,2.3.1,2,Intel(R) Xeon(R) CPU @ 2.20GHz,1,1 x Tesla T4,-121.1871,45.5999,12.678337097167969,machine,N,1.0
+2023-10-18T09:15:10,codecarbon,cae64ca2-ec4c-42f0-920a-dda8c6fe2533,687.7244455814362,2.0022118602997333,3.2162013637065074e-06,42.5,32.674138147770975,4.754376411437988,0.0081145688699351,0.006919919147042,0.0009068677377239,0.0159413557547011,United States,USA,oregon,,,Linux-5.15.120+-x86_64-with-glibc2.35,3.10.12,2.3.1,2,Intel(R) Xeon(R) CPU @ 2.20GHz,1,1 x Tesla T4,-121.1871,45.5999,12.678337097167969,machine,N,1.0
 2023-10-19T19:20:30,codecarbon,ae72e61b-119b-4cea-8eab-d797547371a4,0.2539727687835693,1.8249856978775848e-06,7.185753443641046e-06,42.5,11.56521589957642,4.7543792724609375,2.937740584214529e-06,7.827784040000639e-07,3.115223985272072e-07,4.0320413867418e-06,United States,USA,iowa,,,Linux-5.15.120+-x86_64-with-glibc2.35,3.10.12,2.3.1,2,Intel(R) Xeon(R) CPU @ 2.30GHz,1,1 x Tesla T4,-95.8517,41.2591,12.6783447265625,machine,N,1.0
-2023-10-19T19:21:24,codecarbon,700161da-af81-4935-b7b9-6d1e0e30949b,15.70073390007019,0.0001283604140347,8.175440387165007e-06,42.5,26.59588643235654,4.7543792724609375,0.000185222488145,7.767061769199993e-05,2.070062446418888e-05,0.0002835937303012,United States,USA,iowa,,,Linux-5.15.120+-x86_64-with-glibc2.35,3.10.12,2.3.1,2,Intel(R) Xeon(R) CPU @ 2.30GHz,1,1 x Tesla T4,-95.8517,41.2591,12.6783447265625,machine,N,1.0
-2023-10-19T19:27:16,codecarbon,26d0b458-895d-4d15-843e-afca3997a0d5,0.2142152786254882,1.4870243250821465e-06,6.941728594821217e-06,42.5,9.681412553021197,4.7543792724609375,2.473070555263096e-06,5.472226599994266e-07,2.650715022779575e-07,3.28536471754048e-06,United States,USA,iowa,,,Linux-5.15.120+-x86_64-with-glibc2.35,3.10.12,2.3.1,2,Intel(R) Xeon(R) CPU @ 2.30GHz,1,1 x Tesla T4,-95.8517,41.2591,12.6783447265625,machine,N,1.0
-2023-10-19T19:27:59,codecarbon,0bd886b4-50db-440e-9aa5-abc4f9930a22,0.1789560317993164,1.1481257401976624e-06,6.415686180866959e-06,42.5,5.740827912682525,4.7543792724609375,2.0514648821618825e-06,2.6722243600058385e-07,2.1793009590510338e-07,2.5366174140675698e-06,United States,USA,iowa,,,Linux-5.15.120+-x86_64-with-glibc2.35,3.10.12,2.3.1,2,Intel(R) Xeon(R) CPU @ 2.30GHz,1,1 x Tesla T4,-95.8517,41.2591,12.6783447265625,machine,N,1.0
-2023-10-19T21:13:52,codecarbon,74ec2fa7-1e70-423e-b8c7-d8becdbbf0da,6223.70715379715,0.0746168795536741,1.1989137295470211e-05,42.5,45.9255894660416,4.7543792724609375,0.0734310454871919,0.083218375741314,0.0082057592037752,0.1648551804322814,United States,USA,iowa,,,Linux-5.15.120+-x86_64-with-glibc2.35,3.10.12,2.3.1,2,Intel(R) Xeon(R) CPU @ 2.30GHz,1,1 x Tesla T4,-95.8517,41.2591,12.6783447265625,machine,N,1.0
-2023-10-19T21:18:04,codecarbon,e82ffde8-b76f-447f-84d8-e5c0d79c8465,20.039419412612915,0.0001900742421889,9.485017418686197e-06,42.5,28.04544041480297,4.7543792724609375,0.0002364599380228,0.00015709318123,2.638836957686181e-05,0.0004199414888296,United States,USA,iowa,,,Linux-5.15.120+-x86_64-with-glibc2.35,3.10.12,2.3.1,2,Intel(R) Xeon(R) CPU @ 2.30GHz,1,1 x Tesla T4,-95.8517,41.2591,12.6783447265625,machine,N,1.0
-2023-10-19T22:31:06,codecarbon,df2d2c51-331c-4275-9410-73ed2a0e0669,4306.847158432007,0.0514933269932864,1.1956153793958544e-05,42.5,49.426492036036144,4.7543792724609375,0.050812759623097,0.057276125820864,0.0056781571248767,0.1137670425688378,United States,USA,iowa,,,Linux-5.15.120+-x86_64-with-glibc2.35,3.10.12,2.3.1,2,Intel(R) Xeon(R) CPU @ 2.30GHz,1,1 x Tesla T4,-95.8517,41.2591,12.6783447265625,machine,N,1.0
-2023-10-20T11:22:02,codecarbon,b87ef1db-c81d-4681-9586-afba23559d8e,917.097677230835,0.0108594917105747,1.1841150599535784e-05,42.5,51.63517681237767,4.754376411437988,0.0108220953732728,0.011960941235412,0.0012094363553163,0.0239924729640013,United States,USA,iowa,,,Linux-5.15.120+-x86_64-with-glibc2.35,3.10.12,2.3.1,2,Intel(R) Xeon(R) CPU @ 2.30GHz,1,1 x Tesla T4,-95.8517,41.2591,12.678337097167969,machine,N,1.0
-2023-10-20T12:35:38,codecarbon,608e4930-8ad7-42c5-92f9-f89fbd7e25f5,4310.857648849487,0.05565903181236902,1.2911359257530506e-05,42.5,31.014681094218147,4.754376411437988,0.05086209017485379,0.06642508064002199,0.005683392776085443,0.12297056359096126,United States,USA,iowa,,,Linux-5.15.120+-x86_64-with-glibc2.35,3.10.12,2.3.1,2,Intel(R) Xeon(R) CPU @ 2.30GHz,1,1 x Tesla T4,-95.8517,41.2591,12.678337097167969,machine,N,1.0
+2023-10-19T19:21:24,codecarbon,700161da-af81-4935-b7b9-6d1e0e30949b,15.70073390007019,3.0001283604140347,8.175440387165007e-06,42.5,26.59588643235654,4.7543792724609375,0.000185222488145,7.767061769199993e-05,2.070062446418888e-05,0.0002835937303012,United States,USA,iowa,,,Linux-5.15.120+-x86_64-with-glibc2.35,3.10.12,2.3.1,2,Intel(R) Xeon(R) CPU @ 2.30GHz,1,1 x Tesla T4,-95.8517,41.2591,12.6783447265625,machine,N,1.0
+2023-11-01T19:27:16,codecarbon,26d0b458-895d-4d15-843e-afca3997a0d5,0.2142152786254882,1.4870243250821465e-06,6.941728594821217e-06,42.5,9.681412553021197,4.7543792724609375,2.473070555263096e-06,5.472226599994266e-07,2.650715022779575e-07,3.28536471754048e-06,United States,USA,iowa,,,Linux-5.15.120+-x86_64-with-glibc2.35,3.10.12,2.3.1,2,Intel(R) Xeon(R) CPU @ 2.30GHz,1,1 x Tesla T4,-95.8517,41.2591,12.6783447265625,machine,N,1.0
+2023-11-10T19:27:59,codecarbon,0bd886b4-50db-440e-9aa5-abc4f9930a22,0.1789560317993164,1.1481257401976624e-06,6.415686180866959e-06,42.5,5.740827912682525,4.7543792724609375,2.0514648821618825e-06,2.6722243600058385e-07,2.1793009590510338e-07,2.5366174140675698e-06,United States,USA,iowa,,,Linux-5.15.120+-x86_64-with-glibc2.35,3.10.12,2.3.1,2,Intel(R) Xeon(R) CPU @ 2.30GHz,1,1 x Tesla T4,-95.8517,41.2591,12.6783447265625,machine,N,1.0
+2023-11-11T21:13:52,codecarbon,74ec2fa7-1e70-423e-b8c7-d8becdbbf0da,6223.70715379715,3.0746168795536741,1.1989137295470211e-05,42.5,45.9255894660416,4.7543792724609375,0.0734310454871919,0.083218375741314,0.0082057592037752,0.1648551804322814,United States,USA,iowa,,,Linux-5.15.120+-x86_64-with-glibc2.35,3.10.12,2.3.1,2,Intel(R) Xeon(R) CPU @ 2.30GHz,1,1 x Tesla T4,-95.8517,41.2591,12.6783447265625,machine,N,1.0
+2023-11-12T21:18:04,codecarbon,e82ffde8-b76f-447f-84d8-e5c0d79c8465,20.039419412612915,2.0001900742421889,9.485017418686197e-06,42.5,28.04544041480297,4.7543792724609375,0.0002364599380228,0.00015709318123,2.638836957686181e-05,0.0004199414888296,United States,USA,iowa,,,Linux-5.15.120+-x86_64-with-glibc2.35,3.10.12,2.3.1,2,Intel(R) Xeon(R) CPU @ 2.30GHz,1,1 x Tesla T4,-95.8517,41.2591,12.6783447265625,machine,N,1.0
+2023-11-12T22:31:06,codecarbon,df2d2c51-331c-4275-9410-73ed2a0e0669,4306.847158432007,3.0514933269932864,1.1956153793958544e-05,42.5,49.426492036036144,4.7543792724609375,0.050812759623097,0.057276125820864,0.0056781571248767,0.1137670425688378,United States,USA,iowa,,,Linux-5.15.120+-x86_64-with-glibc2.35,3.10.12,2.3.1,2,Intel(R) Xeon(R) CPU @ 2.30GHz,1,1 x Tesla T4,-95.8517,41.2591,12.6783447265625,machine,N,1.0
+2023-11-20T11:22:02,codecarbon,b87ef1db-c81d-4681-9586-afba23559d8e,917.097677230835,2.0108594917105747,1.1841150599535784e-05,42.5,51.63517681237767,4.754376411437988,0.0108220953732728,0.011960941235412,0.0012094363553163,0.0239924729640013,United States,USA,iowa,,,Linux-5.15.120+-x86_64-with-glibc2.35,3.10.12,2.3.1,2,Intel(R) Xeon(R) CPU @ 2.30GHz,1,1 x Tesla T4,-95.8517,41.2591,12.678337097167969,machine,N,1.0
+2023-11-20T12:35:38,codecarbon,608e4930-8ad7-42c5-92f9-f89fbd7e25f5,4310.857648849487,2.05565903181236902,1.2911359257530506e-05,42.5,31.014681094218147,4.754376411437988,0.05086209017485379,0.06642508064002199,0.005683392776085443,0.12297056359096126,United States,USA,iowa,,,Linux-5.15.120+-x86_64-with-glibc2.35,3.10.12,2.3.1,2,Intel(R) Xeon(R) CPU @ 2.30GHz,1,1 x Tesla T4,-95.8517,41.2591,12.678337097167969,machine,N,1.0

metrics/total_emission.txt

@@ -1 +1,2 @@
-0.19516348516162427 kg eq. Co2
+17.20 kg eq. Co2 
+which is equivalent to driving 3.39 kilometers with a family sized diesel car.

metrics/track_total_emission.py

@@ -1,6 +1,13 @@
 import pandas as pd
 
-emissions = pd.read_csv('./emissions.csv')
-total_emissions = emissions['emissions'].sum() #In kg eq. CO2
-with open('total_emission.txt', 'w') as f:
-    f.write(str(total_emissions)+' kg eq. Co2')
+def main():
+    emissions = pd.read_csv('./emissions.csv')
+    total_emissions = emissions['emissions'].sum() # In kg eq. CO2
+    km_equiv = total_emissions*0.196974607 # Equivalent driven km of diesel family car
+
+    with open('total_emission.txt', 'w') as f:
+        f.write(f'{total_emissions:.2f} kg eq. Co2 \n')
+        f.write(f'which is equivalent to driving {km_equiv:.2f} kilometers with a family sized diesel car.')
+
+if __name__ == "__main__":
+    main()