Merge branch 'develop' of https://github.com/MLOps-essi-upc/MLOps_Whe…

…reIsWally into develop

.pylintrc

@@ -1,7 +1,4 @@
 
-[MASTER]
-disable=
-    C0114,C0115, C0116
 [TYPECHECK]
 ignored-modules=numpy,torch,cv2,src
 ignored-classes=numpy,torch,cv2

metrics/track_total_emission.py

@@ -2,5 +2,5 @@
 
 emissions = pd.read_csv('./emissions.csv')
 total_emissions = emissions['emissions'].sum() #In kg eq. CO2
-with open('total_emission.txt', 'w') as f:
-    f.write(str(total_emissions)+' kg eq. Co2')
+with open('total_emission.txt', 'w',  encoding='utf-8') as f:
+    f.write(str(total_emissions)+' kg eq. Co2')

src/__init__.py

@@ -1,3 +1,6 @@
+"""
+Src init
+"""
 from pathlib import Path
 
 from dotenv import load_dotenv

src/app/backend/api.py

@@ -28,7 +28,7 @@
 )
 
 
-Instrumentator().instrument(app).expose(app) #Prometheus metric tracking
+Instrumentator().instrument(app).expose(app)  # Prometheus metric tracking
 
 
 def construct_response(f):
@@ -78,7 +78,6 @@ async def wrap(request: Request, *args, **kwargs):
     return wrap
 
 
-
 @app.on_event("startup")
 def _load_models():
     """Loads all pickled models found in `MODELS_DIR` and adds them to `models_list`"""
@@ -113,6 +112,7 @@ def _index(request: Request):
     }
     return response
 
+
 @app.get("/models", tags=["Prediction"])
 @construct_response
 def _get_models_list(request: Request, type: str = None):

src/app/frontend/web/apiConstants.py

@@ -1,3 +1,4 @@
+"""Module that defines api constants"""
 # API_ROOT_URL="http://localhost:5000/"
 API_ROOT_URL = "http://10.4.41.34:5000/"
 PREDICT = API_ROOT_URL + "predict/"

src/app/frontend/web/pages/views.py

@@ -39,5 +39,6 @@ def upload_image(request):
 
     return JsonResponse({"status": "error"})
 
+
 def home(request):
     return render(request, "pages/home.html", {})

src/features/__init__.py

@@ -0,0 +1,3 @@
+"""
+Features init
+"""

src/features/process_data.py

@@ -1,3 +1,4 @@
+"""Module process data"""
 # importing libraries
 import os
 import shutil
@@ -12,6 +13,7 @@
 
 
 def noise_removal(img_path):
+    """Function for noise removal."""
     # Reading image from folder where it is stored
     img = cv2.imread(img_path)
     # denoising of image saving it into dst image
@@ -21,26 +23,36 @@ def noise_removal(img_path):
 
 
 def rgb2gray(img_path):
+    """Function rgb2gray."""
     img = cv2.imread(img_path)
     gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
     # blurred = cv2.GaussianBlur(gray, (5, 5), 0)
     return gray
 
 def get_images_path(directory_name):
+    """Function that gets images path."""
     imgs_path={}
     for file in listdir(directory_name):
         if isfile(join(directory_name, file)):
             imgs_path[file] = join(directory_name /file)
     return imgs_path
 
 def auto_canny_edge_detection(path, sigma=0.33):
+    """
+    Perform automatic Canny edge detection on an image.
+    This function reads an image from the given file path, then applies the Canny edge
+    detection algorithm with threshold values determined automatically. The thresholds are
+    set based on the median pixel intensity of the image, adjusted by a factor defined by
+    the 'sigma' parameter.
+    """
     image = cv2.imread(path)
     median = np.median(image)
     lower_value = int(max(0, (1.0-sigma) * median))
     upper_value = int(min(255, (1.0+sigma) * median))
     return cv2.Canny(image, lower_value, upper_value)
 
 def image_processing(src_imgs_dir,dst_imgs_dir):
+    """Function that processes images"""
     imgs_path=get_images_path(src_imgs_dir)
 
     for image_name,path in imgs_path.items():
@@ -49,13 +61,23 @@ def image_processing(src_imgs_dir,dst_imgs_dir):
         cv2.imwrite(str(dst_imgs_dir / image_name),gray)
 
 def copy_labels(src,dest):
+    """
+    Copy all files from a source directory to a destination directory.
+    This function lists all files in the source directory specified by 'src',
+    and then copies each file to the destination directory specified by 'dest'.
+    It does not copy subdirectories or their contents, only files directly
+    within the source directory.
+    """
     src_files = os.listdir(src)
     for file_name in src_files:
         full_file_name = os.path.join(src, file_name)
         if os.path.isfile(full_file_name):
             shutil.copy(full_file_name, dest)
 
 def main():
+    """
+    Main function
+    """
     train_imgs_dir=RAW_DATA_DIR / "train/images"
     valid_imgs_dir=RAW_DATA_DIR / "valid/images"
     test_imgs_dir=RAW_DATA_DIR / "test/images"

src/models/__init__.py

@@ -1,3 +1,6 @@
+"""
+Init models
+"""
 from dotenv import load_dotenv
 
 load_dotenv()

tests/api_test/test_api.py

@@ -1,3 +1,4 @@
+"""Module for testing the api"""
 # pylint: disable=redefined-outer-name
 import os
 from http import HTTPStatus
@@ -10,10 +11,12 @@
 
 @pytest.fixture(scope="module", autouse=True)
 def client():
+    """Function to get the client"""
     with TestClient(app) as client:
         return client
 
 def test_get_main(client):
+    """Function to test a get to the main page"""
     response = client.get("/")
     assert response.status_code == HTTPStatus.OK.value
     response_body = response.json()
@@ -25,6 +28,7 @@ def test_get_main(client):
 
 
 def test_predict_with_invalid_file(client):
+    """Function to test a prediction with an invalid file"""
     current_dir = os.path.dirname(os.path.abspath(__file__))
     file_path = os.path.join(current_dir, 'testing_file.txt')
     with open(file_path, 'rb') as file:
@@ -34,6 +38,7 @@ def test_predict_with_invalid_file(client):
 
 
 def test_predict_with_valid_image_wally_found_model_all(client):
+    """Function to test a prediction with a valid image using the model that finds all characters"""
     current_dir = os.path.dirname(os.path.abspath(__file__))
     file_path = os.path.join(current_dir, 'testing_img.png')
     with open(file_path, 'rb') as file:
@@ -47,6 +52,7 @@ def test_predict_with_valid_image_wally_found_model_all(client):
 
 
 def test_predict_with_valid_image_wally_not_found(client):
+    """Function to test a prediction with a valid image that does not find wally"""
     current_dir = os.path.dirname(os.path.abspath(__file__))
     file_path = os.path.join(current_dir, 'testing_not_found_img.jpeg')
     with open(file_path, 'rb') as file:
@@ -61,6 +67,7 @@ def test_predict_with_valid_image_wally_not_found(client):
 
 
 def test_predict_with_valid_image_model_wally(client):
+    """Function to test a prediction with a valid image using the model that only finds wally"""
     current_dir = os.path.dirname(os.path.abspath(__file__))
     file_path = os.path.join(current_dir, 'testing_img.png')
     with open(file_path, 'rb') as file:
@@ -71,4 +78,3 @@ def test_predict_with_valid_image_model_wally(client):
     assert "boxes" in  response_body
     assert 'orig_shape' in response_body['boxes']
     assert len(response_body['boxes']['orig_shape']) > 0
-

tests/code_test/unit_tests/test_labels_data.py

@@ -1,3 +1,4 @@
+"""Module for unit testing data labels."""
 # pylint: disable=redefined-outer-name
 from os import listdir
 from os.path import isfile, join
@@ -10,39 +11,48 @@
 
 @pytest.fixture
 def validation_data_path():
+    """Function that validates data path."""
     return PROCESSED_DATA_DIR
 
 @pytest.fixture
 def labels_path():
+    """Function that gets labels path."""
     return PROCESSED_DATA_DIR / "valid/labels"
 
 @pytest.fixture
 def raw_images_path():
+    """Function that gets raw images path."""
     images_path=RAW_DATA_DIR / "train/images"
     data_path = [join(images_path /f) for f in listdir(images_path) if isfile(join(images_path, f))]
     return data_path
 
 #validation_labels function's tests
 def test_validation_labels_obj_not_none(labels_path):
+    """Function that tests validation labels"""
     assert get_validation_labels(labels_path) is not None
 
 
 def test_validation_labels_is_right_data_type(labels_path):
+    """Function that tests validation labels"""
     assert isinstance(get_validation_labels(labels_path), dict)
 
 
 def test_validation_labels_obj_is_none():
+    """Function that tests validation labels"""
     assert get_validation_labels(None) is None
 
 
 def test_validation_labels_path_fails(labels_path):
+    """Function that tests validation labels"""
     with pytest.raises(FileNotFoundError):
         assert isinstance(get_validation_labels(labels_path / "poo"), dict)
 
 
 #load_validation_data function's tests
 def test_validation_data_is_all_objects_returned(validation_data_path):
+    """Function that tests validation data is all objects"""
     assert len(load_validation_data(validation_data_path))==2
 
 def test_validation_data_is_right_data_type(validation_data_path):
+    """Function that tests that validation data has the right data type"""
     assert isinstance(load_validation_data(validation_data_path)[0], list)

tests/code_test/unit_tests/test_noise_removal.py

@@ -1,3 +1,4 @@
+"""Module for unit testing noise removal."""
 # pylint: disable=redefined-outer-name
 from os import listdir
 from os.path import isfile, join
@@ -12,25 +13,31 @@
 
 @pytest.fixture
 def raw_images_path():
+    """Functions that gets raw images path"""
     images_path=RAW_DATA_DIR / "train/images"
     data_path = [join(images_path /f) for f in listdir(images_path) if isfile(join(images_path, f))]
     return data_path
 
 
 def is_similar(image1, image2):
+    """Functions that returns if two images are similar in terms of both their
+    shape and the pixel value """
     return image1.shape == image2.shape and not np.bitwise_xor(image1,image2).any()
 
 def test_noise_removal_is_obj_not_none(raw_images_path):
+    """Functions that tests if noise removal is not none"""
     denoised=noise_removal(raw_images_path[0])
     assert denoised is not None
 
 
 def test_noise_removal_is_equal_shape(raw_images_path):
+    """Functions that tests if noise removal is equal to shape"""
     img = cv2.imread(raw_images_path[0])
     denoised=noise_removal(raw_images_path[0])
     assert img.shape == denoised.shape
 
 def test_noise_removal_is_not_same_image(raw_images_path):
+    """Functions that tests if noise removal is not the same image"""
     img = cv2.imread(raw_images_path[0])
     denoised=noise_removal(raw_images_path[0])
     assert (np.bitwise_xor(img,denoised).any())

tests/model_test/model_test.py

@@ -1,3 +1,4 @@
+"""Module for model testing."""
 # pylint: disable=redefined-outer-name
 import pickle
 
@@ -11,14 +12,17 @@
 
 @pytest.fixture
 def yov8_model():
+    """Function that loads yolo8 model."""
     with open(MODELS_DIR / "yolov8_model.pkl", "rb") as file:
         return pickle.load(file)
 
 @pytest.fixture
 def get_validation_data():
+    """Function that gets validation data."""
     return load_validation_data(PROCESSED_DATA_DIR)
 
 def test_model_expected_value(yov8_model, get_validation_data):
+    """Function that tests model expected value."""
     x, _ = get_validation_data  # pylint: disable=invalid-name
 
     val_predictions = yov8_model.predict(x,imgsz=640, conf=0.0033) # pylint: disable=unused-variable