Merge branch 'main' of https://github.com/MLOps-essi-upc/MLOps_WhereI…

…sWally

.gitignore

@@ -4,4 +4,3 @@
 /.idea
 .idea
 .DS_Store
-

data.dvc

@@ -1,5 +1,5 @@
 outs:
     - md5: b61170ca4037816bfbfa03ed2b6381fa.dir
       size: 1954792904
-      path: data
+      path: data/raw
       nfiles: 8023

dvc.yaml

@@ -1,14 +1,39 @@
 stages:
+  process_data:
+    cmd: python -m src.features.process_data
+    deps:
+    - data/raw/train
+    - data/raw/valid
+    - data/raw/test
+
+    outs:
+    - data/processed/train
+    - data/processed/valid
+    - data/processed/test
+
   train:
-    cmd: python src/models/train.py
+    cmd: python -m src.models.train
     deps:
-    - data/yolov8-format
+    - data/processed/train/images
+    - data/processed/train/labels
     - src/models/train.py
-    - src/models/params.yaml
+    - params.yaml
     outs:
     - models/model.pt
     - reports/train_params.yaml:
         cache: false
     metrics:
     - reports/train_metrics.csv:
-        cache: false
+        cache: false
+
+  evaluate:
+    cmd: python -m src.models.evaluate
+    deps:
+    - data/processed/valid
+    - models/model.pt
+    metrics:
+    - metrics/scores.json:
+        cache: false
+
+
+

metrics/scores.json

@@ -0,0 +1,4 @@
+{
+    "map": 0.6000000238418579,
+    "mar_10": 0.6000000238418579
+}

models/.gitignore

@@ -0,0 +1 @@
+/model.pt

src/models/params.yaml → params.yaml

@@ -4,4 +4,4 @@ seed: 0
 imgsz: 200
 batch: 8 
 epochs: 1
-name: 'yolov8s_local_run' # experiment name
+name: 'yolov8s_local_n' # experiment name

reports/train_metrics.csv

@@ -0,0 +1,98 @@
+task: detect
+mode: train
+model: yolov8n.pt
+data: C:\Users\xange\Downloads\MDS2023\MLOPS\Nueva carpeta\MLOps_WhereIsWally\data\processed\data.yaml
+epochs: 1
+patience: 50
+batch: 8
+imgsz: 200
+save: true
+save_period: -1
+cache: false
+device: null
+workers: 8
+project: null
+name: yolov8s_local_n9
+exist_ok: false
+pretrained: true
+optimizer: auto
+verbose: true
+seed: 0
+deterministic: true
+single_cls: false
+rect: false
+cos_lr: false
+close_mosaic: 10
+resume: false
+amp: true
+fraction: 1.0
+profile: false
+freeze: null
+overlap_mask: true
+mask_ratio: 4
+dropout: 0.0
+val: true
+split: val
+save_json: false
+save_hybrid: false
+conf: null
+iou: 0.7
+max_det: 300
+half: false
+dnn: false
+plots: true
+source: null
+show: false
+save_txt: false
+save_conf: false
+save_crop: false
+show_labels: true
+show_conf: true
+vid_stride: 1
+stream_buffer: false
+line_width: null
+visualize: false
+augment: false
+agnostic_nms: false
+classes: null
+retina_masks: false
+boxes: true
+format: torchscript
+keras: false
+optimize: false
+int8: false
+dynamic: false
+simplify: false
+opset: null
+workspace: 4
+nms: false
+lr0: 0.01
+lrf: 0.01
+momentum: 0.937
+weight_decay: 0.0005
+warmup_epochs: 3.0
+warmup_momentum: 0.8
+warmup_bias_lr: 0.1
+box: 7.5
+cls: 0.5
+dfl: 1.5
+pose: 12.0
+kobj: 1.0
+label_smoothing: 0.0
+nbs: 64
+hsv_h: 0.015
+hsv_s: 0.7
+hsv_v: 0.4
+degrees: 0.0
+translate: 0.1
+scale: 0.5
+shear: 0.0
+perspective: 0.0
+flipud: 0.0
+fliplr: 0.5
+mosaic: 1.0
+mixup: 0.0
+copy_paste: 0.0
+cfg: null
+tracker: botsort.yaml
+save_dir: runs\detect\yolov8s_local_n9

reports/train_params.yaml

@@ -0,0 +1,98 @@
+task: detect
+mode: train
+model: yolov8n.pt
+data: C:\Users\xange\Downloads\MDS2023\MLOPS\Nueva carpeta\MLOps_WhereIsWally\data\processed\data.yaml
+epochs: 1
+patience: 50
+batch: 8
+imgsz: 200
+save: true
+save_period: -1
+cache: false
+device: null
+workers: 8
+project: null
+name: yolov8s_local_n9
+exist_ok: false
+pretrained: true
+optimizer: auto
+verbose: true
+seed: 0
+deterministic: true
+single_cls: false
+rect: false
+cos_lr: false
+close_mosaic: 10
+resume: false
+amp: true
+fraction: 1.0
+profile: false
+freeze: null
+overlap_mask: true
+mask_ratio: 4
+dropout: 0.0
+val: true
+split: val
+save_json: false
+save_hybrid: false
+conf: null
+iou: 0.7
+max_det: 300
+half: false
+dnn: false
+plots: true
+source: null
+show: false
+save_txt: false
+save_conf: false
+save_crop: false
+show_labels: true
+show_conf: true
+vid_stride: 1
+stream_buffer: false
+line_width: null
+visualize: false
+augment: false
+agnostic_nms: false
+classes: null
+retina_masks: false
+boxes: true
+format: torchscript
+keras: false
+optimize: false
+int8: false
+dynamic: false
+simplify: false
+opset: null
+workspace: 4
+nms: false
+lr0: 0.01
+lrf: 0.01
+momentum: 0.937
+weight_decay: 0.0005
+warmup_epochs: 3.0
+warmup_momentum: 0.8
+warmup_bias_lr: 0.1
+box: 7.5
+cls: 0.5
+dfl: 1.5
+pose: 12.0
+kobj: 1.0
+label_smoothing: 0.0
+nbs: 64
+hsv_h: 0.015
+hsv_s: 0.7
+hsv_v: 0.4
+degrees: 0.0
+translate: 0.1
+scale: 0.5
+shear: 0.0
+perspective: 0.0
+flipud: 0.0
+fliplr: 0.5
+mosaic: 1.0
+mixup: 0.0
+copy_paste: 0.0
+cfg: null
+tracker: botsort.yaml
+save_dir: runs\detect\yolov8s_local_n9

requirements.txt

@@ -192,3 +192,4 @@ websocket-client==1.6.4
 Werkzeug==3.0.0
 widgetsnbextension==4.0.9
 zipp==3.17.0
+map-boxes==1.0.5

src/__init__.py

@@ -11,4 +11,6 @@
 
 METRICS_DIR = ROOT_DIR / "metrics"
 MODELS_DIR = ROOT_DIR / "models"
-
+REPORTS_DIR= ROOT_DIR / "reports"
+DATA_YAML_DIR= ROOT_DIR / "data/processed/data.yaml"
+ARTIFACTS_DIR= ROOT_DIR / "runs/detect"

src/features/process_data.py

@@ -0,0 +1,88 @@
+# importing libraries 
+import numpy as np 
+import cv2 
+from matplotlib import pyplot as plt 
+from os import listdir
+from os.path import isfile, join
+from src import PROCESSED_DATA_DIR,RAW_DATA_DIR
+from pathlib import Path
+import shutil
+import os
+
+def noise_removal(img_path):
+    # Reading image from folder where it is stored 
+    img = cv2.imread(img_path) 
+    # denoising of image saving it into dst image 
+    denoised = cv2.fastNlMeansDenoisingColored(img, None, 10, 10, 7, 15) 
+    # cv2.imwrite(dst_path,denoised)
+    return denoised
+
+def rgb2gray(img_path):
+    img = cv2.imread(img_path)
+    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
+    # blurred = cv2.GaussianBlur(gray, (5, 5), 0)
+    return gray
+
+def get_images_path(directory_name):
+    imgs_path={}
+    for f in listdir(directory_name):
+        if isfile(join(directory_name, f)):
+            imgs_path[f] = join(directory_name /f)
+    return imgs_path
+
+
+def auto_canny_edge_detection(path, sigma=0.33):
+    image = cv2.imread(path)
+    md = np.median(image)
+    lower_value = int(max(0, (1.0-sigma) * md))
+    upper_value = int(min(255, (1.0+sigma) * md))
+    return cv2.Canny(image, lower_value, upper_value)
+
+
+def image_processing(src_imgs_dir,dst_imgs_dir):
+    imgs_path=get_images_path(src_imgs_dir)
+
+    for image_name,path in imgs_path.items():
+        denoised=noise_removal(path)
+        gray=cv2.cvtColor(denoised,cv2.COLOR_BGR2GRAY)
+        cv2.imwrite(str(dst_imgs_dir / image_name),gray)
+
+
+def copy_labels(src,dest):
+    src_files = os.listdir(src)
+    for file_name in src_files:
+        full_file_name = os.path.join(src, file_name)
+        if os.path.isfile(full_file_name):
+            shutil.copy(full_file_name, dest) 
+
+
+def main():
+    train_imgs_dir=RAW_DATA_DIR / "train/images"
+    valid_imgs_dir=RAW_DATA_DIR / "valid/images"
+    test_imgs_dir=RAW_DATA_DIR / "test/images"
+    processed_train_imgs_dir=PROCESSED_DATA_DIR / "train/images"
+    processed_test_imgs_dir=PROCESSED_DATA_DIR / "test/images"
+    processed_valid_imgs_dir=PROCESSED_DATA_DIR / "valid/images"
+
+    train_labels=PROCESSED_DATA_DIR / "train/labels"
+    test_labels=PROCESSED_DATA_DIR / "test/labels"
+    valid_labels=PROCESSED_DATA_DIR / "valid/labels"
+
+    Path(processed_train_imgs_dir).mkdir(parents=True, exist_ok=True)
+    Path(processed_test_imgs_dir).mkdir(parents=True, exist_ok=True)
+    Path(processed_valid_imgs_dir).mkdir(parents=True, exist_ok=True)
+
+    Path(train_labels).mkdir(parents=True, exist_ok=True)
+    Path(test_labels).mkdir(parents=True, exist_ok=True)
+    Path(valid_labels).mkdir(parents=True, exist_ok=True)
+
+    copy_labels(RAW_DATA_DIR / "train/labels",train_labels)
+    copy_labels(RAW_DATA_DIR / "test/labels",test_labels)
+    copy_labels(RAW_DATA_DIR / "valid/labels",valid_labels)
+
+    image_processing(train_imgs_dir,processed_train_imgs_dir)
+    image_processing(test_imgs_dir,processed_test_imgs_dir)
+    image_processing(valid_imgs_dir,processed_valid_imgs_dir)
+
+if __name__ == "__main__":
+    main()

src/models/__init__.py

@@ -0,0 +1,3 @@
+from dotenv import load_dotenv
+
+load_dotenv()