From bbc2752c34b95ec97c8f772f4d1e0b92acfbc6c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Alori?= Date: Mon, 30 May 2022 17:45:41 -0300 Subject: [PATCH] Add mot metrics GitHub action (#114) * Added basic motmetrics test * Update tests to new norfair api * Run our 4 tests in parallel in github actions * Remove empty test * Move metrics.txt file Co-authored-by: Leonardo Agis --- .github/workflows/ci.yml | 14 ++- .../motmetrics4norfair/motmetrics4norfair.py | 2 +- norfair/video.py | 2 + tests/metrics.txt | 23 ++++ tests/test_mot_metrics.py | 110 ++++++++++++++++++ tests/test_norfair.py | 6 - tox.ini | 7 +- 7 files changed, 152 insertions(+), 12 deletions(-) create mode 100644 tests/metrics.txt create mode 100644 tests/test_mot_metrics.py delete mode 100644 tests/test_norfair.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c853b81..f9078b13 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: build-test: runs-on: ubuntu-latest strategy: - max-parallel: 3 + max-parallel: 4 matrix: python-version: [3.6, 3.7, 3.8, 3.9] steps: @@ -36,8 +36,16 @@ jobs: python -m pip install --upgrade pip pip install tox tox-gh-actions - - name: Test with tox - run: tox + - name: Run tests + run: | + python3 -m venv ./venv + source venv/bin/activate + pip install --upgrade pip + python --version + poetry install -E metrics + curl -O https://motchallenge.net/data/MOT17Labels.zip + unzip MOT17Labels.zip + pytest -s tests/ # # Build & upload release to PyPI. diff --git a/demos/motmetrics4norfair/motmetrics4norfair.py b/demos/motmetrics4norfair/motmetrics4norfair.py index 66d5b6d5..635d1cd8 100644 --- a/demos/motmetrics4norfair/motmetrics4norfair.py +++ b/demos/motmetrics4norfair/motmetrics4norfair.py @@ -155,4 +155,4 @@ def keypoints_distance(detected_pose, tracked_pose): accumulator.print_metrics() if args.save_metrics: - accumulator.save_metrics(save_path=output_path) \ No newline at end of file + accumulator.save_metrics(save_path=output_path) diff --git a/norfair/video.py b/norfair/video.py index 5d8e7930..9017718b 100644 --- a/norfair/video.py +++ b/norfair/video.py @@ -12,6 +12,8 @@ from rich import print from rich.progress import BarColumn, Progress, ProgressColumn, TimeRemainingColumn +from norfair import metrics + from .utils import get_terminal_size diff --git a/tests/metrics.txt b/tests/metrics.txt new file mode 100644 index 00000000..aca15857 --- /dev/null +++ b/tests/metrics.txt @@ -0,0 +1,23 @@ + IDF1 IDP IDR Rcll Prcn GT MT PT ML FP FN IDs FM MOTA MOTP IDt IDa IDm +MOT17-04-FRCNN 55.7% 73.9% 44.6% 56.3% 93.2% 83 18 43 22 1962 20778 90 104 52.0% 0.107 5 88 3 +MOT17-11-FRCNN 54.6% 68.6% 45.3% 61.5% 93.1% 75 19 33 23 431 3631 64 61 56.3% 0.101 15 52 5 +MOT17-05-SDP 58.5% 67.6% 51.5% 66.7% 87.6% 133 32 81 20 653 2301 134 140 55.4% 0.165 48 99 15 +MOT17-10-FRCNN 39.9% 44.7% 36.0% 61.0% 75.8% 57 16 35 6 2504 5013 319 313 39.0% 0.173 125 193 10 +MOT17-05-DPM 36.1% 57.1% 26.4% 38.0% 82.2% 133 10 58 65 570 4291 96 96 28.3% 0.242 33 72 12 +MOT17-11-DPM 38.8% 49.7% 31.8% 54.2% 84.5% 75 12 24 39 935 4321 88 64 43.4% 0.217 20 69 4 +MOT17-04-DPM 29.0% 43.1% 21.9% 42.5% 83.6% 83 7 44 32 3965 27336 401 420 33.3% 0.210 43 352 1 +MOT17-10-SDP 47.4% 51.8% 43.6% 74.2% 88.1% 57 30 24 3 1292 3316 308 289 61.7% 0.198 139 165 6 +MOT17-11-SDP 55.4% 61.0% 50.8% 75.8% 91.1% 75 34 30 11 697 2285 103 101 67.3% 0.140 43 69 13 +MOT17-13-SDP 54.4% 66.4% 46.1% 57.0% 82.1% 110 45 28 37 1444 5008 229 209 42.6% 0.201 73 166 18 +MOT17-09-DPM 33.4% 37.7% 30.0% 59.9% 75.4% 26 4 18 4 1042 2137 119 113 38.1% 0.262 41 81 3 +MOT17-13-DPM 21.3% 59.9% 12.9% 18.0% 83.5% 110 5 28 77 416 9543 120 125 13.4% 0.268 35 94 9 +MOT17-05-FRCNN 54.7% 72.0% 44.1% 54.7% 89.2% 133 24 68 41 457 3136 95 96 46.7% 0.181 44 68 17 +MOT17-02-FRCNN 33.9% 53.8% 24.7% 36.6% 79.7% 62 7 26 29 1736 11783 119 131 26.6% 0.134 31 95 8 +MOT17-09-SDP 53.2% 63.8% 45.6% 67.6% 94.6% 26 12 14 0 204 1726 52 55 62.8% 0.130 21 37 6 +MOT17-09-FRCNN 54.0% 72.4% 43.1% 58.6% 98.5% 26 7 17 2 49 2207 40 39 56.9% 0.095 14 30 4 +MOT17-02-SDP 34.4% 42.9% 28.7% 51.0% 76.1% 62 11 39 12 2979 9103 268 284 33.5% 0.182 82 190 8 +MOT17-02-DPM 16.2% 41.0% 10.1% 20.2% 81.6% 62 5 14 43 843 14834 111 112 15.0% 0.246 19 91 1 +MOT17-13-FRCNN 47.0% 52.9% 42.3% 58.8% 73.6% 110 29 57 24 2455 4802 371 366 34.5% 0.185 117 264 20 +MOT17-10-DPM 28.0% 45.8% 20.2% 37.3% 84.6% 57 6 20 31 871 8051 127 154 29.5% 0.248 15 118 6 +MOT17-04-SDP 66.2% 74.7% 59.5% 77.6% 97.4% 83 48 26 9 1001 10672 225 254 75.0% 0.132 89 136 5 +OVERALL 45.3% 59.5% 36.6% 53.6% 87.2% 1638 381 727 530 26506 156274 3479 3526 44.7% 0.163 1052 2529 174 \ No newline at end of file diff --git a/tests/test_mot_metrics.py b/tests/test_mot_metrics.py new file mode 100644 index 00000000..7cff7dff --- /dev/null +++ b/tests/test_mot_metrics.py @@ -0,0 +1,110 @@ +import os.path + +import numpy as np +import pandas as pd + +from norfair import Tracker, metrics + +DATASET_PATH = "train" +MOTA_ERROR_THRESHOLD = 0.0 + +FRAME_SKIP_PERIOD = 1 +DETECTION_THRESHOLD = 0.01 +DISTANCE_THRESHOLD = 0.9 +DIAGONAL_PROPORTION_THRESHOLD = 1 / 18 +POINTWISE_HIT_COUNTER_MAX = 3 +HIT_COUNTER_MAX = 2 + +def keypoints_distance(detected_pose, tracked_pose): + norm_orders = [1, 2, np.inf] + distances = 0 + diagonal = 0 + + hor_min_pt = min(detected_pose.points[:, 0]) + hor_max_pt = max(detected_pose.points[:, 0]) + ver_min_pt = min(detected_pose.points[:, 1]) + ver_max_pt = max(detected_pose.points[:, 1]) + + # Set keypoint_dist_threshold based on object size, and calculate + # distance between detections and tracker estimations + for p in norm_orders: + distances += np.linalg.norm( + detected_pose.points - tracked_pose.estimate, ord=p, axis=1 + ) + diagonal += np.linalg.norm( + [hor_max_pt - hor_min_pt, ver_max_pt - ver_min_pt], ord=p + ) + + distances = distances / len(norm_orders) + + keypoint_dist_threshold = diagonal * DIAGONAL_PROPORTION_THRESHOLD + + match_num = np.count_nonzero( + (distances < keypoint_dist_threshold) + * (detected_pose.scores > DETECTION_THRESHOLD) + * (tracked_pose.last_detection.scores > DETECTION_THRESHOLD) + ) + return 1 / (1 + match_num) + +def test_mot_metrics(): + """Tests that Norfair's MOT metrics didn't get worse + + Configurable so that it allows some margin on how much worse metrics could get before + the test fails. Margin configured through MOTA_ERROR_THRESHOLD. + + Raises: + If the previous metrics file its not found. + """ + # Load previous metrics + try: + previous_metrics = pd.read_fwf('tests/metrics.txt') + previous_metrics.columns = [column_name.lower() for column_name in previous_metrics.columns] + previous_metrics = previous_metrics.set_index(previous_metrics.columns[0]) + except FileNotFoundError as e: + raise e + + accumulator = metrics.Accumulators() + sequences_paths = [element.path for element in os.scandir(DATASET_PATH) if element.is_dir()] + for input_path in sequences_paths: + # Search vertical resolution in seqinfo.ini + seqinfo_path = os.path.join(input_path, "seqinfo.ini") + info_file = metrics.InformationFile(file_path=seqinfo_path) + + all_detections = metrics.DetectionFileParser( + input_path=input_path, information_file=info_file + ) + + tracker = Tracker( + distance_function=keypoints_distance, + distance_threshold=DISTANCE_THRESHOLD, + detection_threshold=DETECTION_THRESHOLD, + pointwise_hit_counter_max=POINTWISE_HIT_COUNTER_MAX, + hit_counter_max=HIT_COUNTER_MAX, + ) + + # Initialize accumulator for this video + accumulator.create_accumulator(input_path=input_path, information_file=info_file) + + for frame_number, detections in enumerate(all_detections): + if frame_number % FRAME_SKIP_PERIOD == 0: + tracked_objects = tracker.update( + detections=detections, period=FRAME_SKIP_PERIOD + ) + else: + detections = [] + tracked_objects = tracker.update() + + accumulator.update(predictions=tracked_objects) + + accumulator.compute_metrics() + new_metrics = accumulator.summary_dataframe + new_metrics.columns = [column_name.lower() for column_name in new_metrics.columns] + + # Unify the scores to be able to compare them. new metrics is the percentage + # expressed between 0 and 1, the previous metrics have the percentage as a string + # with the % character at the end + new_overall_mota = new_metrics.loc["OVERALL", "mota"] * 100 + previous_overall_mota = float(previous_metrics.loc["OVERALL", "mota"][:-1]) + + accumulator.print_metrics() + assert new_overall_mota >= previous_overall_mota * (1 - MOTA_ERROR_THRESHOLD), f"New overall MOTA score: {new_overall_mota} is too low, previous overall MOTA score: {previous_overall_mota}" diff --git a/tests/test_norfair.py b/tests/test_norfair.py deleted file mode 100644 index a7dcc424..00000000 --- a/tests/test_norfair.py +++ /dev/null @@ -1,6 +0,0 @@ -import norfair - - -def test_norfair(): - # TODO: write actual tests - assert True diff --git a/tox.ini b/tox.ini index 1d0a8d11..45be08b3 100644 --- a/tox.ini +++ b/tox.ini @@ -13,5 +13,8 @@ python = allowlist_externals = poetry commands = ; Need this so dev dependencies (like pytest) are installed - poetry install --no-root -v - poetry run pytest tests/ + poetry install --no-root -v -E metrics + ; Download the needed files to perform the MOT metrics test + curl -O https://motchallenge.net/data/MOT17Labels.zip + unzip MOT17Labels.zip + poetry run pytest -s tests/