From 752ad3b96a929a5e5d20289c8119da5af040f714 Mon Sep 17 00:00:00 2001 From: Mathieu Pillard Date: Thu, 7 Nov 2024 14:45:57 +0100 Subject: [PATCH] Add command to delete erroneously added overgrowth NeedsHumanReview & reset due dates (#22832) * Add command to delete erroneously added overgrowth NeedsHumanReview & reset due dates * Add route --- .../management/commands/process_addons.py | 18 +++ src/olympia/addons/tasks.py | 24 ++++ src/olympia/addons/tests/test_commands.py | 111 +++++++++++++++++- src/olympia/lib/settings_base.py | 3 + 4 files changed, 154 insertions(+), 2 deletions(-) diff --git a/src/olympia/addons/management/commands/process_addons.py b/src/olympia/addons/management/commands/process_addons.py index 94ba953f5aa8..6a9c0ea38a8f 100644 --- a/src/olympia/addons/management/commands/process_addons.py +++ b/src/olympia/addons/management/commands/process_addons.py @@ -6,7 +6,9 @@ from olympia.abuse.models import AbuseReport from olympia.addons.models import Addon from olympia.addons.tasks import ( + ERRONEOUSLY_ADDED_OVERGROWTH_DATE_RANGE, delete_addons, + delete_erroneously_added_overgrowth_needshumanreview, extract_colors_from_static_themes, find_inconsistencies_between_es_and_db, recreate_theme_previews, @@ -22,6 +24,7 @@ from olympia.devhub.tasks import get_preview_sizes, recreate_previews from olympia.lib.crypto.tasks import bump_and_resign_addons from olympia.ratings.tasks import addon_rating_aggregates +from olympia.reviewers.models import NeedsHumanReview from olympia.reviewers.tasks import recalculate_post_review_weight from olympia.versions.tasks import delete_list_theme_previews @@ -144,6 +147,21 @@ def get_recalc_needed_filters(): 'task': addon_rating_aggregates, 'queryset_filters': [Q(status=amo.STATUS_APPROVED)], }, + # https://github.com/mozilla/addons/issues/15141 + 'delete_erroneously_added_overgrowth_needshumanreview': { + 'task': delete_erroneously_added_overgrowth_needshumanreview, + 'queryset_filters': [ + Q( + versions__needshumanreview__reason=( + NeedsHumanReview.REASONS.HOTNESS_THRESHOLD + ), + versions__needshumanreview__created__range=( + ERRONEOUSLY_ADDED_OVERGROWTH_DATE_RANGE + ), + versions__needshumanreview__is_active=True, + ) + ], + }, } def add_arguments(self, parser): diff --git a/src/olympia/addons/tasks.py b/src/olympia/addons/tasks.py index 68f6779f626a..613f2c3f5d00 100644 --- a/src/olympia/addons/tasks.py +++ b/src/olympia/addons/tasks.py @@ -1,6 +1,7 @@ import hashlib import mimetypes import os +from datetime import date from django.core.files.storage import default_storage as storage from django.db import transaction @@ -563,3 +564,26 @@ def flag_high_hotness_according_to_review_tier(): NeedsHumanReview.set_on_addons_latest_signed_versions( qs, NeedsHumanReview.REASONS.HOTNESS_THRESHOLD ) + + +ERRONEOUSLY_ADDED_OVERGROWTH_DATE_RANGE = ( + date(2024, 11, 5), + date(2024, 11, 7), +) + + +@task +@use_primary_db +def delete_erroneously_added_overgrowth_needshumanreview(addon_ids, **kw): + addons = Addon.unfiltered.filter(pk__in=addon_ids).no_transforms() + for addon in addons: + log.info( + 'Deleting erroneously added NHR and updating due dates for %s', addon.pk + ) + NeedsHumanReview.objects.filter( + version__addon=addon, + reason=NeedsHumanReview.REASONS.HOTNESS_THRESHOLD, + created__range=ERRONEOUSLY_ADDED_OVERGROWTH_DATE_RANGE, + is_active=True, + ).delete() + addon.update_all_due_dates() diff --git a/src/olympia/addons/tests/test_commands.py b/src/olympia/addons/tests/test_commands.py index 216c68ec9443..e4ad9500fbea 100644 --- a/src/olympia/addons/tests/test_commands.py +++ b/src/olympia/addons/tests/test_commands.py @@ -1,6 +1,6 @@ import random from contextlib import contextmanager -from datetime import timedelta +from datetime import datetime, timedelta from unittest import mock from django.conf import settings @@ -27,7 +27,7 @@ from olympia.applications.models import AppVersion from olympia.files.models import FileValidation from olympia.ratings.models import Rating, RatingAggregate -from olympia.reviewers.models import AutoApprovalSummary +from olympia.reviewers.models import AutoApprovalSummary, NeedsHumanReview from olympia.versions.models import ApplicationsVersions, Version, VersionPreview @@ -573,3 +573,110 @@ def test_delete_list_theme_previews(): assert VersionPreview.objects.filter(id=other_firefox_preview.id).exists() assert VersionPreview.objects.filter(id=other_amo_preview.id).exists() assert not VersionPreview.objects.filter(id=other_old_list_preview.id).exists() + + +@pytest.mark.django_db +def test_delete_erroneously_added_overgrowth_needshumanreview(): + user_factory(pk=settings.TASK_USER_ID, display_name='Mozilla') + should_no_longer_have_nhr_or_due_date = [ + # Basic case + addon_factory() + .current_version.needshumanreview_set.create( + reason=NeedsHumanReview.REASONS.HOTNESS_THRESHOLD, + created=datetime(2024, 11, 7), + ) + .version.addon, + # Somehow has 2 bogus NHR, one active one not + NeedsHumanReview.objects.create( + reason=NeedsHumanReview.REASONS.HOTNESS_THRESHOLD, + created=datetime(2024, 11, 7), + is_active=False, + version=addon_factory() + .current_version.needshumanreview_set.create( + reason=NeedsHumanReview.REASONS.HOTNESS_THRESHOLD, + created=datetime(2024, 11, 7), + ) + .version, + ).version.addon, + # Has 2 versions both with the bogus NHR + NeedsHumanReview.objects.create( + version=version_factory( + addon=addon_factory() + .current_version.needshumanreview_set.create( + reason=NeedsHumanReview.REASONS.HOTNESS_THRESHOLD, + created=datetime(2024, 11, 7), + ) + .version.addon, + channel=amo.CHANNEL_UNLISTED, + ), + reason=NeedsHumanReview.REASONS.HOTNESS_THRESHOLD, + created=datetime(2024, 11, 7), + ).version.addon, + ] + should_still_have_nhr_and_due_date = [ + # Outside the range + addon_factory() + .current_version.needshumanreview_set.create( + reason=NeedsHumanReview.REASONS.HOTNESS_THRESHOLD, + created=datetime(2024, 11, 3), + ) + .version.addon, + # In the range but wrong reason + addon_factory() + .current_version.needshumanreview_set.create( + reason=NeedsHumanReview.REASONS.UNKNOWN, + created=datetime(2024, 11, 6), + ) + .version.addon, + ] + should_no_longer_have_hotness_nhr_but_still_other_nhr_and_due_date = [ + # Has both a bogus NHR (inside the range, reason we care about) and + # another valid one. + NeedsHumanReview.objects.create( + reason=NeedsHumanReview.REASONS.HOTNESS_THRESHOLD, + created=datetime(2024, 1, 1), + version=addon_factory() + .current_version.needshumanreview_set.create( + reason=NeedsHumanReview.REASONS.HOTNESS_THRESHOLD, + created=datetime(2024, 11, 7), + ) + .version, + ).version.addon, + # Has 2 versions, one with bogus NHR but other version has a different, + # valid (because date outside the range) NHR for that same reason. + NeedsHumanReview.objects.create( + version=version_factory( + addon=addon_factory() + .current_version.needshumanreview_set.create( + reason=NeedsHumanReview.REASONS.HOTNESS_THRESHOLD, + created=datetime(2024, 11, 7), + ) + .version.addon, + channel=amo.CHANNEL_UNLISTED, + ), + reason=NeedsHumanReview.REASONS.HOTNESS_THRESHOLD, + ).version.addon, + ] + call_command( + 'process_addons', task='delete_erroneously_added_overgrowth_needshumanreview' + ) + for addon in should_no_longer_have_nhr_or_due_date: + assert not addon.versions.filter(due_date__isnull=False).exists() + assert not NeedsHumanReview.objects.filter( + version__addon=addon, is_active=True + ).exists() + for addon in should_still_have_nhr_and_due_date: + assert addon.versions.filter(due_date__isnull=False).exists() + assert NeedsHumanReview.objects.filter( + version__addon=addon, is_active=True + ).exists() + for addon in should_no_longer_have_hotness_nhr_but_still_other_nhr_and_due_date: + assert addon.versions.filter(due_date__isnull=False).exists() + assert NeedsHumanReview.objects.filter( + version__addon=addon, is_active=True + ).exists() + assert NeedsHumanReview.objects.filter( + version__addon=addon, + is_active=True, + reason=NeedsHumanReview.REASONS.HOTNESS_THRESHOLD, + ).exists() diff --git a/src/olympia/lib/settings_base.py b/src/olympia/lib/settings_base.py index b38d3e6f44b6..ec6ef09a1dfa 100644 --- a/src/olympia/lib/settings_base.py +++ b/src/olympia/lib/settings_base.py @@ -850,6 +850,9 @@ def get_db_config(environ_var, atomic_requests=True): # Adhoc # A queue to be used for one-off tasks that could be resource intensive or # tasks we want completely separate from the rest. + 'olympia.addons.tasks.delete_erroneously_added_overgrowth_needshumanreview': { + 'queue': 'adhoc' + }, 'olympia.addons.tasks.find_inconsistencies_between_es_and_db': {'queue': 'adhoc'}, 'olympia.search.management.commands.reindex.create_new_index': {'queue': 'adhoc'}, 'olympia.search.management.commands.reindex.delete_indexes': {'queue': 'adhoc'},