From d83d1e1dd09bae502c5bb3181a179c0d324ee19f Mon Sep 17 00:00:00 2001 From: Luke Sabor <32885230+lsabor@users.noreply.github.com> Date: Thu, 16 Jan 2025 18:53:27 -0800 Subject: [PATCH] leaderboard time tweaks (#1926) add finalize flag help text on leaderboard time fields filtering adjustments to update leaderbaord finalize operation add force update leaderboard to admin panel add finalize leaderboard to admin panel add prize pool to leaderboard make all time fields default to grab from project improve admin documentation --- projects/models.py | 3 + scoring/admin.py | 48 ++++- ...nalized_leaderboard_prize_pool_and_more.py | 79 ++++++++ scoring/models.py | 185 +++++++++++------- scoring/utils.py | 76 ++++--- .../leaderboard_action_descriptions.html | 15 ++ 6 files changed, 310 insertions(+), 96 deletions(-) create mode 100644 scoring/migrations/0007_leaderboard_finalized_leaderboard_prize_pool_and_more.py create mode 100644 templates/admin/scoring/leaderboard_action_descriptions.html diff --git a/projects/models.py b/projects/models.py index 266714d9c..fe09343d7 100644 --- a/projects/models.py +++ b/projects/models.py @@ -288,10 +288,12 @@ def __str__(self): def save(self, *args, **kwargs): creating = not self.pk + # Check if the primary leaderboard is associated with this project if self.primary_leaderboard and self.primary_leaderboard.project != self: raise ValueError( "Primary leaderboard must be associated with this project." ) + super().save(*args, **kwargs) if ( creating @@ -300,6 +302,7 @@ def save(self, *args, **kwargs): in ( self.ProjectTypes.TOURNAMENT, self.ProjectTypes.QUESTION_SERIES, + self.ProjectTypes.COMMUNITY, ) ): # create default leaderboard when creating a new tournament/question series diff --git a/scoring/admin.py b/scoring/admin.py index dae4ae1a6..0748b5b55 100644 --- a/scoring/admin.py +++ b/scoring/admin.py @@ -70,14 +70,22 @@ def get_queryset(self, request): @admin.register(Leaderboard) class LeaderboardAdmin(admin.ModelAdmin): + change_list_template = "admin/scoring/leaderboard_action_descriptions.html" search_fields = ["name", "project", "score_type"] - list_display = ["__str__", "id", "project", "score_type"] + list_display = ["__str__", "id", "project", "score_type", "finalized"] autocomplete_fields = ["project"] list_filter = [ AutocompleteFilterFactory("Project", "project"), + "score_type", + "finalized", ] inlines = [LeaderboardEntryInline] - actions = ["make_primary_leaderboard", "update_leaderboards"] + actions = [ + "make_primary_leaderboard", + "update_leaderboards", + "force_update_leaderboards", + "force_finalize_and_asign_medals_leaderboards", + ] def make_primary_leaderboard(self, request, queryset): for leaderboard in queryset: @@ -91,16 +99,42 @@ def make_primary_leaderboard(self, request, queryset): messages.SUCCESS, ) - make_primary_leaderboard.short_description = ( - "Make selected leaderboards their project's primary_leaderboard" - ) + make_primary_leaderboard.short_description = "Make Primary Leaderboard" def update_leaderboards(self, request, queryset): leaderboard: Leaderboard for leaderboard in queryset: - update_project_leaderboard(leaderboard.project, leaderboard) + update_project_leaderboard( + leaderboard.project, + leaderboard, + ) - update_leaderboards.short_description = "Update selected Leaderboards" + update_leaderboards.short_description = "Update Leaderboards" + + def force_update_leaderboards(self, request, queryset): + leaderboard: Leaderboard + for leaderboard in queryset: + update_project_leaderboard( + leaderboard.project, + leaderboard, + force_update=True, + ) + + force_update_leaderboards.short_description = "Force Update Leaderboards" + + def force_finalize_and_asign_medals_leaderboards(self, request, queryset): + leaderboard: Leaderboard + for leaderboard in queryset: + update_project_leaderboard( + leaderboard.project, + leaderboard, + force_update=True, + force_finalize=True, + ) + + force_finalize_and_asign_medals_leaderboards.short_description = ( + "Force Update, Finalize, and Assign Medals/Prizes" + ) @admin.register(LeaderboardEntry) diff --git a/scoring/migrations/0007_leaderboard_finalized_leaderboard_prize_pool_and_more.py b/scoring/migrations/0007_leaderboard_finalized_leaderboard_prize_pool_and_more.py new file mode 100644 index 000000000..e59dbb7b0 --- /dev/null +++ b/scoring/migrations/0007_leaderboard_finalized_leaderboard_prize_pool_and_more.py @@ -0,0 +1,79 @@ +# Generated by Django 5.1.4 on 2025-01-16 19:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("scoring", "0006_alter_leaderboardentry_excluded_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="leaderboard", + name="finalized", + field=models.BooleanField( + default=False, + help_text="If true, this Leaderboard's entries cannot be updated except by a manual action in the admin panel. Automatically set to True the first time this leaderboard is updated after the finalize_time.", + ), + ), + migrations.AddField( + model_name="leaderboard", + name="prize_pool", + field=models.DecimalField( + blank=True, + decimal_places=2, + default=None, + help_text="Optional. If not set, the Project's prize_pool will be used instead.\n
- If the Project has a prize pool, but this leaderboard has none, set this to 0.\n ", + max_digits=15, + null=True, + ), + ), + migrations.AlterField( + model_name="leaderboard", + name="end_time", + field=models.DateTimeField( + blank=True, + help_text="Optional (required for global leaderboards).\n
- Global Leaderboards: filters for questions that have a scheduled_close_time before this (plus a grace period). Automatically set, do not change.\n
- Non-Global Leaderboards: has no effect on question filtering.\n
- Filtering MedalExclusionRecords: MedalExclusionRecords that have a start_time less than this (and no end_time or an end_time later that this Leaderboard's start_time) will be triggered. If not set, this Leaderboard's finalize_time will be used instead - it is recommended not to use this field unless required.\n ", + null=True, + ), + ), + migrations.AlterField( + model_name="leaderboard", + name="finalize_time", + field=models.DateTimeField( + blank=True, + help_text="Optional. If not set, the Project's close_date will be used instead.\n
- For all Leaderboards: used to filter out questions that have a resolution_set_time after this (as they were resolved after this Leaderboard was finalized).\n
- Filtering MedalExclusionRecords: If set and end_time is not set, MedalExclusionRecords that have a start_time less than this (and no end_time or an end_time later that this Leaderboard's start_time) will be triggered.\n ", + null=True, + ), + ), + migrations.AlterField( + model_name="leaderboard", + name="score_type", + field=models.CharField( + choices=[ + ("peer_tournament", "Peer Tournament"), + ("spot_peer_tournament", "Spot Peer Tournament"), + ("relative_legacy_tournament", "Relative Legacy Tournament"), + ("baseline_global", "Baseline Global"), + ("peer_global", "Peer Global"), + ("peer_global_legacy", "Peer Global Legacy"), + ("comment_insight", "Comment Insight"), + ("question_writing", "Question Writing"), + ("manual", "Manual"), + ], + help_text="\n \n \n \n \n \n \n \n \n \n \n
peer_tournament Sum of peer scores. Most likely what you want.
spot_peer_tournament Sum of spot peer scores.
relative_legacy Old site scoring.
baseline_global Sum of baseline scores.
peer_global Coverage-weighted average of peer scores.
peer_global_legacy Average of peer scores.
comment_insight H-index of upvotes for comments on questions.
question_writing H-index of number of forecasters / 10 on questions.
manual Does not automatically update. Manually set all entries.
\n ", + max_length=200, + ), + ), + migrations.AlterField( + model_name="leaderboard", + name="start_time", + field=models.DateTimeField( + blank=True, + help_text="Optional (required for global leaderboards). If not set, the Project's open_date will be used instead.\n
- Global Leaderboards: filters for questions that have an open time after this. Automatically set, do not change.\n
- Non-Global Leaderboards: has no effect on question filtering.\n
- Filtering MedalExclusionRecords: MedalExclusionRecords that have no end_time or an end_time greater than this (and a start_time before this Leaderboard's end_time or finalize_time) will be triggered.\n ", + null=True, + ), + ), + ] diff --git a/scoring/models.py b/scoring/models.py index d0a6ae426..2dd66762b 100644 --- a/scoring/models.py +++ b/scoring/models.py @@ -126,12 +126,12 @@ class Leaderboard(TimeStampedModel): ) class ScoreTypes(models.TextChoices): - RELATIVE_LEGACY_TOURNAMENT = "relative_legacy_tournament" - PEER_GLOBAL = "peer_global" - PEER_GLOBAL_LEGACY = "peer_global_legacy" PEER_TOURNAMENT = "peer_tournament" SPOT_PEER_TOURNAMENT = "spot_peer_tournament" + RELATIVE_LEGACY_TOURNAMENT = "relative_legacy_tournament" BASELINE_GLOBAL = "baseline_global" + PEER_GLOBAL = "peer_global" + PEER_GLOBAL_LEGACY = "peer_global_legacy" COMMENT_INSIGHT = "comment_insight" QUESTION_WRITING = "question_writing" MANUAL = "manual" @@ -162,10 +162,63 @@ def get_base_score(cls, score_type: str) -> Score.ScoreTypes: "Question Writing leaderboards do not have base scores" ) - score_type = models.CharField(max_length=200, choices=ScoreTypes.choices) - start_time = models.DateTimeField(null=True, blank=True) - end_time = models.DateTimeField(null=True, blank=True) - finalize_time = models.DateTimeField(null=True, blank=True) + score_type = models.CharField( + max_length=200, + choices=ScoreTypes.choices, + help_text=""" + + + + + + + + + + +
peer_tournament Sum of peer scores. Most likely what you want.
spot_peer_tournament Sum of spot peer scores.
relative_legacy Old site scoring.
baseline_global Sum of baseline scores.
peer_global Coverage-weighted average of peer scores.
peer_global_legacy Average of peer scores.
comment_insight H-index of upvotes for comments on questions.
question_writing H-index of number of forecasters / 10 on questions.
manual Does not automatically update. Manually set all entries.
+ """, + ) + start_time = models.DateTimeField( + null=True, + blank=True, + help_text="""Optional (required for global leaderboards). If not set, the Project's open_date will be used instead. +
- Global Leaderboards: filters for questions that have an open time after this. Automatically set, do not change. +
- Non-Global Leaderboards: has no effect on question filtering. +
- Filtering MedalExclusionRecords: MedalExclusionRecords that have no end_time or an end_time greater than this (and a start_time before this Leaderboard's end_time or finalize_time) will be triggered. + """, + ) + end_time = models.DateTimeField( + null=True, + blank=True, + help_text="""Optional (required for global leaderboards). +
- Global Leaderboards: filters for questions that have a scheduled_close_time before this (plus a grace period). Automatically set, do not change. +
- Non-Global Leaderboards: has no effect on question filtering. +
- Filtering MedalExclusionRecords: MedalExclusionRecords that have a start_time less than this (and no end_time or an end_time later that this Leaderboard's start_time) will be triggered. If not set, this Leaderboard's finalize_time will be used instead - it is recommended not to use this field unless required. + """, + ) + finalize_time = models.DateTimeField( + null=True, + blank=True, + help_text="""Optional. If not set, the Project's close_date will be used instead. +
- For all Leaderboards: used to filter out questions that have a resolution_set_time after this (as they were resolved after this Leaderboard was finalized). +
- Filtering MedalExclusionRecords: If set and end_time is not set, MedalExclusionRecords that have a start_time less than this (and no end_time or an end_time later that this Leaderboard's start_time) will be triggered. + """, + ) + finalized = models.BooleanField( + default=False, + help_text="If true, this Leaderboard's entries cannot be updated except by a manual action in the admin panel. Automatically set to True the first time this leaderboard is updated after the finalize_time.", + ) + prize_pool = models.DecimalField( + default=None, + decimal_places=2, + max_digits=15, + null=True, + blank=True, + help_text="""Optional. If not set, the Project's prize_pool will be used instead. +
- If the Project has a prize pool, but this leaderboard has none, set this to 0. + """, + ) def __str__(self): if self.name: @@ -179,74 +232,72 @@ def get_questions(self) -> QuerySet[Question]: related_posts__post__curation_status=Post.CurationStatus.APPROVED ) - if self.project and self.project.type == Project.ProjectTypes.SITE_MAIN: - # global leaderboard - if self.start_time is None or self.end_time is None: - raise ValueError("Global leaderboards must have start and end times") - - questions = questions.filter_public().filter( - related_posts__post__in=Post.objects.filter_for_main_feed() - ) - - if self.score_type == self.ScoreTypes.COMMENT_INSIGHT: - # post must be published - return questions.filter( - related_posts__post__published_at__lt=self.end_time - ) - elif self.score_type == self.ScoreTypes.QUESTION_WRITING: - # post must be published, and can't be resolved before the start_time - # of the leaderboard - return questions.filter( - Q(scheduled_close_time__gte=self.start_time) - & ( - Q(actual_close_time__isnull=True) - | Q(actual_close_time__gte=self.start_time) - ), - related_posts__post__published_at__lt=self.end_time, + if not (self.project and self.project.type == Project.ProjectTypes.SITE_MAIN): + # normal Project leaderboard + if self.project: + questions = questions.filter( + Q(related_posts__post__projects=self.project) + | Q(related_posts__post__default_project=self.project), ) + return questions.distinct("id") - close_grace_period = timedelta(days=3) - resolve_grace_period = timedelta(days=100) + # global leaderboard + if self.start_time is None or self.end_time is None: + raise ValueError("Global leaderboards must have start and end times") + + questions = questions.filter_public().filter( + related_posts__post__in=Post.objects.filter_for_main_feed() + ) - questions = questions.filter( - Q(actual_resolve_time__isnull=True) - | Q(actual_resolve_time__lte=self.end_time + resolve_grace_period), - open_time__gte=self.start_time, - open_time__lt=self.end_time, - scheduled_close_time__lte=self.end_time + close_grace_period, + if self.score_type == self.ScoreTypes.COMMENT_INSIGHT: + # post must be published + return questions.filter(related_posts__post__published_at__lt=self.end_time) + elif self.score_type == self.ScoreTypes.QUESTION_WRITING: + # post must be published, and can't be resolved before the start_time + # of the leaderboard + return questions.filter( + Q(scheduled_close_time__gte=self.start_time) + & ( + Q(actual_close_time__isnull=True) + | Q(actual_close_time__gte=self.start_time) + ), + related_posts__post__published_at__lt=self.end_time, ) - gl_dates = global_leaderboard_dates() - checked_intervals: list[tuple[datetime, datetime]] = [] - for start, end in gl_dates[::-1]: # must be in reverse order, biggest first - if ( - (self.start_time, self.end_time) == (start, end) - or start < self.start_time - or self.end_time < end - ): - continue - to_add = True - for checked_start, checked_end in checked_intervals: - if checked_start < start and end < checked_end: - to_add = False - break - if to_add: - checked_intervals.append((start, end)) - questions = questions.filter( - Q(open_time__lt=start) - | Q(scheduled_close_time__gt=end + close_grace_period) - | Q(actual_resolve_time__gt=end + resolve_grace_period) - ) + close_grace_period = timedelta(days=3) + resolve_grace_period = timedelta(days=100) - return questions + questions = questions.filter( + Q(actual_resolve_time__isnull=True) + | Q(actual_resolve_time__lte=self.end_time + resolve_grace_period), + open_time__gte=self.start_time, + open_time__lt=self.end_time, + scheduled_close_time__lte=self.end_time + close_grace_period, + ) - if self.project: - questions = questions.filter( - Q(related_posts__post__projects=self.project) - | Q(related_posts__post__default_project=self.project) - ) + gl_dates = global_leaderboard_dates() + checked_intervals: list[tuple[datetime, datetime]] = [] + for start, end in gl_dates[::-1]: # must be in reverse order, biggest first + if ( + (self.start_time, self.end_time) == (start, end) + or start < self.start_time + or self.end_time < end + ): + continue + to_add = True + for checked_start, checked_end in checked_intervals: + if checked_start < start and end < checked_end: + to_add = False + break + if to_add: + checked_intervals.append((start, end)) + questions = questions.filter( + Q(open_time__lt=start) + | Q(scheduled_close_time__gt=end + close_grace_period) + | Q(actual_resolve_time__gt=end + resolve_grace_period) + ) - return questions.distinct("id") + return questions def name_and_slug_for_global_leaderboard_dates( diff --git a/scoring/utils.py b/scoring/utils.py index 4570ff10c..88fddebb6 100644 --- a/scoring/utils.py +++ b/scoring/utils.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from decimal import Decimal from io import StringIO +import logging import numpy as np from django.db import transaction @@ -29,6 +30,8 @@ from utils.the_math.formulas import string_location_to_bucket_index from utils.the_math.measures import decimal_h_index +logger = logging.getLogger(__name__) + def score_question( question: Question, @@ -82,8 +85,12 @@ def generate_scoring_leaderboard_entries( "score_type": score_type, } - if leaderboard.finalize_time: - qs_filters["question__scheduled_close_time__lte"] = leaderboard.finalize_time + finalize_time = leaderboard.finalize_time or ( + leaderboard.project.close_date if leaderboard.project else None + ) + if finalize_time: + qs_filters["question__scheduled_close_time__lte"] = finalize_time + qs_filters["question__resolution_set_time__lte"] = finalize_time archived_scores = ArchivedScore.objects.filter(**qs_filters).prefetch_related( "question" @@ -96,9 +103,10 @@ def generate_scoring_leaderboard_entries( aggregation_method=OuterRef("aggregation_method"), score_type=Leaderboard.ScoreTypes.get_base_score(leaderboard.score_type), ) - if leaderboard.finalize_time: + if finalize_time: archived_scores_subquery = archived_scores_subquery.filter( - question__scheduled_close_time__lte=leaderboard.finalize_time + question__scheduled_close_time__lte=finalize_time, + question__resolution_set_time__lte=finalize_time, ) calculated_scores = calculated_scores.annotate( @@ -299,20 +307,25 @@ def assign_ranks( # set up exclusions exclusion_records = MedalExclusionRecord.objects.all() - if leaderboard.start_time: + start_time = leaderboard.start_time or ( + leaderboard.project.start_date if leaderboard.project else None + ) + end_time = leaderboard.end_time or ( + leaderboard.project.close_date if leaderboard.project else None + ) + finalize_time = leaderboard.finalize_time or ( + leaderboard.project.close_date if leaderboard.project else None + ) + if start_time: exclusion_records = exclusion_records.filter( - Q(end_time__isnull=True) | Q(end_time__gte=leaderboard.start_time) + Q(end_time__isnull=True) | Q(end_time__gte=start_time) ) - if leaderboard.end_time: + if end_time: # only exclude by end_time if it's set - exclusion_records = exclusion_records.filter( - start_time__lte=leaderboard.end_time - ) - elif leaderboard.finalize_time: + exclusion_records = exclusion_records.filter(start_time__lte=end_time) + elif finalize_time: # if end_time is not set, use finalize_time - exclusion_records = exclusion_records.filter( - start_time__lte=leaderboard.finalize_time - ) + exclusion_records = exclusion_records.filter(start_time__lte=finalize_time) excluded_ids: set[int | None] = set( exclusion_records.values_list("user_id", flat=True) ) @@ -398,6 +411,8 @@ def assign_prizes( def update_project_leaderboard( project: Project | None = None, leaderboard: Leaderboard | None = None, + force_update: bool = False, + force_finalize: bool = False, ) -> list[LeaderboardEntry]: if project is None and leaderboard is None: raise ValueError("Either project or leaderboard must be provided") @@ -408,6 +423,11 @@ def update_project_leaderboard( raise ValueError("Leaderboard not found") if leaderboard.score_type == Leaderboard.ScoreTypes.MANUAL: + logger.info("%s is manual, not updating", leaderboard.name) + return list(leaderboard.entries.all().order_by("rank")) + + if not force_update and leaderboard.finalized: + logger.warning("%s is already finalized, not updating", leaderboard.name) return list(leaderboard.entries.all().order_by("rank")) # new entries @@ -424,22 +444,32 @@ def update_project_leaderboard( new_entries = assign_prize_percentages(new_entries) # check if we're ready to finalize with medals and prizes + finalize_time = leaderboard.finalize_time or ( + project.close_date if project else None + ) if ( - ( - leaderboard.project.type + project + and ( + project.type in [ Project.ProjectTypes.SITE_MAIN, Project.ProjectTypes.TOURNAMENT, ] ) - and leaderboard.finalize_time - and (timezone.now() >= leaderboard.finalize_time) + and (force_finalize or (finalize_time and (timezone.now() >= finalize_time))) ): # assign medals new_entries = assign_medals(new_entries) # add prize if applicable - if project.prize_pool: - new_entries = assign_prizes(new_entries, project.prize_pool) + prize_pool = ( + leaderboard.prize_pool + if leaderboard.prize_pool is not None + else project.prize_pool + ) + if prize_pool: + new_entries = assign_prizes(new_entries, prize_pool) + # set finalize + Leaderboard.objects.filter(pk=leaderboard.pk).update(finalized=True) # save entries previous_entries_map = { @@ -644,10 +674,12 @@ def get_contributions( if leaderboard.finalize_time: calculated_scores = calculated_scores.filter( - question__scheduled_close_time__lte=leaderboard.finalize_time + question__scheduled_close_time__lte=leaderboard.finalize_time, + question__resolution_set_time__lte=leaderboard.finalize_time, ).prefetch_related("question") archived_scores = archived_scores.filter( - question__scheduled_close_time__lte=leaderboard.finalize_time + question__scheduled_close_time__lte=leaderboard.finalize_time, + question__resolution_set_time__lte=leaderboard.finalize_time, ).prefetch_related("question") scores = list(archived_scores) diff --git a/templates/admin/scoring/leaderboard_action_descriptions.html b/templates/admin/scoring/leaderboard_action_descriptions.html new file mode 100644 index 000000000..ddf025b2d --- /dev/null +++ b/templates/admin/scoring/leaderboard_action_descriptions.html @@ -0,0 +1,15 @@ +{% extends "admin/change_list.html" %} +{% load i18n %} + +{% block object-tools %} +
+

Available Actions:

+ + {{ block.super }} +
+{% endblock %} \ No newline at end of file