diff --git a/projects/models.py b/projects/models.py index 266714d9c..fe09343d7 100644 --- a/projects/models.py +++ b/projects/models.py @@ -288,10 +288,12 @@ def __str__(self): def save(self, *args, **kwargs): creating = not self.pk + # Check if the primary leaderboard is associated with this project if self.primary_leaderboard and self.primary_leaderboard.project != self: raise ValueError( "Primary leaderboard must be associated with this project." ) + super().save(*args, **kwargs) if ( creating @@ -300,6 +302,7 @@ def save(self, *args, **kwargs): in ( self.ProjectTypes.TOURNAMENT, self.ProjectTypes.QUESTION_SERIES, + self.ProjectTypes.COMMUNITY, ) ): # create default leaderboard when creating a new tournament/question series diff --git a/scoring/admin.py b/scoring/admin.py index dae4ae1a6..0748b5b55 100644 --- a/scoring/admin.py +++ b/scoring/admin.py @@ -70,14 +70,22 @@ def get_queryset(self, request): @admin.register(Leaderboard) class LeaderboardAdmin(admin.ModelAdmin): + change_list_template = "admin/scoring/leaderboard_action_descriptions.html" search_fields = ["name", "project", "score_type"] - list_display = ["__str__", "id", "project", "score_type"] + list_display = ["__str__", "id", "project", "score_type", "finalized"] autocomplete_fields = ["project"] list_filter = [ AutocompleteFilterFactory("Project", "project"), + "score_type", + "finalized", ] inlines = [LeaderboardEntryInline] - actions = ["make_primary_leaderboard", "update_leaderboards"] + actions = [ + "make_primary_leaderboard", + "update_leaderboards", + "force_update_leaderboards", + "force_finalize_and_asign_medals_leaderboards", + ] def make_primary_leaderboard(self, request, queryset): for leaderboard in queryset: @@ -91,16 +99,42 @@ def make_primary_leaderboard(self, request, queryset): messages.SUCCESS, ) - make_primary_leaderboard.short_description = ( - "Make selected leaderboards their project's primary_leaderboard" - ) + make_primary_leaderboard.short_description = "Make Primary Leaderboard" def update_leaderboards(self, request, queryset): leaderboard: Leaderboard for leaderboard in queryset: - update_project_leaderboard(leaderboard.project, leaderboard) + update_project_leaderboard( + leaderboard.project, + leaderboard, + ) - update_leaderboards.short_description = "Update selected Leaderboards" + update_leaderboards.short_description = "Update Leaderboards" + + def force_update_leaderboards(self, request, queryset): + leaderboard: Leaderboard + for leaderboard in queryset: + update_project_leaderboard( + leaderboard.project, + leaderboard, + force_update=True, + ) + + force_update_leaderboards.short_description = "Force Update Leaderboards" + + def force_finalize_and_asign_medals_leaderboards(self, request, queryset): + leaderboard: Leaderboard + for leaderboard in queryset: + update_project_leaderboard( + leaderboard.project, + leaderboard, + force_update=True, + force_finalize=True, + ) + + force_finalize_and_asign_medals_leaderboards.short_description = ( + "Force Update, Finalize, and Assign Medals/Prizes" + ) @admin.register(LeaderboardEntry) diff --git a/scoring/migrations/0007_leaderboard_finalized_leaderboard_prize_pool_and_more.py b/scoring/migrations/0007_leaderboard_finalized_leaderboard_prize_pool_and_more.py new file mode 100644 index 000000000..e59dbb7b0 --- /dev/null +++ b/scoring/migrations/0007_leaderboard_finalized_leaderboard_prize_pool_and_more.py @@ -0,0 +1,79 @@ +# Generated by Django 5.1.4 on 2025-01-16 19:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("scoring", "0006_alter_leaderboardentry_excluded_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="leaderboard", + name="finalized", + field=models.BooleanField( + default=False, + help_text="If true, this Leaderboard's entries cannot be updated except by a manual action in the admin panel. Automatically set to True the first time this leaderboard is updated after the finalize_time.", + ), + ), + migrations.AddField( + model_name="leaderboard", + name="prize_pool", + field=models.DecimalField( + blank=True, + decimal_places=2, + default=None, + help_text="Optional. If not set, the Project's prize_pool will be used instead.\n
- If the Project has a prize pool, but this leaderboard has none, set this to 0.\n ", + max_digits=15, + null=True, + ), + ), + migrations.AlterField( + model_name="leaderboard", + name="end_time", + field=models.DateTimeField( + blank=True, + help_text="Optional (required for global leaderboards).\n
- Global Leaderboards: filters for questions that have a scheduled_close_time before this (plus a grace period). Automatically set, do not change.\n
- Non-Global Leaderboards: has no effect on question filtering.\n
- Filtering MedalExclusionRecords: MedalExclusionRecords that have a start_time less than this (and no end_time or an end_time later that this Leaderboard's start_time) will be triggered. If not set, this Leaderboard's finalize_time will be used instead - it is recommended not to use this field unless required.\n ", + null=True, + ), + ), + migrations.AlterField( + model_name="leaderboard", + name="finalize_time", + field=models.DateTimeField( + blank=True, + help_text="Optional. If not set, the Project's close_date will be used instead.\n
- For all Leaderboards: used to filter out questions that have a resolution_set_time after this (as they were resolved after this Leaderboard was finalized).\n
- Filtering MedalExclusionRecords: If set and end_time is not set, MedalExclusionRecords that have a start_time less than this (and no end_time or an end_time later that this Leaderboard's start_time) will be triggered.\n ", + null=True, + ), + ), + migrations.AlterField( + model_name="leaderboard", + name="score_type", + field=models.CharField( + choices=[ + ("peer_tournament", "Peer Tournament"), + ("spot_peer_tournament", "Spot Peer Tournament"), + ("relative_legacy_tournament", "Relative Legacy Tournament"), + ("baseline_global", "Baseline Global"), + ("peer_global", "Peer Global"), + ("peer_global_legacy", "Peer Global Legacy"), + ("comment_insight", "Comment Insight"), + ("question_writing", "Question Writing"), + ("manual", "Manual"), + ], + help_text="\n \n \n \n \n \n \n \n \n \n \n
peer_tournament Sum of peer scores. Most likely what you want.
spot_peer_tournament Sum of spot peer scores.
relative_legacy Old site scoring.
baseline_global Sum of baseline scores.
peer_global Coverage-weighted average of peer scores.
peer_global_legacy Average of peer scores.
comment_insight H-index of upvotes for comments on questions.
question_writing H-index of number of forecasters / 10 on questions.
manual Does not automatically update. Manually set all entries.
\n ", + max_length=200, + ), + ), + migrations.AlterField( + model_name="leaderboard", + name="start_time", + field=models.DateTimeField( + blank=True, + help_text="Optional (required for global leaderboards). If not set, the Project's open_date will be used instead.\n
- Global Leaderboards: filters for questions that have an open time after this. Automatically set, do not change.\n
- Non-Global Leaderboards: has no effect on question filtering.\n
- Filtering MedalExclusionRecords: MedalExclusionRecords that have no end_time or an end_time greater than this (and a start_time before this Leaderboard's end_time or finalize_time) will be triggered.\n ", + null=True, + ), + ), + ] diff --git a/scoring/models.py b/scoring/models.py index d0a6ae426..2dd66762b 100644 --- a/scoring/models.py +++ b/scoring/models.py @@ -126,12 +126,12 @@ class Leaderboard(TimeStampedModel): ) class ScoreTypes(models.TextChoices): - RELATIVE_LEGACY_TOURNAMENT = "relative_legacy_tournament" - PEER_GLOBAL = "peer_global" - PEER_GLOBAL_LEGACY = "peer_global_legacy" PEER_TOURNAMENT = "peer_tournament" SPOT_PEER_TOURNAMENT = "spot_peer_tournament" + RELATIVE_LEGACY_TOURNAMENT = "relative_legacy_tournament" BASELINE_GLOBAL = "baseline_global" + PEER_GLOBAL = "peer_global" + PEER_GLOBAL_LEGACY = "peer_global_legacy" COMMENT_INSIGHT = "comment_insight" QUESTION_WRITING = "question_writing" MANUAL = "manual" @@ -162,10 +162,63 @@ def get_base_score(cls, score_type: str) -> Score.ScoreTypes: "Question Writing leaderboards do not have base scores" ) - score_type = models.CharField(max_length=200, choices=ScoreTypes.choices) - start_time = models.DateTimeField(null=True, blank=True) - end_time = models.DateTimeField(null=True, blank=True) - finalize_time = models.DateTimeField(null=True, blank=True) + score_type = models.CharField( + max_length=200, + choices=ScoreTypes.choices, + help_text=""" + + + + + + + + + + +
peer_tournament Sum of peer scores. Most likely what you want.
spot_peer_tournament Sum of spot peer scores.
relative_legacy Old site scoring.
baseline_global Sum of baseline scores.
peer_global Coverage-weighted average of peer scores.
peer_global_legacy Average of peer scores.
comment_insight H-index of upvotes for comments on questions.
question_writing H-index of number of forecasters / 10 on questions.
manual Does not automatically update. Manually set all entries.
+ """, + ) + start_time = models.DateTimeField( + null=True, + blank=True, + help_text="""Optional (required for global leaderboards). If not set, the Project's open_date will be used instead. +
- Global Leaderboards: filters for questions that have an open time after this. Automatically set, do not change. +
- Non-Global Leaderboards: has no effect on question filtering. +
- Filtering MedalExclusionRecords: MedalExclusionRecords that have no end_time or an end_time greater than this (and a start_time before this Leaderboard's end_time or finalize_time) will be triggered. + """, + ) + end_time = models.DateTimeField( + null=True, + blank=True, + help_text="""Optional (required for global leaderboards). +
- Global Leaderboards: filters for questions that have a scheduled_close_time before this (plus a grace period). Automatically set, do not change. +
- Non-Global Leaderboards: has no effect on question filtering. +
- Filtering MedalExclusionRecords: MedalExclusionRecords that have a start_time less than this (and no end_time or an end_time later that this Leaderboard's start_time) will be triggered. If not set, this Leaderboard's finalize_time will be used instead - it is recommended not to use this field unless required. + """, + ) + finalize_time = models.DateTimeField( + null=True, + blank=True, + help_text="""Optional. If not set, the Project's close_date will be used instead. +
- For all Leaderboards: used to filter out questions that have a resolution_set_time after this (as they were resolved after this Leaderboard was finalized). +
- Filtering MedalExclusionRecords: If set and end_time is not set, MedalExclusionRecords that have a start_time less than this (and no end_time or an end_time later that this Leaderboard's start_time) will be triggered. + """, + ) + finalized = models.BooleanField( + default=False, + help_text="If true, this Leaderboard's entries cannot be updated except by a manual action in the admin panel. Automatically set to True the first time this leaderboard is updated after the finalize_time.", + ) + prize_pool = models.DecimalField( + default=None, + decimal_places=2, + max_digits=15, + null=True, + blank=True, + help_text="""Optional. If not set, the Project's prize_pool will be used instead. +
- If the Project has a prize pool, but this leaderboard has none, set this to 0. + """, + ) def __str__(self): if self.name: @@ -179,74 +232,72 @@ def get_questions(self) -> QuerySet[Question]: related_posts__post__curation_status=Post.CurationStatus.APPROVED ) - if self.project and self.project.type == Project.ProjectTypes.SITE_MAIN: - # global leaderboard - if self.start_time is None or self.end_time is None: - raise ValueError("Global leaderboards must have start and end times") - - questions = questions.filter_public().filter( - related_posts__post__in=Post.objects.filter_for_main_feed() - ) - - if self.score_type == self.ScoreTypes.COMMENT_INSIGHT: - # post must be published - return questions.filter( - related_posts__post__published_at__lt=self.end_time - ) - elif self.score_type == self.ScoreTypes.QUESTION_WRITING: - # post must be published, and can't be resolved before the start_time - # of the leaderboard - return questions.filter( - Q(scheduled_close_time__gte=self.start_time) - & ( - Q(actual_close_time__isnull=True) - | Q(actual_close_time__gte=self.start_time) - ), - related_posts__post__published_at__lt=self.end_time, + if not (self.project and self.project.type == Project.ProjectTypes.SITE_MAIN): + # normal Project leaderboard + if self.project: + questions = questions.filter( + Q(related_posts__post__projects=self.project) + | Q(related_posts__post__default_project=self.project), ) + return questions.distinct("id") - close_grace_period = timedelta(days=3) - resolve_grace_period = timedelta(days=100) + # global leaderboard + if self.start_time is None or self.end_time is None: + raise ValueError("Global leaderboards must have start and end times") + + questions = questions.filter_public().filter( + related_posts__post__in=Post.objects.filter_for_main_feed() + ) - questions = questions.filter( - Q(actual_resolve_time__isnull=True) - | Q(actual_resolve_time__lte=self.end_time + resolve_grace_period), - open_time__gte=self.start_time, - open_time__lt=self.end_time, - scheduled_close_time__lte=self.end_time + close_grace_period, + if self.score_type == self.ScoreTypes.COMMENT_INSIGHT: + # post must be published + return questions.filter(related_posts__post__published_at__lt=self.end_time) + elif self.score_type == self.ScoreTypes.QUESTION_WRITING: + # post must be published, and can't be resolved before the start_time + # of the leaderboard + return questions.filter( + Q(scheduled_close_time__gte=self.start_time) + & ( + Q(actual_close_time__isnull=True) + | Q(actual_close_time__gte=self.start_time) + ), + related_posts__post__published_at__lt=self.end_time, ) - gl_dates = global_leaderboard_dates() - checked_intervals: list[tuple[datetime, datetime]] = [] - for start, end in gl_dates[::-1]: # must be in reverse order, biggest first - if ( - (self.start_time, self.end_time) == (start, end) - or start < self.start_time - or self.end_time < end - ): - continue - to_add = True - for checked_start, checked_end in checked_intervals: - if checked_start < start and end < checked_end: - to_add = False - break - if to_add: - checked_intervals.append((start, end)) - questions = questions.filter( - Q(open_time__lt=start) - | Q(scheduled_close_time__gt=end + close_grace_period) - | Q(actual_resolve_time__gt=end + resolve_grace_period) - ) + close_grace_period = timedelta(days=3) + resolve_grace_period = timedelta(days=100) - return questions + questions = questions.filter( + Q(actual_resolve_time__isnull=True) + | Q(actual_resolve_time__lte=self.end_time + resolve_grace_period), + open_time__gte=self.start_time, + open_time__lt=self.end_time, + scheduled_close_time__lte=self.end_time + close_grace_period, + ) - if self.project: - questions = questions.filter( - Q(related_posts__post__projects=self.project) - | Q(related_posts__post__default_project=self.project) - ) + gl_dates = global_leaderboard_dates() + checked_intervals: list[tuple[datetime, datetime]] = [] + for start, end in gl_dates[::-1]: # must be in reverse order, biggest first + if ( + (self.start_time, self.end_time) == (start, end) + or start < self.start_time + or self.end_time < end + ): + continue + to_add = True + for checked_start, checked_end in checked_intervals: + if checked_start < start and end < checked_end: + to_add = False + break + if to_add: + checked_intervals.append((start, end)) + questions = questions.filter( + Q(open_time__lt=start) + | Q(scheduled_close_time__gt=end + close_grace_period) + | Q(actual_resolve_time__gt=end + resolve_grace_period) + ) - return questions.distinct("id") + return questions def name_and_slug_for_global_leaderboard_dates( diff --git a/scoring/utils.py b/scoring/utils.py index 4570ff10c..88fddebb6 100644 --- a/scoring/utils.py +++ b/scoring/utils.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from decimal import Decimal from io import StringIO +import logging import numpy as np from django.db import transaction @@ -29,6 +30,8 @@ from utils.the_math.formulas import string_location_to_bucket_index from utils.the_math.measures import decimal_h_index +logger = logging.getLogger(__name__) + def score_question( question: Question, @@ -82,8 +85,12 @@ def generate_scoring_leaderboard_entries( "score_type": score_type, } - if leaderboard.finalize_time: - qs_filters["question__scheduled_close_time__lte"] = leaderboard.finalize_time + finalize_time = leaderboard.finalize_time or ( + leaderboard.project.close_date if leaderboard.project else None + ) + if finalize_time: + qs_filters["question__scheduled_close_time__lte"] = finalize_time + qs_filters["question__resolution_set_time__lte"] = finalize_time archived_scores = ArchivedScore.objects.filter(**qs_filters).prefetch_related( "question" @@ -96,9 +103,10 @@ def generate_scoring_leaderboard_entries( aggregation_method=OuterRef("aggregation_method"), score_type=Leaderboard.ScoreTypes.get_base_score(leaderboard.score_type), ) - if leaderboard.finalize_time: + if finalize_time: archived_scores_subquery = archived_scores_subquery.filter( - question__scheduled_close_time__lte=leaderboard.finalize_time + question__scheduled_close_time__lte=finalize_time, + question__resolution_set_time__lte=finalize_time, ) calculated_scores = calculated_scores.annotate( @@ -299,20 +307,25 @@ def assign_ranks( # set up exclusions exclusion_records = MedalExclusionRecord.objects.all() - if leaderboard.start_time: + start_time = leaderboard.start_time or ( + leaderboard.project.start_date if leaderboard.project else None + ) + end_time = leaderboard.end_time or ( + leaderboard.project.close_date if leaderboard.project else None + ) + finalize_time = leaderboard.finalize_time or ( + leaderboard.project.close_date if leaderboard.project else None + ) + if start_time: exclusion_records = exclusion_records.filter( - Q(end_time__isnull=True) | Q(end_time__gte=leaderboard.start_time) + Q(end_time__isnull=True) | Q(end_time__gte=start_time) ) - if leaderboard.end_time: + if end_time: # only exclude by end_time if it's set - exclusion_records = exclusion_records.filter( - start_time__lte=leaderboard.end_time - ) - elif leaderboard.finalize_time: + exclusion_records = exclusion_records.filter(start_time__lte=end_time) + elif finalize_time: # if end_time is not set, use finalize_time - exclusion_records = exclusion_records.filter( - start_time__lte=leaderboard.finalize_time - ) + exclusion_records = exclusion_records.filter(start_time__lte=finalize_time) excluded_ids: set[int | None] = set( exclusion_records.values_list("user_id", flat=True) ) @@ -398,6 +411,8 @@ def assign_prizes( def update_project_leaderboard( project: Project | None = None, leaderboard: Leaderboard | None = None, + force_update: bool = False, + force_finalize: bool = False, ) -> list[LeaderboardEntry]: if project is None and leaderboard is None: raise ValueError("Either project or leaderboard must be provided") @@ -408,6 +423,11 @@ def update_project_leaderboard( raise ValueError("Leaderboard not found") if leaderboard.score_type == Leaderboard.ScoreTypes.MANUAL: + logger.info("%s is manual, not updating", leaderboard.name) + return list(leaderboard.entries.all().order_by("rank")) + + if not force_update and leaderboard.finalized: + logger.warning("%s is already finalized, not updating", leaderboard.name) return list(leaderboard.entries.all().order_by("rank")) # new entries @@ -424,22 +444,32 @@ def update_project_leaderboard( new_entries = assign_prize_percentages(new_entries) # check if we're ready to finalize with medals and prizes + finalize_time = leaderboard.finalize_time or ( + project.close_date if project else None + ) if ( - ( - leaderboard.project.type + project + and ( + project.type in [ Project.ProjectTypes.SITE_MAIN, Project.ProjectTypes.TOURNAMENT, ] ) - and leaderboard.finalize_time - and (timezone.now() >= leaderboard.finalize_time) + and (force_finalize or (finalize_time and (timezone.now() >= finalize_time))) ): # assign medals new_entries = assign_medals(new_entries) # add prize if applicable - if project.prize_pool: - new_entries = assign_prizes(new_entries, project.prize_pool) + prize_pool = ( + leaderboard.prize_pool + if leaderboard.prize_pool is not None + else project.prize_pool + ) + if prize_pool: + new_entries = assign_prizes(new_entries, prize_pool) + # set finalize + Leaderboard.objects.filter(pk=leaderboard.pk).update(finalized=True) # save entries previous_entries_map = { @@ -644,10 +674,12 @@ def get_contributions( if leaderboard.finalize_time: calculated_scores = calculated_scores.filter( - question__scheduled_close_time__lte=leaderboard.finalize_time + question__scheduled_close_time__lte=leaderboard.finalize_time, + question__resolution_set_time__lte=leaderboard.finalize_time, ).prefetch_related("question") archived_scores = archived_scores.filter( - question__scheduled_close_time__lte=leaderboard.finalize_time + question__scheduled_close_time__lte=leaderboard.finalize_time, + question__resolution_set_time__lte=leaderboard.finalize_time, ).prefetch_related("question") scores = list(archived_scores) diff --git a/templates/admin/scoring/leaderboard_action_descriptions.html b/templates/admin/scoring/leaderboard_action_descriptions.html new file mode 100644 index 000000000..ddf025b2d --- /dev/null +++ b/templates/admin/scoring/leaderboard_action_descriptions.html @@ -0,0 +1,15 @@ +{% extends "admin/change_list.html" %} +{% load i18n %} + +{% block object-tools %} +
+

Available Actions:

+ + {{ block.super }} +
+{% endblock %} \ No newline at end of file