From e6e1a66cbb5cebc84c682bbe5acccaa587d23719 Mon Sep 17 00:00:00 2001 From: Anuj Gupta <84966248+Anuj-Gupta4@users.noreply.github.com> Date: Mon, 13 Jan 2025 21:17:37 +0545 Subject: [PATCH] feat(backend): add filters for submission date in submission table and downloads (#2077) * feat: add filters for submission date in submission table and downloads * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * feat: add submitted date range filter for submission downloads * feat: refactor date range filter formatting in submission download --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../app/submissions/submission_crud.py | 8 +-- .../app/submissions/submission_routes.py | 61 +++++++++++++------ src/backend/tests/test_submission_routes.py | 10 ++- 3 files changed, 53 insertions(+), 26 deletions(-) diff --git a/src/backend/app/submissions/submission_crud.py b/src/backend/app/submissions/submission_crud.py index dc563da764..a27217e3e6 100644 --- a/src/backend/app/submissions/submission_crud.py +++ b/src/backend/app/submissions/submission_crud.py @@ -109,20 +109,20 @@ # return final_zip_file_path -async def gather_all_submission_csvs(project: DbProject): +async def gather_all_submission_csvs(project: DbProject, filters: dict): """Gather all of the submission CSVs for a project. Generate a single zip with all submissions. """ log.info(f"Downloading all CSV submissions for project {project.id}") xform = get_odk_form(project.odk_credentials) - file = xform.getSubmissionMedia(project.odkid, project.odk_form_id) + file = xform.getSubmissionMedia(project.odkid, project.odk_form_id, filters) return file.content -async def download_submission_in_json(project: DbProject): +async def download_submission_in_json(project: DbProject, filters: dict): """Download submission data from ODK Central.""" - if data := await get_submission_by_project(project, {}): + if data := await get_submission_by_project(project, filters): json_data = data else: json_data = None diff --git a/src/backend/app/submissions/submission_routes.py b/src/backend/app/submissions/submission_routes.py index 2e767705b2..f9494276e5 100644 --- a/src/backend/app/submissions/submission_routes.py +++ b/src/backend/app/submissions/submission_routes.py @@ -65,6 +65,11 @@ async def read_submissions( async def download_submission( project_user: Annotated[ProjectUserDict, Depends(project_contributors)], export_json: bool = True, + submitted_date_range: Optional[str] = Query( + None, + title="Submitted Date Range", + description="Date range in format (e.g., 'YYYY-MM-DD,YYYY-MM-DD')", + ), ): """Download the submissions for a given project. @@ -74,12 +79,23 @@ async def download_submission( Union[list[dict], File]: JSON of submissions, or submission file. """ project = project_user.get("project") + filters = None + if submitted_date_range: + start_date, end_date = submitted_date_range.split(",") + filters = { + "$filter": ( + f"__system/submissionDate ge {start_date}T00:00:00+00:00 " + f"and __system/submissionDate le {end_date}T23:59:59.999+00:00" + ) + } if not export_json: - file_content = await submission_crud.gather_all_submission_csvs(project) + file_content = await submission_crud.gather_all_submission_csvs( + project, filters + ) headers = {"Content-Disposition": f"attachment; filename={project.slug}.zip"} return Response(file_content, headers=headers) - return await submission_crud.download_submission_in_json(project) + return await submission_crud.download_submission_in_json(project, filters) # # FIXME 07/06/2024 since osm-fieldwork update @@ -263,8 +279,10 @@ async def submission_table( task_id: Optional[int] = None, submitted_by: Optional[str] = None, review_state: Optional[str] = None, - submitted_date: Optional[str] = Query( - None, title="Submitted Date", description="Date in format (e.g., 'YYYY-MM-DD')" + submitted_date_range: Optional[str] = Query( + None, + title="Submitted Date Range", + description="Date range in format (e.g., 'YYYY-MM-DD,YYYY-MM-DD')", ), ): """This api returns the submission table of a project. @@ -278,31 +296,29 @@ async def submission_table( "$wkt": True, } - if submitted_date: + if submitted_date_range: + start_date, end_date = submitted_date_range.split(",") filters["$filter"] = ( "__system/submissionDate ge {}T00:00:00+00:00 " "and __system/submissionDate le {}T23:59:59.999+00:00" - ).format(submitted_date, submitted_date) - - if submitted_by: - if "$filter" in filters: - filters["$filter"] += f"and (username eq '{submitted_by}')" - else: - filters["$filter"] = f"username eq '{submitted_by}'" + ).format(start_date, end_date) if review_state: - if "$filter" in filters: - filters["$filter"] += f" and (__system/reviewState eq '{review_state}')" - else: - filters["$filter"] = f"__system/reviewState eq '{review_state}'" + review_filter = f"__system/reviewState eq '{review_state}'" + filters["$filter"] = ( + f"{filters['$filter']} and {review_filter}" + if "$filter" in filters + else review_filter + ) data = await submission_crud.get_submission_by_project(project, filters) total_count = data.get("@odata.count", 0) submissions = data.get("value", []) - instance_ids = [] - for submission in submissions: - if submission["__system"]["attachmentsPresent"] != 0: - instance_ids.append(submission["__id"]) + instance_ids = [ + sub["__id"] + for sub in submissions + if sub["__system"].get("attachmentsPresent", 0) != 0 + ] if instance_ids: background_task_id = await DbBackgroundTask.create( @@ -324,6 +340,11 @@ async def submission_table( if task_id: submissions = [sub for sub in submissions if sub.get("task_id") == str(task_id)] + if submitted_by: + submissions = [ + sub for sub in submissions if sub.get("username") == submitted_by + ] + start_index = (page - 1) * results_per_page end_index = start_index + results_per_page paginated_submissions = submissions[start_index:end_index] diff --git a/src/backend/tests/test_submission_routes.py b/src/backend/tests/test_submission_routes.py index 7b1bdc2585..4fd1c2e2c3 100644 --- a/src/backend/tests/test_submission_routes.py +++ b/src/backend/tests/test_submission_routes.py @@ -48,8 +48,11 @@ async def test_download_submission_json(client, submission): """Test downloading submissions as JSON.""" odk_project = submission["project"] + date = submission["submission_data"].createdAt.strftime("%Y-%m-%d") + response = await client.get( - f"/submission/download?project_id={odk_project.id}&export_json=true" + f"/submission/download?project_id={odk_project.id}" + f"&submitted_date_range={date},{date}&export_json=true" ) assert response.status_code == 200, ( @@ -76,8 +79,11 @@ async def test_download_submission_file(client, submission): """Test downloading submissions as a ZIP file.""" odk_project = submission["project"] + date = submission["submission_data"].createdAt.strftime("%Y-%m-%d") + response = await client.get( - f"/submission/download?project_id={odk_project.id}&export_json=false" + f"/submission/download?project_id={odk_project.id}" + f"&submitted_date_range={date},{date}&export_json=false" ) assert response.status_code == 200, (