diff --git a/README.rst b/README.rst index 856f288..81006b6 100644 --- a/README.rst +++ b/README.rst @@ -35,7 +35,7 @@ Custom QA / data management reports using SQL VIEWS =================================================== Although not absolutely necessary, it is convenient to base a QA report on an SQL VIEW. As -issues are resolved, the SQL VIEW reflects the changes. +data issues are resolved, the SQL VIEW reflects the current data. A QA report based on an SQL VIEW can be represented by a model class. By registering the model class in Admin, all the functionality of the ModelAdmin class is available to show the report. @@ -46,15 +46,14 @@ First, within your EDC project, create a `_reports` app. For example `Met meta_edc meta_edc/meta_reports meta_edc/meta_reports/admin - meta_edc/meta_reports/admin/unmanaged - meta_edc/meta_reports/admin/unmanaged/my_view_in_sql_admin.py - meta_edc/meta_reports/admin/report_note_admin.py + meta_edc/meta_reports/admin/dbviews + meta_edc/meta_reports/admin/dbviews/my_view_in_sql_admin.py meta_edc/meta_reports/migrations meta_edc/meta_reports/migrations/0001_myviewinsql.py meta_edc/meta_reports/models - meta_edc/meta_reports/models/unmanaged - meta_edc/meta_reports/models/unmanaged/my_view_in_sql.py - meta_edc/meta_reports/models/unmanaged/my_view_in_sql.sql + meta_edc/meta_reports/models/dbviews + meta_edc/meta_reports/models/dbviews/mymodel/unmanaged_model.py + meta_edc/meta_reports/models/dbviews/mymodel/view_definition.py meta_edc/meta_reports/admin_site.py meta_edc/meta_reports/apps.py meta_edc/ ... @@ -81,16 +80,43 @@ Now that you have created the basic structure for the Reports App, create an SQL * Column ``report_model`` is in label_lower format. * Suffix the view name with ``_view``. +To manage SQL view code, we use ``django_dbviews``. This module helps by using migrations to manage changes to the SQL view code. + + +The ``view_defintion.py`` might look like this: + .. code-block:: sql - create view my_view_in_sql_view as ( - select *, uuid() as 'id', now() as 'created', - 'meta_reports.myviewinsql' as report_model + from edc_qareports.sql_generator import SqlViewGenerator + + def get_view_definition() -> dict: + subquery = """ + select subject_identifier, site_id, appt_datetime, `first_value`, + `second_value`, `third_value`, + datediff(`third_date`, `first_date`) as `interval_days`, + datediff(now(), `first_date`) as `from_now_days` from ( - select distinct `subject_identifier`, `site_id`, col1, col2, col3 - from some_crf_table - where col1 is null - ) as A + select subject_identifier, site_id, appt_datetime, + FIRST_VALUE(visit_code) OVER w as `first_value`, + NTH_VALUE(visit_code, 2) OVER w as `second_value`, + NTH_VALUE(visit_code, 3) OVER w as `third_value`, + FIRST_VALUE(appt_datetime) OVER w as `first_date`, + NTH_VALUE(appt_datetime, 3) OVER w as `third_date` + from edc_appointment_appointment where visit_code_sequence=0 and appt_status="New" + and appt_datetime <= now() + WINDOW w as (PARTITION BY subject_identifier order by appt_datetime ROWS UNBOUNDED PRECEDING) + ) as B + where `second_value` is not null and `third_value` is not null + """ # noqa + sql_view = SqlViewGenerator( + report_model="meta_reports.unattendedthreeinrow", + ordering=["subject_identifier", "site_id"], + ) + return { + "django.db.backends.mysql": sql_view.as_mysql(subquery), + "django.db.backends.postgresql": sql_view.as_postgres(subquery), + "django.db.backends.sqlite3": sql_view.as_sqlite(subquery), + } Using a model class to represent your QA Report +++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/edc_qareports/sql_generator/__init__.py b/edc_qareports/sql_generator/__init__.py index 41c0d3d..779a65b 100644 --- a/edc_qareports/sql_generator/__init__.py +++ b/edc_qareports/sql_generator/__init__.py @@ -1,3 +1,4 @@ -from .generate_subquery_for_missing_values import generate_subquery_for_missing_values -from .select_from import SelectFrom +from .qa_case import QaCase, QaCaseError from .sql_view_generator import SqlViewGenerator +from .subquery import Subquery +from .subquery_from_dict import subquery_from_dict diff --git a/edc_qareports/sql_generator/qa_case.py b/edc_qareports/sql_generator/qa_case.py new file mode 100644 index 0000000..61cbca1 --- /dev/null +++ b/edc_qareports/sql_generator/qa_case.py @@ -0,0 +1,43 @@ +from dataclasses import dataclass, field + +from django.apps import apps as django_apps +from django.db import OperationalError, connection + +from .subquery_from_dict import subquery_from_dict + + +class QaCaseError(Exception): + pass + + +@dataclass(kw_only=True) +class QaCase: + label: str = None + dbtable: str = None + label_lower: str = None + fld_name: str | None = None + where: str | None = None + list_tables: list[tuple[str, str, str]] | None = field(default_factory=list) + + def __post_init__(self): + if self.fld_name is None and self.where is None: + raise QaCaseError("Expected either 'fld_name' or 'where'. Got None for both.") + elif self.fld_name is not None and self.where is not None: + raise QaCaseError("Expected either 'fld_name' or 'where', not both.") + + @property + def sql(self): + return subquery_from_dict([self.__dict__]) + + @property + def model_cls(self): + return django_apps.get_model(self.label_lower) + + def fetchall(self): + sql = subquery_from_dict([self.__dict__]) + with connection.cursor() as cursor: + try: + cursor.execute(sql) + except OperationalError as e: + raise QaCaseError(f"{e}. See {self}.") + return cursor.fetchall() diff --git a/edc_qareports/sql_generator/sql_view_generator.py b/edc_qareports/sql_generator/sql_view_generator.py index 53673e2..730e4e2 100644 --- a/edc_qareports/sql_generator/sql_view_generator.py +++ b/edc_qareports/sql_generator/sql_view_generator.py @@ -6,11 +6,11 @@ @dataclass(kw_only=True) class SqlViewGenerator: """A class to generate SQL view statements given a select - statement. + statement or subquery. - Generated SQL is compatible with mysql, pgsql and sqlite3. + Generated SQL is compatible with mysql, postgres and sqlite3. - The given `select_statment` is not validated. + The given `subquery` is not validated. For use with view definitions. """ diff --git a/edc_qareports/sql_generator/select_from.py b/edc_qareports/sql_generator/subquery.py similarity index 98% rename from edc_qareports/sql_generator/select_from.py rename to edc_qareports/sql_generator/subquery.py index 831e312..0e25a47 100644 --- a/edc_qareports/sql_generator/select_from.py +++ b/edc_qareports/sql_generator/subquery.py @@ -2,7 +2,7 @@ @dataclass(kw_only=True) -class SelectFrom: +class Subquery: label: str = None label_lower: str = None dbtable: str = None diff --git a/edc_qareports/sql_generator/generate_subquery_for_missing_values.py b/edc_qareports/sql_generator/subquery_from_dict.py similarity index 56% rename from edc_qareports/sql_generator/generate_subquery_for_missing_values.py rename to edc_qareports/sql_generator/subquery_from_dict.py index b27a5a8..a3f50ed 100644 --- a/edc_qareports/sql_generator/generate_subquery_for_missing_values.py +++ b/edc_qareports/sql_generator/subquery_from_dict.py @@ -1,8 +1,15 @@ -from .select_from import SelectFrom +from __future__ import annotations +from typing import TYPE_CHECKING -def generate_subquery_for_missing_values( - cases: list[dict[str:str, str:str, str:str]], +from .subquery import Subquery + +if TYPE_CHECKING: + from .qa_case import QaCase + + +def subquery_from_dict( + cases: list[dict[str:str, str:str, str:str] | QaCase], as_list: bool | None = False, ) -> str | list: """Returns an SQL select statement as a union of the select @@ -20,10 +27,13 @@ def generate_subquery_for_missing_values( Note: `list_field` is the CRF id field, for example: left join as on crf.=.id """ - select_from_list = [] + subqueries = [] for case in cases: - select_from = SelectFrom(**case) - select_from_list.append(select_from.sql) + try: + subquery = case.sql + except AttributeError: + subquery = Subquery(**case).sql + subqueries.append(subquery) if as_list: - return select_from_list - return " UNION ".join(select_from_list) + return subqueries + return " UNION ".join(subqueries)