Merge pull request #98 from TRON-Bioinformatics/develop

Release 2.1.0

covigator/__init__.py

@@ -1,4 +1,4 @@
-VERSION = "v2.0.0"
+VERSION = "v2.1.0"
 ANALYSIS_PIPELINE_VERSION = "v0.15.0"
 
 MISSENSE_VARIANT = "missense_variant"

covigator/dashboard/dashboard.py

@@ -36,6 +36,7 @@
 ENA_HREF = "/ena"
 DOWNLOAD_HREF = "/download"
 ACKNOWLEDGEMENTS_HREF = "/acknowledgements"
+FEEDBACK_HREF = "https://feedback.userreport.com/ca0af557-f613-475f-b1d4-0519d4186f69/"
 TAB_STYLE = {"color": "#003c78", 'margin-right': '15px'}
 
 ID_TAB_CONTENT = "tab-content"
@@ -124,6 +125,9 @@ def serve_layout(self):
                                                 dbc.DropdownMenuItem(
                                                     "Acknowledgements", href=ACKNOWLEDGEMENTS_HREF,
                                                     style={'font-size': '150%', "color": "#003c78"}),
+                                                dbc.DropdownMenuItem(
+                                                    "Feedback", href=FEEDBACK_HREF, target="_blank",
+                                                    style={'font-size': '150%', "color": "#003c78"}),
                                             ],
                                             align_end=True,
                                             size="lg",
@@ -193,8 +197,8 @@ def get_application(self) -> dash.Dash:
                 #
                 # Necessary for "true" mobile support.
                 {
-                  'name': 'viewport',
-                  'content': 'width=device-width, initial-scale=1.0'
+                    'name': 'viewport',
+                    'content': 'width=device-width, initial-scale=1.0'
                 }
             ]
         )

covigator/dashboard/figures/lineages.py

@@ -55,42 +55,63 @@ def discrete_background_color_bins(df, n_bins=5, columns='all', colors='Blues'):
 
 class LineageFigures(Figures):
 
-    def get_lineages_plot(self, data_source: str, countries=None, lineages=None, time_period=14):
+    def get_lineages_plot(self, data_source: str, date_start, date_end, countries=None, lineages=None, time_period=14, prevalence=0):
         logger.debug("Getting data on samples by country...")
         data = self.queries.get_accumulated_lineages_by_country(
             data_source=data_source, countries=countries, lineages=lineages)
         graph = dcc.Markdown("""**No data for the current selection**""")
         if data is not None and data.shape[0] > 0:
+            # Filter data based on start and end range
+            data = data.loc[(data.date >= date_start) & (data.date <= date_end)]
             logger.debug("Prepare plot on samples by lineage...")
             lineages = list(data.sort_values("cumsum", ascending=False).lineage.unique())
-
+            if prevalence > 0:
+                # Calculate cumulative sum for each lineage in selected time interval
+                cumsum_for_range = data.groupby('lineage')[['count_y']].sum()
+                cumsum_for_range = cumsum_for_range.reset_index(level="lineage").rename(columns={'count_y': 'cumsum_for_range'})
+                # Add cumulative sums grouped by lineages to get total number of samples in time interval
+                total_in_range = cumsum_for_range["cumsum_for_range"].sum()
+                cumsum_for_range["prevalence_in_range"] = cumsum_for_range["cumsum_for_range"] / total_in_range
+                # Get lineages with certain prevalence in time interval
+                prevalent_lineages = cumsum_for_range.loc[
+                    cumsum_for_range.prevalence_in_range > prevalence].get('lineage').tolist()
+                # Group non-prevalent lineages into others and sum columns ratio_per_date, cumsum and count
+                others = data.loc[~data.lineage.isin(prevalent_lineages)].groupby('date')\
+                    [['ratio_per_date', 'cumsum', 'count']].sum().reset_index()
+                others['lineage'] = 'others'
+                data = data.loc[data.lineage.isin(prevalent_lineages),
+                    ['date', 'lineage', 'ratio_per_date', 'cumsum', 'count']]
+                # Merge others back with top prevalent lineages
+                data = pd.concat([data, others], ignore_index=True)
+                data.sort_values(by='date', inplace=True)
+
+            data = data.loc[:, ['date', 'lineage', 'ratio_per_date', 'cumsum', 'count']]
             fig = make_subplots(rows=2, cols=1)
 
+            fig1_hovertemplate = "num.samples=%{y}, % daily samples=%{customdata[0]:.0%}, increment=%{customdata[1]}"
             fig1 = px.area(
                 data, x="date", y="cumsum", color="lineage",
                 category_orders={"lineage": lineages[::-1]},
-                labels={"ratio_per_date": "% daily samples", "cumsum": "num. samples", "count": "increment"},
-                hover_data=["ratio_per_date", "count"],
+                custom_data=["ratio_per_date", "count"],
                 color_discrete_sequence=px.colors.qualitative.Vivid)
-            fig1.update_traces(line=dict(width=0.5), showlegend=False)
+            fig1.update_traces(line=dict(width=0.5), showlegend=False, hovertemplate=fig1_hovertemplate)
 
             # Perform smoothing by using a simple moving average
             # If time_period is False the un-smoothed data is plotted
-            smooth_data = data.loc[:, ['date', 'lineage', 'ratio_per_date', 'cumsum', 'count']]
             if time_period:
                 # Group data by lineage to calculate correct average for each lineage in df
-                sma_df = smooth_data.groupby('lineage')[['lineage', 'ratio_per_date']]
+                sma_df = data.groupby('lineage')[['lineage', 'ratio_per_date']]
                 sma_df = sma_df.rolling(time_period, min_periods=1).mean()
                 sma_df = sma_df.reset_index(level='lineage')[['ratio_per_date']]
                 # Update columns with moving average over selected period
-                smooth_data.update(sma_df)
+                data.update(sma_df)
+            # Custom hover tooltip label for lineages plot
+            fig2_hovertemplate = "daily samples: %{y}"
             fig2 = px.area(
-                smooth_data, x="date", y="ratio_per_date", color="lineage",
+                data, x="date", y="ratio_per_date", color="lineage",
                 category_orders={"lineage": lineages[::-1]},
-                labels={"ratio_per_date": "% daily samples", "cumsum": "num. samples", "count": "increment"},
-                hover_data=["cumsum", "count"],
                 color_discrete_sequence=px.colors.qualitative.Vivid)
-            fig2.update_traces(line=dict(width=0.5))
+            fig2.update_traces(line=dict(width=0.5), hovertemplate=fig2_hovertemplate)
 
             for trace in fig1["data"]:
                 fig.append_trace(trace, row=1, col=1)
@@ -111,6 +132,7 @@ def get_lineages_plot(self, data_source: str, countries=None, lineages=None, tim
                     'title': '% samples'
                 },
                 height=700,
+                hovermode="x unified"
             )
             fig.update_xaxes(showspikes=True)
 

covigator/dashboard/tabs/acknowledgements.py

@@ -86,13 +86,7 @@ def get_tab_acknowledgements():
                 ]),
 
                 html.Br(),
-                html.P("We value your feedback!"),
-                html.P("If you found an error or have a feature request, please, report it:"),
-                html.Iframe(srcDoc="""
-                        <a class="github-button" href="https://github.com/tron-bioinformatics/covigator/issues/new?assignees=&labels=bug&template=bug_report.md&title=" data-color-scheme="no-preference: light; light: light; dark: light;" data-size="large" data-show-count="false" aria-label="Issue tron-bioinformatics/covigator on GitHub">Issue</a>
-                        <a class="github-button" href="https://github.com/tron-bioinformatics/covigator/issues/new?assignees=&labels=bug&template=feature_request.md&title=" data-color-scheme="no-preference: light; light: light; dark: light;" data-size="large" data-show-count="false" aria-label="Issue tron-bioinformatics/covigator on GitHub">Feature request</a>
-                        <script async defer src="https://buttons.github.io/buttons.js"></script>
-                       """, style={'border': 0, 'height': '50px'}),
+
                 html.P("If you liked our work support us in GitHub:"),
                 html.Iframe(srcDoc="""
                             <a class="github-button" href="https://github.com/tron-bioinformatics/covigator" data-color-scheme="no-preference: light; light: light; dark: light;" data-icon="octicon-star" data-size="large" data-show-count="true" aria-label="Star tron-bioinformatics/covigator on GitHub">Star</a>

covigator/dashboard/tabs/lineages.py

@@ -1,10 +1,11 @@
+from datetime import timedelta, datetime
 from dash import dcc
 import dash_bootstrap_components as dbc
 from dash import html
 from dash.dependencies import Output, Input, State
 from sqlalchemy.orm import Session
 from covigator.dashboard.figures.lineages import LineageFigures
-from covigator.dashboard.tabs import get_mini_container, print_number
+from covigator.dashboard.tabs import get_mini_container, print_number, MONTH_PATTERN
 from covigator.database.model import DataSource
 from covigator.database.queries import Queries
 
@@ -15,6 +16,10 @@
 ID_DROPDOWN_COUNTRY = 'lineages-dropdown-country'
 ID_DROPDOWN_LINEAGE = 'lineages-dropdown-lineage'
 ID_DROPDOWN_PERIOD = 'lineages-dropdown-period'
+ID_DROPDOWN_LINEAGE_DATE_RANGE_END = 'lineages-dropdown-date-range-end'
+ID_DROPDOWN_LINEAGE_DATE_RANGE_START = 'lineages-dropdown-date-range-start'
+ID_DROPDOWN_LINEAGE_DATE_RANGE_END_DIV = 'lineages-dropdown-date-range-end-div'
+ID_SLIDER_PREVALENCE = 'lineages-slider-prevalence'
 ID_LINEAGES_GRAPH = 'lineages-graph'
 ID_LINEAGES_TABLE = 'lineages-table'
 
@@ -45,6 +50,12 @@ def get_lineages_tab_graphs():
 def get_lineages_tab_left_bar(queries: Queries, data_source: DataSource):
 
     lineages = queries.get_lineages(source=data_source.name)
+    # Get all available months from collection_date column in sample table
+    months = queries.get_sample_months(MONTH_PATTERN, data_source=data_source.name)
+    today = datetime.now()
+    today_formatted = today.strftime(MONTH_PATTERN)
+    oneyearago = today - timedelta(days=356)
+    oneyearago_formatted = oneyearago.strftime(MONTH_PATTERN)
 
     return html.Div(
         className="two columns",
@@ -108,6 +119,36 @@ def get_lineages_tab_left_bar(queries: Queries, data_source: DataSource):
                 multi=False
             ),
             html.Br(),
+            dcc.Markdown("""Select a start and end date"""),
+            html.Div(children=[
+                dcc.Dropdown(
+                    id=ID_DROPDOWN_LINEAGE_DATE_RANGE_START,
+                    options=[{'label': c, 'value': c} for c in months],
+                    value=oneyearago_formatted,
+                    multi=False,
+                    clearable=False
+                ),
+                html.Div(
+                    id=ID_DROPDOWN_LINEAGE_DATE_RANGE_END_DIV,
+                    children=dcc.Dropdown(
+                        id=ID_DROPDOWN_LINEAGE_DATE_RANGE_END,
+                        options=[{'label': c, 'value': c} for c in months],
+                        value=today_formatted,
+                        multi=False,
+                        clearable=False
+                    ))]),
+            html.Br(),
+            dcc.Markdown("""Minimum prevalence of lineage in the time interval to be plotted"""),
+            dcc.Slider(
+                id=ID_SLIDER_PREVALENCE,
+                min=0.0,
+                max=0.2,
+                step=0.01,
+                value=0.01,
+                marks={i: '{}'.format(i) for i in [0, 0.05, 0.1, 0.15, 0.2]},
+                tooltip=dict(always_visible=False, placement="right")
+            ),
+            html.Br(),
             html.P("Select a single lineage to explore its corresponding mutations."),
             html.Button('Apply', id=ID_APPLY_BUTTOM),
         ])
@@ -123,6 +164,10 @@ def set_callbacks_lineages_tab(app, session: Session):
     lineages_ena = queries.get_lineages(DataSource.ENA.name)
     lineages_covid19_portal = queries.get_lineages(DataSource.COVID19_PORTAL.name)
 
+    # Get months from ENA/Covid19 table
+    months_from_db_ena = queries.get_sample_months(MONTH_PATTERN, data_source=DataSource.ENA.name)
+    months_from_db_covid19_portal = queries.get_sample_months(MONTH_PATTERN, data_source=DataSource.COVID19_PORTAL.name)
+
     @app.callback(
         Output(ID_DROPDOWN_COUNTRY, 'options'),
         Input(ID_DROPDOWN_DATA_SOURCE, 'value'))
@@ -151,6 +196,27 @@ def set_lineages(source):
             lineages = [{'label': c, 'value': c} for c in lineages_covid19_portal]
         return lineages
 
+    @app.callback(
+        Output(ID_DROPDOWN_LINEAGE_DATE_RANGE_END_DIV, 'children'),
+        Input(ID_DROPDOWN_LINEAGE_DATE_RANGE_START, 'value'),
+        Input(ID_DROPDOWN_DATA_SOURCE, 'value')
+    )
+    def update_dropdown_end_date(start_date, data_source):
+        today = datetime.now()
+        today_formatted = today.strftime(MONTH_PATTERN)
+        months = []
+        if data_source == DataSource.ENA.name:
+            months = [m for m in months_from_db_ena if m >= start_date]
+        elif data_source == DataSource.COVID19_PORTAL.name:
+            months = [m for m in months_from_db_covid19_portal if m >= start_date]
+        return dcc.Dropdown(
+            id=ID_DROPDOWN_LINEAGE_DATE_RANGE_END,
+            options=[{'label': c, 'value': c} for c in months],
+            value=today_formatted,
+            multi=False,
+            clearable=False
+        )
+
     @app.callback(
         Output(ID_LINEAGES_GRAPH, 'children'),
         inputs=[Input(ID_APPLY_BUTTOM, 'n_clicks')],
@@ -159,14 +225,21 @@ def set_lineages(source):
             State(ID_DROPDOWN_COUNTRY, 'value'),
             State(ID_DROPDOWN_LINEAGE, 'value'),
             State(ID_DROPDOWN_PERIOD, 'value'),
+            State(ID_DROPDOWN_LINEAGE_DATE_RANGE_START, 'value'),
+            State(ID_DROPDOWN_LINEAGE_DATE_RANGE_END, 'value'),
+            State(ID_SLIDER_PREVALENCE, 'value')
         ],
         suppress_callback_exceptions=True)
-    def update_lineages_plot(_, data_source, countries, lineages, time_period):
+    def update_lineages_plot(_, data_source, countries, lineages, time_period, date_start, date_end, prevalence):
         return html.Div(children=figures.get_lineages_plot(
             data_source=data_source,
             countries=countries,
             lineages=lineages,
-            time_period=time_period))
+            time_period=time_period,
+            date_start=date_start,
+            date_end=date_end,
+            prevalence=prevalence
+        ))
 
     @app.callback(
         Output(ID_LINEAGES_TABLE, 'children'),