new alerts front end

Dockerfile

@@ -21,6 +21,7 @@ RUN apt-get -qq update && apt-get -qq install \
       php-xdebug \
       gettext \
       rsync \
+      mariadb-client \
     --no-install-recommends && \
     rm -r /var/lib/apt/lists/*
 

classes/Utility/Alert.php

@@ -9,6 +9,26 @@
  */
 
 class Alert {
+    public static function sectionToTitle($section) {
+        $section_map = [
+            "uk" => gettext('All UK'),
+            "debates" => gettext('House of Commons debates'),
+            "whalls" => gettext('Westminster Hall debates'),
+            "lords" => gettext('House of Lords debates'),
+            "wrans" => gettext('Written answers'),
+            "wms" => gettext('Written ministerial statements'),
+            "standing" => gettext('Bill Committees'),
+            "future" => gettext('Future Business'),
+            "ni" => gettext('Northern Ireland Assembly Debates'),
+            "scotland" => gettext('All Scotland'),
+            "sp" => gettext('Scottish Parliament Debates'),
+            "spwrans" => gettext('Scottish Parliament Written answers'),
+            "wales" => gettext('Welsh parliament record'),
+            "lmqs" => gettext('Questions to the Mayor of London'),
+        ];
+
+        return $section_map[$section];
+    }
     public static function detailsToCriteria($details) {
         $criteria = [];
 
@@ -20,6 +40,10 @@ public static function detailsToCriteria($details) {
             $criteria[] = 'speaker:' . $details['pid'];
         }
 
+        if (!empty($details['search_section'])) {
+            $criteria[] = 'section:' . $details['search_section'];
+        }
+
         $criteria = join(' ', $criteria);
         return $criteria;
     }
@@ -34,6 +58,7 @@ public static function forUser($email) {
         $alerts = [];
         foreach ($q as $row) {
             $criteria = self::prettifyCriteria($row['criteria']);
+            $parts = self::prettifyCriteria($row['criteria'], true);
             $token = $row['alert_id'] . '-' . $row['registrationtoken'];
 
             $status = 'confirmed';
@@ -43,36 +68,87 @@ public static function forUser($email) {
                 $status = 'suspended';
             }
 
-            $alerts[] = [
+            $alert = [
                 'token' => $token,
                 'status' => $status,
                 'criteria' => $criteria,
                 'raw' => $row['criteria'],
+                'keywords' => [],
+                'exclusions' => [],
+                'sections' => [],
             ];
+
+            $alert = array_merge($alert, $parts);
+
+            $alerts[] = $alert;
         }
 
         return $alerts;
     }
 
-    public static function prettifyCriteria($alert_criteria) {
+    public static function prettifyCriteria($alert_criteria, $as_parts = false) {
         $text = '';
+        $parts = ['words' => [], 'sections' => [], 'exclusions' => [], 'match_all' => true];
         if ($alert_criteria) {
-            $criteria = explode(' ', $alert_criteria);
+            # check for phrases
+            if (strpos($alert_criteria, ' OR ') !== false) {
+                $parts['match_all'] = false;
+            }
+            $alert_criteria = str_replace(' OR ', ' ', $alert_criteria);
+            $alert_criteria = str_replace(['(', ')'], '', $alert_criteria);
+            if (strpos($alert_criteria, '"') !== false) {
+                # match phrases
+                preg_match_all('/"([^"]*)"/', $alert_criteria, $phrases);
+                # and then remove them from the criteria
+                $alert_criteria = trim(preg_replace('/ +/', ' ', str_replace($phrases[0], "", $alert_criteria)));
+
+                # and then create an array with the words and phrases
+                $criteria = [];
+                if ( $alert_criteria != "") {
+                    $criteria = explode(' ', $alert_criteria);
+                }
+                $criteria = array_merge($criteria, $phrases[1]);
+            } else {
+                $criteria = explode(' ', $alert_criteria);
+            }
             $words = [];
+            $exclusions = [];
+            $sections = [];
+            $sections_verbose = [];
             $spokenby = array_values(\MySociety\TheyWorkForYou\Utility\Search::speakerNamesForIDs($alert_criteria));
 
             foreach ($criteria as $c) {
-                if (!preg_match('#^speaker:(\d+)#', $c, $m)) {
+                if (preg_match('#^section:(\w+)#', $c, $m)) {
+                    $sections[] = $m[1];
+                    $sections_verbose[] = self::sectionToTitle($m[1]);
+                } elseif (strpos($c, '-') === 0) {
+                    $exclusions[] = str_replace('-', '', $c);
+                } elseif (!preg_match('#^speaker:(\d+)#', $c, $m)) {
                     $words[] = $c;
                 }
             }
             if ($spokenby && count($words)) {
                 $text = implode(' or ', $spokenby) . ' mentions [' . implode(' ', $words) . ']';
+                $parts['spokenby'] = $spokenby;
+                $parts['words'] = $words;
             } elseif (count($words)) {
                 $text = '[' . implode(' ', $words) . ']' . ' is mentioned';
+                $parts['words'] = $words;
             } elseif ($spokenby) {
                 $text = implode(' or ', $spokenby) . " speaks";
+                $parts['spokenby'] = $spokenby;
             }
+
+            if ($sections) {
+                $text = $text . " in " . implode(' or ', $sections_verbose);
+                $parts['sections'] = $sections;
+                $parts['sections_verbose'] = $sections_verbose;
+            }
+
+            $parts['exclusions'] = $exclusions;
+        }
+        if ($as_parts) {
+            return $parts;
         }
         return $text;
     }

classes/Utility/Search.php

@@ -244,17 +244,25 @@ public static function searchMemberDbLookupWithNames($searchstring, $current_onl
      * Given a search term, find constituencies by name or postcode.
      *
      * @param string $searchterm The term to search for.
+     * @param bool $mp_only if true (default) only return westminster constituency if using a postcode, otherwise return all.
      *
      * @return array A list of the array of constituencies, then a boolean
      *               saying whether it was a postcode used.
      */
 
-    public static function searchConstituenciesByQuery($searchterm) {
+    public static function searchConstituenciesByQuery($searchterm, $mp_only=true) {
         if (validate_postcode($searchterm)) {
             // Looks like a postcode - can we find the constituency?
-            $constituency = Postcode::postcodeToConstituency($searchterm);
-            if ($constituency) {
-                return [ [$constituency], true ];
+            if ($mp_only) {
+                $constituency = Postcode::postcodeToConstituency($searchterm);
+                if ($constituency) {
+                    return [ [$constituency], true ];
+                }
+            } else {
+                $constituencies = Postcode::postcodeToConstituencies($searchterm);
+                if ($constituencies) {
+                    return [ $constituencies, true ];
+                }
             }
         }
 
@@ -297,6 +305,28 @@ public static function speakerNamesForIDs($searchstring) {
         return $speakers;
     }
 
+    /**
+     * get list of members of speaker IDs from search string
+     *
+     * @param string      $searchstring       The search string with the speaker:NNN text
+     *
+     * @return array Array with the speaker id string as key and speaker name as value
+     */
+
+    public static function membersForIDs($searchstring) {
+        $criteria = explode(' ', $searchstring);
+        $speakers = [];
+
+        foreach ($criteria as $c) {
+            if (preg_match('#^speaker:(\d+)#', $c, $m)) {
+                $MEMBER = new \MEMBER(['person_id' => $m[1]]);
+                $speakers[$m[1]] = $MEMBER;
+            }
+        }
+
+        return $speakers;
+    }
+
     /**
      * replace speaker:NNNN with speaker:Name in search string
      *

db/0025-add-vector-search-suggestions.sql

@@ -0,0 +1,5 @@
+CREATE TABLE `vector_search_suggestions` (
+  `search_term` varchar(100) NOT NULL default '',
+  `search_suggestion` varchar(100) NOT NULL default '',
+  KEY `search_term` (`search_term`)
+);

db/schema.sql

@@ -214,6 +214,12 @@ CREATE TABLE `postcode_lookup` (
   PRIMARY KEY  (`postcode`)
 );
 
+CREATE TABLE `vector_search_suggestions` (
+  `search_term` varchar(100) NOT NULL default '',
+  `search_suggestion` varchar(100) NOT NULL default '',
+  KEY `search_term` (`search_term`)
+);
+
 -- each time we index, we increment the batch number;
 -- can use this to speed up search
 CREATE TABLE `indexbatch` (

scripts/import_search_suggestions.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+"""
+import_search_suggestions.py - Import vector search suggestions
+
+See python scripts/import_search_suggestions.py --help for usage.
+
+"""
+
+import re
+import sys
+from pathlib import Path
+from typing import cast
+from warnings import filterwarnings
+
+import MySQLdb
+import pandas as pd
+import rich_click as click
+from pylib.mysociety import config
+from rich import print
+from rich.prompt import Prompt
+
+repository_path = Path(__file__).parent.parent
+
+config.set_file(repository_path / "conf" / "general")
+
+# suppress warnings about using mysqldb in pandas
+filterwarnings(
+    "ignore",
+    category=UserWarning,
+    message=".*pandas only supports SQLAlchemy connectable.*",
+)
+
+
+@click.group()
+def cli():
+    pass
+
+
+def get_twfy_db_connection() -> MySQLdb.Connection:
+    db_connection = cast(
+        MySQLdb.Connection,
+        MySQLdb.connect(
+            host=config.get("TWFY_DB_HOST"),
+            db=config.get("TWFY_DB_NAME"),
+            user=config.get("TWFY_DB_USER"),
+            passwd=config.get("TWFY_DB_PASS"),
+            charset="utf8",
+        ),
+    )
+    return db_connection
+
+
+def df_to_db(df: pd.DataFrame, verbose: bool = False):
+    """
+    add search suggestions to the database
+    """
+    df = df.dropna(how="any")
+    db_connection = get_twfy_db_connection()
+
+    with db_connection.cursor() as cursor:
+        # just remove everything and re-insert it all rather than trying to update things
+        cursor.execute("DELETE FROM vector_search_suggestions")
+        insert_command = "INSERT INTO vector_search_suggestions (search_term, search_suggestion) VALUES (%s, %s)"
+        suggestion_data = [
+            (row["original_query"], row["match"]) for _, row in df.iterrows()
+        ]
+        cursor.executemany(insert_command, suggestion_data)
+    db_connection.commit()
+
+    if verbose:
+        print(f"[green]{len(df)} rows updated.")
+
+    db_connection.close()
+
+
+def url_to_db(url: str, verbose: bool = False):
+    """
+    Pipe external URL into the update process.
+    """
+    df = pd.read_csv(url)
+
+    df_to_db(df, verbose=verbose)
+
+
+def file_to_db(file: str, verbose: bool = False):
+    """
+    Pipe file into the update process.
+    """
+    df = pd.read_csv(file)
+
+    df_to_db(df, verbose=verbose)
+
+
+@cli.command()
+@click.option(
+    "--url",
+    required=False,
+    default=None,
+    help="A csv file to update search suggestions from.",
+)
+@click.option(
+    "--file",
+    required=False,
+    default=None,
+    help="A csv file to update search suggestions from.",
+)
+@click.option("--verbose", is_flag=True, help="Show verbose output")
+def update_vector_search_suggestions(url: str, file: str, verbose: bool = False):
+    """
+    Update the vector search suggestions
+    """
+    if file:
+        file_to_db(file, verbose=verbose)
+    elif url:
+        url_to_db(url, verbose=verbose)
+
+
+@cli.command()
+def count_suggestions():
+    """
+    for diagnostics to check import has worked
+    """
+    db_connection = get_twfy_db_connection()
+    with db_connection.cursor() as cursor:
+        cursor.execute(
+            "select count(*) as num_suggestions from vector_search_suggestions"
+        )
+        count = cursor.fetchone()[0]
+        print(f"There are {count} suggestions in the db")
+
+    db_connection.close()
+
+
+def main():
+    cli()
+
+
+if __name__ == "__main__":
+    main()