Database API - updates and test conformance

.github/workflows/auto_update_script_contents.yml

@@ -31,6 +31,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
+          version: '1.7.0'
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true

.github/workflows/autoblack.yml

@@ -1,5 +1,9 @@
 name: autoblack
 on: [pull_request]
+
+env:
+  PYTHON_VERSION: "3.12.1"
+
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -9,10 +13,10 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.ref }}
           repository: ${{ github.event.pull_request.head.repo.full_name }}
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Load cached Poetry installation
         uses: actions/cache@v2
@@ -22,6 +26,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
+          version: '1.7.0'
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true

.github/workflows/documentation.yml

@@ -39,6 +39,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
+          version: '1.7.0'
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true

.github/workflows/lint.yml

@@ -1,20 +1,19 @@
 name: Lint
 on: [pull_request]
 
+env:
+  PYTHON_VERSION: "3.12.1"
+
 jobs:
   build:
     runs-on: ubuntu-latest
-    strategy:
-      max-parallel: 4
-      matrix:
-        python-version: [3.8]
 
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Load cached Poetry installation
         uses: actions/cache@v2
@@ -24,6 +23,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
+          version: '1.7.0'
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true

.github/workflows/poetry_pypi_release.yml

@@ -11,7 +11,6 @@ jobs:
       - uses: actions/checkout@v3
         with:
           ref: master
-          token: ${{ secrets.SPLINK_TOKEN }}
       - name: Install poetry
         run: pipx install poetry
       - uses: actions/setup-python@v4

.github/workflows/pytest_benchmark_comment.yml

@@ -37,6 +37,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
+          version: '1.7.0'
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true

.github/workflows/pytest_benchmark_commit.yml

@@ -39,6 +39,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
+          version: '1.7.0'
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true

.github/workflows/run_demos_examples.yml

@@ -41,6 +41,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
+          version: '1.7.0'
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true

.github/workflows/run_demos_tutorials.yml

@@ -40,6 +40,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
+          version: '1.7.0'
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true

CHANGELOG.md

@@ -7,10 +7,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- Ability to block on array columns by specifying `arrays_to_explode` in your blocking rule. ([#1692](https://github.com/moj-analytical-services/splink/pull/1692))
+
 ### Changed
 
+- Splink now fully parallelises data linkage when using DuckDB ([#1796](https://github.com/moj-analytical-services/splink/pull/1796))
+
+### Fixed
+
+- Allow salting in EM training ([#1832](https://github.com/moj-analytical-services/splink/pull/1832))
+
+## [3.9.10] - 2023-12-07
+
+### Changed
+
+- Remove unused code from Athena linker ([#1775](https://github.com/moj-analytical-services/splink/pull/1775))
+- Add argument for `register_udfs_automatically` ([#1774](https://github.com/moj-analytical-services/splink/pull/1774))
+
 ### Fixed
 
+- Fixed issue with `_source_dataset_col` and `_source_dataset_input_column` ([#1731](https://github.com/moj-analytical-services/splink/pull/1731))
+- Delete cached tables before resetting the cache ([#1752](https://github.com/moj-analytical-services/splink/pull/1752)
 
 ## [3.9.9] - 2023-11-14
 
@@ -46,6 +65,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Corrected path for Spark `.jar` file containing UDFs to work correctly for Spark < 3.0 ([#1622](https://github.com/moj-analytical-services/splink/pull/1622))
 - Spark UDF `damerau_levensthein` is now only registered for Spark >= 3.0, as it is not compatible with earlier versions ([#1622](https://github.com/moj-analytical-services/splink/pull/1622))
 
-[unreleased]: https://github.com/moj-analytical-services/splink/compare/3.9.9...HEAD
+[unreleased]: https://github.com/moj-analytical-services/splink/compare/3.9.10...HEAD
+[3.9.10]: https://github.com/moj-analytical-services/splink/compare/v3.9.9...3.9.10
 [3.9.9]: https://github.com/moj-analytical-services/splink/compare/v3.9.8...3.9.9
 [3.9.8]: https://github.com/moj-analytical-services/splink/compare/v3.9.7...v3.9.8

CONTRIBUTING.md

@@ -18,7 +18,7 @@ Contributions to Splink are not limited to the code. Feedback and input on our d
 
 Behind the scenes, the Splink documentation is split into 2 parts:
 
-- The [Tutorials](./docs/demos/00_Tutorial_Introduction.ipynb) and [Example Notebooks](./docs/examples_index.md) are stored in a separate repo - [splink_demos](https://github.com/moj-analytical-services/splink_demos)
+- The [Tutorials](./docs/demos/tutorials/00_Tutorial_Introduction.ipynb) and [Example Notebooks](./docs/demos/examples/examples_index.md) are stored in a separate repo - [splink_demos](https://github.com/moj-analytical-services/splink_demos)
 - Everything else is stored in the Splink repo either in:
     - the [docs folder](https://github.com/moj-analytical-services/splink/tree/master/docs)
     - the Splink code itself. E.g. docstrings from [linker.py](https://github.com/moj-analytical-services/splink/blob/master/splink/linker.py) feed directly into the [Linker API docs](./docs/linker.md).

README.md

@@ -109,7 +109,7 @@ Should you require a more bare-bones version of Splink **without DuckDB**, pleas
 
 The following code demonstrates how to estimate the parameters of a deduplication model, use it to identify duplicate records, and then use clustering to generate an estimated unique person ID.
 
-For more detailed tutorial, please see [here](https://moj-analytical-services.github.io/splink/demos/00_Tutorial_Introduction.html).
+For more detailed tutorial, please see [here](https://moj-analytical-services.github.io/splink/demos/tutorials/00_Tutorial_Introduction.html).
 
 ```py
 from splink.duckdb.linker import DuckDBLinker
@@ -166,13 +166,16 @@ To find the best place to ask a question, report a bug or get general advice, pl
 
 ## Awards
 
-🥇 Analysis in Government Awards 2020: Innovative Methods: [Winner](https://www.gov.uk/government/news/launch-of-the-analysis-in-government-awards)
+🥇 Analysis in Government Awards 2020: Innovative Methods - [Winner](https://www.gov.uk/government/news/launch-of-the-analysis-in-government-awards)
 
 🥇 MoJ DASD Awards 2020: Innovation and Impact - Winner
 
 🥇 Analysis in Government Awards 2022: People's Choice Award - [Winner](https://analysisfunction.civilservice.gov.uk/news/announcing-the-winner-of-the-first-analysis-in-government-peoples-choice-award/)
 
-🥈 Analysis in Government Awards 2022: Innovative Methods [Runner up](https://twitter.com/gov_analysis/status/1616073633692274689?s=20&t=6TQyNLJRjnhsfJy28Zd6UQ)
+🥈 Analysis in Government Awards 2022: Innovative Methods - [Runner up](https://twitter.com/gov_analysis/status/1616073633692274689?s=20&t=6TQyNLJRjnhsfJy28Zd6UQ)
+
+🥈 Civil Service Awards 2023: Best Use of Data, Science, and Technology - [Runner up](https://www.civilserviceawards.com/best-use-of-data-science-and-technology-award-2/)
+
 
 ## Citation
 

docs/blocking_rule_library.md

@@ -6,9 +6,9 @@ toc_depth: 2
 ---
 # Documentation for `blocking_rules_library`
 
-The `blocking_rules_library` contains a series of pre-made blocking rules available for use in the construction of blocking rule strategies and em training blocks [as described in this topic guide](./topic_guides/drivers_of_performance.html#blocking-rules).
+The `blocking_rules_library` contains a series of pre-made blocking rules available for use in the construction of blocking rule strategies and em training blocks [as described in this topic guide](./topic_guides/blocking/blocking_rules.md).
 
-These conform to a more performant standard that is outlined in detail [here](./topic_guides/drivers_of_performance.html#blocking-rules).
+These conform to a more performant standard that is outlined in detail [here](./topic_guides/performance/drivers_of_performance.html#blocking-rules).
 
 
 The detailed API for each of these are outlined below.

docs/blog/.authors.yml

@@ -1,9 +1,10 @@
-robin-l:
-  name: Robin Linacre
-  description: Creator
-  avatar: https://github.com/robinl.png
+authors:
+  robin-l:
+    name: Robin Linacre
+    description: Creator
+    avatar: https://github.com/robinl.png
 
-ross-k:
-  name: Ross Kennedy
-  description: Maintainer
-  avatar: https://github.com/rossken.png
+  ross-k:
+    name: Ross Kennedy
+    description: Maintainer
+    avatar: https://github.com/rossken.png

docs/blog/posts/2023-07-27-feature_update.md

@@ -1,5 +1,5 @@
 ---
-date: 2022-07-27
+date: 2023-07-27
 authors:
   - ross-k
   - robin-l

docs/blog/posts/2023-12-06-feature_update.md

@@ -0,0 +1,119 @@
+---
+date: 2023-12-06
+authors:
+  - ross-k
+categories:
+  - Feature Updates
+---
+
+# Splink Updates - December 2023
+
+Welcome to the second installment of the Splink Blog! 
+
+Here are some of the highlights from the second half of 2023, and a taste of what is in store for 2024!
+
+<!-- more -->
+
+Latest Splink version: [v3.9.10](https://github.com/moj-analytical-services/splink/releases/tag/v3.9.10)
+
+## :bar_chart: Charts Gallery
+
+The Splink docs site now has a [Charts Gallery](../../charts/index.md) to show off all of the charts that come out-of-the-box with Splink to make linking easier. 
+
+[![](../posts/img/charts_gallery.png){ width="400" }](../../charts/index.md)
+
+Each chart now has an explanation of:
+
+1. What the chart shows
+2. How to interpret it
+3. Actions to take as a result
+
+This is the first step on a longer term journey to provide more guidance on how to evaluate Splink models and linkages, so watch this space for more in the coming months!
+
+## :chart_with_upwards_trend: New Charts
+
+We are always adding more charts to Splink - to understand how these charts are built see our new [Charts Developer Guide](../../dev_guides/charts/understanding_and_editing_charts.md).
+
+Two of our latest additions are:
+
+### :material-matrix: Confusion Matrix
+
+When evaluating any classification model, a confusion matrix is a useful tool for summarizing performance by representing counts of true positive, true negative, false positive, and false negative predictions.
+
+Splink now has its own [confusion matrix chart](../../charts/confusion_matrix_from_labels_table.ipynb) to show how model performance changes with a given match weight threshold. 
+
+[![](./img/confusion_matrix.png){ width="400" }](../../charts/confusion_matrix_from_labels_table.ipynb)
+
+Note, labelled data is required to generate this chart.
+
+### :material-table: Completeness Chart
+
+When linking multiple datasets together, one of the most important factors for a successful linkage is the number of common fields across the datasets. 
+
+Splink now has the [completeness chart](../../charts/completeness_chart.ipynb) which gives a simple view of how well populated fields are across datasets.
+
+[![](./img/completeness_chart.png)](../../charts/completeness_chart.ipynb)
+
+
+## :clipboard: Settings Validation
+
+The [Settings dictionary](../../settings_dict_guide.md) is central to everything in Splink. It defines everything from the sql dialect of your backend to how features are compared in Splink model. 
+
+A common sticking point with users is how easy it is to make small errors when defining the Settings dictionary, resulting in unhelpful error messages.
+
+To address this issue, the [Settings Validator](../../dev_guides/settings_validation/settings_validation_overview.md) provides clear, user-friendly feedback on what the issue is and how to fix it.
+
+
+## :simple-adblock: Blocking Rule Library (Improved)
+
+In our [previous blog](../posts/2023-12-06-feature_update.md#no_entry_sign-drop-support-for-python-37) we introduced the Blocking Rule Library (BRL) built upon the `exact_match_rule` function. When testing this functionality we found it pretty verbose, particularly when blocking on multiple columns, so figured we could do better. From Splink v3.9.6 we introduced the `block_on` function to supercede `exact_match_rule`.
+
+For example, a block on `first_name` and `surname` now looks like:
+
+```py
+from splink.duckdb.blocking_rule_library import block_on
+block_on(["first_name", "surname"])  
+```
+
+as opposed to
+
+```py
+import splink.duckdb.blocking_rule_library as brl
+brl.and_(
+  brl.exact_match_rule("first_name"),
+  brl.exact_match_rule("surname")
+)
+```
+
+All of the [tutorials](../../demos/tutorials/03_Blocking.ipynb), [example notebooks](../../demos/examples/examples_index.md) and [API docs](../../blocking_rule_library.md) have been updated to use `block_on`.
+
+## :electric_plug: Backend Specific Installs
+
+Some users have had difficulties downloading Splink due to additional dependencies, some of which may not be relevant for the backend they are using. To solve this, you can now install a minimal version of Splink for your given SQL engine.
+
+For example, to install Splink purely for Spark use the command:
+
+```bsh
+pip install 'splink[spark]'
+```
+
+See the [Getting Started page](../../getting_started.md#backend-specific-installs) for further guidance.
+
+## :no_entry_sign: Drop support for python 3.7
+
+From Splink 3.9.7, support has been dropped for python 3.7. This decision has been made to manage dependency clashes in the back end of Splink.
+
+If you are working with python 3.7, please revert to Splink 3.9.6.
+
+```bsh
+pip install splink==3.9.6
+```
+
+## :soon: What's in the pipeline?
+
+* :four:   Work on **Splink 4** is currently underway
+* :material-thumbs-up-down:   More guidance on how to evaluate Splink models and linkages
+
+
+
+

docs/charts/profile_columns.ipynb

@@ -309,7 +309,7 @@
     "\n",
     "To take this skew into account, we can build Splink models with **Term Frequency Adjustments**. These adjustments will increase the amount of evidence for rare matching values and reduce the amount of evidence for common matching values.\n",
     "\n",
-    "To understand how these work in more detail, check out the [Term Frequency Adjustments Topic Guide](../comparisons/term-frequency.md)\n",
+    "To understand how these work in more detail, check out the [Term Frequency Adjustments Topic Guide](../topic_guides/comparisons/term-frequency.md)\n",
     "\n",
     "<hr>"
    ]

docs/comparison_helpers.md

@@ -5,7 +5,7 @@ tags:
 ---
 # Documentation for `comparison_helpers` functions
 
-The `comparison_helpers` functions are a set of functions to help users create better comparisons by helping them understand [string comparators](./topic_guides/choosing_comparators.ipynb#comparing-string-similarity-and-distance-scores) (fuzzy matching) and [phonetic matching](./topic_guides/choosing_comparators.ipynb#phonetic-matching).
+The `comparison_helpers` functions are a set of functions to help users create better comparisons by helping them understand [string comparators](./topic_guides/choosing_comparators.ipynb#comparing-string-similarity-and-distance-scores) (fuzzy matching) and [phonetic matching](./topic_guides/comparisons/choosing_comparators.ipynb#phonetic-matching).
 
 The detailed API for each of these are outlined below.
 

docs/comparison_level_library.md

@@ -16,8 +16,8 @@ toc_depth: 2
 # Documentation for `comparison_level_library`
 
 The `comparison_level_library` contains pre-made comparison levels available for use to
-construct custom comparisons [as described in this topic guide](./topic_guides/customising_comparisons.html#method-3-comparisonlevels).
-However, not every comparison level is available for every [Splink-compatible SQL backend](./topic_guides/backends.html).
+construct custom comparisons [as described in this topic guide](./topic_guides/comparisons/customising_comparisons.html#method-3-comparisonlevels).
+However, not every comparison level is available for every [Splink-compatible SQL backend](./topic_guides/splink_fundamentals/backends.html).
 
 The pre-made Splink comparison levels available for each SQL dialect are as given in this table:
 

docs/comparison_library.md

@@ -12,8 +12,8 @@ toc_depth: 2
 ---
 # Documentation for `comparison_library`
 
-The `comparison_library` contains pre-made comparisons available for use directly [as described in this topic guide](./topic_guides/customising_comparisons.html#method-1-using-the-comparisonlibrary).
-However, not every comparison is available for every [Splink-compatible SQL backend](./topic_guides/backends.html).
+The `comparison_library` contains pre-made comparisons available for use directly [as described in this topic guide](./topic_guides/comparisons/customising_comparisons.html#method-1-using-the-comparisonlibrary).
+However, not every comparison is available for every [Splink-compatible SQL backend](./topic_guides/splink_fundamentals/backends/backends.html).
 
 The pre-made Splink comparisons available for each SQL dialect are as given in this table: