Merge branch 'develop' into task/RDMP-165-setup-using-yaml-file

.github/pull_request_template.md

@@ -0,0 +1,24 @@
+## Proposed Change
+
+Summarise your proposed changes here, including any notes for reviewers.
+
+## Type of change
+
+What types of changes does your code introduce? Tick all that apply.
+
+-   [ ] Bugfix (non-breaking change which fixes an issue)
+-   [ ] New Feature (non-breaking change which adds functionality)
+-   [ ] Breaking Change (fix or feature that would cause existing functionality to not work as expected)
+-   [ ] Documentation-Only Update
+-   [ ] Other  (if none of the other choices apply)
+
+## Checklist
+
+By opening this PR, I confirm that I have:
+
+-   [ ] Ensured that the PR branch is in sync with the target branch (i.e. it is automatically merge-able)
+-   [ ] Created or updated any tests if relevant
+-   [ ] Have validated this change against the [Test Plan](https://github.com/HicServices/RDMP/blob/develop/Documentation/CodeTutorials/TestPlan.md)
+-   [ ] Requested a review by one of the repository maintainers
+-   [ ] Have written new documentation or updated existing documentation to detail any new or updated functionality and how to use it
+-   [ ] Have added an entry into the changelog

.github/workflows/build.yml

@@ -66,7 +66,7 @@ jobs:
           mv `find coverage -type f` db-ui.lcov
           dotnet test Rdmp.Core.Tests/Rdmp.Core.Tests.csproj --nologo --collect:"XPlat Code Coverage" --no-build --verbosity minimal -c Release --results-directory coverage -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=lcov
           mv `find coverage -type f` db-core.lcov
-      - uses: coverallsapp/github-action@v2.2.3
+      - uses: coverallsapp/github-action@v2.3.0
         with:
           github-token: ${{ secrets.github_token }}
           files: ./db-ui.lcov ./db-core.lcov
@@ -128,7 +128,7 @@ jobs:
           mv `find coverage -type f` fs-ui.lcov
           dotnet test Rdmp.Core.Tests/Rdmp.Core.Tests.csproj --nologo --collect:"XPlat Code Coverage" --no-build --verbosity minimal -c Release --results-directory coverage -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=lcov
           mv `find coverage -type f` fs-core.lcov
-      - uses: coverallsapp/github-action@v2.2.3
+      - uses: coverallsapp/github-action@v2.3.0
         with:
           github-token: ${{ secrets.github_token }}
           files: ./fs-ui.lcov ./fs-core.lcov

CHANGELOG.md

@@ -1,16 +1,22 @@
 
+
 # Changelog
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [8.1.6] = Unreleased
+## [8.1.6] - Unreleased
 
 ## Changed
 
+- Improve error messages for Multi-ExtractionIdentifier extractions
 - Add Microsoft.Bcl.AsyncInterfaces 6.0.0 for plugin dependancy tree
 - Add prompt to reanem container when adding a cohort filter
+- Fix to remove stack trace button from non error popups
+- Add ability to set Extraction Categort as "Not Extractable"
+- Replace BadMedicine v1.2.1 with SynthEHR v2.0.0
+- Fix issue with RDMP being slow to load when having numerous Load Metadatas
 
 ## [8.1.5] - 2024-04-03
 

Directory.Packages.props

@@ -4,7 +4,7 @@
 		<PackageVersion Include="Equ" Version="2.3.0"/>
 		<PackageVersion Include="ExcelNumberFormat" Version="1.1.0"/>
 		<PackageVersion Include="FluentFTP" Version="50.0.1"/>
-		<PackageVersion Include="HIC.BadMedicine" Version="1.2.1"/>
+		<PackageVersion Include="HIC.SynthEHR" Version="2.0.0"/>
 		<PackageVersion Include="HIC.FAnsiSql" Version="3.2.2"/>
 		<PackageVersion Include="LibArchive.Net" Version="0.1.5"/>
 		<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0">
@@ -20,9 +20,9 @@
 		<PackageVersion Include="Spectre.Console" Version="0.49.1"/>
 		<PackageVersion Include="SSH.NET" Version="2024.0.0"/>
 		<PackageVersion Include="Terminal.Gui" Version="1.16.0"/>
-		<PackageVersion Include="YamlDotNet" Version="15.1.2"/>
+		<PackageVersion Include="YamlDotNet" Version="15.1.4"/>
 		<PackageVersion Include="ConsoleControl" Version="1.3.0"/>
-		<PackageVersion Include="Autoupdater.NET.Official" Version="1.8.5"/>
+		<PackageVersion Include="Autoupdater.NET.Official" Version="1.8.6"/>
 		<PackageVersion Include="DockPanelSuite.ThemeVS2015" Version="3.1.0"/>
 		<PackageVersion Include="System.Threading.ThreadPool" Version="4.3.0"/>
 		<PackageVersion Include="WeCantSpell.Hunspell" Version="5.0.0"/>

Documentation/CodeTutorials/Packages.md

@@ -25,7 +25,7 @@
 | LibArchive.Net | [GitHub](https://github.com/jas88/libarchive.net) | [BSD](https://opensource.org/license/bsd-2-clause/) | Access archive formats without the LZMA bugs of SharpCompress | |
 | [NLog](https://nlog-project.org/) | [GitHub](https://github.com/NLog/NLog) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | |
 | HIC.FAnsiSql |[GitHub](https://github.com/HicServices/FAnsiSql) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | [DBMS] abstraction layer |
-| HIC.BadMedicine | [GitHub](https://github.com/HicServices/BadMedicine) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Generate Test Datasets for tests/exericses |
+| HIC.SynthEHR | [GitHub](https://github.com/HicServices/SynthEHR) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Generate Test Datasets for tests/exericses |
 | SSH.NET  | [GitHub](https://github.com/sshnet/SSH.NET)  | [MIT](https://github.com/sshnet/SSH.NET/blob/develop/LICENSE) | Enables fetching files from SFTP servers |
 | Moq 4 | [GitHub](https://github.com/moq/moq4) |[BSD 3](https://github.com/moq/moq4/blob/master/License.txt)  | Mock objects during unit testing |
 | [Newtonsoft.Json](https://www.newtonsoft.com/json) | [GitHub](https://github.com/JamesNK/Newtonsoft.Json) | [MIT](https://opensource.org/licenses/MIT) | Serialization of objects for sharing/transmission |

Documentation/CodeTutorials/PluginWriting.md

@@ -1436,7 +1436,7 @@ When packaging `dotnet publish` results you can exclude dlls that already come w
 ```
 <files>
     <file src="Plugin\windows\bin\$configuration$\net6.0-windows\win-x64\publish\*"
-          exclude="**\BadMedicine.Core.dll;**\FAnsi.*;**\MapsDirectlyToDatabaseTable.dll;**\MySql.Data.dll;**\Oracle.ManagedDataAccess.dll;**\Rdmp.Core.dll;**\NPOI.*;**\Renci.*;**\MathNet.Numerics.dll*;**\Rdmp.UI.dll;**\ScintillaNET.dll;**\ReusableUIComponents.dll;**\ObjectListView.dll;**\WeifenLuo.WinFormsUI.Docking*"
+          exclude="**\SynthEHR.Core.dll;**\FAnsi.*;**\MapsDirectlyToDatabaseTable.dll;**\MySql.Data.dll;**\Oracle.ManagedDataAccess.dll;**\Rdmp.Core.dll;**\NPOI.*;**\Renci.*;**\MathNet.Numerics.dll*;**\Rdmp.UI.dll;**\ScintillaNET.dll;**\ReusableUIComponents.dll;**\ObjectListView.dll;**\WeifenLuo.WinFormsUI.Docking*"
           target="lib\windows" />
 </files>
 ```

Documentation/CodeTutorials/TestPlan.md

@@ -0,0 +1,56 @@
+# RDMP Test Plan
+The RDMP test plan outlines the questions that should be asked of new changes to help ensure the correct changes are being made at the correct time.
+These questions are designed to help highlight any steps in the testing process that may have been missed.
+
+If you are looking to release a new version with this change, please look at the [Release Testing](#release_testing) section below.
+
+## Generating Test Data
+There is functionality to generate example data built into RDMP. This functionality has a 'nightmare' mode, which can generate large amounts of data for testing purposes.
+This mode can be enabled when creating platform databases by adding the 'Nightmare' flag to the example data settings during platform creation.
+Additional data can be used by using the 'Factor' flag to increase the amount of data, this will multiply the number of objects created by your specified Factor.
+
+## Questions To Ask
+### Does this change add a database migration?
+If the change adds a database migration, ask yourself the following questions:
+* Is the change backwards compatable?
+* Have I tested the change with a fresh install of RDMP?
+* Have I tested the change through the upgrade path with populated data that my change affects?
+* Should this change be included in a patch release? or be part of the next minor release? Or even a major release?
+    * Patch releases contain small improvements and should be usable interchangably with other versions within the same minor version e.g v8.1.4 and v8.1.5
+    * Minor released contain new functionality and should be backwards compatable with other versions in the same major version e.g. v8.1.0 and v8.2.0
+    * Major releases contain changes that are not backwards compatable
+
+### Does this change add new functionality?
+If the change adds new functionality, ask yourself the following questions:
+* Is the functionality usable via the GUI? If no ,why not?
+* Is the functionality usable via the CLI? If no, why not?
+* Has the functionalty been covered via unit tests?
+* Has the functionalty been manually tested?
+    * Has the happy path been tested?
+        * This is the expected path, where users are paying attention and are on their best behaviour
+    * Has the sad path been tested?
+        * This is where the user tries to be as obtuse as possible
+* Does this change do any data processing? If so, check the performance questions below
+
+### Does this change have any impact on performance?
+If the change adds or amends functionality that processes data
+* Is the functionality performant?
+    * Can the space/time complexity of the functions be reduced? 
+* Does the functionality handle large datasets (>1GB) efficiently?
+
+
+### What assumptions have been made?
+* Have any assumptions about how this functionality will be used been made?
+* Have any assumptions about the input data been made? Can these assumptions be extracted out into configuration?
+
+## <a name="release_testing"></a>Release Testing
+The release testing process should be completed once the release is feature complete and no more code changes are due to take place.
+This testing is to ensure that all code changes made during this release play nicely together and have no unintended side effcts.
+It may be useful at this point to revisit each piece of functionality and ensure they work as expected and perform some user acceptance testing on them, in light with the questions above.
+### What's changed?
+* Does the changelog accuratly reflect the pull request changes made?
+* If there are database migrations, are they all correctly sequenced and not overlapping?
+* Does the release work as epxected with a fresh install and via the upgrade path?
+* Is all functionality documented?
+* Are all version numbers bumped correctly?
+* Do all of the managed plugins work with the new release without issues or warnings?

Documentation/CodeTutorials/UserManual.md

@@ -129,7 +129,7 @@ _\* Unless using [file system backend](./YamlRepository.md)_
 ## Example Data
 Example data can be setup on install by ticking the 'Example Datasets' checkbox during platform database setup.
 
-New example data can be generated through the `Diagnostics=>Generate Test  Data` menu. Or with the [BadMedicine](https://github.com/HicServices/BadMedicine) command line tool. 
+New example data can be generated through the `Diagnostics=>Generate Test  Data` menu. Or with the [SynthEHR](https://github.com/HicServices/SynthEHR) command line tool. 
 
 ## Importing a flat file as a new dataset
 If you have some CSV files (or Excel/Fixed Width) that you want to load into your database, you can do so with the RDMP bulk import tool.
@@ -384,7 +384,7 @@ drop trigger Prescribing_OnUpdate;
 
 Each data load in RDMP has a single [LoadMetadata] object which acts as the root for the configuration.  A given configuration will load one or more tables (which must have primary keys).  The tables loaded includes all the tables that underly the [Catalogues] which are ­associated with the [LoadMetadata].  For example if a Catalogue 'Biochemistry' includes a join on two tables 'Header' and 'Results' then a [LoadMetadata] which includes 'Biochemistry' would include both the 'Header' and 'Results' tables.
 
-Create a new Biochemistry CSV file with [BadMedicine] and add a primary key to the 'Biochemistry' table in your example data.  The key should be a composite primary key of:
+Create a new Biochemistry CSV file with [SynthEHR] and add a primary key to the 'Biochemistry' table in your example data.  The key should be a composite primary key of:
 
 - chi
 - SampleDate
@@ -458,7 +458,7 @@ Set the following properties:
 
 ### Running the load
 
-Place the Biochemistry.csv file generated by [BadMedicine] into the ForLoading directory of your load (e.g. `C:\temp\biochem\Data\ForLoading`).
+Place the Biochemistry.csv file generated by [SynthEHR] into the ForLoading directory of your load (e.g. `C:\temp\biochem\Data\ForLoading`).
 
 Right click the [LoadMetadata] and select 'Check and Execute'
 
@@ -838,7 +838,7 @@ _The question mark after ExtractionConfiguration means match all where the Proje
 
 
 [AggregateConfiguration]: ./Glossary.md#AggregateConfiguration
-[BadMedicine]: https://github.com/HicServices/BadMedicine
+[SynthEHR]: https://github.com/HicServices/SynthEHR
 [Catalogue]: ./Glossary.md#Catalogue
 [Catalogues]: ./Glossary.md#Catalogue
 [CohortIdentificationConfiguration]: ./Glossary.md#CohortIdentificationConfiguration

Rdmp.Core.Tests/Curation/Unit/ExerciseData/TestBiochemistryCreation.cs

@@ -6,8 +6,8 @@
 
 using System;
 using System.IO;
-using BadMedicine;
-using BadMedicine.Datasets;
+using SynthEHR;
+using SynthEHR.Datasets;
 using NUnit.Framework;
 
 namespace Rdmp.Core.Tests.Curation.Unit.ExerciseData;

Rdmp.Core.Tests/Curation/Unit/ExerciseData/TestDemographyCreation.cs

@@ -6,8 +6,8 @@
 
 using System;
 using System.IO;
-using BadMedicine;
-using BadMedicine.Datasets;
+using SynthEHR;
+using SynthEHR.Datasets;
 using NUnit.Framework;
 
 namespace Rdmp.Core.Tests.Curation.Unit.ExerciseData;

Rdmp.Core.Tests/Curation/Unit/ExerciseData/TestPrescribingCreation.cs

@@ -7,8 +7,8 @@
 using System;
 using System.IO;
 using System.Linq;
-using BadMedicine;
-using BadMedicine.Datasets;
+using SynthEHR;
+using SynthEHR.Datasets;
 using NUnit.Framework;
 
 namespace Rdmp.Core.Tests.Curation.Unit.ExerciseData;

Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationTest.cs

@@ -8,7 +8,7 @@
 using System.Data;
 using System.IO;
 using System.Linq;
-using BadMedicine.Datasets;
+using SynthEHR.Datasets;
 using FAnsi.Discovery;
 using NUnit.Framework;
 using Rdmp.Core.Curation.Data;

Rdmp.Core.Tests/QueryCaching/QueryCachingCrossServerTests.cs

@@ -7,8 +7,8 @@
 using System;
 using System.Linq;
 using System.Threading;
-using BadMedicine;
-using BadMedicine.Datasets;
+using SynthEHR;
+using SynthEHR.Datasets;
 using FAnsi;
 using FAnsi.Discovery;
 using NUnit.Framework;

Rdmp.Core/CohortCreation/Execution/ExamplePluginCohortCompiler.cs

@@ -9,7 +9,7 @@
 using System.Data;
 using System.Linq;
 using System.Threading;
-using BadMedicine;
+using SynthEHR;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.Curation.Data.Aggregation;
 using Rdmp.Core.QueryCaching.Aggregation;

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractionCategory.cs

@@ -69,17 +69,25 @@ public override void Execute()
             Show("Cannot set the Extraction Category to 'Core' for a  Project Specific Catalogue item. It will be saved as 'Project Specific'.");
         }
 
-        if (ExecuteWithCommit(() => ExecuteImpl(c.Value), $"Set ExtractionCategory to '{c}'", _extractionInformations))
+        if (ExecuteWithCommit(() => ExecuteImpl(c.Value), c == ExtractionCategory.NotExtractable ? "Make Not Extractable" : $"Set ExtractionCategory to '{c}'", _extractionInformations))
             //publish the root Catalogue
             Publish(_extractionInformations.First());
     }
 
     private void ExecuteImpl(ExtractionCategory category)
     {
+
         foreach (var ei in _extractionInformations)
         {
-            ei.ExtractionCategory = category;
-            ei.SaveToDatabase();
+            if (category == ExtractionCategory.NotExtractable)
+            {
+                ei.DeleteInDatabase();
+            }
+            else
+            {
+                ei.ExtractionCategory = category;
+                ei.SaveToDatabase();
+            }
         }
     }
 }

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandGenerateTestData.cs

@@ -7,13 +7,13 @@
 using System;
 using System.IO;
 using System.Linq;
-using BadMedicine;
-using BadMedicine.Datasets;
+using SynthEHR;
+using SynthEHR.Datasets;
 
 namespace Rdmp.Core.CommandExecution.AtomicCommands;
 
 /// <summary>
-/// Generates CSV files on disk for RDMP example datasets (based on BadMedicine library)
+/// Generates CSV files on disk for RDMP example datasets (based on SynthEHR library)
 /// </summary>
 public class ExecuteCommandGenerateTestData : BasicCommandExecution
 {

Rdmp.Core/CommandLine/DatabaseCreation/ExampleDatasetsCreation.cs

@@ -9,8 +9,8 @@
 using System.IO;
 using System.Linq;
 using System.Text.RegularExpressions;
-using BadMedicine;
-using BadMedicine.Datasets;
+using SynthEHR;
+using SynthEHR.Datasets;
 using FAnsi;
 using FAnsi.Discovery;
 using FAnsi.Discovery.ConnectionStringDefaults;
@@ -641,7 +641,7 @@ private ICatalogue ImportCatalogue(ITableInfo ti)
         var forwardEngineer = new ForwardEngineerCatalogue(ti, ti.ColumnInfos);
         forwardEngineer.ExecuteForwardEngineering(out var cata, out _, out var eis);
 
-        //get descriptions of the columns from BadMedicine
+        //get descriptions of the columns from SynthEHR
         cata.Description = Trim(Descriptions.Get(cata.Name));
         if (cata.Description != null)
         {

Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs

@@ -7,8 +7,8 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using BadMedicine;
-using BadMedicine.Datasets;
+using SynthEHR;
+using SynthEHR.Datasets;
 using FAnsi.Discovery;
 using Rdmp.Core.CohortCommitting.Pipeline;
 using Rdmp.Core.Curation.Data;

Rdmp.Core/Curation/Data/ExtractionCategory.cs

@@ -45,5 +45,10 @@ public enum ExtractionCategory
     /// <summary>
     /// Value can only be used for fetching ExtractionInformations.  This means that all will be returned.  You cannot set a column to have an ExtractionCategory of Any
     /// </summary>
-    Any
+    Any,
+
+    /// <summary>
+    /// Value used for improved UI experience, will be set to null when executed
+    /// </summary>
+    NotExtractable
 }

Rdmp.Core/Curation/Data/SafeDirectoryCatalog.cs

@@ -71,7 +71,7 @@ public static class SafeDirectoryCatalog
         "autoupdater.net.resources.dll",
         "azure.core.dll",
         "azure.identity.dll",
-        "badmedicine.core.dll",
+        "syntehr.core.dll",
         "bouncycastle.crypto.dll",
         "clretwrc.dll",
         "clrjit.dll",

Rdmp.Core/DataExport/Checks/SelectedDataSetsChecker.cs

@@ -137,7 +137,7 @@ public void Check(ICheckNotifier notifier)
         {
             notifier.OnCheckPerformed(
                 new CheckEventArgs(
-                    $"Could not generate valid extraction SQL for dataset {ds} in configuration {config}",
+                    $"Could not generate valid extraction SQL for dataset {ds} in configuration {config}. {e.Message}",
                     CheckResult.Fail, e));
             return;
         }

Rdmp.Core/Icons/IconProvision/StateBasedIconProviders/ExtractionInformationStateBasedIconProvider.cs

@@ -35,6 +35,8 @@ internal sealed class ExtractionInformationStateBasedIconProvider : IObjectState
 
     private static readonly Image<Rgba32> NoIconAvailable = Image.Load<Rgba32>(CatalogueIcons.NoIconAvailable);
 
+    private static readonly Image<Rgba32> ExtractionInformationNotExtractable = IconOverlayProvider.GetOverlayNoCache(ExtractionInformationCore, OverlayKind.Delete);
+
     public Image<Rgba32> GetImageIfSupportedObject(object o)
     {
         if (o is ExtractionCategory cat)
@@ -67,6 +69,7 @@ private static Image<Rgba32> GetImage(ExtractionCategory category)
             ExtractionCategory.Deprecated => ExtractionInformationDeprecated,
             ExtractionCategory.ProjectSpecific => ExtractionInformationProjectSpecific,
             ExtractionCategory.Any => NoIconAvailable,
+            ExtractionCategory.NotExtractable => ExtractionInformationNotExtractable,
             _ => throw new ArgumentOutOfRangeException(nameof(category))
         };
     }

Rdmp.Core/Logging/TableLoadInfo.cs

@@ -8,7 +8,7 @@
 using System.Data;
 using System.Threading;
 using Amazon.Auth.AccessControlPolicy;
-using BadMedicine;
+using SynthEHR;
 using FAnsi.Connections;
 using FAnsi.Discovery;
 using Rdmp.Core.ReusableLibraryCode.Settings;