tidy up from codeql

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandIdentifyCHIInCatalogue.cs

@@ -22,6 +22,17 @@ public class ExecuteCommandIdentifyCHIInCatalogue : BasicCommandExecution, IAtom
     private readonly bool _bailOutEarly;
     private readonly Dictionary<string, List<string>> _allowLists = new();
 
+    private readonly string PotentialCHI = "Potential CHI";
+    private readonly string Context = "Context";
+    private readonly string SourceColumnName = "Source Column Name";
+    private readonly string PKValue = "PK Value";
+    private readonly string PKColumn = "PK Column";
+    private readonly string ReplacementIndex = "replacementIndex";
+    private readonly string RDMP_ALL = "RDMP_ALL";
+
+    public DataTable foundChis = new();
+
+
     public ExecuteCommandIdentifyCHIInCatalogue(IBasicActivateItems activator, [DemandsInitialization("The catalogue to search")] ICatalogue catalogue, bool bailOutEarly = false, string allowListLocation = null) : base(activator)
     {
         _catalouge = catalogue;
@@ -51,33 +62,12 @@ public static string WrapCHIInContext(string chi, string source, int padding = 2
         return $"{source[Math.Max(0, foundIndex - padding)..foundIndex]}{chi}{source[(foundIndex + chi.Length)..Math.Min(foundIndex + chi.Length + padding, source.Length)]}";
     }
 
-
-    private readonly string PotentialCHI = "Potential CHI";
-    private readonly string Context = "Context";
-    private readonly string SourceColumnName = "Source Column Name";
-    private readonly string PKValue = "PK Value";
-    private readonly string PKColumn = "PK Column";
-    private readonly string ReplacementIndex = "replacementIndex";
-    private readonly string RDMP_ALL = "RDMP_ALL";
-
-
     private void HandleFoundCHI(string foundChi, string contextValue, string columnName, string pkValue, string pkColumn)
     {
         if (pkColumn.Split(".").Last().Replace("[", "").Replace("]", "") == columnName.Replace("[", "").Replace("]", "")) return; //don't redact PKs, it gets messy
-        //if (foundChis.Rows.Count == 0)
-        //{
-        //    // init
-        //    foundChis.Columns.Add(PotentialCHI);
-        //    foundChis.Columns.Add(Context);
-        //    foundChis.Columns.Add(SourceColumnName);
-        //    foundChis.Columns.Add(PKValue);
-        //    foundChis.Columns.Add(PKColumn);
-        //    foundChis.Columns.Add(ReplacementIndex);
-        //}
         var shrunkContext = WrapCHIInContext(foundChi, contextValue);
         foundChis.Rows.Add(foundChi, shrunkContext, columnName, pkValue, pkColumn, contextValue.IndexOf(foundChi));
     }
-    public DataTable foundChis = new();
 
     public override void Execute()
     {
@@ -102,7 +92,6 @@ public override void Execute()
             int idxOfLastSplit = column.LastIndexOf('.');
             var columnName = column[(idxOfLastSplit + 1)..];
             var server = _catalouge.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false);
-            // what if there is no pk?
             var pkColumns = _catalouge.CatalogueItems.Select(x => x.ColumnInfo).Where(x => x.IsPrimaryKey);
             if (pkColumns.Any())
             {
@@ -133,6 +122,7 @@ public override void Execute()
                         }
                     }
                 }
+                dt.Dispose();
             }
             else
             {

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRedactCHIsFromCatalogue.cs

@@ -3,35 +3,22 @@
 // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
 // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 // You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
-using Rdmp.Core.CommandExecution.AtomicCommands;
-using Rdmp.Core.CommandExecution;
-using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using Rdmp.Core.Curation.Data;
-using YamlDotNet.Serialization;
-using System.IO;
 using Rdmp.Core.ReusableLibraryCode.DataAccess;
 using System.Data;
-using static NPOI.HSSF.Util.HSSFColor;
-using Rdmp.Core.Curation.Data.Defaults;
-using TB.ComponentModel;
-using FAnsi.Discovery.TableCreation;
-using Rdmp.Core.DataFlowPipeline;
 using Microsoft.Data.SqlClient;
 
 namespace Rdmp.Core.CommandExecution.AtomicCommands;
 
 public class ExecuteCommandRedactCHIsFromCatalogue : BasicCommandExecution, IAtomicCommand
 {
 
-    private ICatalogue _catalogue;
-    private IBasicActivateItems _activator;
-    private readonly Dictionary<string, List<string>> _allowLists = new();
+    private readonly ICatalogue _catalogue;
+    private readonly IBasicActivateItems _activator;
     public int redactionCount = 0;
-    private string _allowListLocation = "";
+    private readonly string _allowListLocation = "";
 
     public ExecuteCommandRedactCHIsFromCatalogue(IBasicActivateItems activator, [DemandsInitialization("The catalogue to search")] ICatalogue catalogue, string allowListLocation = null) : base(activator)
     {
@@ -51,7 +38,6 @@ public override void Execute()
             redactionCount++;
             var result = row;
             var foundChi = result.ItemArray[0].ToString();
-            var columnValue = result.ItemArray[1].ToString();
             var column = result.ItemArray[2].ToString();
             var catalogueItem = _catalogue.CatalogueItems.Where(ci => ci.Name == column).First();
             var name = catalogueItem.ColumnInfo.Name;
@@ -70,17 +56,23 @@ public override void Execute()
             using (var con = (SqlConnection)catalogue.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false).GetConnection())
             {
                 con.Open();
-                var da = new SqlDataAdapter(new SqlCommand(fetchSQL, con));
+                var sqlCommand = new SqlCommand(fetchSQL, con);
+                var da = new SqlDataAdapter(sqlCommand);
                 da.Fill(existingResultsDT);
+                da.Dispose();
+                sqlCommand.Dispose();
                 if (existingResultsDT.Rows.Count > 0 && existingResultsDT.Rows[0].ItemArray.Length > 0)
                 {
                     var currentContext = existingResultsDT.Rows[0].ItemArray[0].ToString();
                     var newContext = currentContext.Replace(foundChi, redactedString);
                     var updateSQL = $"update {table} set {column}='{newContext}' where {pkColumn} = '{pkValue}' and {column}='{currentContext}'";
                     var updateCmd = new SqlCommand(updateSQL, con);
                     updateCmd.ExecuteNonQuery();
+                    updateCmd.Dispose();
                 }
             }
+            existingResultsDT.Dispose();
         }
+        results.Dispose();
     }
 }

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRevertRedactedCHI.cs

@@ -6,19 +6,15 @@
 using Microsoft.Data.SqlClient;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.ReusableLibraryCode.DataAccess;
-using System;
-using System.Collections.Generic;
 using System.Data;
 using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace Rdmp.Core.CommandExecution.AtomicCommands;
 
 public class ExecuteCommandRevertRedactedCHI : BasicCommandExecution, IAtomicCommand
 {
-    RedactedCHI _redactedCHI;
-    IBasicActivateItems _activator;
+    private readonly RedactedCHI _redactedCHI;
+    private readonly IBasicActivateItems _activator;
 
     public ExecuteCommandRevertRedactedCHI(IBasicActivateItems activator, [DemandsInitialization("Redacted CHI to Revert")] RedactedCHI redaction) : base(activator)
     {
@@ -29,6 +25,7 @@ public ExecuteCommandRevertRedactedCHI(IBasicActivateItems activator, [DemandsIn
     public override void Execute()
     {
         base.Execute();
+        //todo i think there is some duplication here with other files, may be able to use a helper
         var redactedString = new string('#', _redactedCHI.PotentialCHI.Length);
         var fetchSQL = $"select {_redactedCHI.ColumnName} from {_redactedCHI.TableName} where {_redactedCHI.PKColumnName} = '{_redactedCHI.PKValue}' and charindex('{redactedString}',{_redactedCHI.ColumnName},{_redactedCHI.ReplacementIndex}) >0";
         var existingResultsDT = new DataTable();
@@ -37,18 +34,22 @@ public override void Execute()
         using (var con = (SqlConnection)catalogue.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false).GetConnection())
         {
             con.Open();
-            var da = new SqlDataAdapter(new SqlCommand(fetchSQL, con));
+            var sqlCommand = new SqlCommand(fetchSQL, con);
+            var da = new SqlDataAdapter(sqlCommand);
             da.Fill(existingResultsDT);
+            sqlCommand.Dispose();
+            da.Dispose();
             if (existingResultsDT.Rows.Count > 0 && existingResultsDT.Rows[0].ItemArray.Length > 0)
             {
                 var currentContext = existingResultsDT.Rows[0].ItemArray[0].ToString();
                 var newContext = currentContext.Replace(redactedString, _redactedCHI.PotentialCHI);
                 var updateSQL = $"update {_redactedCHI.TableName} set {_redactedCHI.ColumnName}='{newContext}' where {_redactedCHI.PKColumnName} = '{_redactedCHI.PKValue}' and {_redactedCHI.ColumnName}='{currentContext}'";
                 var updateCmd = new SqlCommand(updateSQL, con);
                 updateCmd.ExecuteNonQuery();
+                updateCmd.Dispose();
             }
             _redactedCHI.DeleteInDatabase();
-
         }
+        existingResultsDT.Dispose();
     }
 }

Rdmp.Core/CommandExecution/ExecuteCommandPasteClipboardAsNewCatalogueItems.cs

@@ -45,8 +45,8 @@ public ExecuteCommandPasteClipboardAsNewCatalogueItems(IBasicActivateItems activ
         _clipboardContentGetter = clipboardContentGetter;
     }
 
-    //public override Image<Rgba32> GetImage(IIconProvider iconProvider) =>
-    //    iconProvider.GetImage(RDMPConcept.Clipboard, OverlayKind.Import);
+    public override Image<Rgba32> GetImage(IIconProvider iconProvider) =>
+        iconProvider.GetImage(RDMPConcept.Clipboard, OverlayKind.Import);
 
     public override void Execute()
     {

Rdmp.Core/Curation/Data/IRedactedCHI.cs

@@ -3,21 +3,13 @@
 // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
 // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 // You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
-using FAnsi.Naming;
-using Rdmp.Core.Curation.Data.Cohort;
 using Rdmp.Core.MapsDirectlyToDatabaseTable;
 using Rdmp.Core.Repositories;
-using Rdmp.Core.ReusableLibraryCode.Checks;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace Rdmp.Core.Curation.Data;
 
 /// <summary>
-/// This class stores the redacted CHIs that are found during data load of via catalogue mutilation
+/// This class stores the redacted CHIs that are found during data load or via catalogue mutilation
 /// </summary>
 public interface IRedactedCHI : IMapsDirectlyToDatabaseTable
 {

Rdmp.Core/Curation/Data/RedactedCHI.cs

@@ -3,21 +3,10 @@
 // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
 // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 // You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.using Amazon.Auth.AccessControlPolicy;
-using Rdmp.Core.Icons.IconProvision;
-using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes;
 using Rdmp.Core.Repositories;
-using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision;
-using SixLabors.ImageSharp.PixelFormats;
-using SixLabors.ImageSharp;
-using System;
 using System.Collections.Generic;
 using System.Data.Common;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using Rdmp.Core.MapsDirectlyToDatabaseTable;
 using System.Diagnostics.CodeAnalysis;
-using Rdmp.Core.Curation.Data;
 
 
 
@@ -80,15 +69,15 @@ public RedactedCHI(ICatalogueRepository catalogueRepository, string potentialCHI
     {
         catalogueRepository.InsertAndHydrate(this, new Dictionary<string, object>
         {
-            {"potentialCHI", potentialCHI },{"replacementIndex",replacementIndex},{"tableName",table},{"PKValue",pkValue},{"pkColumnName",pkColumnName},{"columnName",columnName}
+            {"PotentialCHI", potentialCHI },{"ReplacementIndex",replacementIndex},{"TableName",table},{"PKValue",pkValue},{"PKColumnName",pkColumnName},{"ColumnName",columnName}
         });
     }
 
     public RedactedCHI() { }
     public RedactedCHI(ICatalogueRepository repository, DbDataReader r)
        : base(repository, r)
     {
-        PotentialCHI = r["PotentialChi"].ToString();
+        PotentialCHI = r["PotentialCHI"].ToString();
         ReplacementIndex = int.Parse(r["ReplacementIndex"].ToString());
         TableName = r["TableName"].ToString();
         PKColumnName = r["PKColumnName"].ToString();

Rdmp.Core/DataFlowPipeline/CHIColumnFinder.cs

@@ -90,7 +90,7 @@ public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener
         var listFile = new Lazy<StreamWriter>(() =>
             {
                 var stream = fileLocation is not null ? File.AppendText(fileLocation) : null;
-                if (stream?.BaseStream.Length == 0)
+                if (stream?.BaseStream.Length == 0 && _csvColumns is not null)
                     stream.WriteLine(_csvColumns);
                 return stream;
             },

Rdmp.Core/DataLoad/Engine/LoadExecution/Components/Runtime/ExecuteCHIRedactionStage.cs

@@ -4,8 +4,6 @@
 // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 // You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.using Amazon.Auth.AccessControlPolicy;
 using FAnsi.Discovery;
-using Rdmp.Core.CommandExecution;
-using Rdmp.Core.CommandExecution.AtomicCommands;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.Curation.Data.DataLoad;
 using Rdmp.Core.DataFlowPipeline;
@@ -14,10 +12,7 @@
 using System;
 using System.Collections.Generic;
 using System.Data;
-using System.Diagnostics;
 using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace Rdmp.Core.DataLoad.Engine.LoadExecution.Components.Runtime;
 
@@ -65,7 +60,6 @@ private void RedactCHIs(ITableInfo tableInfo)
         List<String> _allowList = new();
         if (_allowLists is not null && _allowLists.TryGetValue("RDMP_ALL", out var _extractionSpecificAllowances))
             _allowList.AddRange(_extractionSpecificAllowances);
-        var y = tbl.ToString();
         if (_allowLists is not null && _allowLists.TryGetValue(tbl.ToString(), out var _catalogueSpecificAllowances))
             _allowList.AddRange(_catalogueSpecificAllowances.ToList());
         foreach (DataColumn col in dt.Columns)
@@ -81,16 +75,16 @@ private void RedactCHIs(ITableInfo tableInfo)
                         var replacementIdex = row[col].ToString().IndexOf(foundChi);
                         var foundTable = tbl.GetFullyQualifiedName().Replace(_loadStage.ToString(), "").Split("..")[1].Replace("[", "").Replace("]", "");
                         var catalogue = _job.RepositoryLocator.CatalogueRepository.GetAllObjects<Catalogue>().Where(catalogue => catalogue.Name == foundTable).First();
-                        var pkValue = "fake";
+                        var pkValue = "Unknown";
                         var pkColumnName = "Unknown";
                         if (catalogue != null)
                         {
                             //this can probably be tidied up
-                            Console.WriteLine(foundTable);
+                            //is likely code duplication with another file
                             var pkColumnInfo = catalogue.CatalogueItems.Select(x => x.ColumnInfo).Where(x => x.IsPrimaryKey && x.Name.Contains(foundTable)).First(); //there may be more, but we just need one
-                            pkColumnName = pkColumnInfo.Name;
                             if (pkColumnInfo != null)
                             {
+                                pkColumnName = pkColumnInfo.Name;
                                 var pkName = pkColumnInfo.Name.Split(".").Last().Replace("[", "").Replace("]", "");
                                 var arrayNames = (from DataColumn x
                                                   in dt.Columns.Cast<DataColumn>()
@@ -99,15 +93,22 @@ in dt.Columns.Cast<DataColumn>()
                                 pkValue = row[index].ToString();
                             }
                         }
-                        var ft = tbl.GetFullyQualifiedName().Replace("_RAW", ""); //TODO
+                        var ft = tbl.GetFullyQualifiedName();
+                        if(_loadStage == LoadStage.AdjustRaw)
+                        {
+                            ft = ft.Replace("_RAW", "");
+                        }
+                        if (_loadStage == LoadStage.AdjustStaging)
+                        {
+                            ft = ft.Replace("_STAGING", "");
+                        }
                         ft = ft.Replace("..", ".[dbo].");
                         if (pkColumnName.Split(".").Last().Replace("[","").Replace("]","") != $"{col.ColumnName}")
                         {
-                            Console.WriteLine("hi");
                             var rc = new RedactedCHI(_job.RepositoryLocator.CatalogueRepository, foundChi, replacementIdex, ft, pkValue, pkColumnName.Split(".").Last(), $"[{col.ColumnName}]");
                             rc.SaveToDatabase();
                             var redactionString = "##########";
-                            row[col] = row[col].ToString().Replace(foundChi, redactionString.Substring(0, Math.Min(foundChi.Length, redactionString.Length)));
+                            row[col] = row[col].ToString().Replace(foundChi, redactionString[..Math.Min(foundChi.Length, redactionString.Length)]);
                         }
                     }
                     else

Rdmp.Core/DataLoad/Modules/Mutilators/CHIRedactionMutilator.cs

@@ -11,13 +11,8 @@
 using Rdmp.Core.DataLoad.Engine.Mutilators;
 using Rdmp.Core.ReusableLibraryCode.Checks;
 using Rdmp.Core.ReusableLibraryCode.Progress;
-using System;
 using System.Collections.Generic;
-using System.Diagnostics;
 using System.IO;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using YamlDotNet.Serialization;
 
 namespace Rdmp.Core.DataLoad.Modules.Mutilators;