Skip to content

Commit

Permalink
Merge pull request #1807 from HicServices/task/RDMP-170-delta-issue
Browse files Browse the repository at this point in the history 
Task/rdmp 170 Improve RemoteTableAttacher
  • Loading branch information
@bpeacock001
bpeacock001 authored May 20, 2024
2 parents f4112ab + 77dfd57 commit 1c58209
Show file tree
Hide file tree
Showing 4 changed files with 30 additions and 10 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

- Improve error messages for Multi-ExtractionIdentifier extractions
- Add Microsoft.Bcl.AsyncInterfaces 6.0.0 for plugin dependancy tree
- Allow for column selection when using the RemoteTableAttacher
- Add prompt to reanem container when adding a cohort filter
- Fix to remove stack trace button from non error popups
- Add ability to set Extraction Categort as "Not Extractable"
Expand Down
12 changes: 10 additions & 2 deletions Documentation/DataLoadEngine/RemoteAttachers.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ The full configuration options are
| Delta Reading Look Forward Days | If using the Delta Reading fetch duration, this is how many days forward in time you wish to look each time |
| Set Delta Reading To Last Seen Date Post Load | Optional overwrite to the Delta Reading fetch option. Will use the most recently seen date in the fetched data rather than the adding the forward look days amount onto the stored minimum date |
| Culture | Optionally specify a custom date format |
| Explicit Date Time Format | Optionally specify a specific datetime format
| Explicit Date Time Format | Optionally specify a specific datetime format
| Selected Columns | Optionally select which columns you wish to pull from the remote server (defaults to "*" )

## Configuring the Remote Database Attacher
The Remote Database Attacher has a number of configuration options the required fields are:
Expand All @@ -68,4 +69,11 @@ The Remote Database Attacher has a number of configuration options the required
| Delta Reading Look Forward Days | If using the Delta Reading fetch duration, this is how many days forward in time you wish to look each time |
| Set Delta Reading To Last Seen Date Post Load | Optional overwrite to the Delta Reading fetch option. Will use the most recently seen date in the fetched data rather than the adding the forward look days amount onto the stored minimum date |
| Culture | Optionally specify a custom date format |
| Explicit Date Time Format | Optionally specify a specific datetime format
| Explicit Date Time Format | Optionally specify a specific datetime format
| Selected Columns | Optionally select which columns you wish to pull from the remote server (defaults to "*" )


## Using Historical Loading with custom SQL
By default, historical fetch configuration will be ignored in custome select SQL.
To enable the use of hostorical fetching, include the string "$RDMPDefinedWhereClause" in the where clause of your query
e.e. "Select * from Biochemistry Where $RDMPDefinedWhereClause"
23 changes: 17 additions & 6 deletions Rdmp.Core/DataLoad/Modules/Attachers/RemoteTableAttacher.cs
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
// Copyright (c) The University of Dundee 2018-2019
// Copyright (c) The University of Dundee 2018-2024
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Data;
using System.Diagnostics;
using System.Linq;
using System.Text.RegularExpressions;
using FAnsi;
Expand All @@ -15,7 +16,6 @@
using Rdmp.Core.Curation.Data.DataLoad;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataFlowPipeline.Requirements;
using Rdmp.Core.DataLoad.Engine.Attachers;
using Rdmp.Core.DataLoad.Engine.Job;
using Rdmp.Core.DataLoad.Engine.Job.Scheduling;
using Rdmp.Core.DataLoad.Engine.Pipeline.Destinations;
Expand All @@ -27,6 +27,7 @@
using Rdmp.Core.ReusableLibraryCode.DataAccess;
using Rdmp.Core.ReusableLibraryCode.Progress;
using TypeGuesser;
using static NPOI.HSSF.Util.HSSFColor;

namespace Rdmp.Core.DataLoad.Modules.Attachers;

Expand Down Expand Up @@ -94,6 +95,9 @@ public RemoteTableAttacher() : base()
DefaultValue = DatabaseType.MicrosoftSQLServer)]
public DatabaseType DatabaseType { get; set; }

[DemandsInitialization("The Columns you wish to pull from the remote table", DefaultValue = "*")]
public string SelectedColumns { get; set; } = "*";

private const string StartDateParameter = "@startDate";
private const string EndDateParameter = "@endDate";

Expand Down Expand Up @@ -359,9 +363,16 @@ public override ExitCodeType Attach(IDataLoadJob job, GracefulCancellationToken

var syntax = _remoteDatabase.Server.GetQuerySyntaxHelper();

var sql = !string.IsNullOrWhiteSpace(RemoteSelectSQL)
? RemoteSelectSQL
: $"Select * from {syntax.EnsureWrapped(RemoteTableName)} {SqlHistoricalDataFilter(job.LoadMetadata,DatabaseType)}";
string sql;
if (!string.IsNullOrWhiteSpace(RemoteSelectSQL))
{
var injectedWhereClause = SqlHistoricalDataFilter(job.LoadMetadata, DatabaseType).Replace(" WHERE", "");
sql = Regex.Replace(RemoteSelectSQL, "\\$RDMPDefinedWhereClause", injectedWhereClause);
}
else
{
sql = $"Select {SelectedColumns} from {syntax.EnsureWrapped(RemoteTableName)} {SqlHistoricalDataFilter(job.LoadMetadata, DatabaseType)}";
}

var scheduleMismatch = false;
//if there is a load progress
Expand Down Expand Up @@ -457,7 +468,7 @@ public override ExitCodeType Attach(IDataLoadJob job, GracefulCancellationToken

if (SetDeltaReadingToLatestSeenDatePostLoad)
{
FindMostRecentDateInLoadedData(rawSyntax, _dbInfo.Server.DatabaseType ,rawTableName, job);
FindMostRecentDateInLoadedData(rawSyntax, _dbInfo.Server.DatabaseType, rawTableName, job);
}


Expand Down
4 changes: 2 additions & 2 deletions rdmp-client.xml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<item>
<version>8.1.4.0</version>
<url>https://github.com/HicServices/RDMP/releases/download/v8.1.4/rdmp-8.1.4-client.zip</url>
<version>8.1.6.0</version>
<url>https://github.com/HicServices/RDMP/releases/download/v8.1.4/rdmp-8.1.6-client.zip</url>
<changelog>https://github.com/HicServices/RDMP/blob/main/CHANGELOG.md#7</changelog>
<mandatory>true</mandatory>
</item>

0 comments on commit 1c58209

Please sign in to comment.