Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cnde 2274 2 #75

Open
wants to merge 34 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
e0aedab
added log table
ndduc01 Feb 18, 2025
41a16f2
added data log object
ndduc01 Feb 18, 2025
113e5e1
added log table and repos
ndduc01 Feb 18, 2025
7fcf736
added support for log table
ndduc01 Feb 19, 2025
e380539
clean up some log
ndduc01 Feb 19, 2025
543e1bc
update record for poll config
ndduc01 Feb 19, 2025
66806da
update config table
ndduc01 Feb 19, 2025
7e16ac9
clean up
ndduc01 Feb 19, 2025
060d94b
update query
ndduc01 Feb 19, 2025
8ae94f0
refactoring
ndduc01 Feb 19, 2025
bcea3f3
update poll config table
ndduc01 Feb 19, 2025
a558ff0
added query for local test creation
ndduc01 Feb 20, 2025
9157788
update to support no pagination query
ndduc01 Feb 20, 2025
7b26326
update query
ndduc01 Feb 20, 2025
4745bb9
update API logic
ndduc01 Feb 20, 2025
85ee44f
setting sClinet is null
ndduc01 Feb 20, 2025
1ada385
clean up the code
ndduc01 Feb 20, 2025
c823fce
update
ndduc01 Feb 21, 2025
d6274cd
updated based on test
ndduc01 Feb 21, 2025
3248d15
clean up
ndduc01 Feb 21, 2025
a9d5dab
clean up
ndduc01 Feb 21, 2025
7fc09de
clean up repos methods, move duplicate into the centralize localtion
ndduc01 Feb 21, 2025
4e7864b
clean up
ndduc01 Feb 21, 2025
7e11908
clean up
ndduc01 Feb 21, 2025
982ca6c
clean up
ndduc01 Feb 21, 2025
76b182a
Merge branch 'main' of https://github.com/CDCgov/NEDSS-NNDSS into CND…
ndduc01 Feb 21, 2025
24f2fee
clean up
ndduc01 Feb 21, 2025
b7db66a
update config
ndduc01 Feb 21, 2025
e14d57c
clean up unit test
ndduc01 Feb 21, 2025
8d37663
added clean up EDX
ndduc01 Feb 21, 2025
d08fd15
clean up
ndduc01 Feb 21, 2025
8098c39
clean up and disable web service
ndduc01 Feb 21, 2025
2428346
update read me
ndduc01 Feb 21, 2025
e10763a
implemented upsert single
ndduc01 Feb 22, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,13 @@ public ResponseEntity<String> dataSync(@PathVariable String tableName, @RequestP
@RequestHeader(name = "endRow", defaultValue = "0", required = false) String endRow,
@RequestHeader(name = "initialLoad", defaultValue = "false", required = false) String initialLoadApplied,
@RequestHeader(name = "allowNull", defaultValue = "false", required = false) String allowNull,
@RequestHeader(name = "version", defaultValue = "") String version,
@RequestHeader(name = "noPagination", defaultValue = "false") String noPagination,
HttpServletRequest request) throws DataExchangeException {
if (version == null || version.isEmpty()) {
throw new DataExchangeException("Version is Missing");
}

if (request != null) {
String clientIp = getClientIp(request); // Retrieve client IP
logger.info("Fetching Data for Data Availability, Table: {}, Client IP: {}", tableName, clientIp);
Expand All @@ -166,7 +172,7 @@ public ResponseEntity<String> dataSync(@PathVariable String tableName, @RequestP

var ts = convertTimestampFromString(timestamp);
var base64CompressedData = dataExchangeGenericService.getDataForDataSync(tableName, ts, startRow, endRow, Boolean.parseBoolean(initialLoadApplied),
Boolean.parseBoolean(allowNull));
Boolean.parseBoolean(allowNull), Boolean.parseBoolean(noPagination));
return new ResponseEntity<>(base64CompressedData, HttpStatus.OK);
}

Expand Down Expand Up @@ -204,7 +210,13 @@ public ResponseEntity<String> dataSync(@PathVariable String tableName, @RequestP
@GetMapping(path = "/api/datasync/count/{tableName}")
public ResponseEntity<Integer> dataSyncTotalRecords(@PathVariable String tableName,
@RequestParam String timestamp,
@RequestHeader(name = "initialLoad", defaultValue = "false", required = false) String initialLoadApplied) throws DataExchangeException {
@RequestHeader(name = "initialLoad", defaultValue = "false", required = false) String initialLoadApplied,
@RequestHeader(name = "version", defaultValue = "") String version
) throws DataExchangeException {
if (version == null || version.isEmpty()) {
throw new DataExchangeException("Version is Missing");
}

logger.info("Fetching Data Count for Data Availability, Table {}", tableName);
var ts = convertTimestampFromString(timestamp);
var res = dataExchangeGenericService.getTotalRecord(tableName, Boolean.parseBoolean(initialLoadApplied), ts);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ else if (sourceDb.equalsIgnoreCase("NBS_ODSE")) {
}

public String getDataForDataSync(String tableName, String timeStamp, String startRow, String endRow,
boolean initialLoad, boolean allowNull) throws DataExchangeException {
boolean initialLoad, boolean allowNull, boolean noPagination) throws DataExchangeException {

DataSyncConfig dataConfig = getConfigByTableName(tableName);

Expand All @@ -110,7 +110,7 @@ public String getDataForDataSync(String tableName, String timeStamp, String star

try {
Callable<String> callable = () -> {
String baseQuery = preparePaginationQuery(dataConfig, timeStamp, startRow, endRow, initialLoad, allowNull);
String baseQuery = preparePaginationQuery(dataConfig, timeStamp, startRow, endRow, initialLoad, allowNull, noPagination);

List<Map<String, Object>> data = executeQueryForData(baseQuery, dataConfig.getSourceDb());

Expand All @@ -132,13 +132,16 @@ public String getDataForDataSync(String tableName, String timeStamp, String star
}

private String preparePaginationQuery(DataSyncConfig dataConfig, String timeStamp, String startRow,
String endRow, boolean initialLoad, boolean allowNull) {
String endRow, boolean initialLoad, boolean allowNull, boolean noPagination) {

String baseQuery;

if (allowNull && dataConfig.getQueryWithNullTimeStamp() != null && !dataConfig.getQueryWithNullTimeStamp().isEmpty()) {
baseQuery = dataConfig.getQueryWithNullTimeStamp().replaceAll(";", "") ; //NOSONAR
}
else if (noPagination || dataConfig.getQueryWithPagination().isEmpty()) {
baseQuery = dataConfig.getQuery().replaceAll(";", "") ; //NOSONAR
}
else
{
baseQuery = dataConfig.getQueryWithPagination()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
public interface IDataExchangeGenericService {
Integer getTotalRecord(String tableName, boolean initialLoad, String timestamp) throws DataExchangeException;
String getDataForDataSync(String tableName, String timeStamp, String startRow, String endRow,
boolean initialLoad, boolean allowNull) throws DataExchangeException;
boolean initialLoad, boolean allowNull, boolean noPagination) throws DataExchangeException;
String decodeAndDecompress(String base64EncodedData) throws DataExchangeException;

}
Loading