diff --git a/.DS_Store b/.DS_Store index bd29cd5d..895a83d4 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/DataExchangeGenericService.java b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/DataExchangeGenericService.java index 650058f3..6b311868 100644 --- a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/DataExchangeGenericService.java +++ b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/DataExchangeGenericService.java @@ -29,16 +29,19 @@ public class DataExchangeGenericService implements IDataExchangeGenericService { private final JdbcTemplate jdbcTemplate; private final JdbcTemplate srteJdbcTemplate; private final JdbcTemplate rdbModernJdbcTemplate; + private final JdbcTemplate odseJdbcTemplate; private final Gson gson; public DataExchangeGenericService(DataSyncConfigRepository dataSyncConfigRepository, @Qualifier("rdbJdbcTemplate") JdbcTemplate jdbcTemplate, @Qualifier("srteJdbcTemplate") JdbcTemplate srteJdbcTemplate, - @Qualifier("rdbModernJdbcTemplate") JdbcTemplate rdbModernJdbcTemplate) { + @Qualifier("rdbModernJdbcTemplate") JdbcTemplate rdbModernJdbcTemplate, + @Qualifier("odseJdbcTemplate") JdbcTemplate odseJdbcTemplate) { this.dataSyncConfigRepository = dataSyncConfigRepository; this.jdbcTemplate = jdbcTemplate; this.srteJdbcTemplate = srteJdbcTemplate; this.rdbModernJdbcTemplate = rdbModernJdbcTemplate; + this.odseJdbcTemplate = odseJdbcTemplate; this.gson = new GsonBuilder() .registerTypeAdapter(Timestamp.class, TimestampAdapter.getTimestampSerializer()) @@ -74,7 +77,11 @@ private Integer executeQueryForTotalRecords(String query, String sourceDb) throw return srteJdbcTemplate.queryForObject(query, Integer.class); } else if (sourceDb.equalsIgnoreCase(DB_RDB_MODERN)) { return rdbModernJdbcTemplate.queryForObject(query, Integer.class); - } else { + } + else if (sourceDb.equalsIgnoreCase("NBS_ODSE")) { + return odseJdbcTemplate.queryForObject(query, Integer.class); + } + else { throw new DataExchangeException("Database Not Supported: " + sourceDb); } } catch (Exception e) { @@ -131,7 +138,11 @@ private List> executeQueryForData(String query, String sourc return jdbcTemplate.queryForList(query); } else if (sourceDb.equalsIgnoreCase(DB_RDB_MODERN)) { return rdbModernJdbcTemplate.queryForList(query); - } else { + } else if (sourceDb.equalsIgnoreCase("NBS_ODSE")) { + return odseJdbcTemplate.queryForList(query); + } + + else { throw new DataExchangeException("DB IS NOT SUPPORTED: " + sourceDb); } } diff --git a/nnd-data-exchange-service/src/main/resources/query/create/generate_data_rdb.sql b/nnd-data-exchange-service/src/main/resources/query/create/generate_data_rdb.sql index bdd9e3cb..877f7748 100644 --- a/nnd-data-exchange-service/src/main/resources/query/create/generate_data_rdb.sql +++ b/nnd-data-exchange-service/src/main/resources/query/create/generate_data_rdb.sql @@ -503,3 +503,49 @@ VALUES WHERE RowNum BETWEEN :startRow AND :endRow;') ; + +INSERT INTO [RDB].[dbo].[data_sync_config] +(table_name, source_db, query, query_with_null_timestamp, query_count, query_with_pagination) +VALUES + ('EDX_ACTIVITY_LOG', 'NBS_ODSE', + 'SELECT * + FROM EDX_ACTIVITY_LOG + WHERE record_status_time :operator :timestamp;', + NULL, + 'SELECT COUNT(*) + FROM EDX_ACTIVITY_LOG + WHERE record_status_time :operator :timestamp;', + 'WITH PaginatedResults AS ( + SELECT *, ROW_NUMBER() OVER (ORDER BY record_status_time ASC) AS RowNum + FROM EDX_ACTIVITY_LOG + WHERE record_status_time :operator :timestamp + ) + SELECT * FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;') +; + +INSERT INTO [RDB].[dbo].[data_sync_config] +(table_name, source_db, query, query_with_null_timestamp, query_count, query_with_pagination) +VALUES + ('EDX_ACTIVITY_DETAIL_LOG', 'NBS_ODSE', + 'SELECT logg.record_status_time, detail.* + FROM EDX_ACTIVITY_DETAIL_LOG AS detail + INNER JOIN EDX_ACTIVITY_LOG AS logg + ON detail.edx_activity_log_uid = logg.edx_activity_log_uid + WHERE logg.record_status_time :operator :timestamp;', + NULL, + 'SELECT COUNT(*) + FROM EDX_ACTIVITY_DETAIL_LOG AS detail + INNER JOIN EDX_ACTIVITY_LOG AS logg + ON detail.edx_activity_log_uid = logg.edx_activity_log_uid + WHERE logg.record_status_time :operator :timestamp;', + 'WITH PaginatedResults AS ( + SELECT logg.record_status_time, detail.*, ROW_NUMBER() OVER (ORDER BY logg.record_status_time ASC) AS RowNum + FROM EDX_ACTIVITY_DETAIL_LOG AS detail + INNER JOIN EDX_ACTIVITY_LOG AS logg + ON detail.edx_activity_log_uid = logg.edx_activity_log_uid + WHERE logg.record_status_time :operator :timestamp + ) + SELECT * FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;') +; \ No newline at end of file diff --git a/nnd-data-poll-service/.DS_Store b/nnd-data-poll-service/.DS_Store index b99ade0f..8bcfc2fe 100644 Binary files a/nnd-data-poll-service/.DS_Store and b/nnd-data-poll-service/.DS_Store differ diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/DataIngestDataSourceConfig.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/DataIngestDataSourceConfig.java index 295436d3..97941b03 100644 --- a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/DataIngestDataSourceConfig.java +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/DataIngestDataSourceConfig.java @@ -8,6 +8,7 @@ import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Lazy; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.orm.jpa.JpaTransactionManager; import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; @@ -42,6 +43,7 @@ public class DataIngestDataSourceConfig { private String dbUserPassword; @Bean(name = "ingestDataSource") + @Lazy public DataSource ingestDataSource() { DataSourceBuilder dataSourceBuilder = DataSourceBuilder.create(); diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/MsgDataSourceConfig.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/MsgDataSourceConfig.java deleted file mode 100644 index d69f0a3f..00000000 --- a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/MsgDataSourceConfig.java +++ /dev/null @@ -1,76 +0,0 @@ -//package gov.cdc.nnddatapollservice.configuration; -// -// -//import jakarta.persistence.EntityManagerFactory; -//import org.springframework.beans.factory.annotation.Qualifier; -//import org.springframework.beans.factory.annotation.Value; -//import org.springframework.boot.jdbc.DataSourceBuilder; -//import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder; -//import org.springframework.context.annotation.Bean; -//import org.springframework.context.annotation.Configuration; -//import org.springframework.data.jpa.repository.config.EnableJpaRepositories; -//import org.springframework.orm.jpa.JpaTransactionManager; -//import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; -//import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; -//import org.springframework.transaction.PlatformTransactionManager; -//import org.springframework.transaction.annotation.EnableTransactionManagement; -// -//import javax.sql.DataSource; -//import java.util.HashMap; -// -//@Configuration -//@EnableTransactionManagement -//@EnableJpaRepositories( -// entityManagerFactoryRef = "msgEntityManagerFactory", -// transactionManagerRef = "msgTransactionManager", -// basePackages = { -// "gov.cdc.nnddatapollservice.repository.msg" -// } -//) -//public class MsgDataSourceConfig { -// @Value("${spring.datasource.driverClassName}") -// private String driverClassName; -// -// @Value("${spring.datasource.msg.url}") -// private String dbUrl; -// -// @Value("${spring.datasource.username}") -// private String dbUserName; -// -// @Value("${spring.datasource.password}") -// private String dbUserPassword; -// -// @Bean(name = "msgDataSource") -// public DataSource msgDataSource() { -// DataSourceBuilder dataSourceBuilder = DataSourceBuilder.create(); -// -// dataSourceBuilder.driverClassName(driverClassName); -// dataSourceBuilder.url(dbUrl); -// dataSourceBuilder.username(dbUserName); -// dataSourceBuilder.password(dbUserPassword); -// -// return dataSourceBuilder.build(); -// } -// -// @Bean(name = "msgEntityManagerFactoryBuilder") -// public EntityManagerFactoryBuilder msgEntityManagerFactoryBuilder() { -// return new EntityManagerFactoryBuilder(new HibernateJpaVendorAdapter(), new HashMap<>(), null); -// } -// -// @Bean(name = "msgEntityManagerFactory") -// public LocalContainerEntityManagerFactoryBean msgEntityManagerFactory( -// EntityManagerFactoryBuilder msgEntityManagerFactoryBuilder, -// @Qualifier("msgDataSource") DataSource msgDataSource ) { -// return msgEntityManagerFactoryBuilder -// .dataSource(msgDataSource) -// .packages("gov.cdc.nnddatapollservice.repository.msg.model") -// .persistenceUnit("msg") -// .build(); -// } -// -// @Bean(name = "msgTransactionManager") -// public PlatformTransactionManager msgTransactionManager( -// @Qualifier("msgEntityManagerFactory") EntityManagerFactory msgEntityManagerFactory ) { -// return new JpaTransactionManager(msgEntityManagerFactory); -// } -//} diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/NbsOdseDataSourceConfig.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/NbsOdseDataSourceConfig.java new file mode 100644 index 00000000..07014d50 --- /dev/null +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/NbsOdseDataSourceConfig.java @@ -0,0 +1,83 @@ +package gov.cdc.nnddatapollservice.configuration; + +import jakarta.persistence.EntityManagerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Lazy; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.orm.jpa.JpaTransactionManager; +import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; +import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; +import org.springframework.transaction.PlatformTransactionManager; + +import javax.sql.DataSource; +import java.util.HashMap; + +@Configuration +@EnableJpaRepositories( + entityManagerFactoryRef = "nbsOdseEntityManagerFactory", + transactionManagerRef = "nbsOdseTransactionManager", + basePackages = { + "gov.cdc.nnddatapollservice.repository.nbs_odse", + } +) +public class NbsOdseDataSourceConfig { + @Value("${spring.datasource.driverClassName}") + private String driverClassName; + + @Value("${spring.datasource.odse.url}") + private String dbUrl; + + @Value("${spring.datasource.username}") + private String dbUserName; + + @Value("${spring.datasource.password}") + private String dbUserPassword; + + @Bean(name = "nbsOdseDataSource") + @Lazy + public DataSource nbsOdseDataSource() { + DataSourceBuilder dataSourceBuilder = DataSourceBuilder.create(); + + dataSourceBuilder.driverClassName(driverClassName); + dataSourceBuilder.url(dbUrl); + dataSourceBuilder.username(dbUserName); + dataSourceBuilder.password(dbUserPassword); + + return dataSourceBuilder.build(); + } + @Bean(name = "nbsOdseJdbcTemplate") + public JdbcTemplate nbsOdseJdbcTemplate(@Qualifier("nbsOdseDataSource") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + + // JPA Configurations + + @Bean(name = "nbsOdseEntityManagerFactoryBuilder") + public EntityManagerFactoryBuilder nbsOdseEntityManagerFactoryBuilder() { + return new EntityManagerFactoryBuilder(new HibernateJpaVendorAdapter(), new HashMap<>(), null); + } + + @Bean(name = "nbsOdseEntityManagerFactory") + public LocalContainerEntityManagerFactoryBean nbsOdseEntityManagerFactory( + @Qualifier("nbsOdseEntityManagerFactoryBuilder") EntityManagerFactoryBuilder builder, + @Qualifier("nbsOdseDataSource") DataSource dataSource) { + return builder + .dataSource(dataSource) + .packages("gov.cdc.nnddatapollservice.repository.nbs_odse") // Adjust package for your entities + .persistenceUnit("nbsodse") + .build(); + } + + @Bean(name = "nbsOdseTransactionManager") + public PlatformTransactionManager nbsOdseTransactionManager( + @Qualifier("nbsOdseEntityManagerFactory") EntityManagerFactory entityManagerFactory) { + return new JpaTransactionManager(entityManagerFactory); + } +} \ No newline at end of file diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/OdseDataSourceConfig.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/OdseDataSourceConfig.java deleted file mode 100644 index 498ee785..00000000 --- a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/OdseDataSourceConfig.java +++ /dev/null @@ -1,78 +0,0 @@ -//package gov.cdc.nnddatapollservice.configuration; -// -//import jakarta.persistence.EntityManagerFactory; -//import org.springframework.beans.factory.annotation.Qualifier; -//import org.springframework.beans.factory.annotation.Value; -//import org.springframework.boot.jdbc.DataSourceBuilder; -//import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder; -//import org.springframework.context.annotation.Bean; -//import org.springframework.context.annotation.Configuration; -//import org.springframework.context.annotation.Primary; -//import org.springframework.data.jpa.repository.config.EnableJpaRepositories; -//import org.springframework.orm.jpa.JpaTransactionManager; -//import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; -//import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; -//import org.springframework.transaction.PlatformTransactionManager; -//import org.springframework.transaction.annotation.EnableTransactionManagement; -// -//import javax.sql.DataSource; -//import java.util.HashMap; -// -// -//@Configuration -//@EnableTransactionManagement -//@EnableJpaRepositories( -// entityManagerFactoryRef = "odseEntityManagerFactory", -// transactionManagerRef = "odseTransactionManager", -// basePackages = { -// "gov.cdc.nnddatapollservice.repository.odse" -// } -//) -//public class OdseDataSourceConfig { -// @Value("${spring.datasource.driverClassName}") -// private String driverClassName; -// -// @Value("${spring.datasource.odse.url}") -// private String dbUrl; -// -// @Value("${spring.datasource.username}") -// private String dbUserName; -// -// @Value("${spring.datasource.password}") -// private String dbUserPassword; -// -// @Bean(name = "odseDataSource") -// public DataSource odseDataSource() { -// DataSourceBuilder dataSourceBuilder = DataSourceBuilder.create(); -// -// dataSourceBuilder.driverClassName(driverClassName); -// dataSourceBuilder.url(dbUrl); -// dataSourceBuilder.username(dbUserName); -// dataSourceBuilder.password(dbUserPassword); -// -// return dataSourceBuilder.build(); -// } -// -// @Bean(name = "odseEntityManagerFactoryBuilder") -// public EntityManagerFactoryBuilder odseEntityManagerFactoryBuilder() { -// return new EntityManagerFactoryBuilder(new HibernateJpaVendorAdapter(), new HashMap<>(), null); -// } -// -// @Bean(name = "odseEntityManagerFactory") -// public LocalContainerEntityManagerFactoryBean odseEntityManagerFactory( -// EntityManagerFactoryBuilder odseEntityManagerFactoryBuilder, -// @Qualifier("odseDataSource") DataSource odseDataSource ) { -// return odseEntityManagerFactoryBuilder -// .dataSource(odseDataSource) -// .packages("gov.cdc.nnddatapollservice.repository.odse.model") -// .persistenceUnit("odse") -// .build(); -// } -// -// @Primary -// @Bean(name = "odseTransactionManager") -// public PlatformTransactionManager odseTransactionManager( -// @Qualifier("odseEntityManagerFactory") EntityManagerFactory odseEntityManagerFactory ) { -// return new JpaTransactionManager(odseEntityManagerFactory); -// } -//} diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/RdbDataSourceConfig.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/RdbDataSourceConfig.java index 59831fbe..40b2ab10 100644 --- a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/RdbDataSourceConfig.java +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/RdbDataSourceConfig.java @@ -7,6 +7,7 @@ import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Lazy; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.orm.jpa.JpaTransactionManager; @@ -40,6 +41,7 @@ public class RdbDataSourceConfig { private String dbUserPassword; @Bean(name = "rdbDataSource") + @Lazy public DataSource rdbDataSource() { DataSourceBuilder dataSourceBuilder = DataSourceBuilder.create(); diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/RdbModernDataSourceConfig.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/RdbModernDataSourceConfig.java deleted file mode 100644 index df26f42d..00000000 --- a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/RdbModernDataSourceConfig.java +++ /dev/null @@ -1,82 +0,0 @@ -//package gov.cdc.nnddatapollservice.configuration; -// -//import jakarta.persistence.EntityManagerFactory; -//import org.springframework.beans.factory.annotation.Qualifier; -//import org.springframework.beans.factory.annotation.Value; -//import org.springframework.boot.jdbc.DataSourceBuilder; -//import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder; -//import org.springframework.context.annotation.Bean; -//import org.springframework.context.annotation.Configuration; -//import org.springframework.data.jpa.repository.config.EnableJpaRepositories; -//import org.springframework.jdbc.core.JdbcTemplate; -//import org.springframework.orm.jpa.JpaTransactionManager; -//import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; -//import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; -//import org.springframework.transaction.PlatformTransactionManager; -// -//import javax.sql.DataSource; -//import java.util.HashMap; -// -//@Configuration -//@EnableJpaRepositories( -// entityManagerFactoryRef = "rdbmodernEntityManagerFactory", -// transactionManagerRef = "rdbmodernTransactionManager", -// basePackages = { -// "gov.cdc.nnddatapollservice.repository.rdb_modern", -// } -//) -//public class RdbModernDataSourceConfig { -// -// @Value("${spring.datasource.driverClassName}") -// private String driverClassName; -// -// @Value("${spring.datasource.rdb_modern.url}") -// private String dbUrl; -// -// @Value("${spring.datasource.username}") -// private String dbUserName; -// -// @Value("${spring.datasource.password}") -// private String dbUserPassword; -// -// // DataSource Bean -// @Bean(name = "rdbmodernDataSource") -// public DataSource rdbmodernDataSource() { -// return DataSourceBuilder.create() -// .driverClassName(driverClassName) -// .url(dbUrl) -// .username(dbUserName) -// .password(dbUserPassword) -// .build(); -// } -// -// // JdbcTemplate Bean -// @Bean(name = "rdbmodernJdbcTemplate") -// public JdbcTemplate rdbmodernJdbcTemplate(@Qualifier("rdbmodernDataSource") DataSource dataSource) { -// return new JdbcTemplate(dataSource); -// } -// -// // JPA Configurations -// -// @Bean(name = "rdbModernEntityManagerFactoryBuilder") -// public EntityManagerFactoryBuilder rdbModernEntityManagerFactoryBuilder() { -// return new EntityManagerFactoryBuilder(new HibernateJpaVendorAdapter(), new HashMap<>(), null); -// } -// -// @Bean(name = "rdbmodernEntityManagerFactory") -// public LocalContainerEntityManagerFactoryBean rdbModernEntityManagerFactory( -// @Qualifier("rdbModernEntityManagerFactoryBuilder") EntityManagerFactoryBuilder builder, -// @Qualifier("rdbmodernDataSource") DataSource dataSource) { -// return builder -// .dataSource(dataSource) -// .packages("gov.cdc.nnddatapollservice.repository.rdb_modern") // Adjust package for your entities -// .persistenceUnit("rdb_modern") -// .build(); -// } -// -// @Bean(name = "rdbmodernTransactionManager") -// public PlatformTransactionManager rdbmodernTransactionManager( -// @Qualifier("rdbmodernEntityManagerFactory") EntityManagerFactory entityManagerFactory) { -// return new JpaTransactionManager(entityManagerFactory); -// } -//} diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/SrteDataSourceConfig.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/SrteDataSourceConfig.java deleted file mode 100644 index ce07e33c..00000000 --- a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/SrteDataSourceConfig.java +++ /dev/null @@ -1,41 +0,0 @@ -//package gov.cdc.nnddatapollservice.configuration; -// -//import org.springframework.beans.factory.annotation.Qualifier; -//import org.springframework.beans.factory.annotation.Value; -//import org.springframework.boot.jdbc.DataSourceBuilder; -//import org.springframework.context.annotation.Bean; -//import org.springframework.context.annotation.Configuration; -//import org.springframework.jdbc.core.JdbcTemplate; -// -//import javax.sql.DataSource; -// -//@Configuration -//public class SrteDataSourceConfig { -// @Value("${spring.datasource.driverClassName}") -// private String driverClassName; -// -// @Value("${spring.datasource.srte.url}") -// private String dbUrl; -// -// @Value("${spring.datasource.username}") -// private String dbUserName; -// -// @Value("${spring.datasource.password}") -// private String dbUserPassword; -// -// @Bean(name = "srteDataSource") -// public DataSource srteDataSource() { -// DataSourceBuilder dataSourceBuilder = DataSourceBuilder.create(); -// -// dataSourceBuilder.driverClassName(driverClassName); -// dataSourceBuilder.url(dbUrl); -// dataSourceBuilder.username(dbUserName); -// dataSourceBuilder.password(dbUserPassword); -// -// return dataSourceBuilder.build(); -// } -// @Bean(name = "srteJdbcTemplate") -// public JdbcTemplate srteJdbcTemplate(@Qualifier("srteDataSource") DataSource dataSource) { -// return new JdbcTemplate(dataSource); -// } -//} \ No newline at end of file diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/TimestampAdapter.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/TimestampAdapter.java new file mode 100644 index 00000000..95a3813e --- /dev/null +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/configuration/TimestampAdapter.java @@ -0,0 +1,32 @@ +package gov.cdc.nnddatapollservice.configuration; + +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializer; + +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +@SuppressWarnings("java:S1118") +public class TimestampAdapter { + + @SuppressWarnings("java:S2885") + private static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + + // Serializer: Convert Timestamp to JSON + public static JsonSerializer getTimestampSerializer() { + return (src, typeOfSrc, context) -> new JsonPrimitive(dateFormat.format(src)); + } + + // Deserializer: Convert JSON to Timestamp + public static JsonDeserializer getTimestampDeserializer() { + return (json, typeOfT, context) -> { + try { + return new Timestamp(dateFormat.parse(json.getAsJsonPrimitive().getAsString()).getTime()); + } catch (Exception e) { + throw new JsonParseException(e); + } + }; + } +} \ No newline at end of file diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/constant/ConstantValue.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/constant/ConstantValue.java index b1c2d26f..f34759b5 100644 --- a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/constant/ConstantValue.java +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/constant/ConstantValue.java @@ -3,6 +3,7 @@ public class ConstantValue { public static final String LOG_SUCCESS = "SUCCESS"; public static final String RDB_MODERN = "RDB_MODERN"; + public static final String NBS_ODSE = "NBS_ODSE"; public static final String RDB = "RDB"; public static final String SRTE = "SRTE"; public static final String SQL_LOG = "SQL Log: "; diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/nbs_odse/dao/NbsOdseDataPersistentDAO.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/nbs_odse/dao/NbsOdseDataPersistentDAO.java new file mode 100644 index 00000000..a1f1b3ac --- /dev/null +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/nbs_odse/dao/NbsOdseDataPersistentDAO.java @@ -0,0 +1,114 @@ +package gov.cdc.nnddatapollservice.nbs_odse.dao; + +import com.google.gson.FieldNamingPolicy; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; +import gov.cdc.nnddatapollservice.configuration.TimestampAdapter; +import gov.cdc.nnddatapollservice.nbs_odse.dto.EDXActivityDetailLogDto; +import gov.cdc.nnddatapollservice.nbs_odse.dto.EDXActivityLogDto; +import gov.cdc.nnddatapollservice.rdbmodern.dto.NrtObservationCodedDto; +import gov.cdc.nnddatapollservice.rdbmodern.dto.NrtObservationDto; +import gov.cdc.nnddatapollservice.repository.nbs_odse.EDXActivityDetailLogRepository; +import gov.cdc.nnddatapollservice.repository.nbs_odse.EDXActivityLogRepository; +import gov.cdc.nnddatapollservice.repository.nbs_odse.model.EDXActivityDetailLog; +import gov.cdc.nnddatapollservice.repository.nbs_odse.model.EDXActivityLog; +import gov.cdc.nnddatapollservice.share.HandleError; +import jakarta.transaction.Transactional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Service; + +import java.lang.reflect.Type; +import java.sql.Timestamp; +import java.util.List; +import java.util.UUID; + +import static gov.cdc.nnddatapollservice.constant.ConstantValue.LOG_SUCCESS; + +@Service +public class NbsOdseDataPersistentDAO { + private static Logger logger = LoggerFactory.getLogger(NbsOdseDataPersistentDAO.class); + + private JdbcTemplate jdbcTemplate; + private final HandleError handleError; + + private final EDXActivityLogRepository edxActivityLogRepository; + private final EDXActivityDetailLogRepository edxActivityDetailLogRepository; + + @Value("${datasync.sql_error_handle_log}") + protected String sqlErrorPath = ""; + + @Value("${datasync.data_sync_batch_limit}") + protected Integer batchSize = 1000; + private final Gson gsonNorm = new Gson(); + private final Gson gson = new GsonBuilder() + .registerTypeAdapter(Timestamp.class, TimestampAdapter.getTimestampSerializer()) + .registerTypeAdapter(Timestamp.class, TimestampAdapter.getTimestampDeserializer()) + .setFieldNamingPolicy(FieldNamingPolicy.UPPER_CASE_WITH_UNDERSCORES) + .create(); + + public NbsOdseDataPersistentDAO(@Qualifier("nbsOdseJdbcTemplate") JdbcTemplate jdbcTemplate, HandleError handleError, EDXActivityLogRepository edxActivityLogRepository, EDXActivityDetailLogRepository edxActivityDetailLogRepository) { + this.jdbcTemplate = jdbcTemplate; + this.handleError = handleError; + this.edxActivityLogRepository = edxActivityLogRepository; + this.edxActivityDetailLogRepository = edxActivityDetailLogRepository; + } + + public String saveNbsOdseData(String tableName, String jsonData) { + logger.info("saveNbsOdseData tableName: {}", tableName); + StringBuilder logBuilder = new StringBuilder(LOG_SUCCESS); + if ("EDX_ACTIVITY_LOG".equalsIgnoreCase(tableName)) { + logBuilder = new StringBuilder(LOG_SUCCESS); + Type resultType = new TypeToken>() { + }.getType(); + List list = gson.fromJson(jsonData, resultType); + persistingEdxActivity(list, tableName); + } + else if ("EDX_ACTIVITY_DETAIL_LOG".equalsIgnoreCase(tableName)) { + logBuilder = new StringBuilder(LOG_SUCCESS); + Type resultType = new TypeToken>() { + }.getType(); + List list = gson.fromJson(jsonData, resultType); + persistingEdxActivityDetail(list, tableName); + } + + return logBuilder.toString(); + + } + + + + protected void persistingEdxActivity(List list, String tableName) { + for (EDXActivityLogDto data : list) { + try { + jdbcTemplate.execute("SET IDENTITY_INSERT " + tableName + " ON"); + var domainModel = new EDXActivityLog(data); + edxActivityLogRepository.save(domainModel); + } catch (Exception e) { + logger.error("ERROR occured at record: {}, {}", gsonNorm.toJson(data), e.getMessage()); // NOSONAR + handleError.writeRecordToFileTypedObject(gsonNorm, data, tableName + UUID.randomUUID(), sqlErrorPath + "/RDB_MODERN/" + e.getClass().getSimpleName() + "/" + tableName + "/"); // NOSONAR + } + } + jdbcTemplate.execute("SET IDENTITY_INSERT " + tableName + " OFF"); + + } + + protected void persistingEdxActivityDetail(List list, String tableName) { + for (EDXActivityDetailLogDto data : list) { + try { + jdbcTemplate.execute("SET IDENTITY_INSERT " + tableName + " ON"); + var domainModel = new EDXActivityDetailLog(data); + edxActivityDetailLogRepository.save(domainModel); + } catch (Exception e) { + logger.error("ERROR occured at record: {}, {}", gsonNorm.toJson(data), e.getMessage()); // NOSONAR + handleError.writeRecordToFileTypedObject(gsonNorm, data, tableName + UUID.randomUUID(), sqlErrorPath + "/RDB_MODERN/" + e.getClass().getSimpleName() + "/" + tableName + "/"); // NOSONAR + } + jdbcTemplate.execute("SET IDENTITY_INSERT " + tableName + " OFF"); + } + } + +} diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/nbs_odse/dto/EDXActivityDetailLogDto.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/nbs_odse/dto/EDXActivityDetailLogDto.java new file mode 100644 index 00000000..6627eb51 --- /dev/null +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/nbs_odse/dto/EDXActivityDetailLogDto.java @@ -0,0 +1,31 @@ +package gov.cdc.nnddatapollservice.nbs_odse.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.gson.annotations.SerializedName; +import lombok.*; + +@Getter +@Setter +public class EDXActivityDetailLogDto { + + @SerializedName("edx_activity_detail_log_uid") + private Long edxActivityDetailLogUid; + + @SerializedName("edx_activity_log_uid") + private Long edxActivityLogUid; // Reference to the foreign key (edx_activity_log_uid) + + @SerializedName("record_id") + private String recordId; + + @SerializedName("record_type") + private String recordType; + + @SerializedName("record_name") + private String recordName; + + @SerializedName("log_type") + private String logType; + + @SerializedName("log_comment") + private String logComment; +} \ No newline at end of file diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/nbs_odse/dto/EDXActivityLogDto.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/nbs_odse/dto/EDXActivityLogDto.java new file mode 100644 index 00000000..af62cafa --- /dev/null +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/nbs_odse/dto/EDXActivityLogDto.java @@ -0,0 +1,66 @@ +package gov.cdc.nnddatapollservice.nbs_odse.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.gson.annotations.SerializedName; +import lombok.*; + +import java.sql.Timestamp; +import java.time.LocalDateTime; + +@Setter +@Getter +public class EDXActivityLogDto { + @SerializedName("edx_activity_log_uid") + private Long edxActivityLogUid; + + @SerializedName("source_uid") + private Long sourceUid; + + @SerializedName("target_uid") + private Long targetUid; + + @SerializedName("doc_type") + private String docType; + + @SerializedName("record_status_cd") + private String recordStatusCd; + + @SerializedName("record_status_time") + private Timestamp recordStatusTime; + + @SerializedName("exception_txt") + private String exceptionTxt; + + @SerializedName("imp_exp_ind_cd") + private String impExpIndCd; + + @SerializedName("source_type_cd") + private String sourceTypeCd; + + @SerializedName("target_type_cd") + private String targetTypeCd; + + @SerializedName("business_obj_local_id") + private String businessObjLocalId; + + @SerializedName("doc_nm") + private String docNm; + + @SerializedName("source_nm") + private String sourceNm; + + @SerializedName("algorithm_action") + private String algorithmAction; + + @SerializedName("algorithm_name") + private String algorithmName; + + @SerializedName("message_id") + private String messageId; + + @SerializedName("entity_nm") + private String entityNm; + + @SerializedName("accession_nbr") + private String accessionNbr; +} diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/nbs_odse/service/NbsOdseDataHandlingService.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/nbs_odse/service/NbsOdseDataHandlingService.java new file mode 100644 index 00000000..1b124b73 --- /dev/null +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/nbs_odse/service/NbsOdseDataHandlingService.java @@ -0,0 +1,196 @@ +package gov.cdc.nnddatapollservice.nbs_odse.service; + +import com.google.gson.Gson; +import gov.cdc.nnddatapollservice.exception.DataPollException; +import gov.cdc.nnddatapollservice.nbs_odse.dao.NbsOdseDataPersistentDAO; +import gov.cdc.nnddatapollservice.nbs_odse.service.interfaces.INbsOdseDataHandlingService; +import gov.cdc.nnddatapollservice.rdb.dto.PollDataSyncConfig; +import gov.cdc.nnddatapollservice.service.interfaces.IPollCommonService; +import gov.cdc.nnddatapollservice.service.interfaces.IS3DataService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.List; + +import static gov.cdc.nnddatapollservice.constant.ConstantValue.*; + +@Service +public class NbsOdseDataHandlingService implements INbsOdseDataHandlingService { + private static Logger logger = LoggerFactory.getLogger(NbsOdseDataHandlingService.class); + private final IPollCommonService pollCommonService; + private final NbsOdseDataPersistentDAO nbsOdseDataPersistentDAO; + + private final IS3DataService is3DataService; + @Value("${datasync.store_in_local}") + protected boolean storeJsonInLocalFolder = false; + @Value("${datasync.store_in_S3}") + protected boolean storeJsonInS3 = false; + @Value("${datasync.store_in_sql}") + protected boolean storeInSql = false; + + @Value("${datasync.data_sync_batch_limit}") + protected Integer pullLimit = 0; + + private final Gson gson = new Gson(); + + public NbsOdseDataHandlingService(IPollCommonService pollCommonService, NbsOdseDataPersistentDAO nbsOdseDataPersistentDAO, IS3DataService is3DataService) { + this.pollCommonService = pollCommonService; + this.nbsOdseDataPersistentDAO = nbsOdseDataPersistentDAO; + this.is3DataService = is3DataService; + } + + + public void handlingExchangedData() throws DataPollException { + logger.info("---START RDB_MODERN POLLING---"); + List configTableList = pollCommonService.getTableListFromConfig(); + List rdbModernTablesList = pollCommonService.getTablesConfigListBySOurceDB(configTableList, NBS_ODSE); + logger.info(" RDB_MODERN TableList to be polled: {}", rdbModernTablesList.size()); + + boolean isInitialLoad = pollCommonService.checkPollingIsInitailLoad(rdbModernTablesList); + logger.info("-----INITIAL LOAD: {}", isInitialLoad); + + + for (PollDataSyncConfig pollDataSyncConfig : rdbModernTablesList) { + logger.info("Start polling: Table:{} order:{}", pollDataSyncConfig.getTableName(), pollDataSyncConfig.getTableOrder()); + pollAndPersistRDBMOdernData(pollDataSyncConfig.getTableName(), isInitialLoad); + } + + logger.info("---END RDB_MODERN POLLING---"); + } + + protected void pollAndPersistRDBMOdernData(String tableName, boolean isInitialLoad) { + try { + logger.info("--START--pollAndPersistRDBData for table {}", tableName); + String log = ""; + boolean exceptionAtApiLevel = false; + Integer totalRecordCounts = 0; + String timeStampForPoll = getPollTimestamp(isInitialLoad, tableName); + logger.info("isInitialLoad {}", isInitialLoad); + + logger.info("------lastUpdatedTime to send to exchange api {}", timeStampForPoll); + var timestampWithNull = Timestamp.from(Instant.now()); + //call data exchange service api + try { + totalRecordCounts = pollCommonService.callDataCountEndpoint(tableName, isInitialLoad, timeStampForPoll); + } catch (Exception e) { + pollCommonService.updateLastUpdatedTimeAndLogLocalDir(tableName, timestampWithNull, CRITICAL_COUNT_LOG + log); + throw new DataPollException("TASK FAILED: " + e.getMessage()); + } + + int batchSize = pullLimit; + int totalPages = (int) Math.ceil((double) totalRecordCounts / batchSize); + + try { + var encodedDataWithNull = pollCommonService.callDataExchangeEndpoint(tableName, isInitialLoad, timeStampForPoll, true, "0", "0"); + var rawJsonDataWithNull = pollCommonService.decodeAndDecompress(encodedDataWithNull); + if (storeJsonInS3) { + log = is3DataService.persistToS3MultiPart(NBS_ODSE, rawJsonDataWithNull, tableName, timestampWithNull, isInitialLoad); + pollCommonService.updateLastUpdatedTimeAndLogS3(tableName, timestampWithNull, S3_LOG + log); + } + else if (storeInSql) { + log = nbsOdseDataPersistentDAO.saveNbsOdseData(tableName, rawJsonDataWithNull); + pollCommonService.updateLastUpdatedTimeAndLog(tableName, timestampWithNull, SQL_LOG + log); + } + else { + log = pollCommonService.writeJsonDataToFile(NBS_ODSE, tableName, timestampWithNull, rawJsonDataWithNull); + pollCommonService.updateLastUpdatedTimeAndLogLocalDir(tableName, timestampWithNull, LOCAL_DIR_LOG + log); + } + } catch (Exception e) { + pollCommonService.updateLastUpdatedTimeAndLogLocalDir(tableName, timestampWithNull, CRITICAL_NULL_LOG + e.getMessage()); + throw new DataPollException("TASK FAILED: " + e.getMessage()); + } + + + for (int i = 0; i < totalPages; i++) { + String rawJsonData = ""; + Timestamp timestamp = null; + try { + int startRow = i * batchSize + 1; + int endRow = (i + 1) * batchSize; + + String encodedData = ""; + if (i == 0) { + // First batch pull record will null time stamp + encodedData = pollCommonService.callDataExchangeEndpoint(tableName, isInitialLoad, timeStampForPoll, true, String.valueOf(startRow), String.valueOf(endRow)); + } else { + encodedData = pollCommonService.callDataExchangeEndpoint(tableName, isInitialLoad, timeStampForPoll, false, String.valueOf(startRow), String.valueOf(endRow)); + } + + rawJsonData = pollCommonService.decodeAndDecompress(encodedData); + timestamp = Timestamp.from(Instant.now()); + } catch (Exception e) { + log = e.getMessage(); + exceptionAtApiLevel = true; + } + + updateDataHelper(exceptionAtApiLevel, tableName, timestamp, + rawJsonData, isInitialLoad, log); + + } + } catch (Exception e) { + logger.error("TASK failed. tableName: {}, message: {}", tableName, e.getMessage()); + } + } + + protected String getPollTimestamp(boolean isInitialLoad, String tableName) { + String timeStampForPoll; + if (isInitialLoad) { + timeStampForPoll = pollCommonService.getCurrentTimestamp(); + } else { + if(storeJsonInS3) { + timeStampForPoll = pollCommonService.getLastUpdatedTimeS3(tableName); + } + else if (storeInSql) + { + timeStampForPoll = pollCommonService.getLastUpdatedTime(tableName); + } + else + { + timeStampForPoll = pollCommonService.getLastUpdatedTimeLocalDir(tableName); + } + } + return timeStampForPoll; + } + + protected void updateDataHelper(boolean exceptionAtApiLevel, String tableName, Timestamp timestamp, + String rawJsonData, boolean isInitialLoad, String log) { + try { + if (exceptionAtApiLevel) { + if (storeInSql) { + pollCommonService.updateLastUpdatedTimeAndLog(tableName, timestamp, API_LEVEL + log); + } + else if (storeJsonInS3) + { + pollCommonService.updateLastUpdatedTimeAndLogS3(tableName, timestamp, API_LEVEL + log); + } + else + { + pollCommonService.updateLastUpdatedTimeAndLogLocalDir(tableName, timestamp, API_LEVEL + log); + } + } else { + if (storeJsonInS3) { + log = is3DataService.persistToS3MultiPart(RDB, rawJsonData, tableName, timestamp, isInitialLoad); + pollCommonService.updateLastUpdatedTimeAndLogS3(tableName, timestamp, S3_LOG + log); + } + else if (storeInSql) + { + log = nbsOdseDataPersistentDAO.saveNbsOdseData(tableName, rawJsonData); + pollCommonService.updateLastUpdatedTimeAndLog(tableName, timestamp, SQL_LOG + log); + } + else + { + log = pollCommonService.writeJsonDataToFile(RDB, tableName, timestamp, rawJsonData); + pollCommonService.updateLastUpdatedTimeAndLogLocalDir(tableName, timestamp, LOCAL_DIR_LOG + log); + } + } + } catch (Exception e) { + pollCommonService.updateLastUpdatedTimeAndLogLocalDir(tableName, timestamp, CRITICAL_NON_NULL_LOG + e.getMessage()); + } + } + + +} diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/nbs_odse/service/interfaces/INbsOdseDataHandlingService.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/nbs_odse/service/interfaces/INbsOdseDataHandlingService.java new file mode 100644 index 00000000..1fe4fe77 --- /dev/null +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/nbs_odse/service/interfaces/INbsOdseDataHandlingService.java @@ -0,0 +1,7 @@ +package gov.cdc.nnddatapollservice.nbs_odse.service.interfaces; + +import gov.cdc.nnddatapollservice.exception.DataPollException; + +public interface INbsOdseDataHandlingService { + void handlingExchangedData() throws DataPollException; +} diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/repository/nbs_odse/EDXActivityDetailLogRepository.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/repository/nbs_odse/EDXActivityDetailLogRepository.java new file mode 100644 index 00000000..366199b6 --- /dev/null +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/repository/nbs_odse/EDXActivityDetailLogRepository.java @@ -0,0 +1,9 @@ +package gov.cdc.nnddatapollservice.repository.nbs_odse; + +import gov.cdc.nnddatapollservice.repository.nbs_odse.model.EDXActivityDetailLog; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface EDXActivityDetailLogRepository extends JpaRepository { +} diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/repository/nbs_odse/EDXActivityLogRepository.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/repository/nbs_odse/EDXActivityLogRepository.java new file mode 100644 index 00000000..7abe6891 --- /dev/null +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/repository/nbs_odse/EDXActivityLogRepository.java @@ -0,0 +1,9 @@ +package gov.cdc.nnddatapollservice.repository.nbs_odse; + +import gov.cdc.nnddatapollservice.repository.nbs_odse.model.EDXActivityLog; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface EDXActivityLogRepository extends JpaRepository { +} diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/repository/nbs_odse/model/EDXActivityDetailLog.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/repository/nbs_odse/model/EDXActivityDetailLog.java new file mode 100644 index 00000000..56a65ab7 --- /dev/null +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/repository/nbs_odse/model/EDXActivityDetailLog.java @@ -0,0 +1,50 @@ +package gov.cdc.nnddatapollservice.repository.nbs_odse.model; + +import gov.cdc.nnddatapollservice.nbs_odse.dto.EDXActivityDetailLogDto; +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Entity +@Table(name = "EDX_activity_detail_log") +public class EDXActivityDetailLog { + + @Id + @Column(name = "edx_activity_detail_log_uid", nullable = false) + private Long edxActivityDetailLogUid; + + @Column(name = "edx_activity_log_uid", nullable = false) + private Long edxActivityLogUid; + + @Column(name = "record_id", length = 256) + private String recordId; + + @Column(name = "record_type", length = 50) + private String recordType; + + @Column(name = "record_nm", length = 250) + private String recordName; + + @Column(name = "log_type", length = 50) + private String logType; + + @Column(name = "log_comment", length = 2000) + private String logComment; + + public EDXActivityDetailLog(EDXActivityDetailLogDto dto) { + this.edxActivityDetailLogUid = dto.getEdxActivityDetailLogUid(); + this.edxActivityLogUid = dto.getEdxActivityLogUid(); + this.recordId = dto.getRecordId(); + this.recordType = dto.getRecordType(); + this.recordName = dto.getRecordName(); + this.logType = dto.getLogType(); + this.logComment = dto.getLogComment(); + } +} diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/repository/nbs_odse/model/EDXActivityLog.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/repository/nbs_odse/model/EDXActivityLog.java new file mode 100644 index 00000000..8dde5ec3 --- /dev/null +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/repository/nbs_odse/model/EDXActivityLog.java @@ -0,0 +1,94 @@ +package gov.cdc.nnddatapollservice.repository.nbs_odse.model; + +import gov.cdc.nnddatapollservice.nbs_odse.dto.EDXActivityLogDto; +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.sql.Timestamp; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Entity +@Table(name = "EDX_activity_log") +public class EDXActivityLog { + + @Id + @Column(name = "edx_activity_log_uid", nullable = false) + private Long edxActivityLogUid; + + @Column(name = "source_uid") + private Long sourceUid; + + @Column(name = "target_uid") + private Long targetUid; + + @Column(name = "doc_type", length = 50) + private String docType; + + @Column(name = "record_status_cd", length = 20) + private String recordStatusCd; + + @Column(name = "record_status_time") + private Timestamp recordStatusTime; + + @Lob + @Column(name = "exception_txt") + private String exceptionTxt; + + @Column(name = "imp_exp_ind_cd", length = 1) + private String impExpIndCd; + + @Column(name = "source_type_cd", length = 50) + private String sourceTypeCd; + + @Column(name = "target_type_cd", length = 50) + private String targetTypeCd; + + @Column(name = "business_obj_localId", length = 50) + private String businessObjLocalId; + + @Column(name = "doc_nm", length = 250) + private String docNm; + + @Column(name = "source_nm", length = 250) + private String sourceNm; + + @Column(name = "algorithm_action", length = 10) + private String algorithmAction; + + @Column(name = "algorithm_name", length = 250) + private String algorithmName; + + @Column(name = "Message_id", length = 255) + private String messageId; + + @Column(name = "Entity_nm", length = 255) + private String entityNm; + + @Column(name = "Accession_nbr", length = 100) + private String accessionNbr; + + public EDXActivityLog(EDXActivityLogDto dto) { + this.edxActivityLogUid = dto.getEdxActivityLogUid(); + this.sourceUid = dto.getSourceUid(); + this.targetUid = dto.getTargetUid(); + this.docType = dto.getDocType(); + this.recordStatusCd = dto.getRecordStatusCd(); + this.recordStatusTime = dto.getRecordStatusTime(); + this.exceptionTxt = dto.getExceptionTxt(); + this.impExpIndCd = dto.getImpExpIndCd(); + this.sourceTypeCd = dto.getSourceTypeCd(); + this.targetTypeCd = dto.getTargetTypeCd(); + this.businessObjLocalId = dto.getBusinessObjLocalId(); + this.docNm = dto.getDocNm(); + this.sourceNm = dto.getSourceNm(); + this.algorithmAction = dto.getAlgorithmAction(); + this.algorithmName = dto.getAlgorithmName(); + this.messageId = dto.getMessageId(); + this.entityNm = dto.getEntityNm(); + this.accessionNbr = dto.getAccessionNbr(); + } +} diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/DataPullService.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/DataPullService.java index ae2dca08..1d0dc907 100644 --- a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/DataPullService.java +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/DataPullService.java @@ -4,6 +4,7 @@ import gov.cdc.nnddatapollservice.rdb.service.interfaces.IRdbDataHandlingService; import gov.cdc.nnddatapollservice.rdbmodern.service.interfaces.IRdbModernDataHandlingService; import gov.cdc.nnddatapollservice.service.interfaces.IDataPullService; +import gov.cdc.nnddatapollservice.nbs_odse.service.interfaces.INbsOdseDataHandlingService; import gov.cdc.nnddatapollservice.service.interfaces.INNDDataHandlingService; import gov.cdc.nnddatapollservice.srte.service.interfaces.ISrteDataHandlingService; import lombok.extern.slf4j.Slf4j; @@ -33,21 +34,26 @@ public class DataPullService implements IDataPullService { @Value("${poll.srte.enabled}") private boolean srtePollEnabled; + @Value("${poll.edx_activity.enabled}") + private boolean edxActivityEnabled; + @Value("${poll.single_time_poll_enabled}") private boolean singlePoll; @Value("${datasync.sql_reprocessing_data}") private boolean reprocessFailedSQL = false; + private final INbsOdseDataHandlingService edxActivitySyncService; private final INNDDataHandlingService dataHandlingService; private final IRdbDataHandlingService rdbDataHandlingService; private final IRdbModernDataHandlingService rdbModernDataHandlingService; private final ISrteDataHandlingService srteDataHandlingService; - public DataPullService(INNDDataHandlingService dataHandlingService, + public DataPullService(INbsOdseDataHandlingService edxActivitySyncService, INNDDataHandlingService dataHandlingService, IRdbDataHandlingService rdbDataHandlingService, IRdbModernDataHandlingService rdbModernDataHandlingService, ISrteDataHandlingService srteDataHandlingService) { + this.edxActivitySyncService = edxActivitySyncService; this.dataHandlingService = dataHandlingService; this.rdbDataHandlingService = rdbDataHandlingService; this.rdbModernDataHandlingService = rdbModernDataHandlingService; @@ -56,14 +62,25 @@ public DataPullService(INNDDataHandlingService dataHandlingService, @Scheduled(cron = "${scheduler.cron}", zone = "${scheduler.zone}") public void scheduleNNDDataFetch() throws DataPollException { - if (nndPollEnabled) { - logger.info("CRON STARTED FOR NND"); - logger.info(cron); - logger.info(zone); - dataHandlingService.handlingExchangedData(); - closePoller(); + if (nndPollEnabled) { + logger.info("CRON STARTED FOR NND"); + logger.info(cron); + logger.info(zone); + dataHandlingService.handlingExchangedData(); + closePoller(); + } + } + + @Scheduled(cron = "${scheduler.cron_edx_activity}", zone = "${scheduler.zone}") + public void scheduleEdxActivityDataFetch() throws DataPollException { + if (edxActivityEnabled) { + logger.info("CRON STARTED FOR EDX ACTIVITY"); + logger.info(cron); + logger.info(zone); + edxActivitySyncService.handlingExchangedData(); + closePoller(); + } } -} @Scheduled(cron = "${scheduler.cron_rdb}", zone = "${scheduler.zone}") public void scheduleRDBDataFetch() throws DataPollException { diff --git a/nnd-data-poll-service/src/main/resources/application.yaml b/nnd-data-poll-service/src/main/resources/application.yaml index 014e302f..a83bce14 100644 --- a/nnd-data-poll-service/src/main/resources/application.yaml +++ b/nnd-data-poll-service/src/main/resources/application.yaml @@ -27,6 +27,8 @@ poll: enabled: ${RDB_MODERN_POLL_ENABLED:false} srte: enabled: ${SRTE_POLL_ENABLED:false} + edx_activity: + enabled: ${EDX_ACTIVITY_ENABLED:false} datasync: store_in_sql: ${DATASYNC_STORE_SQL:false} @@ -65,6 +67,7 @@ scheduler: cron_rdb: ${NBS_RDB_CRON:0 */1 * * * * } cron_srte: ${NBS_SRTE_CRON:0 */1 * * * * } cron_rdb_modern: ${NBS_RDB_MODERN_CRON:0 */1 * * * * } + cron_edx_activity: ${NBS_EDX_ACTIVITY_CRON:0 */1 * * * * } # "0 */5 0-2 * * *" # every 5 min from 12 am to 3 am # 0 * 0-2 * * * @@ -84,13 +87,15 @@ spring: username: ${OP_DBUSER} password: ${OP_DBPASSWORD} ingest: - url: jdbc:sqlserver://${OP_DBSERVER};databaseName=${OP_DBNAME};encrypt=true;trustServerCertificate=true; + url: jdbc:sqlserver://${OP_DBSERVER:};databaseName=${OP_DBNAME:};encrypt=true;trustServerCertificate=true; rdb: - url: jdbc:sqlserver://${OP_DBSERVER};databaseName=${RDB_DBNAME};encrypt=true;trustServerCertificate=true; + url: jdbc:sqlserver://${OP_DBSERVER:};databaseName=${RDB_DBNAME:};encrypt=true;trustServerCertificate=true; rdb_modern: - url: jdbc:sqlserver://${OP_DBSERVER};databaseName=${RDB_MODERN_DBNAME};encrypt=true;trustServerCertificate=true; + url: jdbc:sqlserver://${OP_DBSERVER:};databaseName=${RDB_MODERN_DBNAME:};encrypt=true;trustServerCertificate=true; srte: - url: jdbc:sqlserver://${OP_DBSERVER};databaseName=${SRTE_DBNAME};encrypt=true;trustServerCertificate=true; + url: jdbc:sqlserver://${OP_DBSERVER:};databaseName=${SRTE_DBNAME:};encrypt=true;trustServerCertificate=true; + odse: + url: jdbc:sqlserver://${OP_DBSERVER:};databaseName=${ODSE_DBNAME:};encrypt=true;trustServerCertificate=true; jpa: show-sql: true hibernate: diff --git a/nnd-data-poll-service/src/main/resources/sql/rdb/poll_config_insert.sql b/nnd-data-poll-service/src/main/resources/sql/rdb/poll_config_insert.sql index 1d8f7fbb..2e75124b 100644 --- a/nnd-data-poll-service/src/main/resources/sql/rdb/poll_config_insert.sql +++ b/nnd-data-poll-service/src/main/resources/sql/rdb/poll_config_insert.sql @@ -165,3 +165,9 @@ VALUES('NRT_OBSERVATION', 'RDB_MODERN',1); INSERT INTO RDB.dbo.poll_data_sync_config (table_name, source_db,table_order) VALUES('NRT_OBSERVATION_CODED', 'RDB_MODERN',2); +INSERT INTO RDB.dbo.poll_data_sync_config +(table_name, source_db,table_order) +VALUES('EDX_ACTIVITY_LOG', 'NBS_ODSE',1); +INSERT INTO RDB.dbo.poll_data_sync_config +(table_name, source_db,table_order) +VALUES('EDX_ACTIVITY_DETAIL_LOG', 'NBS_ODSE',2);