Cnde 1957 - data sync for edx activities (#55)

* added query for edx activity * added support for edx activity * update sql script * update poll service * data sync
CDCgov · Dec 13, 2024 · 1ea06c9 · 1ea06c9
1 parent 4cc2338
commit 1ea06c9
Show file tree

Hide file tree

Showing 25 changed files with 796 additions and 292 deletions.
diff --git a/.DS_Store b/.DS_Store
diff --git a/...vice/src/main/java/gov/cdc/nnddataexchangeservice/service/DataExchangeGenericService.java b/...vice/src/main/java/gov/cdc/nnddataexchangeservice/service/DataExchangeGenericService.java
@@ -29,16 +29,19 @@ public class DataExchangeGenericService implements IDataExchangeGenericService {
     private final JdbcTemplate jdbcTemplate;
     private final JdbcTemplate srteJdbcTemplate;
     private final JdbcTemplate rdbModernJdbcTemplate;
+    private final JdbcTemplate odseJdbcTemplate;
     private final Gson gson;
 
     public DataExchangeGenericService(DataSyncConfigRepository dataSyncConfigRepository,
                                       @Qualifier("rdbJdbcTemplate") JdbcTemplate jdbcTemplate,
                                       @Qualifier("srteJdbcTemplate")  JdbcTemplate srteJdbcTemplate,
-                                      @Qualifier("rdbModernJdbcTemplate")  JdbcTemplate rdbModernJdbcTemplate) {
+                                      @Qualifier("rdbModernJdbcTemplate")  JdbcTemplate rdbModernJdbcTemplate,
+                                      @Qualifier("odseJdbcTemplate") JdbcTemplate odseJdbcTemplate) {
         this.dataSyncConfigRepository = dataSyncConfigRepository;
         this.jdbcTemplate = jdbcTemplate;
         this.srteJdbcTemplate = srteJdbcTemplate;
         this.rdbModernJdbcTemplate = rdbModernJdbcTemplate;
+        this.odseJdbcTemplate = odseJdbcTemplate;
 
         this.gson = new GsonBuilder()
                 .registerTypeAdapter(Timestamp.class, TimestampAdapter.getTimestampSerializer())
@@ -74,7 +77,11 @@ private Integer executeQueryForTotalRecords(String query, String sourceDb) throw
                 return srteJdbcTemplate.queryForObject(query, Integer.class);
             } else if (sourceDb.equalsIgnoreCase(DB_RDB_MODERN)) {
                 return rdbModernJdbcTemplate.queryForObject(query, Integer.class);
-            } else {
+            }
+            else if (sourceDb.equalsIgnoreCase("NBS_ODSE")) {
+                return odseJdbcTemplate.queryForObject(query, Integer.class);
+            }
+            else {
                 throw new DataExchangeException("Database Not Supported: " + sourceDb);
             }
         } catch (Exception e) {
@@ -131,7 +138,11 @@ private List<Map<String, Object>> executeQueryForData(String query, String sourc
             return jdbcTemplate.queryForList(query);
         } else if (sourceDb.equalsIgnoreCase(DB_RDB_MODERN)) {
             return rdbModernJdbcTemplate.queryForList(query);
-        } else {
+        } else if (sourceDb.equalsIgnoreCase("NBS_ODSE")) {
+            return odseJdbcTemplate.queryForList(query);
+        }
+
+        else {
             throw new DataExchangeException("DB IS NOT SUPPORTED: " + sourceDb);
         }
     }

diff --git a/nnd-data-exchange-service/src/main/resources/query/create/generate_data_rdb.sql b/nnd-data-exchange-service/src/main/resources/query/create/generate_data_rdb.sql
@@ -503,3 +503,49 @@ VALUES
     WHERE RowNum BETWEEN :startRow AND :endRow;')
 ;
 
+
+INSERT INTO [RDB].[dbo].[data_sync_config]
+(table_name, source_db, query, query_with_null_timestamp, query_count, query_with_pagination)
+VALUES
+    ('EDX_ACTIVITY_LOG', 'NBS_ODSE',
+    'SELECT *
+     FROM EDX_ACTIVITY_LOG
+     WHERE record_status_time :operator :timestamp;',
+    NULL,
+    'SELECT COUNT(*)
+     FROM EDX_ACTIVITY_LOG
+     WHERE record_status_time :operator :timestamp;',
+    'WITH PaginatedResults AS (
+        SELECT *, ROW_NUMBER() OVER (ORDER BY record_status_time ASC) AS RowNum
+        FROM EDX_ACTIVITY_LOG
+        WHERE record_status_time :operator :timestamp
+    )
+    SELECT * FROM PaginatedResults
+    WHERE RowNum BETWEEN :startRow AND :endRow;')
+;
+
+INSERT INTO [RDB].[dbo].[data_sync_config]
+(table_name, source_db, query, query_with_null_timestamp, query_count, query_with_pagination)
+VALUES
+    ('EDX_ACTIVITY_DETAIL_LOG', 'NBS_ODSE',
+    'SELECT logg.record_status_time, detail.*
+     FROM EDX_ACTIVITY_DETAIL_LOG AS detail
+     INNER JOIN EDX_ACTIVITY_LOG AS logg
+     ON detail.edx_activity_log_uid = logg.edx_activity_log_uid
+     WHERE logg.record_status_time :operator :timestamp;',
+    NULL,
+    'SELECT COUNT(*)
+     FROM EDX_ACTIVITY_DETAIL_LOG AS detail
+     INNER JOIN EDX_ACTIVITY_LOG AS logg
+     ON detail.edx_activity_log_uid = logg.edx_activity_log_uid
+     WHERE logg.record_status_time :operator :timestamp;',
+    'WITH PaginatedResults AS (
+        SELECT logg.record_status_time, detail.*, ROW_NUMBER() OVER (ORDER BY logg.record_status_time ASC) AS RowNum
+        FROM EDX_ACTIVITY_DETAIL_LOG AS detail
+        INNER JOIN EDX_ACTIVITY_LOG AS logg
+        ON detail.edx_activity_log_uid = logg.edx_activity_log_uid
+        WHERE logg.record_status_time :operator :timestamp
+    )
+    SELECT * FROM PaginatedResults
+    WHERE RowNum BETWEEN :startRow AND :endRow;')
+;
diff --git a/nnd-data-poll-service/.DS_Store b/nnd-data-poll-service/.DS_Store
diff --git a/...ce/src/main/java/gov/cdc/nnddatapollservice/configuration/DataIngestDataSourceConfig.java b/...ce/src/main/java/gov/cdc/nnddatapollservice/configuration/DataIngestDataSourceConfig.java
@@ -8,6 +8,7 @@
 import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Lazy;
 import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
 import org.springframework.orm.jpa.JpaTransactionManager;
 import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
@@ -42,6 +43,7 @@ public class DataIngestDataSourceConfig {
     private String dbUserPassword;
 
     @Bean(name = "ingestDataSource")
+    @Lazy
     public DataSource ingestDataSource() {
         DataSourceBuilder dataSourceBuilder = DataSourceBuilder.create();
 

diff --git a/...l-service/src/main/java/gov/cdc/nnddatapollservice/configuration/MsgDataSourceConfig.java b/...l-service/src/main/java/gov/cdc/nnddatapollservice/configuration/MsgDataSourceConfig.java
diff --git a/...rvice/src/main/java/gov/cdc/nnddatapollservice/configuration/NbsOdseDataSourceConfig.java b/...rvice/src/main/java/gov/cdc/nnddatapollservice/configuration/NbsOdseDataSourceConfig.java
@@ -0,0 +1,83 @@
+package gov.cdc.nnddatapollservice.configuration;
+
+import jakarta.persistence.EntityManagerFactory;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.jdbc.DataSourceBuilder;
+import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.orm.jpa.JpaTransactionManager;
+import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
+import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter;
+import org.springframework.transaction.PlatformTransactionManager;
+
+import javax.sql.DataSource;
+import java.util.HashMap;
+
+@Configuration
+@EnableJpaRepositories(
+        entityManagerFactoryRef = "nbsOdseEntityManagerFactory",
+        transactionManagerRef = "nbsOdseTransactionManager",
+        basePackages = {
+                "gov.cdc.nnddatapollservice.repository.nbs_odse",
+        }
+)
+public class NbsOdseDataSourceConfig {
+    @Value("${spring.datasource.driverClassName}")
+    private String driverClassName;
+
+    @Value("${spring.datasource.odse.url}")
+    private String dbUrl;
+
+    @Value("${spring.datasource.username}")
+    private String dbUserName;
+
+    @Value("${spring.datasource.password}")
+    private String dbUserPassword;
+
+    @Bean(name = "nbsOdseDataSource")
+    @Lazy
+    public DataSource nbsOdseDataSource() {
+        DataSourceBuilder dataSourceBuilder = DataSourceBuilder.create();
+
+        dataSourceBuilder.driverClassName(driverClassName);
+        dataSourceBuilder.url(dbUrl);
+        dataSourceBuilder.username(dbUserName);
+        dataSourceBuilder.password(dbUserPassword);
+
+        return dataSourceBuilder.build();
+    }
+    @Bean(name = "nbsOdseJdbcTemplate")
+    public JdbcTemplate nbsOdseJdbcTemplate(@Qualifier("nbsOdseDataSource") DataSource dataSource) {
+        return new JdbcTemplate(dataSource);
+    }
+
+
+    // JPA Configurations
+
+    @Bean(name = "nbsOdseEntityManagerFactoryBuilder")
+    public EntityManagerFactoryBuilder nbsOdseEntityManagerFactoryBuilder() {
+        return new EntityManagerFactoryBuilder(new HibernateJpaVendorAdapter(), new HashMap<>(), null);
+    }
+
+    @Bean(name = "nbsOdseEntityManagerFactory")
+    public LocalContainerEntityManagerFactoryBean nbsOdseEntityManagerFactory(
+            @Qualifier("nbsOdseEntityManagerFactoryBuilder") EntityManagerFactoryBuilder builder,
+            @Qualifier("nbsOdseDataSource") DataSource dataSource) {
+        return builder
+                .dataSource(dataSource)
+                .packages("gov.cdc.nnddatapollservice.repository.nbs_odse") // Adjust package for your entities
+                .persistenceUnit("nbsodse")
+                .build();
+    }
+
+    @Bean(name = "nbsOdseTransactionManager")
+    public PlatformTransactionManager nbsOdseTransactionManager(
+            @Qualifier("nbsOdseEntityManagerFactory") EntityManagerFactory entityManagerFactory) {
+        return new JpaTransactionManager(entityManagerFactory);
+    }
+}
diff --git a/...-service/src/main/java/gov/cdc/nnddatapollservice/configuration/OdseDataSourceConfig.java b/...-service/src/main/java/gov/cdc/nnddatapollservice/configuration/OdseDataSourceConfig.java
diff --git a/...l-service/src/main/java/gov/cdc/nnddatapollservice/configuration/RdbDataSourceConfig.java b/...l-service/src/main/java/gov/cdc/nnddatapollservice/configuration/RdbDataSourceConfig.java
@@ -7,6 +7,7 @@
 import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Lazy;
 import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.orm.jpa.JpaTransactionManager;
@@ -40,6 +41,7 @@ public class RdbDataSourceConfig {
     private String dbUserPassword;
 
     @Bean(name = "rdbDataSource")
+    @Lazy
     public DataSource rdbDataSource() {
         DataSourceBuilder dataSourceBuilder = DataSourceBuilder.create();