Skip to content

Commit

Permalink
Add ability to run small tests with appropriate batching
Browse files Browse the repository at this point in the history
  • Loading branch information
@dconneely
dconneely committed Jan 13, 2025
1 parent c311de0 commit 27eae5e
Show file tree
Hide file tree
Showing 4 changed files with 18 additions and 10 deletions.
7 changes: 5 additions & 2 deletions premigconcor/src/main/java/uk/gov/justice/laadces/premigconcor/Runner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ class Runner implements ApplicationRunner {
private static final String PATH_CSV_OUTPUT_MISSING_CONCOR = "./premigconcor-concorMissing.csv";
private static final String PATH_CSV_OUTPUT_FOUND_FDC = "./premigconcor-fdcCases.csv";
private static final String PATH_CSV_OUTPUT_MISSING_FDC = "./premigconcor-fdcMissing.csv";
private static final int MAX_COUNT_MAAT_IDS = 5000; // truncate data set if non-negative.

private final MigrationScopeRepository migrationScopeRepository;
private final ConcorContributionRepository concorContributionRepository;
Expand All @@ -40,8 +41,10 @@ class Runner implements ApplicationRunner {
@Override
public void run(final ApplicationArguments args) throws Exception {
log.info("Entering run...");
final var maatIds = migrationScopeRepository.findAll();
log.info("Found {} maatIds from migration database", maatIds.size());
final var allMaatIds = migrationScopeRepository.findAll();
log.info("Found {} maatIds from migration database", allMaatIds.size());
final var maatIds = MAX_COUNT_MAAT_IDS < 0 ? allMaatIds : allMaatIds.subList(0, Math.min(MAX_COUNT_MAAT_IDS, allMaatIds.size()));
log.info("Truncating to {} maatIds to be processed", maatIds.size());

final var foundConcors = new HashSet<CaseMigration>();
final var missingConcors = new TreeSet<Long>();
Expand Down
10 changes: 6 additions & 4 deletions .../main/java/uk/gov/justice/laadces/premigconcor/dao/maat/ConcorContributionRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@

@Repository
public class ConcorContributionRepository {
private static final int BATCH_SIZE = 990;
private static final int MAATDB_BATCH_QUERY_SIZE = 990; // concor_contributions query batch size
private static final long INTDB_BATCH_ID_DIVISOR = 250L; // case_migration batch_id size
private final JdbcClient maat;

public ConcorContributionRepository(@Qualifier("maatJdbcClient") final JdbcClient maat) {
Expand All @@ -21,8 +22,9 @@ public ConcorContributionRepository(@Qualifier("maatJdbcClient") final JdbcClien

public void addLatestIdsByMaatIds(final List<Long> maatIds, final Collection<CaseMigration> foundConcors, final Collection<Long> missingConcors) {
final int count = maatIds.size();
for (int i = 0; i < count; i += BATCH_SIZE) {
final var subList = maatIds.subList(i, Math.min(count, i + BATCH_SIZE));
final long[] batchIndex = {0L}; // Needs to be "effectively final".
for (int i = 0; i < count; i += MAATDB_BATCH_QUERY_SIZE) {
final var subList = maatIds.subList(i, Math.min(count, i + MAATDB_BATCH_QUERY_SIZE));
final var paramSource = new MapSqlParameterSource("maatIds", subList);
final var set = new TreeSet<>(subList);
foundConcors.addAll(maat.sql("""
Expand All @@ -36,7 +38,7 @@ WHERE rep_id IN (:maatIds) AND status = 'SENT'
final long concorContributionId = rs.getLong(1);
final long maatId = rs.getLong(2);
set.remove(maatId);
return CaseMigration.ofConcorContribution(maatId, concorContributionId, (long) rowNum);
return CaseMigration.ofConcorContribution(maatId, concorContributionId, (batchIndex[0]++) / INTDB_BATCH_ID_DIVISOR);
})
.list());
if (!set.isEmpty()) {
Expand Down
10 changes: 6 additions & 4 deletions ...src/main/java/uk/gov/justice/laadces/premigconcor/dao/maat/FdcContributionRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@

@Repository
public class FdcContributionRepository {
private static final int BATCH_SIZE = 990;
private static final int MAATDB_BATCH_QUERY_SIZE = 990; // fdc_contributions query batch size
private static final long INTDB_BATCH_ID_DIVISOR = 500L; // case_migration batch_id size
private final JdbcClient maat;

public FdcContributionRepository(@Qualifier("maatJdbcClient") final JdbcClient maat) {
Expand All @@ -21,8 +22,9 @@ public FdcContributionRepository(@Qualifier("maatJdbcClient") final JdbcClient m

public void addLatestIdsByMaatIds(final List<Long> maatIds, final Collection<CaseMigration> foundFdcs, final Collection<Long> missingFdcs) {
final int count = maatIds.size();
for (int i = 0; i < count; i += BATCH_SIZE) {
final var subList = maatIds.subList(i, Math.min(count, i + BATCH_SIZE));
final long[] batchIndex = {0L}; // Needs to be "effectively final".
for (int i = 0; i < count; i += MAATDB_BATCH_QUERY_SIZE) {
final var subList = maatIds.subList(i, Math.min(count, i + MAATDB_BATCH_QUERY_SIZE));
final var paramSource = new MapSqlParameterSource("maatIds", subList);
final var set = new TreeSet<>(subList);
foundFdcs.addAll(maat.sql("""
Expand All @@ -36,7 +38,7 @@ WHERE rep_id IN (:maatIds)
final long fdcId = rs.getLong(1);
final long maatId = rs.getLong(2);
set.remove(maatId);
return CaseMigration.ofFdcContribution(maatId, fdcId, (long) rowNum);
return CaseMigration.ofFdcContribution(maatId, fdcId, (batchIndex[0]++) / INTDB_BATCH_ID_DIVISOR);
})
.list());
if (!set.isEmpty()) {
Expand Down
1 change: 1 addition & 0 deletions ...main/java/uk/gov/justice/laadces/premigconcor/dao/migration/MigrationScopeRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ public List<Long> findAll() {
SELECT DISTINCT CAST(clientcasereference AS INTEGER) AS maat_id
FROM transform.laacasedetails
WHERE clientcasereference ~ '^[1-9][0-9]*$'
ORDER BY 1 DESC
""").query(Long.class).list();
}
}

0 comments on commit 27eae5e

Please sign in to comment.