feat(restoreIndices): update restore indices available parameters (#549)

* feat(restoreIndices): update restore indices available parameters * update chart
acryldata · Feb 3, 2025 · ca0c8a1 · ca0c8a1
1 parent 06bb272
commit ca0c8a1
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 2 deletions.
diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml
@@ -4,7 +4,7 @@ description: A Helm chart for DataHub
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.5.4
+version: 0.5.5
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: 0.15.0

diff --git a/charts/datahub/templates/datahub-upgrade/datahub-nocode-migration-job.yml b/charts/datahub/templates/datahub-upgrade/datahub-nocode-migration-job.yml
@@ -1,4 +1,4 @@
-{{- if .Values.datahubUpgrade.enabled -}}
+{{- if .Values.datahubUpgrade.noCodeDataMigration.enabled -}}
 apiVersion: batch/v1
 kind: Job
 metadata:

diff --git a/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml b/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml
@@ -75,6 +75,42 @@ spec:
                 - "batchSize={{ .Values.datahubUpgrade.batchSize }}"
                 - "-a"
                 - "batchDelayMs={{ .Values.datahubUpgrade.batchDelayMs }}"
+                {{- if .Values.datahubUpgrade.restoreIndices.args.lePitEpochMs }}
+                - "-a"
+                - "lePitEpochMs={{ .Values.datahubUpgrade.restoreIndices.args.lePitEpochMs }}"
+                {{- end}}
+                {{- if .Values.datahubUpgrade.restoreIndices.args.gePitEpochMs }}
+                - "-a"
+                - "gePitEpochMs={{ .Values.datahubUpgrade.restoreIndices.args.gePitEpochMs }}"
+                {{- end}}
+                {{- if .Values.datahubUpgrade.restoreIndices.args.lastUrn }}
+                - "-a"
+                - "lastUrn={{ .Values.datahubUpgrade.restoreIndices.args.lastUrn }}"
+                {{- end}}
+                {{- if .Values.datahubUpgrade.restoreIndices.args.lastAspect}}
+                - "-a"
+                - "lastAspect={{ .Values.datahubUpgrade.restoreIndices.args.lastAspect }}"
+                {{- end}}
+                {{- if .Values.datahubUpgrade.restoreIndices.args.urnBasedPagination}}
+                - "-a"
+                - "urnBasedPagination={{ .Values.datahubUpgrade.restoreIndices.args.urnBasedPagination }}"
+                {{- end}}
+                {{- if .Values.datahubUpgrade.restoreIndices.args.aspectNames}}
+                - "-a"
+                - "aspectNames={{ .Values.datahubUpgrade.restoreIndices.args.aspectNames }}"
+                {{- end}}
+                {{- if .Values.datahubUpgrade.restoreIndices.args.urnLike}}
+                - "-a"
+                - "urnLike={{ .Values.datahubUpgrade.restoreIndices.args.urnLike }}"
+                {{- end}}
+                {{- if .Values.datahubUpgrade.restoreIndices.args.startingOffset}}
+                - "-a"
+                - "startingOffset={{ .Values.datahubUpgrade.restoreIndices.args.startingOffset }}"
+                {{- end}}
+                {{- if .Values.datahubUpgrade.restoreIndices.args.numThreads}}
+                - "-a"
+                - "numThreads={{ .Values.datahubUpgrade.restoreIndices.args.numThreads }}"
+                {{- end}}
               {{- end }}
               env:
                 {{- include "datahub.upgrade.env" . | nindent 16 }}

diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml
@@ -283,6 +283,7 @@ datahubUpgrade:
   batchSize: 1000
   batchDelayMs: 100
   noCodeDataMigration:
+    enabled: false
     image:
       # Add custom command / arguments to this job.  Useful if you need a custom startup or shutdown script
       # to run
@@ -331,7 +332,28 @@ datahubUpgrade:
       # Add custom command / arguments to this job.  Useful if you need a custom startup or shutdown script
       # to run
       command:
+      # Overrides the entire set of upgrade job arguments, for if you want to specify exactly whi
       args: []
+    # Alternative for overriding specific args instead of changing the entire set and overriding defaults
+    args:
+      # Restore only rows with less than a certain epoch millisecond timestamp
+      lePitEpochMs:
+      # Restore only rows with greater than a certain epoch millisecond timestamp
+      gePitEpochMs:
+      # Used in conjunction with urnBasedPagination, resume from a particular urn
+      lastUrn:
+      # Used in conjunction with urnBasedPagination and lastUrn, resume from a particular urn for a particular aspect to avoid reprocessing certain aspects
+      lastAspect:
+      # Uses a key based paging strategy when scrolling through SQL rows instead of offset. Often faster for large sets of data
+      urnBasedPagination:
+      # Only restore for certain aspects, comma separated list
+      aspectNames:
+      # A SQL "like" statement that matches against the urn column
+      urnLike:
+      # Used with default paging, start from a specified offset
+      startingOffset:
+      # Specifies number of threads for processing pages of rows to restore
+      numThreads:
     resources:
       limits:
         cpu: 500m