Reuse Esti components

treeverse · Jan 2, 2025 · 3ac0e8b · 3ac0e8b
1 parent 54977e5
commit 3ac0e8b
Show file tree

Hide file tree

Showing 11 changed files with 36 additions and 31 deletions.
diff --git a/.github/workflows/compatibility-tests.yaml b/.github/workflows/compatibility-tests.yaml
@@ -174,7 +174,7 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       TAG: ${{ matrix.lakefs_version }}
-      REPO: treeverse
+      REGISTRY: treeverse
       SPARK_TAG: 3
     steps:
       - name: Check-out code
@@ -207,7 +207,7 @@ jobs:
         with:
           compose-directory: test/spark
         env:
-          REPO: treeverse
+          REGISTRY: treeverse
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           LAKEFS_DATABASE_TYPE: postgres
@@ -252,7 +252,7 @@ jobs:
       CLIENT_VERSION: ${{ matrix.client_version }}
       TAG: ${{ needs.deploy-image.outputs.tag }}
       IMAGE_ID: ${{ needs.deploy-image.outputs.image_id }}
-      REPO: treeverse
+      REGISTRY: treeverse
       SPARK_TAG: 3
     steps:
       - name: Check-out code
@@ -297,7 +297,7 @@ jobs:
         with:
           compose-directory: test/spark
         env:
-          REPO: treeverse
+          REGISTRY: treeverse
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           LAKEFS_DATABASE_TYPE: postgres

diff --git a/.github/workflows/esti.yaml b/.github/workflows/esti.yaml
@@ -281,7 +281,7 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       TAG: ${{ needs.deploy-image.outputs.tag }}
-      REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }}
+      REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }}
     steps:
       - name: Check-out code
         uses: actions/checkout@v4
@@ -328,7 +328,7 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       TAG: ${{ needs.deploy-image.outputs.tag }}
-      REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }}
+      REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }}
 
     steps:
       - name: Check-out code
@@ -438,7 +438,7 @@ jobs:
             sonnet_jar: sonnets-311/target/sonnets-311/scala-2.12/sonnets-311_2.12-0.1.0.jar
     env:
       TAG: ${{ needs.deploy-image.outputs.tag }}
-      REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }}
+      REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }}
       SPARK_TAG: ${{ matrix.spark.tag }}
     steps:
       - name: Check-out code
@@ -547,7 +547,7 @@ jobs:
     env:
       LAKEFS_TAG: ${{ needs.deploy-image.outputs.tag }}
       EXPORT_TAG: ${{ needs.deploy-rclone-export-image.outputs.tag }}
-      REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }}
+      REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }}
 
     steps:
       - name: Check-out code
@@ -596,7 +596,7 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       TAG: ${{ needs.deploy-image.outputs.tag }}
-      REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }}
+      REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }}
     steps:
       - name: Check-out code
         uses: actions/checkout@v4
@@ -654,7 +654,7 @@ jobs:
     needs: [check-secrets, deploy-image, login-to-amazon-ecr]
     env:
       TAG: ${{ needs.deploy-image.outputs.tag }}
-      REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }}
+      REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }}
     steps:
       - name: Check-out code
         uses: actions/checkout@v4
@@ -694,7 +694,7 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       SPARK_TAG: 3.2.1
-      REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }}
+      REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }}
       TAG: ${{ needs.deploy-image.outputs.tag }}
     steps:
       - name: Check-out code
@@ -753,7 +753,7 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       TAG: ${{ needs.deploy-image.outputs.tag }}
-      REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }}
+      REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }}
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       LAKEFS_DATABASE_CONNECTION_STRING: "" # Override lakeFS docker compose settings
@@ -830,7 +830,7 @@ jobs:
         branch_ownership: [false, true]
     env:
       TAG: ${{ needs.deploy-image.outputs.tag }}
-      REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }}
+      REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }}
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
     steps:
@@ -904,7 +904,7 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       TAG: ${{ needs.deploy-image.outputs.tag }}
-      REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }}
+      REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }}
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
     steps:
@@ -958,7 +958,7 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       TAG: ${{ needs.deploy-image.outputs.tag }}
-      REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }}
+      REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }}
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
@@ -1019,7 +1019,7 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       TAG: ${{ needs.deploy-image.outputs.tag }}
-      REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }}
+      REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }}
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
@@ -1098,7 +1098,7 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       TAG: ${{ needs.deploy-image.outputs.tag }}
-      REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }}
+      REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }}
       LAKEFS_INSTALLATION_ACCESS_KEY_ID: AKIAIOSFDNN7EXAMPLEQ
       LAKEFS_INSTALLATION_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
 
@@ -1398,7 +1398,7 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       TAG: ${{ needs.deploy-image.outputs.tag }}
-      REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }}
+      REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }}
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
     steps:

diff --git a/esti/lakefs_util.go b/esti/lakefs_util.go
@@ -23,7 +23,7 @@ func LakefsWithParamsWithBasicAuth(connectionString string, basicAuth bool) stri
 }
 
 func lakefsLocation() string {
-	return viper.GetString("binaries_dir") + "/lakefs"
+	return viper.GetString("binaries_dir") + "/" + viper.GetString("lakefs_binary_name")
 }
 
 func LakefsWithBasicAuth() string {

diff --git a/esti/ops/docker-compose-dynamodb.yaml b/esti/ops/docker-compose-dynamodb.yaml
@@ -1,7 +1,7 @@
 version: "3"
 services:
   lakefs:
-    image: "${REPO:-treeverse}/lakefs:${TAG:-dev}"
+    image: "${REGISTRY:-treeverse}/${REPO:-lakefs}:${TAG:-dev}"
     command: "${COMMAND:-run}"
     ports:
       - "8000:8000"
@@ -29,7 +29,6 @@ services:
       - LAKEFS_DATABASE_DYNAMODB_AWS_ACCESS_KEY_ID=AKIAIO5FODNN7EXAMPLE
       - LAKEFS_DATABASE_DYNAMODB_AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K3MDENG/bPxRfiCYEXAMPLEKEY
       - LAKEFS_USAGE_REPORT_ENABLED=false
-    entrypoint: ["/app/lakefs", "run"]
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
@@ -59,18 +58,21 @@ services:
       - ESTI_AWS_SECRET_ACCESS_KEY
       - ESTI_ENDPOINT_URL=http://lakefs:8000
       - ESTI_BINARIES_DIR=/app
+      - ESTI_LAKEFS_BINARY_NAME=${ESTI_LAKEFS_BINARY_NAME:-lakefs}
       - ESTI_DATABASE_KV_ENABLED
       - ESTI_KV_MIGRATION=${ESTI_KV_MIGRATION:-none}
       - ESTI_POST_MIGRATE
       - ESTI_GOTEST_FLAGS
       - ESTI_FLAGS
+      - ESTI_SKIP_TESTS=${ESTI_SKIP_TESTS:-TestUnifiedGC}
+      - LAKEFS_ROOT
     working_dir: /lakefs
     command:
       - /bin/sh
       - -c
       - |
         apk add --no-cache util-linux
-        go test -v $ESTI_GOTEST_FLAGS ./esti --system-tests $ESTI_FLAGS --skip=".*GC"
+        go test -v $ESTI_GOTEST_FLAGS ./esti --system-tests $ESTI_FLAGS --skip $SKIP_TESTS
     volumes:
       - lakefs-code:/lakefs
       - lakefs-app:/app:ro

diff --git a/esti/ops/docker-compose-external-db.yaml b/esti/ops/docker-compose-external-db.yaml
@@ -1,7 +1,7 @@
 version: "3"
 services:
   lakefs:
-    image: "${REPO:-treeverse}/lakefs:${TAG:-dev}"
+    image: "${REGISTRY:-treeverse}/${REPO:-lakefs}:${TAG:-dev}"
     command: "${COMMAND:-run}"
     ports:
       - "8000:8000"
@@ -29,7 +29,6 @@ services:
       - AZURE_CLIENT_SECRET
       - AZURE_TENANT_ID
       - LAKEFS_USAGE_REPORT_ENABLED=false
-    entrypoint: ["/app/lakefs", "run"]
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
@@ -54,8 +53,10 @@ services:
       - ESTI_AWS_SECRET_ACCESS_KEY
       - ESTI_ENDPOINT_URL=http://lakefs:8000
       - ESTI_BINARIES_DIR=/app
+      - ESTI_LAKEFS_BINARY_NAME=${ESTI_LAKEFS_BINARY_NAME:-lakefs}
       - ESTI_GOTEST_FLAGS
       - ESTI_FLAGS
+      - ESTI_SKIP_TESTS=${ESTI_SKIP_TESTS:-TestUnifiedGC}
       - ESTI_LARGE_OBJECT_PATH
       - ESTI_FORCE_PATH_STYLE
       - ESTI_AZURE_STORAGE_ACCOUNT
@@ -66,7 +67,7 @@ services:
       - -c
       - |
         apk add --no-cache util-linux
-        go test -timeout 20m -v $ESTI_GOTEST_FLAGS ./esti --system-tests $ESTI_FLAGS --skip=".*GC"
+        go test -timeout 20m -v $ESTI_GOTEST_FLAGS ./esti --system-tests $ESTI_FLAGS --skip $SKIP_TESTS
     volumes:
       - lakefs-code:/lakefs
       - lakefs-app:/app:ro

diff --git a/esti/ops/docker-compose.yaml b/esti/ops/docker-compose.yaml
@@ -1,7 +1,7 @@
 version: "3"
 services:
   lakefs:
-    image: "${REPO:-treeverse}/lakefs:${TAG:-dev}"
+    image: "${REGISTRY:-treeverse}/${REPO:-lakefs}:${TAG:-dev}"
     command: "${COMMAND:-run}"
     ports:
       - "8000:8000"
@@ -62,6 +62,7 @@ services:
       - ESTI_AWS_SECRET_ACCESS_KEY
       - ESTI_ENDPOINT_URL=http://lakefs:8000
       - ESTI_BINARIES_DIR=/app
+      - ESTI_LAKEFS_BINARY_NAME=${ESTI_LAKEFS_BINARY_NAME:-lakefs}
       - ESTI_DATABASE_CONNECTION_STRING=postgres://lakefs:lakefs@postgres/postgres?sslmode=disable
       - ESTI_GOTEST_FLAGS
       - ESTI_FLAGS

diff --git a/pkg/testutil/setup.go b/pkg/testutil/setup.go
@@ -57,6 +57,7 @@ func SetupTestingEnv(params *SetupTestingEnvParams) (logging.Logger, apigen.Clie
 	viper.SetDefault("glue_export_hooks_database", "export-hooks-esti")
 	viper.SetDefault("glue_export_region", "us-east-1")
 	viper.SetDefault("binaries_dir", filepath.Join(currDir, ".."))
+	viper.SetDefault("lakefs_binary_name", filepath.Join(currDir, "lakefs"))
 	viper.SetDefault("azure_storage_account", "")
 	viper.SetDefault("azure_storage_access_key", "")
 	viper.SetDefault("large_object_path", "")

diff --git a/test/lakectl_metastore/docker-compose.yml b/test/lakectl_metastore/docker-compose.yml
@@ -1,7 +1,7 @@
 version: "3.7"
 services:
   lakefs:
-    image: "${REPO:-treeverse}/lakefs:${TAG:-latest}"
+    image: "${REGISTRY:-treeverse}/${REPO:-lakefs}:${TAG:-latest}"
     container_name: lakefs
     ports:
       - "8000:8000"

diff --git a/test/lakefsfs_contract/docker-compose.yaml b/test/lakefsfs_contract/docker-compose.yaml
@@ -28,7 +28,7 @@ services:
       timeout: 20s
       retries: 3
   lakefs:
-    image: "${REPO}/lakefs:${TAG}"
+    image: "${REGISTRY:-treeverse}/${REPO:-lakefs}:${TAG}"
     ports:
       - "8000:8000"
     networks:

diff --git a/test/rclone_export/docker-compose.yaml b/test/rclone_export/docker-compose.yaml
@@ -8,7 +8,7 @@ services:
       POSTGRES_PASSWORD: lakefs
 
   lakefs:
-    image: "${REPO:-treeverse}/lakefs:${LAKEFS_TAG:-latest}"
+    image: "${REGISTRY:-treeverse}/lakefs:${LAKEFS_TAG:-latest}"
     container_name: lakefs
     ports:
       - "8000:8000"
@@ -32,7 +32,7 @@ services:
     entrypoint: ["/app/wait-for", "postgres:5432", "--", "/app/lakefs", "run"]
 
   lakefs-export:
-    image: "${REPO:-treeverse}/lakefs-rclone-export:${EXPORT_TAG:-latest}"
+    image: "${REGISTRY:-treeverse}/lakefs-rclone-export:${EXPORT_TAG:-latest}"
     profiles: ["client"]
     environment:
       - LAKEFS_ENDPOINT=http://lakefs:8000

diff --git a/test/spark/docker-compose.yaml b/test/spark/docker-compose.yaml
@@ -1,6 +1,6 @@
 x-lakefs-common:
   &lakefs-common
-  image: "${REPO:-treeverse}/lakefs:${TAG:-latest}"
+  image: "${REGISTRY:-treeverse}/lakefs:${TAG:-latest}"
   environment:
     - LAKEFS_GATEWAYS_S3_DOMAIN_NAME=s3.docker.lakefs.io,s3.local.lakefs.io
     - LAKEFS_AUTH_ENCRYPT_SECRET_KEY=some random secret string