From c7f722c607e6495e1333e5db0b8ec7766f125168 Mon Sep 17 00:00:00 2001 From: Nir Ozery Date: Tue, 31 Dec 2024 18:02:24 -0500 Subject: [PATCH] Reuse Esti components --- .github/workflows/compatibility-tests.yaml | 8 +++---- .github/workflows/esti.yaml | 28 +++++++++++----------- esti/lakectl_util.go | 3 ++- esti/lakefs_util.go | 2 +- esti/ops/docker-compose-dynamodb.yaml | 5 ++-- esti/ops/docker-compose-external-db.yaml | 4 ++-- esti/ops/docker-compose.yaml | 3 ++- pkg/testutil/setup.go | 2 ++ test/lakectl_metastore/docker-compose.yml | 2 +- test/lakefsfs_contract/docker-compose.yaml | 2 +- test/rclone_export/docker-compose.yaml | 4 ++-- test/spark/docker-compose.yaml | 2 +- 12 files changed, 35 insertions(+), 30 deletions(-) diff --git a/.github/workflows/compatibility-tests.yaml b/.github/workflows/compatibility-tests.yaml index 2e28d8f48d8..66a47e12931 100644 --- a/.github/workflows/compatibility-tests.yaml +++ b/.github/workflows/compatibility-tests.yaml @@ -174,7 +174,7 @@ jobs: runs-on: ubuntu-22.04 env: TAG: ${{ matrix.lakefs_version }} - REPO: treeverse + REGISTRY: treeverse SPARK_TAG: 3 steps: - name: Check-out code @@ -207,7 +207,7 @@ jobs: with: compose-directory: test/spark env: - REPO: treeverse + REGISTRY: treeverse AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} LAKEFS_DATABASE_TYPE: postgres @@ -252,7 +252,7 @@ jobs: CLIENT_VERSION: ${{ matrix.client_version }} TAG: ${{ needs.deploy-image.outputs.tag }} IMAGE_ID: ${{ needs.deploy-image.outputs.image_id }} - REPO: treeverse + REGISTRY: treeverse SPARK_TAG: 3 steps: - name: Check-out code @@ -297,7 +297,7 @@ jobs: with: compose-directory: test/spark env: - REPO: treeverse + REGISTRY: treeverse AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} LAKEFS_DATABASE_TYPE: postgres diff --git a/.github/workflows/esti.yaml b/.github/workflows/esti.yaml index 2cb43e8434c..010771b3ef2 100644 --- a/.github/workflows/esti.yaml +++ b/.github/workflows/esti.yaml @@ -281,7 +281,7 @@ jobs: runs-on: ubuntu-22.04 env: TAG: ${{ needs.deploy-image.outputs.tag }} - REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }} + REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }} steps: - name: Check-out code uses: actions/checkout@v4 @@ -328,7 +328,7 @@ jobs: runs-on: ubuntu-22.04 env: TAG: ${{ needs.deploy-image.outputs.tag }} - REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }} + REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }} steps: - name: Check-out code @@ -438,7 +438,7 @@ jobs: sonnet_jar: sonnets-311/target/sonnets-311/scala-2.12/sonnets-311_2.12-0.1.0.jar env: TAG: ${{ needs.deploy-image.outputs.tag }} - REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }} + REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }} SPARK_TAG: ${{ matrix.spark.tag }} steps: - name: Check-out code @@ -547,7 +547,7 @@ jobs: env: LAKEFS_TAG: ${{ needs.deploy-image.outputs.tag }} EXPORT_TAG: ${{ needs.deploy-rclone-export-image.outputs.tag }} - REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }} + REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }} steps: - name: Check-out code @@ -596,7 +596,7 @@ jobs: runs-on: ubuntu-22.04 env: TAG: ${{ needs.deploy-image.outputs.tag }} - REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }} + REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }} steps: - name: Check-out code uses: actions/checkout@v4 @@ -654,7 +654,7 @@ jobs: needs: [check-secrets, deploy-image, login-to-amazon-ecr] env: TAG: ${{ needs.deploy-image.outputs.tag }} - REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }} + REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }} steps: - name: Check-out code uses: actions/checkout@v4 @@ -694,7 +694,7 @@ jobs: runs-on: ubuntu-22.04 env: SPARK_TAG: 3.2.1 - REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }} + REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }} TAG: ${{ needs.deploy-image.outputs.tag }} steps: - name: Check-out code @@ -753,7 +753,7 @@ jobs: runs-on: ubuntu-22.04 env: TAG: ${{ needs.deploy-image.outputs.tag }} - REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }} + REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} LAKEFS_DATABASE_CONNECTION_STRING: "" # Override lakeFS docker compose settings @@ -830,7 +830,7 @@ jobs: branch_ownership: [false, true] env: TAG: ${{ needs.deploy-image.outputs.tag }} - REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }} + REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} steps: @@ -904,7 +904,7 @@ jobs: runs-on: ubuntu-22.04 env: TAG: ${{ needs.deploy-image.outputs.tag }} - REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }} + REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} steps: @@ -958,7 +958,7 @@ jobs: runs-on: ubuntu-22.04 env: TAG: ${{ needs.deploy-image.outputs.tag }} - REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }} + REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} @@ -1019,7 +1019,7 @@ jobs: runs-on: ubuntu-22.04 env: TAG: ${{ needs.deploy-image.outputs.tag }} - REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }} + REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} @@ -1098,7 +1098,7 @@ jobs: runs-on: ubuntu-22.04 env: TAG: ${{ needs.deploy-image.outputs.tag }} - REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }} + REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }} LAKEFS_INSTALLATION_ACCESS_KEY_ID: AKIAIOSFDNN7EXAMPLEQ LAKEFS_INSTALLATION_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY @@ -1398,7 +1398,7 @@ jobs: runs-on: ubuntu-22.04 env: TAG: ${{ needs.deploy-image.outputs.tag }} - REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }} + REGISTRY: ${{ needs.login-to-amazon-ecr.outputs.registry }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} steps: diff --git a/esti/lakectl_util.go b/esti/lakectl_util.go index ef2fa55273e..7294499f4de 100644 --- a/esti/lakectl_util.go +++ b/esti/lakectl_util.go @@ -183,7 +183,8 @@ func RunCmdAndVerifyFailureWithFile(t *testing.T, cmd string, isTerminal bool, g func runCmdAndVerifyWithFile(t *testing.T, cmd, goldenFile string, expectFail, isTerminal bool, vars map[string]string) { t.Helper() - goldenFile = "golden/" + goldenFile + ".golden" + goldenDir := viper.GetString("golden_dir") + goldenFile = goldenDir + goldenFile + ".golden" if *update { updateGoldenFile(t, cmd, isTerminal, goldenFile, vars) diff --git a/esti/lakefs_util.go b/esti/lakefs_util.go index 3be1c17838f..d4a7fb94fd4 100644 --- a/esti/lakefs_util.go +++ b/esti/lakefs_util.go @@ -23,7 +23,7 @@ func LakefsWithParamsWithBasicAuth(connectionString string, basicAuth bool) stri } func lakefsLocation() string { - return viper.GetString("binaries_dir") + "/lakefs" + return viper.GetString("binaries_dir") + "/" + viper.GetString("lakefs_binary_name") } func LakefsWithBasicAuth() string { diff --git a/esti/ops/docker-compose-dynamodb.yaml b/esti/ops/docker-compose-dynamodb.yaml index 7805c6a9bd9..4af364fed7b 100644 --- a/esti/ops/docker-compose-dynamodb.yaml +++ b/esti/ops/docker-compose-dynamodb.yaml @@ -1,7 +1,7 @@ version: "3" services: lakefs: - image: "${REPO:-treeverse}/lakefs:${TAG:-dev}" + image: "${REGISTRY:-treeverse}/${REPO:-lakefs}:${TAG:-dev}" command: "${COMMAND:-run}" ports: - "8000:8000" @@ -29,7 +29,6 @@ services: - LAKEFS_DATABASE_DYNAMODB_AWS_ACCESS_KEY_ID=AKIAIO5FODNN7EXAMPLE - LAKEFS_DATABASE_DYNAMODB_AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K3MDENG/bPxRfiCYEXAMPLEKEY - LAKEFS_USAGE_REPORT_ENABLED=false - entrypoint: ["/app/lakefs", "run"] extra_hosts: - "host.docker.internal:host-gateway" @@ -59,11 +58,13 @@ services: - ESTI_AWS_SECRET_ACCESS_KEY - ESTI_ENDPOINT_URL=http://lakefs:8000 - ESTI_BINARIES_DIR=/app + - ESTI_LAKEFS_BINARY_NAME=${ESTI_LAKEFS_BINARY_NAME:-lakefs} - ESTI_DATABASE_KV_ENABLED - ESTI_KV_MIGRATION=${ESTI_KV_MIGRATION:-none} - ESTI_POST_MIGRATE - ESTI_GOTEST_FLAGS - ESTI_FLAGS + - LAKEFS_ROOT working_dir: /lakefs command: - /bin/sh diff --git a/esti/ops/docker-compose-external-db.yaml b/esti/ops/docker-compose-external-db.yaml index 1ea73a657b7..09a444fa54a 100644 --- a/esti/ops/docker-compose-external-db.yaml +++ b/esti/ops/docker-compose-external-db.yaml @@ -1,7 +1,7 @@ version: "3" services: lakefs: - image: "${REPO:-treeverse}/lakefs:${TAG:-dev}" + image: "${REGISTRY:-treeverse}/${REPO:-lakefs}:${TAG:-dev}" command: "${COMMAND:-run}" ports: - "8000:8000" @@ -29,7 +29,6 @@ services: - AZURE_CLIENT_SECRET - AZURE_TENANT_ID - LAKEFS_USAGE_REPORT_ENABLED=false - entrypoint: ["/app/lakefs", "run"] extra_hosts: - "host.docker.internal:host-gateway" @@ -54,6 +53,7 @@ services: - ESTI_AWS_SECRET_ACCESS_KEY - ESTI_ENDPOINT_URL=http://lakefs:8000 - ESTI_BINARIES_DIR=/app + - ESTI_LAKEFS_BINARY_NAME=${ESTI_LAKEFS_BINARY_NAME:-lakefs} - ESTI_GOTEST_FLAGS - ESTI_FLAGS - ESTI_LARGE_OBJECT_PATH diff --git a/esti/ops/docker-compose.yaml b/esti/ops/docker-compose.yaml index cb640ec8004..3c8a0e80601 100644 --- a/esti/ops/docker-compose.yaml +++ b/esti/ops/docker-compose.yaml @@ -1,7 +1,7 @@ version: "3" services: lakefs: - image: "${REPO:-treeverse}/lakefs:${TAG:-dev}" + image: "${REGISTRY:-treeverse}/${REPO:-lakefs}:${TAG:-dev}" command: "${COMMAND:-run}" ports: - "8000:8000" @@ -62,6 +62,7 @@ services: - ESTI_AWS_SECRET_ACCESS_KEY - ESTI_ENDPOINT_URL=http://lakefs:8000 - ESTI_BINARIES_DIR=/app + - ESTI_LAKEFS_BINARY_NAME=${ESTI_LAKEFS_BINARY_NAME:-lakefs} - ESTI_DATABASE_CONNECTION_STRING=postgres://lakefs:lakefs@postgres/postgres?sslmode=disable - ESTI_GOTEST_FLAGS - ESTI_FLAGS diff --git a/pkg/testutil/setup.go b/pkg/testutil/setup.go index 2616589c5d4..8a79873f0c6 100644 --- a/pkg/testutil/setup.go +++ b/pkg/testutil/setup.go @@ -57,6 +57,8 @@ func SetupTestingEnv(params *SetupTestingEnvParams) (logging.Logger, apigen.Clie viper.SetDefault("glue_export_hooks_database", "export-hooks-esti") viper.SetDefault("glue_export_region", "us-east-1") viper.SetDefault("binaries_dir", filepath.Join(currDir, "..")) + viper.SetDefault("lakefs_binary_name", filepath.Join(currDir, "lakefs")) + viper.SetDefault("golden_dir", "golden/") viper.SetDefault("azure_storage_account", "") viper.SetDefault("azure_storage_access_key", "") viper.SetDefault("large_object_path", "") diff --git a/test/lakectl_metastore/docker-compose.yml b/test/lakectl_metastore/docker-compose.yml index 02db99b5cf1..6861bcc5074 100644 --- a/test/lakectl_metastore/docker-compose.yml +++ b/test/lakectl_metastore/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.7" services: lakefs: - image: "${REPO:-treeverse}/lakefs:${TAG:-latest}" + image: "${REGISTRY:-treeverse}/${REPO:-lakefs}:${TAG:-latest}" container_name: lakefs ports: - "8000:8000" diff --git a/test/lakefsfs_contract/docker-compose.yaml b/test/lakefsfs_contract/docker-compose.yaml index 935f9518bd0..d8d5b9555df 100644 --- a/test/lakefsfs_contract/docker-compose.yaml +++ b/test/lakefsfs_contract/docker-compose.yaml @@ -28,7 +28,7 @@ services: timeout: 20s retries: 3 lakefs: - image: "${REPO}/lakefs:${TAG}" + image: "${REGISTRY:-treeverse}/${REPO:-lakefs}:${TAG}" ports: - "8000:8000" networks: diff --git a/test/rclone_export/docker-compose.yaml b/test/rclone_export/docker-compose.yaml index 9861ac85c57..a8ce8f2344d 100644 --- a/test/rclone_export/docker-compose.yaml +++ b/test/rclone_export/docker-compose.yaml @@ -8,7 +8,7 @@ services: POSTGRES_PASSWORD: lakefs lakefs: - image: "${REPO:-treeverse}/lakefs:${LAKEFS_TAG:-latest}" + image: "${REGISTRY:-treeverse}/lakefs:${LAKEFS_TAG:-latest}" container_name: lakefs ports: - "8000:8000" @@ -32,7 +32,7 @@ services: entrypoint: ["/app/wait-for", "postgres:5432", "--", "/app/lakefs", "run"] lakefs-export: - image: "${REPO:-treeverse}/lakefs-rclone-export:${EXPORT_TAG:-latest}" + image: "${REGISTRY:-treeverse}/lakefs-rclone-export:${EXPORT_TAG:-latest}" profiles: ["client"] environment: - LAKEFS_ENDPOINT=http://lakefs:8000 diff --git a/test/spark/docker-compose.yaml b/test/spark/docker-compose.yaml index 788878b09cc..20742363e14 100644 --- a/test/spark/docker-compose.yaml +++ b/test/spark/docker-compose.yaml @@ -1,6 +1,6 @@ x-lakefs-common: &lakefs-common - image: "${REPO:-treeverse}/lakefs:${TAG:-latest}" + image: "${REGISTRY:-treeverse}/lakefs:${TAG:-latest}" environment: - LAKEFS_GATEWAYS_S3_DOMAIN_NAME=s3.docker.lakefs.io,s3.local.lakefs.io - LAKEFS_AUTH_ENCRYPT_SECRET_KEY=some random secret string