From e7fb7b8f4d65d228f92c07ec79150a94ced954ca Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 23 Oct 2024 11:43:22 +0300 Subject: [PATCH 01/58] Revert "docs: clarify auditing availble on enterprise (#8289)" This reverts commit e12badc61311809390186938dc8d97847dce0a03. --- docs/reference/auditing.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/reference/auditing.md b/docs/reference/auditing.md index bcf7ad1a3f6..cfebd88fbfd 100644 --- a/docs/reference/auditing.md +++ b/docs/reference/auditing.md @@ -17,7 +17,10 @@ lakeFS Enterprise {: .label .label-purple } {: .note} -> Auditing is only available for [lakeFS Cloud]({% link cloud/index.md %}) and [lakeFS Enterprise]({% link enterprise/index.md %}). +> Auditing is only available for [lakeFS Cloud]({% link cloud/index.md %}). + +{: .warning } +> Please note, as of Jan 2024, the queryable interface within the lakeFS Cloud UI has been removed in favor of direct access to lakeFS audit logs. This document now describes how to set up and query this information using [AWS Glue](https://aws.amazon.com/glue/) as a reference. The lakeFS audit log allows you to view all relevant user action information in a clear and organized table, including when the action was performed, by whom, and what it was they did. From 33ca670059041fd6dc7b062f171a1a3a958a8a63 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Thu, 2 Jan 2025 14:55:47 +0100 Subject: [PATCH 02/58] got a decent error --- pkg/api/helpers/errors.go | 5 +++-- pkg/api/helpers/upload.go | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/api/helpers/errors.go b/pkg/api/helpers/errors.go index 7b6113770fd..2ebc63997f9 100644 --- a/pkg/api/helpers/errors.go +++ b/pkg/api/helpers/errors.go @@ -19,8 +19,9 @@ var ( ErrUnsupportedProtocol = errors.New("unsupported protocol") // ErrRequestFailed is an error returned for failing lakeFS server replies. - ErrRequestFailed = errors.New("request failed") - ErrConflict = errors.New("conflict") + ErrRequestFailed = errors.New("request failed") + ErrConflict = errors.New("conflict") + ErrProtectedBranch = errors.New("writing to protected branch") ) const minHTTPErrorStatusCode = 400 diff --git a/pkg/api/helpers/upload.go b/pkg/api/helpers/upload.go index 27e09027a6f..156adf01816 100644 --- a/pkg/api/helpers/upload.go +++ b/pkg/api/helpers/upload.go @@ -386,6 +386,9 @@ func (u *presignUpload) uploadObject(ctx context.Context) (*apigen.ObjectStats, if linkResp.JSON200 != nil { return linkResp.JSON200, nil } + if linkResp.JSON403 != nil { + return nil, fmt.Errorf("link object to backing store: %w (%s)", ErrProtectedBranch, linkResp.Status()) + } if linkResp.JSON409 != nil { return nil, ErrConflict } From 3c5adddd1a29ef89f4830b02d5cc27431667c715 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Sun, 5 Jan 2025 11:02:55 +0100 Subject: [PATCH 03/58] works --- pkg/api/helpers/upload.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/api/helpers/upload.go b/pkg/api/helpers/upload.go index 156adf01816..9ccbccd5d0e 100644 --- a/pkg/api/helpers/upload.go +++ b/pkg/api/helpers/upload.go @@ -20,6 +20,7 @@ import ( "github.com/treeverse/lakefs/pkg/api/apigen" "github.com/treeverse/lakefs/pkg/api/apiutil" "github.com/treeverse/lakefs/pkg/block/azure" + "github.com/treeverse/lakefs/pkg/graveler" "github.com/treeverse/lakefs/pkg/httputil" "golang.org/x/sync/errgroup" ) @@ -387,7 +388,7 @@ func (u *presignUpload) uploadObject(ctx context.Context) (*apigen.ObjectStats, return linkResp.JSON200, nil } if linkResp.JSON403 != nil { - return nil, fmt.Errorf("link object to backing store: %w (%s)", ErrProtectedBranch, linkResp.Status()) + return nil, graveler.ErrWriteToProtectedBranch } if linkResp.JSON409 != nil { return nil, ErrConflict From 5ed4479edaea4ffa9ea0e7e80a734f79333ea77c Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Sun, 5 Jan 2025 11:07:11 +0100 Subject: [PATCH 04/58] without bs --- pkg/api/helpers/errors.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/api/helpers/errors.go b/pkg/api/helpers/errors.go index 2ebc63997f9..7b6113770fd 100644 --- a/pkg/api/helpers/errors.go +++ b/pkg/api/helpers/errors.go @@ -19,9 +19,8 @@ var ( ErrUnsupportedProtocol = errors.New("unsupported protocol") // ErrRequestFailed is an error returned for failing lakeFS server replies. - ErrRequestFailed = errors.New("request failed") - ErrConflict = errors.New("conflict") - ErrProtectedBranch = errors.New("writing to protected branch") + ErrRequestFailed = errors.New("request failed") + ErrConflict = errors.New("conflict") ) const minHTTPErrorStatusCode = 400 From 4aee694caadb6a336b9206d7450add6e82adb6be Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Mon, 6 Jan 2025 09:32:45 +0100 Subject: [PATCH 05/58] changes, working --- docs/reference/auditing.md | 5 +---- pkg/api/helpers/upload.go | 17 +++++++++-------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/docs/reference/auditing.md b/docs/reference/auditing.md index cfebd88fbfd..bcf7ad1a3f6 100644 --- a/docs/reference/auditing.md +++ b/docs/reference/auditing.md @@ -17,10 +17,7 @@ lakeFS Enterprise {: .label .label-purple } {: .note} -> Auditing is only available for [lakeFS Cloud]({% link cloud/index.md %}). - -{: .warning } -> Please note, as of Jan 2024, the queryable interface within the lakeFS Cloud UI has been removed in favor of direct access to lakeFS audit logs. This document now describes how to set up and query this information using [AWS Glue](https://aws.amazon.com/glue/) as a reference. +> Auditing is only available for [lakeFS Cloud]({% link cloud/index.md %}) and [lakeFS Enterprise]({% link enterprise/index.md %}). The lakeFS audit log allows you to view all relevant user action information in a clear and organized table, including when the action was performed, by whom, and what it was they did. diff --git a/pkg/api/helpers/upload.go b/pkg/api/helpers/upload.go index 9ccbccd5d0e..59f1dcedce0 100644 --- a/pkg/api/helpers/upload.go +++ b/pkg/api/helpers/upload.go @@ -20,7 +20,6 @@ import ( "github.com/treeverse/lakefs/pkg/api/apigen" "github.com/treeverse/lakefs/pkg/api/apiutil" "github.com/treeverse/lakefs/pkg/block/azure" - "github.com/treeverse/lakefs/pkg/graveler" "github.com/treeverse/lakefs/pkg/httputil" "golang.org/x/sync/errgroup" ) @@ -384,16 +383,18 @@ func (u *presignUpload) uploadObject(ctx context.Context) (*apigen.ObjectStats, if err != nil { return nil, fmt.Errorf("link object to backing store: %w", err) } + if linkResp.JSON200 != nil { return linkResp.JSON200, nil } - if linkResp.JSON403 != nil { - return nil, graveler.ErrWriteToProtectedBranch - } - if linkResp.JSON409 != nil { - return nil, ErrConflict - } - return nil, fmt.Errorf("link object to backing store: %w (%s)", ErrRequestFailed, linkResp.Status()) + return nil, ResponseAsError(linkResp) + // if linkResp.JSON403 != nil { + // return nil, graveler.ErrWriteToProtectedBranch + // } + // if linkResp.JSON409 != nil { + // return nil, ErrConflict + // } + // return nil, fmt.Errorf("link object to backing store: %w (%s)", ErrRequestFailed, linkResp.Status()) } func (u *presignUpload) Upload(ctx context.Context) (*apigen.ObjectStats, error) { From cce44f3735d8ed5d9e56511ff23befac2aa57792 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Tue, 7 Jan 2025 09:55:17 +0100 Subject: [PATCH 06/58] lets see if something breaks --- pkg/local/sync.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/local/sync.go b/pkg/local/sync.go index fa82508334d..24dac82c01f 100644 --- a/pkg/local/sync.go +++ b/pkg/local/sync.go @@ -140,7 +140,7 @@ func (s *SyncManager) downloadFile(ctx context.Context, remote *uri.URI, path, d if errors.Is(err, syscall.EISDIR) && sizeBytes == 0 { return nil // no further action required! } - return fmt.Errorf("could not create file '%s': %w", destination, err) + return helpers.ResponseAsError(err) //fmt.Errorf("could not create file '%s': %w", destination, err) } defer func() { err = f.Close() @@ -193,7 +193,8 @@ func (s *SyncManager) downloadFile(ctx context.Context, remote *uri.URI, path, d _, err = io.Copy(f, barReader) if err != nil { - return fmt.Errorf("could not write file '%s': %w", destination, err) + return helpers.ResponseAsError(err) + // return fmt.Errorf("could not write file '%s': %w", destination, err) } } return nil @@ -388,7 +389,8 @@ func (s *SyncManager) deleteRemote(ctx context.Context, remote *uri.URI, change return } if resp.StatusCode() != http.StatusNoContent { - return fmt.Errorf("could not delete object: HTTP %d: %w", resp.StatusCode(), helpers.ErrRequestFailed) + return helpers.ResponseAsError(resp) + //return fmt.Errorf("could not delete object: HTTP %d: %w", resp.StatusCode(), helpers.ErrRequestFailed) } return } From 36d9215a954bb149f25ff8747d7b4376d771aed7 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 8 Jan 2025 09:26:52 +0100 Subject: [PATCH 07/58] testing the tests --- esti/lakectl_local_test.go | 105 +++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 21a8861d48b..8898b890e33 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -486,6 +486,111 @@ func TestLakectlLocal_pull(t *testing.T) { } } +func TestLakectlLocal_commitProtetedBranch(t *testing.T) { + tmpDir := t.TempDir() + fd, err := os.CreateTemp(tmpDir, "") + require.NoError(t, err) + require.NoError(t, fd.Close()) + repoName := generateUniqueRepositoryName() + storage := generateUniqueStorageNamespace(repoName) + vars := map[string]string{ + "REPO": repoName, + "STORAGE": storage, + "BRANCH": mainBranch, + "REF": mainBranch, + "PREFIX": "", + } + + runCmd(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, false, vars) + runCmd(t, Lakectl()+" log lakefs://"+repoName+"/"+mainBranch, false, false, vars) + runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ "+mainBranch, false, false, vars) + prefix := "images" + objects := []string{ + "ro_1k.1", + "ro_1k.2", + "ro_1k.3", + prefix + "/1.png", + prefix + "/2.png", + prefix + "/3.png", + prefix + "/subdir/1.png", + prefix + "/subdir/2.png", + prefix + "/subdir/3.png", + } + + tests := []struct { + name string + prefix string + presign bool + }{ + { + name: "root", + prefix: "", + presign: false, + }, + { + name: "root-presign", + prefix: "", + presign: true, + }, + { + name: prefix, + prefix: prefix, + presign: false, + }, + { + name: prefix + "-presign", + prefix: prefix, + presign: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.presign { + // Skip due to bug on Azure https://github.com/treeverse/lakeFS/issues/6426 + requireBlockstoreType(t, block.BlockstoreTypeS3, block.BlockstoreTypeGS) + } + dataDir, err := os.MkdirTemp(tmpDir, "") + require.NoError(t, err) + deleted := prefix + "/subdir/deleted.png" + + localCreateTestData(t, vars, append(objects, deleted)) + + runCmd(t, Lakectl()+" branch create lakefs://"+repoName+"/"+tt.name+" --source lakefs://"+repoName+"/"+mainBranch, false, false, vars) + + vars["LOCAL_DIR"] = dataDir + vars["PREFIX"] = "" + vars["BRANCH"] = tt.name + vars["REF"] = tt.name + presign := fmt.Sprintf(" --pre-sign=%v ", tt.presign) + RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+repoName+"/"+vars["BRANCH"]+"/"+vars["PREFIX"]+presign+dataDir, false, "Successfully cloned lakefs://${REPO}/${REF}/${PREFIX} to ${LOCAL_DIR}.", vars) + + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "No diff found", vars) + + // Modify local folder - add and remove files + require.NoError(t, os.MkdirAll(filepath.Join(dataDir, "subdir"), os.ModePerm)) + require.NoError(t, os.MkdirAll(filepath.Join(dataDir, "subdir-a"), os.ModePerm)) + fd, err = os.Create(filepath.Join(dataDir, "subdir", "test.txt")) + require.NoError(t, err) + fd, err = os.Create(filepath.Join(dataDir, "subdir-a", "test.txt")) + require.NoError(t, err) + fd, err = os.Create(filepath.Join(dataDir, "test.txt")) + require.NoError(t, err) + require.NoError(t, fd.Close()) + require.NoError(t, os.Remove(filepath.Join(dataDir, deleted))) + + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ test.txt", vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ subdir/test.txt", vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ subdir-a/test.txt", vars) + + // Commit changes to branch + RunCmdAndVerifyContainsText(t, Lakectl()+" local commit -m test"+presign+dataDir, false, "cannot write to protected branch", vars) + + // Check no diff after commit + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "No diff found", vars) + }) + } + +} func TestLakectlLocal_commit(t *testing.T) { tmpDir := t.TempDir() fd, err := os.CreateTemp(tmpDir, "") From 03ad31715a11f21170f0be0e863a7fa2934bdcdb Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 8 Jan 2025 10:16:10 +0100 Subject: [PATCH 08/58] testing the tests#2 --- esti/lakectl_local_test.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 8898b890e33..19f1c5e992b 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -487,6 +487,18 @@ func TestLakectlLocal_pull(t *testing.T) { } func TestLakectlLocal_commitProtetedBranch(t *testing.T) { + + // repoName := generateUniqueRepositoryName() + // storage := generateUniqueStorageNamespace(repoName) + // vars := map[string]string{ + // "REPO": repoName, + // "STORAGE": storage, + // "BRANCH": mainBranch, + // } + // RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, "lakectl_repo_create", vars) + + // RunCmdAndVerifySuccessWithFile(t, Lakectl()+" branch-protect add lakefs://"+repoName+" "+mainBranch, false, "lakectl_empty", vars) + // RunCmdAndVerifySuccessWithFile(t, Lakectl()+" branch-protect list lakefs://"+repoName, false, "lakectl_branch_protection_list.term", vars) tmpDir := t.TempDir() fd, err := os.CreateTemp(tmpDir, "") require.NoError(t, err) @@ -501,9 +513,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { "PREFIX": "", } - runCmd(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, false, vars) runCmd(t, Lakectl()+" log lakefs://"+repoName+"/"+mainBranch, false, false, vars) - runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ "+mainBranch, false, false, vars) prefix := "images" objects := []string{ "ro_1k.1", @@ -556,6 +566,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { localCreateTestData(t, vars, append(objects, deleted)) runCmd(t, Lakectl()+" branch create lakefs://"+repoName+"/"+tt.name+" --source lakefs://"+repoName+"/"+mainBranch, false, false, vars) + runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ "+tt.name, false, false, vars) vars["LOCAL_DIR"] = dataDir vars["PREFIX"] = "" @@ -585,8 +596,8 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { // Commit changes to branch RunCmdAndVerifyContainsText(t, Lakectl()+" local commit -m test"+presign+dataDir, false, "cannot write to protected branch", vars) - // Check no diff after commit - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "No diff found", vars) + // // Check no diff after commit + // RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "No diff found", vars) }) } From 22bd27e1d3c0ec30161cf41a4bbbda1e73da29a2 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 8 Jan 2025 10:27:24 +0100 Subject: [PATCH 09/58] testing the tests#2 --- esti/lakectl_local_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 19f1c5e992b..71ea54e2dea 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -512,7 +512,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { "REF": mainBranch, "PREFIX": "", } - + runCmd(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, false, vars) runCmd(t, Lakectl()+" log lakefs://"+repoName+"/"+mainBranch, false, false, vars) prefix := "images" objects := []string{ From ca686cef1d51ab8cb371717715c4a17018533c03 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 8 Jan 2025 10:45:15 +0100 Subject: [PATCH 10/58] testing the tests#2 --- esti/lakectl_local_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 71ea54e2dea..5263521f1be 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -566,7 +566,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { localCreateTestData(t, vars, append(objects, deleted)) runCmd(t, Lakectl()+" branch create lakefs://"+repoName+"/"+tt.name+" --source lakefs://"+repoName+"/"+mainBranch, false, false, vars) - runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ "+tt.name, false, false, vars) + runCmd(t, Lakectl()+" branch-protect add "+tt.name, false, false, vars) vars["LOCAL_DIR"] = dataDir vars["PREFIX"] = "" From c32d126bbae332174b9b1e6caf67f03b5d02b4ec Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 8 Jan 2025 11:03:03 +0100 Subject: [PATCH 11/58] test test tests --- esti/lakectl_local_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 5263521f1be..d4b4e620b2f 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -566,7 +566,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { localCreateTestData(t, vars, append(objects, deleted)) runCmd(t, Lakectl()+" branch create lakefs://"+repoName+"/"+tt.name+" --source lakefs://"+repoName+"/"+mainBranch, false, false, vars) - runCmd(t, Lakectl()+" branch-protect add "+tt.name, false, false, vars) + runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ "+tt.name, false, false, vars) vars["LOCAL_DIR"] = dataDir vars["PREFIX"] = "" @@ -594,7 +594,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ subdir-a/test.txt", vars) // Commit changes to branch - RunCmdAndVerifyContainsText(t, Lakectl()+" local commit -m test"+presign+dataDir, false, "cannot write to protected branch", vars) + RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test"+presign+dataDir, false, "cannot write to protected branch", vars) // // Check no diff after commit // RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "No diff found", vars) From c7e1ca7d5ddd54a37213bdbdd747af82ad997254 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 8 Jan 2025 12:09:24 +0100 Subject: [PATCH 12/58] test test tests --- esti/lakectl_local_test.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index d4b4e620b2f..28ff9076ae8 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -565,14 +565,15 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { localCreateTestData(t, vars, append(objects, deleted)) - runCmd(t, Lakectl()+" branch create lakefs://"+repoName+"/"+tt.name+" --source lakefs://"+repoName+"/"+mainBranch, false, false, vars) - runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ "+tt.name, false, false, vars) - vars["LOCAL_DIR"] = dataDir vars["PREFIX"] = "" vars["BRANCH"] = tt.name vars["REF"] = tt.name presign := fmt.Sprintf(" --pre-sign=%v ", tt.presign) + + runCmd(t, Lakectl()+" branch create lakefs://"+repoName+"/"+tt.name+" --source lakefs://"+repoName+"/"+mainBranch, false, false, vars) + runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ "+tt.name, false, false, vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+repoName+"/"+vars["BRANCH"]+"/"+vars["PREFIX"]+presign+dataDir, false, "Successfully cloned lakefs://${REPO}/${REF}/${PREFIX} to ${LOCAL_DIR}.", vars) RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "No diff found", vars) @@ -596,8 +597,6 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { // Commit changes to branch RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test"+presign+dataDir, false, "cannot write to protected branch", vars) - // // Check no diff after commit - // RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "No diff found", vars) }) } From 5495d4e36b2978c7cac8602725d3f657055e6669 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 8 Jan 2025 12:23:43 +0100 Subject: [PATCH 13/58] test test tests --- esti/lakectl_local_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 28ff9076ae8..b2b6eacb00d 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -596,6 +596,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { // Commit changes to branch RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test"+presign+dataDir, false, "cannot write to protected branch", vars) + runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ "+tt.name, false, false, vars) }) } From c1af8bb38b6d4306bb0146815ee66e2fc766d626 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 8 Jan 2025 13:09:25 +0100 Subject: [PATCH 14/58] test test tests --- esti/lakectl_local_test.go | 93 ++++++++++++++------------------------ 1 file changed, 35 insertions(+), 58 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index b2b6eacb00d..b97d7cca2e2 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -488,17 +488,6 @@ func TestLakectlLocal_pull(t *testing.T) { func TestLakectlLocal_commitProtetedBranch(t *testing.T) { - // repoName := generateUniqueRepositoryName() - // storage := generateUniqueStorageNamespace(repoName) - // vars := map[string]string{ - // "REPO": repoName, - // "STORAGE": storage, - // "BRANCH": mainBranch, - // } - // RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, "lakectl_repo_create", vars) - - // RunCmdAndVerifySuccessWithFile(t, Lakectl()+" branch-protect add lakefs://"+repoName+" "+mainBranch, false, "lakectl_empty", vars) - // RunCmdAndVerifySuccessWithFile(t, Lakectl()+" branch-protect list lakefs://"+repoName, false, "lakectl_branch_protection_list.term", vars) tmpDir := t.TempDir() fd, err := os.CreateTemp(tmpDir, "") require.NoError(t, err) @@ -513,7 +502,6 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { "PREFIX": "", } runCmd(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, false, vars) - runCmd(t, Lakectl()+" log lakefs://"+repoName+"/"+mainBranch, false, false, vars) prefix := "images" objects := []string{ "ro_1k.1", @@ -528,75 +516,64 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { } tests := []struct { - name string - prefix string - presign bool + name string + prefix string + addOrDelete bool }{ { - name: "root", - prefix: "", - presign: false, - }, - { - name: "root-presign", - prefix: "", - presign: true, - }, - { - name: prefix, - prefix: prefix, - presign: false, + name: "test1", + prefix: "", + addOrDelete: true, }, { - name: prefix + "-presign", - prefix: prefix, - presign: true, + name: "test2", + prefix: prefix, + addOrDelete: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if tt.presign { - // Skip due to bug on Azure https://github.com/treeverse/lakeFS/issues/6426 - requireBlockstoreType(t, block.BlockstoreTypeS3, block.BlockstoreTypeGS) - } dataDir, err := os.MkdirTemp(tmpDir, "") require.NoError(t, err) deleted := prefix + "/subdir/deleted.png" - localCreateTestData(t, vars, append(objects, deleted)) - vars["LOCAL_DIR"] = dataDir - vars["PREFIX"] = "" vars["BRANCH"] = tt.name vars["REF"] = tt.name - presign := fmt.Sprintf(" --pre-sign=%v ", tt.presign) + vars["PREFIX"] = tt.prefix - runCmd(t, Lakectl()+" branch create lakefs://"+repoName+"/"+tt.name+" --source lakefs://"+repoName+"/"+mainBranch, false, false, vars) - runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ "+tt.name, false, false, vars) + runCmd(t, Lakectl()+" branch create lakefs://"+repoName+"/"+vars["BRANCH"]+" --source lakefs://"+repoName+"/"+mainBranch, false, false, vars) + localCreateTestData(t, vars, append(objects, deleted)) + runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ "+vars["BRANCH"], false, false, vars) - RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+repoName+"/"+vars["BRANCH"]+"/"+vars["PREFIX"]+presign+dataDir, false, "Successfully cloned lakefs://${REPO}/${REF}/${PREFIX} to ${LOCAL_DIR}.", vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+repoName+"/"+vars["BRANCH"]+"/"+vars["PREFIX"]+dataDir, false, "Successfully cloned lakefs://${REPO}/${REF}/${PREFIX} to ${LOCAL_DIR}.", vars) RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "No diff found", vars) - // Modify local folder - add and remove files - require.NoError(t, os.MkdirAll(filepath.Join(dataDir, "subdir"), os.ModePerm)) - require.NoError(t, os.MkdirAll(filepath.Join(dataDir, "subdir-a"), os.ModePerm)) - fd, err = os.Create(filepath.Join(dataDir, "subdir", "test.txt")) - require.NoError(t, err) - fd, err = os.Create(filepath.Join(dataDir, "subdir-a", "test.txt")) - require.NoError(t, err) - fd, err = os.Create(filepath.Join(dataDir, "test.txt")) - require.NoError(t, err) - require.NoError(t, fd.Close()) - require.NoError(t, os.Remove(filepath.Join(dataDir, deleted))) + // Modify local folder - add or remove files + if tt.addOrDelete { + require.NoError(t, os.MkdirAll(filepath.Join(dataDir, "subdir"), os.ModePerm)) + require.NoError(t, os.MkdirAll(filepath.Join(dataDir, "subdir-a"), os.ModePerm)) + fd, err = os.Create(filepath.Join(dataDir, "subdir", "test.txt")) + require.NoError(t, err) + fd, err = os.Create(filepath.Join(dataDir, "subdir-a", "test.txt")) + require.NoError(t, err) + fd, err = os.Create(filepath.Join(dataDir, "test.txt")) + require.NoError(t, err) + require.NoError(t, fd.Close()) + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ test.txt", vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ subdir/test.txt", vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ subdir-a/test.txt", vars) - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ test.txt", vars) - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ subdir/test.txt", vars) - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ subdir-a/test.txt", vars) + } else { + require.NoError(t, os.Remove(filepath.Join(dataDir, deleted))) + //RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ removed ║ test.txt", vars) + + } // Commit changes to branch - RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test"+presign+dataDir, false, "cannot write to protected branch", vars) - runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ "+tt.name, false, false, vars) + RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test", false, "cannot write to protected branch", vars) + runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ "+vars["BRANCH"], false, false, vars) }) } From 700035a227cc8fb8d660188046e0977f6dd7c1ad Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 8 Jan 2025 13:39:10 +0100 Subject: [PATCH 15/58] test test tests --- esti/lakectl_local_test.go | 31 ++++++------------------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index b97d7cca2e2..f8dac4a0ed9 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -487,11 +487,6 @@ func TestLakectlLocal_pull(t *testing.T) { } func TestLakectlLocal_commitProtetedBranch(t *testing.T) { - - tmpDir := t.TempDir() - fd, err := os.CreateTemp(tmpDir, "") - require.NoError(t, err) - require.NoError(t, fd.Close()) repoName := generateUniqueRepositoryName() storage := generateUniqueStorageNamespace(repoName) vars := map[string]string{ @@ -499,65 +494,51 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { "STORAGE": storage, "BRANCH": mainBranch, "REF": mainBranch, - "PREFIX": "", } runCmd(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, false, vars) - prefix := "images" objects := []string{ "ro_1k.1", "ro_1k.2", "ro_1k.3", - prefix + "/1.png", - prefix + "/2.png", - prefix + "/3.png", - prefix + "/subdir/1.png", - prefix + "/subdir/2.png", - prefix + "/subdir/3.png", } tests := []struct { name string - prefix string addOrDelete bool }{ { name: "test1", - prefix: "", addOrDelete: true, }, { name: "test2", - prefix: prefix, addOrDelete: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + fd, err := os.CreateTemp(tmpDir, "") + require.NoError(t, err) + require.NoError(t, fd.Close()) dataDir, err := os.MkdirTemp(tmpDir, "") require.NoError(t, err) - deleted := prefix + "/subdir/deleted.png" + deleted := "deleted.png" vars["LOCAL_DIR"] = dataDir vars["BRANCH"] = tt.name vars["REF"] = tt.name - vars["PREFIX"] = tt.prefix runCmd(t, Lakectl()+" branch create lakefs://"+repoName+"/"+vars["BRANCH"]+" --source lakefs://"+repoName+"/"+mainBranch, false, false, vars) localCreateTestData(t, vars, append(objects, deleted)) runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ "+vars["BRANCH"], false, false, vars) - RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+repoName+"/"+vars["BRANCH"]+"/"+vars["PREFIX"]+dataDir, false, "Successfully cloned lakefs://${REPO}/${REF}/${PREFIX} to ${LOCAL_DIR}.", vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+repoName+"/"+vars["BRANCH"]+"/ "+dataDir, false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "No diff found", vars) // Modify local folder - add or remove files if tt.addOrDelete { - require.NoError(t, os.MkdirAll(filepath.Join(dataDir, "subdir"), os.ModePerm)) - require.NoError(t, os.MkdirAll(filepath.Join(dataDir, "subdir-a"), os.ModePerm)) - fd, err = os.Create(filepath.Join(dataDir, "subdir", "test.txt")) - require.NoError(t, err) - fd, err = os.Create(filepath.Join(dataDir, "subdir-a", "test.txt")) - require.NoError(t, err) fd, err = os.Create(filepath.Join(dataDir, "test.txt")) require.NoError(t, err) require.NoError(t, fd.Close()) From ce1031e7b0a5bb91021999ed9e5d1c5169b7937f Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 8 Jan 2025 14:11:40 +0100 Subject: [PATCH 16/58] test test tests --- esti/lakectl_local_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index f8dac4a0ed9..6fc89ee7521 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -542,9 +542,9 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { fd, err = os.Create(filepath.Join(dataDir, "test.txt")) require.NoError(t, err) require.NoError(t, fd.Close()) - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ test.txt", vars) - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ subdir/test.txt", vars) - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ subdir-a/test.txt", vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ test.txt", vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ subdir/test.txt", vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ subdir-a/test.txt", vars) } else { require.NoError(t, os.Remove(filepath.Join(dataDir, deleted))) @@ -553,7 +553,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { } // Commit changes to branch - RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test", false, "cannot write to protected branch", vars) + RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test"+dataDir, false, "cannot write to protected branch", vars) runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ "+vars["BRANCH"], false, false, vars) }) From e086c14f90aba0c0f5345ef367f64fb7de7abb75 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 8 Jan 2025 14:26:14 +0100 Subject: [PATCH 17/58] test test tests --- esti/lakectl_local_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 6fc89ee7521..eabb3987bc1 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -543,8 +543,6 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { require.NoError(t, err) require.NoError(t, fd.Close()) RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ test.txt", vars) - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ subdir/test.txt", vars) - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ subdir-a/test.txt", vars) } else { require.NoError(t, os.Remove(filepath.Join(dataDir, deleted))) @@ -553,7 +551,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { } // Commit changes to branch - RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test"+dataDir, false, "cannot write to protected branch", vars) + RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+dataDir, false, "cannot write to protected branch", vars) runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ "+vars["BRANCH"], false, false, vars) }) From 45f611dfce4d7d834ca6eb2abade6376ed6343fe Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 8 Jan 2025 16:05:47 +0100 Subject: [PATCH 18/58] test test tests --- esti/lakectl_local_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index eabb3987bc1..39c70a065df 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -549,7 +549,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { //RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ removed ║ test.txt", vars) } - + runCmdAndVerifyContainsText(t, Lakectl()+"local branch-protect list lakefs://"+repoName+"/"+dataDir, false, false, vars["BRANCH"], vars) // Commit changes to branch RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+dataDir, false, "cannot write to protected branch", vars) runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ "+vars["BRANCH"], false, false, vars) From 36f38b59db30be6d6e411de440c788e74cebbc5f Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 8 Jan 2025 16:07:03 +0100 Subject: [PATCH 19/58] test test tests --- esti/lakectl_local_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 39c70a065df..056034edba6 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -549,7 +549,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { //RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ removed ║ test.txt", vars) } - runCmdAndVerifyContainsText(t, Lakectl()+"local branch-protect list lakefs://"+repoName+"/"+dataDir, false, false, vars["BRANCH"], vars) + runCmdAndVerifyContainsText(t, Lakectl()+"local branch-protect list lakefs://"+repoName+"/ "+dataDir, false, false, vars["BRANCH"], vars) // Commit changes to branch RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+dataDir, false, "cannot write to protected branch", vars) runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ "+vars["BRANCH"], false, false, vars) From c92bf21f3910dafdb0267ae7bd4ec8e7e2c61a1f Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 8 Jan 2025 16:27:06 +0100 Subject: [PATCH 20/58] test test tests --- esti/lakectl_local_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 056034edba6..be425c06322 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -549,7 +549,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { //RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ removed ║ test.txt", vars) } - runCmdAndVerifyContainsText(t, Lakectl()+"local branch-protect list lakefs://"+repoName+"/ "+dataDir, false, false, vars["BRANCH"], vars) + runCmdAndVerifyContainsText(t, Lakectl()+"local branch-protect list lakefs://"+repoName+"/ ", false, false, vars["BRANCH"], vars) // Commit changes to branch RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+dataDir, false, "cannot write to protected branch", vars) runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ "+vars["BRANCH"], false, false, vars) From fdeb73e9cb0becc69fb0feaf3b5775d1bb9c8feb Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 8 Jan 2025 16:37:01 +0100 Subject: [PATCH 21/58] test test tests --- esti/lakectl_local_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index be425c06322..f471baaa7b7 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -549,7 +549,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { //RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ removed ║ test.txt", vars) } - runCmdAndVerifyContainsText(t, Lakectl()+"local branch-protect list lakefs://"+repoName+"/ ", false, false, vars["BRANCH"], vars) + runCmdAndVerifyContainsText(t, Lakectl()+" local branch-protect list lakefs://"+repoName+"/ ", false, false, vars["BRANCH"], vars) // Commit changes to branch RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+dataDir, false, "cannot write to protected branch", vars) runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ "+vars["BRANCH"], false, false, vars) From c8c3c6f7721ecd367dbf09a285c1f671262d0f5a Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 8 Jan 2025 16:51:36 +0100 Subject: [PATCH 22/58] test test tests --- esti/lakectl_local_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index f471baaa7b7..c388f64745e 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -549,7 +549,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { //RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ removed ║ test.txt", vars) } - runCmdAndVerifyContainsText(t, Lakectl()+" local branch-protect list lakefs://"+repoName+"/ ", false, false, vars["BRANCH"], vars) + runCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars["BRANCH"], vars) // Commit changes to branch RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+dataDir, false, "cannot write to protected branch", vars) runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ "+vars["BRANCH"], false, false, vars) From bdaa9a04fd22ee45c46ff4546397539acbc03f18 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 8 Jan 2025 21:27:16 +0100 Subject: [PATCH 23/58] back to beginning --- pkg/api/helpers/upload.go | 13 ++++--------- pkg/local/sync.go | 8 +++----- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/pkg/api/helpers/upload.go b/pkg/api/helpers/upload.go index 59f1dcedce0..27e09027a6f 100644 --- a/pkg/api/helpers/upload.go +++ b/pkg/api/helpers/upload.go @@ -383,18 +383,13 @@ func (u *presignUpload) uploadObject(ctx context.Context) (*apigen.ObjectStats, if err != nil { return nil, fmt.Errorf("link object to backing store: %w", err) } - if linkResp.JSON200 != nil { return linkResp.JSON200, nil } - return nil, ResponseAsError(linkResp) - // if linkResp.JSON403 != nil { - // return nil, graveler.ErrWriteToProtectedBranch - // } - // if linkResp.JSON409 != nil { - // return nil, ErrConflict - // } - // return nil, fmt.Errorf("link object to backing store: %w (%s)", ErrRequestFailed, linkResp.Status()) + if linkResp.JSON409 != nil { + return nil, ErrConflict + } + return nil, fmt.Errorf("link object to backing store: %w (%s)", ErrRequestFailed, linkResp.Status()) } func (u *presignUpload) Upload(ctx context.Context) (*apigen.ObjectStats, error) { diff --git a/pkg/local/sync.go b/pkg/local/sync.go index 24dac82c01f..4008c6154b9 100644 --- a/pkg/local/sync.go +++ b/pkg/local/sync.go @@ -140,7 +140,7 @@ func (s *SyncManager) downloadFile(ctx context.Context, remote *uri.URI, path, d if errors.Is(err, syscall.EISDIR) && sizeBytes == 0 { return nil // no further action required! } - return helpers.ResponseAsError(err) //fmt.Errorf("could not create file '%s': %w", destination, err) + fmt.Errorf("could not create file '%s': %w", destination, err) } defer func() { err = f.Close() @@ -193,8 +193,7 @@ func (s *SyncManager) downloadFile(ctx context.Context, remote *uri.URI, path, d _, err = io.Copy(f, barReader) if err != nil { - return helpers.ResponseAsError(err) - // return fmt.Errorf("could not write file '%s': %w", destination, err) + return fmt.Errorf("could not write file '%s': %w", destination, err) } } return nil @@ -389,8 +388,7 @@ func (s *SyncManager) deleteRemote(ctx context.Context, remote *uri.URI, change return } if resp.StatusCode() != http.StatusNoContent { - return helpers.ResponseAsError(resp) - //return fmt.Errorf("could not delete object: HTTP %d: %w", resp.StatusCode(), helpers.ErrRequestFailed) + return fmt.Errorf("could not delete object: HTTP %d: %w", resp.StatusCode(), helpers.ErrRequestFailed) } return } From 57c109c348c3e9560e5c9da7a6c1078f8c0475fa Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 8 Jan 2025 21:28:14 +0100 Subject: [PATCH 24/58] back to beginning --- pkg/local/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local/sync.go b/pkg/local/sync.go index 4008c6154b9..fa82508334d 100644 --- a/pkg/local/sync.go +++ b/pkg/local/sync.go @@ -140,7 +140,7 @@ func (s *SyncManager) downloadFile(ctx context.Context, remote *uri.URI, path, d if errors.Is(err, syscall.EISDIR) && sizeBytes == 0 { return nil // no further action required! } - fmt.Errorf("could not create file '%s': %w", destination, err) + return fmt.Errorf("could not create file '%s': %w", destination, err) } defer func() { err = f.Close() From f46df4877d708b5d02746c7af16009f158f61c9a Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 8 Jan 2025 23:45:24 +0100 Subject: [PATCH 25/58] looks fresh --- pkg/api/helpers/upload.go | 2 +- pkg/local/sync.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/api/helpers/upload.go b/pkg/api/helpers/upload.go index 27e09027a6f..061c5433f11 100644 --- a/pkg/api/helpers/upload.go +++ b/pkg/api/helpers/upload.go @@ -389,7 +389,7 @@ func (u *presignUpload) uploadObject(ctx context.Context) (*apigen.ObjectStats, if linkResp.JSON409 != nil { return nil, ErrConflict } - return nil, fmt.Errorf("link object to backing store: %w (%s)", ErrRequestFailed, linkResp.Status()) + return nil, fmt.Errorf("link object to backing store: %w (%s)", ResponseAsError(linkResp), linkResp.Status()) } func (u *presignUpload) Upload(ctx context.Context) (*apigen.ObjectStats, error) { diff --git a/pkg/local/sync.go b/pkg/local/sync.go index fa82508334d..e28ec46d5d9 100644 --- a/pkg/local/sync.go +++ b/pkg/local/sync.go @@ -388,7 +388,7 @@ func (s *SyncManager) deleteRemote(ctx context.Context, remote *uri.URI, change return } if resp.StatusCode() != http.StatusNoContent { - return fmt.Errorf("could not delete object: HTTP %d: %w", resp.StatusCode(), helpers.ErrRequestFailed) + return fmt.Errorf("could not delete object: HTTP %d: %w", resp.StatusCode(), helpers.ResponseAsError(resp)) } return } From 58ef767254522c0679902fd4d4d8cffb74c96ddf Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Sun, 12 Jan 2025 09:32:26 +0100 Subject: [PATCH 26/58] yalla --- esti/lakectl_local_test.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index c388f64745e..6f8c82753f3 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -503,16 +503,16 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { } tests := []struct { - name string - addOrDelete bool + name string + Delete bool }{ { - name: "test1", - addOrDelete: true, + name: "test1", + Delete: true, }, { - name: "test2", - addOrDelete: false, + name: "test2", + Delete: false, }, } for _, tt := range tests { @@ -538,16 +538,14 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "No diff found", vars) // Modify local folder - add or remove files - if tt.addOrDelete { + if tt.Delete { + require.NoError(t, os.Remove(filepath.Join(dataDir, deleted))) + + } else { fd, err = os.Create(filepath.Join(dataDir, "test.txt")) require.NoError(t, err) require.NoError(t, fd.Close()) RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ test.txt", vars) - - } else { - require.NoError(t, os.Remove(filepath.Join(dataDir, deleted))) - //RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ removed ║ test.txt", vars) - } runCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars["BRANCH"], vars) // Commit changes to branch From 40a6ba2a8a3b055694d3c4d9a7d26f8a847768e1 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Sun, 12 Jan 2025 11:42:09 +0100 Subject: [PATCH 27/58] testing list --- esti/lakectl_local_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 6f8c82753f3..f24925014cc 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -548,6 +548,8 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ test.txt", vars) } runCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars["BRANCH"], vars) + str := runCmd(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars["BRANCH"], vars) + fmt.Println("itamar testing", str) // Commit changes to branch RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+dataDir, false, "cannot write to protected branch", vars) runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ "+vars["BRANCH"], false, false, vars) From 72d105ee75624505d748353a37ab46cabea9be49 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Sun, 12 Jan 2025 11:44:04 +0100 Subject: [PATCH 28/58] testing list --- esti/lakectl_local_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index f24925014cc..7b811182d12 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -548,7 +548,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ test.txt", vars) } runCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars["BRANCH"], vars) - str := runCmd(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars["BRANCH"], vars) + str := runCmd(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars) fmt.Println("itamar testing", str) // Commit changes to branch RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+dataDir, false, "cannot write to protected branch", vars) From 230aff51c19a1cbf95dba597d21c31aab6625f3b Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Sun, 12 Jan 2025 11:57:38 +0100 Subject: [PATCH 29/58] testing list --- esti/lakectl_local_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 7b811182d12..ab6bb654e6f 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -531,7 +531,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { runCmd(t, Lakectl()+" branch create lakefs://"+repoName+"/"+vars["BRANCH"]+" --source lakefs://"+repoName+"/"+mainBranch, false, false, vars) localCreateTestData(t, vars, append(objects, deleted)) - runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ "+vars["BRANCH"], false, false, vars) + runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ *", false, false, vars) RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+repoName+"/"+vars["BRANCH"]+"/ "+dataDir, false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) @@ -547,12 +547,12 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { require.NoError(t, fd.Close()) RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ test.txt", vars) } - runCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars["BRANCH"], vars) + //runCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars["BRANCH"], vars) str := runCmd(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars) fmt.Println("itamar testing", str) // Commit changes to branch RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+dataDir, false, "cannot write to protected branch", vars) - runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ "+vars["BRANCH"], false, false, vars) + runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ *", false, false, vars) }) } From 291c18190d683bf39793b6c0ecca517f3568b7a2 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Sun, 12 Jan 2025 12:03:18 +0100 Subject: [PATCH 30/58] *** --- esti/lakectl_local_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index ab6bb654e6f..2ec84438516 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -531,7 +531,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { runCmd(t, Lakectl()+" branch create lakefs://"+repoName+"/"+vars["BRANCH"]+" --source lakefs://"+repoName+"/"+mainBranch, false, false, vars) localCreateTestData(t, vars, append(objects, deleted)) - runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ *", false, false, vars) + runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ '*'", false, false, vars) RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+repoName+"/"+vars["BRANCH"]+"/ "+dataDir, false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) @@ -552,7 +552,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { fmt.Println("itamar testing", str) // Commit changes to branch RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+dataDir, false, "cannot write to protected branch", vars) - runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ *", false, false, vars) + //runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ \"*\"", false, false, vars) }) } From 524a00000c4a0ab5634dfd7fb688e0b2a52af18a Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Sun, 12 Jan 2025 12:28:12 +0100 Subject: [PATCH 31/58] fs upload test --- esti/lakectl_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go index 5195bb267bf..3a28427294e 100644 --- a/esti/lakectl_test.go +++ b/esti/lakectl_test.go @@ -726,6 +726,22 @@ func getStorageConfig(t *testing.T) *apigen.StorageConfig { } return storageResp.JSON200 } +func TestLakectlFsUpload_protectedBranch(t *testing.T) { + repoName := generateUniqueRepositoryName() + storage := generateUniqueStorageNamespace(repoName) + vars := map[string]string{ + "REPO": repoName, + "STORAGE": storage, + "BRANCH": mainBranch, + } + RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, "lakectl_repo_create", vars) + runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ '*'", false, false, vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, "*", vars) + + vars["FILE_PATH"] = "data/ro/ro_1k.0_sep/" + RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, "cannot write to protected branch", vars) + +} func TestLakectlFsPresign(t *testing.T) { config := getStorageConfig(t) From 3923c45a7fc2c638d2925f3f1e938285e9c55a09 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Sun, 12 Jan 2025 12:44:17 +0100 Subject: [PATCH 32/58] fs upload test --- esti/lakectl_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go index 3a28427294e..638714193ee 100644 --- a/esti/lakectl_test.go +++ b/esti/lakectl_test.go @@ -738,9 +738,8 @@ func TestLakectlFsUpload_protectedBranch(t *testing.T) { runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ '*'", false, false, vars) RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, "*", vars) - vars["FILE_PATH"] = "data/ro/ro_1k.0_sep/" + vars["FILE_PATH"] = "data/ro/ro_1k.0" RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, "cannot write to protected branch", vars) - } func TestLakectlFsPresign(t *testing.T) { From edd352d2cac185e671f8ba965b21796bf0fe4b13 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Sun, 12 Jan 2025 12:55:52 +0100 Subject: [PATCH 33/58] fs upload test --- esti/lakectl_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go index 638714193ee..57ca55ebb22 100644 --- a/esti/lakectl_test.go +++ b/esti/lakectl_test.go @@ -739,7 +739,7 @@ func TestLakectlFsUpload_protectedBranch(t *testing.T) { RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, "*", vars) vars["FILE_PATH"] = "data/ro/ro_1k.0" - RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, "cannot write to protected branch", vars) + RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, "cannot write to protected branch\n403 Forbidden\n", vars) } func TestLakectlFsPresign(t *testing.T) { From c6c2c9a55d79a9edbd7e244cc920d68fca4bed65 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Sun, 12 Jan 2025 15:14:57 +0100 Subject: [PATCH 34/58] with extre checks --- esti/lakectl_local_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 2ec84438516..e0f9a85f6ba 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -551,6 +551,11 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { str := runCmd(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars) fmt.Println("itamar testing", str) // Commit changes to branch + runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) + + RunCmdAndVerifyFailureContainsText(t, Lakectl()+" fs upload -s files/ro_1k lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+"fail_file.txt", false, "cannot write to protected branch", vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" local list", false, repoName+"/"+vars["BRANCH"], vars) + RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+dataDir, false, "cannot write to protected branch", vars) //runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ \"*\"", false, false, vars) From c3b9fca9d8c56ffeef9a6ae0d407139bf6fc6dc6 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Sun, 12 Jan 2025 15:28:08 +0100 Subject: [PATCH 35/58] with extre checks --- esti/lakectl_local_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index e0f9a85f6ba..8d52e81c2d1 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -551,7 +551,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { str := runCmd(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars) fmt.Println("itamar testing", str) // Commit changes to branch - runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) + //runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) RunCmdAndVerifyFailureContainsText(t, Lakectl()+" fs upload -s files/ro_1k lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+"fail_file.txt", false, "cannot write to protected branch", vars) RunCmdAndVerifyContainsText(t, Lakectl()+" local list", false, repoName+"/"+vars["BRANCH"], vars) From 2743e1ba0fa530a3f73c59963903abb6e8495d30 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Sun, 12 Jan 2025 15:42:03 +0100 Subject: [PATCH 36/58] with extre checks --- esti/lakectl_local_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 8d52e81c2d1..cc7ea7c1649 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -553,7 +553,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { // Commit changes to branch //runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) - RunCmdAndVerifyFailureContainsText(t, Lakectl()+" fs upload -s files/ro_1k lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+"fail_file.txt", false, "cannot write to protected branch", vars) + //RunCmdAndVerifyFailureContainsText(t, Lakectl()+" fs upload -s files/ro_1k lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+"fail_file.txt", false, "cannot write to protected branch", vars) RunCmdAndVerifyContainsText(t, Lakectl()+" local list", false, repoName+"/"+vars["BRANCH"], vars) RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+dataDir, false, "cannot write to protected branch", vars) From b64a79ec8744b639840d2ff59fbd405da960599d Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Sun, 12 Jan 2025 16:02:01 +0100 Subject: [PATCH 37/58] switched tests location --- esti/lakectl_local_test.go | 77 ------------------------------------- esti/lakectl_test.go | 78 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 77 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index cc7ea7c1649..21a8861d48b 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -486,83 +486,6 @@ func TestLakectlLocal_pull(t *testing.T) { } } -func TestLakectlLocal_commitProtetedBranch(t *testing.T) { - repoName := generateUniqueRepositoryName() - storage := generateUniqueStorageNamespace(repoName) - vars := map[string]string{ - "REPO": repoName, - "STORAGE": storage, - "BRANCH": mainBranch, - "REF": mainBranch, - } - runCmd(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, false, vars) - objects := []string{ - "ro_1k.1", - "ro_1k.2", - "ro_1k.3", - } - - tests := []struct { - name string - Delete bool - }{ - { - name: "test1", - Delete: true, - }, - { - name: "test2", - Delete: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tmpDir := t.TempDir() - fd, err := os.CreateTemp(tmpDir, "") - require.NoError(t, err) - require.NoError(t, fd.Close()) - dataDir, err := os.MkdirTemp(tmpDir, "") - require.NoError(t, err) - deleted := "deleted.png" - - vars["LOCAL_DIR"] = dataDir - vars["BRANCH"] = tt.name - vars["REF"] = tt.name - - runCmd(t, Lakectl()+" branch create lakefs://"+repoName+"/"+vars["BRANCH"]+" --source lakefs://"+repoName+"/"+mainBranch, false, false, vars) - localCreateTestData(t, vars, append(objects, deleted)) - runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ '*'", false, false, vars) - - RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+repoName+"/"+vars["BRANCH"]+"/ "+dataDir, false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) - - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "No diff found", vars) - - // Modify local folder - add or remove files - if tt.Delete { - require.NoError(t, os.Remove(filepath.Join(dataDir, deleted))) - - } else { - fd, err = os.Create(filepath.Join(dataDir, "test.txt")) - require.NoError(t, err) - require.NoError(t, fd.Close()) - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ test.txt", vars) - } - //runCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars["BRANCH"], vars) - str := runCmd(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars) - fmt.Println("itamar testing", str) - // Commit changes to branch - //runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) - - //RunCmdAndVerifyFailureContainsText(t, Lakectl()+" fs upload -s files/ro_1k lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+"fail_file.txt", false, "cannot write to protected branch", vars) - RunCmdAndVerifyContainsText(t, Lakectl()+" local list", false, repoName+"/"+vars["BRANCH"], vars) - - RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+dataDir, false, "cannot write to protected branch", vars) - //runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ \"*\"", false, false, vars) - - }) - } - -} func TestLakectlLocal_commit(t *testing.T) { tmpDir := t.TempDir() fd, err := os.CreateTemp(tmpDir, "") diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go index 57ca55ebb22..9b92dfd0df5 100644 --- a/esti/lakectl_test.go +++ b/esti/lakectl_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "path/filepath" "strings" "testing" "time" @@ -741,6 +742,83 @@ func TestLakectlFsUpload_protectedBranch(t *testing.T) { vars["FILE_PATH"] = "data/ro/ro_1k.0" RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, "cannot write to protected branch\n403 Forbidden\n", vars) } +func TestLakectlLocal_commitProtetedBranch(t *testing.T) { + repoName := generateUniqueRepositoryName() + storage := generateUniqueStorageNamespace(repoName) + vars := map[string]string{ + "REPO": repoName, + "STORAGE": storage, + "BRANCH": mainBranch, + "REF": mainBranch, + } + runCmd(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, false, vars) + objects := []string{ + "ro_1k.1", + "ro_1k.2", + "ro_1k.3", + } + + tests := []struct { + name string + Delete bool + }{ + { + name: "test1", + Delete: true, + }, + { + name: "test2", + Delete: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + fd, err := os.CreateTemp(tmpDir, "") + require.NoError(t, err) + require.NoError(t, fd.Close()) + dataDir, err := os.MkdirTemp(tmpDir, "") + require.NoError(t, err) + deleted := "deleted.png" + + vars["LOCAL_DIR"] = dataDir + vars["BRANCH"] = tt.name + vars["REF"] = tt.name + + runCmd(t, Lakectl()+" branch create lakefs://"+repoName+"/"+vars["BRANCH"]+" --source lakefs://"+repoName+"/"+mainBranch, false, false, vars) + localCreateTestData(t, vars, append(objects, deleted)) + runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ '*'", false, false, vars) + + RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+repoName+"/"+vars["BRANCH"]+"/ "+dataDir, false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) + + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "No diff found", vars) + + // Modify local folder - add or remove files + if tt.Delete { + require.NoError(t, os.Remove(filepath.Join(dataDir, deleted))) + + } else { + fd, err = os.Create(filepath.Join(dataDir, "test.txt")) + require.NoError(t, err) + require.NoError(t, fd.Close()) + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ test.txt", vars) + } + //runCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars["BRANCH"], vars) + str := runCmd(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars) + fmt.Println("itamar testing", str) + // Commit changes to branch + //runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) + + //RunCmdAndVerifyFailureContainsText(t, Lakectl()+" fs upload -s files/ro_1k lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+"fail_file.txt", false, "cannot write to protected branch", vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" local list", false, repoName+"/"+vars["BRANCH"], vars) + + RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+dataDir, false, "cannot write to protected branch", vars) + //runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ \"*\"", false, false, vars) + + }) + } + +} func TestLakectlFsPresign(t *testing.T) { config := getStorageConfig(t) From 7004bff89eac2343e8abb1a0a50af5d4a48f6a11 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Sun, 12 Jan 2025 16:15:32 +0100 Subject: [PATCH 38/58] switched tests location --- esti/lakectl_local_test.go | 77 +++++++++++++++++++++++++++++++++++++ esti/lakectl_test.go | 78 -------------------------------------- 2 files changed, 77 insertions(+), 78 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 21a8861d48b..7f800d5fba3 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -486,6 +486,83 @@ func TestLakectlLocal_pull(t *testing.T) { } } +func TestLakectlLocal_commitProtetedBranch(t *testing.T) { + repoName := generateUniqueRepositoryName() + storage := generateUniqueStorageNamespace(repoName) + vars := map[string]string{ + "REPO": repoName, + "STORAGE": storage, + "BRANCH": mainBranch, + "REF": mainBranch, + } + runCmd(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, false, vars) + objects := []string{ + "ro_1k.1", + "ro_1k.2", + "ro_1k.3", + } + + tests := []struct { + name string + Delete bool + }{ + { + name: "test1", + Delete: true, + }, + { + name: "test2", + Delete: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + fd, err := os.CreateTemp(tmpDir, "") + require.NoError(t, err) + require.NoError(t, fd.Close()) + dataDir, err := os.MkdirTemp(tmpDir, "") + require.NoError(t, err) + deleted := "deleted.png" + + vars["LOCAL_DIR"] = dataDir + vars["BRANCH"] = tt.name + vars["REF"] = tt.name + + runCmd(t, Lakectl()+" branch create lakefs://"+repoName+"/"+vars["BRANCH"]+" --source lakefs://"+repoName+"/"+mainBranch, false, false, vars) + localCreateTestData(t, vars, append(objects, deleted)) + runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ '*'", false, false, vars) + + RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+repoName+"/"+vars["BRANCH"]+"/ "+dataDir, false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) + + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "No diff found", vars) + + // Modify local folder - add or remove files + if tt.Delete { + require.NoError(t, os.Remove(filepath.Join(dataDir, deleted))) + + } else { + fd, err = os.Create(filepath.Join(dataDir, "test.txt")) + require.NoError(t, err) + require.NoError(t, fd.Close()) + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ test.txt", vars) + } + //runCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars["BRANCH"], vars) + str := runCmd(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars) + fmt.Println("itamar testing", str) + // Commit changes to branch + //runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) + + //RunCmdAndVerifyFailureContainsText(t, Lakectl()+" fs upload -s files/ro_1k lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+"fail_file.txt", false, "cannot write to protected branch", vars) + //RunCmdAndVerifyContainsText(t, Lakectl()+" local list", false, repoName+"/"+vars["BRANCH"], vars) + + RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+dataDir, false, "cannot write to protected branch", vars) + //runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ \"*\"", false, false, vars) + + }) + } + +} func TestLakectlLocal_commit(t *testing.T) { tmpDir := t.TempDir() fd, err := os.CreateTemp(tmpDir, "") diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go index 9b92dfd0df5..57ca55ebb22 100644 --- a/esti/lakectl_test.go +++ b/esti/lakectl_test.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "os" - "path/filepath" "strings" "testing" "time" @@ -742,83 +741,6 @@ func TestLakectlFsUpload_protectedBranch(t *testing.T) { vars["FILE_PATH"] = "data/ro/ro_1k.0" RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, "cannot write to protected branch\n403 Forbidden\n", vars) } -func TestLakectlLocal_commitProtetedBranch(t *testing.T) { - repoName := generateUniqueRepositoryName() - storage := generateUniqueStorageNamespace(repoName) - vars := map[string]string{ - "REPO": repoName, - "STORAGE": storage, - "BRANCH": mainBranch, - "REF": mainBranch, - } - runCmd(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, false, vars) - objects := []string{ - "ro_1k.1", - "ro_1k.2", - "ro_1k.3", - } - - tests := []struct { - name string - Delete bool - }{ - { - name: "test1", - Delete: true, - }, - { - name: "test2", - Delete: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tmpDir := t.TempDir() - fd, err := os.CreateTemp(tmpDir, "") - require.NoError(t, err) - require.NoError(t, fd.Close()) - dataDir, err := os.MkdirTemp(tmpDir, "") - require.NoError(t, err) - deleted := "deleted.png" - - vars["LOCAL_DIR"] = dataDir - vars["BRANCH"] = tt.name - vars["REF"] = tt.name - - runCmd(t, Lakectl()+" branch create lakefs://"+repoName+"/"+vars["BRANCH"]+" --source lakefs://"+repoName+"/"+mainBranch, false, false, vars) - localCreateTestData(t, vars, append(objects, deleted)) - runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ '*'", false, false, vars) - - RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+repoName+"/"+vars["BRANCH"]+"/ "+dataDir, false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) - - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "No diff found", vars) - - // Modify local folder - add or remove files - if tt.Delete { - require.NoError(t, os.Remove(filepath.Join(dataDir, deleted))) - - } else { - fd, err = os.Create(filepath.Join(dataDir, "test.txt")) - require.NoError(t, err) - require.NoError(t, fd.Close()) - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ test.txt", vars) - } - //runCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars["BRANCH"], vars) - str := runCmd(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars) - fmt.Println("itamar testing", str) - // Commit changes to branch - //runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) - - //RunCmdAndVerifyFailureContainsText(t, Lakectl()+" fs upload -s files/ro_1k lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+"fail_file.txt", false, "cannot write to protected branch", vars) - RunCmdAndVerifyContainsText(t, Lakectl()+" local list", false, repoName+"/"+vars["BRANCH"], vars) - - RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+dataDir, false, "cannot write to protected branch", vars) - //runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ \"*\"", false, false, vars) - - }) - } - -} func TestLakectlFsPresign(t *testing.T) { config := getStorageConfig(t) From c5b659960dd7c918464e30ce18df40df8c259a14 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Mon, 13 Jan 2025 10:24:18 +0100 Subject: [PATCH 39/58] reduced --- esti/lakectl_local_test.go | 94 +++++++++++++------------------------- 1 file changed, 31 insertions(+), 63 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 7f800d5fba3..a85b0b4fd2c 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -489,80 +489,48 @@ func TestLakectlLocal_pull(t *testing.T) { func TestLakectlLocal_commitProtetedBranch(t *testing.T) { repoName := generateUniqueRepositoryName() storage := generateUniqueStorageNamespace(repoName) - vars := map[string]string{ - "REPO": repoName, - "STORAGE": storage, - "BRANCH": mainBranch, - "REF": mainBranch, - } - runCmd(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, false, vars) - objects := []string{ - "ro_1k.1", - "ro_1k.2", - "ro_1k.3", - } + tmpDir := t.TempDir() + fd, err := os.CreateTemp(tmpDir, "") + require.NoError(t, err) + require.NoError(t, fd.Close()) + dataDir, err := os.MkdirTemp(tmpDir, "") + require.NoError(t, err) - tests := []struct { - name string - Delete bool - }{ - { - name: "test1", - Delete: true, - }, - { - name: "test2", - Delete: false, - }, + vars := map[string]string{ + "REPO": repoName, + "STORAGE": storage, + "BRANCH": mainBranch, + "REF": mainBranch, + "LOCAL_DIR": dataDir, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tmpDir := t.TempDir() - fd, err := os.CreateTemp(tmpDir, "") - require.NoError(t, err) - require.NoError(t, fd.Close()) - dataDir, err := os.MkdirTemp(tmpDir, "") - require.NoError(t, err) - deleted := "deleted.png" - - vars["LOCAL_DIR"] = dataDir - vars["BRANCH"] = tt.name - vars["REF"] = tt.name - - runCmd(t, Lakectl()+" branch create lakefs://"+repoName+"/"+vars["BRANCH"]+" --source lakefs://"+repoName+"/"+mainBranch, false, false, vars) - localCreateTestData(t, vars, append(objects, deleted)) - runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ '*'", false, false, vars) + runCmd(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+storage, false, false, vars) - RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+repoName+"/"+vars["BRANCH"]+"/ "+dataDir, false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) + runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "No diff found", vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/ "+vars["LOCAL_DIR"], false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "No diff found", vars) - // Modify local folder - add or remove files - if tt.Delete { - require.NoError(t, os.Remove(filepath.Join(dataDir, deleted))) + fd, err = os.Create(filepath.Join(dataDir, "test.txt")) + require.NoError(t, err) + require.NoError(t, fd.Close()) + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "local ║ added ║ test.txt", vars) - } else { - fd, err = os.Create(filepath.Join(dataDir, "test.txt")) - require.NoError(t, err) - require.NoError(t, fd.Close()) - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+dataDir, false, "local ║ added ║ test.txt", vars) - } - //runCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars["BRANCH"], vars) - str := runCmd(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, false, vars) - fmt.Println("itamar testing", str) - // Commit changes to branch - //runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) + runCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+vars["REPO"]+"/ ", false, false, "*", vars) + str := runCmd(t, Lakectl()+" branch-protect list lakefs://"+vars["REPO"]+"/ ", false, false, vars) + fmt.Println("itamar testing", str) + // Commit changes to branch + //runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) - //RunCmdAndVerifyFailureContainsText(t, Lakectl()+" fs upload -s files/ro_1k lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+"fail_file.txt", false, "cannot write to protected branch", vars) - //RunCmdAndVerifyContainsText(t, Lakectl()+" local list", false, repoName+"/"+vars["BRANCH"], vars) + //RunCmdAndVerifyFailureContainsText(t, Lakectl()+" fs upload -s files/ro_1k lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+"fail_file.txt", false, "cannot write to protected branch", vars) + //RunCmdAndVerifyContainsText(t, Lakectl()+" local list", false, repoName+"/"+vars["BRANCH"], vars) - RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+dataDir, false, "cannot write to protected branch", vars) - //runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ \"*\"", false, false, vars) + RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+vars["LOCAL_DIR"], false, "cannot write to protected branch", vars) + //RunCmdAndVerifyFailureContainsText(t, Lakectl()+" fs rm "+dataDir, false, "cannot write to protected branch", vars) - }) - } + //runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ \"*\"", false, false, vars) } + func TestLakectlLocal_commit(t *testing.T) { tmpDir := t.TempDir() fd, err := os.CreateTemp(tmpDir, "") From 99b25bc6de0b6998281cb75c2f3357bd599394b2 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Mon, 13 Jan 2025 10:50:08 +0100 Subject: [PATCH 40/58] works --- esti/lakectl_local_test.go | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index a85b0b4fd2c..75ef862f36a 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -503,32 +503,20 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { "REF": mainBranch, "LOCAL_DIR": dataDir, } - runCmd(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+storage, false, false, vars) + runCmd(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, false, vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) - + // Cloning local dir RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/ "+vars["LOCAL_DIR"], false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "No diff found", vars) - + // Adding file to local dir fd, err = os.Create(filepath.Join(dataDir, "test.txt")) require.NoError(t, err) require.NoError(t, fd.Close()) RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "local ║ added ║ test.txt", vars) - runCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+vars["REPO"]+"/ ", false, false, "*", vars) - str := runCmd(t, Lakectl()+" branch-protect list lakefs://"+vars["REPO"]+"/ ", false, false, vars) - fmt.Println("itamar testing", str) - // Commit changes to branch - //runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) - - //RunCmdAndVerifyFailureContainsText(t, Lakectl()+" fs upload -s files/ro_1k lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+"fail_file.txt", false, "cannot write to protected branch", vars) - //RunCmdAndVerifyContainsText(t, Lakectl()+" local list", false, repoName+"/"+vars["BRANCH"], vars) - + // Try to commit local dir, expect failure RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+vars["LOCAL_DIR"], false, "cannot write to protected branch", vars) - //RunCmdAndVerifyFailureContainsText(t, Lakectl()+" fs rm "+dataDir, false, "cannot write to protected branch", vars) - - //runCmd(t, Lakectl()+" branch-protect delete lakefs://"+repoName+"/ \"*\"", false, false, vars) - } func TestLakectlLocal_commit(t *testing.T) { From a62d71ae5ee457f6a2c5beb5f585136f0f71a952 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Mon, 13 Jan 2025 11:42:55 +0100 Subject: [PATCH 41/58] extended tests --- esti/lakectl_local_test.go | 6 ++++++ esti/lakectl_test.go | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 75ef862f36a..e49f3e4d68d 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -504,6 +504,8 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { "LOCAL_DIR": dataDir, } runCmd(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, false, vars) + vars["FILE_PATH"] = "ro_1k.0" + runCmd(t, Lakectl()+" fs upload lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, false, vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) // Cloning local dir @@ -517,6 +519,10 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { // Try to commit local dir, expect failure RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+vars["LOCAL_DIR"], false, "cannot write to protected branch", vars) + + // Try delete file from local dir and then commit + require.NoError(t, os.Remove(filepath.Join(dataDir, vars["FILE_PATH"]))) + RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+vars["LOCAL_DIR"], false, "cannot write to protected branch", vars) } func TestLakectlLocal_commit(t *testing.T) { diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go index 57ca55ebb22..bf53f663706 100644 --- a/esti/lakectl_test.go +++ b/esti/lakectl_test.go @@ -741,6 +741,22 @@ func TestLakectlFsUpload_protectedBranch(t *testing.T) { vars["FILE_PATH"] = "data/ro/ro_1k.0" RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, "cannot write to protected branch\n403 Forbidden\n", vars) } +func TestLakectlFsRm_protectedBranch(t *testing.T) { + repoName := generateUniqueRepositoryName() + storage := generateUniqueStorageNamespace(repoName) + vars := map[string]string{ + "REPO": repoName, + "STORAGE": storage, + "BRANCH": mainBranch, + } + RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, "lakectl_repo_create", vars) + vars["FILE_PATH"] = "data/ro/ro_1k.0" + runCmd(t, Lakectl()+" fs upload lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, false, vars) + runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ '*'", false, false, vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, "*", vars) + + RunCmdAndVerifyFailure(t, Lakectl()+" fs rm lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"], false, "cannot write to protected branch\n403 Forbidden\n", vars) +} func TestLakectlFsPresign(t *testing.T) { config := getStorageConfig(t) From b7cd2d9a001a551a337f4b629789f8536f51a0c4 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Mon, 13 Jan 2025 14:53:51 +0100 Subject: [PATCH 42/58] hopefuly now --- esti/lakectl_local_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index e49f3e4d68d..ee74002ffe3 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -506,6 +506,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { runCmd(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, false, vars) vars["FILE_PATH"] = "ro_1k.0" runCmd(t, Lakectl()+" fs upload lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, false, vars) + runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) // Cloning local dir From b33d289b81e3967328ff0df9f743f7e3f9ea05ff Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Mon, 13 Jan 2025 15:13:09 +0100 Subject: [PATCH 43/58] weirrrd --- esti/lakectl_local_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index ee74002ffe3..608e7533eee 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -504,9 +504,9 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { "LOCAL_DIR": dataDir, } runCmd(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, false, vars) - vars["FILE_PATH"] = "ro_1k.0" - runCmd(t, Lakectl()+" fs upload lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, false, vars) - runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) + // vars["FILE_PATH"] = "ro_1k.0" + // runCmd(t, Lakectl()+" fs upload lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, false, vars) + // runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) // Cloning local dir From 3bfb306ccc9a788367c3cf363a189ee97fe00d69 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Mon, 13 Jan 2025 15:21:38 +0100 Subject: [PATCH 44/58] maybe --- esti/lakectl_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go index bf53f663706..b89cb3da8d8 100644 --- a/esti/lakectl_test.go +++ b/esti/lakectl_test.go @@ -738,7 +738,7 @@ func TestLakectlFsUpload_protectedBranch(t *testing.T) { runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ '*'", false, false, vars) RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, "*", vars) - vars["FILE_PATH"] = "data/ro/ro_1k.0" + vars["FILE_PATH"] = "ro_1k.0" RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, "cannot write to protected branch\n403 Forbidden\n", vars) } func TestLakectlFsRm_protectedBranch(t *testing.T) { @@ -749,13 +749,13 @@ func TestLakectlFsRm_protectedBranch(t *testing.T) { "STORAGE": storage, "BRANCH": mainBranch, } - RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, "lakectl_repo_create", vars) - vars["FILE_PATH"] = "data/ro/ro_1k.0" - runCmd(t, Lakectl()+" fs upload lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, false, vars) - runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ '*'", false, false, vars) - RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, "*", vars) - - RunCmdAndVerifyFailure(t, Lakectl()+" fs rm lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"], false, "cannot write to protected branch\n403 Forbidden\n", vars) + RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, "lakectl_repo_create", vars) + vars["FILE_PATH"] = "ro_1k.0" + runCmd(t, Lakectl()+" fs upload lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, false, vars) + runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) + runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+vars["REPO"]+"/ ", false, "*", vars) + RunCmdAndVerifyFailure(t, Lakectl()+" fs rm lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"], false, "cannot write to protected branch\n403 Forbidden\n", vars) } func TestLakectlFsPresign(t *testing.T) { From 0d67997823c59768d1f0b78f8cd08147bb3ad00b Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Mon, 13 Jan 2025 19:03:12 +0100 Subject: [PATCH 45/58] lets seeee --- esti/lakectl_local_test.go | 8 +++++--- esti/lakectl_test.go | 10 ++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 608e7533eee..f87258951ad 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -495,6 +495,7 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { require.NoError(t, fd.Close()) dataDir, err := os.MkdirTemp(tmpDir, "") require.NoError(t, err) + file := "ro_1k.0" vars := map[string]string{ "REPO": repoName, @@ -502,11 +503,12 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { "BRANCH": mainBranch, "REF": mainBranch, "LOCAL_DIR": dataDir, + "FILE_PATH": file, } runCmd(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, false, vars) - // vars["FILE_PATH"] = "ro_1k.0" - // runCmd(t, Lakectl()+" fs upload lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, false, vars) - // runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) + + runCmd(t, Lakectl()+" fs upload lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, false, vars) + runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) // Cloning local dir diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go index b89cb3da8d8..abaa08c294c 100644 --- a/esti/lakectl_test.go +++ b/esti/lakectl_test.go @@ -744,13 +744,15 @@ func TestLakectlFsUpload_protectedBranch(t *testing.T) { func TestLakectlFsRm_protectedBranch(t *testing.T) { repoName := generateUniqueRepositoryName() storage := generateUniqueStorageNamespace(repoName) + file := "ro_1k.0" vars := map[string]string{ - "REPO": repoName, - "STORAGE": storage, - "BRANCH": mainBranch, + "REPO": repoName, + "STORAGE": storage, + "BRANCH": mainBranch, + "FILE_PATH": file, } + RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, "lakectl_repo_create", vars) - vars["FILE_PATH"] = "ro_1k.0" runCmd(t, Lakectl()+" fs upload lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, false, vars) runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) From 9d0d3c1b7747fdaa1819db876077685ec92dae08 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Mon, 13 Jan 2025 22:07:23 +0100 Subject: [PATCH 46/58] hi there yalla --- esti/lakectl_local_test.go | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index f87258951ad..a70140e4372 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -495,7 +495,6 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { require.NoError(t, fd.Close()) dataDir, err := os.MkdirTemp(tmpDir, "") require.NoError(t, err) - file := "ro_1k.0" vars := map[string]string{ "REPO": repoName, @@ -503,13 +502,8 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { "BRANCH": mainBranch, "REF": mainBranch, "LOCAL_DIR": dataDir, - "FILE_PATH": file, } runCmd(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, false, vars) - - runCmd(t, Lakectl()+" fs upload lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, false, vars) - runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) - runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) // Cloning local dir RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/ "+vars["LOCAL_DIR"], false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) @@ -522,12 +516,42 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { // Try to commit local dir, expect failure RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+vars["LOCAL_DIR"], false, "cannot write to protected branch", vars) +} +func TestLakectlLocal_RmcommitProtetedBranch(t *testing.T) { + repoName := generateUniqueRepositoryName() + storage := generateUniqueStorageNamespace(repoName) + tmpDir := t.TempDir() + fd, err := os.CreateTemp(tmpDir, "") + require.NoError(t, err) + require.NoError(t, fd.Close()) + dataDir, err := os.MkdirTemp(tmpDir, "") + require.NoError(t, err) + file := "ro_1k.0" + + vars := map[string]string{ + "REPO": repoName, + "STORAGE": storage, + "BRANCH": mainBranch, + "REF": mainBranch, + "LOCAL_DIR": dataDir, + "FILE_PATH": file, + } + runCmd(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, false, vars) + + runCmd(t, Lakectl()+" fs upload lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, false, vars) + runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) + + runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) + // Cloning local dir + RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/ "+vars["LOCAL_DIR"], false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "No diff found", vars) + + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "local ║ added ║ test.txt", vars) // Try delete file from local dir and then commit require.NoError(t, os.Remove(filepath.Join(dataDir, vars["FILE_PATH"]))) RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+vars["LOCAL_DIR"], false, "cannot write to protected branch", vars) } - func TestLakectlLocal_commit(t *testing.T) { tmpDir := t.TempDir() fd, err := os.CreateTemp(tmpDir, "") From c7bb775af30533efcca84dbb8b0a74274c2c7774 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Mon, 13 Jan 2025 22:54:09 +0100 Subject: [PATCH 47/58] yalla --- esti/lakectl_local_test.go | 4 +--- esti/lakectl_test.go | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index a70140e4372..8bff8c27233 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -526,7 +526,7 @@ func TestLakectlLocal_RmcommitProtetedBranch(t *testing.T) { require.NoError(t, fd.Close()) dataDir, err := os.MkdirTemp(tmpDir, "") require.NoError(t, err) - file := "ro_1k.0" + file := "data/ro/ro_1k.0" vars := map[string]string{ "REPO": repoName, @@ -546,8 +546,6 @@ func TestLakectlLocal_RmcommitProtetedBranch(t *testing.T) { RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/ "+vars["LOCAL_DIR"], false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "No diff found", vars) - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "local ║ added ║ test.txt", vars) - // Try delete file from local dir and then commit require.NoError(t, os.Remove(filepath.Join(dataDir, vars["FILE_PATH"]))) RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+vars["LOCAL_DIR"], false, "cannot write to protected branch", vars) diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go index abaa08c294c..f198c47ea4c 100644 --- a/esti/lakectl_test.go +++ b/esti/lakectl_test.go @@ -744,7 +744,7 @@ func TestLakectlFsUpload_protectedBranch(t *testing.T) { func TestLakectlFsRm_protectedBranch(t *testing.T) { repoName := generateUniqueRepositoryName() storage := generateUniqueStorageNamespace(repoName) - file := "ro_1k.0" + file := "data/ro_1k.0" vars := map[string]string{ "REPO": repoName, "STORAGE": storage, From eaaeb66ddd3107f487e7f9bfe3a29365a71d1428 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 15 Jan 2025 11:52:29 +0100 Subject: [PATCH 48/58] check if fs upload is the problem --- esti/lakectl_local_test.go | 19 ++++++++++++------- esti/lakectl_test.go | 2 ++ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 8bff8c27233..7f2a4ec7739 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -486,7 +486,7 @@ func TestLakectlLocal_pull(t *testing.T) { } } -func TestLakectlLocal_commitProtetedBranch(t *testing.T) { +func TestLakectlLocal_commitProtectedBranch(t *testing.T) { repoName := generateUniqueRepositoryName() storage := generateUniqueStorageNamespace(repoName) tmpDir := t.TempDir() @@ -517,7 +517,8 @@ func TestLakectlLocal_commitProtetedBranch(t *testing.T) { // Try to commit local dir, expect failure RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+vars["LOCAL_DIR"], false, "cannot write to protected branch", vars) } -func TestLakectlLocal_RmcommitProtetedBranch(t *testing.T) { + +func TestLakectlLocal_RmCommitProtectedBranch(t *testing.T) { repoName := generateUniqueRepositoryName() storage := generateUniqueStorageNamespace(repoName) tmpDir := t.TempDir() @@ -526,7 +527,7 @@ func TestLakectlLocal_RmcommitProtetedBranch(t *testing.T) { require.NoError(t, fd.Close()) dataDir, err := os.MkdirTemp(tmpDir, "") require.NoError(t, err) - file := "data/ro/ro_1k.0" + file := "ro_1k.0" vars := map[string]string{ "REPO": repoName, @@ -538,18 +539,22 @@ func TestLakectlLocal_RmcommitProtetedBranch(t *testing.T) { } runCmd(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, false, vars) - runCmd(t, Lakectl()+" fs upload lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, false, vars) - runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) - - runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) // Cloning local dir RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/ "+vars["LOCAL_DIR"], false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "No diff found", vars) + // locally add a file and commit + fd, err = os.Create(filepath.Join(dataDir, file)) + require.NoError(t, err) + require.NoError(t, fd.Close()) + RunCmdAndVerifyContainsText(t, Lakectl()+" local commit "+vars["LOCAL_DIR"], false, "Commit for branch \""+vars["BRANCH"]+"\" completed.", vars) + runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) + // Try delete file from local dir and then commit require.NoError(t, os.Remove(filepath.Join(dataDir, vars["FILE_PATH"]))) RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+vars["LOCAL_DIR"], false, "cannot write to protected branch", vars) } + func TestLakectlLocal_commit(t *testing.T) { tmpDir := t.TempDir() fd, err := os.CreateTemp(tmpDir, "") diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go index f198c47ea4c..4b2520969b8 100644 --- a/esti/lakectl_test.go +++ b/esti/lakectl_test.go @@ -726,6 +726,7 @@ func getStorageConfig(t *testing.T) *apigen.StorageConfig { } return storageResp.JSON200 } + func TestLakectlFsUpload_protectedBranch(t *testing.T) { repoName := generateUniqueRepositoryName() storage := generateUniqueStorageNamespace(repoName) @@ -741,6 +742,7 @@ func TestLakectlFsUpload_protectedBranch(t *testing.T) { vars["FILE_PATH"] = "ro_1k.0" RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, "cannot write to protected branch\n403 Forbidden\n", vars) } + func TestLakectlFsRm_protectedBranch(t *testing.T) { repoName := generateUniqueRepositoryName() storage := generateUniqueStorageNamespace(repoName) From e953d8fb10226f15d4e845886316c0f6d7af1941 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 15 Jan 2025 12:06:45 +0100 Subject: [PATCH 49/58] kadima --- esti/lakectl_local_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 7f2a4ec7739..ca08ad7f701 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -547,7 +547,7 @@ func TestLakectlLocal_RmCommitProtectedBranch(t *testing.T) { fd, err = os.Create(filepath.Join(dataDir, file)) require.NoError(t, err) require.NoError(t, fd.Close()) - RunCmdAndVerifyContainsText(t, Lakectl()+" local commit "+vars["LOCAL_DIR"], false, "Commit for branch \""+vars["BRANCH"]+"\" completed.", vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" local commit "+vars["LOCAL_DIR"]+" -m test", false, "Commit for branch \""+vars["BRANCH"]+"\" completed.", vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) // Try delete file from local dir and then commit From bc06a232fedab1be018fdc5e1e9fce53a6b5a327 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 15 Jan 2025 13:41:44 +0100 Subject: [PATCH 50/58] changes --- esti/lakectl_local_test.go | 9 ++++++--- esti/lakectl_test.go | 8 ++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index ca08ad7f701..c292179d4e3 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -495,6 +495,7 @@ func TestLakectlLocal_commitProtectedBranch(t *testing.T) { require.NoError(t, fd.Close()) dataDir, err := os.MkdirTemp(tmpDir, "") require.NoError(t, err) + file := "test.txt" vars := map[string]string{ "REPO": repoName, @@ -502,6 +503,7 @@ func TestLakectlLocal_commitProtectedBranch(t *testing.T) { "BRANCH": mainBranch, "REF": mainBranch, "LOCAL_DIR": dataDir, + "FILE": file, } runCmd(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, false, vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) @@ -509,7 +511,7 @@ func TestLakectlLocal_commitProtectedBranch(t *testing.T) { RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/ "+vars["LOCAL_DIR"], false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "No diff found", vars) // Adding file to local dir - fd, err = os.Create(filepath.Join(dataDir, "test.txt")) + fd, err = os.Create(filepath.Join(vars["LOCAL_DIR"], vars["FILE"])) require.NoError(t, err) require.NoError(t, fd.Close()) RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "local ║ added ║ test.txt", vars) @@ -544,14 +546,15 @@ func TestLakectlLocal_RmCommitProtectedBranch(t *testing.T) { RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "No diff found", vars) // locally add a file and commit - fd, err = os.Create(filepath.Join(dataDir, file)) + fd, err = os.Create(filepath.Join(vars["LOCAL_DIR"], vars["FILE_PATH"])) require.NoError(t, err) require.NoError(t, fd.Close()) RunCmdAndVerifyContainsText(t, Lakectl()+" local commit "+vars["LOCAL_DIR"]+" -m test", false, "Commit for branch \""+vars["BRANCH"]+"\" completed.", vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) // Try delete file from local dir and then commit - require.NoError(t, os.Remove(filepath.Join(dataDir, vars["FILE_PATH"]))) + require.NoError(t, os.Remove(filepath.Join(vars["LOCAL_DIR"], vars["FILE_PATH"]))) + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "local ║ removed ║ "+vars["FILE_PATH"], vars) RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+vars["LOCAL_DIR"], false, "cannot write to protected branch", vars) } diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go index 4b2520969b8..f109da54d3b 100644 --- a/esti/lakectl_test.go +++ b/esti/lakectl_test.go @@ -735,12 +735,12 @@ func TestLakectlFsUpload_protectedBranch(t *testing.T) { "STORAGE": storage, "BRANCH": mainBranch, } - RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, "lakectl_repo_create", vars) - runCmd(t, Lakectl()+" branch-protect add lakefs://"+repoName+"/ '*'", false, false, vars) - RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+repoName+"/ ", false, "*", vars) + RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, "lakectl_repo_create", vars) + runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+vars["REPO"]+"/ ", false, "*", vars) vars["FILE_PATH"] = "ro_1k.0" - RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, "cannot write to protected branch\n403 Forbidden\n", vars) + RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, "cannot write to protected branch\n403 Forbidden\n", vars) } func TestLakectlFsRm_protectedBranch(t *testing.T) { From 267cfed155f73f6a9ac0c6518df9031586d44d69 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 15 Jan 2025 15:06:27 +0100 Subject: [PATCH 51/58] check upload before rm --- esti/lakectl_test.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go index f109da54d3b..a46053ff631 100644 --- a/esti/lakectl_test.go +++ b/esti/lakectl_test.go @@ -747,11 +747,13 @@ func TestLakectlFsRm_protectedBranch(t *testing.T) { repoName := generateUniqueRepositoryName() storage := generateUniqueStorageNamespace(repoName) file := "data/ro_1k.0" + file2 := "data/ro_2k.0" vars := map[string]string{ - "REPO": repoName, - "STORAGE": storage, - "BRANCH": mainBranch, - "FILE_PATH": file, + "REPO": repoName, + "STORAGE": storage, + "BRANCH": mainBranch, + "FILE_PATH": file, + "FILE_PATH2": file2, } RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, "lakectl_repo_create", vars) @@ -759,6 +761,7 @@ func TestLakectlFsRm_protectedBranch(t *testing.T) { runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+vars["REPO"]+"/ ", false, "*", vars) + RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH2"]+" -s files/ro_1k", false, "cannot write to protected branch\n403 Forbidden\n", vars) RunCmdAndVerifyFailure(t, Lakectl()+" fs rm lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"], false, "cannot write to protected branch\n403 Forbidden\n", vars) } From 7e553d3b158f4a11aad272d22640860eeb2e4aac Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 15 Jan 2025 15:29:35 +0100 Subject: [PATCH 52/58] whats going on --- esti/lakectl_test.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go index a46053ff631..f30bb42ce2f 100644 --- a/esti/lakectl_test.go +++ b/esti/lakectl_test.go @@ -746,21 +746,19 @@ func TestLakectlFsUpload_protectedBranch(t *testing.T) { func TestLakectlFsRm_protectedBranch(t *testing.T) { repoName := generateUniqueRepositoryName() storage := generateUniqueStorageNamespace(repoName) - file := "data/ro_1k.0" - file2 := "data/ro_2k.0" vars := map[string]string{ - "REPO": repoName, - "STORAGE": storage, - "BRANCH": mainBranch, - "FILE_PATH": file, - "FILE_PATH2": file2, + "REPO": repoName, + "STORAGE": storage, + "BRANCH": mainBranch, } RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, "lakectl_repo_create", vars) + vars["FILE_PATH"] = "ro_1k.0" runCmd(t, Lakectl()+" fs upload lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, false, vars) runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+vars["REPO"]+"/ ", false, "*", vars) + vars["FILE_PATH2"] = "ro_2k.0" RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH2"]+" -s files/ro_1k", false, "cannot write to protected branch\n403 Forbidden\n", vars) RunCmdAndVerifyFailure(t, Lakectl()+" fs rm lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"], false, "cannot write to protected branch\n403 Forbidden\n", vars) } From 15212dfa24b36848a160e6c0766933d02a5cd89d Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 15 Jan 2025 15:46:52 +0100 Subject: [PATCH 53/58] changed name --- esti/lakectl_local_test.go | 20 ++++++++++---------- esti/lakectl_test.go | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index c292179d4e3..cf7c92a61e9 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -531,7 +531,7 @@ func TestLakectlLocal_RmCommitProtectedBranch(t *testing.T) { require.NoError(t, err) file := "ro_1k.0" - vars := map[string]string{ + variables := map[string]string{ "REPO": repoName, "STORAGE": storage, "BRANCH": mainBranch, @@ -539,23 +539,23 @@ func TestLakectlLocal_RmCommitProtectedBranch(t *testing.T) { "LOCAL_DIR": dataDir, "FILE_PATH": file, } - runCmd(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, false, vars) + runCmd(t, Lakectl()+" repo create lakefs://"+variables["REPO"]+" "+variables["STORAGE"], false, false, variables) // Cloning local dir - RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/ "+vars["LOCAL_DIR"], false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "No diff found", vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+variables["REPO"]+"/"+variables["BRANCH"]+"/ "+variables["LOCAL_DIR"], false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", variables) + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+variables["LOCAL_DIR"], false, "No diff found", variables) // locally add a file and commit - fd, err = os.Create(filepath.Join(vars["LOCAL_DIR"], vars["FILE_PATH"])) + fd, err = os.Create(filepath.Join(variables["LOCAL_DIR"], variables["FILE_PATH"])) require.NoError(t, err) require.NoError(t, fd.Close()) - RunCmdAndVerifyContainsText(t, Lakectl()+" local commit "+vars["LOCAL_DIR"]+" -m test", false, "Commit for branch \""+vars["BRANCH"]+"\" completed.", vars) - runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" local commit "+variables["LOCAL_DIR"]+" -m test", false, "Commit for branch \""+variables["BRANCH"]+"\" completed.", variables) + runCmd(t, Lakectl()+" branch-protect add lakefs://"+variables["REPO"]+"/ '*'", false, false, variables) // Try delete file from local dir and then commit - require.NoError(t, os.Remove(filepath.Join(vars["LOCAL_DIR"], vars["FILE_PATH"]))) - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "local ║ removed ║ "+vars["FILE_PATH"], vars) - RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+vars["LOCAL_DIR"], false, "cannot write to protected branch", vars) + require.NoError(t, os.Remove(filepath.Join(variables["LOCAL_DIR"], variables["FILE_PATH"]))) + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+variables["LOCAL_DIR"], false, "local ║ removed ║ "+variables["FILE_PATH"], variables) + RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+variables["LOCAL_DIR"], false, "cannot write to protected branch", variables) } func TestLakectlLocal_commit(t *testing.T) { diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go index f30bb42ce2f..7addaf6299c 100644 --- a/esti/lakectl_test.go +++ b/esti/lakectl_test.go @@ -746,21 +746,21 @@ func TestLakectlFsUpload_protectedBranch(t *testing.T) { func TestLakectlFsRm_protectedBranch(t *testing.T) { repoName := generateUniqueRepositoryName() storage := generateUniqueStorageNamespace(repoName) - vars := map[string]string{ + variables := map[string]string{ "REPO": repoName, "STORAGE": storage, "BRANCH": mainBranch, } - RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, "lakectl_repo_create", vars) - vars["FILE_PATH"] = "ro_1k.0" - runCmd(t, Lakectl()+" fs upload lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, false, vars) - runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) - runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) - RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+vars["REPO"]+"/ ", false, "*", vars) - vars["FILE_PATH2"] = "ro_2k.0" - RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH2"]+" -s files/ro_1k", false, "cannot write to protected branch\n403 Forbidden\n", vars) - RunCmdAndVerifyFailure(t, Lakectl()+" fs rm lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"], false, "cannot write to protected branch\n403 Forbidden\n", vars) + RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+variables["REPO"]+" "+variables["STORAGE"], false, "lakectl_repo_create", variables) + variables["FILE_PATH"] = "ro_1k.0" + runCmd(t, Lakectl()+" fs upload lakefs://"+variables["REPO"]+"/"+variables["BRANCH"]+"/"+variables["FILE_PATH"]+" -s files/ro_1k", false, false, variables) + runCmd(t, Lakectl()+" commit lakefs://"+variables["REPO"]+"/"+variables["BRANCH"]+" --allow-empty-message -m \" \"", false, false, variables) + runCmd(t, Lakectl()+" branch-protect add lakefs://"+variables["REPO"]+"/ '*'", false, false, variables) + RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+variables["REPO"]+"/ ", false, "*", variables) + variables["FILE_PATH2"] = "ro_2k.0" + RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+variables["REPO"]+"/"+variables["BRANCH"]+"/"+variables["FILE_PATH2"]+" -s files/ro_1k", false, "cannot write to protected branch\n403 Forbidden\n", variables) + RunCmdAndVerifyFailure(t, Lakectl()+" fs rm lakefs://"+variables["REPO"]+"/"+variables["BRANCH"]+"/"+variables["FILE_PATH"], false, "cannot write to protected branch\n403 Forbidden\n", variables) } func TestLakectlFsPresign(t *testing.T) { From 3d27ed31e35c7a54d565d5eb65d99f8c80f131a5 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 15 Jan 2025 16:00:48 +0100 Subject: [PATCH 54/58] with sleeping --- esti/lakectl_local_test.go | 26 +++++++++++++++----------- esti/lakectl_test.go | 23 ++++++++++++----------- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index cf7c92a61e9..3b0cd60f896 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/stretchr/testify/require" "github.com/treeverse/lakefs/pkg/block" @@ -507,6 +508,8 @@ func TestLakectlLocal_commitProtectedBranch(t *testing.T) { } runCmd(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, false, vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) + // sleep in order to overcome branch-protect caching problem + time.Sleep(10 * time.Second) // Cloning local dir RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/ "+vars["LOCAL_DIR"], false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "No diff found", vars) @@ -515,7 +518,6 @@ func TestLakectlLocal_commitProtectedBranch(t *testing.T) { require.NoError(t, err) require.NoError(t, fd.Close()) RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "local ║ added ║ test.txt", vars) - // Try to commit local dir, expect failure RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+vars["LOCAL_DIR"], false, "cannot write to protected branch", vars) } @@ -531,7 +533,7 @@ func TestLakectlLocal_RmCommitProtectedBranch(t *testing.T) { require.NoError(t, err) file := "ro_1k.0" - variables := map[string]string{ + vars := map[string]string{ "REPO": repoName, "STORAGE": storage, "BRANCH": mainBranch, @@ -539,23 +541,25 @@ func TestLakectlLocal_RmCommitProtectedBranch(t *testing.T) { "LOCAL_DIR": dataDir, "FILE_PATH": file, } - runCmd(t, Lakectl()+" repo create lakefs://"+variables["REPO"]+" "+variables["STORAGE"], false, false, variables) + runCmd(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, false, vars) // Cloning local dir - RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+variables["REPO"]+"/"+variables["BRANCH"]+"/ "+variables["LOCAL_DIR"], false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", variables) - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+variables["LOCAL_DIR"], false, "No diff found", variables) + RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/ "+vars["LOCAL_DIR"], false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "No diff found", vars) // locally add a file and commit - fd, err = os.Create(filepath.Join(variables["LOCAL_DIR"], variables["FILE_PATH"])) + fd, err = os.Create(filepath.Join(vars["LOCAL_DIR"], vars["FILE_PATH"])) require.NoError(t, err) require.NoError(t, fd.Close()) - RunCmdAndVerifyContainsText(t, Lakectl()+" local commit "+variables["LOCAL_DIR"]+" -m test", false, "Commit for branch \""+variables["BRANCH"]+"\" completed.", variables) - runCmd(t, Lakectl()+" branch-protect add lakefs://"+variables["REPO"]+"/ '*'", false, false, variables) + RunCmdAndVerifyContainsText(t, Lakectl()+" local commit "+vars["LOCAL_DIR"]+" -m test", false, "Commit for branch \""+vars["BRANCH"]+"\" completed.", vars) + runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) + // sleep in order to overcome branch-protect caching problem + time.Sleep(10 * time.Second) // Try delete file from local dir and then commit - require.NoError(t, os.Remove(filepath.Join(variables["LOCAL_DIR"], variables["FILE_PATH"]))) - RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+variables["LOCAL_DIR"], false, "local ║ removed ║ "+variables["FILE_PATH"], variables) - RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+variables["LOCAL_DIR"], false, "cannot write to protected branch", variables) + require.NoError(t, os.Remove(filepath.Join(vars["LOCAL_DIR"], vars["FILE_PATH"]))) + RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "local ║ removed ║ "+vars["FILE_PATH"], vars) + RunCmdAndVerifyFailureContainsText(t, Lakectl()+" local commit -m test "+vars["LOCAL_DIR"], false, "cannot write to protected branch", vars) } func TestLakectlLocal_commit(t *testing.T) { diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go index 7addaf6299c..7ae315a10f2 100644 --- a/esti/lakectl_test.go +++ b/esti/lakectl_test.go @@ -738,7 +738,8 @@ func TestLakectlFsUpload_protectedBranch(t *testing.T) { RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, "lakectl_repo_create", vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+vars["REPO"]+"/ ", false, "*", vars) - + // sleep in order to overcome branch-protect caching problem + time.Sleep(10 * time.Second) vars["FILE_PATH"] = "ro_1k.0" RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, "cannot write to protected branch\n403 Forbidden\n", vars) } @@ -746,21 +747,21 @@ func TestLakectlFsUpload_protectedBranch(t *testing.T) { func TestLakectlFsRm_protectedBranch(t *testing.T) { repoName := generateUniqueRepositoryName() storage := generateUniqueStorageNamespace(repoName) - variables := map[string]string{ + vars := map[string]string{ "REPO": repoName, "STORAGE": storage, "BRANCH": mainBranch, } - RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+variables["REPO"]+" "+variables["STORAGE"], false, "lakectl_repo_create", variables) - variables["FILE_PATH"] = "ro_1k.0" - runCmd(t, Lakectl()+" fs upload lakefs://"+variables["REPO"]+"/"+variables["BRANCH"]+"/"+variables["FILE_PATH"]+" -s files/ro_1k", false, false, variables) - runCmd(t, Lakectl()+" commit lakefs://"+variables["REPO"]+"/"+variables["BRANCH"]+" --allow-empty-message -m \" \"", false, false, variables) - runCmd(t, Lakectl()+" branch-protect add lakefs://"+variables["REPO"]+"/ '*'", false, false, variables) - RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+variables["REPO"]+"/ ", false, "*", variables) - variables["FILE_PATH2"] = "ro_2k.0" - RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+variables["REPO"]+"/"+variables["BRANCH"]+"/"+variables["FILE_PATH2"]+" -s files/ro_1k", false, "cannot write to protected branch\n403 Forbidden\n", variables) - RunCmdAndVerifyFailure(t, Lakectl()+" fs rm lakefs://"+variables["REPO"]+"/"+variables["BRANCH"]+"/"+variables["FILE_PATH"], false, "cannot write to protected branch\n403 Forbidden\n", variables) + RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, "lakectl_repo_create", vars) + vars["FILE_PATH"] = "ro_1k.0" + runCmd(t, Lakectl()+" fs upload lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, false, vars) + runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) + runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) + // sleep in order to overcome branch-protect caching problem + time.Sleep(10 * time.Second) + RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+vars["REPO"]+"/ ", false, "*", vars) + RunCmdAndVerifyFailure(t, Lakectl()+" fs rm lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"], false, "cannot write to protected branch\n403 Forbidden\n", vars) } func TestLakectlFsPresign(t *testing.T) { From b07c7bc711c8d29485ab27224afd4322a0b736c1 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 15 Jan 2025 16:21:45 +0100 Subject: [PATCH 55/58] with sleeping --- esti/lakectl_local_test.go | 8 ++++---- esti/lakectl_test.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 3b0cd60f896..71804ca83e5 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -508,8 +508,8 @@ func TestLakectlLocal_commitProtectedBranch(t *testing.T) { } runCmd(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, false, vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) - // sleep in order to overcome branch-protect caching problem - time.Sleep(10 * time.Second) + // BranchUpdateMaxInterval - sleep in order to overcome branch update caching + time.Sleep(6 * time.Second) // Cloning local dir RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/ "+vars["LOCAL_DIR"], false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "No diff found", vars) @@ -553,8 +553,8 @@ func TestLakectlLocal_RmCommitProtectedBranch(t *testing.T) { require.NoError(t, fd.Close()) RunCmdAndVerifyContainsText(t, Lakectl()+" local commit "+vars["LOCAL_DIR"]+" -m test", false, "Commit for branch \""+vars["BRANCH"]+"\" completed.", vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) - // sleep in order to overcome branch-protect caching problem - time.Sleep(10 * time.Second) + // BranchUpdateMaxInterval - sleep in order to overcome branch update caching + time.Sleep(6 * time.Second) // Try delete file from local dir and then commit require.NoError(t, os.Remove(filepath.Join(vars["LOCAL_DIR"], vars["FILE_PATH"]))) diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go index 7ae315a10f2..6ef297a80c8 100644 --- a/esti/lakectl_test.go +++ b/esti/lakectl_test.go @@ -738,8 +738,8 @@ func TestLakectlFsUpload_protectedBranch(t *testing.T) { RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, "lakectl_repo_create", vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+vars["REPO"]+"/ ", false, "*", vars) - // sleep in order to overcome branch-protect caching problem - time.Sleep(10 * time.Second) + // BranchUpdateMaxInterval - sleep in order to overcome branch update caching + time.Sleep(6 * time.Second) vars["FILE_PATH"] = "ro_1k.0" RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, "cannot write to protected branch\n403 Forbidden\n", vars) } @@ -758,8 +758,8 @@ func TestLakectlFsRm_protectedBranch(t *testing.T) { runCmd(t, Lakectl()+" fs upload lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, false, vars) runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) - // sleep in order to overcome branch-protect caching problem - time.Sleep(10 * time.Second) + // BranchUpdateMaxInterval - sleep in order to overcome branch update caching + time.Sleep(6 * time.Second) RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+vars["REPO"]+"/ ", false, "*", vars) RunCmdAndVerifyFailure(t, Lakectl()+" fs rm lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"], false, "cannot write to protected branch\n403 Forbidden\n", vars) } From 7e839e509009f160644a274d6fc14455d0667e7c Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 15 Jan 2025 18:08:57 +0100 Subject: [PATCH 56/58] yallla --- esti/lakectl_local_test.go | 5 +++-- esti/lakectl_test.go | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 71804ca83e5..de00cfc95dd 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/treeverse/lakefs/pkg/block" "github.com/treeverse/lakefs/pkg/fileutil" + "github.com/treeverse/lakefs/pkg/graveler" "github.com/treeverse/lakefs/pkg/local" "github.com/treeverse/lakefs/pkg/uri" "golang.org/x/exp/slices" @@ -509,7 +510,7 @@ func TestLakectlLocal_commitProtectedBranch(t *testing.T) { runCmd(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, false, vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) // BranchUpdateMaxInterval - sleep in order to overcome branch update caching - time.Sleep(6 * time.Second) + time.Sleep(graveler.BranchUpdateMaxInterval * time.Second) // Cloning local dir RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/ "+vars["LOCAL_DIR"], false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "No diff found", vars) @@ -554,7 +555,7 @@ func TestLakectlLocal_RmCommitProtectedBranch(t *testing.T) { RunCmdAndVerifyContainsText(t, Lakectl()+" local commit "+vars["LOCAL_DIR"]+" -m test", false, "Commit for branch \""+vars["BRANCH"]+"\" completed.", vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) // BranchUpdateMaxInterval - sleep in order to overcome branch update caching - time.Sleep(6 * time.Second) + time.Sleep(graveler.BranchUpdateMaxInterval * time.Second) // Try delete file from local dir and then commit require.NoError(t, os.Remove(filepath.Join(vars["LOCAL_DIR"], vars["FILE_PATH"]))) diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go index 6ef297a80c8..b27dcf428ed 100644 --- a/esti/lakectl_test.go +++ b/esti/lakectl_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" "github.com/treeverse/lakefs/pkg/api/apigen" + "github.com/treeverse/lakefs/pkg/graveler" ) var emptyVars = make(map[string]string) @@ -739,7 +740,7 @@ func TestLakectlFsUpload_protectedBranch(t *testing.T) { runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+vars["REPO"]+"/ ", false, "*", vars) // BranchUpdateMaxInterval - sleep in order to overcome branch update caching - time.Sleep(6 * time.Second) + time.Sleep(graveler.BranchUpdateMaxInterval * time.Second) vars["FILE_PATH"] = "ro_1k.0" RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, "cannot write to protected branch\n403 Forbidden\n", vars) } @@ -759,7 +760,7 @@ func TestLakectlFsRm_protectedBranch(t *testing.T) { runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) // BranchUpdateMaxInterval - sleep in order to overcome branch update caching - time.Sleep(6 * time.Second) + time.Sleep(graveler.BranchUpdateMaxInterval * time.Second) RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+vars["REPO"]+"/ ", false, "*", vars) RunCmdAndVerifyFailure(t, Lakectl()+" fs rm lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"], false, "cannot write to protected branch\n403 Forbidden\n", vars) } From 1f52a580f5336c2c0b2ca4495795e4088882a574 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Wed, 15 Jan 2025 18:46:05 +0100 Subject: [PATCH 57/58] yallla --- esti/lakectl_local_test.go | 5 ++--- esti/lakectl_test.go | 6 ++++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index de00cfc95dd..818ec020b36 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -14,7 +14,6 @@ import ( "github.com/stretchr/testify/require" "github.com/treeverse/lakefs/pkg/block" "github.com/treeverse/lakefs/pkg/fileutil" - "github.com/treeverse/lakefs/pkg/graveler" "github.com/treeverse/lakefs/pkg/local" "github.com/treeverse/lakefs/pkg/uri" "golang.org/x/exp/slices" @@ -510,7 +509,7 @@ func TestLakectlLocal_commitProtectedBranch(t *testing.T) { runCmd(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, false, vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) // BranchUpdateMaxInterval - sleep in order to overcome branch update caching - time.Sleep(graveler.BranchUpdateMaxInterval * time.Second) + time.Sleep(branchProtectTimeout * time.Second) // Cloning local dir RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/ "+vars["LOCAL_DIR"], false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "No diff found", vars) @@ -555,7 +554,7 @@ func TestLakectlLocal_RmCommitProtectedBranch(t *testing.T) { RunCmdAndVerifyContainsText(t, Lakectl()+" local commit "+vars["LOCAL_DIR"]+" -m test", false, "Commit for branch \""+vars["BRANCH"]+"\" completed.", vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) // BranchUpdateMaxInterval - sleep in order to overcome branch update caching - time.Sleep(graveler.BranchUpdateMaxInterval * time.Second) + time.Sleep(branchProtectTimeout * time.Second) // Try delete file from local dir and then commit require.NoError(t, os.Remove(filepath.Join(vars["LOCAL_DIR"], vars["FILE_PATH"]))) diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go index b27dcf428ed..d09a331eadb 100644 --- a/esti/lakectl_test.go +++ b/esti/lakectl_test.go @@ -15,6 +15,8 @@ import ( var emptyVars = make(map[string]string) +const branchProtectTimeout = graveler.BranchUpdateMaxInterval + 1 + func TestLakectlHelp(t *testing.T) { RunCmdAndVerifySuccessWithFile(t, Lakectl(), false, "lakectl_help", emptyVars) RunCmdAndVerifySuccessWithFile(t, Lakectl()+" --help", false, "lakectl_help", emptyVars) @@ -740,7 +742,7 @@ func TestLakectlFsUpload_protectedBranch(t *testing.T) { runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+vars["REPO"]+"/ ", false, "*", vars) // BranchUpdateMaxInterval - sleep in order to overcome branch update caching - time.Sleep(graveler.BranchUpdateMaxInterval * time.Second) + time.Sleep(branchProtectTimeout * time.Second) vars["FILE_PATH"] = "ro_1k.0" RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, "cannot write to protected branch\n403 Forbidden\n", vars) } @@ -760,7 +762,7 @@ func TestLakectlFsRm_protectedBranch(t *testing.T) { runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) // BranchUpdateMaxInterval - sleep in order to overcome branch update caching - time.Sleep(graveler.BranchUpdateMaxInterval * time.Second) + time.Sleep(branchProtectTimeout * time.Second) RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+vars["REPO"]+"/ ", false, "*", vars) RunCmdAndVerifyFailure(t, Lakectl()+" fs rm lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"], false, "cannot write to protected branch\n403 Forbidden\n", vars) } From 1b64392dde40fb9f438b54c792560ad81fe0fb89 Mon Sep 17 00:00:00 2001 From: Itamar Yuran Date: Sun, 19 Jan 2025 09:34:59 +0100 Subject: [PATCH 58/58] yallla --- esti/lakectl_local_test.go | 5 ++--- esti/lakectl_test.go | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/esti/lakectl_local_test.go b/esti/lakectl_local_test.go index 818ec020b36..6a9bc18b5c4 100644 --- a/esti/lakectl_local_test.go +++ b/esti/lakectl_local_test.go @@ -509,7 +509,7 @@ func TestLakectlLocal_commitProtectedBranch(t *testing.T) { runCmd(t, Lakectl()+" repo create lakefs://"+vars["REPO"]+" "+vars["STORAGE"], false, false, vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) // BranchUpdateMaxInterval - sleep in order to overcome branch update caching - time.Sleep(branchProtectTimeout * time.Second) + time.Sleep(branchProtectTimeout) // Cloning local dir RunCmdAndVerifyContainsText(t, Lakectl()+" local clone lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/ "+vars["LOCAL_DIR"], false, "Successfully cloned lakefs://${REPO}/${REF}/ to ${LOCAL_DIR}.", vars) RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "No diff found", vars) @@ -554,8 +554,7 @@ func TestLakectlLocal_RmCommitProtectedBranch(t *testing.T) { RunCmdAndVerifyContainsText(t, Lakectl()+" local commit "+vars["LOCAL_DIR"]+" -m test", false, "Commit for branch \""+vars["BRANCH"]+"\" completed.", vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) // BranchUpdateMaxInterval - sleep in order to overcome branch update caching - time.Sleep(branchProtectTimeout * time.Second) - + time.Sleep(branchProtectTimeout) // Try delete file from local dir and then commit require.NoError(t, os.Remove(filepath.Join(vars["LOCAL_DIR"], vars["FILE_PATH"]))) RunCmdAndVerifyContainsText(t, Lakectl()+" local status "+vars["LOCAL_DIR"], false, "local ║ removed ║ "+vars["FILE_PATH"], vars) diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go index d09a331eadb..d81ab66db7c 100644 --- a/esti/lakectl_test.go +++ b/esti/lakectl_test.go @@ -15,7 +15,7 @@ import ( var emptyVars = make(map[string]string) -const branchProtectTimeout = graveler.BranchUpdateMaxInterval + 1 +const branchProtectTimeout = graveler.BranchUpdateMaxInterval + time.Second func TestLakectlHelp(t *testing.T) { RunCmdAndVerifySuccessWithFile(t, Lakectl(), false, "lakectl_help", emptyVars) @@ -742,7 +742,7 @@ func TestLakectlFsUpload_protectedBranch(t *testing.T) { runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+vars["REPO"]+"/ ", false, "*", vars) // BranchUpdateMaxInterval - sleep in order to overcome branch update caching - time.Sleep(branchProtectTimeout * time.Second) + time.Sleep(branchProtectTimeout) vars["FILE_PATH"] = "ro_1k.0" RunCmdAndVerifyFailure(t, Lakectl()+" fs upload lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"]+" -s files/ro_1k", false, "cannot write to protected branch\n403 Forbidden\n", vars) } @@ -762,7 +762,7 @@ func TestLakectlFsRm_protectedBranch(t *testing.T) { runCmd(t, Lakectl()+" commit lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+" --allow-empty-message -m \" \"", false, false, vars) runCmd(t, Lakectl()+" branch-protect add lakefs://"+vars["REPO"]+"/ '*'", false, false, vars) // BranchUpdateMaxInterval - sleep in order to overcome branch update caching - time.Sleep(branchProtectTimeout * time.Second) + time.Sleep(branchProtectTimeout) RunCmdAndVerifyContainsText(t, Lakectl()+" branch-protect list lakefs://"+vars["REPO"]+"/ ", false, "*", vars) RunCmdAndVerifyFailure(t, Lakectl()+" fs rm lakefs://"+vars["REPO"]+"/"+vars["BRANCH"]+"/"+vars["FILE_PATH"], false, "cannot write to protected branch\n403 Forbidden\n", vars) }