Size tracking component

.github/workflows/code-coverage.yml

@@ -254,6 +254,37 @@ jobs:
           sudo fusermount -u ${{ env.MOUNT_DIR }}
           sleep 5
 
+      - name: Cleanup S3 Storage before size tracker test
+        run: go test -timeout 120m -v test/s3cleanup/s3cleanup_test.go
+        env:
+          AWS_ACCESS_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}"
+          AWS_SECRET_ACCESS_KEY: "${{ secrets.S3TEST_SECRET_KEY }}"
+          AWS_REGION: "${{ secrets.S3TEST_REGION }}"
+          S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}"
+          S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}"
+
+      - name: Create Config File - S3 with Size Tracker
+        env:
+          S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}"
+          S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}"
+          S3_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}"
+          S3_REGION: "${{ secrets.S3TEST_REGION }}"
+          S3_SECRET_KEY: "${{ secrets.S3TEST_SECRET_KEY }}"
+        run: "./cloudfuse.test gen-test-config --config-file=s3_key_size_tracker.yaml --temp-path=${{ env.TEMP_DIR }} --output-file=${{ env.cloudfuse_CFG }}"
+
+      - name: S3 Coverage with Size Tracker
+        run: |-
+          rm -rf ${{ env.MOUNT_DIR }}/*
+          rm -rf ${{ env.TEMP_DIR }}/*
+          ./cloudfuse.test -test.v -test.coverprofile=${{ env.WORK_DIR }}/cloudfuse_s3.cov mount ${{ env.MOUNT_DIR }} --config-file=${{ env.cloudfuse_CFG }} --foreground=true &
+          sleep 10
+          ps -aux | grep cloudfuse
+          cd test/e2e_tests
+          go test -v -timeout=7200s ./... -args -mnt-path=${{ env.MOUNT_DIR }} -tmp-path=${{ env.TEMP_DIR }} -size-tracker=true
+          cd -
+          sudo fusermount -u ${{ env.MOUNT_DIR }}
+          sleep 5
+
       - name: Create Config File - Stream
         env:
           NIGHTLY_STO_ACC_NAME: "${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}"
@@ -782,6 +813,39 @@ jobs:
           kill $pid
           sleep 5
 
+      - name: Cleanup S3 Storage before size tracker test
+        run: go test -timeout 120m -v test/s3cleanup/s3cleanup_test.go
+        env:
+          AWS_ACCESS_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}"
+          AWS_SECRET_ACCESS_KEY: "${{ secrets.S3TEST_SECRET_KEY }}"
+          AWS_REGION: "${{ secrets.S3TEST_REGION }}"
+          S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}"
+          S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}"
+
+      - name: Create Config File - S3 with Size Tracker
+        shell: bash
+        env:
+          S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}"
+          S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}"
+          S3_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}"
+          S3_REGION: "${{ secrets.S3TEST_REGION }}"
+          S3_SECRET_KEY: "${{ secrets.S3TEST_SECRET_KEY }}"
+        run: "./cloudfuse.test gen-test-config --config-file=s3_key_size_tracker.yaml --temp-path=${{ env.TEMP_DIR }} --output-file=${{ env.cloudfuse_CFG }}"
+
+      - name: S3 Coverage with Size Tracker
+        shell: bash
+        run: |-
+          rm -rf ${{ env.MOUNT_DIR }}/*
+          rm -rf ${{ env.TEMP_DIR }}/*
+          ./cloudfuse.test -test.v -test.coverprofile=${{ env.WORK_DIR }}/cloudfuse_s3.cov mount ${{ env.MOUNT_DIR }} --config-file=${{ env.cloudfuse_CFG }} --foreground=true &
+          sleep 10
+          pid=`ps -a | grep cloudfuse | tr -s ' ' | cut -d ' ' -f2`
+          cd test/e2e_tests
+          go test -v -timeout=7200s ./... -args -mnt-path=${{ env.MOUNT_DIR }} -tmp-path=${{ env.TEMP_DIR }} -quick-test=true -stream-direct-test=true -size-tracker=true
+          cd -
+          kill $pid
+          sleep 5
+
       - name: Create Config File - Stream
         shell: bash
         env:
@@ -1050,3 +1114,12 @@ jobs:
         env:
           STORAGE_ACCOUNT_NAME: "${{ secrets.AZTEST_ADLS_ACC_NAME }}"
           STORAGE_ACCOUNT_Key: "${{ secrets.AZTEST_ADLS_KEY }}"
+
+      - name: Cleanup S3 Storage
+        run: go test -timeout 120m -v test/s3cleanup/s3cleanup_test.go
+        env:
+          AWS_ACCESS_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}"
+          AWS_SECRET_ACCESS_KEY: "${{ secrets.S3TEST_SECRET_KEY }}"
+          AWS_REGION: "${{ secrets.S3TEST_REGION }}"
+          S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}"
+          S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}"

cmd/import_windows.go

@@ -34,5 +34,6 @@ import (
 	_ "github.com/Seagate/cloudfuse/component/libfuse"
 	_ "github.com/Seagate/cloudfuse/component/loopback"
 	_ "github.com/Seagate/cloudfuse/component/s3storage"
+	_ "github.com/Seagate/cloudfuse/component/size_tracker"
 	_ "github.com/Seagate/cloudfuse/component/stream"
 )

cmd/imports_linux.go

@@ -35,5 +35,6 @@ import (
 	_ "github.com/Seagate/cloudfuse/component/libfuse"
 	_ "github.com/Seagate/cloudfuse/component/loopback"
 	_ "github.com/Seagate/cloudfuse/component/s3storage"
+	_ "github.com/Seagate/cloudfuse/component/size_tracker"
 	_ "github.com/Seagate/cloudfuse/component/stream"
 )

cmd/mount.go

@@ -158,16 +158,9 @@ func (opt *mountOptions) validate(skipNonEmptyMount bool) error {
 		common.DefaultLogFilePath = common.JoinUnixFilepath(common.DefaultWorkDir, "cloudfuse.log")
 	}
 
-	f, err := os.Stat(common.ExpandPath(common.DefaultWorkDir))
-	if err == nil && !f.IsDir() {
-		return fmt.Errorf("default work dir '%s' is not a directory", common.DefaultWorkDir)
-	}
-
-	if err != nil && os.IsNotExist(err) {
-		// create the default work dir
-		if err = os.MkdirAll(common.ExpandPath(common.DefaultWorkDir), 0755); err != nil {
-			return fmt.Errorf("failed to create default work dir [%s]", err.Error())
-		}
+	err := common.CreateDefaultDirectory()
+	if err != nil {
+		return fmt.Errorf("Failed to create default work dir [%s]", err.Error())
 	}
 
 	opt.Logging.LogFilePath = common.ExpandPath(opt.Logging.LogFilePath)

common/util.go

@@ -369,3 +369,18 @@ func IsDriveLetter(path string) bool {
 	match, _ := regexp.MatchString(pattern, path)
 	return match
 }
+
+func CreateDefaultDirectory() error {
+	dir, err := os.Stat(ExpandPath(DefaultWorkDir))
+	if err == nil && !dir.IsDir() {
+		return err
+	}
+
+	if err != nil && os.IsNotExist(err) {
+		// create the default work dir
+		if err = os.MkdirAll(ExpandPath(DefaultWorkDir), 0755); err != nil {
+			return err
+		}
+	}
+	return nil
+}

component/loopback/loopback_fs.go

@@ -182,6 +182,7 @@ func (lfs *LoopbackFS) CreateFile(options internal.CreateFileOptions) (*handlema
 	}
 	handle := handlemap.NewHandle(options.Name)
 	handle.SetFileObject(f)
+	handlemap.Add(handle)
 
 	return handle, nil
 }
@@ -212,6 +213,7 @@ func (lfs *LoopbackFS) OpenFile(options internal.OpenFileOptions) (*handlemap.Ha
 	}
 	handle := handlemap.NewHandle(options.Name)
 	handle.SetFileObject(f)
+	handlemap.Add(handle)
 	return handle, nil
 }
 
@@ -224,14 +226,23 @@ func (lfs *LoopbackFS) CloseFile(options internal.CloseFileOptions) error {
 		return syscall.EBADF
 	}
 
+	handlemap.Delete(options.Handle.ID)
 	return f.Close()
 }
 
 func (lfs *LoopbackFS) RenameFile(options internal.RenameFileOptions) error {
 	log.Trace("LoopbackFS::RenameFile : %s -> %s", options.Src, options.Dst)
 	oldPath := filepath.Join(lfs.path, options.Src)
 	newPath := filepath.Join(lfs.path, options.Dst)
-	return os.Rename(oldPath, newPath)
+	err := os.Rename(oldPath, newPath)
+	handlemap.GetHandles().Range(func(key, value any) bool {
+		handle := value.(*handlemap.Handle)
+		if handle.Path == options.Src {
+			handle.Path = options.Dst
+		}
+		return true
+	})
+	return err
 }
 
 func (lfs *LoopbackFS) ReadLink(options internal.ReadLinkOptions) (string, error) {
@@ -266,10 +277,6 @@ func (lfs *LoopbackFS) ReadInBuffer(options internal.ReadInBufferOptions) (int,
 	options.Handle.RLock()
 	defer options.Handle.RUnlock()
 
-	if f == nil {
-		log.Err("LoopbackFS::ReadInBuffer : error [invalid file object]")
-		return 0, os.ErrInvalid
-	}
 	return f.ReadAt(options.Data, options.Offset)
 }