From a88d7c54a647349ac97f7f12e8a06d87a10912d0 Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Fri, 17 Jan 2025 16:40:20 -0700 Subject: [PATCH 01/23] Initial commit for size tracking --- cmd/import_windows.go | 1 + cmd/imports_linux.go | 1 + component/loopback/loopback_fs.go | 6 + component/size_tracker/journal.go | 103 ++++++ component/size_tracker/size_tracker.go | 380 ++++++++++++++++++++ component/size_tracker/size_tracker_test.go | 380 ++++++++++++++++++++ internal/component.go | 2 +- internal/component.template | 7 +- 8 files changed, 876 insertions(+), 4 deletions(-) create mode 100644 component/size_tracker/journal.go create mode 100644 component/size_tracker/size_tracker.go create mode 100644 component/size_tracker/size_tracker_test.go diff --git a/cmd/import_windows.go b/cmd/import_windows.go index 8c350895d..60692e95e 100644 --- a/cmd/import_windows.go +++ b/cmd/import_windows.go @@ -34,5 +34,6 @@ import ( _ "github.com/Seagate/cloudfuse/component/libfuse" _ "github.com/Seagate/cloudfuse/component/loopback" _ "github.com/Seagate/cloudfuse/component/s3storage" + _ "github.com/Seagate/cloudfuse/component/size_tracker" _ "github.com/Seagate/cloudfuse/component/stream" ) diff --git a/cmd/imports_linux.go b/cmd/imports_linux.go index 2027f256a..6c7f9f00b 100644 --- a/cmd/imports_linux.go +++ b/cmd/imports_linux.go @@ -35,5 +35,6 @@ import ( _ "github.com/Seagate/cloudfuse/component/libfuse" _ "github.com/Seagate/cloudfuse/component/loopback" _ "github.com/Seagate/cloudfuse/component/s3storage" + _ "github.com/Seagate/cloudfuse/component/size_tracker" _ "github.com/Seagate/cloudfuse/component/stream" ) diff --git a/component/loopback/loopback_fs.go b/component/loopback/loopback_fs.go index 80651d1c5..30992c88c 100644 --- a/component/loopback/loopback_fs.go +++ b/component/loopback/loopback_fs.go @@ -91,6 +91,12 @@ func (lfs *LoopbackFS) Start(ctx context.Context) error { return nil } +func (lfs *LoopbackFS) Stop() error { + log.Info("Stopping Loopback FS") + os.RemoveAll(lfs.path) + return nil +} + func (lfs *LoopbackFS) Priority() internal.ComponentPriority { return internal.EComponentPriority.Consumer() } diff --git a/component/size_tracker/journal.go b/component/size_tracker/journal.go new file mode 100644 index 000000000..927b88abc --- /dev/null +++ b/component/size_tracker/journal.go @@ -0,0 +1,103 @@ +/* + Licensed under the MIT License . + Copyright © 2023-2025 Seagate Technology LLC and/or its Affiliates + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package size_tracker + +import ( + "encoding/binary" + "os" + "sync" + + "github.com/Seagate/cloudfuse/common" +) + +var journalFile string + +type MountSize struct { + size uint64 + file *os.File + mu sync.Mutex +} + +func CreateSizeJournal(filename string) (*MountSize, error) { + journalFile = common.JoinUnixFilepath(common.DefaultWorkDir, filename) + f, err := os.OpenFile(journalFile, os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + return nil, err + } + defer f.Close() + + fileInfo, err := f.Stat() + if err != nil { + return nil, err + } + + var size uint64 + + if fileInfo.Size() >= 8 { + buf := make([]byte, 8) + if _, err := f.ReadAt(buf, 0); err != nil { + return nil, err + } + + size = binary.BigEndian.Uint64(buf) + } + + return &MountSize{size: size, file: f, mu: sync.Mutex{}}, nil +} + +func (s *MountSize) GetSize() uint64 { + s.mu.Lock() + defer s.mu.Unlock() + return s.size +} + +func (s *MountSize) Add(delta uint64) uint64 { + return s.updateSize(delta) +} + +func (s *MountSize) Subtract(delta uint64) uint64 { + return s.updateSize(-delta) +} + +func (s *MountSize) CloseFile() error { + s.mu.Lock() + defer s.mu.Unlock() + return s.file.Close() +} + +func (s *MountSize) updateSize(delta uint64) uint64 { + s.mu.Lock() + defer s.mu.Unlock() + s.size += delta + _ = s.writeSizeToFile() + return s.size +} + +func (s *MountSize) writeSizeToFile() error { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, s.size) + + if _, err := s.file.WriteAt(buf, 0); err != nil { + return err + } + + return s.file.Sync() +} diff --git a/component/size_tracker/size_tracker.go b/component/size_tracker/size_tracker.go new file mode 100644 index 000000000..7dccb3a26 --- /dev/null +++ b/component/size_tracker/size_tracker.go @@ -0,0 +1,380 @@ +/* + Licensed under the MIT License . + + Copyright © 2023-2025 Seagate Technology LLC and/or its Affiliates + Copyright © 2020-2024 Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package size_tracker + +import ( + "context" + "fmt" + + "github.com/Seagate/cloudfuse/common" + "github.com/Seagate/cloudfuse/common/config" + "github.com/Seagate/cloudfuse/common/log" + "github.com/Seagate/cloudfuse/internal" + "github.com/Seagate/cloudfuse/internal/handlemap" +) + +/* NOTES: + - Component shall have a structure which inherits "internal.BaseComponent" to participate in pipeline + - Component shall register a name and its constructor to participate in pipeline (add by default by generator) + - Order of calls : Constructor -> Configure -> Start ..... -> Stop + - To read any new setting from config file follow the Configure method default comments +*/ + +// Common structure for Component +type SizeTracker struct { + internal.BaseComponent + mountSize *MountSize +} + +// Structure defining your config parameters +type SizeTrackerOptions struct { + JournalName string `config:"journal-name" yaml:"journal-name,omitempty"` +} + +const compName = "size_tracker" +const blockSize = int64(4096) +const default_journal_name = "directory_size.dat" + +// Verification to check satisfaction criteria with Component Interface +var _ internal.Component = &SizeTracker{} + +func (st *SizeTracker) Name() string { + return compName +} + +func (st *SizeTracker) SetName(name string) { + st.BaseComponent.SetName(name) +} + +func (st *SizeTracker) SetNextComponent(nc internal.Component) { + st.BaseComponent.SetNextComponent(nc) +} + +// Start : Pipeline calls this method to start the component functionality +// +// this shall not block the call otherwise pipeline will not start +func (st *SizeTracker) Start(ctx context.Context) error { + log.Trace("SizeTracker::Start : Starting component %s", st.Name()) + return nil +} + +// Stop : Stop the component functionality and kill all threads started +func (st *SizeTracker) Stop() error { + log.Trace("SizeTracker::Stop : Stopping component %s", st.Name()) + _ = st.mountSize.CloseFile() + return nil +} + +// Configure : Pipeline will call this method after constructor so that you can read config and initialize yourself +// +// Return failure if any config is not valid to exit the process +func (st *SizeTracker) Configure(_ bool) error { + log.Trace("SizeTracker::Configure : %s", st.Name()) + + // >> If you do not need any config parameters remove below code and return nil + conf := SizeTrackerOptions{} + err := config.UnmarshalKey(st.Name(), &conf) + if err != nil { + log.Err("SizeTracker::Configure : config error [invalid config attributes]") + return fmt.Errorf("SizeTracker: config error [invalid config attributes]") + } + + if config.IsSet(compName + ".journal-name") { + st.mountSize, err = CreateSizeJournal(conf.JournalName) + } else { + st.mountSize, err = CreateSizeJournal(default_journal_name) + } + + return err +} + +func (st *SizeTracker) Priority() internal.ComponentPriority { + return internal.EComponentPriority.LevelOne() +} + +// OnConfigChange : If component has registered, on config file change this method is called +func (st *SizeTracker) OnConfigChange() { +} + +// Directory operations +func (st *SizeTracker) DeleteDir(options internal.DeleteDirOptions) error { + // Libfuse only allows deleting empty directories, so we should not need to update the size here + return st.NextComponent().DeleteDir(options) +} + +func (st *SizeTracker) RenameDir(options internal.RenameDirOptions) error { + // Rename dir should not allow renaming files into a directory that already exists so we should not + // need to update the size here. + return st.NextComponent().RenameDir(options) +} + +// File operations +func (st *SizeTracker) CreateFile(options internal.CreateFileOptions) (*handlemap.Handle, error) { + attr, getAttrErr := st.NextComponent().GetAttr(internal.GetAttrOptions{Name: options.Name}) + + handle, err := st.NextComponent().CreateFile(options) + + // File already exists but create succeeded so remove old file size + if err == nil && getAttrErr == nil { + st.mountSize.Subtract(uint64(attr.Size)) + } + + return handle, err +} + +func (st *SizeTracker) DeleteFile(options internal.DeleteFileOptions) error { + attr, getAttrErr := st.NextComponent().GetAttr(internal.GetAttrOptions{Name: options.Name}) + + err := st.NextComponent().DeleteFile(options) + + // File already exists and delete succeeded so remove old file size + if err == nil && getAttrErr == nil { + st.mountSize.Subtract(uint64(attr.Size)) + } + + return err +} + +func (st *SizeTracker) RenameFile(options internal.RenameFileOptions) error { + dstAttr, dstErr := st.NextComponent().GetAttr(internal.GetAttrOptions{Name: options.Dst}) + + err := st.NextComponent().RenameFile(options) + + // If dst already exista and rename succeeds, remove overwritten dst size + if dstErr == nil && err == nil { + st.mountSize.Subtract(uint64(dstAttr.Size)) + } + + return err +} + +func (st *SizeTracker) WriteFile(options internal.WriteFileOptions) (int, error) { + var origSize int64 + attr, getAttrErr1 := st.NextComponent().GetAttr(internal.GetAttrOptions{Name: options.Handle.Path}) + if getAttrErr1 == nil { + origSize = attr.Size + } else { + log.Err("SizeTracker::WriteFile : Unable to get attr for file %s. Current tracked size is invalid. Error: : %v", options.Handle.Path, getAttrErr1) + } + + bytesWritten, err := st.NextComponent().WriteFile(options) + if err != nil { + return bytesWritten, err + } + + var newSize int64 + attr, getAttrErr2 := st.NextComponent().GetAttr(internal.GetAttrOptions{Name: options.Handle.Path}) + if getAttrErr2 == nil { + newSize = attr.Size + } else { + log.Err("SizeTracker::WriteFile : Unable to get attr for file %s. Current tracked size is invalid. Error: : %v", options.Handle.Path, getAttrErr2) + } + + if getAttrErr1 != nil || getAttrErr2 != nil { + return bytesWritten, nil + } + + diff := newSize - origSize + + // File already exists and CopyFromFile succeeded subtract difference in file size + if diff < 0 { + st.mountSize.Subtract(uint64(-diff)) + } else { + st.mountSize.Add(uint64(diff)) + } + + return bytesWritten, nil +} + +func (st *SizeTracker) TruncateFile(options internal.TruncateFileOptions) error { + var origSize int64 + attr, getAttrErr := st.NextComponent().GetAttr(internal.GetAttrOptions{Name: options.Name}) + if getAttrErr == nil { + origSize = attr.Size + } + + err := st.NextComponent().TruncateFile(options) + newSize := options.Size - origSize + + // File already exists and truncate succeeded subtract difference in file size + if err == nil && getAttrErr == nil && newSize < 0 { + st.mountSize.Subtract(uint64(-newSize)) + } else if err == nil && attr == nil && newSize >= 0 { + st.mountSize.Add(uint64(newSize)) + } + + return err +} + +func (st *SizeTracker) CopyFromFile(options internal.CopyFromFileOptions) error { + var origSize int64 + attr, err := st.NextComponent().GetAttr(internal.GetAttrOptions{Name: options.Name}) + if err == nil { + origSize = attr.Size + } + + err = st.NextComponent().CopyFromFile(options) + if err != nil { + return err + } + fileInfo, err := options.File.Stat() + if err != nil { + return nil + } + newSize := fileInfo.Size() - origSize + + // File already exists and CopyFromFile succeeded subtract difference in file size + if newSize < 0 { + st.mountSize.Subtract(uint64(-newSize)) + } else { + st.mountSize.Add(uint64(newSize)) + } + + return nil +} + +func (st *SizeTracker) FlushFile(options internal.FlushFileOptions) error { + var origSize int64 + attr, getAttrErr1 := st.NextComponent().GetAttr(internal.GetAttrOptions{Name: options.Handle.Path}) + if getAttrErr1 == nil { + origSize = attr.Size + } else { + log.Err("SizeTracker::FlushFile : Unable to get attr for file %s. Current tracked size is invalid. Error: : %v", options.Handle.Path, getAttrErr1) + } + + err := st.NextComponent().FlushFile(options) + if err != nil { + return err + } + + var newSize int64 + attr, getAttrErr2 := st.NextComponent().GetAttr(internal.GetAttrOptions{Name: options.Handle.Path}) + if getAttrErr2 == nil { + newSize = attr.Size + } else { + log.Err("SizeTracker::FlushFile : Unable to get attr for file %s. Current tracked size is invalid. Error: : %v", options.Handle.Path, getAttrErr2) + } + + if getAttrErr1 != nil || getAttrErr2 != nil { + return nil + } + + diff := newSize - origSize + + // File already exists and CopyFromFile succeeded subtract difference in file size + if diff < 0 { + st.mountSize.Subtract(uint64(-diff)) + } else { + st.mountSize.Add(uint64(diff)) + } + + return nil +} + +// Symlink operations +func (st *SizeTracker) CreateLink(options internal.CreateLinkOptions) error { + attr, _ := st.NextComponent().GetAttr(internal.GetAttrOptions{Name: options.Name}) + + err := st.NextComponent().CreateLink(options) + + // File already exists but create succeeded so remove old file size + if err == nil && attr != nil { + st.mountSize.Subtract(uint64(attr.Size)) + } + + return err +} + +// Filesystem level operations +func (st *SizeTracker) StatFs() (*common.Statfs_t, bool, error) { + log.Trace("SizeTracker::StatFs") + stat := common.Statfs_t{ + Blocks: st.mountSize.GetSize() / uint64(blockSize), + // there is no set capacity limit in cloud storage + // so we use zero for free and avail + // this zero value is used in the libfuse component to recognize that cloud storage responded + Bavail: 0, + Bfree: 0, + Bsize: blockSize, + Ffree: 1e9, + Files: 1e9, + Frsize: blockSize, + Namemax: 255, + } + + log.Debug("SizeTracker::StatFs : responding with free=%d avail=%d blocks=%d (bsize=%d)", stat.Bfree, stat.Bavail, stat.Blocks, stat.Bsize) + + return &stat, true, nil +} + +func (st *SizeTracker) CommitData(opt internal.CommitDataOptions) error { + var origSize int64 + attr, err := st.NextComponent().GetAttr(internal.GetAttrOptions{Name: opt.Name}) + if err == nil { + origSize = attr.Size + } else { + log.Err("SizeTracker::CommitData : Unable to get attr for file %s. Current tracked size is invalid. Error: : %v", opt.Name, err) + } + + err = st.NextComponent().CommitData(opt) + if err != nil { + return err + } + + var newSize int64 + attr, err = st.NextComponent().GetAttr(internal.GetAttrOptions{Name: opt.Name}) + if err == nil { + newSize = attr.Size + } else { + log.Err("SizeTracker::CommitData : Unable to get attr for file %s. Current tracked size is invalid. Error: : %v", opt.Name, err) + } + + diff := newSize - origSize + + // File already exists and CopyFromFile succeeded subtract difference in file size + if diff < 0 { + st.mountSize.Subtract(uint64(-diff)) + } else { + st.mountSize.Add(uint64(diff)) + } + + return nil +} + +// ------------------------- Factory ------------------------------------------- + +// Pipeline will call this method to create your object, initialize your variables here +// << DO NOT DELETE ANY AUTO GENERATED CODE HERE >> +func NewSizeTrackerComponent() internal.Component { + comp := &SizeTracker{} + comp.SetName(compName) + return comp +} + +// On init register this component to pipeline and supply your constructor +func init() { + internal.AddComponent(compName, NewSizeTrackerComponent) +} diff --git a/component/size_tracker/size_tracker_test.go b/component/size_tracker/size_tracker_test.go new file mode 100644 index 000000000..b9c2ed1d4 --- /dev/null +++ b/component/size_tracker/size_tracker_test.go @@ -0,0 +1,380 @@ +/* + Licensed under the MIT License . + + Copyright © 2023-2025 Seagate Technology LLC and/or its Affiliates + Copyright © 2020-2024 Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package size_tracker + +import ( + "context" + "fmt" + "os" + "strconv" + "strings" + "syscall" + "testing" + + "github.com/Seagate/cloudfuse/common" + "github.com/Seagate/cloudfuse/common/config" + "github.com/Seagate/cloudfuse/common/log" + "github.com/Seagate/cloudfuse/component/loopback" + "github.com/Seagate/cloudfuse/internal" + "github.com/Seagate/cloudfuse/internal/handlemap" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type sizeTrackerTestSuite struct { + suite.Suite + assert *assert.Assertions + sizeTracker *SizeTracker + loopback internal.Component + mockCtrl *gomock.Controller + mock *internal.MockComponent +} + +const journal_test_name = "size_tracker_test.dat" + +var emptyConfig = "size_tracker:\n journal-name: " + journal_test_name + +const MB = 1024 * 1024 + +func newLoopbackFS() internal.Component { + loopback := loopback.NewLoopbackFSComponent() + loopback.Configure(true) + + return loopback +} + +func newTestSizeTracker(next internal.Component, configuration string) *SizeTracker { + _ = config.ReadConfigFromReader(strings.NewReader(configuration)) + sizeTracker := NewSizeTrackerComponent() + sizeTracker.SetNextComponent(next) + _ = sizeTracker.Configure(true) + + return sizeTracker.(*SizeTracker) +} + +func (suite *sizeTrackerTestSuite) SetupTest() { + err := log.SetDefaultLogger("silent", common.LogConfig{}) + if err != nil { + panic(fmt.Sprintf("Unable to set silent logger as default: %v", err)) + } + suite.setupTestHelper(emptyConfig) +} + +func (suite *sizeTrackerTestSuite) setupTestHelper(config string) { + suite.assert = assert.New(suite.T()) + + suite.mockCtrl = gomock.NewController(suite.T()) + suite.mock = internal.NewMockComponent(suite.mockCtrl) + suite.loopback = newLoopbackFS() + suite.sizeTracker = newTestSizeTracker(suite.loopback, config) + _ = suite.sizeTracker.Start(context.Background()) +} + +func (suite *sizeTrackerTestSuite) cleanupTest() { + _ = suite.loopback.Stop() + err := suite.sizeTracker.Stop() + if err != nil { + panic(fmt.Sprintf("Unable to stop size tracker [%s]", err.Error())) + } + journal_file := common.JoinUnixFilepath(common.DefaultWorkDir, journal_test_name) + os.Remove(journal_file) + suite.mockCtrl.Finish() +} + +// Tests the default configuration of attribute cache +func (suite *sizeTrackerTestSuite) TestDefault() { + defer suite.cleanupTest() + suite.assert.Equal("size_tracker", suite.sizeTracker.Name()) + print(suite.sizeTracker.mountSize.GetSize()) + suite.assert.EqualValues(uint64(0), suite.sizeTracker.mountSize.GetSize()) +} + +func (suite *sizeTrackerTestSuite) TestDeleteDir() { + defer suite.cleanupTest() + // Setup + + dir := "dir" + path := dir + "/file" + err := suite.sizeTracker.CreateDir(internal.CreateDirOptions{Name: dir, Mode: 0777}) + suite.assert.NoError(err) + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0777}) + suite.assert.NoError(err) + + testData := "test data" + data := []byte(testData) + _, err = suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}) + suite.assert.NoError(err) + suite.assert.EqualValues(len(testData), suite.sizeTracker.mountSize.GetSize()) + + err = suite.sizeTracker.CloseFile(internal.CloseFileOptions{Handle: handle}) + suite.assert.NoError(err) + err = suite.sizeTracker.DeleteFile(internal.DeleteFileOptions{Name: path}) + suite.assert.NoError(err) + + // Delete the directory + err = suite.sizeTracker.DeleteDir(internal.DeleteDirOptions{Name: dir}) + suite.assert.NoError(err) + + // Final size should be 0 + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) +} + +func (suite *sizeTrackerTestSuite) TestRenameDir() { + defer suite.cleanupTest() + + // Setup + src := "src" + dst := "dst" + testData := "test data" + data := []byte(testData) + err := suite.sizeTracker.CreateDir(internal.CreateDirOptions{Name: src, Mode: 0777}) + suite.assert.NoError(err) + path := src + "/file" + for i := 0; i < 5; i++ { + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: path + strconv.Itoa(i), Mode: 0777}) + suite.assert.NoError(err) + _, err = suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}) + suite.assert.NoError(err) + err = suite.sizeTracker.CloseFile(internal.CloseFileOptions{Handle: handle}) + suite.assert.NoError(err) + } + suite.assert.EqualValues(5*len(testData), suite.sizeTracker.mountSize.GetSize()) + + // Rename the directory + err = suite.sizeTracker.RenameDir(internal.RenameDirOptions{Src: src, Dst: dst}) + suite.assert.NoError(err) + + suite.assert.EqualValues(5*len(testData), suite.sizeTracker.mountSize.GetSize()) +} + +func (suite *sizeTrackerTestSuite) TestCreateFile() { + defer suite.cleanupTest() + // Default is to not create empty files on create file to support immutable storage. + path := "file1" + options := internal.CreateFileOptions{Name: path} + _, err := suite.sizeTracker.CreateFile(options) + suite.assert.NoError(err) + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) +} + +func (suite *sizeTrackerTestSuite) TestDeleteFile() { + defer suite.cleanupTest() + path := "file4" + + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0777}) + suite.assert.NoError(err) + testData := "test data" + data := []byte(testData) + _, err = suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}) + suite.assert.NoError(err) + suite.assert.EqualValues(len(testData), suite.sizeTracker.mountSize.GetSize()) + err = suite.sizeTracker.CloseFile(internal.CloseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + err = suite.sizeTracker.DeleteFile(internal.DeleteFileOptions{Name: path}) + suite.assert.NoError(err) + + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) +} + +func (suite *sizeTrackerTestSuite) TestDeleteFileError() { + defer suite.cleanupTest() + path := "file6" + err := suite.sizeTracker.DeleteFile(internal.DeleteFileOptions{Name: path}) + suite.assert.Error(err) + suite.assert.EqualValues(syscall.ENOENT, err) + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) +} + +func (suite *sizeTrackerTestSuite) TestWriteFile() { + defer suite.cleanupTest() + // Setup + file := "file19" + handle, _ := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0777}) + + testData := "test data" + data := []byte(testData) + _, err := suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}) + suite.assert.NoError(err) + suite.assert.EqualValues(len(data), suite.sizeTracker.mountSize.GetSize()) +} + +func (suite *sizeTrackerTestSuite) TestWriteFileErrorBadFd() { + defer suite.cleanupTest() + // Setup + file := "file20" + handle := handlemap.NewHandle(file) + len, err := suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle}) + suite.assert.Error(err) + suite.assert.EqualValues(syscall.EBADF, err) + suite.assert.EqualValues(0, len) + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) +} + +func (suite *sizeTrackerTestSuite) TestFlushFileEmpty() { + defer suite.cleanupTest() + // Setup + file := "file21" + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0777}) + suite.assert.Error(err) + + // Flush the Empty File + err = suite.sizeTracker.FlushFile(internal.FlushFileOptions{Handle: handle}) + suite.assert.NoError(err) + suite.assert.False(handle.Dirty()) + + // Path should be in fake storage + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) +} + +func (suite *sizeTrackerTestSuite) TestFlushFile() { + defer suite.cleanupTest() + // Setup + file := "file22" + handle, _ := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0777}) + testData := "test data" + data := []byte(testData) + _, err := suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}) + suite.assert.Error(err) + + // Flush the Empty File + err = suite.sizeTracker.FlushFile(internal.FlushFileOptions{Handle: handle}) + suite.assert.NoError(err) + + suite.assert.EqualValues(len(testData), suite.sizeTracker.mountSize.GetSize()) +} + +func (suite *sizeTrackerTestSuite) TestFlushFileErrorBadFd() { + defer suite.cleanupTest() + // Setup + file := "file23" + handle := handlemap.NewHandle(file) + handle.Flags.Set(handlemap.HandleFlagDirty) + err := suite.sizeTracker.FlushFile(internal.FlushFileOptions{Handle: handle}) + suite.assert.Error(err) + suite.assert.EqualValues(syscall.EBADF, err) + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) +} + +func (suite *sizeTrackerTestSuite) TestRenameFile() { + defer suite.cleanupTest() + // Setup + src := "source2" + dst := "destination2" + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: src, Mode: 0666}) + suite.assert.NoError(err) + err = suite.sizeTracker.CloseFile(internal.CloseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + testData := "test data" + data := []byte(testData) + _, err = suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}) + suite.assert.Error(err) + + // RenameFile + err = suite.sizeTracker.RenameFile(internal.RenameFileOptions{Src: src, Dst: dst}) + suite.assert.NoError(err) + + suite.assert.EqualValues(len(testData), suite.sizeTracker.mountSize.GetSize()) +} + +func (suite *sizeTrackerTestSuite) TestRenameOpenFile() { + defer suite.cleanupTest() + + src := "source6" + dst := "destination6" + + // create source file + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: src, Mode: 0666}) + suite.assert.NoError(err) + + // rename open file + err = suite.sizeTracker.RenameFile(internal.RenameFileOptions{ + Src: src, + Dst: dst, + }) + suite.assert.NoError(err) + + // write to file handle + data := []byte("newdata") + n, err := suite.sizeTracker.WriteFile(internal.WriteFileOptions{ + Handle: handle, + Data: data, + }) + suite.assert.NoError(err) + suite.assert.Equal(len(data), n) + + // Close file handle + err = suite.sizeTracker.CloseFile(internal.CloseFileOptions{ + Handle: handle, + }) + suite.assert.NoError(err) + + suite.assert.EqualValues(len(data), suite.sizeTracker.mountSize.GetSize()) +} + +func (suite *sizeTrackerTestSuite) TestTruncateFile() { + defer suite.cleanupTest() + // Setup + path := "file30" + handle, _ := suite.loopback.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0777}) + suite.loopback.CloseFile(internal.CloseFileOptions{Handle: handle}) + + size := 1024 + err := suite.sizeTracker.TruncateFile(internal.TruncateFileOptions{Name: path, Size: int64(size)}) + suite.assert.NoError(err) + + suite.assert.EqualValues(size, suite.sizeTracker.mountSize.GetSize()) +} + +func (suite *sizeTrackerTestSuite) TestStatFS() { + defer suite.cleanupTest() + + file := "file41" + handle, _ := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0777}) + data := make([]byte, 1024*1024) + _, err := suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}) + suite.assert.NoError(err) + err = suite.sizeTracker.FlushFile(internal.FlushFileOptions{Handle: handle}) + suite.assert.NoError(err) + stat, ret, err := suite.sizeTracker.StatFs() + suite.assert.True(ret) + suite.assert.NoError(err) + suite.assert.NotEqual(&common.Statfs_t{}, stat) + + suite.assert.Equal(uint64(512), stat.Blocks) + suite.assert.Equal(int64(4096), stat.Bsize) + suite.assert.Equal(int64(4096), stat.Frsize) + suite.assert.Equal(uint64(255), stat.Namemax) +} + +// In order for 'go test' to run this suite, we need to create +// a normal test function and pass our suite to suite.Run +func TestSizeTrackerTestSuite(t *testing.T) { + suite.Run(t, new(sizeTrackerTestSuite)) +} diff --git a/internal/component.go b/internal/component.go index 66bfdf1dc..43bc945a7 100644 --- a/internal/component.go +++ b/internal/component.go @@ -49,7 +49,7 @@ func (ComponentPriority) Consumer() ComponentPriority { } func (ComponentPriority) LevelOne() ComponentPriority { - return ComponentPriority(700) + return ComponentPriority(400) } func (ComponentPriority) LevelTwo() ComponentPriority { diff --git a/internal/component.template b/internal/component.template index ea3609b64..ae18ff0ca 100644 --- a/internal/component.template +++ b/internal/component.template @@ -26,11 +26,12 @@ package import ( - "cloudfuse/common/config" - "cloudfuse/common/log" - "cloudfuse/internal" "context" "fmt" + + "github.com/Seagate/cloudfuse/common/config" + "github.com/Seagate/cloudfuse/common/log" + "github.com/Seagate/cloudfuse/internal" ) /* NOTES: From 748732c43aad17fe6562bd49c53d338732830b55 Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Tue, 21 Jan 2025 11:17:05 -0700 Subject: [PATCH 02/23] Fix some unit tests --- component/loopback/loopback_fs.go | 16 ++-- component/size_tracker/size_tracker_test.go | 88 ++++++++++++--------- 2 files changed, 60 insertions(+), 44 deletions(-) diff --git a/component/loopback/loopback_fs.go b/component/loopback/loopback_fs.go index 30992c88c..295677c72 100644 --- a/component/loopback/loopback_fs.go +++ b/component/loopback/loopback_fs.go @@ -91,12 +91,6 @@ func (lfs *LoopbackFS) Start(ctx context.Context) error { return nil } -func (lfs *LoopbackFS) Stop() error { - log.Info("Stopping Loopback FS") - os.RemoveAll(lfs.path) - return nil -} - func (lfs *LoopbackFS) Priority() internal.ComponentPriority { return internal.EComponentPriority.Consumer() } @@ -237,7 +231,15 @@ func (lfs *LoopbackFS) RenameFile(options internal.RenameFileOptions) error { log.Trace("LoopbackFS::RenameFile : %s -> %s", options.Src, options.Dst) oldPath := filepath.Join(lfs.path, options.Src) newPath := filepath.Join(lfs.path, options.Dst) - return os.Rename(oldPath, newPath) + err := os.Rename(oldPath, newPath) + handlemap.GetHandles().Range(func(key, value any) bool { + handle := value.(*handlemap.Handle) + if handle.Path == options.Src { + handle.Path = options.Dst + } + return true + }) + return err } func (lfs *LoopbackFS) ReadLink(options internal.ReadLinkOptions) (string, error) { diff --git a/component/size_tracker/size_tracker_test.go b/component/size_tracker/size_tracker_test.go index b9c2ed1d4..69c9ea134 100644 --- a/component/size_tracker/size_tracker_test.go +++ b/component/size_tracker/size_tracker_test.go @@ -27,11 +27,12 @@ package size_tracker import ( "context" + "crypto/rand" "fmt" "os" + "path/filepath" "strconv" "strings" - "syscall" "testing" "github.com/Seagate/cloudfuse/common" @@ -48,18 +49,30 @@ import ( type sizeTrackerTestSuite struct { suite.Suite - assert *assert.Assertions - sizeTracker *SizeTracker - loopback internal.Component - mockCtrl *gomock.Controller - mock *internal.MockComponent + assert *assert.Assertions + sizeTracker *SizeTracker + loopback internal.Component + mockCtrl *gomock.Controller + mock *internal.MockComponent + loopback_storage_path string } +var home_dir, _ = os.UserHomeDir() + const journal_test_name = "size_tracker_test.dat" +const MB = 1024 * 1024 -var emptyConfig = "size_tracker:\n journal-name: " + journal_test_name +func getFakeStoragePath(base string) string { + tmp_path := filepath.Join(home_dir, base+randomString(8)) + _ = os.Mkdir(tmp_path, 0777) + return tmp_path +} -const MB = 1024 * 1024 +func randomString(length int) string { + b := make([]byte, length) + rand.Read(b) + return fmt.Sprintf("%x", b)[:length] +} func newLoopbackFS() internal.Component { loopback := loopback.NewLoopbackFSComponent() @@ -82,7 +95,9 @@ func (suite *sizeTrackerTestSuite) SetupTest() { if err != nil { panic(fmt.Sprintf("Unable to set silent logger as default: %v", err)) } - suite.setupTestHelper(emptyConfig) + suite.loopback_storage_path = getFakeStoragePath("loopback") + cfg := fmt.Sprintf("loopbackfs:\n path: %s\n\nsize_tracker:\n journal-name: %s", suite.loopback_storage_path, journal_test_name) + suite.setupTestHelper(cfg) } func (suite *sizeTrackerTestSuite) setupTestHelper(config string) { @@ -92,6 +107,7 @@ func (suite *sizeTrackerTestSuite) setupTestHelper(config string) { suite.mock = internal.NewMockComponent(suite.mockCtrl) suite.loopback = newLoopbackFS() suite.sizeTracker = newTestSizeTracker(suite.loopback, config) + _ = suite.loopback.Start(context.Background()) _ = suite.sizeTracker.Start(context.Background()) } @@ -103,6 +119,7 @@ func (suite *sizeTrackerTestSuite) cleanupTest() { } journal_file := common.JoinUnixFilepath(common.DefaultWorkDir, journal_test_name) os.Remove(journal_file) + os.RemoveAll(suite.loopback_storage_path) suite.mockCtrl.Finish() } @@ -175,11 +192,14 @@ func (suite *sizeTrackerTestSuite) TestRenameDir() { func (suite *sizeTrackerTestSuite) TestCreateFile() { defer suite.cleanupTest() // Default is to not create empty files on create file to support immutable storage. - path := "file1" + path := "file4" options := internal.CreateFileOptions{Name: path} _, err := suite.sizeTracker.CreateFile(options) suite.assert.NoError(err) suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) + + err = suite.sizeTracker.DeleteFile(internal.DeleteFileOptions{Name: path}) + suite.assert.NoError(err) } func (suite *sizeTrackerTestSuite) TestDeleteFile() { @@ -207,7 +227,6 @@ func (suite *sizeTrackerTestSuite) TestDeleteFileError() { path := "file6" err := suite.sizeTracker.DeleteFile(internal.DeleteFileOptions{Name: path}) suite.assert.Error(err) - suite.assert.EqualValues(syscall.ENOENT, err) suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) } @@ -215,11 +234,12 @@ func (suite *sizeTrackerTestSuite) TestWriteFile() { defer suite.cleanupTest() // Setup file := "file19" - handle, _ := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0777}) + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0777}) + suite.assert.NoError(err) testData := "test data" data := []byte(testData) - _, err := suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}) + _, err = suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}) suite.assert.NoError(err) suite.assert.EqualValues(len(data), suite.sizeTracker.mountSize.GetSize()) } @@ -229,10 +249,9 @@ func (suite *sizeTrackerTestSuite) TestWriteFileErrorBadFd() { // Setup file := "file20" handle := handlemap.NewHandle(file) - len, err := suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle}) + length, err := suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle}) suite.assert.Error(err) - suite.assert.EqualValues(syscall.EBADF, err) - suite.assert.EqualValues(0, len) + suite.assert.EqualValues(0, length) suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) } @@ -241,7 +260,7 @@ func (suite *sizeTrackerTestSuite) TestFlushFileEmpty() { // Setup file := "file21" handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0777}) - suite.assert.Error(err) + suite.assert.NoError(err) // Flush the Empty File err = suite.sizeTracker.FlushFile(internal.FlushFileOptions{Handle: handle}) @@ -256,13 +275,15 @@ func (suite *sizeTrackerTestSuite) TestFlushFile() { defer suite.cleanupTest() // Setup file := "file22" - handle, _ := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0777}) + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0777}) + suite.assert.NoError(err) testData := "test data" data := []byte(testData) - _, err := suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}) - suite.assert.Error(err) + _, err = suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}) + suite.assert.NoError(err) + suite.assert.EqualValues(len(testData), suite.sizeTracker.mountSize.GetSize()) - // Flush the Empty File + // Flush the file err = suite.sizeTracker.FlushFile(internal.FlushFileOptions{Handle: handle}) suite.assert.NoError(err) @@ -277,7 +298,6 @@ func (suite *sizeTrackerTestSuite) TestFlushFileErrorBadFd() { handle.Flags.Set(handlemap.HandleFlagDirty) err := suite.sizeTracker.FlushFile(internal.FlushFileOptions{Handle: handle}) suite.assert.Error(err) - suite.assert.EqualValues(syscall.EBADF, err) suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) } @@ -288,18 +308,19 @@ func (suite *sizeTrackerTestSuite) TestRenameFile() { dst := "destination2" handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: src, Mode: 0666}) suite.assert.NoError(err) - err = suite.sizeTracker.CloseFile(internal.CloseFileOptions{Handle: handle}) - suite.assert.NoError(err) testData := "test data" data := []byte(testData) _, err = suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}) - suite.assert.Error(err) + suite.assert.NoError(err) + suite.assert.EqualValues(len(testData), suite.sizeTracker.mountSize.GetSize()) + + err = suite.sizeTracker.CloseFile(internal.CloseFileOptions{Handle: handle}) + suite.assert.NoError(err) // RenameFile err = suite.sizeTracker.RenameFile(internal.RenameFileOptions{Src: src, Dst: dst}) suite.assert.NoError(err) - suite.assert.EqualValues(len(testData), suite.sizeTracker.mountSize.GetSize()) } @@ -314,26 +335,19 @@ func (suite *sizeTrackerTestSuite) TestRenameOpenFile() { suite.assert.NoError(err) // rename open file - err = suite.sizeTracker.RenameFile(internal.RenameFileOptions{ - Src: src, - Dst: dst, - }) + err = suite.sizeTracker.RenameFile(internal.RenameFileOptions{Src: src, Dst: dst}) suite.assert.NoError(err) // write to file handle data := []byte("newdata") - n, err := suite.sizeTracker.WriteFile(internal.WriteFileOptions{ - Handle: handle, - Data: data, - }) + n, err := suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Data: data}) suite.assert.NoError(err) suite.assert.Equal(len(data), n) // Close file handle - err = suite.sizeTracker.CloseFile(internal.CloseFileOptions{ - Handle: handle, - }) + err = suite.sizeTracker.CloseFile(internal.CloseFileOptions{Handle: handle}) suite.assert.NoError(err) + fmt.Println(suite.sizeTracker.mountSize.GetSize()) suite.assert.EqualValues(len(data), suite.sizeTracker.mountSize.GetSize()) } From 080b9f98a1202f826ba872f70045db9469566934 Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Wed, 22 Jan 2025 11:03:18 -0700 Subject: [PATCH 03/23] Pass all unit tests --- component/loopback/loopback_fs.go | 7 +- component/loopback/loopback_fs_test.go | 148 +++++++++++++++- component/size_tracker/size_tracker.go | 8 +- component/size_tracker/size_tracker_test.go | 185 ++++++++++++++++---- 4 files changed, 297 insertions(+), 51 deletions(-) diff --git a/component/loopback/loopback_fs.go b/component/loopback/loopback_fs.go index 295677c72..5667bea56 100644 --- a/component/loopback/loopback_fs.go +++ b/component/loopback/loopback_fs.go @@ -182,6 +182,7 @@ func (lfs *LoopbackFS) CreateFile(options internal.CreateFileOptions) (*handlema } handle := handlemap.NewHandle(options.Name) handle.SetFileObject(f) + handlemap.Add(handle) return handle, nil } @@ -212,6 +213,7 @@ func (lfs *LoopbackFS) OpenFile(options internal.OpenFileOptions) (*handlemap.Ha } handle := handlemap.NewHandle(options.Name) handle.SetFileObject(f) + handlemap.Add(handle) return handle, nil } @@ -224,6 +226,7 @@ func (lfs *LoopbackFS) CloseFile(options internal.CloseFileOptions) error { return syscall.EBADF } + handlemap.Delete(options.Handle.ID) return f.Close() } @@ -274,10 +277,6 @@ func (lfs *LoopbackFS) ReadInBuffer(options internal.ReadInBufferOptions) (int, options.Handle.RLock() defer options.Handle.RUnlock() - if f == nil { - log.Err("LoopbackFS::ReadInBuffer : error [invalid file object]") - return 0, os.ErrInvalid - } return f.ReadAt(options.Data, options.Offset) } diff --git a/component/loopback/loopback_fs_test.go b/component/loopback/loopback_fs_test.go index 83592feab..6a75b5aee 100644 --- a/component/loopback/loopback_fs_test.go +++ b/component/loopback/loopback_fs_test.go @@ -105,7 +105,7 @@ func (suite *LoopbackFSTestSuite) TestCreateDir() { defer suite.cleanupTest() assert := assert.New(suite.T()) - err := suite.lfs.CreateDir(internal.CreateDirOptions{Name: dirTwo, Mode: os.FileMode(0777)}) + err := suite.lfs.CreateDir(internal.CreateDirOptions{Name: dirTwo, Mode: os.FileMode(0755)}) assert.NoError(err, "CreateDir: Failed") suite.DirExists(filepath.Join(testPath, dirTwo)) } @@ -149,7 +149,7 @@ func (suite *LoopbackFSTestSuite) TestCreateFile() { defer suite.cleanupTest() assert := assert.New(suite.T()) - handle, err := suite.lfs.CreateFile(internal.CreateFileOptions{Name: fileEmpty, Mode: os.FileMode(0777)}) + handle, err := suite.lfs.CreateFile(internal.CreateFileOptions{Name: fileEmpty, Mode: os.FileMode(0644)}) assert.NoError(err, "CreateFile: Failed") assert.NotNil(handle) @@ -168,11 +168,118 @@ func (suite *LoopbackFSTestSuite) TestDeleteFile() { assert.NoFileExists(filepath.Join(testPath, fileHello), "DeleteFile: file was not deleted") } +func (suite *LoopbackFSTestSuite) TestRenameFile() { + defer suite.cleanupTest() + assert := assert.New(suite.T()) + + handle, err := suite.lfs.CreateFile(internal.CreateFileOptions{Name: fileEmpty, Mode: os.FileMode(0644)}) + assert.NoError(err, "TestRenameFile: Failed") + assert.NotNil(handle) + assert.FileExists(filepath.Join(testPath, fileEmpty)) + + err = suite.lfs.CloseFile(internal.CloseFileOptions{Handle: handle}) + assert.NoError(err, "TestRenameFile: Failed to close file") + + err = suite.lfs.RenameFile(internal.RenameFileOptions{Src: fileEmpty, Dst: fileHello}) + assert.NoError(err) + + assert.FileExists(filepath.Join(testPath, fileHello)) + assert.NoFileExists(filepath.Join(testPath, fileEmpty)) +} + +func (suite *LoopbackFSTestSuite) TestRenameOpenFile() { + defer suite.cleanupTest() + assert := assert.New(suite.T()) + + handle, err := suite.lfs.CreateFile(internal.CreateFileOptions{Name: fileEmpty, Mode: os.FileMode(0644)}) + assert.NoError(err, "TestRenameOpenFile: Failed") + assert.NotNil(handle) + assert.FileExists(filepath.Join(testPath, fileEmpty)) + + err = suite.lfs.CloseFile(internal.CloseFileOptions{Handle: handle}) + assert.NoError(err, "TestRenameOpenFile: Failed to close file") + + handle, err = suite.lfs.OpenFile(internal.OpenFileOptions{Name: fileEmpty, Flags: os.O_RDONLY, Mode: os.FileMode(0644)}) + assert.NoError(err, "TestRenameOpenFile: Failed to open file") + assert.NotNil(handle) + + err = suite.lfs.RenameFile(internal.RenameFileOptions{Src: fileEmpty, Dst: fileHello}) + assert.NoError(err) + + assert.FileExists(filepath.Join(testPath, fileHello)) + assert.NoFileExists(filepath.Join(testPath, fileEmpty)) + + err = suite.lfs.CloseFile(internal.CloseFileOptions{Handle: handle}) + assert.NoError(err, "TestRenameOpenFile: Failed to close file") +} + +func (suite *LoopbackFSTestSuite) TestRenameWriteFile() { + defer suite.cleanupTest() + assert := assert.New(suite.T()) + + handle, err := suite.lfs.CreateFile(internal.CreateFileOptions{Name: fileEmpty, Mode: os.FileMode(0644)}) + assert.NoError(err, "TestRenameWriteFile: Failed") + assert.NotNil(handle) + assert.FileExists(filepath.Join(testPath, fileEmpty)) + + err = suite.lfs.RenameFile(internal.RenameFileOptions{Src: fileEmpty, Dst: fileHello}) + assert.NoError(err) + + assert.FileExists(filepath.Join(testPath, fileHello)) + assert.NoFileExists(filepath.Join(testPath, fileEmpty)) + + n, err := suite.lfs.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: []byte(quotesText)[:5]}) + assert.NoError(err) + assert.Equal(5, n, "TestRenameWriteFile: failed to write the specified number of bytes") + + err = suite.lfs.CloseFile(internal.CloseFileOptions{Handle: handle}) + assert.NoError(err, "TestRenameWriteFile: Failed to close file") + + info, err := os.Stat(filepath.Join(testPath, fileHello)) + assert.NoError(err, "TestRenameWriteFile: cannot stat file") + assert.Equal(int64(5), info.Size()) +} + +func (suite *LoopbackFSTestSuite) TestRenameWriteFileGetAttr() { + defer suite.cleanupTest() + assert := assert.New(suite.T()) + + handle, err := suite.lfs.CreateFile(internal.CreateFileOptions{Name: fileEmpty, Mode: os.FileMode(0644)}) + assert.NoError(err, "TestRenameWriteFile: Failed") + assert.NotNil(handle) + assert.FileExists(filepath.Join(testPath, fileEmpty)) + + n, err := suite.lfs.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: []byte(quotesText)[:5]}) + assert.NoError(err) + assert.Equal(5, n, "TestRenameWriteFile: failed to write the specified number of bytes") + + attr, err := suite.lfs.GetAttr(internal.GetAttrOptions{Name: fileEmpty}) + assert.NoError(err) + assert.EqualValues(5, attr.Size) + + err = suite.lfs.RenameFile(internal.RenameFileOptions{Src: fileEmpty, Dst: fileHello}) + assert.NoError(err) + + assert.FileExists(filepath.Join(testPath, fileHello)) + assert.NoFileExists(filepath.Join(testPath, fileEmpty)) + + attr, err = suite.lfs.GetAttr(internal.GetAttrOptions{Name: fileHello}) + assert.NoError(err) + assert.EqualValues(5, attr.Size) + + err = suite.lfs.CloseFile(internal.CloseFileOptions{Handle: handle}) + assert.NoError(err, "TestRenameWriteFile: Failed to close file") + + info, err := os.Stat(filepath.Join(testPath, fileHello)) + assert.NoError(err, "TestRenameWriteFile: cannot stat file") + assert.Equal(int64(5), info.Size()) +} + func (suite *LoopbackFSTestSuite) TestReadInBuffer() { defer suite.cleanupTest() assert := assert.New(suite.T()) - handle, err := suite.lfs.OpenFile(internal.OpenFileOptions{Name: fileLorem, Flags: os.O_RDONLY, Mode: os.FileMode(0777)}) + handle, err := suite.lfs.OpenFile(internal.OpenFileOptions{Name: fileLorem, Flags: os.O_RDONLY, Mode: os.FileMode(0644)}) assert.NoError(err, "ReadInBuffer: Failed to open file") assert.NotNil(handle) testCases := []struct { @@ -207,7 +314,7 @@ func (suite *LoopbackFSTestSuite) TestWriteFile() { defer suite.cleanupTest() assert := assert.New(suite.T()) - handle, err := suite.lfs.OpenFile(internal.OpenFileOptions{Name: fileQuotes, Flags: os.O_RDWR | os.O_CREATE, Mode: os.FileMode(0777)}) + handle, err := suite.lfs.OpenFile(internal.OpenFileOptions{Name: fileQuotes, Flags: os.O_RDWR | os.O_CREATE, Mode: os.FileMode(0644)}) assert.NoError(err, "WriteFile: failed to open file") assert.NotNil(handle) @@ -215,6 +322,10 @@ func (suite *LoopbackFSTestSuite) TestWriteFile() { assert.NoError(err) assert.Equal(5, n, "WriteFile: failed to write the specified number of bytes") + attr, err := suite.lfs.GetAttr(internal.GetAttrOptions{Name: fileQuotes}) + assert.NoError(err) + assert.EqualValues(5, attr.Size) + n, err = suite.lfs.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 5, Data: []byte(quotesText)[5:]}) assert.NoError(err) assert.Len([]byte(quotesText)[5:], n, "WriteFile: failed to write specified number of bytes") @@ -222,24 +333,45 @@ func (suite *LoopbackFSTestSuite) TestWriteFile() { err = suite.lfs.CloseFile(internal.CloseFileOptions{Handle: handle}) assert.NoError(err, "WriteFile: Failed to close file") + attr, err = suite.lfs.GetAttr(internal.GetAttrOptions{Name: fileQuotes}) + assert.NoError(err) + assert.EqualValues(len(quotesText), attr.Size) } func (suite *LoopbackFSTestSuite) TestTruncateFile() { defer suite.cleanupTest() assert := assert.New(suite.T()) - handle, err := suite.lfs.OpenFile(internal.OpenFileOptions{Name: fileLorem, Flags: os.O_RDWR, Mode: os.FileMode(0777)}) + handle, err := suite.lfs.OpenFile(internal.OpenFileOptions{Name: fileLorem, Flags: os.O_RDWR, Mode: os.FileMode(0644)}) assert.NoError(err, "TruncateFile: failed to open file") assert.NotNil(handle) - err = suite.lfs.TruncateFile(internal.TruncateFileOptions{Name: fileLorem, Size: 0}) + err = suite.lfs.TruncateFile(internal.TruncateFileOptions{Name: fileLorem, Size: 10}) assert.NoError(err) info, err := os.Stat(filepath.Join(testPath, fileLorem)) assert.NoError(err, "TruncateFile: cannot stat file") - assert.Equal(int64(0), info.Size()) + assert.Equal(int64(10), info.Size()) + + err = suite.lfs.CloseFile(internal.CloseFileOptions{Handle: handle}) + assert.NoError(err, "TruncateFile: Failed to close file") +} + +func (suite *LoopbackFSTestSuite) TestTruncateClosedFile() { + defer suite.cleanupTest() + assert := assert.New(suite.T()) + + handle, err := suite.lfs.OpenFile(internal.OpenFileOptions{Name: fileLorem, Flags: os.O_RDWR, Mode: os.FileMode(0644)}) + assert.NoError(err, "TruncateFile: failed to open file") + assert.NotNil(handle) err = suite.lfs.CloseFile(internal.CloseFileOptions{Handle: handle}) assert.NoError(err, "TruncateFile: Failed to close file") + + err = suite.lfs.TruncateFile(internal.TruncateFileOptions{Name: fileLorem, Size: 10}) + assert.NoError(err) + info, err := os.Stat(filepath.Join(testPath, fileLorem)) + assert.NoError(err, "TruncateFile: cannot stat file") + assert.Equal(int64(10), info.Size()) } func (suite *LoopbackFSTestSuite) TestGetAttr() { @@ -264,7 +396,7 @@ func (suite *LoopbackFSTestSuite) TestStageAndCommitData() { lfs := &LoopbackFS{} lfs.path = common.ExpandPath("~/blocklfstest") - err := os.MkdirAll(lfs.path, os.FileMode(0777)) + err := os.MkdirAll(lfs.path, os.FileMode(0755)) assert.NoError(err) defer os.RemoveAll(lfs.path) diff --git a/component/size_tracker/size_tracker.go b/component/size_tracker/size_tracker.go index 7dccb3a26..1d9fcc0e6 100644 --- a/component/size_tracker/size_tracker.go +++ b/component/size_tracker/size_tracker.go @@ -172,10 +172,10 @@ func (st *SizeTracker) RenameFile(options internal.RenameFileOptions) error { } func (st *SizeTracker) WriteFile(options internal.WriteFileOptions) (int, error) { - var origSize int64 + var oldSize int64 attr, getAttrErr1 := st.NextComponent().GetAttr(internal.GetAttrOptions{Name: options.Handle.Path}) if getAttrErr1 == nil { - origSize = attr.Size + oldSize = attr.Size } else { log.Err("SizeTracker::WriteFile : Unable to get attr for file %s. Current tracked size is invalid. Error: : %v", options.Handle.Path, getAttrErr1) } @@ -197,7 +197,7 @@ func (st *SizeTracker) WriteFile(options internal.WriteFileOptions) (int, error) return bytesWritten, nil } - diff := newSize - origSize + diff := newSize - oldSize // File already exists and CopyFromFile succeeded subtract difference in file size if diff < 0 { @@ -222,7 +222,7 @@ func (st *SizeTracker) TruncateFile(options internal.TruncateFileOptions) error // File already exists and truncate succeeded subtract difference in file size if err == nil && getAttrErr == nil && newSize < 0 { st.mountSize.Subtract(uint64(-newSize)) - } else if err == nil && attr == nil && newSize >= 0 { + } else if err == nil && getAttrErr == nil && newSize >= 0 { st.mountSize.Add(uint64(newSize)) } diff --git a/component/size_tracker/size_tracker_test.go b/component/size_tracker/size_tracker_test.go index 69c9ea134..7cd8eb63a 100644 --- a/component/size_tracker/size_tracker_test.go +++ b/component/size_tracker/size_tracker_test.go @@ -30,6 +30,7 @@ import ( "crypto/rand" "fmt" "os" + "path" "path/filepath" "strconv" "strings" @@ -64,10 +65,14 @@ const MB = 1024 * 1024 func getFakeStoragePath(base string) string { tmp_path := filepath.Join(home_dir, base+randomString(8)) - _ = os.Mkdir(tmp_path, 0777) + _ = os.Mkdir(tmp_path, 0755) return tmp_path } +func generateFileName() string { + return "file" + randomString(8) +} + func randomString(length int) string { b := make([]byte, length) rand.Read(b) @@ -136,10 +141,10 @@ func (suite *sizeTrackerTestSuite) TestDeleteDir() { // Setup dir := "dir" - path := dir + "/file" - err := suite.sizeTracker.CreateDir(internal.CreateDirOptions{Name: dir, Mode: 0777}) + path := path.Join(dir, generateFileName()) + err := suite.sizeTracker.CreateDir(internal.CreateDirOptions{Name: dir, Mode: 0755}) suite.assert.NoError(err) - handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0777}) + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0644}) suite.assert.NoError(err) testData := "test data" @@ -169,11 +174,11 @@ func (suite *sizeTrackerTestSuite) TestRenameDir() { dst := "dst" testData := "test data" data := []byte(testData) - err := suite.sizeTracker.CreateDir(internal.CreateDirOptions{Name: src, Mode: 0777}) + err := suite.sizeTracker.CreateDir(internal.CreateDirOptions{Name: src, Mode: 0755}) suite.assert.NoError(err) - path := src + "/file" + path := path.Join(src, generateFileName()) for i := 0; i < 5; i++ { - handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: path + strconv.Itoa(i), Mode: 0777}) + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: path + strconv.Itoa(i), Mode: 0644}) suite.assert.NoError(err) _, err = suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}) suite.assert.NoError(err) @@ -192,7 +197,7 @@ func (suite *sizeTrackerTestSuite) TestRenameDir() { func (suite *sizeTrackerTestSuite) TestCreateFile() { defer suite.cleanupTest() // Default is to not create empty files on create file to support immutable storage. - path := "file4" + path := generateFileName() options := internal.CreateFileOptions{Name: path} _, err := suite.sizeTracker.CreateFile(options) suite.assert.NoError(err) @@ -204,9 +209,9 @@ func (suite *sizeTrackerTestSuite) TestCreateFile() { func (suite *sizeTrackerTestSuite) TestDeleteFile() { defer suite.cleanupTest() - path := "file4" + path := generateFileName() - handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0777}) + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0644}) suite.assert.NoError(err) testData := "test data" data := []byte(testData) @@ -224,7 +229,7 @@ func (suite *sizeTrackerTestSuite) TestDeleteFile() { func (suite *sizeTrackerTestSuite) TestDeleteFileError() { defer suite.cleanupTest() - path := "file6" + path := generateFileName() err := suite.sizeTracker.DeleteFile(internal.DeleteFileOptions{Name: path}) suite.assert.Error(err) suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) @@ -233,8 +238,8 @@ func (suite *sizeTrackerTestSuite) TestDeleteFileError() { func (suite *sizeTrackerTestSuite) TestWriteFile() { defer suite.cleanupTest() // Setup - file := "file19" - handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0777}) + file := generateFileName() + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0644}) suite.assert.NoError(err) testData := "test data" @@ -244,10 +249,43 @@ func (suite *sizeTrackerTestSuite) TestWriteFile() { suite.assert.EqualValues(len(data), suite.sizeTracker.mountSize.GetSize()) } +func (suite *sizeTrackerTestSuite) TestWriteFileMultiple() { + defer suite.cleanupTest() + // Setup + file := generateFileName() + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0644}) + suite.assert.NoError(err) + + data := make([]byte, 1024*1024) + _, _ = rand.Read(data) + _, err = suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}) + suite.assert.NoError(err) + suite.assert.EqualValues(len(data), suite.sizeTracker.mountSize.GetSize()) + + _, err = suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: int64(len(data)), Data: data}) + suite.assert.NoError(err) + suite.assert.EqualValues(2*len(data), suite.sizeTracker.mountSize.GetSize()) + + _, err = suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 512, Data: data}) + suite.assert.NoError(err) + suite.assert.EqualValues(2*len(data), suite.sizeTracker.mountSize.GetSize()) + + _, err = suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 2*int64(len(data)) + 512, Data: data}) + suite.assert.NoError(err) + suite.assert.EqualValues(3*len(data)+512, suite.sizeTracker.mountSize.GetSize()) + + _, err = suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 3 * int64(len(data)), Data: data}) + suite.assert.NoError(err) + suite.assert.EqualValues(4*len(data), int(suite.sizeTracker.mountSize.GetSize())) + + err = suite.sizeTracker.DeleteFile(internal.DeleteFileOptions{Name: file}) + suite.assert.NoError(err) +} + func (suite *sizeTrackerTestSuite) TestWriteFileErrorBadFd() { defer suite.cleanupTest() // Setup - file := "file20" + file := generateFileName() handle := handlemap.NewHandle(file) length, err := suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle}) suite.assert.Error(err) @@ -258,8 +296,8 @@ func (suite *sizeTrackerTestSuite) TestWriteFileErrorBadFd() { func (suite *sizeTrackerTestSuite) TestFlushFileEmpty() { defer suite.cleanupTest() // Setup - file := "file21" - handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0777}) + file := generateFileName() + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0644}) suite.assert.NoError(err) // Flush the Empty File @@ -274,8 +312,8 @@ func (suite *sizeTrackerTestSuite) TestFlushFileEmpty() { func (suite *sizeTrackerTestSuite) TestFlushFile() { defer suite.cleanupTest() // Setup - file := "file22" - handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0777}) + file := generateFileName() + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0644}) suite.assert.NoError(err) testData := "test data" data := []byte(testData) @@ -293,7 +331,7 @@ func (suite *sizeTrackerTestSuite) TestFlushFile() { func (suite *sizeTrackerTestSuite) TestFlushFileErrorBadFd() { defer suite.cleanupTest() // Setup - file := "file23" + file := generateFileName() handle := handlemap.NewHandle(file) handle.Flags.Set(handlemap.HandleFlagDirty) err := suite.sizeTracker.FlushFile(internal.FlushFileOptions{Handle: handle}) @@ -304,9 +342,9 @@ func (suite *sizeTrackerTestSuite) TestFlushFileErrorBadFd() { func (suite *sizeTrackerTestSuite) TestRenameFile() { defer suite.cleanupTest() // Setup - src := "source2" - dst := "destination2" - handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: src, Mode: 0666}) + src := "src1" + dst := "dst1" + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: src, Mode: 0644}) suite.assert.NoError(err) testData := "test data" @@ -327,64 +365,141 @@ func (suite *sizeTrackerTestSuite) TestRenameFile() { func (suite *sizeTrackerTestSuite) TestRenameOpenFile() { defer suite.cleanupTest() - src := "source6" - dst := "destination6" + src := "src2" + dst := "dst2" // create source file - handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: src, Mode: 0666}) + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: src, Mode: 0644}) suite.assert.NoError(err) + // write to file handle + data := []byte("newdata") + n, err := suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Data: data}) + suite.assert.NoError(err) + suite.assert.Equal(len(data), n) + // rename open file err = suite.sizeTracker.RenameFile(internal.RenameFileOptions{Src: src, Dst: dst}) suite.assert.NoError(err) + // Close file handle + err = suite.sizeTracker.CloseFile(internal.CloseFileOptions{Handle: handle}) + suite.assert.NoError(err) + fmt.Println(suite.sizeTracker.mountSize.GetSize()) + + suite.assert.EqualValues(len(data), suite.sizeTracker.mountSize.GetSize()) + + err = suite.sizeTracker.DeleteFile(internal.DeleteFileOptions{Name: dst}) + suite.assert.NoError(err) +} + +func (suite *sizeTrackerTestSuite) TestRenameWriteFile() { + defer suite.cleanupTest() + + src := "src3" + dst := "dst3" + + // create source file + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: src, Mode: 0644}) + suite.assert.NoError(err) + // write to file handle data := []byte("newdata") - n, err := suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Data: data}) + n, err := suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Data: data, Offset: 0}) + suite.assert.NoError(err) + suite.assert.Equal(len(data), n) + suite.assert.EqualValues(len(data), suite.sizeTracker.mountSize.GetSize()) + + // rename open file + err = suite.sizeTracker.RenameFile(internal.RenameFileOptions{Src: src, Dst: dst}) + suite.assert.NoError(err) + suite.assert.EqualValues(len(data), suite.sizeTracker.mountSize.GetSize()) + + // write to file handle + n, err = suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Data: data, Offset: int64(len(data))}) suite.assert.NoError(err) suite.assert.Equal(len(data), n) + err = suite.sizeTracker.FlushFile(internal.FlushFileOptions{Handle: handle}) + suite.assert.NoError(err) + // Close file handle err = suite.sizeTracker.CloseFile(internal.CloseFileOptions{Handle: handle}) suite.assert.NoError(err) fmt.Println(suite.sizeTracker.mountSize.GetSize()) - suite.assert.EqualValues(len(data), suite.sizeTracker.mountSize.GetSize()) + suite.assert.EqualValues(2*len(data), suite.sizeTracker.mountSize.GetSize()) + + err = suite.sizeTracker.DeleteFile(internal.DeleteFileOptions{Name: dst}) + suite.assert.NoError(err) } func (suite *sizeTrackerTestSuite) TestTruncateFile() { defer suite.cleanupTest() // Setup - path := "file30" - handle, _ := suite.loopback.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0777}) - suite.loopback.CloseFile(internal.CloseFileOptions{Handle: handle}) + path := generateFileName() + handle, err := suite.loopback.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0644}) + suite.assert.NoError(err) + err = suite.loopback.CloseFile(internal.CloseFileOptions{Handle: handle}) + suite.assert.NoError(err) size := 1024 - err := suite.sizeTracker.TruncateFile(internal.TruncateFileOptions{Name: path, Size: int64(size)}) + err = suite.sizeTracker.TruncateFile(internal.TruncateFileOptions{Name: path, Size: int64(size)}) suite.assert.NoError(err) suite.assert.EqualValues(size, suite.sizeTracker.mountSize.GetSize()) + + err = suite.sizeTracker.DeleteFile(internal.DeleteFileOptions{Name: path}) + suite.assert.NoError(err) +} + +func (suite *sizeTrackerTestSuite) TestTruncateFileOpen() { + defer suite.cleanupTest() + // Setup + path := generateFileName() + handle, err := suite.loopback.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0644}) + suite.assert.NoError(err) + + size := 1024 + err = suite.sizeTracker.TruncateFile(internal.TruncateFileOptions{Name: path, Size: int64(size)}) + suite.assert.NoError(err) + + suite.assert.EqualValues(size, suite.sizeTracker.mountSize.GetSize()) + + err = suite.loopback.CloseFile(internal.CloseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + err = suite.sizeTracker.DeleteFile(internal.DeleteFileOptions{Name: path}) + suite.assert.NoError(err) } func (suite *sizeTrackerTestSuite) TestStatFS() { defer suite.cleanupTest() - file := "file41" - handle, _ := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0777}) + file := generateFileName() + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0644}) + suite.assert.NoError(err) data := make([]byte, 1024*1024) - _, err := suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}) + _, _ = rand.Read(data) + _, err = suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}) suite.assert.NoError(err) err = suite.sizeTracker.FlushFile(internal.FlushFileOptions{Handle: handle}) suite.assert.NoError(err) + suite.assert.EqualValues(len(data), suite.sizeTracker.mountSize.GetSize()) stat, ret, err := suite.sizeTracker.StatFs() suite.assert.True(ret) suite.assert.NoError(err) suite.assert.NotEqual(&common.Statfs_t{}, stat) - suite.assert.Equal(uint64(512), stat.Blocks) + suite.assert.Equal(uint64(len(data)/4096), stat.Blocks) suite.assert.Equal(int64(4096), stat.Bsize) suite.assert.Equal(int64(4096), stat.Frsize) suite.assert.Equal(uint64(255), stat.Namemax) + + err = suite.loopback.CloseFile(internal.CloseFileOptions{Handle: handle}) + suite.assert.NoError(err) + err = suite.sizeTracker.DeleteFile(internal.DeleteFileOptions{Name: file}) + suite.assert.NoError(err) } // In order for 'go test' to run this suite, we need to create From d1801451419a8e727dc500eed406339bbd6bfa09 Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Wed, 22 Jan 2025 11:15:28 -0700 Subject: [PATCH 04/23] Cleanup --- component/size_tracker/journal.go | 4 ++++ component/size_tracker/size_tracker.go | 3 +-- component/size_tracker/size_tracker_test.go | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/component/size_tracker/journal.go b/component/size_tracker/journal.go index 927b88abc..9a9371767 100644 --- a/component/size_tracker/journal.go +++ b/component/size_tracker/journal.go @@ -1,14 +1,18 @@ /* Licensed under the MIT License . + Copyright © 2023-2025 Seagate Technology LLC and/or its Affiliates + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE diff --git a/component/size_tracker/size_tracker.go b/component/size_tracker/size_tracker.go index 1d9fcc0e6..b0a022e27 100644 --- a/component/size_tracker/size_tracker.go +++ b/component/size_tracker/size_tracker.go @@ -2,7 +2,6 @@ Licensed under the MIT License . Copyright © 2023-2025 Seagate Technology LLC and/or its Affiliates - Copyright © 2020-2024 Microsoft Corporation. All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -56,7 +55,7 @@ type SizeTrackerOptions struct { const compName = "size_tracker" const blockSize = int64(4096) -const default_journal_name = "directory_size.dat" +const default_journal_name = "mount_size.dat" // Verification to check satisfaction criteria with Component Interface var _ internal.Component = &SizeTracker{} diff --git a/component/size_tracker/size_tracker_test.go b/component/size_tracker/size_tracker_test.go index 7cd8eb63a..3a2db2dc1 100644 --- a/component/size_tracker/size_tracker_test.go +++ b/component/size_tracker/size_tracker_test.go @@ -2,7 +2,6 @@ Licensed under the MIT License . Copyright © 2023-2025 Seagate Technology LLC and/or its Affiliates - Copyright © 2020-2024 Microsoft Corporation. All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From c59096397745d49ab674e22f52ce3368efdbfec4 Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Thu, 23 Jan 2025 09:48:24 -0700 Subject: [PATCH 05/23] Finalize unit tests and fix issue with symlink --- .github/workflows/code-coverage.yml | 46 +++++++++ component/size_tracker/size_tracker.go | 18 +--- component/size_tracker/size_tracker_test.go | 24 +++++ test-scripts/e2e-test.sh | 2 +- test-scripts/e2e-test_windows.ps1 | 2 +- test/e2e_tests/dir_test.go | 101 ++++++++++++++++++-- test/e2e_tests/statfs_linux.go | 38 ++++++++ test/e2e_tests/statfs_windows.go | 48 ++++++++++ testdata/config/azure_key_size_tracker.yaml | 38 ++++++++ testdata/config/s3_key_size_tracker.yaml | 35 +++++++ 10 files changed, 328 insertions(+), 24 deletions(-) create mode 100644 test/e2e_tests/statfs_linux.go create mode 100644 test/e2e_tests/statfs_windows.go create mode 100644 testdata/config/azure_key_size_tracker.yaml create mode 100644 testdata/config/s3_key_size_tracker.yaml diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index fc9196b75..6866ce5db 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -161,6 +161,30 @@ jobs: sudo fusermount -u ${{ env.MOUNT_DIR }} sleep 5 + - name: Create Config File - Block Blob with Size Tracker + env: + NIGHTLY_STO_ACC_NAME: "${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}" + NIGHTLY_STO_ACC_KEY: "${{ secrets.NIGHTLY_STO_BLOB_ACC_KEY }}" + ACCOUNT_TYPE: block + ACCOUNT_ENDPOINT: https://${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}.blob.core.windows.net + VERBOSE_LOG: false + USE_HTTP: false + run: "./cloudfuse.test -test.v -test.coverprofile=${{ env.WORK_DIR }}/cloudfuse_gentest1.cov gen-test-config --config-file=azure_key_size_tracker.yaml --container-name=${{ matrix.containerName }} --temp-path=${{ env.TEMP_DIR }} --output-file=${{ env.cloudfuse_CFG }}" + + - name: Block Blob Coverage with Size Tracker + run: |- + rm -rf ${{ env.MOUNT_DIR }}/* + rm -rf ${{ env.TEMP_DIR }}/* + ./cloudfuse.test -test.v -test.coverprofile=${{ env.WORK_DIR }}/cloudfuse_block.cov mount ${{ env.MOUNT_DIR }} --config-file=${{ env.cloudfuse_CFG }} --foreground=true & + sleep 10 + ps -aux | grep cloudfuse + rm -rf ${{ env.MOUNT_DIR }}/* + cd test/e2e_tests + go test -v -timeout=7200s ./... -args -mnt-path=${{ env.MOUNT_DIR }} -tmp-path=${{ env.TEMP_DIR }} + cd - + sudo fusermount -u ${{ env.MOUNT_DIR }} + sleep 5 + - name: Create Config File - ADLS env: NIGHTLY_STO_ACC_NAME: "${{ secrets.AZTEST_ADLS_ACC_NAME }}" @@ -252,6 +276,28 @@ jobs: sudo fusermount -u ${{ env.MOUNT_DIR }} sleep 5 + - name: Create Config File - S3 with Size Tracker + env: + S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}" + S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}" + S3_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" + S3_REGION: "${{ secrets.S3TEST_REGION }}" + S3_SECRET_KEY: "${{ secrets.S3TEST_SECRET_KEY }}" + run: "./cloudfuse.test gen-test-config --config-file=s3_key_size_tracker.yaml --temp-path=${{ env.TEMP_DIR }} --output-file=${{ env.cloudfuse_CFG }}" + + - name: S3 Coverage with Size Tracker + run: |- + rm -rf ${{ env.MOUNT_DIR }}/* + rm -rf ${{ env.TEMP_DIR }}/* + ./cloudfuse.test -test.v -test.coverprofile=${{ env.WORK_DIR }}/cloudfuse_s3.cov mount ${{ env.MOUNT_DIR }} --config-file=${{ env.cloudfuse_CFG }} --foreground=true & + sleep 10 + ps -aux | grep cloudfuse + cd test/e2e_tests + go test -v -timeout=7200s ./... -args -mnt-path=${{ env.MOUNT_DIR }} -tmp-path=${{ env.TEMP_DIR }} + cd - + sudo fusermount -u ${{ env.MOUNT_DIR }} + sleep 5 + - name: Create Config File - Stream env: NIGHTLY_STO_ACC_NAME: "${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}" diff --git a/component/size_tracker/size_tracker.go b/component/size_tracker/size_tracker.go index b0a022e27..d59916c1a 100644 --- a/component/size_tracker/size_tracker.go +++ b/component/size_tracker/size_tracker.go @@ -149,8 +149,8 @@ func (st *SizeTracker) DeleteFile(options internal.DeleteFileOptions) error { err := st.NextComponent().DeleteFile(options) - // File already exists and delete succeeded so remove old file size - if err == nil && getAttrErr == nil { + // If the file is a symlink then it has no size so don't change the size + if err == nil && getAttrErr == nil && !attr.IsSymlink() { st.mountSize.Subtract(uint64(attr.Size)) } @@ -293,20 +293,6 @@ func (st *SizeTracker) FlushFile(options internal.FlushFileOptions) error { return nil } -// Symlink operations -func (st *SizeTracker) CreateLink(options internal.CreateLinkOptions) error { - attr, _ := st.NextComponent().GetAttr(internal.GetAttrOptions{Name: options.Name}) - - err := st.NextComponent().CreateLink(options) - - // File already exists but create succeeded so remove old file size - if err == nil && attr != nil { - st.mountSize.Subtract(uint64(attr.Size)) - } - - return err -} - // Filesystem level operations func (st *SizeTracker) StatFs() (*common.Statfs_t, bool, error) { log.Trace("SizeTracker::StatFs") diff --git a/component/size_tracker/size_tracker_test.go b/component/size_tracker/size_tracker_test.go index 3a2db2dc1..0a62b86b5 100644 --- a/component/size_tracker/size_tracker_test.go +++ b/component/size_tracker/size_tracker_test.go @@ -472,6 +472,30 @@ func (suite *sizeTrackerTestSuite) TestTruncateFileOpen() { suite.assert.NoError(err) } +func (suite *sizeTrackerTestSuite) TestSymlink() { + defer suite.cleanupTest() + // Setup + file := generateFileName() + symlink := generateFileName() + ".lnk" + handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0644}) + suite.assert.NoError(err) + + testData := "test data" + data := []byte(testData) + _, err = suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}) + suite.assert.NoError(err) + suite.assert.EqualValues(len(data), suite.sizeTracker.mountSize.GetSize()) + + suite.sizeTracker.CreateLink(internal.CreateLinkOptions{Name: symlink, Target: file}) + suite.assert.EqualValues(len(data), suite.sizeTracker.mountSize.GetSize()) + + suite.sizeTracker.DeleteFile(internal.DeleteFileOptions{Name: symlink}) + suite.assert.EqualValues(len(data), suite.sizeTracker.mountSize.GetSize()) + + err = suite.sizeTracker.DeleteFile(internal.DeleteFileOptions{Name: file}) + suite.assert.NoError(err) +} + func (suite *sizeTrackerTestSuite) TestStatFS() { defer suite.cleanupTest() diff --git a/test-scripts/e2e-test.sh b/test-scripts/e2e-test.sh index 7b03320e3..09c36a03e 100755 --- a/test-scripts/e2e-test.sh +++ b/test-scripts/e2e-test.sh @@ -19,7 +19,7 @@ echo "-------------------------------------------------------------------" echo "Starting End-to-End Test" # run test -go test -v -timeout=2h ../test/e2e_tests/data_validation_test.go ../test/e2e_tests/dir_test.go ../test/e2e_tests/file_test.go -args -mnt-path="$MOUNT_DIR" -adls=false -clone=false -tmp-path="$MOUNT_TMP" -quick-test=true -stream-direct-test=false -distro-name="Ubuntu" +go test -v -timeout=2h ../test/e2e_tests/dir_test.go ../test/e2e_tests/statfs_linux.go -args -mnt-path="$MOUNT_DIR" -adls=false -clone=false -tmp-path="$MOUNT_TMP" -stream-direct-test=false # Cleanup test source ./helper/cleanup.sh diff --git a/test-scripts/e2e-test_windows.ps1 b/test-scripts/e2e-test_windows.ps1 index 18d583a3b..35be96048 100755 --- a/test-scripts/e2e-test_windows.ps1 +++ b/test-scripts/e2e-test_windows.ps1 @@ -6,7 +6,7 @@ Write-Output "-------------------------------------------------------------------" Write-Output "Starting e2e Tests" -go test -v -timeout=2h ../test/e2e_tests/data_validation_test.go ../test/e2e_tests/dir_test.go ../test/e2e_tests/file_test.go -args -mnt-path="${mount_dir}" -adls=false -clone=false -tmp-path="${mount_tmp}" -quick-test=true -stream-direct-test=true -distro-name="windows" +go test -v -timeout=2h ../test/e2e_tests/data_validation_test.go ../test/e2e_tests/dir_test.go ../test/e2e_tests/file_test.go ../test/e2e_tests/statfs_windows.go -args -mnt-path="${mount_dir}" -adls=false -clone=false -tmp-path="${mount_tmp}" -quick-test=true -stream-direct-test=true -distro-name="windows" #go test -v -timeout=2h ../test/e2e_tests/file_test.go -args -mnt-path="${mount_dir}" -adls=false -clone=false -stream-direct-test=false -distro-name="windows" diff --git a/test/e2e_tests/dir_test.go b/test/e2e_tests/dir_test.go index 389f8f6ce..db95018f8 100644 --- a/test/e2e_tests/dir_test.go +++ b/test/e2e_tests/dir_test.go @@ -99,16 +99,19 @@ func (suite *dirTestSuite) dirTestCleanup(toRemove []string) { // # Create Directory with a simple name func (suite *dirTestSuite) TestDirCreateSimple() { + suite.EqualValues(0, DiskSize(pathPtr)) dirName := filepath.Join(suite.testPath, "test1") err := os.Mkdir(dirName, 0777) suite.NoError(err) + suite.EqualValues(0, DiskSize(pathPtr)) // cleanup suite.dirTestCleanup([]string{dirName}) } // # Create Directory that already exists func (suite *dirTestSuite) TestDirCreateDuplicate() { + suite.EqualValues(0, DiskSize(pathPtr)) dirName := filepath.Join(suite.testPath, "test1") err := os.Mkdir(dirName, 0777) suite.NoError(err) @@ -121,6 +124,7 @@ func (suite *dirTestSuite) TestDirCreateDuplicate() { suite.Contains(err.Error(), "file exists") } + suite.EqualValues(0, DiskSize(pathPtr)) // cleanup suite.dirTestCleanup([]string{dirName}) } @@ -131,10 +135,12 @@ func (suite *dirTestSuite) TestDirCreateSplChar() { fmt.Println("Skipping TestDirCreateSplChar on Windows") return } + suite.EqualValues(0, DiskSize(pathPtr)) dirName := filepath.Join(suite.testPath, "@#$^&*()_+=-{}[]|?><.,~") err := os.Mkdir(dirName, 0777) suite.NoError(err) + suite.EqualValues(0, DiskSize(pathPtr)) // cleanup suite.dirTestCleanup([]string{dirName}) } @@ -145,16 +151,19 @@ func (suite *dirTestSuite) TestDirCreateSlashChar() { fmt.Println("Skipping TestDirCreateSlashChar on Windows") return } + suite.EqualValues(0, DiskSize(pathPtr)) dirName := filepath.Join(suite.testPath, "PRQ\\STUV") err := os.Mkdir(dirName, 0777) suite.NoError(err) + suite.EqualValues(0, DiskSize(pathPtr)) // cleanup suite.dirTestCleanup([]string{dirName}) } // # Rename a directory func (suite *dirTestSuite) TestDirRename() { + suite.EqualValues(0, DiskSize(pathPtr)) dirName := filepath.Join(suite.testPath, "test1") err := os.Mkdir(dirName, 0777) suite.NoError(err) @@ -165,12 +174,14 @@ func (suite *dirTestSuite) TestDirRename() { suite.NoDirExists(dirName) + suite.EqualValues(0, DiskSize(pathPtr)) // cleanup suite.dirTestCleanup([]string{newName}) } // # Move an empty directory func (suite *dirTestSuite) TestDirMoveEmpty() { + suite.EqualValues(0, DiskSize(pathPtr)) dir2Name := filepath.Join(suite.testPath, "test2") err := os.Mkdir(dir2Name, 0777) suite.NoError(err) @@ -183,12 +194,14 @@ func (suite *dirTestSuite) TestDirMoveEmpty() { time.Sleep(1 * time.Second) suite.NoError(err) + suite.EqualValues(0, DiskSize(pathPtr)) // cleanup suite.dirTestCleanup([]string{dir3Name}) } // # Move an non-empty directory func (suite *dirTestSuite) TestDirMoveNonEmpty() { + suite.EqualValues(0, DiskSize(pathPtr)) dir2Name := filepath.Join(suite.testPath, "test2NE") err := os.Mkdir(dir2Name, 0777) suite.NoError(err) @@ -209,21 +222,25 @@ func (suite *dirTestSuite) TestDirMoveNonEmpty() { time.Sleep(1 * time.Second) suite.NoError(err) + suite.EqualValues(0, DiskSize(pathPtr)) // cleanup - suite.dirTestCleanup([]string{dir3Name}) + suite.dirTestCleanup([]string{file1Name, dir3Name}) } // # Delete non-empty directory func (suite *dirTestSuite) TestDirDeleteEmpty() { + suite.EqualValues(0, DiskSize(pathPtr)) dirName := filepath.Join(suite.testPath, "test1_new") err := os.Mkdir(dirName, 0777) suite.NoError(err) + suite.EqualValues(0, DiskSize(pathPtr)) suite.dirTestCleanup([]string{dirName}) } // # Delete non-empty directory func (suite *dirTestSuite) TestDirDeleteNonEmpty() { + suite.EqualValues(0, DiskSize(pathPtr)) dir3Name := filepath.Join(suite.testPath, "test3NE") err := os.Mkdir(dir3Name, 0777) suite.NoError(err) @@ -240,6 +257,7 @@ func (suite *dirTestSuite) TestDirDeleteNonEmpty() { suite.Contains(err.Error(), "directory not empty") } + suite.EqualValues(0, DiskSize(pathPtr)) // cleanup suite.dirTestCleanup([]string{dir3Name}) } @@ -269,6 +287,7 @@ func (suite *dirTestSuite) TestDirDeleteNonEmpty() { // # Get stats of a directory func (suite *dirTestSuite) TestDirGetStats() { + suite.EqualValues(0, DiskSize(pathPtr)) dirName := filepath.Join(suite.testPath, "test3") err := os.Mkdir(dirName, 0777) suite.NoError(err) @@ -285,6 +304,8 @@ func (suite *dirTestSuite) TestDirGetStats() { suite.Equal("test3", stat.Name()) suite.GreaterOrEqual(float64(1), modTineDiff.Hours()) } + + suite.EqualValues(0, DiskSize(pathPtr)) // Cleanup suite.dirTestCleanup([]string{dirName}) } @@ -296,6 +317,7 @@ func (suite *dirTestSuite) TestDirChmod() { return } if suite.adlsTest == true { + suite.EqualValues(0, DiskSize(pathPtr)) dirName := filepath.Join(suite.testPath, "testchmod") err := os.Mkdir(dirName, 0777) suite.NoError(err) @@ -307,12 +329,14 @@ func (suite *dirTestSuite) TestDirChmod() { suite.NoError(err) suite.Equal("-rwxr--r--", stat.Mode().Perm().String()) + suite.EqualValues(0, DiskSize(pathPtr)) suite.dirTestCleanup([]string{dirName}) } } // # List directory func (suite *dirTestSuite) TestDirList() { + suite.EqualValues(0, DiskSize(pathPtr)) testDir := filepath.Join(suite.testPath, "bigTestDir") err := os.Mkdir(testDir, 0777) suite.NoError(err) @@ -336,6 +360,7 @@ func (suite *dirTestSuite) TestDirList() { suite.NoError(err) suite.Len(files, 4) + suite.EqualValues(0, DiskSize(pathPtr)) // Cleanup suite.dirTestCleanup([]string{testDir}) } @@ -383,6 +408,7 @@ func (suite *dirTestSuite) TestDirRenameFull() { fmt.Println("Skipping this test case for stream direct") return } + suite.EqualValues(0, DiskSize(pathPtr)) dirName := filepath.Join(suite.testPath, "full_dir") newName := filepath.Join(suite.testPath, "full_dir_rename") fileName := filepath.Join(dirName, "test_file_") @@ -399,9 +425,13 @@ func (suite *dirTestSuite) TestDirRenameFull() { suite.NoError(err) } + suite.EqualValues(10*len(suite.medBuff), DiskSize(pathPtr)) + err = os.Rename(dirName, newName) suite.NoError(err) + suite.EqualValues(10*len(suite.medBuff), DiskSize(pathPtr)) + // Deleted directory shall not be present in the container now suite.NoDirExists(dirName) @@ -499,8 +529,9 @@ func (suite *dirTestSuite) TestGitStash() { cmd = exec.Command("tar", "-zxvf", tarName, "--directory", dirName) _, _ = cmd.Output() - os.RemoveAll(dirName) - os.Remove("libfuse.tar.gz") + os.Remove(tarName) + + suite.dirTestCleanup([]string{dirName}) } } @@ -515,6 +546,7 @@ func (suite *dirTestSuite) TestReadDirLink() { return } + suite.EqualValues(0, DiskSize(pathPtr)) dirName := filepath.Join(suite.testPath, "test_hns") err := os.Mkdir(dirName, 0777) suite.NoError(err) @@ -527,6 +559,15 @@ func (suite *dirTestSuite) TestReadDirLink() { err = os.WriteFile(fileName, suite.minBuff, 0777) suite.NoError(err) + // Write three more files so one blokc, 4096 bytes, is filled + for i := 0; i < 3; i++ { + newFile := fileName + strconv.Itoa(i) + err := os.WriteFile(newFile, suite.minBuff, 0777) + suite.NoError(err) + } + + suite.EqualValues(4*len(suite.minBuff), DiskSize(pathPtr)) + symName := filepath.Join(suite.testPath, "dirlink.lnk") err = os.Symlink(dirName, symName) suite.NoError(err) @@ -568,11 +609,61 @@ func (suite *dirTestSuite) TestReadDirLink() { // validating data suite.Equal(data1, data2) + suite.EqualValues(4*len(suite.minBuff), DiskSize(pathPtr)) + suite.dirTestCleanup([]string{dirName}) err = os.Remove(symName) suite.NoError(err) } +func (suite *dirTestSuite) TestStatfs() { + suite.EqualValues(0, DiskSize(pathPtr)) + + dirName := filepath.Join(suite.testPath, "test_statfs") + err := os.Mkdir(dirName, 0777) + suite.NoError(err) + + fileName := filepath.Join(dirName, "small_file_") + for i := 0; i < 12; i++ { + newFile := fileName + strconv.Itoa(i) + err := os.WriteFile(newFile, suite.minBuff, 0777) + suite.NoError(err) + } + suite.EqualValues(12*len(suite.minBuff), DiskSize(pathPtr)) + + for i := 0; i < 12; i++ { + file := fileName + strconv.Itoa(i) + err := os.Truncate(file, 4096) + suite.NoError(err) + } + suite.EqualValues(12*4096, DiskSize(pathPtr)) + + for i := 0; i < 12; i++ { + file := fileName + strconv.Itoa(i) + err := os.WriteFile(file, suite.medBuff, 0777) + suite.NoError(err) + } + suite.EqualValues(12*len(suite.medBuff), DiskSize(pathPtr)) + + renameFile := filepath.Join(dirName, "small_file_rename") + for i := 0; i < 12; i++ { + oldFile := fileName + strconv.Itoa(i) + newFile := renameFile + strconv.Itoa(i) + err := os.Rename(oldFile, newFile) + suite.NoError(err) + } + suite.EqualValues(12*len(suite.medBuff), DiskSize(pathPtr)) + + for i := 0; i < 12; i++ { + file := renameFile + strconv.Itoa(i) + err := os.Truncate(file, 4096) + suite.NoError(err) + } + suite.EqualValues(12*4096, DiskSize(pathPtr)) + + suite.dirTestCleanup([]string{dirName}) +} + // -------------- Main Method ------------------- func TestDirTestSuite(t *testing.T) { initDirFlags() @@ -584,14 +675,11 @@ func TestDirTestSuite(t *testing.T) { // Generate random test dir name where our End to End test run is contained testDirName := getTestDirName(10) - fmt.Println(testDirName) // Create directory for testing the End to End test on mount path dirTest.testPath = filepath.Join(pathPtr, testDirName) - fmt.Println(dirTest.testPath) dirTest.testCachePath = filepath.Join(tempPathPtr, testDirName) - fmt.Println(dirTest.testCachePath) if adlsPtr == "true" || adlsPtr == "True" { fmt.Println("ADLS Testing...") @@ -629,5 +717,6 @@ func init() { regDirTestFlag(&adlsPtr, "adls", "", "Account is ADLS or not") regDirTestFlag(&clonePtr, "clone", "", "Git clone test is enable or not") regDirTestFlag(&tempPathPtr, "tmp-path", "", "Cache dir path") + regDirTestFlag(&streamDirectPtr, "stream-direct-test", "false", "Run stream direct tests") regDirTestFlag(&enableSymlinkADLS, "enable-symlink-adls", "false", "Enable symlink support for ADLS accounts") } diff --git a/test/e2e_tests/statfs_linux.go b/test/e2e_tests/statfs_linux.go new file mode 100644 index 000000000..c87ecd6cc --- /dev/null +++ b/test/e2e_tests/statfs_linux.go @@ -0,0 +1,38 @@ +//go:build linux + +/* + Licensed under the MIT License . + + Copyright © 2023-2025 Seagate Technology LLC and/or its Affiliates + Copyright © 2020-2024 Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package e2e_tests + +import ( + "golang.org/x/sys/unix" +) + +func DiskSize(path string) int { + buf := &unix.Statfs_t{} + _ = unix.Statfs(path, buf) + return int((buf.Blocks - buf.Bavail) * uint64(buf.Bsize)) +} diff --git a/test/e2e_tests/statfs_windows.go b/test/e2e_tests/statfs_windows.go new file mode 100644 index 000000000..f46de2f04 --- /dev/null +++ b/test/e2e_tests/statfs_windows.go @@ -0,0 +1,48 @@ +//go:build windows + +/* + Licensed under the MIT License . + + Copyright © 2023-2025 Seagate Technology LLC and/or its Affiliates + Copyright © 2020-2024 Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package e2e_tests + +import ( + "golang.org/x/sys/windows" +) + +func DiskSize(path string) int { + var free, total, avail uint64 + + // Get path to the cache + pathPtr, err := windows.UTF16PtrFromString(path) + if err != nil { + return 0 + } + err = windows.GetDiskFreeSpaceEx(pathPtr, &free, &total, &avail) + if err != nil { + return 0 + } + + return int(total) +} diff --git a/testdata/config/azure_key_size_tracker.yaml b/testdata/config/azure_key_size_tracker.yaml new file mode 100644 index 000000000..34a942f72 --- /dev/null +++ b/testdata/config/azure_key_size_tracker.yaml @@ -0,0 +1,38 @@ +logging: + level: log_debug + file-path: "cloudfuse-logs.txt" + type: base + +components: + - libfuse + - file_cache + - size_tracker + - attr_cache + - azstorage + +libfuse: + attribute-expiration-sec: 0 + entry-expiration-sec: 0 + negative-entry-expiration-sec: 0 + ignore-open-flags: true + +file_cache: + path: { 1 } + timeout-sec: 30 + max-size-mb: 2048 + allow-non-empty-temp: true + cleanup-on-start: true + +attr_cache: + timeout-sec: 3600 + enable-symlinks: true + +azstorage: + type: { ACCOUNT_TYPE } + endpoint: { ACCOUNT_ENDPOINT } + use-http: { USE_HTTP } + account-name: { NIGHTLY_STO_ACC_NAME } + account-key: { NIGHTLY_STO_ACC_KEY } + mode: key + container: { 0 } + tier: hot diff --git a/testdata/config/s3_key_size_tracker.yaml b/testdata/config/s3_key_size_tracker.yaml new file mode 100644 index 000000000..453092ec7 --- /dev/null +++ b/testdata/config/s3_key_size_tracker.yaml @@ -0,0 +1,35 @@ +logging: + level: log_debug + file-path: "lyvecloudfuse-logs.txt" + type: base + +components: + - libfuse + - file_cache + - size_tracker + - attr_cache + - s3storage + +libfuse: + attribute-expiration-sec: 0 + entry-expiration-sec: 0 + negative-entry-expiration-sec: 0 + ignore-open-flags: true + +file_cache: + path: { 1 } + timeout-sec: 30 + max-size-mb: 2048 + allow-non-empty-temp: true + cleanup-on-start: true + +attr_cache: + timeout-sec: 3600 + enable-symlinks: true + +s3storage: + bucket-name: { S3_BUCKET_NAME } + endpoint: { S3_ENDPOINT } + key-id: { S3_KEY_ID } + region: { S3_REGION } + secret-key: { S3_SECRET_KEY } From d6c332c95d73f4fc2a8f428ba8aff56de921e6b4 Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Thu, 23 Jan 2025 09:53:50 -0700 Subject: [PATCH 06/23] Fix spelling --- test/e2e_tests/dir_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e_tests/dir_test.go b/test/e2e_tests/dir_test.go index db95018f8..4d718626e 100644 --- a/test/e2e_tests/dir_test.go +++ b/test/e2e_tests/dir_test.go @@ -559,7 +559,7 @@ func (suite *dirTestSuite) TestReadDirLink() { err = os.WriteFile(fileName, suite.minBuff, 0777) suite.NoError(err) - // Write three more files so one blokc, 4096 bytes, is filled + // Write three more files so one block, 4096 bytes, is filled for i := 0; i < 3; i++ { newFile := fileName + strconv.Itoa(i) err := os.WriteFile(newFile, suite.minBuff, 0777) From 98710e208dd7977bd48df30415eac7f45384f55d Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Thu, 23 Jan 2025 12:27:28 -0700 Subject: [PATCH 07/23] Fix size test to run only for S3, cleanup tests for Windows --- .github/workflows/code-coverage.yml | 4 +- component/size_tracker/size_tracker.go | 7 +- component/size_tracker/size_tracker_test.go | 21 ++- test/e2e_tests/dir_test.go | 154 +++++++++++++++----- test/s3cleanup/s3cleanup_test.go | 110 ++++++++++++++ 5 files changed, 250 insertions(+), 46 deletions(-) create mode 100644 test/s3cleanup/s3cleanup_test.go diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 6866ce5db..1d2f2391a 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -249,7 +249,7 @@ jobs: sleep 10 ps -aux | grep cloudfuse cd test/e2e_tests - go test -v -timeout=7200s ./... -args -mnt-path=${{ env.MOUNT_DIR }} -tmp-path=${{ env.TEMP_DIR }} + go test -v -timeout=7200s ./... -args -mnt-path=${{ env.MOUNT_DIR }} -tmp-path=${{ env.TEMP_DIR }} -s3storage=true cd - sudo fusermount -u ${{ env.MOUNT_DIR }} sleep 5 @@ -293,7 +293,7 @@ jobs: sleep 10 ps -aux | grep cloudfuse cd test/e2e_tests - go test -v -timeout=7200s ./... -args -mnt-path=${{ env.MOUNT_DIR }} -tmp-path=${{ env.TEMP_DIR }} + go test -v -timeout=7200s ./... -args -mnt-path=${{ env.MOUNT_DIR }} -tmp-path=${{ env.TEMP_DIR }} -s3storage=true cd - sudo fusermount -u ${{ env.MOUNT_DIR }} sleep 5 diff --git a/component/size_tracker/size_tracker.go b/component/size_tracker/size_tracker.go index d59916c1a..2ccddb68a 100644 --- a/component/size_tracker/size_tracker.go +++ b/component/size_tracker/size_tracker.go @@ -55,6 +55,7 @@ type SizeTrackerOptions struct { const compName = "size_tracker" const blockSize = int64(4096) +const emptyDirSize = uint64(4096) const default_journal_name = "mount_size.dat" // Verification to check satisfaction criteria with Component Interface @@ -118,12 +119,6 @@ func (st *SizeTracker) Priority() internal.ComponentPriority { func (st *SizeTracker) OnConfigChange() { } -// Directory operations -func (st *SizeTracker) DeleteDir(options internal.DeleteDirOptions) error { - // Libfuse only allows deleting empty directories, so we should not need to update the size here - return st.NextComponent().DeleteDir(options) -} - func (st *SizeTracker) RenameDir(options internal.RenameDirOptions) error { // Rename dir should not allow renaming files into a directory that already exists so we should not // need to update the size here. diff --git a/component/size_tracker/size_tracker_test.go b/component/size_tracker/size_tracker_test.go index 0a62b86b5..ad6a696cf 100644 --- a/component/size_tracker/size_tracker_test.go +++ b/component/size_tracker/size_tracker_test.go @@ -31,6 +31,7 @@ import ( "os" "path" "path/filepath" + "runtime" "strconv" "strings" "testing" @@ -198,10 +199,13 @@ func (suite *sizeTrackerTestSuite) TestCreateFile() { // Default is to not create empty files on create file to support immutable storage. path := generateFileName() options := internal.CreateFileOptions{Name: path} - _, err := suite.sizeTracker.CreateFile(options) + h, err := suite.sizeTracker.CreateFile(options) suite.assert.NoError(err) suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) + err = suite.sizeTracker.CloseFile(internal.CloseFileOptions{Handle: h}) + suite.assert.NoError(err) + err = suite.sizeTracker.DeleteFile(internal.DeleteFileOptions{Name: path}) suite.assert.NoError(err) } @@ -277,6 +281,9 @@ func (suite *sizeTrackerTestSuite) TestWriteFileMultiple() { suite.assert.NoError(err) suite.assert.EqualValues(4*len(data), int(suite.sizeTracker.mountSize.GetSize())) + err = suite.sizeTracker.CloseFile(internal.CloseFileOptions{Handle: handle}) + suite.assert.NoError(err) + err = suite.sizeTracker.DeleteFile(internal.DeleteFileOptions{Name: file}) suite.assert.NoError(err) } @@ -362,6 +369,10 @@ func (suite *sizeTrackerTestSuite) TestRenameFile() { } func (suite *sizeTrackerTestSuite) TestRenameOpenFile() { + if runtime.GOOS == "windows" { + fmt.Println("Skipping test on Windows") + return + } defer suite.cleanupTest() src := "src2" @@ -393,6 +404,10 @@ func (suite *sizeTrackerTestSuite) TestRenameOpenFile() { } func (suite *sizeTrackerTestSuite) TestRenameWriteFile() { + if runtime.GOOS == "windows" { + fmt.Println("Skipping test on Windows") + return + } defer suite.cleanupTest() src := "src3" @@ -473,6 +488,10 @@ func (suite *sizeTrackerTestSuite) TestTruncateFileOpen() { } func (suite *sizeTrackerTestSuite) TestSymlink() { + if runtime.GOOS == "windows" { + fmt.Println("Skipping test on Windows") + return + } defer suite.cleanupTest() // Setup file := generateFileName() diff --git a/test/e2e_tests/dir_test.go b/test/e2e_tests/dir_test.go index 4d718626e..f02e9740b 100644 --- a/test/e2e_tests/dir_test.go +++ b/test/e2e_tests/dir_test.go @@ -48,6 +48,7 @@ type dirTestSuite struct { suite.Suite testPath string adlsTest bool + s3storageTest bool testCachePath string minBuff []byte medBuff []byte @@ -60,6 +61,7 @@ var adlsPtr string var clonePtr string var streamDirectPtr string var enableSymlinkADLS string +var s3storagePtr string func regDirTestFlag(p *string, name string, value string, usage string) { if flag.Lookup(name) == nil { @@ -78,6 +80,7 @@ func initDirFlags() { clonePtr = getDirTestFlag("clone") streamDirectPtr = getDirTestFlag("stream-direct-test") enableSymlinkADLS = getDirTestFlag("enable-symlink-adls") + s3storagePtr = getDirTestFlag("s3storage") } func getTestDirName(n int) string { @@ -99,19 +102,25 @@ func (suite *dirTestSuite) dirTestCleanup(toRemove []string) { // # Create Directory with a simple name func (suite *dirTestSuite) TestDirCreateSimple() { - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } dirName := filepath.Join(suite.testPath, "test1") err := os.Mkdir(dirName, 0777) suite.NoError(err) - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } // cleanup suite.dirTestCleanup([]string{dirName}) } // # Create Directory that already exists func (suite *dirTestSuite) TestDirCreateDuplicate() { - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } dirName := filepath.Join(suite.testPath, "test1") err := os.Mkdir(dirName, 0777) suite.NoError(err) @@ -124,7 +133,9 @@ func (suite *dirTestSuite) TestDirCreateDuplicate() { suite.Contains(err.Error(), "file exists") } - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } // cleanup suite.dirTestCleanup([]string{dirName}) } @@ -135,12 +146,16 @@ func (suite *dirTestSuite) TestDirCreateSplChar() { fmt.Println("Skipping TestDirCreateSplChar on Windows") return } - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } dirName := filepath.Join(suite.testPath, "@#$^&*()_+=-{}[]|?><.,~") err := os.Mkdir(dirName, 0777) suite.NoError(err) - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } // cleanup suite.dirTestCleanup([]string{dirName}) } @@ -151,19 +166,25 @@ func (suite *dirTestSuite) TestDirCreateSlashChar() { fmt.Println("Skipping TestDirCreateSlashChar on Windows") return } - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } dirName := filepath.Join(suite.testPath, "PRQ\\STUV") err := os.Mkdir(dirName, 0777) suite.NoError(err) - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } // cleanup suite.dirTestCleanup([]string{dirName}) } // # Rename a directory func (suite *dirTestSuite) TestDirRename() { - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } dirName := filepath.Join(suite.testPath, "test1") err := os.Mkdir(dirName, 0777) suite.NoError(err) @@ -174,14 +195,18 @@ func (suite *dirTestSuite) TestDirRename() { suite.NoDirExists(dirName) - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } // cleanup suite.dirTestCleanup([]string{newName}) } // # Move an empty directory func (suite *dirTestSuite) TestDirMoveEmpty() { - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } dir2Name := filepath.Join(suite.testPath, "test2") err := os.Mkdir(dir2Name, 0777) suite.NoError(err) @@ -194,14 +219,18 @@ func (suite *dirTestSuite) TestDirMoveEmpty() { time.Sleep(1 * time.Second) suite.NoError(err) - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } // cleanup suite.dirTestCleanup([]string{dir3Name}) } // # Move an non-empty directory func (suite *dirTestSuite) TestDirMoveNonEmpty() { - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } dir2Name := filepath.Join(suite.testPath, "test2NE") err := os.Mkdir(dir2Name, 0777) suite.NoError(err) @@ -222,25 +251,33 @@ func (suite *dirTestSuite) TestDirMoveNonEmpty() { time.Sleep(1 * time.Second) suite.NoError(err) - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } // cleanup suite.dirTestCleanup([]string{file1Name, dir3Name}) } // # Delete non-empty directory func (suite *dirTestSuite) TestDirDeleteEmpty() { - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } dirName := filepath.Join(suite.testPath, "test1_new") err := os.Mkdir(dirName, 0777) suite.NoError(err) - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } suite.dirTestCleanup([]string{dirName}) } // # Delete non-empty directory func (suite *dirTestSuite) TestDirDeleteNonEmpty() { - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } dir3Name := filepath.Join(suite.testPath, "test3NE") err := os.Mkdir(dir3Name, 0777) suite.NoError(err) @@ -257,7 +294,9 @@ func (suite *dirTestSuite) TestDirDeleteNonEmpty() { suite.Contains(err.Error(), "directory not empty") } - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } // cleanup suite.dirTestCleanup([]string{dir3Name}) } @@ -287,7 +326,9 @@ func (suite *dirTestSuite) TestDirDeleteNonEmpty() { // # Get stats of a directory func (suite *dirTestSuite) TestDirGetStats() { - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } dirName := filepath.Join(suite.testPath, "test3") err := os.Mkdir(dirName, 0777) suite.NoError(err) @@ -305,7 +346,9 @@ func (suite *dirTestSuite) TestDirGetStats() { suite.GreaterOrEqual(float64(1), modTineDiff.Hours()) } - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } // Cleanup suite.dirTestCleanup([]string{dirName}) } @@ -317,7 +360,9 @@ func (suite *dirTestSuite) TestDirChmod() { return } if suite.adlsTest == true { - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } dirName := filepath.Join(suite.testPath, "testchmod") err := os.Mkdir(dirName, 0777) suite.NoError(err) @@ -329,14 +374,18 @@ func (suite *dirTestSuite) TestDirChmod() { suite.NoError(err) suite.Equal("-rwxr--r--", stat.Mode().Perm().String()) - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } suite.dirTestCleanup([]string{dirName}) } } // # List directory func (suite *dirTestSuite) TestDirList() { - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } testDir := filepath.Join(suite.testPath, "bigTestDir") err := os.Mkdir(testDir, 0777) suite.NoError(err) @@ -360,7 +409,9 @@ func (suite *dirTestSuite) TestDirList() { suite.NoError(err) suite.Len(files, 4) - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } // Cleanup suite.dirTestCleanup([]string{testDir}) } @@ -408,7 +459,10 @@ func (suite *dirTestSuite) TestDirRenameFull() { fmt.Println("Skipping this test case for stream direct") return } - suite.EqualValues(0, DiskSize(pathPtr)) + + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } dirName := filepath.Join(suite.testPath, "full_dir") newName := filepath.Join(suite.testPath, "full_dir_rename") fileName := filepath.Join(dirName, "test_file_") @@ -425,12 +479,16 @@ func (suite *dirTestSuite) TestDirRenameFull() { suite.NoError(err) } - suite.EqualValues(10*len(suite.medBuff), DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(10*len(suite.medBuff), DiskSize(pathPtr)) + } err = os.Rename(dirName, newName) suite.NoError(err) - suite.EqualValues(10*len(suite.medBuff), DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(10*len(suite.medBuff), DiskSize(pathPtr)) + } // Deleted directory shall not be present in the container now suite.NoDirExists(dirName) @@ -546,7 +604,9 @@ func (suite *dirTestSuite) TestReadDirLink() { return } - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } dirName := filepath.Join(suite.testPath, "test_hns") err := os.Mkdir(dirName, 0777) suite.NoError(err) @@ -566,7 +626,9 @@ func (suite *dirTestSuite) TestReadDirLink() { suite.NoError(err) } - suite.EqualValues(4*len(suite.minBuff), DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(4*len(suite.minBuff), DiskSize(pathPtr)) + } symName := filepath.Join(suite.testPath, "dirlink.lnk") err = os.Symlink(dirName, symName) @@ -609,7 +671,9 @@ func (suite *dirTestSuite) TestReadDirLink() { // validating data suite.Equal(data1, data2) - suite.EqualValues(4*len(suite.minBuff), DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(4*len(suite.minBuff), DiskSize(pathPtr)) + } suite.dirTestCleanup([]string{dirName}) err = os.Remove(symName) @@ -617,7 +681,9 @@ func (suite *dirTestSuite) TestReadDirLink() { } func (suite *dirTestSuite) TestStatfs() { - suite.EqualValues(0, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(0, DiskSize(pathPtr)) + } dirName := filepath.Join(suite.testPath, "test_statfs") err := os.Mkdir(dirName, 0777) @@ -629,21 +695,27 @@ func (suite *dirTestSuite) TestStatfs() { err := os.WriteFile(newFile, suite.minBuff, 0777) suite.NoError(err) } - suite.EqualValues(12*len(suite.minBuff), DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(12*len(suite.minBuff), DiskSize(pathPtr)) + } for i := 0; i < 12; i++ { file := fileName + strconv.Itoa(i) err := os.Truncate(file, 4096) suite.NoError(err) } - suite.EqualValues(12*4096, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(12*4096, DiskSize(pathPtr)) + } for i := 0; i < 12; i++ { file := fileName + strconv.Itoa(i) err := os.WriteFile(file, suite.medBuff, 0777) suite.NoError(err) } - suite.EqualValues(12*len(suite.medBuff), DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(12*len(suite.medBuff), DiskSize(pathPtr)) + } renameFile := filepath.Join(dirName, "small_file_rename") for i := 0; i < 12; i++ { @@ -652,14 +724,18 @@ func (suite *dirTestSuite) TestStatfs() { err := os.Rename(oldFile, newFile) suite.NoError(err) } - suite.EqualValues(12*len(suite.medBuff), DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(12*len(suite.medBuff), DiskSize(pathPtr)) + } for i := 0; i < 12; i++ { file := renameFile + strconv.Itoa(i) err := os.Truncate(file, 4096) suite.NoError(err) } - suite.EqualValues(12*4096, DiskSize(pathPtr)) + if suite.s3storageTest { + suite.EqualValues(12*4096, DiskSize(pathPtr)) + } suite.dirTestCleanup([]string{dirName}) } @@ -684,8 +760,11 @@ func TestDirTestSuite(t *testing.T) { if adlsPtr == "true" || adlsPtr == "True" { fmt.Println("ADLS Testing...") dirTest.adlsTest = true + } else if s3storagePtr == "true" || adlsPtr == "True" { + fmt.Println("S3 Storage Testing...") + dirTest.s3storageTest = true } else { - fmt.Println("BLOCK Blob Testing...") + fmt.Println("Azure Block Blob Testing...") } // Sanity check in the off chance the same random name was generated twice and was still around somehow @@ -719,4 +798,5 @@ func init() { regDirTestFlag(&tempPathPtr, "tmp-path", "", "Cache dir path") regDirTestFlag(&streamDirectPtr, "stream-direct-test", "false", "Run stream direct tests") regDirTestFlag(&enableSymlinkADLS, "enable-symlink-adls", "false", "Enable symlink support for ADLS accounts") + regDirTestFlag(&s3storagePtr, "s3storage", "false", "Using S3 Storage") } diff --git a/test/s3cleanup/s3cleanup_test.go b/test/s3cleanup/s3cleanup_test.go new file mode 100644 index 000000000..9a1cf83c9 --- /dev/null +++ b/test/s3cleanup/s3cleanup_test.go @@ -0,0 +1,110 @@ +//go:build !unittest +// +build !unittest + +/* + Licensed under the MIT License . + + Copyright © 2023-2025 Seagate Technology LLC and/or its Affiliates + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package s3_cleanup + +import ( + "context" + "log" + "os" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + +func getS3Client() (*s3.Client, error) { + cfg, err := config.LoadDefaultConfig(context.TODO()) + if err != nil { + return nil, err + } + + endpoint := os.Getenv("S3_ENDPOINT") + if endpoint == "" { + log.Fatal("S3_ENDPOINT environment variable not set") + } + + return s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(endpoint) + }), nil +} + +func TestDeleteAllObjects(t *testing.T) { + ctx := context.Background() + client, err := getS3Client() + if err != nil { + log.Fatal(err) + } + + bucket := os.Getenv("S3_BUCKET_NAME") + if bucket == "" { + log.Fatal("S3_BUCKET_NAME environment variable not set") + } + + paginator := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + }) + + var objectsToDelete []types.ObjectIdentifier + + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + log.Fatal(err) + } + + for _, obj := range page.Contents { + objectsToDelete = append(objectsToDelete, types.ObjectIdentifier{ + Key: obj.Key, + }) + } + + if len(objectsToDelete) > 0 { + err = deleteObjects(ctx, client, bucket, objectsToDelete) + if err != nil { + log.Fatal(err) + } + } + objectsToDelete = nil + } +} + +func deleteObjects(ctx context.Context, client *s3.Client, bucket string, objects []types.ObjectIdentifier) error { + _, err := client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: aws.String(bucket), + Delete: &types.Delete{ + Objects: objects, + }, + }) + return err +} + +func TestMain(m *testing.M) { + m.Run() +} From f513d7fadd7760d23544a72a5f462cebeb9e8c5e Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Thu, 23 Jan 2025 12:47:08 -0700 Subject: [PATCH 08/23] Fix test failurs on Windows in loopback --- component/loopback/loopback_fs_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/component/loopback/loopback_fs_test.go b/component/loopback/loopback_fs_test.go index 6a75b5aee..46f3d8013 100644 --- a/component/loopback/loopback_fs_test.go +++ b/component/loopback/loopback_fs_test.go @@ -30,6 +30,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "testing" "github.com/Seagate/cloudfuse/common" @@ -188,6 +189,10 @@ func (suite *LoopbackFSTestSuite) TestRenameFile() { } func (suite *LoopbackFSTestSuite) TestRenameOpenFile() { + if runtime.GOOS == "windows" { + fmt.Println("Skipping test on Windows") + return + } defer suite.cleanupTest() assert := assert.New(suite.T()) @@ -214,6 +219,10 @@ func (suite *LoopbackFSTestSuite) TestRenameOpenFile() { } func (suite *LoopbackFSTestSuite) TestRenameWriteFile() { + if runtime.GOOS == "windows" { + fmt.Println("Skipping test on Windows") + return + } defer suite.cleanupTest() assert := assert.New(suite.T()) @@ -241,6 +250,10 @@ func (suite *LoopbackFSTestSuite) TestRenameWriteFile() { } func (suite *LoopbackFSTestSuite) TestRenameWriteFileGetAttr() { + if runtime.GOOS == "windows" { + fmt.Println("Skipping test on Windows") + return + } defer suite.cleanupTest() assert := assert.New(suite.T()) From 2306396bea270c9ec02e48373c95a952c85fbc4d Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Thu, 23 Jan 2025 12:48:49 -0700 Subject: [PATCH 09/23] Fix journaling on Windows --- component/size_tracker/journal.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/component/size_tracker/journal.go b/component/size_tracker/journal.go index 9a9371767..9de804e91 100644 --- a/component/size_tracker/journal.go +++ b/component/size_tracker/journal.go @@ -27,6 +27,7 @@ package size_tracker import ( "encoding/binary" "os" + "path/filepath" "sync" "github.com/Seagate/cloudfuse/common" @@ -41,7 +42,7 @@ type MountSize struct { } func CreateSizeJournal(filename string) (*MountSize, error) { - journalFile = common.JoinUnixFilepath(common.DefaultWorkDir, filename) + journalFile = filepath.Join(common.DefaultWorkDir, filename) f, err := os.OpenFile(journalFile, os.O_CREATE|os.O_RDWR, 0644) if err != nil { return nil, err From 4f443b648e743eca70e0291851d326f809420769 Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Thu, 23 Jan 2025 12:49:19 -0700 Subject: [PATCH 10/23] Fix linting and don't run size_tracker test on blob storage --- .github/workflows/code-coverage.yml | 42 +++++++++++--------------- component/size_tracker/size_tracker.go | 1 - 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 1d2f2391a..1016be41f 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -161,30 +161,6 @@ jobs: sudo fusermount -u ${{ env.MOUNT_DIR }} sleep 5 - - name: Create Config File - Block Blob with Size Tracker - env: - NIGHTLY_STO_ACC_NAME: "${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}" - NIGHTLY_STO_ACC_KEY: "${{ secrets.NIGHTLY_STO_BLOB_ACC_KEY }}" - ACCOUNT_TYPE: block - ACCOUNT_ENDPOINT: https://${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}.blob.core.windows.net - VERBOSE_LOG: false - USE_HTTP: false - run: "./cloudfuse.test -test.v -test.coverprofile=${{ env.WORK_DIR }}/cloudfuse_gentest1.cov gen-test-config --config-file=azure_key_size_tracker.yaml --container-name=${{ matrix.containerName }} --temp-path=${{ env.TEMP_DIR }} --output-file=${{ env.cloudfuse_CFG }}" - - - name: Block Blob Coverage with Size Tracker - run: |- - rm -rf ${{ env.MOUNT_DIR }}/* - rm -rf ${{ env.TEMP_DIR }}/* - ./cloudfuse.test -test.v -test.coverprofile=${{ env.WORK_DIR }}/cloudfuse_block.cov mount ${{ env.MOUNT_DIR }} --config-file=${{ env.cloudfuse_CFG }} --foreground=true & - sleep 10 - ps -aux | grep cloudfuse - rm -rf ${{ env.MOUNT_DIR }}/* - cd test/e2e_tests - go test -v -timeout=7200s ./... -args -mnt-path=${{ env.MOUNT_DIR }} -tmp-path=${{ env.TEMP_DIR }} - cd - - sudo fusermount -u ${{ env.MOUNT_DIR }} - sleep 5 - - name: Create Config File - ADLS env: NIGHTLY_STO_ACC_NAME: "${{ secrets.AZTEST_ADLS_ACC_NAME }}" @@ -276,6 +252,15 @@ jobs: sudo fusermount -u ${{ env.MOUNT_DIR }} sleep 5 + - name: Cleanup S3 Storage before size tracker test + run: go test -timeout 120m -v test/s3cleanup/s3cleanup_test.go + env: + AWS_ACCESS_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets.S3TEST_SECRET_KEY }}" + S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}" + S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}" + S3_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" + - name: Create Config File - S3 with Size Tracker env: S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}" @@ -1094,3 +1079,12 @@ jobs: env: STORAGE_ACCOUNT_NAME: "${{ secrets.AZTEST_ADLS_ACC_NAME }}" STORAGE_ACCOUNT_Key: "${{ secrets.AZTEST_ADLS_KEY }}" + + - name: Cleanup S3 Storage + run: go test -timeout 120m -v test/s3cleanup/s3cleanup_test.go + env: + AWS_ACCESS_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets.S3TEST_SECRET_KEY }}" + S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}" + S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}" + S3_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" diff --git a/component/size_tracker/size_tracker.go b/component/size_tracker/size_tracker.go index 2ccddb68a..e59f8eaa5 100644 --- a/component/size_tracker/size_tracker.go +++ b/component/size_tracker/size_tracker.go @@ -55,7 +55,6 @@ type SizeTrackerOptions struct { const compName = "size_tracker" const blockSize = int64(4096) -const emptyDirSize = uint64(4096) const default_journal_name = "mount_size.dat" // Verification to check satisfaction criteria with Component Interface From fa4eba6fd3f5855b229b693ad2a5e35ab7a200e7 Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Thu, 23 Jan 2025 13:19:03 -0700 Subject: [PATCH 11/23] Fix code coverage tests for Linux, add them for Windows --- .github/workflows/code-coverage.yml | 35 ++++++++++- test/e2e_tests/dir_test.go | 90 ++++++++++++++--------------- 2 files changed, 78 insertions(+), 47 deletions(-) diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 1016be41f..83f822797 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -225,7 +225,7 @@ jobs: sleep 10 ps -aux | grep cloudfuse cd test/e2e_tests - go test -v -timeout=7200s ./... -args -mnt-path=${{ env.MOUNT_DIR }} -tmp-path=${{ env.TEMP_DIR }} -s3storage=true + go test -v -timeout=7200s ./... -args -mnt-path=${{ env.MOUNT_DIR }} -tmp-path=${{ env.TEMP_DIR }} -size-tracker=true cd - sudo fusermount -u ${{ env.MOUNT_DIR }} sleep 5 @@ -278,7 +278,7 @@ jobs: sleep 10 ps -aux | grep cloudfuse cd test/e2e_tests - go test -v -timeout=7200s ./... -args -mnt-path=${{ env.MOUNT_DIR }} -tmp-path=${{ env.TEMP_DIR }} -s3storage=true + go test -v -timeout=7200s ./... -args -mnt-path=${{ env.MOUNT_DIR }} -tmp-path=${{ env.TEMP_DIR }} -size-tracker=true cd - sudo fusermount -u ${{ env.MOUNT_DIR }} sleep 5 @@ -811,6 +811,37 @@ jobs: kill $pid sleep 5 + - name: Cleanup S3 Storage before size tracker test + run: go test -timeout 120m -v test/s3cleanup/s3cleanup_test.go + env: + AWS_ACCESS_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets.S3TEST_SECRET_KEY }}" + S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}" + S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}" + S3_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" + + - name: Create Config File - S3 with Size Tracker + env: + S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}" + S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}" + S3_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" + S3_REGION: "${{ secrets.S3TEST_REGION }}" + S3_SECRET_KEY: "${{ secrets.S3TEST_SECRET_KEY }}" + run: "./cloudfuse.test gen-test-config --config-file=s3_key_size_tracker.yaml --temp-path=${{ env.TEMP_DIR }} --output-file=${{ env.cloudfuse_CFG }}" + + - name: S3 Coverage with Size Tracker + run: |- + rm -rf ${{ env.MOUNT_DIR }}/* + rm -rf ${{ env.TEMP_DIR }}/* + ./cloudfuse.test -test.v -test.coverprofile=${{ env.WORK_DIR }}/cloudfuse_s3.cov mount ${{ env.MOUNT_DIR }} --config-file=${{ env.cloudfuse_CFG }} --foreground=true & + sleep 10 + pid=`ps -a | grep cloudfuse | tr -s ' ' | cut -d ' ' -f2` + cd test/e2e_tests + go test -v -timeout=7200s ./... -args -mnt-path=${{ env.MOUNT_DIR }} -tmp-path=${{ env.TEMP_DIR }} -quick-test=true -stream-direct-test=true -size-tracker=true + cd - + kill $pid + sleep 5 + - name: Create Config File - Stream shell: bash env: diff --git a/test/e2e_tests/dir_test.go b/test/e2e_tests/dir_test.go index f02e9740b..8195f22f7 100644 --- a/test/e2e_tests/dir_test.go +++ b/test/e2e_tests/dir_test.go @@ -48,7 +48,7 @@ type dirTestSuite struct { suite.Suite testPath string adlsTest bool - s3storageTest bool + sizeTracker bool testCachePath string minBuff []byte medBuff []byte @@ -61,7 +61,7 @@ var adlsPtr string var clonePtr string var streamDirectPtr string var enableSymlinkADLS string -var s3storagePtr string +var sizeTrackerPtr string func regDirTestFlag(p *string, name string, value string, usage string) { if flag.Lookup(name) == nil { @@ -80,7 +80,7 @@ func initDirFlags() { clonePtr = getDirTestFlag("clone") streamDirectPtr = getDirTestFlag("stream-direct-test") enableSymlinkADLS = getDirTestFlag("enable-symlink-adls") - s3storagePtr = getDirTestFlag("s3storage") + sizeTrackerPtr = getDirTestFlag("size-tracker") } func getTestDirName(n int) string { @@ -102,14 +102,14 @@ func (suite *dirTestSuite) dirTestCleanup(toRemove []string) { // # Create Directory with a simple name func (suite *dirTestSuite) TestDirCreateSimple() { - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } dirName := filepath.Join(suite.testPath, "test1") err := os.Mkdir(dirName, 0777) suite.NoError(err) - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } // cleanup @@ -118,7 +118,7 @@ func (suite *dirTestSuite) TestDirCreateSimple() { // # Create Directory that already exists func (suite *dirTestSuite) TestDirCreateDuplicate() { - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } dirName := filepath.Join(suite.testPath, "test1") @@ -133,7 +133,7 @@ func (suite *dirTestSuite) TestDirCreateDuplicate() { suite.Contains(err.Error(), "file exists") } - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } // cleanup @@ -146,14 +146,14 @@ func (suite *dirTestSuite) TestDirCreateSplChar() { fmt.Println("Skipping TestDirCreateSplChar on Windows") return } - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } dirName := filepath.Join(suite.testPath, "@#$^&*()_+=-{}[]|?><.,~") err := os.Mkdir(dirName, 0777) suite.NoError(err) - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } // cleanup @@ -166,14 +166,14 @@ func (suite *dirTestSuite) TestDirCreateSlashChar() { fmt.Println("Skipping TestDirCreateSlashChar on Windows") return } - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } dirName := filepath.Join(suite.testPath, "PRQ\\STUV") err := os.Mkdir(dirName, 0777) suite.NoError(err) - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } // cleanup @@ -182,7 +182,7 @@ func (suite *dirTestSuite) TestDirCreateSlashChar() { // # Rename a directory func (suite *dirTestSuite) TestDirRename() { - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } dirName := filepath.Join(suite.testPath, "test1") @@ -195,7 +195,7 @@ func (suite *dirTestSuite) TestDirRename() { suite.NoDirExists(dirName) - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } // cleanup @@ -204,7 +204,7 @@ func (suite *dirTestSuite) TestDirRename() { // # Move an empty directory func (suite *dirTestSuite) TestDirMoveEmpty() { - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } dir2Name := filepath.Join(suite.testPath, "test2") @@ -219,7 +219,7 @@ func (suite *dirTestSuite) TestDirMoveEmpty() { time.Sleep(1 * time.Second) suite.NoError(err) - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } // cleanup @@ -228,7 +228,7 @@ func (suite *dirTestSuite) TestDirMoveEmpty() { // # Move an non-empty directory func (suite *dirTestSuite) TestDirMoveNonEmpty() { - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } dir2Name := filepath.Join(suite.testPath, "test2NE") @@ -251,7 +251,7 @@ func (suite *dirTestSuite) TestDirMoveNonEmpty() { time.Sleep(1 * time.Second) suite.NoError(err) - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } // cleanup @@ -260,14 +260,14 @@ func (suite *dirTestSuite) TestDirMoveNonEmpty() { // # Delete non-empty directory func (suite *dirTestSuite) TestDirDeleteEmpty() { - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } dirName := filepath.Join(suite.testPath, "test1_new") err := os.Mkdir(dirName, 0777) suite.NoError(err) - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } suite.dirTestCleanup([]string{dirName}) @@ -275,7 +275,7 @@ func (suite *dirTestSuite) TestDirDeleteEmpty() { // # Delete non-empty directory func (suite *dirTestSuite) TestDirDeleteNonEmpty() { - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } dir3Name := filepath.Join(suite.testPath, "test3NE") @@ -294,7 +294,7 @@ func (suite *dirTestSuite) TestDirDeleteNonEmpty() { suite.Contains(err.Error(), "directory not empty") } - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } // cleanup @@ -326,7 +326,7 @@ func (suite *dirTestSuite) TestDirDeleteNonEmpty() { // # Get stats of a directory func (suite *dirTestSuite) TestDirGetStats() { - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } dirName := filepath.Join(suite.testPath, "test3") @@ -346,7 +346,7 @@ func (suite *dirTestSuite) TestDirGetStats() { suite.GreaterOrEqual(float64(1), modTineDiff.Hours()) } - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } // Cleanup @@ -360,7 +360,7 @@ func (suite *dirTestSuite) TestDirChmod() { return } if suite.adlsTest == true { - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } dirName := filepath.Join(suite.testPath, "testchmod") @@ -374,7 +374,7 @@ func (suite *dirTestSuite) TestDirChmod() { suite.NoError(err) suite.Equal("-rwxr--r--", stat.Mode().Perm().String()) - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } suite.dirTestCleanup([]string{dirName}) @@ -383,7 +383,7 @@ func (suite *dirTestSuite) TestDirChmod() { // # List directory func (suite *dirTestSuite) TestDirList() { - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } testDir := filepath.Join(suite.testPath, "bigTestDir") @@ -409,7 +409,7 @@ func (suite *dirTestSuite) TestDirList() { suite.NoError(err) suite.Len(files, 4) - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } // Cleanup @@ -460,7 +460,7 @@ func (suite *dirTestSuite) TestDirRenameFull() { return } - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } dirName := filepath.Join(suite.testPath, "full_dir") @@ -479,14 +479,14 @@ func (suite *dirTestSuite) TestDirRenameFull() { suite.NoError(err) } - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(10*len(suite.medBuff), DiskSize(pathPtr)) } err = os.Rename(dirName, newName) suite.NoError(err) - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(10*len(suite.medBuff), DiskSize(pathPtr)) } @@ -604,7 +604,7 @@ func (suite *dirTestSuite) TestReadDirLink() { return } - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } dirName := filepath.Join(suite.testPath, "test_hns") @@ -626,7 +626,7 @@ func (suite *dirTestSuite) TestReadDirLink() { suite.NoError(err) } - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(4*len(suite.minBuff), DiskSize(pathPtr)) } @@ -671,7 +671,7 @@ func (suite *dirTestSuite) TestReadDirLink() { // validating data suite.Equal(data1, data2) - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(4*len(suite.minBuff), DiskSize(pathPtr)) } @@ -681,7 +681,7 @@ func (suite *dirTestSuite) TestReadDirLink() { } func (suite *dirTestSuite) TestStatfs() { - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(0, DiskSize(pathPtr)) } @@ -695,7 +695,7 @@ func (suite *dirTestSuite) TestStatfs() { err := os.WriteFile(newFile, suite.minBuff, 0777) suite.NoError(err) } - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(12*len(suite.minBuff), DiskSize(pathPtr)) } @@ -704,7 +704,7 @@ func (suite *dirTestSuite) TestStatfs() { err := os.Truncate(file, 4096) suite.NoError(err) } - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(12*4096, DiskSize(pathPtr)) } @@ -713,7 +713,7 @@ func (suite *dirTestSuite) TestStatfs() { err := os.WriteFile(file, suite.medBuff, 0777) suite.NoError(err) } - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(12*len(suite.medBuff), DiskSize(pathPtr)) } @@ -724,7 +724,7 @@ func (suite *dirTestSuite) TestStatfs() { err := os.Rename(oldFile, newFile) suite.NoError(err) } - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(12*len(suite.medBuff), DiskSize(pathPtr)) } @@ -733,7 +733,7 @@ func (suite *dirTestSuite) TestStatfs() { err := os.Truncate(file, 4096) suite.NoError(err) } - if suite.s3storageTest { + if suite.sizeTracker { suite.EqualValues(12*4096, DiskSize(pathPtr)) } @@ -760,11 +760,11 @@ func TestDirTestSuite(t *testing.T) { if adlsPtr == "true" || adlsPtr == "True" { fmt.Println("ADLS Testing...") dirTest.adlsTest = true - } else if s3storagePtr == "true" || adlsPtr == "True" { - fmt.Println("S3 Storage Testing...") - dirTest.s3storageTest = true - } else { - fmt.Println("Azure Block Blob Testing...") + } + + if sizeTrackerPtr == "true" || adlsPtr == "True" { + fmt.Println("Size Tracker Testing...") + dirTest.sizeTracker = true } // Sanity check in the off chance the same random name was generated twice and was still around somehow @@ -798,5 +798,5 @@ func init() { regDirTestFlag(&tempPathPtr, "tmp-path", "", "Cache dir path") regDirTestFlag(&streamDirectPtr, "stream-direct-test", "false", "Run stream direct tests") regDirTestFlag(&enableSymlinkADLS, "enable-symlink-adls", "false", "Enable symlink support for ADLS accounts") - regDirTestFlag(&s3storagePtr, "s3storage", "false", "Using S3 Storage") + regDirTestFlag(&sizeTrackerPtr, "size-tracker", "false", "Using size_tracker component") } From c7fe19acd3484be0a9d7b70bcec08148a04ad278 Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Thu, 23 Jan 2025 13:26:36 -0700 Subject: [PATCH 12/23] Debug failur --- component/size_tracker/size_tracker_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/component/size_tracker/size_tracker_test.go b/component/size_tracker/size_tracker_test.go index ad6a696cf..03ca0534e 100644 --- a/component/size_tracker/size_tracker_test.go +++ b/component/size_tracker/size_tracker_test.go @@ -90,7 +90,8 @@ func newTestSizeTracker(next internal.Component, configuration string) *SizeTrac _ = config.ReadConfigFromReader(strings.NewReader(configuration)) sizeTracker := NewSizeTrackerComponent() sizeTracker.SetNextComponent(next) - _ = sizeTracker.Configure(true) + err := sizeTracker.Configure(true) + fmt.Println("Result from Configure is: ", err) return sizeTracker.(*SizeTracker) } From a63c8bc36baf66e8163a87465eaa5521fe40052c Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Thu, 23 Jan 2025 13:56:09 -0700 Subject: [PATCH 13/23] Create default working directory if it does not exist when journaling --- cmd/mount.go | 13 +++---------- common/util.go | 15 +++++++++++++++ component/size_tracker/journal.go | 8 +++++++- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/cmd/mount.go b/cmd/mount.go index 43de78f32..37363816d 100644 --- a/cmd/mount.go +++ b/cmd/mount.go @@ -158,16 +158,9 @@ func (opt *mountOptions) validate(skipNonEmptyMount bool) error { common.DefaultLogFilePath = common.JoinUnixFilepath(common.DefaultWorkDir, "cloudfuse.log") } - f, err := os.Stat(common.ExpandPath(common.DefaultWorkDir)) - if err == nil && !f.IsDir() { - return fmt.Errorf("default work dir '%s' is not a directory", common.DefaultWorkDir) - } - - if err != nil && os.IsNotExist(err) { - // create the default work dir - if err = os.MkdirAll(common.ExpandPath(common.DefaultWorkDir), 0755); err != nil { - return fmt.Errorf("failed to create default work dir [%s]", err.Error()) - } + err := common.CreateDefaultDirectory() + if err != nil { + return fmt.Errorf("Failed to create default work dir [%s]", err.Error()) } opt.Logging.LogFilePath = common.ExpandPath(opt.Logging.LogFilePath) diff --git a/common/util.go b/common/util.go index 054f516d6..88ce3b68d 100644 --- a/common/util.go +++ b/common/util.go @@ -369,3 +369,18 @@ func IsDriveLetter(path string) bool { match, _ := regexp.MatchString(pattern, path) return match } + +func CreateDefaultDirectory() error { + dir, err := os.Stat(ExpandPath(DefaultWorkDir)) + if err == nil && !dir.IsDir() { + return err + } + + if err != nil && os.IsNotExist(err) { + // create the default work dir + if err = os.MkdirAll(ExpandPath(DefaultWorkDir), 0755); err != nil { + return err + } + } + return nil +} diff --git a/component/size_tracker/journal.go b/component/size_tracker/journal.go index 9de804e91..9e1e448be 100644 --- a/component/size_tracker/journal.go +++ b/component/size_tracker/journal.go @@ -26,6 +26,7 @@ package size_tracker import ( "encoding/binary" + "fmt" "os" "path/filepath" "sync" @@ -42,7 +43,12 @@ type MountSize struct { } func CreateSizeJournal(filename string) (*MountSize, error) { - journalFile = filepath.Join(common.DefaultWorkDir, filename) + journalFile = filepath.Join(common.ExpandPath(common.DefaultWorkDir), filename) + err := common.CreateDefaultDirectory() + if err != nil { + return nil, fmt.Errorf("Failed to create default work dir [%s]", err.Error()) + } + f, err := os.OpenFile(journalFile, os.O_CREATE|os.O_RDWR, 0644) if err != nil { return nil, err From ce925ed4685c7cc702e48ce6b968d60775253e11 Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Thu, 23 Jan 2025 15:44:20 -0700 Subject: [PATCH 14/23] Fix code coverage tests --- .github/workflows/code-coverage.yml | 4 ++-- test/s3cleanup/s3cleanup_test.go | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 83f822797..e57191bca 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -225,7 +225,7 @@ jobs: sleep 10 ps -aux | grep cloudfuse cd test/e2e_tests - go test -v -timeout=7200s ./... -args -mnt-path=${{ env.MOUNT_DIR }} -tmp-path=${{ env.TEMP_DIR }} -size-tracker=true + go test -v -timeout=7200s ./... -args -mnt-path=${{ env.MOUNT_DIR }} -tmp-path=${{ env.TEMP_DIR }} cd - sudo fusermount -u ${{ env.MOUNT_DIR }} sleep 5 @@ -257,9 +257,9 @@ jobs: env: AWS_ACCESS_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" AWS_SECRET_ACCESS_KEY: "${{ secrets.S3TEST_SECRET_KEY }}" + AWS_REGION: "${{ secrets.S3TEST_REGION }}" S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}" S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}" - S3_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" - name: Create Config File - S3 with Size Tracker env: diff --git a/test/s3cleanup/s3cleanup_test.go b/test/s3cleanup/s3cleanup_test.go index 9a1cf83c9..c075cc95d 100644 --- a/test/s3cleanup/s3cleanup_test.go +++ b/test/s3cleanup/s3cleanup_test.go @@ -40,7 +40,12 @@ import ( ) func getS3Client() (*s3.Client, error) { - cfg, err := config.LoadDefaultConfig(context.TODO()) + region := os.Getenv("AWS_REGION") + if region == "" { + log.Fatal("AWS_REGION environment variable not set") + } + + cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(region)) if err != nil { return nil, err } From 5d158c2efded9b2f6ca03a21d1de2821836b8796 Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Thu, 23 Jan 2025 16:39:31 -0700 Subject: [PATCH 15/23] Fix windows code coverage error --- .github/workflows/code-coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index e57191bca..e0ae0d549 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -816,9 +816,9 @@ jobs: env: AWS_ACCESS_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" AWS_SECRET_ACCESS_KEY: "${{ secrets.S3TEST_SECRET_KEY }}" + AWS_REGION: "${{ secrets.S3TEST_REGION }}" S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}" S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}" - S3_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" - name: Create Config File - S3 with Size Tracker env: From 0af0481b1057f248ed98760acd9c6b767c47a6f1 Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Thu, 23 Jan 2025 17:06:45 -0700 Subject: [PATCH 16/23] Fix issue with bash shell --- .github/workflows/code-coverage.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index e0ae0d549..68d4b9aa2 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -821,6 +821,7 @@ jobs: S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}" - name: Create Config File - S3 with Size Tracker + shell: bash env: S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}" S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}" @@ -830,6 +831,7 @@ jobs: run: "./cloudfuse.test gen-test-config --config-file=s3_key_size_tracker.yaml --temp-path=${{ env.TEMP_DIR }} --output-file=${{ env.cloudfuse_CFG }}" - name: S3 Coverage with Size Tracker + shell: bash run: |- rm -rf ${{ env.MOUNT_DIR }}/* rm -rf ${{ env.TEMP_DIR }}/* From 2ef55a2a11b3daba9616f208499f5f391f70631b Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Fri, 24 Jan 2025 08:28:42 -0700 Subject: [PATCH 17/23] Fix dir tests for Windows --- test/e2e_tests/dir_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/e2e_tests/dir_test.go b/test/e2e_tests/dir_test.go index 8195f22f7..c8dab52b3 100644 --- a/test/e2e_tests/dir_test.go +++ b/test/e2e_tests/dir_test.go @@ -549,6 +549,7 @@ func (suite *dirTestSuite) TestGitStash() { suite.NoError(err) suite.EqualValues(10, n) suite.EqualValues("TestString", string(data)) + _ = f.Close() cmd = exec.Command("git", "status") cliOut, err = cmd.Output() @@ -717,6 +718,8 @@ func (suite *dirTestSuite) TestStatfs() { suite.EqualValues(12*len(suite.medBuff), DiskSize(pathPtr)) } + time.Sleep(time.Second * 1) + renameFile := filepath.Join(dirName, "small_file_rename") for i := 0; i < 12; i++ { oldFile := fileName + strconv.Itoa(i) From b28e7ddbd8857e742a120006f3f62f503802005a Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Fri, 24 Jan 2025 11:01:58 -0700 Subject: [PATCH 18/23] Fix flaky tests on Windows --- test-scripts/e2e-test_windows.ps1 | 4 +++- test-scripts/helper/mount.ps1 | 12 ++++++++++++ test/e2e_tests/dir_test.go | 7 +++++-- test/e2e_tests/statfs_windows.go | 2 +- 4 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 test-scripts/helper/mount.ps1 diff --git a/test-scripts/e2e-test_windows.ps1 b/test-scripts/e2e-test_windows.ps1 index 35be96048..26e00207e 100755 --- a/test-scripts/e2e-test_windows.ps1 +++ b/test-scripts/e2e-test_windows.ps1 @@ -6,10 +6,12 @@ Write-Output "-------------------------------------------------------------------" Write-Output "Starting e2e Tests" +. ".\helper\mount.ps1" + go test -v -timeout=2h ../test/e2e_tests/data_validation_test.go ../test/e2e_tests/dir_test.go ../test/e2e_tests/file_test.go ../test/e2e_tests/statfs_windows.go -args -mnt-path="${mount_dir}" -adls=false -clone=false -tmp-path="${mount_tmp}" -quick-test=true -stream-direct-test=true -distro-name="windows" #go test -v -timeout=2h ../test/e2e_tests/file_test.go -args -mnt-path="${mount_dir}" -adls=false -clone=false -stream-direct-test=false -distro-name="windows" -#go test -v -timeout=2h ../test/e2e_tests/dir_test.go -args -mnt-path="${mount_dir}" -adls=false -clone=false -stream-direct-test=true +#go test -v -timeout=2h ../test/e2e_tests/dir_test.go ../test/e2e_tests/statfs_windows.go -args -mnt-path="${mount_dir}" -adls=false -clone=false -stream-direct-test=true #go test -v -timeout=2h ../test/e2e_tests/data_validation_test.go -args -mnt-path="${mount_dir}" -adls=false -tmp-path="${mount_tmp}" -quick-test=true -stream-direct-test=false -distro-name="windows" diff --git a/test-scripts/helper/mount.ps1 b/test-scripts/helper/mount.ps1 new file mode 100644 index 000000000..2e437fcc9 --- /dev/null +++ b/test-scripts/helper/mount.ps1 @@ -0,0 +1,12 @@ +# Load variables +. ".\helper\var.ps1" + +# Mount step +Write-Host "Mounting into mount directory" +& "${work_dir}\cloudfuse" mount "${mount_dir}" --config-file="${work_dir}\config.yaml" + +Start-Sleep -Seconds 5 + +# Check for mount +Write-Host "Checking for mount" +Get-Process | Where-Object { $_.Name -like "*cloudfuse*" } \ No newline at end of file diff --git a/test/e2e_tests/dir_test.go b/test/e2e_tests/dir_test.go index c8dab52b3..de3a18f41 100644 --- a/test/e2e_tests/dir_test.go +++ b/test/e2e_tests/dir_test.go @@ -696,6 +696,7 @@ func (suite *dirTestSuite) TestStatfs() { err := os.WriteFile(newFile, suite.minBuff, 0777) suite.NoError(err) } + time.Sleep(time.Second * 2) if suite.sizeTracker { suite.EqualValues(12*len(suite.minBuff), DiskSize(pathPtr)) } @@ -705,6 +706,7 @@ func (suite *dirTestSuite) TestStatfs() { err := os.Truncate(file, 4096) suite.NoError(err) } + time.Sleep(time.Second * 2) if suite.sizeTracker { suite.EqualValues(12*4096, DiskSize(pathPtr)) } @@ -714,12 +716,11 @@ func (suite *dirTestSuite) TestStatfs() { err := os.WriteFile(file, suite.medBuff, 0777) suite.NoError(err) } + time.Sleep(time.Second * 2) if suite.sizeTracker { suite.EqualValues(12*len(suite.medBuff), DiskSize(pathPtr)) } - time.Sleep(time.Second * 1) - renameFile := filepath.Join(dirName, "small_file_rename") for i := 0; i < 12; i++ { oldFile := fileName + strconv.Itoa(i) @@ -727,6 +728,7 @@ func (suite *dirTestSuite) TestStatfs() { err := os.Rename(oldFile, newFile) suite.NoError(err) } + time.Sleep(time.Second * 2) if suite.sizeTracker { suite.EqualValues(12*len(suite.medBuff), DiskSize(pathPtr)) } @@ -736,6 +738,7 @@ func (suite *dirTestSuite) TestStatfs() { err := os.Truncate(file, 4096) suite.NoError(err) } + time.Sleep(time.Second * 2) if suite.sizeTracker { suite.EqualValues(12*4096, DiskSize(pathPtr)) } diff --git a/test/e2e_tests/statfs_windows.go b/test/e2e_tests/statfs_windows.go index f46de2f04..7f2b6092e 100644 --- a/test/e2e_tests/statfs_windows.go +++ b/test/e2e_tests/statfs_windows.go @@ -44,5 +44,5 @@ func DiskSize(path string) int { return 0 } - return int(total) + return int(total - free) } From 912a35b44aea8bb7720cc78a550a54697d3963e5 Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Fri, 24 Jan 2025 11:36:25 -0700 Subject: [PATCH 19/23] Fix missing aws region --- .github/workflows/code-coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 0e1a7a9d1..102bd951d 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -1120,6 +1120,6 @@ jobs: env: AWS_ACCESS_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" AWS_SECRET_ACCESS_KEY: "${{ secrets.S3TEST_SECRET_KEY }}" + AWS_REGION: "${{ secrets.S3TEST_REGION }}" S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}" S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}" - S3_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" From 12e1ac4eaaf9cc403f7bdaa4810bbca50d534b62 Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Fri, 24 Jan 2025 11:36:40 -0700 Subject: [PATCH 20/23] Fix journaling issue on Windows --- component/size_tracker/journal.go | 11 ++++--- component/size_tracker/size_tracker.go | 40 +++++++++++++++++++------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/component/size_tracker/journal.go b/component/size_tracker/journal.go index 9e1e448be..1149000b0 100644 --- a/component/size_tracker/journal.go +++ b/component/size_tracker/journal.go @@ -53,7 +53,6 @@ func CreateSizeJournal(filename string) (*MountSize, error) { if err != nil { return nil, err } - defer f.Close() fileInfo, err := f.Stat() if err != nil { @@ -80,11 +79,11 @@ func (s *MountSize) GetSize() uint64 { return s.size } -func (s *MountSize) Add(delta uint64) uint64 { +func (s *MountSize) Add(delta uint64) (uint64, error) { return s.updateSize(delta) } -func (s *MountSize) Subtract(delta uint64) uint64 { +func (s *MountSize) Subtract(delta uint64) (uint64, error) { return s.updateSize(-delta) } @@ -94,12 +93,12 @@ func (s *MountSize) CloseFile() error { return s.file.Close() } -func (s *MountSize) updateSize(delta uint64) uint64 { +func (s *MountSize) updateSize(delta uint64) (uint64, error) { s.mu.Lock() defer s.mu.Unlock() s.size += delta - _ = s.writeSizeToFile() - return s.size + err := s.writeSizeToFile() + return s.size, err } func (s *MountSize) writeSizeToFile() error { diff --git a/component/size_tracker/size_tracker.go b/component/size_tracker/size_tracker.go index e59f8eaa5..2bc6d8008 100644 --- a/component/size_tracker/size_tracker.go +++ b/component/size_tracker/size_tracker.go @@ -192,11 +192,15 @@ func (st *SizeTracker) WriteFile(options internal.WriteFileOptions) (int, error) diff := newSize - oldSize + var journalErr error // File already exists and CopyFromFile succeeded subtract difference in file size if diff < 0 { - st.mountSize.Subtract(uint64(-diff)) + _, journalErr = st.mountSize.Subtract(uint64(-diff)) } else { - st.mountSize.Add(uint64(diff)) + _, journalErr = st.mountSize.Add(uint64(diff)) + } + if journalErr != nil { + log.Err("SizeTracker::WriteFile : Unable to journal size. Error: %v", journalErr) } return bytesWritten, nil @@ -212,11 +216,15 @@ func (st *SizeTracker) TruncateFile(options internal.TruncateFileOptions) error err := st.NextComponent().TruncateFile(options) newSize := options.Size - origSize + var journalErr error // File already exists and truncate succeeded subtract difference in file size if err == nil && getAttrErr == nil && newSize < 0 { - st.mountSize.Subtract(uint64(-newSize)) + _, journalErr = st.mountSize.Subtract(uint64(-newSize)) } else if err == nil && getAttrErr == nil && newSize >= 0 { - st.mountSize.Add(uint64(newSize)) + _, journalErr = st.mountSize.Add(uint64(newSize)) + } + if journalErr != nil { + log.Err("SizeTracker::WriteFile : Unable to journal size. Error: %v", journalErr) } return err @@ -239,11 +247,15 @@ func (st *SizeTracker) CopyFromFile(options internal.CopyFromFileOptions) error } newSize := fileInfo.Size() - origSize + var journalErr error // File already exists and CopyFromFile succeeded subtract difference in file size if newSize < 0 { - st.mountSize.Subtract(uint64(-newSize)) + _, journalErr = st.mountSize.Subtract(uint64(-newSize)) } else { - st.mountSize.Add(uint64(newSize)) + _, journalErr = st.mountSize.Add(uint64(newSize)) + } + if journalErr != nil { + log.Err("SizeTracker::WriteFile : Unable to journal size. Error: %v", journalErr) } return nil @@ -277,11 +289,15 @@ func (st *SizeTracker) FlushFile(options internal.FlushFileOptions) error { diff := newSize - origSize + var journalErr error // File already exists and CopyFromFile succeeded subtract difference in file size if diff < 0 { - st.mountSize.Subtract(uint64(-diff)) + _, journalErr = st.mountSize.Subtract(uint64(-diff)) } else { - st.mountSize.Add(uint64(diff)) + _, journalErr = st.mountSize.Add(uint64(diff)) + } + if journalErr != nil { + log.Err("SizeTracker::WriteFile : Unable to journal size. Error: %v", journalErr) } return nil @@ -333,11 +349,15 @@ func (st *SizeTracker) CommitData(opt internal.CommitDataOptions) error { diff := newSize - origSize + var journalErr error // File already exists and CopyFromFile succeeded subtract difference in file size if diff < 0 { - st.mountSize.Subtract(uint64(-diff)) + _, journalErr = st.mountSize.Subtract(uint64(-diff)) } else { - st.mountSize.Add(uint64(diff)) + _, journalErr = st.mountSize.Add(uint64(diff)) + } + if journalErr != nil { + log.Err("SizeTracker::WriteFile : Unable to journal size. Error: %v", journalErr) } return nil From 641b31e3a665dd76476e204a8bade48f6fab3a90 Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Fri, 24 Jan 2025 12:23:04 -0700 Subject: [PATCH 21/23] Fix windows size tracker test errors --- component/size_tracker/journal.go | 20 +++++++-------- component/size_tracker/size_tracker.go | 23 +++++++++++------ component/size_tracker/size_tracker_test.go | 28 ++++++++++++++++++--- 3 files changed, 51 insertions(+), 20 deletions(-) diff --git a/component/size_tracker/journal.go b/component/size_tracker/journal.go index 1149000b0..ffe7c9a74 100644 --- a/component/size_tracker/journal.go +++ b/component/size_tracker/journal.go @@ -80,25 +80,25 @@ func (s *MountSize) GetSize() uint64 { } func (s *MountSize) Add(delta uint64) (uint64, error) { - return s.updateSize(delta) + s.mu.Lock() + defer s.mu.Unlock() + s.size += delta + err := s.writeSizeToFile() + return s.size, err } func (s *MountSize) Subtract(delta uint64) (uint64, error) { - return s.updateSize(-delta) -} - -func (s *MountSize) CloseFile() error { s.mu.Lock() defer s.mu.Unlock() - return s.file.Close() + s.size -= delta + err := s.writeSizeToFile() + return s.size, err } -func (s *MountSize) updateSize(delta uint64) (uint64, error) { +func (s *MountSize) CloseFile() error { s.mu.Lock() defer s.mu.Unlock() - s.size += delta - err := s.writeSizeToFile() - return s.size, err + return s.file.Close() } func (s *MountSize) writeSizeToFile() error { diff --git a/component/size_tracker/size_tracker.go b/component/size_tracker/size_tracker.go index 2bc6d8008..4e67d6150 100644 --- a/component/size_tracker/size_tracker.go +++ b/component/size_tracker/size_tracker.go @@ -132,7 +132,10 @@ func (st *SizeTracker) CreateFile(options internal.CreateFileOptions) (*handlema // File already exists but create succeeded so remove old file size if err == nil && getAttrErr == nil { - st.mountSize.Subtract(uint64(attr.Size)) + _, journalErr := st.mountSize.Subtract(uint64(attr.Size)) + if journalErr != nil { + log.Err("SizeTracker::CreateFile : Unable to journal size. Error: %v", journalErr) + } } return handle, err @@ -145,7 +148,10 @@ func (st *SizeTracker) DeleteFile(options internal.DeleteFileOptions) error { // If the file is a symlink then it has no size so don't change the size if err == nil && getAttrErr == nil && !attr.IsSymlink() { - st.mountSize.Subtract(uint64(attr.Size)) + _, journalErr := st.mountSize.Subtract(uint64(attr.Size)) + if journalErr != nil { + log.Err("SizeTracker::DeleteFile : Unable to journal size. Error: %v", journalErr) + } } return err @@ -158,7 +164,10 @@ func (st *SizeTracker) RenameFile(options internal.RenameFileOptions) error { // If dst already exista and rename succeeds, remove overwritten dst size if dstErr == nil && err == nil { - st.mountSize.Subtract(uint64(dstAttr.Size)) + _, journalErr := st.mountSize.Subtract(uint64(dstAttr.Size)) + if journalErr != nil { + log.Err("SizeTracker::RenameFile : Unable to journal size. Error: %v", journalErr) + } } return err @@ -224,7 +233,7 @@ func (st *SizeTracker) TruncateFile(options internal.TruncateFileOptions) error _, journalErr = st.mountSize.Add(uint64(newSize)) } if journalErr != nil { - log.Err("SizeTracker::WriteFile : Unable to journal size. Error: %v", journalErr) + log.Err("SizeTracker::TruncateFile : Unable to journal size. Error: %v", journalErr) } return err @@ -255,7 +264,7 @@ func (st *SizeTracker) CopyFromFile(options internal.CopyFromFileOptions) error _, journalErr = st.mountSize.Add(uint64(newSize)) } if journalErr != nil { - log.Err("SizeTracker::WriteFile : Unable to journal size. Error: %v", journalErr) + log.Err("SizeTracker::CopyFromFile : Unable to journal size. Error: %v", journalErr) } return nil @@ -297,7 +306,7 @@ func (st *SizeTracker) FlushFile(options internal.FlushFileOptions) error { _, journalErr = st.mountSize.Add(uint64(diff)) } if journalErr != nil { - log.Err("SizeTracker::WriteFile : Unable to journal size. Error: %v", journalErr) + log.Err("SizeTracker::FlushFile : Unable to journal size. Error: %v", journalErr) } return nil @@ -357,7 +366,7 @@ func (st *SizeTracker) CommitData(opt internal.CommitDataOptions) error { _, journalErr = st.mountSize.Add(uint64(diff)) } if journalErr != nil { - log.Err("SizeTracker::WriteFile : Unable to journal size. Error: %v", journalErr) + log.Err("SizeTracker::CommitData : Unable to journal size. Error: %v", journalErr) } return nil diff --git a/component/size_tracker/size_tracker_test.go b/component/size_tracker/size_tracker_test.go index 03ca0534e..4a5e955fc 100644 --- a/component/size_tracker/size_tracker_test.go +++ b/component/size_tracker/size_tracker_test.go @@ -90,8 +90,7 @@ func newTestSizeTracker(next internal.Component, configuration string) *SizeTrac _ = config.ReadConfigFromReader(strings.NewReader(configuration)) sizeTracker := NewSizeTrackerComponent() sizeTracker.SetNextComponent(next) - err := sizeTracker.Configure(true) - fmt.Println("Result from Configure is: ", err) + _ = sizeTracker.Configure(true) return sizeTracker.(*SizeTracker) } @@ -133,12 +132,12 @@ func (suite *sizeTrackerTestSuite) cleanupTest() { func (suite *sizeTrackerTestSuite) TestDefault() { defer suite.cleanupTest() suite.assert.Equal("size_tracker", suite.sizeTracker.Name()) - print(suite.sizeTracker.mountSize.GetSize()) suite.assert.EqualValues(uint64(0), suite.sizeTracker.mountSize.GetSize()) } func (suite *sizeTrackerTestSuite) TestDeleteDir() { defer suite.cleanupTest() + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) // Setup dir := "dir" @@ -169,6 +168,7 @@ func (suite *sizeTrackerTestSuite) TestDeleteDir() { func (suite *sizeTrackerTestSuite) TestRenameDir() { defer suite.cleanupTest() + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) // Setup src := "src" @@ -197,6 +197,7 @@ func (suite *sizeTrackerTestSuite) TestRenameDir() { func (suite *sizeTrackerTestSuite) TestCreateFile() { defer suite.cleanupTest() + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) // Default is to not create empty files on create file to support immutable storage. path := generateFileName() options := internal.CreateFileOptions{Name: path} @@ -213,6 +214,7 @@ func (suite *sizeTrackerTestSuite) TestCreateFile() { func (suite *sizeTrackerTestSuite) TestDeleteFile() { defer suite.cleanupTest() + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) path := generateFileName() handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0644}) @@ -233,6 +235,7 @@ func (suite *sizeTrackerTestSuite) TestDeleteFile() { func (suite *sizeTrackerTestSuite) TestDeleteFileError() { defer suite.cleanupTest() + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) path := generateFileName() err := suite.sizeTracker.DeleteFile(internal.DeleteFileOptions{Name: path}) suite.assert.Error(err) @@ -241,6 +244,7 @@ func (suite *sizeTrackerTestSuite) TestDeleteFileError() { func (suite *sizeTrackerTestSuite) TestWriteFile() { defer suite.cleanupTest() + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) // Setup file := generateFileName() handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0644}) @@ -251,10 +255,17 @@ func (suite *sizeTrackerTestSuite) TestWriteFile() { _, err = suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}) suite.assert.NoError(err) suite.assert.EqualValues(len(data), suite.sizeTracker.mountSize.GetSize()) + + err = suite.sizeTracker.CloseFile(internal.CloseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + err = suite.sizeTracker.DeleteFile(internal.DeleteFileOptions{Name: file}) + suite.assert.NoError(err) } func (suite *sizeTrackerTestSuite) TestWriteFileMultiple() { defer suite.cleanupTest() + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) // Setup file := generateFileName() handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0644}) @@ -291,6 +302,7 @@ func (suite *sizeTrackerTestSuite) TestWriteFileMultiple() { func (suite *sizeTrackerTestSuite) TestWriteFileErrorBadFd() { defer suite.cleanupTest() + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) // Setup file := generateFileName() handle := handlemap.NewHandle(file) @@ -302,6 +314,7 @@ func (suite *sizeTrackerTestSuite) TestWriteFileErrorBadFd() { func (suite *sizeTrackerTestSuite) TestFlushFileEmpty() { defer suite.cleanupTest() + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) // Setup file := generateFileName() handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0644}) @@ -318,6 +331,7 @@ func (suite *sizeTrackerTestSuite) TestFlushFileEmpty() { func (suite *sizeTrackerTestSuite) TestFlushFile() { defer suite.cleanupTest() + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) // Setup file := generateFileName() handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0644}) @@ -337,6 +351,7 @@ func (suite *sizeTrackerTestSuite) TestFlushFile() { func (suite *sizeTrackerTestSuite) TestFlushFileErrorBadFd() { defer suite.cleanupTest() + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) // Setup file := generateFileName() handle := handlemap.NewHandle(file) @@ -348,6 +363,7 @@ func (suite *sizeTrackerTestSuite) TestFlushFileErrorBadFd() { func (suite *sizeTrackerTestSuite) TestRenameFile() { defer suite.cleanupTest() + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) // Setup src := "src1" dst := "dst1" @@ -375,6 +391,7 @@ func (suite *sizeTrackerTestSuite) TestRenameOpenFile() { return } defer suite.cleanupTest() + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) src := "src2" dst := "dst2" @@ -410,6 +427,7 @@ func (suite *sizeTrackerTestSuite) TestRenameWriteFile() { return } defer suite.cleanupTest() + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) src := "src3" dst := "dst3" @@ -451,6 +469,7 @@ func (suite *sizeTrackerTestSuite) TestRenameWriteFile() { func (suite *sizeTrackerTestSuite) TestTruncateFile() { defer suite.cleanupTest() + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) // Setup path := generateFileName() handle, err := suite.loopback.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0644}) @@ -470,6 +489,7 @@ func (suite *sizeTrackerTestSuite) TestTruncateFile() { func (suite *sizeTrackerTestSuite) TestTruncateFileOpen() { defer suite.cleanupTest() + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) // Setup path := generateFileName() handle, err := suite.loopback.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0644}) @@ -494,6 +514,7 @@ func (suite *sizeTrackerTestSuite) TestSymlink() { return } defer suite.cleanupTest() + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) // Setup file := generateFileName() symlink := generateFileName() + ".lnk" @@ -518,6 +539,7 @@ func (suite *sizeTrackerTestSuite) TestSymlink() { func (suite *sizeTrackerTestSuite) TestStatFS() { defer suite.cleanupTest() + suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize()) file := generateFileName() handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0644}) From 2359475359dc8cf09e4e1464d6ffdfe69f9fa795 Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Mon, 27 Jan 2025 08:23:59 -0700 Subject: [PATCH 22/23] Update component/size_tracker/size_tracker.go Co-authored-by: Michael Habinsky Signed-off-by: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> --- component/size_tracker/size_tracker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/component/size_tracker/size_tracker.go b/component/size_tracker/size_tracker.go index 4e67d6150..1bbc530e8 100644 --- a/component/size_tracker/size_tracker.go +++ b/component/size_tracker/size_tracker.go @@ -162,7 +162,7 @@ func (st *SizeTracker) RenameFile(options internal.RenameFileOptions) error { err := st.NextComponent().RenameFile(options) - // If dst already exista and rename succeeds, remove overwritten dst size + // If dst already exists and rename succeeds, remove overwritten dst size if dstErr == nil && err == nil { _, journalErr := st.mountSize.Subtract(uint64(dstAttr.Size)) if journalErr != nil { From 80dbc22e1999e9aa03b4ce6afc55cff2a10d956c Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Mon, 27 Jan 2025 11:06:11 -0700 Subject: [PATCH 23/23] Avoid second GetAttr call in size tracker write file --- component/size_tracker/size_tracker.go | 15 ++------------- component/size_tracker/size_tracker_test.go | 5 +---- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/component/size_tracker/size_tracker.go b/component/size_tracker/size_tracker.go index e59f8eaa5..3ecdab595 100644 --- a/component/size_tracker/size_tracker.go +++ b/component/size_tracker/size_tracker.go @@ -177,22 +177,11 @@ func (st *SizeTracker) WriteFile(options internal.WriteFileOptions) (int, error) if err != nil { return bytesWritten, err } - - var newSize int64 - attr, getAttrErr2 := st.NextComponent().GetAttr(internal.GetAttrOptions{Name: options.Handle.Path}) - if getAttrErr2 == nil { - newSize = attr.Size - } else { - log.Err("SizeTracker::WriteFile : Unable to get attr for file %s. Current tracked size is invalid. Error: : %v", options.Handle.Path, getAttrErr2) - } - - if getAttrErr1 != nil || getAttrErr2 != nil { - return bytesWritten, nil - } + newSize := max(oldSize, options.Offset+int64(len(options.Data))) diff := newSize - oldSize - // File already exists and CopyFromFile succeeded subtract difference in file size + // File already exists and WriteFile succeeded subtract difference in file size if diff < 0 { st.mountSize.Subtract(uint64(-diff)) } else { diff --git a/component/size_tracker/size_tracker_test.go b/component/size_tracker/size_tracker_test.go index 03ca0534e..606f956cc 100644 --- a/component/size_tracker/size_tracker_test.go +++ b/component/size_tracker/size_tracker_test.go @@ -90,8 +90,7 @@ func newTestSizeTracker(next internal.Component, configuration string) *SizeTrac _ = config.ReadConfigFromReader(strings.NewReader(configuration)) sizeTracker := NewSizeTrackerComponent() sizeTracker.SetNextComponent(next) - err := sizeTracker.Configure(true) - fmt.Println("Result from Configure is: ", err) + _ = sizeTracker.Configure(true) return sizeTracker.(*SizeTracker) } @@ -396,7 +395,6 @@ func (suite *sizeTrackerTestSuite) TestRenameOpenFile() { // Close file handle err = suite.sizeTracker.CloseFile(internal.CloseFileOptions{Handle: handle}) suite.assert.NoError(err) - fmt.Println(suite.sizeTracker.mountSize.GetSize()) suite.assert.EqualValues(len(data), suite.sizeTracker.mountSize.GetSize()) @@ -441,7 +439,6 @@ func (suite *sizeTrackerTestSuite) TestRenameWriteFile() { // Close file handle err = suite.sizeTracker.CloseFile(internal.CloseFileOptions{Handle: handle}) suite.assert.NoError(err) - fmt.Println(suite.sizeTracker.mountSize.GetSize()) suite.assert.EqualValues(2*len(data), suite.sizeTracker.mountSize.GetSize())