From 96a77b20ff85509d6a4d4432edc2040dd9a7e071 Mon Sep 17 00:00:00 2001 From: John Starich Date: Tue, 27 Jul 2021 22:27:13 -0500 Subject: [PATCH] Finish removing afero in favor of hackpadfs --- go.mod | 3 - go.sum | 4 - internal/fs/fs_other.go | 4 +- internal/fs/null_file.go | 9 +- internal/fs/pipe.go | 2 +- internal/fs/stdout.go | 6 +- internal/fs/unimplemented.go | 30 ++----- internal/interop/error.go | 23 ++--- internal/js/fs/chown.go | 2 +- internal/js/fs/fs.go | 1 - internal/js/fs/overlay.go | 62 ------------- internal/tarfs/fs_test.go | 166 ++++++++++------------------------- 12 files changed, 66 insertions(+), 246 deletions(-) diff --git a/go.mod b/go.mod index 0c6fc2b..9aaf33a 100644 --- a/go.mod +++ b/go.mod @@ -11,10 +11,7 @@ require ( github.com/machinebox/progress v0.2.0 github.com/mattn/go-tty v0.0.3 github.com/pkg/errors v0.9.1 - github.com/spf13/afero v1.3.0 github.com/stretchr/testify v1.5.1 go.uber.org/atomic v1.6.0 mvdan.cc/sh/v3 v3.1.2 ) - -replace github.com/spf13/afero v1.3.0 => github.com/johnstarich/afero v1.3.2-0.20210214021553-81c4e4e83b19 diff --git a/go.sum b/go.sum index 518fb0d..ca657da 100644 --- a/go.sum +++ b/go.sum @@ -17,10 +17,6 @@ github.com/hack-pad/hackpadfs v0.0.0-20210721061701-8173de332f89 h1:N78BVvRpFvF1 github.com/hack-pad/hackpadfs v0.0.0-20210721061701-8173de332f89/go.mod h1:zXCMPoXvOxqv/hQh5drtWQmJZ1MtKKejvF+qaI+aPmo= github.com/hack-pad/hackpadfs v0.0.0-20210728023418-c0d176de33ad h1:X98BtOtlhzejdiKec+i0xmTl2UFuYcuv/Vl243vSZro= github.com/hack-pad/hackpadfs v0.0.0-20210728023418-c0d176de33ad/go.mod h1:Ecy1tNXrYbhYs5Bdd9MnEBOvDTCjLMfI99hTQ9oV5uA= -github.com/johnstarich/afero v1.3.2-0.20200824034706-e0c81fb79d7b h1:8XJ1QRVoSx+rcCoch3GI3lOCEIWLu7bsoU5xd+GO3sk= -github.com/johnstarich/afero v1.3.2-0.20200824034706-e0c81fb79d7b/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= -github.com/johnstarich/afero v1.3.2-0.20210214021553-81c4e4e83b19 h1:dF789piTv5rw9ztiRTsTEWyVeVo90AC+5W1XIPawBEo= -github.com/johnstarich/afero v1.3.2-0.20210214021553-81c4e4e83b19/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/johnstarich/go v0.1.2 h1:F9dlaaVfeouu87o0phoCutz41FlzrFYKzwC4gV/DRlg= github.com/johnstarich/go/datasize v0.0.1 h1:Hjswen8gwmO7trXtQ8Xl8NUOQAKKm1wvus3/xK5eHGY= github.com/johnstarich/go/datasize v0.0.1/go.mod h1:4eHLMGz7Q5uCmZeS9rZdahvAih1QmBg1EW3bBXTJpi4= diff --git a/internal/fs/fs_other.go b/internal/fs/fs_other.go index 43a3a67..c51cb98 100644 --- a/internal/fs/fs_other.go +++ b/internal/fs/fs_other.go @@ -2,10 +2,8 @@ package fs -import "github.com/spf13/afero" - type persistFs struct { - afero.Fs + hackpadfs.FS } func newPersistDB(name string, relaxedDurability bool, shouldCache ShouldCacher) (*persistFs, error) { diff --git a/internal/fs/null_file.go b/internal/fs/null_file.go index ebf99cb..bab7f53 100644 --- a/internal/fs/null_file.go +++ b/internal/fs/null_file.go @@ -5,14 +5,14 @@ import ( "os" "time" - "github.com/spf13/afero" + "github.com/hack-pad/hackpadfs" ) type nullFile struct { name string } -func newNullFile(name string) afero.File { +func newNullFile(name string) hackpadfs.File { return nullFile{name: name} } @@ -22,13 +22,8 @@ func (f nullFile) ReadAt(p []byte, off int64) (n int, err error) { return 0, io func (f nullFile) Seek(offset int64, whence int) (int64, error) { return 0, nil } func (f nullFile) Write(p []byte) (n int, err error) { return len(p), nil } func (f nullFile) WriteAt(p []byte, off int64) (n int, err error) { return len(p), nil } -func (f nullFile) Name() string { return f.name } -func (f nullFile) Readdir(count int) ([]os.FileInfo, error) { return nil, nil } -func (f nullFile) Readdirnames(n int) ([]string, error) { return nil, nil } func (f nullFile) Stat() (os.FileInfo, error) { return nullStat{f}, nil } -func (f nullFile) Sync() error { return nil } func (f nullFile) Truncate(size int64) error { return nil } -func (f nullFile) WriteString(s string) (ret int, err error) { return len(s), nil } type nullStat struct { f nullFile diff --git a/internal/fs/pipe.go b/internal/fs/pipe.go index 4b3dc5f..2071940 100644 --- a/internal/fs/pipe.go +++ b/internal/fs/pipe.go @@ -70,7 +70,7 @@ func (p pipeStat) Sys() interface{} { return nil } func (p *pipeChan) Stat() (os.FileInfo, error) { return &pipeStat{ - name: p.Name(), + name: "", size: int64(len(p.buf)), mode: os.ModeNamedPipe, }, nil diff --git a/internal/fs/stdout.go b/internal/fs/stdout.go index 523b5a9..fed054d 100644 --- a/internal/fs/stdout.go +++ b/internal/fs/stdout.go @@ -5,13 +5,13 @@ import ( "sync" "time" + "github.com/hack-pad/hackpadfs" "github.com/johnstarich/go-wasm/log" - "github.com/spf13/afero" ) var ( - stdout afero.File = &bufferedLogger{name: "dev/stdout", printFn: log.Print} - stderr afero.File = &bufferedLogger{name: "dev/stderr", printFn: log.Error} + stdout hackpadfs.File = &bufferedLogger{name: "dev/stdout", printFn: log.Print} + stderr hackpadfs.File = &bufferedLogger{name: "dev/stderr", printFn: log.Error} ) type bufferedLogger struct { diff --git a/internal/fs/unimplemented.go b/internal/fs/unimplemented.go index 0b0ae8f..f2d2c81 100644 --- a/internal/fs/unimplemented.go +++ b/internal/fs/unimplemented.go @@ -3,35 +3,15 @@ package fs import ( "os" + "github.com/hack-pad/hackpadfs" "github.com/johnstarich/go-wasm/internal/interop" ) -// unimplementedFile can be embedded in special files like /dev/null to provide a default unimplemented afero.File interface +var _ hackpadfs.File = &unimplementedFile{} + +// unimplementedFile can be embedded in special files like /dev/null to provide a default unimplemented hackpadfs.File interface type unimplementedFile struct{} func (f unimplementedFile) Close() error { return interop.ErrNotImplemented } func (f unimplementedFile) Read(p []byte) (n int, err error) { return 0, interop.ErrNotImplemented } -func (f unimplementedFile) ReadAt(p []byte, off int64) (n int, err error) { - return 0, interop.ErrNotImplemented -} -func (f unimplementedFile) Seek(offset int64, whence int) (int64, error) { - return 0, interop.ErrNotImplemented -} -func (f unimplementedFile) Write(p []byte) (n int, err error) { return 0, interop.ErrNotImplemented } -func (f unimplementedFile) WriteAt(p []byte, off int64) (n int, err error) { - return 0, interop.ErrNotImplemented -} - -func (f unimplementedFile) Name() string { return "" } -func (f unimplementedFile) Readdir(count int) ([]os.FileInfo, error) { - return nil, interop.ErrNotImplemented -} -func (f unimplementedFile) Readdirnames(n int) ([]string, error) { - return nil, interop.ErrNotImplemented -} -func (f unimplementedFile) Stat() (os.FileInfo, error) { return nil, interop.ErrNotImplemented } -func (f unimplementedFile) Sync() error { return interop.ErrNotImplemented } -func (f unimplementedFile) Truncate(size int64) error { return interop.ErrNotImplemented } -func (f unimplementedFile) WriteString(s string) (ret int, err error) { - return 0, interop.ErrNotImplemented -} +func (f unimplementedFile) Stat() (os.FileInfo, error) { return nil, interop.ErrNotImplemented } diff --git a/internal/interop/error.go b/internal/interop/error.go index c63e6c4..8109f43 100644 --- a/internal/interop/error.go +++ b/internal/interop/error.go @@ -3,13 +3,12 @@ package interop import ( "fmt" "io" - "os" "os/exec" + "github.com/hack-pad/hackpadfs" "github.com/johnstarich/go-wasm/internal/common" "github.com/johnstarich/go-wasm/log" "github.com/pkg/errors" - "github.com/spf13/afero" ) var ( @@ -55,24 +54,20 @@ func mapToErrNo(err error, debugMessage string) string { return mapToErrNo(err.Unwrap(), debugMessage) } switch err { - case io.EOF, os.ErrNotExist, exec.ErrNotFound: + case io.EOF, exec.ErrNotFound: return "ENOENT" - case os.ErrExist: - return "EEXIST" - case os.ErrPermission: - return "EPERM" - } - switch err.Error() { - case os.ErrClosed.Error(), afero.ErrFileClosed.Error(): - return "EBADF" // if it was already closed, then the file descriptor was invalid } switch { - case os.IsNotExist(err): + case errors.Is(err, hackpadfs.ErrClosed): + return "EBADF" // if it was already closed, then the file descriptor was invalid + case errors.Is(err, hackpadfs.ErrNotExist): return "ENOENT" - case os.IsExist(err): + case errors.Is(err, hackpadfs.ErrExist): return "EEXIST" - case afero.IsDirErr(err): + case errors.Is(err, hackpadfs.ErrIsDir): return "EISDIR" + case errors.Is(err, hackpadfs.ErrPermission): + return "EPERM" default: log.Errorf("Unknown error type: (%T) %+v\n\n%s", err, err, debugMessage) return "EPERM" diff --git a/internal/js/fs/chown.go b/internal/js/fs/chown.go index 063eacd..779671a 100644 --- a/internal/js/fs/chown.go +++ b/internal/js/fs/chown.go @@ -25,6 +25,6 @@ func chownSync(args []js.Value) (interface{}, error) { } func Chown(path string, uid, gid int) error { - // TODO no-op, consider adding user and group ID support to afero + // TODO no-op, consider adding user and group ID support to hackpadfs return nil } diff --git a/internal/js/fs/fs.go b/internal/js/fs/fs.go index 3502e23..37c36eb 100644 --- a/internal/js/fs/fs.go +++ b/internal/js/fs/fs.go @@ -78,7 +78,6 @@ func Init() { global.Set("getMounts", js.FuncOf(getMounts)) global.Set("destroyMount", js.FuncOf(destroyMount)) - global.Set("overlayZip", js.FuncOf(overlayZip)) global.Set("overlayTarGzip", js.FuncOf(overlayTarGzip)) global.Set("overlayIndexedDB", js.FuncOf(overlayIndexedDB)) global.Set("dumpZip", js.FuncOf(dumpZip)) diff --git a/internal/js/fs/overlay.go b/internal/js/fs/overlay.go index 6760540..ae63303 100644 --- a/internal/js/fs/overlay.go +++ b/internal/js/fs/overlay.go @@ -3,12 +3,9 @@ package fs import ( - "archive/zip" - "bytes" "context" "errors" "io" - "io/ioutil" "net/http" "net/url" "path" @@ -19,8 +16,6 @@ import ( "github.com/hack-pad/hackpadfs" "github.com/hack-pad/hackpadfs/indexeddb" "github.com/machinebox/progress" - "github.com/spf13/afero" - "github.com/spf13/afero/zipfs" "github.com/johnstarich/go-wasm/internal/common" "github.com/johnstarich/go-wasm/internal/fs" @@ -31,63 +26,6 @@ import ( "github.com/johnstarich/go/datasize" ) -func overlayZip(this js.Value, args []js.Value) interface{} { - resolve, reject, prom := promise.New() - log.Debug("Backgrounding overlay request") - go func() { - err := OverlayZip(args) - if err != nil { - reject(interop.WrapAsJSError(err, "Failed overlaying zip FS")) - } else { - log.Debug("Successfully overlayed zip FS") - resolve(nil) - } - }() - return prom -} - -func OverlayZip(args []js.Value) error { - if len(args) != 2 { - return errors.New("overlayZip: mount path and zip URL path is required") - } - - mountPath := args[0].String() - zipPath := args[1].String() - log.Debug("Downloading overlay zip FS: ", zipPath) - u, err := url.Parse(zipPath) - if err != nil { - return err - } - resp, err := http.Get(u.Path) // only download from current server, not just any URL - if err != nil { - return err - } - log.Debug("Download response received. Reading body...") - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - log.Debug("Finished reading download data. Overlaying FS...") - if err := resp.Body.Close(); err != nil { - return err - } - - z, err := zip.NewReader(bytes.NewReader(body), resp.ContentLength) - if err != nil { - return err - } - return fs.Overlay(mountPath, &zipFS{zipfs.New(z)}) -} - -type zipFS struct { - afero.Fs -} - -func (z *zipFS) Open(name string) (hackpadfs.File, error) { - return z.Fs.Open(name) -} - func overlayIndexedDB(this js.Value, args []js.Value) interface{} { resolve, reject, prom := promise.New() go func() { diff --git a/internal/tarfs/fs_test.go b/internal/tarfs/fs_test.go index 49464bd..2278888 100644 --- a/internal/tarfs/fs_test.go +++ b/internal/tarfs/fs_test.go @@ -5,65 +5,46 @@ import ( "bytes" "compress/gzip" "io" - "os" - goPath "path" - "path/filepath" "runtime" - "strings" "testing" "time" - "github.com/johnstarich/go-wasm/internal/fstest" - "github.com/johnstarich/go-wasm/internal/fsutil" + "github.com/hack-pad/hackpadfs" + "github.com/hack-pad/hackpadfs/fstest" + "github.com/hack-pad/hackpadfs/mem" "github.com/pkg/errors" - "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -type swappableFS struct { - afero.Fs -} - -func newSwappableFS(fs afero.Fs) *swappableFS { - return &swappableFS{fs} -} - -func (s *swappableFS) Swap(fs afero.Fs) { - s.Fs = fs -} - -func TestFs(t *testing.T) { - tarFS := newSwappableFS(nil) - memFS := newSwappableFS(afero.NewMemMapFs()) +func TestFS(t *testing.T) { + options := fstest.FSOptions{ + Name: "tar", + Setup: fstest.TestSetupFunc(func(tb testing.TB) (fstest.SetupFS, func() hackpadfs.FS) { + setupFS, err := mem.NewFS() + require.NoError(tb, err) - rebuildTarFromMem := func() error { - tmpTarFS, err := newTarFromFS(memFS) - if err == nil { - tarFS.Swap(tmpTarFS) - } - return err - } - require.NoError(t, rebuildTarFromMem()) - - cleanup := func() error { - memFS.Swap(afero.NewMemMapFs()) - err := rebuildTarFromMem() - return err + return setupFS, func() hackpadfs.FS { + return newTarFromFS(tb, setupFS) + } + }), } - - fstest.RunReadOnly(t, tarFS, memFS, cleanup, rebuildTarFromMem) + fstest.FS(t, options) + fstest.File(t, options) } -func newTarFromFS(src afero.Fs) (*FS, error) { +func newTarFromFS(tb testing.TB, src hackpadfs.FS) *FS { r, err := buildTarFromFS(src) - if err != nil { - return nil, err - } - return New(r, afero.NewMemMapFs()) + require.NoError(tb, err) + memFS, err := mem.NewFS() + require.NoError(tb, err) + + fs, err := New(r, memFS) + require.NoError(tb, err) + return fs } -func buildTarFromFS(src afero.Fs) (io.Reader, error) { +func buildTarFromFS(src hackpadfs.FS) (io.Reader, error) { var buf bytes.Buffer compressor := gzip.NewWriter(&buf) defer compressor.Close() @@ -71,12 +52,16 @@ func buildTarFromFS(src afero.Fs) (io.Reader, error) { archive := tar.NewWriter(compressor) defer archive.Close() - err := afero.Walk(src, "/", copyTarWalk(src, archive)) + err := hackpadfs.WalkDir(src, "/", copyTarWalk(src, archive)) return &buf, errors.Wrap(err, "Failed building tar from FS walk") } -func copyTarWalk(src afero.Fs, archive *tar.Writer) filepath.WalkFunc { - return func(path string, info os.FileInfo, err error) error { +func copyTarWalk(src hackpadfs.FS, archive *tar.Writer) hackpadfs.WalkDirFunc { + return func(path string, dir hackpadfs.DirEntry, err error) error { + if err != nil { + return err + } + info, err := dir.Info() if err != nil { return err } @@ -92,7 +77,7 @@ func copyTarWalk(src afero.Fs, archive *tar.Writer) filepath.WalkFunc { if err != nil { return err } - fileBytes, err := afero.ReadFile(src, path) + fileBytes, err := hackpadfs.ReadFile(src, path) if err != nil { return err } @@ -104,44 +89,45 @@ func copyTarWalk(src afero.Fs, archive *tar.Writer) filepath.WalkFunc { func TestNewFromFs(t *testing.T) { for _, tc := range []struct { description string - do func(t *testing.T, fs afero.Fs) + do func(t *testing.T, fs hackpadfs.FS) }{ { description: "empty", - do: func(t *testing.T, fs afero.Fs) {}, + do: func(t *testing.T, fs hackpadfs.FS) {}, }, { description: "one file", - do: func(t *testing.T, fs afero.Fs) { - _, err := fs.Create("foo") + do: func(t *testing.T, fs hackpadfs.FS) { + _, err := hackpadfs.Create(fs, "foo") require.NoError(t, err) }, }, { description: "one dir", - do: func(t *testing.T, fs afero.Fs) { - err := fs.Mkdir("foo", 0700) + do: func(t *testing.T, fs hackpadfs.FS) { + err := hackpadfs.Mkdir(fs, "foo", 0700) require.NoError(t, err) }, }, { description: "dir with one nested file", - do: func(t *testing.T, fs afero.Fs) { - err := fs.Mkdir("foo", 0700) + do: func(t *testing.T, fs hackpadfs.FS) { + err := hackpadfs.Mkdir(fs, "foo", 0700) require.NoError(t, err) - _, err = fs.Create("foo/bar") + _, err = hackpadfs.Create(fs, "foo/bar") require.NoError(t, err) }, }, } { t.Run(tc.description, func(t *testing.T) { - memFS := afero.NewMemMapFs() + memFS, err := mem.NewFS() + require.NoError(t, err) tc.do(t, memFS) timer := time.NewTimer(50 * time.Millisecond) done := make(chan struct{}) go func() { - tarFS, err := newTarFromFS(memFS) + tarFS := newTarFromFS(t, memFS) assert.NoError(t, err) assert.NotNil(t, tarFS) close(done) @@ -158,67 +144,3 @@ func TestNewFromFs(t *testing.T) { }) } } - -func TestDirsFromPath(t *testing.T) { - for _, tc := range []struct { - description string - path string - expect []string - }{ - { - description: "empty path", - path: "", - expect: []string{"/"}, - }, - { - description: "slash", - path: "/", - expect: []string{"/"}, - }, - { - description: "file", - path: "/foo", - expect: []string{"/"}, - }, - { - description: "dir", - path: "/foo/", - expect: []string{"/foo", "/"}, - }, - { - description: "nested dir", - path: "/foo/bar/", - expect: []string{"/foo/bar", "/foo", "/"}, - }, - { - description: "nested file", - path: "/foo/bar", - expect: []string{"/foo", "/"}, - }, - } { - t.Run(tc.description, func(t *testing.T) { - assert.Equal(t, tc.expect, dirsFromPath(tc.path)) - }) - } -} - -// dirsFromPath returns all directory segments in 'path'. Assumes 'path' is a raw header name from a tar. -func dirsFromPath(path string) []string { - var dirs []string - if strings.HasSuffix(path, pathSeparator) { - // denotes a tar directory path, so clean it and add it before pop - path = fsutil.NormalizePath(path) - dirs = append(dirs, path) - } - if path == pathSeparator { - return dirs - } - path = fsutil.NormalizePath(path) // make absolute + clean - path = goPath.Dir(path) // pop normal files from the end - var prevPath string - for ; path != prevPath; path = goPath.Dir(path) { - dirs = append(dirs, path) - prevPath = path - } - return dirs -}