archive pseudo-vcs driver: indexing code in archives (e.g. zip, tar) without extracting files

go.mod

@@ -4,5 +4,6 @@ go 1.16
 
 require (
 	github.com/blang/semver/v4 v4.0.0
+	github.com/mholt/archiver/v4 v4.0.0-alpha.8
 	golang.org/x/mod v0.10.0
 )

index/archive.go

@@ -0,0 +1,126 @@
+package index
+
+import (
+	"context"
+	"io/fs"
+	"os"
+
+	"github.com/hound-search/hound/codesearch/index"
+	"github.com/hound-search/hound/config"
+	"github.com/mholt/archiver/v4"
+)
+
+func stringInSlice(a string, list []string) bool {
+	for _, b := range list {
+		if b == a {
+			return true
+		}
+	}
+	return false
+}
+
+func isArchiveRepo(repo *config.Repo) bool {
+	return stringInSlice(repo.Vcs, []string{"archive"})
+}
+
+func openArchive(repo *config.Repo) (fs.FS, error) {
+	ctx := context.Background()
+	return archiver.FileSystem(ctx, repo.Url)
+}
+
+func indexArchive(opt *IndexOptions, repo *config.Repo, ix *index.IndexWriter) ([]*ExcludedFile, error) {
+	fsys, err := openArchive(repo)
+	if err != nil {
+		return nil, err
+	}
+
+	excluded := []*ExcludedFile{}
+
+	err = fs.WalkDir(fsys, ".", func(path string, info fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		name := info.Name()
+		// already relative paths in archive
+		rel := path
+
+		// Is this file considered "special", this means it's not even a part
+		// of the source repository (like .git or .svn).
+		if containsString(opt.SpecialFiles, name) {
+			if info.IsDir() {
+				return fs.SkipDir
+			}
+			return nil
+		}
+
+		if path == "." {
+			// special case for archives
+			return nil
+		}
+
+		if opt.ExcludeDotFiles && name[0] == '.' {
+			if info.IsDir() {
+				return fs.SkipDir
+			}
+
+			excluded = append(excluded, &ExcludedFile{
+				rel,
+				reasonDotFile,
+			})
+			return nil
+		}
+
+		if info.IsDir() {
+			return nil
+		}
+
+		if info.Type()&os.ModeType & ^os.ModeSymlink != 0 {
+			excluded = append(excluded, &ExcludedFile{
+				rel,
+				reasonInvalidMode,
+			})
+			return nil
+		}
+
+		// is text file
+		{
+			r, err := fsys.Open(path)
+			if err != nil {
+				return err
+			}
+			defer r.Close()
+
+			txt, err := isTextReader(r)
+			if err != nil {
+				return err
+			}
+
+			if !txt {
+				excluded = append(excluded, &ExcludedFile{
+					rel,
+					reasonNotText,
+				})
+				return nil
+			}
+		}
+
+		r, err := fsys.Open(path)
+		if err != nil {
+			return err
+		}
+		defer r.Close()
+
+		reasonForExclusion := ix.Add(rel, r)
+		if reasonForExclusion != "" {
+			excluded = append(excluded, &ExcludedFile{rel, reasonForExclusion})
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return excluded, nil
+}

index/archive_test.go

@@ -0,0 +1,73 @@
+package index
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/hound-search/hound/config"
+	"github.com/mholt/archiver/v4"
+)
+
+func buildIndexForZip() (*IndexRef, error) {
+	rev := "n/a"
+
+	dir, err := os.MkdirTemp(os.TempDir(), "hound")
+	if err != nil {
+		return nil, err
+	}
+
+	url := filepath.Join(dir, "archive.zip")
+	if err := func() error {
+		files, err := archiver.FilesFromDisk(nil, map[string]string{
+			thisDir(): "",
+		})
+
+		out, err := os.Create(url)
+		if err != nil {
+			return err
+		}
+		defer out.Close()
+
+		format := archiver.CompressedArchive{
+			Archival: archiver.Zip{
+				SelectiveCompression: true,
+			},
+		}
+
+		return format.Archive(context.Background(), out, files)
+	}(); err != nil {
+		return nil, err
+	}
+
+	opt := &IndexOptions{
+		SpecialFiles: []string{".git"},
+	}
+
+	return Build(opt, dir, "/not_existent_dir", &config.Repo{
+		Url: url,
+		Vcs: "archive",
+	}, rev)
+}
+
+func TestSearchForZip(t *testing.T) {
+	// Build an index
+	ref, err := buildIndexForZip()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ref.Remove() //nolint
+
+	// Make sure the ref can be opened.
+	idx, err := ref.Open()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer idx.Close()
+
+	// Make sure we can carry out a search
+	if _, err := idx.Search("5a1c0dac2d9b3ea4085b30dd14375c18eab993d5", &SearchOptions{}); err != nil {
+		t.Fatal(err)
+	}
+}