Help with running embedded with S3 Sync

[heynemann]

What would you like to be added or enhanced

I'm trying to understand how to run in embedded mode with S3 storage sync. Haven't found any examples in doc and been trying to understand the codebase for it, but it's been hard. I've managed to get the store to work locally and managed to create an S3 Remote Storage but haven't found how to plug em.

Why is this needed

To run immudb embedded with S3 sync.

Additional context

[ostafen]

Hi, @heynemann, thanks for the feedback.
I'll get back to you shortly with a code snippet for running immudb with remote S3 storage.

[heynemann]

Any news @ostafen ?

[ostafen]

Hi, @heynemann, the following snippet should fit your scenario:

package main

import (
	"context"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/codenotary/immudb/embedded/appendable"
	"github.com/codenotary/immudb/embedded/appendable/multiapp"
	"github.com/codenotary/immudb/embedded/appendable/remoteapp"
	"github.com/codenotary/immudb/embedded/remotestorage/s3"
	"github.com/codenotary/immudb/embedded/store"
)

func getS3RemotePath(rootPath, subPath string) string {
	fsPath := filepath.Join(rootPath, subPath)

	return strings.ReplaceAll(
		fsPath+"/",
		string(filepath.Separator), "/",
	)
}

type Options struct {
	S3Endpoint            string
	S3RoleEnabled         bool
	S3Role                string
	S3AccessKeyID         string
	S3SecretKey           string
	S3BucketName          string
	S3Location            string
	S3PathPrefix          string
	S3InstanceMetadataURL string
}

func main() {
	var s3Opts Options // set your s3 options

	remoteStorage, err := s3.Open(
		s3Opts.S3Endpoint,
		s3Opts.S3RoleEnabled,
		s3Opts.S3Role,
		s3Opts.S3AccessKeyID,
		s3Opts.S3SecretKey,
		s3Opts.S3BucketName,
		s3Opts.S3Location,
		s3Opts.S3PathPrefix,
		s3Opts.S3InstanceMetadataURL,
	)
	if err != nil {
		panic(err)
	}

	opts := store.DefaultOptions()

	opts.WithAppFactory(func(rootPath, subPath string, opts *multiapp.Options) (appendable.Appendable, error) {
		remoteAppOpts := remoteapp.DefaultOptions()
		remoteAppOpts.Options = *opts

		s3Path := getS3RemotePath(rootPath, subPath)
		return remoteapp.Open(
			filepath.Join(rootPath, subPath),
			s3Path,
			remoteStorage,
			remoteAppOpts,
		)
	}).
		WithFileSize(1 << 20). // Reduce file size for better cache granularity
		WithAppRemoveFunc(func(rootPath, subPath string) error {
			s3Path := getS3RemotePath(rootPath, subPath)

			err = os.RemoveAll(filepath.Join(rootPath, subPath))
			if err != nil {
				return err
			}
			return remoteStorage.RemoveAll(context.Background(), s3Path)
		})

	store, err := store.Open("./data", opts)
	if err != nil {
		log.Fatal(err)
	}
	defer store.Close()
       // use your store
}

Let me know if you need further assistance.