fix: Working on Writer (#14)

Signed-off-by: Piyush Singariya <piyushsingariya@datazip.io>
Co-authored-by: hashcode-ankit <ankit@datazip.io>

.dockerignore

@@ -1 +1,2 @@
 .git
+test/

.gitignore

@@ -2,3 +2,6 @@ g5
 .vscode
 test
 .DS_Store
+__debug_bin*
+go.sum
+go.work.sum

Dockerfile

@@ -1,17 +1,17 @@
-FROM golang:1.22-alpine as base
+FROM golang:1.23-alpine as base
 
 ADD . /home/app/
 # ADD . /home/app/
-WORKDIR /home/app/drivers/postgres/
+WORKDIR /home/app/drivers/mongodb/
 
 RUN gofmt -l -s -w .
 RUN go build -o dsynk main.go
 RUN mv dsynk /
-RUN mv generated.json /
+# RUN mv generated.json /
 
 FROM golang:1.21-alpine
 COPY --from=base /dsynk /home/
-COPY --from=base /generated.json /home/generated.json
+# COPY --from=base /generated.json /home/generated.json
 ADD . /home
 
 LABEL io.eggwhite.version=2.0.24

connector.go

@@ -5,8 +5,8 @@ import (
 
 	"github.com/datazip-inc/olake/logger"
 	protocol "github.com/datazip-inc/olake/protocol"
-	"github.com/datazip-inc/olake/safego"
 	_ "github.com/datazip-inc/olake/writers/local"
+	"github.com/piyushsingariya/relec/safego"
 )
 
 func RegisterDriver(driver protocol.Driver) {

constants/constants.go

@@ -1,5 +1,8 @@
 package constants
 
 const (
-	ParquetFileExt = "parquet"
+	ParquetFileExt       = "parquet"
+	MongoPrimaryID       = "_id"
+	MongoPrimaryIDPrefix = `ObjectID("`
+	MongoPrimaryIDSuffix = `")`
 )

drivers/google-sheets/go.mod

@@ -13,39 +13,46 @@ require (
 require (
 	cloud.google.com/go/compute v1.20.1 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect
-	github.com/apache/thrift v0.14.2 // indirect
+	github.com/apache/thrift v0.16.0 // indirect
 	github.com/brainicorn/ganno v0.0.0-20220304182003-e638228cd865 // indirect
 	github.com/brainicorn/goblex v0.0.0-20210908194630-cfe0cfdf87dd // indirect
+	github.com/felixge/fgprof v0.9.5 // indirect
+	github.com/fraugster/parquet-go v0.12.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.22.1 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/golang/snappy v0.0.3 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect
+	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/joomcode/errorx v1.1.0 // indirect
-	github.com/klauspost/compress v1.15.9 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/mitchellh/hashstructure v1.1.0 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
-	github.com/pierrec/lz4/v4 v4.1.8 // indirect
-	github.com/piyushsingariya/relec v0.0.18 // indirect
+	github.com/piyushsingariya/relec v0.0.20 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/rogpeppe/go-internal v1.13.1 // indirect
+	github.com/shirou/gopsutil/v4 v4.24.8 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/cobra v1.8.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/xitongsys/parquet-go v1.6.2 // indirect
 	github.com/xitongsys/parquet-go-source v0.0.0-20241021075129-b732d2ac9c9b // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	golang.org/x/crypto v0.24.0 // indirect
 	golang.org/x/net v0.26.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
 	golang.org/x/sys v0.24.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 	golang.org/x/tools v0.22.0 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

drivers/hubspot/go.mod

@@ -8,15 +8,22 @@ require (
 )
 
 require (
-	github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect
-	github.com/apache/thrift v0.14.2 // indirect
-	github.com/golang/snappy v0.0.3 // indirect
-	github.com/klauspost/compress v1.15.9 // indirect
-	github.com/pierrec/lz4/v4 v4.1.8 // indirect
+	github.com/apache/thrift v0.16.0 // indirect
+	github.com/felixge/fgprof v0.9.5 // indirect
+	github.com/fraugster/parquet-go v0.12.0 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect
+	github.com/gorilla/mux v1.8.1 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/rogpeppe/go-internal v1.13.1 // indirect
+	github.com/shirou/gopsutil/v4 v4.24.8 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/xitongsys/parquet-go v1.6.2 // indirect
 	github.com/xitongsys/parquet-go-source v0.0.0-20241021075129-b732d2ac9c9b // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 )
 
 require (
@@ -35,7 +42,7 @@ require (
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mitchellh/hashstructure v1.1.0 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
-	github.com/piyushsingariya/relec v0.0.18 // indirect
+	github.com/piyushsingariya/relec v0.0.20
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/cobra v1.8.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect