more consistent workflow

packaging/init.conf

@@ -10,7 +10,4 @@ BONDSTER_BCO_LAKE_HOSTNAME=127.0.0.1
 BONDSTER_BCO_HTTP_PORT=4001
 BONDSTER_BCO_SERVER_KEY=/etc/bondster-bco/secrets/domain.local.key
 BONDSTER_BCO_SERVER_CERT=/etc/bondster-bco/secrets/domain.local.crt
-BONDSTER_BCO_ENCRYPTION_KEY=/etc/bondster-bco/secrets/fs_encryption.key
-BONDSTER_BCO_METRICS_CONTINUOUS=true
-BONDSTER_BCO_METRICS_REFRESHRATE=1s
-BONDSTER_BCO_METRICS_OUTPUT=/opt/bondster-bco/metrics/metrics.json
+BONDSTER_BCO_ENCRYPTION_KEY=/etc/bondster-bco/secrets/fs_encryption.key

services/bondster-bco-import/go.mod

@@ -5,7 +5,7 @@ go 1.17
 require (
 	github.com/DataDog/datadog-go v4.8.3+incompatible
 	github.com/jancajthaml-openbank/actor-system v1.5.4
-	github.com/jancajthaml-openbank/local-fs v1.2.6
+	github.com/jancajthaml-openbank/local-fs v1.2.8
 	github.com/rs/zerolog v1.26.1
 	github.com/stretchr/testify v1.7.0
 )

services/bondster-bco-import/go.sum

@@ -9,8 +9,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/jancajthaml-openbank/actor-system v1.5.4 h1:1Eqn4A62uBPkEPW/0k9VxwSqdCdWKUWGlRQEQG9oPtU=
 github.com/jancajthaml-openbank/actor-system v1.5.4/go.mod h1:nIHD5as3cIrWQKj/YH1qJcEoGDJ9bLspb/rEfRwCv4U=
-github.com/jancajthaml-openbank/local-fs v1.2.6 h1:KF9a1A3VN7Qj1epurckNV2shoVS6MREyN9tcOXgvtjI=
-github.com/jancajthaml-openbank/local-fs v1.2.6/go.mod h1:W19XcWPceSnj+ZE8UOZapMlinVIxq8xJsGogTfv80XM=
+github.com/jancajthaml-openbank/local-fs v1.2.8 h1:66HspCnsoPgrrNqU/O2EShpkdcuoqDXtSOzTli3Igeo=
+github.com/jancajthaml-openbank/local-fs v1.2.8/go.mod h1:W19XcWPceSnj+ZE8UOZapMlinVIxq8xJsGogTfv80XM=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/pebbe/zmq4 v1.2.7 h1:6EaX83hdFSRUEhgzSW1E/SPoTS3JeYZgYkBvwdcrA9A=
 github.com/pebbe/zmq4 v1.2.7/go.mod h1:nqnPueOapVhE2wItZ0uOErngczsJdLOGkebMxaO8r48=

services/bondster-bco-import/integration/worker.go

@@ -62,8 +62,6 @@ func (bondster BondsterImport) Setup() error {
 
 // Work performs import roundtrip
 func (bondster BondsterImport) Work() {
-	log.Info().Msg("Import statements from Bondster Marketplace")
-
 	tokens, err := bondster.getActiveTokens()
 	if err != nil {
 		log.Error().Err(err).Msg("unable to get active tokens")

services/bondster-bco-import/integration/workflow.go

@@ -88,7 +88,7 @@ func importAccountsFromStatemets(
 	idsNeedingConfirmation := make([]string, 0)
 
 	for _, id := range ids {
-		exists, err := plaintextStorage.Exists("token/" + token.ID + "/statements/" + currency + "/" + id + "/accounts")
+		exists, err := plaintextStorage.Exists("token/" + token.ID + "/statements/" + currency + "/" + id + "/ack_account")
 		if err != nil {
 			log.Warn().Err(err).Msgf("Unable to check if statement %s/%s/%s accounts exists", token.ID, currency, id)
 			continue
@@ -126,7 +126,7 @@ func importAccountsFromStatemets(
 			Tenant:         tenant,
 			Name:           account,
 			Currency:       currency,
-			Format:         "BONDSTER_TECHNICAL",
+			Format:         "BONDSTER_VIRTUAL",
 			IsBalanceCheck: false,
 		}
 		err = vaultClient.CreateAccount(request)
@@ -137,7 +137,7 @@ func importAccountsFromStatemets(
 	}
 
 	for _, id := range idsNeedingConfirmation {
-		err = plaintextStorage.TouchFile("token/" + token.ID + "/statements/" + currency + "/" + id + "/accounts")
+		err = plaintextStorage.TouchFile("token/" + token.ID + "/statements/" + currency + "/" + id + "/ack_account")
 		if err != nil {
 			log.Warn().Err(err).Msgf("Unable to mark account discovery for %s/%s/%s", token.ID, currency, id)
 		}
@@ -165,7 +165,7 @@ func importTransactionsFromStatemets(
 	}
 
 	for _, id := range ids {
-		exists, err := plaintextStorage.Exists("token/" + token.ID + "/statements/" + currency + "/" + id + "/done")
+		exists, err := plaintextStorage.Exists("token/" + token.ID + "/statements/" + currency + "/" + id + "/ack_transfer")
 		if err != nil {
 			log.Warn().Msgf("Unable to check if statement %s/%s/%s done exists", token.ID, currency, id)
 			continue
@@ -226,7 +226,7 @@ func importTransactionsFromStatemets(
 
 		metrics.TransactionImported(1)
 
-		err = plaintextStorage.TouchFile("token/" + token.ID + "/statements/" + currency + "/" + id + "/done")
+		err = plaintextStorage.TouchFile("token/" + token.ID + "/statements/" + currency + "/" + id + "/ack_transfer")
 		if err != nil {
 			log.Warn().Msgf("Unable to mark statement done for %s/%s/%s", token.ID, currency, id)
 			continue
@@ -248,7 +248,7 @@ func downloadStatements(
 	if len(ids) == 0 {
 		return startTime
 	}
-	log.Debug().Msgf("Will synchronize %d statements in %s currency", len(ids), currency)
+	log.Debug().Msgf("Will download %d statements in %s currency", len(ids), currency)
 	statements, err := bondsterClient.GetStatements(currency, ids)
 	if err != nil {
 		log.Warn().Msgf("Unable to download statements details for currency %s", currency)
@@ -330,9 +330,10 @@ func downloadStatementsForCurrency(
 	lastTime := *lastSyncedTime
 	endTime := time.Now()
 
-	log.Info().Msgf("Token %s discovering new statements for currency %s between %s and %s", token.ID, currency, lastTime.Format("2006-01-02T15:04:05Z0700"), endTime.Format("2006-01-02T15:04:05Z0700"))
-
 	for _, interval := range timeshift.PartitionInterval(lastTime, endTime) {
+
+		log.Debug().Msgf("Token %s discovering new statements for currency %s between %s and %s", token.ID, currency, interval.StartTime.Format("2006-01-02T15:04:05Z0700"), interval.EndTime.Format("2006-01-02T15:04:05Z0700"))
+
 		ids, err := bondsterClient.GetStatementIdsInInterval(currency, interval)
 		if err != nil {
 			log.Warn().Err(err).Msgf("Unable to obtain transaction ids for token %s currency %s", token.ID, currency)
@@ -347,9 +348,9 @@ func downloadStatementsForCurrency(
 			if exists {
 				continue
 			}
-			err = plaintextStorage.TouchFile("token/" + token.ID + "/statements/" + currency + "/" + id + "/mark")
+			err = plaintextStorage.Mkdir("token/" + token.ID + "/statements/" + currency + "/" + id)
 			if err != nil {
-				log.Warn().Err(err).Msgf("Unable to mark transaction %s as known for token %s currency %s", id, token.ID, currency)
+				log.Warn().Err(err).Msgf("Unable to ensure statement partition %s as known for token %s currency %s", id, token.ID, currency)
 				return
 			}
 		}
@@ -374,7 +375,6 @@ func downloadStatementsForCurrency(
 			log.Warn().Msgf("unable to update token %s", token.ID)
 		}
 	}
-
 }
 
 // DownloadStatements download new statements from bonster gateway
@@ -420,6 +420,8 @@ func (workflow Workflow) DownloadStatements() {
 }
 
 func (workflow Workflow) CreateAccounts() {
+	log.Debug().Msgf("token %s creating accounts from statements", workflow.Token.ID)
+
 	currencies := workflow.Token.GetCurrencies()
 
 	// FIXME better with daemon support and cancelation
@@ -440,6 +442,8 @@ func (workflow Workflow) CreateAccounts() {
 }
 
 func (workflow Workflow) CreateTransactions() {
+	log.Debug().Msgf("token %s creating transactions from statements", workflow.Token.ID)
+
 	currencies := workflow.Token.GetCurrencies()
 
 	// FIXME better with daemon support and cancelation

services/bondster-bco-import/persistence/token.go

@@ -38,7 +38,7 @@ func LoadTokens(storage localfs.Storage) ([]model.Token, error) {
 		token := model.Token{
 			ID: id,
 		}
-		if HydrateToken(storage, &token) != nil {
+		if HydrateToken(storage, &token) == nil {
 			result[i] = token
 		}
 	}

services/bondster-bco-rest/go.mod

@@ -5,7 +5,7 @@ go 1.17
 require (
 	github.com/coreos/go-systemd/v22 v22.3.2
 	github.com/jancajthaml-openbank/actor-system v1.5.4
-	github.com/jancajthaml-openbank/local-fs v1.2.6
+	github.com/jancajthaml-openbank/local-fs v1.2.8
 	github.com/labstack/echo/v4 v4.6.1
 	github.com/rs/xid v1.3.0
 	github.com/rs/zerolog v1.26.1

services/bondster-bco-rest/go.sum

@@ -7,8 +7,8 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/jancajthaml-openbank/actor-system v1.5.4 h1:1Eqn4A62uBPkEPW/0k9VxwSqdCdWKUWGlRQEQG9oPtU=
 github.com/jancajthaml-openbank/actor-system v1.5.4/go.mod h1:nIHD5as3cIrWQKj/YH1qJcEoGDJ9bLspb/rEfRwCv4U=
-github.com/jancajthaml-openbank/local-fs v1.2.6 h1:KF9a1A3VN7Qj1epurckNV2shoVS6MREyN9tcOXgvtjI=
-github.com/jancajthaml-openbank/local-fs v1.2.6/go.mod h1:W19XcWPceSnj+ZE8UOZapMlinVIxq8xJsGogTfv80XM=
+github.com/jancajthaml-openbank/local-fs v1.2.8 h1:66HspCnsoPgrrNqU/O2EShpkdcuoqDXtSOzTli3Igeo=
+github.com/jancajthaml-openbank/local-fs v1.2.8/go.mod h1:W19XcWPceSnj+ZE8UOZapMlinVIxq8xJsGogTfv80XM=
 github.com/labstack/echo/v4 v4.6.1 h1:OMVsrnNFzYlGSdaiYGHbgWQnr+JM7NG+B9suCPie14M=
 github.com/labstack/echo/v4 v4.6.1/go.mod h1:RnjgMWNDB9g/HucVWhQYNQP9PvbYf6adqftqryo7s9k=
 github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=

services/bondster-bco-rest/model/model.go

@@ -26,10 +26,10 @@ import (
 
 // Token represents metadata of token entity
 type Token struct {
-	ID        string    `json:"-"`
-	CreatedAt time.Time `json:"-"`
-	Username  string    `json:"username"`
-	Password  string    `json:"password"`
+	ID        string
+	CreatedAt time.Time
+	Username  string
+	Password  string
 }
 
 // MarshalJSON serializes Token as json