diff --git a/packaging/init.conf b/packaging/init.conf index eb40aff..2c6135b 100644 --- a/packaging/init.conf +++ b/packaging/init.conf @@ -10,7 +10,4 @@ BONDSTER_BCO_LAKE_HOSTNAME=127.0.0.1 BONDSTER_BCO_HTTP_PORT=4001 BONDSTER_BCO_SERVER_KEY=/etc/bondster-bco/secrets/domain.local.key BONDSTER_BCO_SERVER_CERT=/etc/bondster-bco/secrets/domain.local.crt -BONDSTER_BCO_ENCRYPTION_KEY=/etc/bondster-bco/secrets/fs_encryption.key -BONDSTER_BCO_METRICS_CONTINUOUS=true -BONDSTER_BCO_METRICS_REFRESHRATE=1s -BONDSTER_BCO_METRICS_OUTPUT=/opt/bondster-bco/metrics/metrics.json +BONDSTER_BCO_ENCRYPTION_KEY=/etc/bondster-bco/secrets/fs_encryption.key \ No newline at end of file diff --git a/services/bondster-bco-import/go.mod b/services/bondster-bco-import/go.mod index bb5b4b0..a0ce865 100644 --- a/services/bondster-bco-import/go.mod +++ b/services/bondster-bco-import/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/DataDog/datadog-go v4.8.3+incompatible github.com/jancajthaml-openbank/actor-system v1.5.4 - github.com/jancajthaml-openbank/local-fs v1.2.6 + github.com/jancajthaml-openbank/local-fs v1.2.8 github.com/rs/zerolog v1.26.1 github.com/stretchr/testify v1.7.0 ) diff --git a/services/bondster-bco-import/go.sum b/services/bondster-bco-import/go.sum index 1aff3eb..b7626f4 100644 --- a/services/bondster-bco-import/go.sum +++ b/services/bondster-bco-import/go.sum @@ -9,8 +9,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/jancajthaml-openbank/actor-system v1.5.4 h1:1Eqn4A62uBPkEPW/0k9VxwSqdCdWKUWGlRQEQG9oPtU= github.com/jancajthaml-openbank/actor-system v1.5.4/go.mod h1:nIHD5as3cIrWQKj/YH1qJcEoGDJ9bLspb/rEfRwCv4U= -github.com/jancajthaml-openbank/local-fs v1.2.6 h1:KF9a1A3VN7Qj1epurckNV2shoVS6MREyN9tcOXgvtjI= -github.com/jancajthaml-openbank/local-fs v1.2.6/go.mod h1:W19XcWPceSnj+ZE8UOZapMlinVIxq8xJsGogTfv80XM= +github.com/jancajthaml-openbank/local-fs v1.2.8 h1:66HspCnsoPgrrNqU/O2EShpkdcuoqDXtSOzTli3Igeo= +github.com/jancajthaml-openbank/local-fs v1.2.8/go.mod h1:W19XcWPceSnj+ZE8UOZapMlinVIxq8xJsGogTfv80XM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/pebbe/zmq4 v1.2.7 h1:6EaX83hdFSRUEhgzSW1E/SPoTS3JeYZgYkBvwdcrA9A= github.com/pebbe/zmq4 v1.2.7/go.mod h1:nqnPueOapVhE2wItZ0uOErngczsJdLOGkebMxaO8r48= diff --git a/services/bondster-bco-import/integration/worker.go b/services/bondster-bco-import/integration/worker.go index 9274e14..6b34f0b 100644 --- a/services/bondster-bco-import/integration/worker.go +++ b/services/bondster-bco-import/integration/worker.go @@ -62,8 +62,6 @@ func (bondster BondsterImport) Setup() error { // Work performs import roundtrip func (bondster BondsterImport) Work() { - log.Info().Msg("Import statements from Bondster Marketplace") - tokens, err := bondster.getActiveTokens() if err != nil { log.Error().Err(err).Msg("unable to get active tokens") diff --git a/services/bondster-bco-import/integration/workflow.go b/services/bondster-bco-import/integration/workflow.go index be1b2a5..e6801f2 100644 --- a/services/bondster-bco-import/integration/workflow.go +++ b/services/bondster-bco-import/integration/workflow.go @@ -88,7 +88,7 @@ func importAccountsFromStatemets( idsNeedingConfirmation := make([]string, 0) for _, id := range ids { - exists, err := plaintextStorage.Exists("token/" + token.ID + "/statements/" + currency + "/" + id + "/accounts") + exists, err := plaintextStorage.Exists("token/" + token.ID + "/statements/" + currency + "/" + id + "/ack_account") if err != nil { log.Warn().Err(err).Msgf("Unable to check if statement %s/%s/%s accounts exists", token.ID, currency, id) continue @@ -126,7 +126,7 @@ func importAccountsFromStatemets( Tenant: tenant, Name: account, Currency: currency, - Format: "BONDSTER_TECHNICAL", + Format: "BONDSTER_VIRTUAL", IsBalanceCheck: false, } err = vaultClient.CreateAccount(request) @@ -137,7 +137,7 @@ func importAccountsFromStatemets( } for _, id := range idsNeedingConfirmation { - err = plaintextStorage.TouchFile("token/" + token.ID + "/statements/" + currency + "/" + id + "/accounts") + err = plaintextStorage.TouchFile("token/" + token.ID + "/statements/" + currency + "/" + id + "/ack_account") if err != nil { log.Warn().Err(err).Msgf("Unable to mark account discovery for %s/%s/%s", token.ID, currency, id) } @@ -165,7 +165,7 @@ func importTransactionsFromStatemets( } for _, id := range ids { - exists, err := plaintextStorage.Exists("token/" + token.ID + "/statements/" + currency + "/" + id + "/done") + exists, err := plaintextStorage.Exists("token/" + token.ID + "/statements/" + currency + "/" + id + "/ack_transfer") if err != nil { log.Warn().Msgf("Unable to check if statement %s/%s/%s done exists", token.ID, currency, id) continue @@ -226,7 +226,7 @@ func importTransactionsFromStatemets( metrics.TransactionImported(1) - err = plaintextStorage.TouchFile("token/" + token.ID + "/statements/" + currency + "/" + id + "/done") + err = plaintextStorage.TouchFile("token/" + token.ID + "/statements/" + currency + "/" + id + "/ack_transfer") if err != nil { log.Warn().Msgf("Unable to mark statement done for %s/%s/%s", token.ID, currency, id) continue @@ -248,7 +248,7 @@ func downloadStatements( if len(ids) == 0 { return startTime } - log.Debug().Msgf("Will synchronize %d statements in %s currency", len(ids), currency) + log.Debug().Msgf("Will download %d statements in %s currency", len(ids), currency) statements, err := bondsterClient.GetStatements(currency, ids) if err != nil { log.Warn().Msgf("Unable to download statements details for currency %s", currency) @@ -330,9 +330,10 @@ func downloadStatementsForCurrency( lastTime := *lastSyncedTime endTime := time.Now() - log.Info().Msgf("Token %s discovering new statements for currency %s between %s and %s", token.ID, currency, lastTime.Format("2006-01-02T15:04:05Z0700"), endTime.Format("2006-01-02T15:04:05Z0700")) - for _, interval := range timeshift.PartitionInterval(lastTime, endTime) { + + log.Debug().Msgf("Token %s discovering new statements for currency %s between %s and %s", token.ID, currency, interval.StartTime.Format("2006-01-02T15:04:05Z0700"), interval.EndTime.Format("2006-01-02T15:04:05Z0700")) + ids, err := bondsterClient.GetStatementIdsInInterval(currency, interval) if err != nil { log.Warn().Err(err).Msgf("Unable to obtain transaction ids for token %s currency %s", token.ID, currency) @@ -347,9 +348,9 @@ func downloadStatementsForCurrency( if exists { continue } - err = plaintextStorage.TouchFile("token/" + token.ID + "/statements/" + currency + "/" + id + "/mark") + err = plaintextStorage.Mkdir("token/" + token.ID + "/statements/" + currency + "/" + id) if err != nil { - log.Warn().Err(err).Msgf("Unable to mark transaction %s as known for token %s currency %s", id, token.ID, currency) + log.Warn().Err(err).Msgf("Unable to ensure statement partition %s as known for token %s currency %s", id, token.ID, currency) return } } @@ -374,7 +375,6 @@ func downloadStatementsForCurrency( log.Warn().Msgf("unable to update token %s", token.ID) } } - } // DownloadStatements download new statements from bonster gateway @@ -420,6 +420,8 @@ func (workflow Workflow) DownloadStatements() { } func (workflow Workflow) CreateAccounts() { + log.Debug().Msgf("token %s creating accounts from statements", workflow.Token.ID) + currencies := workflow.Token.GetCurrencies() // FIXME better with daemon support and cancelation @@ -440,6 +442,8 @@ func (workflow Workflow) CreateAccounts() { } func (workflow Workflow) CreateTransactions() { + log.Debug().Msgf("token %s creating transactions from statements", workflow.Token.ID) + currencies := workflow.Token.GetCurrencies() // FIXME better with daemon support and cancelation diff --git a/services/bondster-bco-import/persistence/token.go b/services/bondster-bco-import/persistence/token.go index b9a4ab3..2973702 100644 --- a/services/bondster-bco-import/persistence/token.go +++ b/services/bondster-bco-import/persistence/token.go @@ -38,7 +38,7 @@ func LoadTokens(storage localfs.Storage) ([]model.Token, error) { token := model.Token{ ID: id, } - if HydrateToken(storage, &token) != nil { + if HydrateToken(storage, &token) == nil { result[i] = token } } diff --git a/services/bondster-bco-rest/go.mod b/services/bondster-bco-rest/go.mod index 7f692aa..61ebd14 100644 --- a/services/bondster-bco-rest/go.mod +++ b/services/bondster-bco-rest/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/coreos/go-systemd/v22 v22.3.2 github.com/jancajthaml-openbank/actor-system v1.5.4 - github.com/jancajthaml-openbank/local-fs v1.2.6 + github.com/jancajthaml-openbank/local-fs v1.2.8 github.com/labstack/echo/v4 v4.6.1 github.com/rs/xid v1.3.0 github.com/rs/zerolog v1.26.1 diff --git a/services/bondster-bco-rest/go.sum b/services/bondster-bco-rest/go.sum index 910fc61..bc8f26b 100644 --- a/services/bondster-bco-rest/go.sum +++ b/services/bondster-bco-rest/go.sum @@ -7,8 +7,8 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/jancajthaml-openbank/actor-system v1.5.4 h1:1Eqn4A62uBPkEPW/0k9VxwSqdCdWKUWGlRQEQG9oPtU= github.com/jancajthaml-openbank/actor-system v1.5.4/go.mod h1:nIHD5as3cIrWQKj/YH1qJcEoGDJ9bLspb/rEfRwCv4U= -github.com/jancajthaml-openbank/local-fs v1.2.6 h1:KF9a1A3VN7Qj1epurckNV2shoVS6MREyN9tcOXgvtjI= -github.com/jancajthaml-openbank/local-fs v1.2.6/go.mod h1:W19XcWPceSnj+ZE8UOZapMlinVIxq8xJsGogTfv80XM= +github.com/jancajthaml-openbank/local-fs v1.2.8 h1:66HspCnsoPgrrNqU/O2EShpkdcuoqDXtSOzTli3Igeo= +github.com/jancajthaml-openbank/local-fs v1.2.8/go.mod h1:W19XcWPceSnj+ZE8UOZapMlinVIxq8xJsGogTfv80XM= github.com/labstack/echo/v4 v4.6.1 h1:OMVsrnNFzYlGSdaiYGHbgWQnr+JM7NG+B9suCPie14M= github.com/labstack/echo/v4 v4.6.1/go.mod h1:RnjgMWNDB9g/HucVWhQYNQP9PvbYf6adqftqryo7s9k= github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= diff --git a/services/bondster-bco-rest/model/model.go b/services/bondster-bco-rest/model/model.go index 22e3b9a..a96ae6f 100644 --- a/services/bondster-bco-rest/model/model.go +++ b/services/bondster-bco-rest/model/model.go @@ -26,10 +26,10 @@ import ( // Token represents metadata of token entity type Token struct { - ID string `json:"-"` - CreatedAt time.Time `json:"-"` - Username string `json:"username"` - Password string `json:"password"` + ID string + CreatedAt time.Time + Username string + Password string } // MarshalJSON serializes Token as json