diff --git a/.github/workflows/horizon.yml b/.github/workflows/horizon.yml index b2dcf77ef1..10d53badab 100644 --- a/.github/workflows/horizon.yml +++ b/.github/workflows/horizon.yml @@ -35,7 +35,7 @@ jobs: HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_USE_DB: true PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.3.1-2007.4ede19620.focal PROTOCOL_21_CORE_DOCKER_IMG: stellar/stellar-core:21.3.1-2007.4ede19620.focal - PROTOCOL_21_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.4.1 + PROTOCOL_21_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.5.1 PROTOCOL_22_CORE_DEBIAN_PKG_VERSION: 22.0.0-2095.rc2.1bccbc921.focal PROTOCOL_22_CORE_DOCKER_IMG: stellar/stellar-core:22.0.0-2095.rc2.1bccbc921.focal PROTOCOL_22_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:22.0.0-rc2-99 diff --git a/ingest/CHANGELOG.md b/ingest/CHANGELOG.md index ed168de74e..3d3835a799 100644 --- a/ingest/CHANGELOG.md +++ b/ingest/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## Pending + +### New Features +* Create new package `ingest/cdp` for new components which will assist towards writing data transformation pipelines as part of [Composable Data Platform](https://stellar.org/blog/developers/composable-data-platform). +* Add new functional producer, `cdp.ApplyLedgerMetadata`. A new function which enables a private instance of `BufferedStorageBackend` to perfrom the role of a producer operator in streaming pipeline designs. It will emit pre-computed `LedgerCloseMeta` from a chosen `DataStore`. The stream can use `ApplyLedgerMetadata` as the origin of `LedgerCloseMeta`, providing a callback function which acts as the next operator in the stream, receiving the `LedgerCloseMeta`. [5462](https://github.com/stellar/go/pull/5462). + ### Stellar Core Protocol 21 Configuration Update: * BucketlistDB is now the default database for stellar-core, replacing the experimental option. As a result, the `EXPERIMENTAL_BUCKETLIST_DB` configuration parameter has been deprecated. * A new mandatory parameter, `DEPRECATED_SQL_LEDGER_STATE`, has been added with a default value of false which equivalent to `EXPERIMENTAL_BUCKETLIST_DB` being set to true. diff --git a/ingest/cdp/producer.go b/ingest/cdp/producer.go new file mode 100644 index 0000000000..0c5ff6de49 --- /dev/null +++ b/ingest/cdp/producer.go @@ -0,0 +1,147 @@ +package cdp + +import ( + "context" + "fmt" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/stellar/go/ingest/ledgerbackend" + "github.com/stellar/go/support/datastore" + "github.com/stellar/go/support/log" + "github.com/stellar/go/support/ordered" + "github.com/stellar/go/xdr" +) + +// provide testing hooks to inject mocks of these +var datastoreFactory = datastore.NewDataStore + +// Generate a default buffered storage config with values +// set to optimize buffered performance to some degree based +// on number of ledgers per file expected in the underlying +// datastore used by an instance of BufferedStorageBackend. +// +// these numbers were derived empirically from benchmarking analysis: +// https://github.com/stellar/go/issues/5390 +// +// ledgersPerFile - number of ledgers per file from remote datastore schema. +// return - preconfigured instance of BufferedStorageBackendConfig +func DefaultBufferedStorageBackendConfig(ledgersPerFile uint32) ledgerbackend.BufferedStorageBackendConfig { + + config := ledgerbackend.BufferedStorageBackendConfig{ + RetryLimit: 5, + RetryWait: 30 * time.Second, + } + + switch { + case ledgersPerFile < 2: + config.BufferSize = 500 + config.NumWorkers = 5 + return config + case ledgersPerFile < 101: + config.BufferSize = 10 + config.NumWorkers = 5 + return config + default: + config.BufferSize = 10 + config.NumWorkers = 2 + return config + } +} + +type PublisherConfig struct { + // Registry, optional, include to capture buffered storage backend metrics + Registry *prometheus.Registry + // RegistryNamespace, optional, include to emit buffered storage backend + // under this namespace + RegistryNamespace string + // BufferedStorageConfig, required + BufferedStorageConfig ledgerbackend.BufferedStorageBackendConfig + //DataStoreConfig, required + DataStoreConfig datastore.DataStoreConfig + // Log, optional, if nil uses go default logger + Log *log.Entry +} + +// ApplyLedgerMetadata - creates an internal instance +// of BufferedStorageBackend using provided config and emits +// ledger metadata for the requested range by invoking the provided callback +// once per ledger. +// +// The function is blocking, it will only return when a bounded range +// is completed, the ctx is canceled, or an error occurs. +// +// ledgerRange - the requested range, can be bounded or unbounded. +// +// publisherConfig - PublisherConfig. Provide configuration settings for DataStore +// and BufferedStorageBackend. Use DefaultBufferedStorageBackendConfig() to create +// optimized BufferedStorageBackendConfig. +// +// ctx - the context. Caller uses this to cancel the internal ledger processing, +// when canceled, the function will return asap with that error. +// +// callback - function. Invoked for every LedgerCloseMeta. If callback invocation +// returns an error, the processing will stop and return an error asap. +// +// return - error, function only returns if requested range is bounded or an error occured. +// nil will be returned only if bounded range requested and completed processing with no errors. +// otherwise return will always be an error. +func ApplyLedgerMetadata(ledgerRange ledgerbackend.Range, + publisherConfig PublisherConfig, + ctx context.Context, + callback func(xdr.LedgerCloseMeta) error) error { + + logger := publisherConfig.Log + if logger == nil { + logger = log.DefaultLogger + } + + dataStore, err := datastoreFactory(ctx, publisherConfig.DataStoreConfig) + if err != nil { + return fmt.Errorf("failed to create datastore: %w", err) + } + + var ledgerBackend ledgerbackend.LedgerBackend + ledgerBackend, err = ledgerbackend.NewBufferedStorageBackend(publisherConfig.BufferedStorageConfig, dataStore) + if err != nil { + return fmt.Errorf("failed to create buffered storage backend: %w", err) + } + + if publisherConfig.Registry != nil { + ledgerBackend = ledgerbackend.WithMetrics(ledgerBackend, publisherConfig.Registry, publisherConfig.RegistryNamespace) + } + + if ledgerRange.Bounded() && ledgerRange.To() <= ledgerRange.From() { + return fmt.Errorf("invalid end value for bounded range, must be greater than start") + } + + if !ledgerRange.Bounded() && ledgerRange.To() > 0 { + return fmt.Errorf("invalid end value for unbounded range, must be zero") + } + + from := ordered.Max(2, ledgerRange.From()) + ledgerBackend.PrepareRange(ctx, ledgerRange) + + for ledgerSeq := from; ledgerSeq <= ledgerRange.To() || !ledgerRange.Bounded(); ledgerSeq++ { + var ledgerCloseMeta xdr.LedgerCloseMeta + + logger.WithField("sequence", ledgerSeq).Info("Requesting ledger from the backend...") + startTime := time.Now() + ledgerCloseMeta, err = ledgerBackend.GetLedger(ctx, ledgerSeq) + + if err != nil { + return fmt.Errorf("error getting ledger, %w", err) + } + + log.WithFields(log.F{ + "sequence": ledgerSeq, + "duration": time.Since(startTime).Seconds(), + }).Info("Ledger returned from the backend") + + err = callback(ledgerCloseMeta) + if err != nil { + return fmt.Errorf("received an error from callback invocation: %w", err) + } + } + return nil +} diff --git a/ingest/cdp/producer_test.go b/ingest/cdp/producer_test.go new file mode 100644 index 0000000000..ba3c10bc9b --- /dev/null +++ b/ingest/cdp/producer_test.go @@ -0,0 +1,303 @@ +package cdp + +import ( + "bytes" + "context" + "fmt" + "io" + "math" + "os" + "testing" + "time" + + "github.com/stellar/go/ingest/ledgerbackend" + "github.com/stellar/go/support/compressxdr" + "github.com/stellar/go/support/datastore" + "github.com/stellar/go/support/errors" + "github.com/stellar/go/xdr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func TestDefaultBSBConfigs(t *testing.T) { + smallConfig := ledgerbackend.BufferedStorageBackendConfig{ + RetryLimit: 5, + RetryWait: 30 * time.Second, + BufferSize: 500, + NumWorkers: 5, + } + + mediumConfig := ledgerbackend.BufferedStorageBackendConfig{ + RetryLimit: 5, + RetryWait: 30 * time.Second, + BufferSize: 10, + NumWorkers: 5, + } + + largeConfig := ledgerbackend.BufferedStorageBackendConfig{ + RetryLimit: 5, + RetryWait: 30 * time.Second, + BufferSize: 10, + NumWorkers: 2, + } + + assert.Equal(t, DefaultBufferedStorageBackendConfig(1), smallConfig) + assert.Equal(t, DefaultBufferedStorageBackendConfig(2), mediumConfig) + assert.Equal(t, DefaultBufferedStorageBackendConfig(100), mediumConfig) + assert.Equal(t, DefaultBufferedStorageBackendConfig(101), largeConfig) + assert.Equal(t, DefaultBufferedStorageBackendConfig(1000), largeConfig) +} + +func TestBSBProducerFn(t *testing.T) { + startLedger := uint32(2) + endLedger := uint32(3) + ctx := context.Background() + ledgerRange := ledgerbackend.BoundedRange(startLedger, endLedger) + mockDataStore := createMockdataStore(t, startLedger, endLedger, 64000) + dsConfig := datastore.DataStoreConfig{} + pubConfig := PublisherConfig{ + DataStoreConfig: dsConfig, + BufferedStorageConfig: DefaultBufferedStorageBackendConfig(1), + } + + // inject the mock datastore using the package private testing factory override + datastoreFactory = func(ctx context.Context, datastoreConfig datastore.DataStoreConfig) (datastore.DataStore, error) { + assert.Equal(t, datastoreConfig, dsConfig) + return mockDataStore, nil + } + + expectedLcmSeqWasPublished := []bool{false, false} + + appCallback := func(lcm xdr.LedgerCloseMeta) error { + if lcm.MustV0().LedgerHeader.Header.LedgerSeq == 2 { + if expectedLcmSeqWasPublished[0] { + assert.Fail(t, "producer fn had multiple callback invocations for same lcm") + } + expectedLcmSeqWasPublished[0] = true + } + if lcm.MustV0().LedgerHeader.Header.LedgerSeq == 3 { + if expectedLcmSeqWasPublished[1] { + assert.Fail(t, "producer fn had multiple callback invocations for same lcm") + } + expectedLcmSeqWasPublished[1] = true + } + return nil + } + + assert.Nil(t, ApplyLedgerMetadata(ledgerRange, pubConfig, ctx, appCallback)) + assert.Equal(t, expectedLcmSeqWasPublished, []bool{true, true}, "producer fn did not invoke callback for all expected lcm") +} + +func TestBSBProducerFnDataStoreError(t *testing.T) { + ctx := context.Background() + ledgerRange := ledgerbackend.BoundedRange(uint32(2), uint32(3)) + pubConfig := PublisherConfig{ + DataStoreConfig: datastore.DataStoreConfig{}, + BufferedStorageConfig: ledgerbackend.BufferedStorageBackendConfig{}, + } + + datastoreFactory = func(ctx context.Context, datastoreConfig datastore.DataStoreConfig) (datastore.DataStore, error) { + return &datastore.MockDataStore{}, errors.New("uhoh") + } + + appCallback := func(lcm xdr.LedgerCloseMeta) error { + return nil + } + + assert.ErrorContains(t, + ApplyLedgerMetadata(ledgerRange, pubConfig, ctx, appCallback), + "failed to create datastore:") +} + +func TestBSBProducerFnConfigError(t *testing.T) { + ctx := context.Background() + ledgerRange := ledgerbackend.BoundedRange(uint32(2), uint32(3)) + pubConfig := PublisherConfig{ + DataStoreConfig: datastore.DataStoreConfig{}, + BufferedStorageConfig: ledgerbackend.BufferedStorageBackendConfig{}, + } + mockDataStore := new(datastore.MockDataStore) + appCallback := func(lcm xdr.LedgerCloseMeta) error { + return nil + } + + datastoreFactory = func(_ context.Context, _ datastore.DataStoreConfig) (datastore.DataStore, error) { + return mockDataStore, nil + } + assert.ErrorContains(t, + ApplyLedgerMetadata(ledgerRange, pubConfig, ctx, appCallback), + "failed to create buffered storage backend") + mockDataStore.AssertExpectations(t) +} + +func TestBSBProducerFnInvalidRange(t *testing.T) { + ctx := context.Background() + pubConfig := PublisherConfig{ + DataStoreConfig: datastore.DataStoreConfig{}, + BufferedStorageConfig: DefaultBufferedStorageBackendConfig(1), + } + mockDataStore := new(datastore.MockDataStore) + mockDataStore.On("GetSchema").Return(datastore.DataStoreSchema{ + LedgersPerFile: 1, + FilesPerPartition: 1, + }) + + appCallback := func(lcm xdr.LedgerCloseMeta) error { + return nil + } + + datastoreFactory = func(_ context.Context, _ datastore.DataStoreConfig) (datastore.DataStore, error) { + return mockDataStore, nil + } + + assert.ErrorContains(t, + ApplyLedgerMetadata(ledgerbackend.BoundedRange(uint32(3), uint32(2)), pubConfig, ctx, appCallback), + "invalid end value for bounded range, must be greater than start") + mockDataStore.AssertExpectations(t) +} + +func TestBSBProducerFnGetLedgerError(t *testing.T) { + ctx := context.Background() + pubConfig := PublisherConfig{ + DataStoreConfig: datastore.DataStoreConfig{}, + BufferedStorageConfig: DefaultBufferedStorageBackendConfig(1), + } + // we don't want to let buffer do real retries, force the first error to propagate + pubConfig.BufferedStorageConfig.RetryLimit = 0 + mockDataStore := new(datastore.MockDataStore) + mockDataStore.On("GetSchema").Return(datastore.DataStoreSchema{ + LedgersPerFile: 1, + FilesPerPartition: 1, + }) + + mockDataStore.On("GetFile", mock.Anything, "FFFFFFFD--2.xdr.zstd").Return(nil, os.ErrNotExist).Once() + // since buffer is multi-worker async, it may get to this on other worker, but not deterministic, + // don't assert on it + mockDataStore.On("GetFile", mock.Anything, "FFFFFFFC--3.xdr.zstd").Return(makeSingleLCMBatch(3), nil).Maybe() + + appCallback := func(lcm xdr.LedgerCloseMeta) error { + return nil + } + + datastoreFactory = func(_ context.Context, _ datastore.DataStoreConfig) (datastore.DataStore, error) { + return mockDataStore, nil + } + assert.ErrorContains(t, + ApplyLedgerMetadata(ledgerbackend.BoundedRange(uint32(2), uint32(3)), pubConfig, ctx, appCallback), + "error getting ledger") + + mockDataStore.AssertExpectations(t) +} + +func TestBSBProducerCallerCancelsCtx(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + pubConfig := PublisherConfig{ + DataStoreConfig: datastore.DataStoreConfig{}, + BufferedStorageConfig: DefaultBufferedStorageBackendConfig(1), + } + + pubConfig.BufferedStorageConfig.NumWorkers = 1 + + // the buffering runs async, test needs to stub datastore methods for potential invocation, + // but is race, since test also cancels the backend context which started the buffer, + // so, not deterministic, no assert on these. + mockDataStore := new(datastore.MockDataStore) + mockDataStore.On("GetSchema").Return(datastore.DataStoreSchema{ + LedgersPerFile: 1, + FilesPerPartition: 1, + }) + + mockDataStore.On("GetFile", mock.Anything, "FFFFFFFD--2.xdr.zstd"). + Run(func(args mock.Arguments) { + cancel() + }). + Return(makeSingleLCMBatch(2), nil) + // this second attempt needs to be mocked, ledger buffer queues this 'next' sequence task automatically + // in getFromLedgerQueue after it receives "FFFFFFFD--2.xdr.zstd", the ctx is not checked then or in + // the async worker routine that receives the task. + mockDataStore.On("GetFile", mock.Anything, "FFFFFFFC--3.xdr.zstd").Return(makeSingleLCMBatch(3), nil) + + appCallback := func(lcm xdr.LedgerCloseMeta) error { + return nil + } + + datastoreFactory = func(_ context.Context, _ datastore.DataStoreConfig) (datastore.DataStore, error) { + return mockDataStore, nil + } + assert.ErrorIs(t, + ApplyLedgerMetadata(ledgerbackend.BoundedRange(uint32(2), uint32(3)), pubConfig, ctx, appCallback), + context.Canceled) +} + +func TestBSBProducerFnCallbackError(t *testing.T) { + ctx := context.Background() + pubConfig := PublisherConfig{ + DataStoreConfig: datastore.DataStoreConfig{}, + BufferedStorageConfig: DefaultBufferedStorageBackendConfig(1), + } + mockDataStore := createMockdataStore(t, 2, 3, 64000) + + appCallback := func(lcm xdr.LedgerCloseMeta) error { + return errors.New("uhoh") + } + + datastoreFactory = func(_ context.Context, _ datastore.DataStoreConfig) (datastore.DataStore, error) { + return mockDataStore, nil + } + assert.ErrorContains(t, + ApplyLedgerMetadata(ledgerbackend.BoundedRange(uint32(2), uint32(3)), pubConfig, ctx, appCallback), + "received an error from callback invocation") +} + +func createMockdataStore(t *testing.T, start, end, partitionSize uint32) *datastore.MockDataStore { + mockDataStore := new(datastore.MockDataStore) + partition := partitionSize - 1 + for i := start; i <= end; i++ { + objectName := fmt.Sprintf("FFFFFFFF--0-%d/%08X--%d.xdr.zstd", partition, math.MaxUint32-i, i) + mockDataStore.On("GetFile", mock.Anything, objectName).Return(makeSingleLCMBatch(i), nil).Once() + } + mockDataStore.On("GetSchema").Return(datastore.DataStoreSchema{ + LedgersPerFile: 1, + FilesPerPartition: partitionSize, + }) + + t.Cleanup(func() { + mockDataStore.AssertExpectations(t) + }) + + return mockDataStore +} + +func makeSingleLCMBatch(seq uint32) io.ReadCloser { + lcm := xdr.LedgerCloseMetaBatch{ + StartSequence: xdr.Uint32(seq), + EndSequence: xdr.Uint32(seq), + LedgerCloseMetas: []xdr.LedgerCloseMeta{ + createLedgerCloseMeta(seq), + }, + } + encoder := compressxdr.NewXDREncoder(compressxdr.DefaultCompressor, lcm) + var buf bytes.Buffer + encoder.WriteTo(&buf) + capturedBuf := buf.Bytes() + reader := bytes.NewReader(capturedBuf) + return io.NopCloser(reader) +} + +func createLedgerCloseMeta(ledgerSeq uint32) xdr.LedgerCloseMeta { + return xdr.LedgerCloseMeta{ + V: int32(0), + V0: &xdr.LedgerCloseMetaV0{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + LedgerSeq: xdr.Uint32(ledgerSeq), + }, + }, + TxSet: xdr.TransactionSet{}, + TxProcessing: nil, + UpgradesProcessing: nil, + ScpInfo: nil, + }, + V1: nil, + } +} diff --git a/ingest/ledgerbackend/buffered_storage_backend_test.go b/ingest/ledgerbackend/buffered_storage_backend_test.go index ca2711c40d..fe3ca0266c 100644 --- a/ingest/ledgerbackend/buffered_storage_backend_test.go +++ b/ingest/ledgerbackend/buffered_storage_backend_test.go @@ -76,7 +76,7 @@ func createMockdataStore(t *testing.T, start, end, partitionSize, count uint32) readCloser = createLCMBatchReader(i, i, count) objectName = fmt.Sprintf("FFFFFFFF--0-%d/%08X--%d.xdr.zstd", partition, math.MaxUint32-i, i) } - mockDataStore.On("GetFile", mock.Anything, objectName).Return(readCloser, nil) + mockDataStore.On("GetFile", mock.Anything, objectName).Return(readCloser, nil).Times(1) } mockDataStore.On("GetSchema").Return(datastore.DataStoreSchema{ LedgersPerFile: count, @@ -160,6 +160,50 @@ func TestNewLedgerBuffer(t *testing.T) { assert.Equal(t, ledgerRange, ledgerBuffer.ledgerRange) } +func TestNewLedgerBufferSizeLessThanRangeSize(t *testing.T) { + startLedger := uint32(10) + endLedger := uint32(30) + bsb := createBufferedStorageBackendForTesting() + bsb.config.NumWorkers = 2 + bsb.config.BufferSize = 10 + ledgerRange := BoundedRange(startLedger, endLedger) + mockDataStore := createMockdataStore(t, startLedger, endLedger, partitionSize, ledgerPerFileCount) + bsb.dataStore = mockDataStore + + ledgerBuffer, err := bsb.newLedgerBuffer(ledgerRange) + assert.Eventually(t, func() bool { return len(ledgerBuffer.ledgerQueue) == 10 }, time.Second*1, time.Millisecond*50) + assert.NoError(t, err) + + for i := startLedger; i < endLedger; i++ { + lcm, err := ledgerBuffer.getFromLedgerQueue(context.Background()) + assert.NoError(t, err) + assert.Equal(t, xdr.Uint32(i), lcm.StartSequence) + } + assert.Equal(t, ledgerRange, ledgerBuffer.ledgerRange) +} + +func TestNewLedgerBufferSizeLargerThanRangeSize(t *testing.T) { + startLedger := uint32(1) + endLedger := uint32(15) + bsb := createBufferedStorageBackendForTesting() + bsb.config.NumWorkers = 2 + bsb.config.BufferSize = 100 + ledgerRange := BoundedRange(startLedger, endLedger) + mockDataStore := createMockdataStore(t, startLedger, endLedger, partitionSize, ledgerPerFileCount) + bsb.dataStore = mockDataStore + + ledgerBuffer, err := bsb.newLedgerBuffer(ledgerRange) + assert.Eventually(t, func() bool { return len(ledgerBuffer.ledgerQueue) == 15 }, time.Second*1, time.Millisecond*50) + assert.NoError(t, err) + + for i := startLedger; i < endLedger; i++ { + lcm, err := ledgerBuffer.getFromLedgerQueue(context.Background()) + assert.NoError(t, err) + assert.Equal(t, xdr.Uint32(i), lcm.StartSequence) + } + assert.Equal(t, ledgerRange, ledgerBuffer.ledgerRange) +} + func TestBSBGetLatestLedgerSequence(t *testing.T) { startLedger := uint32(3) endLedger := uint32(5) @@ -505,7 +549,9 @@ func TestLedgerBufferBoundedObjectNotFound(t *testing.T) { bsb.ledgerBuffer.wg.Wait() _, err := bsb.GetLedger(ctx, 3) - assert.EqualError(t, err, "failed getting next ledger batch from queue: ledger object containing sequence 3 is missing: file does not exist") + assert.ErrorContains(t, err, "ledger object containing sequence 3 is missing") + assert.ErrorContains(t, err, objectName) + assert.ErrorContains(t, err, "file does not exist") } func TestLedgerBufferUnboundedObjectNotFound(t *testing.T) { @@ -571,5 +617,8 @@ func TestLedgerBufferRetryLimit(t *testing.T) { bsb.ledgerBuffer.wg.Wait() _, err := bsb.GetLedger(context.Background(), 3) - assert.EqualError(t, err, "failed getting next ledger batch from queue: maximum retries exceeded for downloading object containing sequence 3: transient error") + assert.ErrorContains(t, err, "failed getting next ledger batch from queue") + assert.ErrorContains(t, err, "maximum retries exceeded for downloading object containing sequence 3") + assert.ErrorContains(t, err, objectName) + assert.ErrorContains(t, err, "transient error") } diff --git a/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go index c8f28974f5..dc5365809b 100644 --- a/ingest/ledgerbackend/captive_core_backend.go +++ b/ingest/ledgerbackend/captive_core_backend.go @@ -112,9 +112,11 @@ type CaptiveStellarCore struct { lastLedger *uint32 // end of current segment if offline, nil if online previousLedgerHash *string - config CaptiveCoreConfig - stellarCoreClient *stellarcore.Client - captiveCoreVersion string // Updates when captive-core restarts + config CaptiveCoreConfig + captiveCoreStartDuration prometheus.Summary + captiveCoreNewDBCounter prometheus.Counter + stellarCoreClient *stellarcore.Client + captiveCoreVersion string // Updates when captive-core restarts } // CaptiveCoreConfig contains all the parameters required to create a CaptiveStellarCore instance @@ -230,7 +232,7 @@ func NewCaptive(config CaptiveCoreConfig) (*CaptiveStellarCore, error) { c.stellarCoreRunnerFactory = func() stellarCoreRunnerInterface { c.setCoreVersion() - return newStellarCoreRunner(config) + return newStellarCoreRunner(config, c.captiveCoreNewDBCounter) } if config.Toml != nil && config.Toml.HTTPPort != 0 { @@ -315,7 +317,27 @@ func (c *CaptiveStellarCore) registerMetrics(registry *prometheus.Registry, name return float64(latest) }, ) - registry.MustRegister(coreSynced, supportedProtocolVersion, latestLedger) + c.captiveCoreStartDuration = prometheus.NewSummary(prometheus.SummaryOpts{ + Namespace: namespace, + Subsystem: "ingest", + Name: "captive_stellar_core_start_duration_seconds", + Help: "duration of start up time when running captive core on an unbounded range, sliding window = 10m", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }) + c.captiveCoreNewDBCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "ingest", + Name: "captive_stellar_core_new_db", + Help: "counter for the number of times we start up captive core with a new buckets db, sliding window = 10m", + }) + + registry.MustRegister( + coreSynced, + supportedProtocolVersion, + latestLedger, + c.captiveCoreStartDuration, + c.captiveCoreNewDBCounter, + ) } func (c *CaptiveStellarCore) getLatestCheckpointSequence() (uint32, error) { @@ -521,12 +543,14 @@ func (c *CaptiveStellarCore) startPreparingRange(ctx context.Context, ledgerRang // Please note that using a BoundedRange, currently, requires a full-trust on // history archive. This issue is being fixed in Stellar-Core. func (c *CaptiveStellarCore) PrepareRange(ctx context.Context, ledgerRange Range) error { + startTime := time.Now() if alreadyPrepared, err := c.startPreparingRange(ctx, ledgerRange); err != nil { return errors.Wrap(err, "error starting prepare range") } else if alreadyPrepared { return nil } + var reportedStartTime bool // the prepared range might be below ledgerRange.from so we // need to seek ahead until we reach ledgerRange.from for seq := c.prepared.from; seq <= ledgerRange.from; seq++ { @@ -534,6 +558,12 @@ func (c *CaptiveStellarCore) PrepareRange(ctx context.Context, ledgerRange Range if err != nil { return errors.Wrapf(err, "Error fast-forwarding to %d", ledgerRange.from) } + if !reportedStartTime { + reportedStartTime = true + if c.captiveCoreStartDuration != nil && !ledgerRange.bounded { + c.captiveCoreStartDuration.Observe(time.Since(startTime).Seconds()) + } + } } return nil diff --git a/ingest/ledgerbackend/captive_core_backend_test.go b/ingest/ledgerbackend/captive_core_backend_test.go index f8161aec25..cb7e8dba7e 100644 --- a/ingest/ledgerbackend/captive_core_backend_test.go +++ b/ingest/ledgerbackend/captive_core_backend_test.go @@ -10,6 +10,8 @@ import ( "sync" "testing" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -224,7 +226,7 @@ func TestCaptivePrepareRange(t *testing.T) { }, nil) cancelCalled := false - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner @@ -234,6 +236,7 @@ func TestCaptivePrepareRange(t *testing.T) { cancelCalled = true }), } + captiveBackend.registerMetrics(prometheus.NewRegistry(), "test") err := captiveBackend.PrepareRange(ctx, BoundedRange(100, 200)) assert.NoError(t, err) @@ -243,6 +246,8 @@ func TestCaptivePrepareRange(t *testing.T) { assert.True(t, cancelCalled) mockRunner.AssertExpectations(t) mockArchive.AssertExpectations(t) + + assert.Equal(t, uint64(0), getStartDurationMetric(captiveBackend).GetSampleCount()) } func TestCaptivePrepareRangeCrash(t *testing.T) { @@ -263,7 +268,7 @@ func TestCaptivePrepareRangeCrash(t *testing.T) { CurrentLedger: uint32(200), }, nil) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner @@ -302,7 +307,7 @@ func TestCaptivePrepareRangeTerminated(t *testing.T) { CurrentLedger: uint32(200), }, nil) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner @@ -339,7 +344,7 @@ func TestCaptivePrepareRangeCloseNotFullyTerminated(t *testing.T) { CurrentLedger: uint32(200), }, nil) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner @@ -367,7 +372,7 @@ func TestCaptivePrepareRange_ErrClosingSession(t *testing.T) { mockRunner.On("getProcessExitError").Return(nil, false) mockRunner.On("context").Return(ctx) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ nextLedger: 300, stellarCoreRunner: mockRunner, } @@ -388,7 +393,7 @@ func TestCaptivePrepareRange_ErrGettingRootHAS(t *testing.T) { On("GetRootHAS"). Return(historyarchive.HistoryArchiveState{}, errors.New("transient error")) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, } @@ -412,7 +417,7 @@ func TestCaptivePrepareRange_FromIsAheadOfRootHAS(t *testing.T) { CurrentLedger: uint32(64), }, nil) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner @@ -460,7 +465,7 @@ func TestCaptivePrepareRangeWithDB_FromIsAheadOfRootHAS(t *testing.T) { CurrentLedger: uint32(64), }, nil) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, useDB: true, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { @@ -499,7 +504,7 @@ func TestCaptivePrepareRange_ToIsAheadOfRootHAS(t *testing.T) { CurrentLedger: uint32(192), }, nil) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner @@ -527,7 +532,7 @@ func TestCaptivePrepareRange_ErrCatchup(t *testing.T) { ctx := context.Background() cancelCalled := false - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner @@ -565,7 +570,7 @@ func TestCaptivePrepareRangeUnboundedRange_ErrRunFrom(t *testing.T) { ctx := context.Background() cancelCalled := false - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner @@ -575,10 +580,13 @@ func TestCaptivePrepareRangeUnboundedRange_ErrRunFrom(t *testing.T) { cancelCalled = true }), } + captiveBackend.registerMetrics(prometheus.NewRegistry(), "test") err := captiveBackend.PrepareRange(ctx, UnboundedRange(128)) assert.EqualError(t, err, "error starting prepare range: opening subprocess: error running stellar-core: transient error") + assert.Equal(t, uint64(0), getStartDurationMetric(captiveBackend).GetSampleCount()) + // make sure we can Close without errors assert.NoError(t, captiveBackend.Close()) assert.True(t, cancelCalled) @@ -587,6 +595,15 @@ func TestCaptivePrepareRangeUnboundedRange_ErrRunFrom(t *testing.T) { mockRunner.AssertExpectations(t) } +func getStartDurationMetric(captiveCore *CaptiveStellarCore) *dto.Summary { + value := &dto.Metric{} + err := captiveCore.captiveCoreStartDuration.Write(value) + if err != nil { + panic(err) + } + return value.GetSummary() +} + func TestCaptivePrepareRangeUnboundedRange_ReuseSession(t *testing.T) { metaChan := make(chan metaResult, 100) @@ -617,21 +634,27 @@ func TestCaptivePrepareRangeUnboundedRange_ReuseSession(t *testing.T) { On("GetLedgerHeader", uint32(65)). Return(xdr.LedgerHeaderHistoryEntry{}, nil) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner }, checkpointManager: historyarchive.NewCheckpointManager(64), } + captiveBackend.registerMetrics(prometheus.NewRegistry(), "test") err := captiveBackend.PrepareRange(ctx, UnboundedRange(65)) assert.NoError(t, err) + assert.Equal(t, uint64(1), getStartDurationMetric(captiveBackend).GetSampleCount()) + assert.Greater(t, getStartDurationMetric(captiveBackend).GetSampleSum(), float64(0)) + captiveBackend.nextLedger = 64 err = captiveBackend.PrepareRange(ctx, UnboundedRange(65)) assert.NoError(t, err) + assert.Equal(t, uint64(1), getStartDurationMetric(captiveBackend).GetSampleCount()) + mockArchive.AssertExpectations(t) mockRunner.AssertExpectations(t) } @@ -665,7 +688,7 @@ func TestGetLatestLedgerSequence(t *testing.T) { On("GetLedgerHeader", uint32(64)). Return(xdr.LedgerHeaderHistoryEntry{}, nil) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner @@ -712,7 +735,7 @@ func TestGetLatestLedgerSequenceRaceCondition(t *testing.T) { On("GetLedgerHeader", mock.Anything). Return(xdr.LedgerHeaderHistoryEntry{}, nil) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner @@ -775,7 +798,7 @@ func TestCaptiveGetLedger(t *testing.T) { CurrentLedger: uint32(200), }, nil) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner @@ -873,7 +896,7 @@ func TestCaptiveGetLedgerCacheLatestLedger(t *testing.T) { }, }, nil).Once() - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner @@ -928,7 +951,7 @@ func TestCaptiveGetLedger_NextLedgerIsDifferentToLedgerFromBuffer(t *testing.T) CurrentLedger: uint32(200), }, nil) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner @@ -978,7 +1001,7 @@ func TestCaptiveGetLedger_NextLedger0RangeFromIsSmallerThanLedgerFromBuffer(t *t On("GetLedgerHeader", uint32(65)). Return(xdr.LedgerHeaderHistoryEntry{}, nil) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner @@ -1081,7 +1104,7 @@ func TestCaptiveGetLedger_ErrReadingMetaResult(t *testing.T) { CurrentLedger: uint32(200), }, nil) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner @@ -1134,7 +1157,7 @@ func TestCaptiveGetLedger_ErrClosingAfterLastLedger(t *testing.T) { CurrentLedger: uint32(200), }, nil) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner @@ -1176,7 +1199,7 @@ func TestCaptiveAfterClose(t *testing.T) { CurrentLedger: uint32(200), }, nil) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner @@ -1230,7 +1253,7 @@ func TestGetLedgerBoundsCheck(t *testing.T) { CurrentLedger: uint32(200), }, nil) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner @@ -1356,7 +1379,7 @@ func TestCaptiveGetLedgerTerminatedUnexpectedly(t *testing.T) { CurrentLedger: uint32(200), }, nil) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner @@ -1410,7 +1433,7 @@ func TestCaptiveUseOfLedgerHashStore(t *testing.T) { Return("mnb", true, nil).Once() cancelCalled := false - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, ledgerHashStore: mockLedgerHashStore, checkpointManager: historyarchive.NewCheckpointManager(64), @@ -1493,7 +1516,7 @@ func TestCaptiveRunFromParams(t *testing.T) { CurrentLedger: uint32(255), }, nil) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, checkpointManager: historyarchive.NewCheckpointManager(64), } @@ -1515,7 +1538,7 @@ func TestCaptiveIsPrepared(t *testing.T) { mockRunner.On("getProcessExitError").Return(nil, false) // c.prepared == nil - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ nextLedger: 0, } @@ -1548,7 +1571,7 @@ func TestCaptiveIsPrepared(t *testing.T) { for _, tc := range tests { t.Run(fmt.Sprintf("next_%d_last_%d_cached_%d_range_%v", tc.nextLedger, tc.lastLedger, tc.cachedLedger, tc.ledgerRange), func(t *testing.T) { - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ stellarCoreRunner: mockRunner, nextLedger: tc.nextLedger, prepared: &tc.preparedRange, @@ -1579,7 +1602,7 @@ func TestCaptiveIsPreparedCoreContextCancelled(t *testing.T) { mockRunner.On("getProcessExitError").Return(nil, false) rang := UnboundedRange(100) - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ nextLedger: 100, prepared: &rang, stellarCoreRunner: mockRunner, @@ -1650,7 +1673,7 @@ func TestCaptivePreviousLedgerCheck(t *testing.T) { mockLedgerHashStore.On("GetLedgerHash", ctx, uint32(299)). Return("", false, nil).Once() - captiveBackend := CaptiveStellarCore{ + captiveBackend := &CaptiveStellarCore{ archive: mockArchive, stellarCoreRunnerFactory: func() stellarCoreRunnerInterface { return mockRunner diff --git a/ingest/ledgerbackend/file_watcher_test.go b/ingest/ledgerbackend/file_watcher_test.go index 7e84bbfcf2..c3e85d643f 100644 --- a/ingest/ledgerbackend/file_watcher_test.go +++ b/ingest/ledgerbackend/file_watcher_test.go @@ -65,7 +65,7 @@ func createFWFixtures(t *testing.T) (*mockHash, *stellarCoreRunner, *fileWatcher Context: context.Background(), Toml: captiveCoreToml, StoragePath: storagePath, - }) + }, nil) fw, err := newFileWatcherWithOptions(runner, ms.hashFile, time.Millisecond) assert.NoError(t, err) @@ -96,7 +96,7 @@ func TestNewFileWatcherError(t *testing.T) { Context: context.Background(), Toml: captiveCoreToml, StoragePath: storagePath, - }) + }, nil) _, err = newFileWatcherWithOptions(runner, ms.hashFile, time.Millisecond) assert.EqualError(t, err, "could not hash captive core binary: test error") diff --git a/ingest/ledgerbackend/ledger_buffer.go b/ingest/ledgerbackend/ledger_buffer.go index 6965461bba..7ee9dda083 100644 --- a/ingest/ledgerbackend/ledger_buffer.go +++ b/ingest/ledgerbackend/ledger_buffer.go @@ -13,6 +13,8 @@ import ( "github.com/stellar/go/support/collections/heap" "github.com/stellar/go/support/compressxdr" "github.com/stellar/go/support/datastore" + "github.com/stellar/go/support/ordered" + "github.com/stellar/go/xdr" ) @@ -54,6 +56,10 @@ func (bsb *BufferedStorageBackend) newLedgerBuffer(ledgerRange Range) (*ledgerBu less := func(a, b ledgerBatchObject) bool { return a.startLedger < b.startLedger } + // ensure BufferSize does not exceed the total range + if ledgerRange.bounded { + bsb.config.BufferSize = uint32(ordered.Min(int(bsb.config.BufferSize), int(ledgerRange.to-ledgerRange.from)+1)) + } pq := heap.New(less, int(bsb.config.BufferSize)) ledgerBuffer := &ledgerBuffer{ @@ -167,7 +173,7 @@ func (lb *ledgerBuffer) downloadLedgerObject(ctx context.Context, sequence uint3 reader, err := lb.dataStore.GetFile(ctx, objectKey) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "unable to retrieve file: %s", objectKey) } defer reader.Close() diff --git a/ingest/ledgerbackend/range.go b/ingest/ledgerbackend/range.go index f0c80695a1..99b4dfc800 100644 --- a/ingest/ledgerbackend/range.go +++ b/ingest/ledgerbackend/range.go @@ -46,6 +46,18 @@ func (r Range) String() string { return fmt.Sprintf("[%d,latest)", r.from) } +func (r Range) Bounded() bool { + return r.bounded +} + +func (r Range) To() uint32 { + return r.to +} + +func (r Range) From() uint32 { + return r.from +} + func (r Range) Contains(other Range) bool { if r.bounded && !other.bounded { return false diff --git a/ingest/ledgerbackend/run_from.go b/ingest/ledgerbackend/run_from.go index 2d02322519..8a424da105 100644 --- a/ingest/ledgerbackend/run_from.go +++ b/ingest/ledgerbackend/run_from.go @@ -6,32 +6,36 @@ import ( "fmt" "runtime" + "github.com/prometheus/client_golang/prometheus" + "github.com/stellar/go/protocols/stellarcore" "github.com/stellar/go/support/log" ) type runFromStream struct { - dir workingDir - from uint32 - hash string - coreCmdFactory coreCmdFactory - log *log.Entry - useDB bool + dir workingDir + from uint32 + hash string + coreCmdFactory coreCmdFactory + log *log.Entry + useDB bool + captiveCoreNewDBCounter prometheus.Counter } -func newRunFromStream(r *stellarCoreRunner, from uint32, hash string) runFromStream { +func newRunFromStream(r *stellarCoreRunner, from uint32, hash string, captiveCoreNewDBCounter prometheus.Counter) runFromStream { // We only use ephemeral directories on windows because there is // no way to terminate captive core gracefully on windows. // Having an ephemeral directory ensures that it is wiped out // whenever we terminate captive core dir := newWorkingDir(r, runtime.GOOS == "windows") return runFromStream{ - dir: dir, - from: from, - hash: hash, - coreCmdFactory: newCoreCmdFactory(r, dir), - log: r.log, - useDB: r.useDB, + dir: dir, + from: from, + hash: hash, + coreCmdFactory: newCoreCmdFactory(r, dir), + log: r.log, + useDB: r.useDB, + captiveCoreNewDBCounter: captiveCoreNewDBCounter, } } @@ -79,6 +83,9 @@ func (s runFromStream) start(ctx context.Context) (cmd cmdI, captiveCorePipe pip } if createNewDB { + if s.captiveCoreNewDBCounter != nil { + s.captiveCoreNewDBCounter.Inc() + } if err = s.dir.remove(); err != nil { return nil, pipe{}, fmt.Errorf("error removing existing storage-dir contents: %w", err) } diff --git a/ingest/ledgerbackend/stellar_core_runner.go b/ingest/ledgerbackend/stellar_core_runner.go index 5245051dce..4f95e94f45 100644 --- a/ingest/ledgerbackend/stellar_core_runner.go +++ b/ingest/ledgerbackend/stellar_core_runner.go @@ -7,6 +7,8 @@ import ( "math/rand" "sync" + "github.com/prometheus/client_golang/prometheus" + "github.com/stellar/go/support/log" ) @@ -67,6 +69,8 @@ type stellarCoreRunner struct { toml *CaptiveCoreToml useDB bool + captiveCoreNewDBCounter prometheus.Counter + log *log.Entry } @@ -79,7 +83,7 @@ func createRandomHexString(n int) string { return string(b) } -func newStellarCoreRunner(config CaptiveCoreConfig) *stellarCoreRunner { +func newStellarCoreRunner(config CaptiveCoreConfig, captiveCoreNewDBCounter prometheus.Counter) *stellarCoreRunner { ctx, cancel := context.WithCancel(config.Context) runner := &stellarCoreRunner{ @@ -91,7 +95,8 @@ func newStellarCoreRunner(config CaptiveCoreConfig) *stellarCoreRunner { log: config.Log, toml: config.Toml, - systemCaller: realSystemCaller{}, + captiveCoreNewDBCounter: captiveCoreNewDBCounter, + systemCaller: realSystemCaller{}, } return runner @@ -104,7 +109,7 @@ func (r *stellarCoreRunner) context() context.Context { // runFrom executes the run command with a starting ledger on the captive core subprocess func (r *stellarCoreRunner) runFrom(from uint32, hash string) error { - return r.startMetaStream(newRunFromStream(r, from, hash)) + return r.startMetaStream(newRunFromStream(r, from, hash, r.captiveCoreNewDBCounter)) } // catchup executes the catchup command on the captive core subprocess diff --git a/ingest/ledgerbackend/stellar_core_runner_test.go b/ingest/ledgerbackend/stellar_core_runner_test.go index f53cd88328..06b9c85fce 100644 --- a/ingest/ledgerbackend/stellar_core_runner_test.go +++ b/ingest/ledgerbackend/stellar_core_runner_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -26,7 +28,7 @@ func TestCloseOffline(t *testing.T) { Context: context.Background(), Toml: captiveCoreToml, StoragePath: "/tmp/captive-core", - }) + }, nil) cmdMock := simpleCommandMock() cmdMock.On("Wait").Return(nil) @@ -68,7 +70,7 @@ func TestCloseOnline(t *testing.T) { Context: context.Background(), Toml: captiveCoreToml, StoragePath: "/tmp/captive-core", - }) + }, nil) cmdMock := simpleCommandMock() cmdMock.On("Wait").Return(nil) @@ -112,7 +114,7 @@ func TestCloseOnlineWithError(t *testing.T) { Context: context.Background(), Toml: captiveCoreToml, StoragePath: "/tmp/captive-core", - }) + }, nil) cmdMock := simpleCommandMock() cmdMock.On("Wait").Return(errors.New("wait error")) @@ -166,7 +168,7 @@ func TestCloseConcurrency(t *testing.T) { Context: context.Background(), Toml: captiveCoreToml, StoragePath: "/tmp/captive-core", - }) + }, nil) cmdMock := simpleCommandMock() cmdMock.On("Wait").Return(errors.New("wait error")).WaitUntil(time.After(time.Millisecond * 300)) @@ -223,7 +225,7 @@ func TestRunFromUseDBLedgersMatch(t *testing.T) { Toml: captiveCoreToml, StoragePath: "/tmp/captive-core", UseDB: true, - }) + }, createNewDBCounter()) cmdMock := simpleCommandMock() cmdMock.On("Wait").Return(nil) @@ -263,6 +265,8 @@ func TestRunFromUseDBLedgersMatch(t *testing.T) { assert.NoError(t, runner.runFrom(100, "hash")) assert.NoError(t, runner.close()) + + assert.Equal(t, float64(0), getNewDBCounterMetric(runner)) } func TestRunFromUseDBLedgersBehind(t *testing.T) { @@ -279,7 +283,7 @@ func TestRunFromUseDBLedgersBehind(t *testing.T) { Toml: captiveCoreToml, StoragePath: "/tmp/captive-core", UseDB: true, - }) + }, createNewDBCounter()) newDBCmdMock := simpleCommandMock() newDBCmdMock.On("Run").Return(nil) @@ -325,6 +329,23 @@ func TestRunFromUseDBLedgersBehind(t *testing.T) { assert.NoError(t, runner.runFrom(100, "hash")) assert.NoError(t, runner.close()) + + assert.Equal(t, float64(0), getNewDBCounterMetric(runner)) +} + +func createNewDBCounter() prometheus.Counter { + return prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "test", Subsystem: "captive_core", Name: "new_db_counter", + }) +} + +func getNewDBCounterMetric(runner *stellarCoreRunner) float64 { + value := &dto.Metric{} + err := runner.captiveCoreNewDBCounter.Write(value) + if err != nil { + panic(err) + } + return value.GetCounter().GetValue() } func TestRunFromUseDBLedgersInFront(t *testing.T) { @@ -341,7 +362,7 @@ func TestRunFromUseDBLedgersInFront(t *testing.T) { Toml: captiveCoreToml, StoragePath: "/tmp/captive-core", UseDB: true, - }) + }, createNewDBCounter()) newDBCmdMock := simpleCommandMock() newDBCmdMock.On("Run").Return(nil) @@ -405,4 +426,5 @@ func TestRunFromUseDBLedgersInFront(t *testing.T) { assert.NoError(t, runner.runFrom(100, "hash")) assert.NoError(t, runner.close()) + assert.Equal(t, float64(1), getNewDBCounterMetric(runner)) } diff --git a/services/galexie/architecture.png b/services/galexie/architecture.png index 85bd6d8b31..c19a313ebe 100644 Binary files a/services/galexie/architecture.png and b/services/galexie/architecture.png differ diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index a444771ca6..65389d580f 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -2,23 +2,34 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). - ## Pending ### Breaking Changes +- `--parallel-job-size` configuration parameter for the `stellar-horizon db reingest` command has been removed. + Job size will be automatically determined based on the number of workers (configuration parameter --parallel-workers), distributing + the range equally among them. The minimum job size will remain 64 ledgers and the start and end ledger range will be rounded to + the nearest checkpoint.([5484](https://github.com/stellar/go/pull/5484)) + - Removed `num_accounts` and `amount` fields from Asset stats related endpoints, and `valid_before` and `valid_after` fields from transaction related endpoints. Issue - [5438](https://github.com/stellar/go/issues/5438), PR - [5478](https://github.com/stellar/go/pull/5478) - These fields have already been functionally deprecated as of release v2.1.0. As a part of this release, these fields are omitted from API Response - Additionally, the `num_accounts` and `amount` columns have been dropped from `exp_asset_stats` table in Postgres. +## 2.32.0 + ### Added - Reingest from pre-computed tx meta on remote cloud storage. ([4911](https://github.com/stellar/go/issues/4911)), ([5374](https://github.com/stellar/go/pull/5374)) - Configure horizon reingestion to obtain ledger tx meta in pre-computed files from a Google Cloud Storage(GCS) location. - Using this option will no longer require a captive core binary be present and it no longer runs a captive core sub-process, instead obtaining the tx meta from the GCS backend. - Horizon supports this new feature with two new parameters `ledgerbackend` and `datastore-config` on the `reingest` command. Refer to [Reingestion README](./internal/ingest/README.md#reingestion). +- Add metrics for reaping of history lookup tables ([5385](https://github.com/stellar/go/pull/5385)). +- Add `--reap-lookup-tables` (defaults to true) which will disable reaping of history lookup tables when set to false. ([5366](https://github.com/stellar/go/pull/5366)). +### Fixed +- Fix ingestion duration metric so it includes time spent reaping history lookup tables ([5361](https://github.com/stellar/go/pull/5361)). +- Optimize query for reaping history lookup tables ([5393](https://github.com/stellar/go/pull/5393)). ## 2.31.0 diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index 92a732e002..58c096a782 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -42,7 +42,6 @@ var ( dbDetectGapsCmd *cobra.Command reingestForce bool parallelWorkers uint - parallelJobSize uint32 retries uint retryBackoffSeconds uint ledgerBackendStr string @@ -118,14 +117,6 @@ func ingestRangeCmdOpts() support.ConfigOptions { FlagDefault: uint(1), Usage: "[optional] if this flag is set to > 1, horizon will parallelize reingestion using the supplied number of workers", }, - { - Name: "parallel-job-size", - ConfigKey: ¶llelJobSize, - OptType: types.Uint32, - Required: false, - FlagDefault: uint32(100000), - Usage: "[optional] parallel workers will run jobs processing ledger batches of the supplied size", - }, { Name: "retries", ConfigKey: &retries, @@ -178,7 +169,7 @@ func ingestRangeCmdOpts() support.ConfigOptions { var dbReingestRangeCmdOpts = ingestRangeCmdOpts() var dbFillGapsCmdOpts = ingestRangeCmdOpts() -func runDBReingestRange(ledgerRanges []history.LedgerRange, reingestForce bool, parallelWorkers uint, config horizon.Config, storageBackendConfig ingest.StorageBackendConfig) error { +func runDBReingestRange(ledgerRanges []history.LedgerRange, reingestForce bool, parallelWorkers uint, minBatchSize, maxBatchSize uint, config horizon.Config, storageBackendConfig ingest.StorageBackendConfig) error { var err error if reingestForce && parallelWorkers > 1 { @@ -186,9 +177,6 @@ func runDBReingestRange(ledgerRanges []history.LedgerRange, reingestForce bool, } maxLedgersPerFlush := ingest.MaxLedgersPerFlush - if parallelJobSize < maxLedgersPerFlush { - maxLedgersPerFlush = parallelJobSize - } ingestConfig := ingest.Config{ NetworkPassphrase: config.NetworkPassphrase, @@ -214,15 +202,12 @@ func runDBReingestRange(ledgerRanges []history.LedgerRange, reingestForce bool, } if parallelWorkers > 1 { - system, systemErr := ingest.NewParallelSystems(ingestConfig, parallelWorkers) + system, systemErr := ingest.NewParallelSystems(ingestConfig, parallelWorkers, minBatchSize, maxBatchSize) if systemErr != nil { return systemErr } - return system.ReingestRange( - ledgerRanges, - parallelJobSize, - ) + return system.ReingestRange(ledgerRanges) } system, systemErr := ingest.NewSystem(ingestConfig) @@ -479,6 +464,7 @@ func DefineDBCommands(rootCmd *cobra.Command, horizonConfig *horizon.Config, hor } } + maxBatchSize := ingest.MaxCaptiveCoreBackendBatchSize var err error var storageBackendConfig ingest.StorageBackendConfig options := horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false} @@ -486,12 +472,8 @@ func DefineDBCommands(rootCmd *cobra.Command, horizonConfig *horizon.Config, hor if storageBackendConfig, err = loadStorageBackendConfig(storageBackendConfigPath); err != nil { return err } - // when using buffered storage, performance observations have noted optimal parallel batch size - // of 100, apply that as default if the flag was absent. - if !viper.IsSet("parallel-job-size") { - parallelJobSize = 100 - } options.NoCaptiveCore = true + maxBatchSize = ingest.MaxBufferedStorageBackendBatchSize } if err = horizon.ApplyFlags(horizonConfig, horizonFlags, options); err != nil { @@ -501,6 +483,8 @@ func DefineDBCommands(rootCmd *cobra.Command, horizonConfig *horizon.Config, hor []history.LedgerRange{{StartSequence: argsUInt32[0], EndSequence: argsUInt32[1]}}, reingestForce, parallelWorkers, + ingest.MinBatchSize, + maxBatchSize, *horizonConfig, storageBackendConfig, ) @@ -541,6 +525,7 @@ func DefineDBCommands(rootCmd *cobra.Command, horizonConfig *horizon.Config, hor withRange = true } + maxBatchSize := ingest.MaxCaptiveCoreBackendBatchSize var err error var storageBackendConfig ingest.StorageBackendConfig options := horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false} @@ -549,6 +534,7 @@ func DefineDBCommands(rootCmd *cobra.Command, horizonConfig *horizon.Config, hor return err } options.NoCaptiveCore = true + maxBatchSize = ingest.MaxBufferedStorageBackendBatchSize } if err = horizon.ApplyFlags(horizonConfig, horizonFlags, options); err != nil { @@ -569,7 +555,7 @@ func DefineDBCommands(rootCmd *cobra.Command, horizonConfig *horizon.Config, hor hlog.Infof("found gaps %v", gaps) } - return runDBReingestRangeFn(gaps, reingestForce, parallelWorkers, *horizonConfig, storageBackendConfig) + return runDBReingestRangeFn(gaps, reingestForce, parallelWorkers, ingest.MinBatchSize, maxBatchSize, *horizonConfig, storageBackendConfig) }, } diff --git a/services/horizon/cmd/db_test.go b/services/horizon/cmd/db_test.go index 6a00576bd3..a2dd5f014c 100644 --- a/services/horizon/cmd/db_test.go +++ b/services/horizon/cmd/db_test.go @@ -25,7 +25,7 @@ type DBCommandsTestSuite struct { } func (s *DBCommandsTestSuite) SetupSuite() { - runDBReingestRangeFn = func([]history.LedgerRange, bool, uint, + runDBReingestRangeFn = func([]history.LedgerRange, bool, uint, uint, uint, horizon.Config, ingest.StorageBackendConfig) error { return nil } @@ -45,66 +45,19 @@ func (s *DBCommandsTestSuite) BeforeTest(suiteName string, testName string) { s.rootCmd = NewRootCmd() } -func (s *DBCommandsTestSuite) TestDefaultParallelJobSizeForBufferedBackend() { +func (s *DBCommandsTestSuite) TestInvalidParameterParallelJobSize() { s.rootCmd.SetArgs([]string{ "db", "reingest", "range", "--db-url", s.db.DSN, "--network", "testnet", "--parallel-workers", "2", + "--parallel-job-size", "10", "--ledgerbackend", "datastore", "--datastore-config", "../internal/ingest/testdata/config.storagebackend.toml", "2", "10"}) - require.NoError(s.T(), s.rootCmd.Execute()) - require.Equal(s.T(), parallelJobSize, uint32(100)) -} - -func (s *DBCommandsTestSuite) TestDefaultParallelJobSizeForCaptiveBackend() { - s.rootCmd.SetArgs([]string{ - "db", "reingest", "range", - "--db-url", s.db.DSN, - "--network", "testnet", - "--stellar-core-binary-path", "/test/core/bin/path", - "--parallel-workers", "2", - "--ledgerbackend", "captive-core", - "2", - "10"}) - - require.NoError(s.T(), s.rootCmd.Execute()) - require.Equal(s.T(), parallelJobSize, uint32(100_000)) -} - -func (s *DBCommandsTestSuite) TestUsesParallelJobSizeWhenSetForCaptive() { - s.rootCmd.SetArgs([]string{ - "db", "reingest", "range", - "--db-url", s.db.DSN, - "--network", "testnet", - "--stellar-core-binary-path", "/test/core/bin/path", - "--parallel-workers", "2", - "--parallel-job-size", "5", - "--ledgerbackend", "captive-core", - "2", - "10"}) - - require.NoError(s.T(), s.rootCmd.Execute()) - require.Equal(s.T(), parallelJobSize, uint32(5)) -} - -func (s *DBCommandsTestSuite) TestUsesParallelJobSizeWhenSetForBuffered() { - s.rootCmd.SetArgs([]string{ - "db", "reingest", "range", - "--db-url", s.db.DSN, - "--network", "testnet", - "--parallel-workers", "2", - "--parallel-job-size", "5", - "--ledgerbackend", "datastore", - "--datastore-config", "../internal/ingest/testdata/config.storagebackend.toml", - "2", - "10"}) - - require.NoError(s.T(), s.rootCmd.Execute()) - require.Equal(s.T(), parallelJobSize, uint32(5)) + require.Equal(s.T(), "unknown flag: --parallel-job-size", s.rootCmd.Execute().Error()) } func (s *DBCommandsTestSuite) TestDbReingestAndFillGapsCmds() { diff --git a/services/horizon/docker/Dockerfile b/services/horizon/docker/Dockerfile index 3c090d203c..2d5c7a01a2 100644 --- a/services/horizon/docker/Dockerfile +++ b/services/horizon/docker/Dockerfile @@ -1,3 +1,4 @@ +# install Core from apt "stable" or "unstable" pool, and horizon from apt "testing" pool FROM ubuntu:focal ARG VERSION diff --git a/services/horizon/docker/Dockerfile.core-testing b/services/horizon/docker/Dockerfile.core-testing new file mode 100644 index 0000000000..c0d1dff8e6 --- /dev/null +++ b/services/horizon/docker/Dockerfile.core-testing @@ -0,0 +1,19 @@ +# install both horizon and core from apt "testing" pool +FROM ubuntu:focal + +ARG VERSION +ARG STELLAR_CORE_VERSION +ARG DEBIAN_FRONTEND=noninteractive +ARG ALLOW_CORE_UNSTABLE=no + +RUN apt-get update && apt-get install -y wget apt-transport-https gnupg2 && \ + wget -qO /etc/apt/trusted.gpg.d/SDF.asc https://apt.stellar.org/SDF.asc && \ + echo "deb https://apt.stellar.org focal testing" | tee -a /etc/apt/sources.list.d/SDF.list && \ + cat /etc/apt/sources.list.d/SDF.list && \ + apt-get update && \ + apt-cache madison stellar-core && eval "apt-get install -y stellar-core${STELLAR_CORE_VERSION+=$STELLAR_CORE_VERSION}" && \ + apt-cache madison stellar-horizon && apt-get install -y stellar-horizon=${VERSION} && \ + apt-get clean && rm -rf /var/lib/apt/lists/* /var/log/*.log /var/log/*/*.log + +EXPOSE 8000 +ENTRYPOINT ["/usr/bin/stellar-horizon"] diff --git a/services/horizon/docker/Dockerfile.stable b/services/horizon/docker/Dockerfile.stable new file mode 100644 index 0000000000..63f268873a --- /dev/null +++ b/services/horizon/docker/Dockerfile.stable @@ -0,0 +1,18 @@ +# install Core from apt "stable"and horizon from apt "stable" pool +FROM ubuntu:focal + +ARG VERSION +ARG STELLAR_CORE_VERSION +ARG DEBIAN_FRONTEND=noninteractive +ARG ALLOW_CORE_UNSTABLE=no + +RUN apt-get update && apt-get install -y wget apt-transport-https gnupg2 && \ + wget -qO /etc/apt/trusted.gpg.d/SDF.asc https://apt.stellar.org/SDF.asc && \ + echo "deb https://apt.stellar.org focal stable" | tee -a /etc/apt/sources.list.d/SDF.list && \ + cat /etc/apt/sources.list.d/SDF.list && \ + apt-get update && apt-cache madison stellar-core && eval "apt-get install -y stellar-core${STELLAR_CORE_VERSION+=$STELLAR_CORE_VERSION}" && \ + apt-cache madison stellar-horizon && apt-get install -y stellar-horizon=${VERSION} && \ + apt-get clean && rm -rf /var/lib/apt/lists/* /var/log/*.log /var/log/*/*.log + +EXPOSE 8000 +ENTRYPOINT ["/usr/bin/stellar-horizon"] diff --git a/services/horizon/docker/Makefile b/services/horizon/docker/Makefile index 51ee19f2fe..26d3c892d0 100644 --- a/services/horizon/docker/Makefile +++ b/services/horizon/docker/Makefile @@ -5,6 +5,7 @@ BUILD_DATE := $(shell date -u +%FT%TZ) TAG ?= stellar/stellar-horizon:$(VERSION) +# build with Core from apt "stable" or "unstable", and horizon from apt "testing" docker-build: ifndef VERSION $(error VERSION environment variable must be set. For example VERSION=2.4.1-101 ) @@ -22,6 +23,34 @@ else -t $(TAG) . endif +# build Core and Horizon from apt "testing" +docker-build-core-testing: +ifndef VERSION + $(error VERSION environment variable must be set. For example VERSION=2.4.1-101 ) +endif +ifndef STELLAR_CORE_VERSION + $(error STELLAR_CORE_VERSION environment variable must be set. ) +else + $(SUDO) docker build --file ./Dockerfile.core-testing --pull $(DOCKER_OPTS) \ + --label org.opencontainers.image.created="$(BUILD_DATE)" \ + --build-arg VERSION=$(VERSION) --build-arg STELLAR_CORE_VERSION=$(STELLAR_CORE_VERSION) \ + -t $(TAG) . +endif + +# build Core and Horizon from apt "stable" +docker-build-core-stable: +ifndef VERSION + $(error VERSION environment variable must be set. For example VERSION=2.4.1-101 ) +endif +ifndef STELLAR_CORE_VERSION + $(error STELLAR_CORE_VERSION environment variable must be set. ) +else + $(SUDO) docker build --file ./Dockerfile.stable --pull $(DOCKER_OPTS) \ + --label org.opencontainers.image.created="$(BUILD_DATE)" \ + --build-arg VERSION=$(VERSION) --build-arg STELLAR_CORE_VERSION=$(STELLAR_CORE_VERSION) \ + -t $(TAG) . +endif + docker-push: ifndef TAG $(error Must set VERSION or TAG environment variable. For example VERSION=2.4.1-101 ) diff --git a/services/horizon/internal/actions/effects.go b/services/horizon/internal/actions/effects.go index e04997aca5..aab7fcb699 100644 --- a/services/horizon/internal/actions/effects.go +++ b/services/horizon/internal/actions/effects.go @@ -51,12 +51,12 @@ type GetEffectsHandler struct { } func (handler GetEffectsHandler) GetResourcePage(w HeaderWriter, r *http.Request) ([]hal.Pageable, error) { - pq, err := GetPageQuery(handler.LedgerState, r, DefaultTOID) + pq, err := GetPageQuery(handler.LedgerState, r) if err != nil { return nil, err } - err = validateCursorWithinHistory(handler.LedgerState, pq) + err = validateAndAdjustCursor(handler.LedgerState, &pq) if err != nil { return nil, err } @@ -72,7 +72,7 @@ func (handler GetEffectsHandler) GetResourcePage(w HeaderWriter, r *http.Request return nil, err } - records, err := loadEffectRecords(r.Context(), historyQ, qp, pq) + records, err := loadEffectRecords(r.Context(), historyQ, qp, pq, handler.LedgerState.CurrentStatus().HistoryElder) if err != nil { return nil, errors.Wrap(err, "loading transaction records") } @@ -94,12 +94,12 @@ func (handler GetEffectsHandler) GetResourcePage(w HeaderWriter, r *http.Request return result, nil } -func loadEffectRecords(ctx context.Context, hq *history.Q, qp EffectsQuery, pq db2.PageQuery) ([]history.Effect, error) { +func loadEffectRecords(ctx context.Context, hq *history.Q, qp EffectsQuery, pq db2.PageQuery, oldestLedger int32) ([]history.Effect, error) { switch { case qp.AccountID != "": - return hq.EffectsForAccount(ctx, qp.AccountID, pq) + return hq.EffectsForAccount(ctx, qp.AccountID, pq, oldestLedger) case qp.LiquidityPoolID != "": - return hq.EffectsForLiquidityPool(ctx, qp.LiquidityPoolID, pq) + return hq.EffectsForLiquidityPool(ctx, qp.LiquidityPoolID, pq, oldestLedger) case qp.OperationID > 0: return hq.EffectsForOperation(ctx, int64(qp.OperationID), pq) case qp.LedgerID > 0: @@ -107,7 +107,7 @@ func loadEffectRecords(ctx context.Context, hq *history.Q, qp EffectsQuery, pq d case qp.TxHash != "": return hq.EffectsForTransaction(ctx, qp.TxHash, pq) default: - return hq.Effects(ctx, pq) + return hq.Effects(ctx, pq, oldestLedger) } } diff --git a/services/horizon/internal/actions/helpers.go b/services/horizon/internal/actions/helpers.go index 2575b13251..99071bfba7 100644 --- a/services/horizon/internal/actions/helpers.go +++ b/services/horizon/internal/actions/helpers.go @@ -3,6 +3,7 @@ package actions import ( "context" "encoding/hex" + "errors" "fmt" "net/http" "net/url" @@ -20,7 +21,6 @@ import ( "github.com/stellar/go/services/horizon/internal/db2" "github.com/stellar/go/services/horizon/internal/ledger" hProblem "github.com/stellar/go/services/horizon/internal/render/problem" - "github.com/stellar/go/support/errors" "github.com/stellar/go/support/ordered" "github.com/stellar/go/support/render/problem" "github.com/stellar/go/toid" @@ -45,8 +45,6 @@ type Opt int const ( // DisableCursorValidation disables cursor validation in GetPageQuery DisableCursorValidation Opt = iota - // DefaultTOID sets a default cursor value in GetPageQuery based on the ledger state - DefaultTOID Opt = iota ) // HeaderWriter is an interface for setting HTTP response headers @@ -171,7 +169,7 @@ func getLimit(r *http.Request, name string, def uint64, max uint64) (uint64, err if asI64 <= 0 { err = errors.New("invalid limit: non-positive value provided") } else if asI64 > int64(max) { - err = errors.Errorf("invalid limit: value provided that is over limit max of %d", max) + err = fmt.Errorf("invalid limit: value provided that is over limit max of %d", max) } if err != nil { @@ -185,14 +183,10 @@ func getLimit(r *http.Request, name string, def uint64, max uint64) (uint64, err // using the results from a call to GetPagingParams() func GetPageQuery(ledgerState *ledger.State, r *http.Request, opts ...Opt) (db2.PageQuery, error) { disableCursorValidation := false - defaultTOID := false for _, opt := range opts { if opt == DisableCursorValidation { disableCursorValidation = true } - if opt == DefaultTOID { - defaultTOID = true - } } cursor, err := getCursor(ledgerState, r, ParamCursor) @@ -221,13 +215,6 @@ func GetPageQuery(ledgerState *ledger.State, r *http.Request, opts ...Opt) (db2. return db2.PageQuery{}, err } - if cursor == "" && defaultTOID { - if pageQuery.Order == db2.OrderAscending { - pageQuery.Cursor = toid.AfterLedger( - ordered.Max(0, ledgerState.CurrentStatus().HistoryElder-1), - ).String() - } - } return pageQuery, nil } @@ -553,19 +540,37 @@ func validateAssetParams(aType, code, issuer, prefix string) error { return nil } -// validateCursorWithinHistory compares the requested page of data against the +// validateAndAdjustCursor compares the requested page of data against the // ledger state of the history database. In the event that the cursor is // guaranteed to return no results, we return a 410 GONE http response. -func validateCursorWithinHistory(ledgerState *ledger.State, pq db2.PageQuery) error { - // an ascending query should never return a gone response: An ascending query - // prior to known history should return results at the beginning of history, - // and an ascending query beyond the end of history should not error out but - // rather return an empty page (allowing code that tracks the procession of - // some resource more easily). - if pq.Order != "desc" { - return nil +// For ascending queries, we adjust the cursor to ensure it starts at +// the oldest available ledger. +func validateAndAdjustCursor(ledgerState *ledger.State, pq *db2.PageQuery) error { + err := validateCursorWithinHistory(ledgerState, *pq) + + if pq.Order == db2.OrderAscending { + // an ascending query should never return a gone response: An ascending query + // prior to known history should return results at the beginning of history, + // and an ascending query beyond the end of history should not error out but + // rather return an empty page (allowing code that tracks the procession of + // some resource more easily). + + // set/modify the cursor for ascending queries to start at the oldest available ledger if it + // precedes the oldest ledger. This avoids inefficient queries caused by index bloat from deleted rows + // that are removed as part of reaping to maintain the retention window. + if pq.Cursor == "" || errors.Is(err, &hProblem.BeforeHistory) { + pq.Cursor = toid.AfterLedger( + ordered.Max(0, ledgerState.CurrentStatus().HistoryElder-1), + ).String() + return nil + } } + return err +} +// validateCursorWithinHistory checks if the cursor is within the known history range. +// If the cursor is before the oldest available ledger, it returns BeforeHistory error. +func validateCursorWithinHistory(ledgerState *ledger.State, pq db2.PageQuery) error { var cursor int64 var err error @@ -596,7 +601,7 @@ func countNonEmpty(params ...interface{}) (int, error) { for _, param := range params { switch param := param.(type) { default: - return 0, errors.Errorf("unexpected type %T", param) + return 0, fmt.Errorf("unexpected type %T", param) case int32: if param != 0 { count++ diff --git a/services/horizon/internal/actions/helpers_test.go b/services/horizon/internal/actions/helpers_test.go index 05e840577e..e393cc357e 100644 --- a/services/horizon/internal/actions/helpers_test.go +++ b/services/horizon/internal/actions/helpers_test.go @@ -16,6 +16,7 @@ import ( horizonContext "github.com/stellar/go/services/horizon/internal/context" "github.com/stellar/go/services/horizon/internal/db2" "github.com/stellar/go/services/horizon/internal/ledger" + hProblem "github.com/stellar/go/services/horizon/internal/render/problem" "github.com/stellar/go/services/horizon/internal/test" "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" @@ -126,11 +127,21 @@ func TestValidateCursorWithinHistory(t *testing.T) { { cursor: "0", order: "asc", - valid: true, + valid: false, }, { cursor: "0-1234", order: "asc", + valid: false, + }, + { + cursor: "1", + order: "asc", + valid: true, + }, + { + cursor: "1-1234", + order: "asc", valid: true, }, } @@ -291,11 +302,10 @@ func TestGetPageQuery(t *testing.T) { tt.Assert.Error(err) } -func TestGetPageQueryCursorDefaultTOID(t *testing.T) { - ascReq := makeTestActionRequest("/foo-bar/blah?limit=2", testURLParams()) - descReq := makeTestActionRequest("/foo-bar/blah?limit=2&order=desc", testURLParams()) - +func TestPageQueryCursorDefaultOrder(t *testing.T) { ledgerState := &ledger.State{} + + // truncated history ledgerState.SetHorizonStatus(ledger.HorizonStatus{ HistoryLatest: 7000, HistoryLatestClosedAt: time.Now(), @@ -303,30 +313,30 @@ func TestGetPageQueryCursorDefaultTOID(t *testing.T) { ExpHistoryLatest: 7000, }) - pq, err := GetPageQuery(ledgerState, ascReq, DefaultTOID) + req := makeTestActionRequest("/foo-bar/blah?limit=2", testURLParams()) + + // default asc, w/o cursor + pq, err := GetPageQuery(ledgerState, req) assert.NoError(t, err) + assert.Empty(t, pq.Cursor) + assert.NoError(t, validateAndAdjustCursor(ledgerState, &pq)) assert.Equal(t, toid.AfterLedger(299).String(), pq.Cursor) assert.Equal(t, uint64(2), pq.Limit) assert.Equal(t, "asc", pq.Order) - pq, err = GetPageQuery(ledgerState, descReq, DefaultTOID) - assert.NoError(t, err) - assert.Equal(t, "", pq.Cursor) - assert.Equal(t, uint64(2), pq.Limit) - assert.Equal(t, "desc", pq.Order) + cursor := toid.AfterLedger(200).String() + reqWithCursor := makeTestActionRequest(fmt.Sprintf("/foo-bar/blah?cursor=%s&limit=2", cursor), testURLParams()) - pq, err = GetPageQuery(ledgerState, ascReq) + // default asc, w/ cursor + pq, err = GetPageQuery(ledgerState, reqWithCursor) assert.NoError(t, err) - assert.Empty(t, pq.Cursor) + assert.Equal(t, cursor, pq.Cursor) + assert.NoError(t, validateAndAdjustCursor(ledgerState, &pq)) + assert.Equal(t, toid.AfterLedger(299).String(), pq.Cursor) assert.Equal(t, uint64(2), pq.Limit) assert.Equal(t, "asc", pq.Order) - pq, err = GetPageQuery(ledgerState, descReq) - assert.NoError(t, err) - assert.Empty(t, pq.Cursor) - assert.Equal(t, "", pq.Cursor) - assert.Equal(t, "desc", pq.Order) - + // full history ledgerState.SetHorizonStatus(ledger.HorizonStatus{ HistoryLatest: 7000, HistoryLatestClosedAt: time.Now(), @@ -334,18 +344,130 @@ func TestGetPageQueryCursorDefaultTOID(t *testing.T) { ExpHistoryLatest: 7000, }) - pq, err = GetPageQuery(ledgerState, ascReq, DefaultTOID) + // default asc, w/o cursor + pq, err = GetPageQuery(ledgerState, req) assert.NoError(t, err) + assert.Empty(t, pq.Cursor) + assert.NoError(t, validateAndAdjustCursor(ledgerState, &pq)) assert.Equal(t, toid.AfterLedger(0).String(), pq.Cursor) assert.Equal(t, uint64(2), pq.Limit) assert.Equal(t, "asc", pq.Order) - pq, err = GetPageQuery(ledgerState, descReq, DefaultTOID) + // default asc, w/ cursor + pq, err = GetPageQuery(ledgerState, reqWithCursor) assert.NoError(t, err) - assert.Equal(t, "", pq.Cursor) + assert.Equal(t, cursor, pq.Cursor) + assert.NoError(t, validateAndAdjustCursor(ledgerState, &pq)) + assert.Equal(t, cursor, pq.Cursor) assert.Equal(t, uint64(2), pq.Limit) - assert.Equal(t, "desc", pq.Order) + assert.Equal(t, "asc", pq.Order) + +} + +func TestGetPageQueryWithoutCursor(t *testing.T) { + ledgerState := &ledger.State{} + + validateCursor := func(limit uint64, order string, expectedCursor string) { + req := makeTestActionRequest(fmt.Sprintf("/foo-bar/blah?limit=%d&order=%s", limit, order), testURLParams()) + pq, err := GetPageQuery(ledgerState, req) + assert.NoError(t, err) + assert.Empty(t, pq.Cursor) + assert.NoError(t, validateAndAdjustCursor(ledgerState, &pq)) + assert.Equal(t, expectedCursor, pq.Cursor) + assert.Equal(t, limit, pq.Limit) + assert.Equal(t, order, pq.Order) + } + + // truncated history + ledgerState.SetHorizonStatus(ledger.HorizonStatus{ + HistoryLatest: 7000, + HistoryLatestClosedAt: time.Now(), + HistoryElder: 300, + ExpHistoryLatest: 7000, + }) + + validateCursor(2, "asc", toid.AfterLedger(299).String()) + validateCursor(2, "desc", "") + + // full history + ledgerState.SetHorizonStatus(ledger.HorizonStatus{ + HistoryLatest: 7000, + HistoryLatestClosedAt: time.Now(), + HistoryElder: 0, + ExpHistoryLatest: 7000, + }) + + validateCursor(2, "asc", toid.AfterLedger(0).String()) + validateCursor(2, "desc", "") +} + +func TestValidateAndAdjustCursor(t *testing.T) { + ledgerState := &ledger.State{} + + validateCursor := func(cursor string, limit uint64, order string, expectedCursor string, expectedError error) { + pq := db2.PageQuery{Cursor: cursor, + Limit: limit, + Order: order, + } + err := validateAndAdjustCursor(ledgerState, &pq) + if expectedError != nil { + assert.EqualError(t, expectedError, err.Error()) + } else { + assert.NoError(t, err) + } + assert.Equal(t, expectedCursor, pq.Cursor) + assert.Equal(t, limit, pq.Limit) + assert.Equal(t, order, pq.Order) + } + + // full history + ledgerState.SetHorizonStatus(ledger.HorizonStatus{ + HistoryLatest: 7000, + HistoryLatestClosedAt: time.Now(), + HistoryElder: 0, + ExpHistoryLatest: 7000, + }) + + // invalid cursor + validateCursor("blah", 2, "asc", "blah", problem.BadRequest) + validateCursor("blah", 2, "desc", "blah", problem.BadRequest) + + validateCursor(toid.AfterLedger(0).String(), 2, "asc", toid.AfterLedger(0).String(), nil) + validateCursor(toid.AfterLedger(200).String(), 2, "asc", toid.AfterLedger(200).String(), nil) + validateCursor(toid.AfterLedger(7001).String(), 2, "asc", toid.AfterLedger(7001).String(), nil) + + validateCursor(toid.AfterLedger(0).String(), 2, "desc", toid.AfterLedger(0).String(), nil) + validateCursor(toid.AfterLedger(200).String(), 2, "desc", toid.AfterLedger(200).String(), nil) + validateCursor(toid.AfterLedger(7001).String(), 2, "desc", toid.AfterLedger(7001).String(), nil) + + // truncated history + ledgerState.SetHorizonStatus(ledger.HorizonStatus{ + HistoryLatest: 7000, + HistoryLatestClosedAt: time.Now(), + HistoryElder: 300, + ExpHistoryLatest: 7000, + }) + // invalid cursor + validateCursor("blah", 2, "asc", "blah", problem.BadRequest) + validateCursor("blah", 2, "desc", "blah", problem.BadRequest) + + // asc order + validateCursor(toid.AfterLedger(0).String(), 2, "asc", toid.AfterLedger(299).String(), nil) + validateCursor(toid.AfterLedger(200).String(), 2, "asc", toid.AfterLedger(299).String(), nil) + validateCursor(toid.AfterLedger(298).String(), 2, "asc", toid.AfterLedger(299).String(), nil) + validateCursor(toid.AfterLedger(299).String(), 2, "asc", toid.AfterLedger(299).String(), nil) + validateCursor(toid.AfterLedger(300).String(), 2, "asc", toid.AfterLedger(300).String(), nil) + validateCursor(toid.AfterLedger(301).String(), 2, "asc", toid.AfterLedger(301).String(), nil) + validateCursor(toid.AfterLedger(7001).String(), 2, "asc", toid.AfterLedger(7001).String(), nil) + + // desc order + validateCursor(toid.AfterLedger(0).String(), 2, "desc", toid.AfterLedger(0).String(), hProblem.BeforeHistory) + validateCursor(toid.AfterLedger(200).String(), 2, "desc", toid.AfterLedger(200).String(), hProblem.BeforeHistory) + validateCursor(toid.AfterLedger(299).String(), 2, "desc", toid.AfterLedger(299).String(), hProblem.BeforeHistory) + validateCursor(toid.AfterLedger(300).String(), 2, "desc", toid.AfterLedger(300).String(), nil) + validateCursor(toid.AfterLedger(320).String(), 2, "desc", toid.AfterLedger(320).String(), nil) + validateCursor(toid.AfterLedger(7001).String(), 2, "desc", toid.AfterLedger(7001).String(), nil) } func TestGetString(t *testing.T) { diff --git a/services/horizon/internal/actions/ledger.go b/services/horizon/internal/actions/ledger.go index 0b3d51b8e1..5fe9c294e4 100644 --- a/services/horizon/internal/actions/ledger.go +++ b/services/horizon/internal/actions/ledger.go @@ -17,12 +17,12 @@ type GetLedgersHandler struct { } func (handler GetLedgersHandler) GetResourcePage(w HeaderWriter, r *http.Request) ([]hal.Pageable, error) { - pq, err := GetPageQuery(handler.LedgerState, r, DefaultTOID) + pq, err := GetPageQuery(handler.LedgerState, r) if err != nil { return nil, err } - err = validateCursorWithinHistory(handler.LedgerState, pq) + err = validateAndAdjustCursor(handler.LedgerState, &pq) if err != nil { return nil, err } @@ -33,7 +33,8 @@ func (handler GetLedgersHandler) GetResourcePage(w HeaderWriter, r *http.Request } var records []history.Ledger - if err = historyQ.Ledgers().Page(pq).Select(r.Context(), &records); err != nil { + err = historyQ.Ledgers().Page(pq, handler.LedgerState.CurrentStatus().HistoryElder).Select(r.Context(), &records) + if err != nil { return nil, err } diff --git a/services/horizon/internal/actions/operation.go b/services/horizon/internal/actions/operation.go index 9ccadb272e..f12dc95eaa 100644 --- a/services/horizon/internal/actions/operation.go +++ b/services/horizon/internal/actions/operation.go @@ -72,12 +72,12 @@ type GetOperationsHandler struct { func (handler GetOperationsHandler) GetResourcePage(w HeaderWriter, r *http.Request) ([]hal.Pageable, error) { ctx := r.Context() - pq, err := GetPageQuery(handler.LedgerState, r, DefaultTOID) + pq, err := GetPageQuery(handler.LedgerState, r) if err != nil { return nil, err } - err = validateCursorWithinHistory(handler.LedgerState, pq) + err = validateAndAdjustCursor(handler.LedgerState, &pq) if err != nil { return nil, err } @@ -122,7 +122,7 @@ func (handler GetOperationsHandler) GetResourcePage(w HeaderWriter, r *http.Requ query.OnlyPayments() } - ops, txs, err := query.Page(pq).Fetch(ctx) + ops, txs, err := query.Page(pq, handler.LedgerState.CurrentStatus().HistoryElder).Fetch(ctx) if err != nil { return nil, err } diff --git a/services/horizon/internal/actions/trade.go b/services/horizon/internal/actions/trade.go index b29ad64715..40c0fa61de 100644 --- a/services/horizon/internal/actions/trade.go +++ b/services/horizon/internal/actions/trade.go @@ -159,12 +159,12 @@ type GetTradesHandler struct { func (handler GetTradesHandler) GetResourcePage(w HeaderWriter, r *http.Request) ([]hal.Pageable, error) { ctx := r.Context() - pq, err := GetPageQuery(handler.LedgerState, r, DefaultTOID) + pq, err := GetPageQuery(handler.LedgerState, r) if err != nil { return nil, err } - err = validateCursorWithinHistory(handler.LedgerState, pq) + err = validateAndAdjustCursor(handler.LedgerState, &pq) if err != nil { return nil, err } @@ -189,19 +189,20 @@ func (handler GetTradesHandler) GetResourcePage(w HeaderWriter, r *http.Request) return nil, err } + oldestLedger := handler.LedgerState.CurrentStatus().HistoryElder if baseAsset != nil { counterAsset, err = qp.Counter() if err != nil { return nil, err } - records, err = historyQ.GetTradesForAssets(ctx, pq, qp.AccountID, qp.TradeType, *baseAsset, *counterAsset) + records, err = historyQ.GetTradesForAssets(ctx, pq, oldestLedger, qp.AccountID, qp.TradeType, *baseAsset, *counterAsset) } else if qp.OfferID != 0 { - records, err = historyQ.GetTradesForOffer(ctx, pq, int64(qp.OfferID)) + records, err = historyQ.GetTradesForOffer(ctx, pq, oldestLedger, int64(qp.OfferID)) } else if qp.PoolID != "" { - records, err = historyQ.GetTradesForLiquidityPool(ctx, pq, qp.PoolID) + records, err = historyQ.GetTradesForLiquidityPool(ctx, pq, oldestLedger, qp.PoolID) } else { - records, err = historyQ.GetTrades(ctx, pq, qp.AccountID, qp.TradeType) + records, err = historyQ.GetTrades(ctx, pq, oldestLedger, qp.AccountID, qp.TradeType) } if err != nil { return nil, err @@ -287,10 +288,7 @@ func (handler GetTradeAggregationsHandler) GetResource(w HeaderWriter, r *http.R if err != nil { return nil, err } - err = validateCursorWithinHistory(handler.LedgerState, pq) - if err != nil { - return nil, err - } + qp := TradeAggregationsQuery{} if err = getParams(&qp, r); err != nil { return nil, err diff --git a/services/horizon/internal/actions/transaction.go b/services/horizon/internal/actions/transaction.go index 6a2fd1c6e2..c1649e7524 100644 --- a/services/horizon/internal/actions/transaction.go +++ b/services/horizon/internal/actions/transaction.go @@ -98,12 +98,12 @@ type GetTransactionsHandler struct { func (handler GetTransactionsHandler) GetResourcePage(w HeaderWriter, r *http.Request) ([]hal.Pageable, error) { ctx := r.Context() - pq, err := GetPageQuery(handler.LedgerState, r, DefaultTOID) + pq, err := GetPageQuery(handler.LedgerState, r) if err != nil { return nil, err } - err = validateCursorWithinHistory(handler.LedgerState, pq) + err = validateAndAdjustCursor(handler.LedgerState, &pq) if err != nil { return nil, err } @@ -119,7 +119,7 @@ func (handler GetTransactionsHandler) GetResourcePage(w HeaderWriter, r *http.Re return nil, err } - records, err := loadTransactionRecords(ctx, historyQ, qp, pq) + records, err := loadTransactionRecords(ctx, historyQ, qp, pq, handler.LedgerState.CurrentStatus().HistoryElder) if err != nil { return nil, errors.Wrap(err, "loading transaction records") } @@ -141,7 +141,7 @@ func (handler GetTransactionsHandler) GetResourcePage(w HeaderWriter, r *http.Re // loadTransactionRecords returns a slice of transaction records of an // account/ledger identified by accountID/ledgerID based on pq and // includeFailedTx. -func loadTransactionRecords(ctx context.Context, hq *history.Q, qp TransactionsQuery, pq db2.PageQuery) ([]history.Transaction, error) { +func loadTransactionRecords(ctx context.Context, hq *history.Q, qp TransactionsQuery, pq db2.PageQuery, oldestLedger int32) ([]history.Transaction, error) { var records []history.Transaction txs := hq.Transactions() @@ -160,7 +160,7 @@ func loadTransactionRecords(ctx context.Context, hq *history.Q, qp TransactionsQ txs.IncludeFailed() } - err := txs.Page(pq).Select(ctx, &records) + err := txs.Page(pq, oldestLedger).Select(ctx, &records) if err != nil { return nil, errors.Wrap(err, "executing transaction records query") } diff --git a/services/horizon/internal/db2/history/account_loader.go b/services/horizon/internal/db2/history/account_loader.go index 9e15920609..f69e7d7f48 100644 --- a/services/horizon/internal/db2/history/account_loader.go +++ b/services/horizon/internal/db2/history/account_loader.go @@ -2,29 +2,10 @@ package history import ( "cmp" - "context" - "database/sql/driver" - "fmt" - "sort" - "strings" - - "github.com/lib/pq" "github.com/stellar/go/support/collections/set" - "github.com/stellar/go/support/db" - "github.com/stellar/go/support/errors" ) -var errSealed = errors.New("cannot register more entries to Loader after calling Exec()") - -// LoaderStats describes the result of executing a history lookup id Loader -type LoaderStats struct { - // Total is the number of elements registered to the Loader - Total int - // Inserted is the number of elements inserted into the lookup table - Inserted int -} - // FutureAccountID represents a future history account. // A FutureAccountID is created by an AccountLoader and // the account id is available after calling Exec() on @@ -38,7 +19,7 @@ type FutureAccountID = future[string, Account] type AccountLoader = loader[string, Account] // NewAccountLoader will construct a new AccountLoader instance. -func NewAccountLoader() *AccountLoader { +func NewAccountLoader(concurrencyMode ConcurrencyMode) *AccountLoader { return &AccountLoader{ sealed: false, set: set.Set[string]{}, @@ -58,287 +39,11 @@ func NewAccountLoader() *AccountLoader { mappingFromRow: func(account Account) (string, int64) { return account.Address, account.ID }, - less: cmp.Less[string], + less: cmp.Less[string], + concurrencyMode: concurrencyMode, } } -type loader[K comparable, T any] struct { - sealed bool - set set.Set[K] - ids map[K]int64 - stats LoaderStats - name string - table string - columnsForKeys func([]K) []columnValues - mappingFromRow func(T) (K, int64) - less func(K, K) bool -} - -type future[K comparable, T any] struct { - key K - loader *loader[K, T] -} - -// Value implements the database/sql/driver Valuer interface. -func (f future[K, T]) Value() (driver.Value, error) { - return f.loader.GetNow(f.key) -} - -// GetFuture registers the given key into the Loader and -// returns a future which will hold the history id for -// the key after Exec() is called. -func (l *loader[K, T]) GetFuture(key K) future[K, T] { - if l.sealed { - panic(errSealed) - } - - l.set.Add(key) - return future[K, T]{ - key: key, - loader: l, - } -} - -// GetNow returns the history id for the given key. -// GetNow should only be called on values which were registered by -// GetFuture() calls. Also, Exec() must be called before any GetNow -// call can succeed. -func (l *loader[K, T]) GetNow(key K) (int64, error) { - if !l.sealed { - return 0, fmt.Errorf(`invalid loader state, - Exec was not called yet to properly seal and resolve %v id`, key) - } - if internalID, ok := l.ids[key]; !ok { - return 0, fmt.Errorf(`loader key %v was not found`, key) - } else { - return internalID, nil - } -} - -// Exec will look up all the history ids for the keys registered in the Loader. -// If there are no history ids for a given set of keys, Exec will insert rows -// into the corresponding history table to establish a mapping between each key and its history id. -func (l *loader[K, T]) Exec(ctx context.Context, session db.SessionInterface) error { - l.sealed = true - if len(l.set) == 0 { - return nil - } - q := &Q{session} - keys := make([]K, 0, len(l.set)) - for key := range l.set { - keys = append(keys, key) - } - // sort entries before inserting rows to prevent deadlocks on acquiring a ShareLock - // https://github.com/stellar/go/issues/2370 - sort.Slice(keys, func(i, j int) bool { - return l.less(keys[i], keys[j]) - }) - - if count, err := l.insert(ctx, q, keys); err != nil { - return err - } else { - l.stats.Total += count - l.stats.Inserted += count - } - - if count, err := l.query(ctx, q, keys); err != nil { - return err - } else { - l.stats.Total += count - } - - return nil -} - -// Stats returns the number of addresses registered in the Loader and the number of rows -// inserted into the history table. -func (l *loader[K, T]) Stats() LoaderStats { - return l.stats -} - -func (l *loader[K, T]) Name() string { - return l.name -} - -func (l *loader[K, T]) filter(keys []K) []K { - if len(l.ids) == 0 { - return keys - } - - remaining := make([]K, 0, len(keys)) - for _, key := range keys { - if _, ok := l.ids[key]; ok { - continue - } - remaining = append(remaining, key) - } - return remaining -} - -func (l *loader[K, T]) updateMap(rows []T) { - for _, row := range rows { - key, id := l.mappingFromRow(row) - l.ids[key] = id - } -} - -func (l *loader[K, T]) insert(ctx context.Context, q *Q, keys []K) (int, error) { - keys = l.filter(keys) - if len(keys) == 0 { - return 0, nil - } - - var rows []T - err := bulkInsert( - ctx, - q, - l.table, - l.columnsForKeys(keys), - &rows, - ) - if err != nil { - return 0, err - } - - l.updateMap(rows) - return len(rows), nil -} - -func (l *loader[K, T]) query(ctx context.Context, q *Q, keys []K) (int, error) { - keys = l.filter(keys) - if len(keys) == 0 { - return 0, nil - } - - var rows []T - err := bulkGet( - ctx, - q, - l.table, - l.columnsForKeys(keys), - &rows, - ) - if err != nil { - return 0, err - } - - l.updateMap(rows) - return len(rows), nil -} - -type columnValues struct { - name string - dbType string - objects []string -} - -func bulkInsert(ctx context.Context, q *Q, table string, fields []columnValues, response interface{}) error { - unnestPart := make([]string, 0, len(fields)) - insertFieldsPart := make([]string, 0, len(fields)) - pqArrays := make([]interface{}, 0, len(fields)) - - // In the code below we are building the bulk insert query which looks like: - // - // WITH rows AS - // (SELECT - // /* unnestPart */ - // unnest(?::type1[]), /* field1 */ - // unnest(?::type2[]), /* field2 */ - // ... - // ) - // INSERT INTO table ( - // /* insertFieldsPart */ - // field1, - // field2, - // ... - // ) - // SELECT * FROM rows ON CONFLICT (field1, field2, ...) DO NOTHING RETURNING * - // - // Using unnest allows to get around the maximum limit of 65,535 query parameters, - // see https://www.postgresql.org/docs/12/limits.html and - // https://klotzandrew.com/blog/postgres-passing-65535-parameter-limit/ - // - // Without using unnest we would have to use multiple insert statements to insert - // all the rows for large datasets. - for _, field := range fields { - unnestPart = append( - unnestPart, - fmt.Sprintf("unnest(?::%s[]) /* %s */", field.dbType, field.name), - ) - insertFieldsPart = append( - insertFieldsPart, - field.name, - ) - pqArrays = append( - pqArrays, - pq.Array(field.objects), - ) - } - columns := strings.Join(insertFieldsPart, ",") - - sql := ` - WITH rows AS - (SELECT ` + strings.Join(unnestPart, ",") + `) - INSERT INTO ` + table + ` - (` + columns + `) - SELECT * FROM rows - ON CONFLICT (` + columns + `) DO NOTHING - RETURNING *` - - return q.SelectRaw( - ctx, - response, - sql, - pqArrays..., - ) -} - -func bulkGet(ctx context.Context, q *Q, table string, fields []columnValues, response interface{}) error { - unnestPart := make([]string, 0, len(fields)) - columns := make([]string, 0, len(fields)) - pqArrays := make([]interface{}, 0, len(fields)) - - // In the code below we are building the bulk get query which looks like: - // - // SELECT * FROM table WHERE (field1, field2, ...) IN - // (SELECT - // /* unnestPart */ - // unnest(?::type1[]), /* field1 */ - // unnest(?::type2[]), /* field2 */ - // ... - // ) - // - // Using unnest allows to get around the maximum limit of 65,535 query parameters, - // see https://www.postgresql.org/docs/12/limits.html and - // https://klotzandrew.com/blog/postgres-passing-65535-parameter-limit/ - // - // Without using unnest we would have to use multiple select statements to obtain - // all the rows for large datasets. - for _, field := range fields { - unnestPart = append( - unnestPart, - fmt.Sprintf("unnest(?::%s[]) /* %s */", field.dbType, field.name), - ) - columns = append( - columns, - field.name, - ) - pqArrays = append( - pqArrays, - pq.Array(field.objects), - ) - } - sql := `SELECT * FROM ` + table + ` WHERE (` + strings.Join(columns, ",") + `) IN - (SELECT ` + strings.Join(unnestPart, ",") + `)` - - return q.SelectRaw( - ctx, - response, - sql, - pqArrays..., - ) -} - // AccountLoaderStub is a stub wrapper around AccountLoader which allows // you to manually configure the mapping of addresses to history account ids type AccountLoaderStub struct { @@ -347,7 +52,7 @@ type AccountLoaderStub struct { // NewAccountLoaderStub returns a new AccountLoaderStub instance func NewAccountLoaderStub() AccountLoaderStub { - return AccountLoaderStub{Loader: NewAccountLoader()} + return AccountLoaderStub{Loader: NewAccountLoader(ConcurrentInserts)} } // Insert updates the wrapped AccountLoader so that the given account diff --git a/services/horizon/internal/db2/history/account_loader_test.go b/services/horizon/internal/db2/history/account_loader_test.go index 9a9fb30445..83b172b40b 100644 --- a/services/horizon/internal/db2/history/account_loader_test.go +++ b/services/horizon/internal/db2/history/account_loader_test.go @@ -8,6 +8,7 @@ import ( "github.com/stellar/go/keypair" "github.com/stellar/go/services/horizon/internal/test" + "github.com/stellar/go/support/db" ) func TestAccountLoader(t *testing.T) { @@ -16,12 +17,18 @@ func TestAccountLoader(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) session := tt.HorizonSession() + testAccountLoader(t, session, ConcurrentInserts) + test.ResetHorizonDB(t, tt.HorizonDB) + testAccountLoader(t, session, ConcurrentDeletes) +} + +func testAccountLoader(t *testing.T, session *db.Session, mode ConcurrencyMode) { var addresses []string for i := 0; i < 100; i++ { addresses = append(addresses, keypair.MustRandom().Address()) } - loader := NewAccountLoader() + loader := NewAccountLoader(mode) for _, address := range addresses { future := loader.GetFuture(address) _, err := future.Value() @@ -58,7 +65,7 @@ func TestAccountLoader(t *testing.T) { // check that Loader works when all the previous values are already // present in the db and also add 10 more rows to insert - loader = NewAccountLoader() + loader = NewAccountLoader(mode) for i := 0; i < 10; i++ { addresses = append(addresses, keypair.MustRandom().Address()) } @@ -85,5 +92,4 @@ func TestAccountLoader(t *testing.T) { assert.Equal(t, account.ID, internalId) assert.Equal(t, account.Address, address) } - } diff --git a/services/horizon/internal/db2/history/asset_loader.go b/services/horizon/internal/db2/history/asset_loader.go index cdd2a0d714..33c5c333dd 100644 --- a/services/horizon/internal/db2/history/asset_loader.go +++ b/services/horizon/internal/db2/history/asset_loader.go @@ -42,7 +42,7 @@ type FutureAssetID = future[AssetKey, Asset] type AssetLoader = loader[AssetKey, Asset] // NewAssetLoader will construct a new AssetLoader instance. -func NewAssetLoader() *AssetLoader { +func NewAssetLoader(concurrencyMode ConcurrencyMode) *AssetLoader { return &AssetLoader{ sealed: false, set: set.Set[AssetKey]{}, @@ -88,6 +88,7 @@ func NewAssetLoader() *AssetLoader { less: func(a AssetKey, b AssetKey) bool { return a.String() < b.String() }, + concurrencyMode: concurrencyMode, } } @@ -99,7 +100,7 @@ type AssetLoaderStub struct { // NewAssetLoaderStub returns a new AssetLoaderStub instance func NewAssetLoaderStub() AssetLoaderStub { - return AssetLoaderStub{Loader: NewAssetLoader()} + return AssetLoaderStub{Loader: NewAssetLoader(ConcurrentInserts)} } // Insert updates the wrapped AssetLoaderStub so that the given asset diff --git a/services/horizon/internal/db2/history/asset_loader_test.go b/services/horizon/internal/db2/history/asset_loader_test.go index ca65cebb7e..e7a0495cad 100644 --- a/services/horizon/internal/db2/history/asset_loader_test.go +++ b/services/horizon/internal/db2/history/asset_loader_test.go @@ -9,6 +9,7 @@ import ( "github.com/stellar/go/keypair" "github.com/stellar/go/services/horizon/internal/test" + "github.com/stellar/go/support/db" "github.com/stellar/go/xdr" ) @@ -40,6 +41,12 @@ func TestAssetLoader(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) session := tt.HorizonSession() + testAssetLoader(t, session, ConcurrentInserts) + test.ResetHorizonDB(t, tt.HorizonDB) + testAssetLoader(t, session, ConcurrentDeletes) +} + +func testAssetLoader(t *testing.T, session *db.Session, mode ConcurrencyMode) { var keys []AssetKey for i := 0; i < 100; i++ { var key AssetKey @@ -66,7 +73,7 @@ func TestAssetLoader(t *testing.T) { keys = append(keys, key) } - loader := NewAssetLoader() + loader := NewAssetLoader(mode) for _, key := range keys { future := loader.GetFuture(key) _, err := future.Value() @@ -109,7 +116,7 @@ func TestAssetLoader(t *testing.T) { // check that Loader works when all the previous values are already // present in the db and also add 10 more rows to insert - loader = NewAssetLoader() + loader = NewAssetLoader(mode) for i := 0; i < 10; i++ { var key AssetKey if i%2 == 0 { diff --git a/services/horizon/internal/db2/history/claimable_balance_loader.go b/services/horizon/internal/db2/history/claimable_balance_loader.go index f775ea4b24..9107d4fb9f 100644 --- a/services/horizon/internal/db2/history/claimable_balance_loader.go +++ b/services/horizon/internal/db2/history/claimable_balance_loader.go @@ -19,7 +19,7 @@ type FutureClaimableBalanceID = future[string, HistoryClaimableBalance] type ClaimableBalanceLoader = loader[string, HistoryClaimableBalance] // NewClaimableBalanceLoader will construct a new ClaimableBalanceLoader instance. -func NewClaimableBalanceLoader() *ClaimableBalanceLoader { +func NewClaimableBalanceLoader(concurrencyMode ConcurrencyMode) *ClaimableBalanceLoader { return &ClaimableBalanceLoader{ sealed: false, set: set.Set[string]{}, @@ -39,6 +39,7 @@ func NewClaimableBalanceLoader() *ClaimableBalanceLoader { mappingFromRow: func(row HistoryClaimableBalance) (string, int64) { return row.BalanceID, row.InternalID }, - less: cmp.Less[string], + less: cmp.Less[string], + concurrencyMode: concurrencyMode, } } diff --git a/services/horizon/internal/db2/history/claimable_balance_loader_test.go b/services/horizon/internal/db2/history/claimable_balance_loader_test.go index f5759015c7..490d5a0f70 100644 --- a/services/horizon/internal/db2/history/claimable_balance_loader_test.go +++ b/services/horizon/internal/db2/history/claimable_balance_loader_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stellar/go/services/horizon/internal/test" + "github.com/stellar/go/support/db" "github.com/stellar/go/xdr" ) @@ -17,6 +18,12 @@ func TestClaimableBalanceLoader(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) session := tt.HorizonSession() + testCBLoader(t, tt, session, ConcurrentInserts) + test.ResetHorizonDB(t, tt.HorizonDB) + testCBLoader(t, tt, session, ConcurrentDeletes) +} + +func testCBLoader(t *testing.T, tt *test.T, session *db.Session, mode ConcurrencyMode) { var ids []string for i := 0; i < 100; i++ { balanceID := xdr.ClaimableBalanceId{ @@ -28,7 +35,7 @@ func TestClaimableBalanceLoader(t *testing.T) { ids = append(ids, id) } - loader := NewClaimableBalanceLoader() + loader := NewClaimableBalanceLoader(mode) var futures []FutureClaimableBalanceID for _, id := range ids { future := loader.GetFuture(id) @@ -70,7 +77,7 @@ func TestClaimableBalanceLoader(t *testing.T) { // check that Loader works when all the previous values are already // present in the db and also add 10 more rows to insert - loader = NewClaimableBalanceLoader() + loader = NewClaimableBalanceLoader(mode) for i := 100; i < 110; i++ { balanceID := xdr.ClaimableBalanceId{ Type: xdr.ClaimableBalanceIdTypeClaimableBalanceIdTypeV0, diff --git a/services/horizon/internal/db2/history/effect.go b/services/horizon/internal/db2/history/effect.go index bdf1e2dfb0..0f0b25ec03 100644 --- a/services/horizon/internal/db2/history/effect.go +++ b/services/horizon/internal/db2/history/effect.go @@ -14,6 +14,8 @@ import ( "github.com/stellar/go/toid" ) +const genesisLedger = 2 + // UnmarshalDetails unmarshals the details of this effect into `dest` func (r *Effect) UnmarshalDetails(dest interface{}) error { if !r.DetailsString.Valid { @@ -70,7 +72,7 @@ func (r *Effect) PagingToken() string { } // Effects returns a page of effects without any filters besides the cursor -func (q *Q) Effects(ctx context.Context, page db2.PageQuery) ([]Effect, error) { +func (q *Q) Effects(ctx context.Context, page db2.PageQuery, oldestLedger int32) ([]Effect, error) { op, idx, err := parseEffectsCursor(page) if err != nil { return nil, err @@ -87,6 +89,9 @@ func (q *Q) Effects(ctx context.Context, page db2.PageQuery) ([]Effect, error) { Where("(heff.history_operation_id, heff.order) > (?, ?)", op, idx). OrderBy("heff.history_operation_id asc, heff.order asc") case "desc": + if lowerBound := lowestLedgerBound(oldestLedger); lowerBound > 0 { + query = query.Where("heff.history_operation_id > ?", lowerBound) + } query = query. Where("(heff.history_operation_id, heff.order) < (?, ?)", op, idx). OrderBy("heff.history_operation_id desc, heff.order desc") @@ -101,14 +106,14 @@ func (q *Q) Effects(ctx context.Context, page db2.PageQuery) ([]Effect, error) { } // EffectsForAccount returns a page of effects for a given account -func (q *Q) EffectsForAccount(ctx context.Context, aid string, page db2.PageQuery) ([]Effect, error) { +func (q *Q) EffectsForAccount(ctx context.Context, aid string, page db2.PageQuery, oldestLedger int32) ([]Effect, error) { var account Account if err := q.AccountByAddress(ctx, &account, aid); err != nil { return nil, err } query := selectEffect.Where("heff.history_account_id = ?", account.ID) - return q.selectEffectsPage(ctx, query, page) + return q.selectEffectsPage(ctx, query, page, oldestLedger) } // EffectsForLedger returns a page of effects for a given ledger sequence @@ -125,7 +130,7 @@ func (q *Q) EffectsForLedger(ctx context.Context, seq int32, page db2.PageQuery) start.ToInt64(), end.ToInt64(), ) - return q.selectEffectsPage(ctx, query, page) + return q.selectEffectsPage(ctx, query, page, 0) } // EffectsForOperation returns a page of effects for a given operation id. @@ -138,11 +143,11 @@ func (q *Q) EffectsForOperation(ctx context.Context, id int64, page db2.PageQuer start.ToInt64(), end.ToInt64(), ) - return q.selectEffectsPage(ctx, query, page) + return q.selectEffectsPage(ctx, query, page, 0) } // EffectsForLiquidityPool returns a page of effects for a given liquidity pool. -func (q *Q) EffectsForLiquidityPool(ctx context.Context, id string, page db2.PageQuery) ([]Effect, error) { +func (q *Q) EffectsForLiquidityPool(ctx context.Context, id string, page db2.PageQuery, oldestLedger int32) ([]Effect, error) { op, _, err := page.CursorInt64Pair(db2.DefaultPairSep) if err != nil { return nil, err @@ -173,6 +178,7 @@ func (q *Q) EffectsForLiquidityPool(ctx context.Context, id string, page db2.Pag "heff.history_operation_id": liquidityPoolOperationIDs, }), page, + oldestLedger, ) } @@ -194,6 +200,7 @@ func (q *Q) EffectsForTransaction(ctx context.Context, hash string, page db2.Pag end.ToInt64(), ), page, + 0, ) } @@ -209,7 +216,14 @@ func parseEffectsCursor(page db2.PageQuery) (int64, int64, error) { return op, idx, nil } -func (q *Q) selectEffectsPage(ctx context.Context, query sq.SelectBuilder, page db2.PageQuery) ([]Effect, error) { +func lowestLedgerBound(oldestLedger int32) int64 { + if oldestLedger <= genesisLedger { + return 0 + } + return toid.AfterLedger(oldestLedger - 1).ToInt64() +} + +func (q *Q) selectEffectsPage(ctx context.Context, query sq.SelectBuilder, page db2.PageQuery, oldestLedger int32) ([]Effect, error) { op, idx, err := parseEffectsCursor(page) if err != nil { return nil, err @@ -230,6 +244,9 @@ func (q *Q) selectEffectsPage(ctx context.Context, query sq.SelectBuilder, page ))`, op, op, op, idx). OrderBy("heff.history_operation_id asc, heff.order asc") case "desc": + if lowerBound := lowestLedgerBound(oldestLedger); lowerBound > 0 { + query = query.Where("heff.history_operation_id > ?", lowerBound) + } query = query. Where(`( heff.history_operation_id <= ? diff --git a/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go b/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go index e1ac998953..2a3b0c9ab8 100644 --- a/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go +++ b/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go @@ -2,6 +2,7 @@ package history import ( "encoding/json" + "fmt" "testing" "github.com/guregu/null" @@ -16,11 +17,11 @@ func TestAddEffect(t *testing.T) { defer tt.Finish() test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - tt.Assert.NoError(q.Begin(tt.Ctx)) + tt.Require.NoError(q.Begin(tt.Ctx)) address := "GAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSTVY" muxedAddres := "MAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSAAAAAAAAAAE2LP26" - accountLoader := NewAccountLoader() + accountLoader := NewAccountLoader(ConcurrentInserts) builder := q.NewEffectBatchInsertBuilder() sequence := int32(56) @@ -37,25 +38,42 @@ func TestAddEffect(t *testing.T) { 3, details, ) - tt.Assert.NoError(err) + tt.Require.NoError(err) - tt.Assert.NoError(accountLoader.Exec(tt.Ctx, q)) - tt.Assert.NoError(builder.Exec(tt.Ctx, q)) - tt.Assert.NoError(q.Commit()) + tt.Require.NoError(accountLoader.Exec(tt.Ctx, q)) + tt.Require.NoError(builder.Exec(tt.Ctx, q)) + tt.Require.NoError(q.Commit()) effects, err := q.Effects(tt.Ctx, db2.PageQuery{ Cursor: "0-0", Order: "asc", Limit: 200, - }) + }, 0) tt.Require.NoError(err) - tt.Assert.Len(effects, 1) + tt.Require.Len(effects, 1) effect := effects[0] - tt.Assert.Equal(address, effect.Account) - tt.Assert.Equal(muxedAddres, effect.AccountMuxed.String) - tt.Assert.Equal(int64(240518172673), effect.HistoryOperationID) - tt.Assert.Equal(int32(1), effect.Order) - tt.Assert.Equal(EffectType(3), effect.Type) - tt.Assert.Equal("{\"amount\": \"1000.0000000\", \"asset_type\": \"native\"}", effect.DetailsString.String) + tt.Require.Equal(address, effect.Account) + tt.Require.Equal(muxedAddres, effect.AccountMuxed.String) + tt.Require.Equal(int64(240518172673), effect.HistoryOperationID) + tt.Require.Equal(int32(1), effect.Order) + tt.Require.Equal(EffectType(3), effect.Type) + tt.Require.Equal("{\"amount\": \"1000.0000000\", \"asset_type\": \"native\"}", effect.DetailsString.String) + + effects, err = q.Effects(tt.Ctx, db2.PageQuery{ + Cursor: fmt.Sprintf("%d-0", toid.New(sequence+2, 0, 0).ToInt64()), + Order: "desc", + Limit: 200, + }, sequence-3) + tt.Require.NoError(err) + tt.Require.Len(effects, 1) + tt.Require.Equal(effects[0], effect) + + effects, err = q.Effects(tt.Ctx, db2.PageQuery{ + Cursor: fmt.Sprintf("%d-0", toid.New(sequence+5, 0, 0).ToInt64()), + Order: "desc", + Limit: 200, + }, sequence+2) + tt.Require.NoError(err) + tt.Require.Empty(effects) } diff --git a/services/horizon/internal/db2/history/effect_test.go b/services/horizon/internal/db2/history/effect_test.go index 19af0ceff8..e3eb79d941 100644 --- a/services/horizon/internal/db2/history/effect_test.go +++ b/services/horizon/internal/db2/history/effect_test.go @@ -23,7 +23,7 @@ func TestEffectsForLiquidityPool(t *testing.T) { // Insert Effect address := "GAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSTVY" muxedAddres := "MAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSAAAAAAAAAAE2LP26" - accountLoader := NewAccountLoader() + accountLoader := NewAccountLoader(ConcurrentInserts) builder := q.NewEffectBatchInsertBuilder() sequence := int32(56) @@ -47,7 +47,7 @@ func TestEffectsForLiquidityPool(t *testing.T) { // Insert Liquidity Pool history liquidityPoolID := "abcde" - lpLoader := NewLiquidityPoolLoader() + lpLoader := NewLiquidityPoolLoader(ConcurrentInserts) operationBuilder := q.NewOperationLiquidityPoolBatchInsertBuilder() tt.Assert.NoError(operationBuilder.Add(opID, lpLoader.GetFuture(liquidityPoolID))) @@ -56,17 +56,34 @@ func TestEffectsForLiquidityPool(t *testing.T) { tt.Assert.NoError(q.Commit()) - var result []Effect - result, err = q.EffectsForLiquidityPool(tt.Ctx, liquidityPoolID, db2.PageQuery{ + var effects []Effect + effects, err = q.EffectsForLiquidityPool(tt.Ctx, liquidityPoolID, db2.PageQuery{ Cursor: "0-0", Order: "asc", Limit: 10, - }) + }, 0) tt.Assert.NoError(err) - tt.Assert.Len(result, 1) - tt.Assert.Equal(result[0].Account, address) + tt.Assert.Len(effects, 1) + effect := effects[0] + tt.Assert.Equal(effect.Account, address) + + effects, err = q.EffectsForLiquidityPool(tt.Ctx, liquidityPoolID, db2.PageQuery{ + Cursor: fmt.Sprintf("%d-0", toid.New(sequence+2, 0, 0).ToInt64()), + Order: "desc", + Limit: 200, + }, sequence-3) + tt.Require.NoError(err) + tt.Require.Len(effects, 1) + tt.Require.Equal(effects[0], effect) + effects, err = q.EffectsForLiquidityPool(tt.Ctx, liquidityPoolID, db2.PageQuery{ + Cursor: fmt.Sprintf("%d-0", toid.New(sequence+5, 0, 0).ToInt64()), + Order: "desc", + Limit: 200, + }, sequence+2) + tt.Require.NoError(err) + tt.Require.Empty(effects) } func TestEffectsForTrustlinesSponsorshipEmptyAssetType(t *testing.T) { @@ -78,7 +95,7 @@ func TestEffectsForTrustlinesSponsorshipEmptyAssetType(t *testing.T) { address := "GAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSTVY" muxedAddres := "MAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSAAAAAAAAAAE2LP26" - accountLoader := NewAccountLoader() + accountLoader := NewAccountLoader(ConcurrentInserts) builder := q.NewEffectBatchInsertBuilder() sequence := int32(56) @@ -160,7 +177,7 @@ func TestEffectsForTrustlinesSponsorshipEmptyAssetType(t *testing.T) { Cursor: "0-0", Order: "asc", Limit: 200, - }) + }, 0) tt.Require.NoError(err) tt.Require.Len(results, len(tests)) diff --git a/services/horizon/internal/db2/history/fee_bump_scenario.go b/services/horizon/internal/db2/history/fee_bump_scenario.go index da6563c732..e161d686d1 100644 --- a/services/horizon/internal/db2/history/fee_bump_scenario.go +++ b/services/horizon/internal/db2/history/fee_bump_scenario.go @@ -288,7 +288,7 @@ func FeeBumpScenario(tt *test.T, q *Q, successful bool) FeeBumpFixture { details, err = json.Marshal(map[string]interface{}{"new_seq": 98}) tt.Assert.NoError(err) - accountLoader := NewAccountLoader() + accountLoader := NewAccountLoader(ConcurrentInserts) err = effectBuilder.Add( accountLoader.GetFuture(account.Address()), diff --git a/services/horizon/internal/db2/history/key_value.go b/services/horizon/internal/db2/history/key_value.go index 3d23451937..9fee9513c2 100644 --- a/services/horizon/internal/db2/history/key_value.go +++ b/services/horizon/internal/db2/history/key_value.go @@ -5,7 +5,6 @@ import ( "database/sql" "fmt" "strconv" - "strings" sq "github.com/Masterminds/squirrel" @@ -207,41 +206,26 @@ func (q *Q) getValueFromStore(ctx context.Context, key string, forUpdate bool) ( return value, nil } -type KeyValuePair struct { - Key string `db:"key"` - Value string `db:"value"` -} - -func (q *Q) getLookupTableReapOffsets(ctx context.Context) (map[string]int64, error) { - keys := make([]string, 0, len(historyLookupTables)) - for table := range historyLookupTables { - keys = append(keys, table+lookupTableReapOffsetSuffix) - } - offsets := map[string]int64{} - var pairs []KeyValuePair - query := sq.Select("key", "value"). +func (q *Q) getLookupTableReapOffset(ctx context.Context, table string) (int64, error) { + query := sq.Select("value"). From("key_value_store"). Where(map[string]interface{}{ - "key": keys, + "key": table + lookupTableReapOffsetSuffix, }) - err := q.Select(ctx, &pairs, query) + var text string + err := q.Get(ctx, &text, query) if err != nil { - return nil, err - } - for _, pair := range pairs { - table := strings.TrimSuffix(pair.Key, lookupTableReapOffsetSuffix) - if _, ok := historyLookupTables[table]; !ok { - return nil, fmt.Errorf("invalid key: %s", pair.Key) - } - - var offset int64 - offset, err = strconv.ParseInt(pair.Value, 10, 64) - if err != nil { - return nil, fmt.Errorf("invalid offset: %s", pair.Value) + if errors.Cause(err) == sql.ErrNoRows { + return 0, nil } - offsets[table] = offset + return 0, err + } + var offset int64 + offset, err = strconv.ParseInt(text, 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid offset: %s for table %s", text, table) } - return offsets, err + return offset, nil } func (q *Q) updateLookupTableReapOffset(ctx context.Context, table string, offset int64) error { diff --git a/services/horizon/internal/db2/history/ledger.go b/services/horizon/internal/db2/history/ledger.go index c2d1f6b3c9..5059cb7f49 100644 --- a/services/horizon/internal/db2/history/ledger.go +++ b/services/horizon/internal/db2/history/ledger.go @@ -72,11 +72,15 @@ func (q *Q) LedgerCapacityUsageStats(ctx context.Context, currentSeq int32, dest } // Page specifies the paging constraints for the query being built by `q`. -func (q *LedgersQ) Page(page db2.PageQuery) *LedgersQ { +func (q *LedgersQ) Page(page db2.PageQuery, oldestLedger int32) *LedgersQ { if q.Err != nil { return q } + if lowerBound := lowestLedgerBound(oldestLedger); lowerBound > 0 && page.Order == "desc" { + q.sql = q.sql. + Where("hl.id > ?", lowerBound) + } q.sql, q.Err = page.ApplyTo(q.sql, "hl.id") return q } diff --git a/services/horizon/internal/db2/history/liquidity_pool_loader.go b/services/horizon/internal/db2/history/liquidity_pool_loader.go index a03caaa988..5da2a7b6fd 100644 --- a/services/horizon/internal/db2/history/liquidity_pool_loader.go +++ b/services/horizon/internal/db2/history/liquidity_pool_loader.go @@ -19,7 +19,7 @@ type FutureLiquidityPoolID = future[string, HistoryLiquidityPool] type LiquidityPoolLoader = loader[string, HistoryLiquidityPool] // NewLiquidityPoolLoader will construct a new LiquidityPoolLoader instance. -func NewLiquidityPoolLoader() *LiquidityPoolLoader { +func NewLiquidityPoolLoader(concurrencyMode ConcurrencyMode) *LiquidityPoolLoader { return &LiquidityPoolLoader{ sealed: false, set: set.Set[string]{}, @@ -39,7 +39,8 @@ func NewLiquidityPoolLoader() *LiquidityPoolLoader { mappingFromRow: func(row HistoryLiquidityPool) (string, int64) { return row.PoolID, row.InternalID }, - less: cmp.Less[string], + less: cmp.Less[string], + concurrencyMode: concurrencyMode, } } @@ -51,7 +52,7 @@ type LiquidityPoolLoaderStub struct { // NewLiquidityPoolLoaderStub returns a new LiquidityPoolLoader instance func NewLiquidityPoolLoaderStub() LiquidityPoolLoaderStub { - return LiquidityPoolLoaderStub{Loader: NewLiquidityPoolLoader()} + return LiquidityPoolLoaderStub{Loader: NewLiquidityPoolLoader(ConcurrentInserts)} } // Insert updates the wrapped LiquidityPoolLoader so that the given liquidity pool diff --git a/services/horizon/internal/db2/history/liquidity_pool_loader_test.go b/services/horizon/internal/db2/history/liquidity_pool_loader_test.go index aec2fcd886..c7a1282760 100644 --- a/services/horizon/internal/db2/history/liquidity_pool_loader_test.go +++ b/services/horizon/internal/db2/history/liquidity_pool_loader_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stellar/go/services/horizon/internal/test" + "github.com/stellar/go/support/db" "github.com/stellar/go/xdr" ) @@ -16,6 +17,12 @@ func TestLiquidityPoolLoader(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) session := tt.HorizonSession() + testLPLoader(t, tt, session, ConcurrentInserts) + test.ResetHorizonDB(t, tt.HorizonDB) + testLPLoader(t, tt, session, ConcurrentDeletes) +} + +func testLPLoader(t *testing.T, tt *test.T, session *db.Session, mode ConcurrencyMode) { var ids []string for i := 0; i < 100; i++ { poolID := xdr.PoolId{byte(i)} @@ -24,7 +31,7 @@ func TestLiquidityPoolLoader(t *testing.T) { ids = append(ids, id) } - loader := NewLiquidityPoolLoader() + loader := NewLiquidityPoolLoader(mode) for _, id := range ids { future := loader.GetFuture(id) _, err := future.Value() @@ -62,7 +69,7 @@ func TestLiquidityPoolLoader(t *testing.T) { // check that Loader works when all the previous values are already // present in the db and also add 10 more rows to insert - loader = NewLiquidityPoolLoader() + loader = NewLiquidityPoolLoader(mode) for i := 100; i < 110; i++ { poolID := xdr.PoolId{byte(i)} var id string diff --git a/services/horizon/internal/db2/history/loader.go b/services/horizon/internal/db2/history/loader.go new file mode 100644 index 0000000000..dc2236accb --- /dev/null +++ b/services/horizon/internal/db2/history/loader.go @@ -0,0 +1,365 @@ +package history + +import ( + "context" + "database/sql/driver" + "fmt" + "sort" + "strings" + + "github.com/lib/pq" + + "github.com/stellar/go/support/collections/set" + "github.com/stellar/go/support/db" +) + +var errSealed = fmt.Errorf("cannot register more entries to Loader after calling Exec()") + +// ConcurrencyMode is used to configure the level of thread-safety for a loader +type ConcurrencyMode int + +func (cm ConcurrencyMode) String() string { + switch cm { + case ConcurrentInserts: + return "ConcurrentInserts" + case ConcurrentDeletes: + return "ConcurrentDeletes" + default: + return "unknown" + } +} + +const ( + _ ConcurrencyMode = iota + // ConcurrentInserts configures the loader to maintain safety when there are multiple loaders + // inserting into the same table concurrently. This ConcurrencyMode is suitable for parallel reingestion. + // Note while ConcurrentInserts is enabled it is not safe to have deletes occurring concurrently on the + // same table. + ConcurrentInserts + // ConcurrentDeletes configures the loader to maintain safety when there is another thread which is invoking + // reapLookupTable() to delete rows from the same table concurrently. This ConcurrencyMode is suitable for + // live ingestion when reaping of lookup tables is enabled. + // Note while ConcurrentDeletes is enabled it is not safe to have multiple threads inserting concurrently to the + // same table. + ConcurrentDeletes +) + +// LoaderStats describes the result of executing a history lookup id Loader +type LoaderStats struct { + // Total is the number of elements registered to the Loader + Total int + // Inserted is the number of elements inserted into the lookup table + Inserted int +} + +type loader[K comparable, T any] struct { + sealed bool + set set.Set[K] + ids map[K]int64 + stats LoaderStats + name string + table string + columnsForKeys func([]K) []columnValues + mappingFromRow func(T) (K, int64) + less func(K, K) bool + concurrencyMode ConcurrencyMode +} + +type future[K comparable, T any] struct { + key K + loader *loader[K, T] +} + +// Value implements the database/sql/driver Valuer interface. +func (f future[K, T]) Value() (driver.Value, error) { + return f.loader.GetNow(f.key) +} + +// GetFuture registers the given key into the Loader and +// returns a future which will hold the history id for +// the key after Exec() is called. +func (l *loader[K, T]) GetFuture(key K) future[K, T] { + if l.sealed { + panic(errSealed) + } + + l.set.Add(key) + return future[K, T]{ + key: key, + loader: l, + } +} + +// GetNow returns the history id for the given key. +// GetNow should only be called on values which were registered by +// GetFuture() calls. Also, Exec() must be called before any GetNow +// call can succeed. +func (l *loader[K, T]) GetNow(key K) (int64, error) { + if !l.sealed { + return 0, fmt.Errorf(`invalid loader state, + Exec was not called yet to properly seal and resolve %v id`, key) + } + if internalID, ok := l.ids[key]; !ok { + return 0, fmt.Errorf(`loader key %v was not found`, key) + } else { + return internalID, nil + } +} + +// Exec will look up all the history ids for the keys registered in the Loader. +// If there are no history ids for a given set of keys, Exec will insert rows +// into the corresponding history table to establish a mapping between each key and its history id. +func (l *loader[K, T]) Exec(ctx context.Context, session db.SessionInterface) error { + l.sealed = true + if len(l.set) == 0 { + return nil + } + q := &Q{session} + keys := make([]K, 0, len(l.set)) + for key := range l.set { + keys = append(keys, key) + } + // sort entries before inserting rows to prevent deadlocks on acquiring a ShareLock + // https://github.com/stellar/go/issues/2370 + sort.Slice(keys, func(i, j int) bool { + return l.less(keys[i], keys[j]) + }) + + if l.concurrencyMode == ConcurrentInserts { + // if there are other ingestion transactions running concurrently, + // we need to first insert the records (with a ON CONFLICT DO NOTHING + // clause). Then, we can query for the remaining records. + // This order (insert first and then query) is important because + // if multiple concurrent transactions try to insert the same record + // only one of them will succeed and the other transactions will omit + // the record from the RETURNING set. + if count, err := l.insert(ctx, q, keys); err != nil { + return err + } else { + l.stats.Total += count + l.stats.Inserted += count + } + + if count, err := l.query(ctx, q, keys, false); err != nil { + return err + } else { + l.stats.Total += count + } + } else if l.concurrencyMode == ConcurrentDeletes { + // if the lookup table reaping transaction is running concurrently, + // we need to lock the rows from the lookup table to ensure that + // the reaper cannot run until after the ingestion transaction has + // been committed. + if count, err := l.query(ctx, q, keys, true); err != nil { + return err + } else { + l.stats.Total += count + } + + // insert whatever records were not found from l.query() + if count, err := l.insert(ctx, q, keys); err != nil { + return err + } else { + l.stats.Total += count + l.stats.Inserted += count + } + } else { + return fmt.Errorf("concurrency mode %v is invalid", l.concurrencyMode) + } + + return nil +} + +// Stats returns the number of addresses registered in the Loader and the number of rows +// inserted into the history table. +func (l *loader[K, T]) Stats() LoaderStats { + return l.stats +} + +func (l *loader[K, T]) Name() string { + return l.name +} + +func (l *loader[K, T]) filter(keys []K) []K { + if len(l.ids) == 0 { + return keys + } + + remaining := make([]K, 0, len(keys)) + for _, key := range keys { + if _, ok := l.ids[key]; ok { + continue + } + remaining = append(remaining, key) + } + return remaining +} + +func (l *loader[K, T]) updateMap(rows []T) { + for _, row := range rows { + key, id := l.mappingFromRow(row) + l.ids[key] = id + } +} + +func (l *loader[K, T]) insert(ctx context.Context, q *Q, keys []K) (int, error) { + keys = l.filter(keys) + if len(keys) == 0 { + return 0, nil + } + + var rows []T + err := bulkInsert( + ctx, + q, + l.table, + l.columnsForKeys(keys), + &rows, + ) + if err != nil { + return 0, err + } + + l.updateMap(rows) + return len(rows), nil +} + +func (l *loader[K, T]) query(ctx context.Context, q *Q, keys []K, lockRows bool) (int, error) { + keys = l.filter(keys) + if len(keys) == 0 { + return 0, nil + } + var suffix string + if lockRows { + suffix = "ORDER BY id ASC FOR KEY SHARE" + } + + var rows []T + err := bulkGet( + ctx, + q, + l.table, + l.columnsForKeys(keys), + &rows, + suffix, + ) + if err != nil { + return 0, err + } + + l.updateMap(rows) + return len(rows), nil +} + +type columnValues struct { + name string + dbType string + objects []string +} + +func bulkInsert(ctx context.Context, q *Q, table string, fields []columnValues, response interface{}) error { + unnestPart := make([]string, 0, len(fields)) + insertFieldsPart := make([]string, 0, len(fields)) + pqArrays := make([]interface{}, 0, len(fields)) + + // In the code below we are building the bulk insert query which looks like: + // + // WITH rows AS + // (SELECT + // /* unnestPart */ + // unnest(?::type1[]), /* field1 */ + // unnest(?::type2[]), /* field2 */ + // ... + // ) + // INSERT INTO table ( + // /* insertFieldsPart */ + // field1, + // field2, + // ... + // ) + // SELECT * FROM rows ON CONFLICT (field1, field2, ...) DO NOTHING RETURNING * + // + // Using unnest allows to get around the maximum limit of 65,535 query parameters, + // see https://www.postgresql.org/docs/12/limits.html and + // https://klotzandrew.com/blog/postgres-passing-65535-parameter-limit/ + // + // Without using unnest we would have to use multiple insert statements to insert + // all the rows for large datasets. + for _, field := range fields { + unnestPart = append( + unnestPart, + fmt.Sprintf("unnest(?::%s[]) /* %s */", field.dbType, field.name), + ) + insertFieldsPart = append( + insertFieldsPart, + field.name, + ) + pqArrays = append( + pqArrays, + pq.Array(field.objects), + ) + } + columns := strings.Join(insertFieldsPart, ",") + + sql := ` + WITH rows AS + (SELECT ` + strings.Join(unnestPart, ",") + `) + INSERT INTO ` + table + ` + (` + columns + `) + SELECT * FROM rows + ON CONFLICT (` + columns + `) DO NOTHING + RETURNING *` + + return q.SelectRaw( + ctx, + response, + sql, + pqArrays..., + ) +} + +func bulkGet(ctx context.Context, q *Q, table string, fields []columnValues, response interface{}, suffix string) error { + unnestPart := make([]string, 0, len(fields)) + columns := make([]string, 0, len(fields)) + pqArrays := make([]interface{}, 0, len(fields)) + + // In the code below we are building the bulk get query which looks like: + // + // SELECT * FROM table WHERE (field1, field2, ...) IN + // (SELECT + // /* unnestPart */ + // unnest(?::type1[]), /* field1 */ + // unnest(?::type2[]), /* field2 */ + // ... + // ) + // + // Using unnest allows to get around the maximum limit of 65,535 query parameters, + // see https://www.postgresql.org/docs/12/limits.html and + // https://klotzandrew.com/blog/postgres-passing-65535-parameter-limit/ + // + // Without using unnest we would have to use multiple select statements to obtain + // all the rows for large datasets. + for _, field := range fields { + unnestPart = append( + unnestPart, + fmt.Sprintf("unnest(?::%s[]) /* %s */", field.dbType, field.name), + ) + columns = append( + columns, + field.name, + ) + pqArrays = append( + pqArrays, + pq.Array(field.objects), + ) + } + sql := `SELECT * FROM ` + table + ` WHERE (` + strings.Join(columns, ",") + `) IN + (SELECT ` + strings.Join(unnestPart, ",") + `) ` + suffix + + return q.SelectRaw( + ctx, + response, + sql, + pqArrays..., + ) +} diff --git a/services/horizon/internal/db2/history/loader_concurrency_test.go b/services/horizon/internal/db2/history/loader_concurrency_test.go new file mode 100644 index 0000000000..e87d901bd8 --- /dev/null +++ b/services/horizon/internal/db2/history/loader_concurrency_test.go @@ -0,0 +1,197 @@ +package history + +import ( + "context" + "database/sql" + "fmt" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/stellar/go/keypair" + "github.com/stellar/go/services/horizon/internal/test" +) + +func TestLoaderConcurrentInserts(t *testing.T) { + tt := test.Start(t) + defer tt.Finish() + test.ResetHorizonDB(t, tt.HorizonDB) + s1 := tt.HorizonSession() + s2 := s1.Clone() + + for _, testCase := range []struct { + mode ConcurrencyMode + pass bool + }{ + {ConcurrentInserts, true}, + {ConcurrentDeletes, false}, + } { + t.Run(fmt.Sprintf("%v", testCase.mode), func(t *testing.T) { + var addresses []string + for i := 0; i < 10; i++ { + addresses = append(addresses, keypair.MustRandom().Address()) + } + + l1 := NewAccountLoader(testCase.mode) + for _, address := range addresses { + l1.GetFuture(address) + } + + for i := 0; i < 5; i++ { + addresses = append(addresses, keypair.MustRandom().Address()) + } + + l2 := NewAccountLoader(testCase.mode) + for _, address := range addresses { + l2.GetFuture(address) + } + + assert.NoError(t, s1.Begin(context.Background())) + assert.NoError(t, l1.Exec(context.Background(), s1)) + + assert.NoError(t, s2.Begin(context.Background())) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + <-time.After(time.Second * 3) + assert.NoError(t, s1.Commit()) + }() + // l2.Exec(context.Background(), s2) will block until s1 + // is committed because s1 and s2 both attempt to insert common + // accounts and, since s1 executed first, s2 must wait until + // s1 terminates. + assert.NoError(t, l2.Exec(context.Background(), s2)) + assert.NoError(t, s2.Commit()) + wg.Wait() + + assert.Equal(t, LoaderStats{ + Total: 10, + Inserted: 10, + }, l1.Stats()) + + if testCase.pass { + assert.Equal(t, LoaderStats{ + Total: 15, + Inserted: 5, + }, l2.Stats()) + } else { + assert.Equal(t, LoaderStats{ + Total: 5, + Inserted: 5, + }, l2.Stats()) + return + } + + q := &Q{s1} + for _, address := range addresses[:10] { + l1Id, err := l1.GetNow(address) + assert.NoError(t, err) + + l2Id, err := l2.GetNow(address) + assert.NoError(t, err) + assert.Equal(t, l1Id, l2Id) + + var account Account + assert.NoError(t, q.AccountByAddress(context.Background(), &account, address)) + assert.Equal(t, account.ID, l1Id) + assert.Equal(t, account.Address, address) + } + + for _, address := range addresses[10:] { + l2Id, err := l2.GetNow(address) + assert.NoError(t, err) + + _, err = l1.GetNow(address) + assert.ErrorContains(t, err, "was not found") + + var account Account + assert.NoError(t, q.AccountByAddress(context.Background(), &account, address)) + assert.Equal(t, account.ID, l2Id) + assert.Equal(t, account.Address, address) + } + }) + } +} + +func TestLoaderConcurrentDeletes(t *testing.T) { + tt := test.Start(t) + defer tt.Finish() + test.ResetHorizonDB(t, tt.HorizonDB) + s1 := tt.HorizonSession() + s2 := s1.Clone() + + for _, testCase := range []struct { + mode ConcurrencyMode + pass bool + }{ + {ConcurrentInserts, false}, + {ConcurrentDeletes, true}, + } { + t.Run(fmt.Sprintf("%v", testCase.mode), func(t *testing.T) { + var addresses []string + for i := 0; i < 10; i++ { + addresses = append(addresses, keypair.MustRandom().Address()) + } + + loader := NewAccountLoader(testCase.mode) + for _, address := range addresses { + loader.GetFuture(address) + } + assert.NoError(t, loader.Exec(context.Background(), s1)) + + var ids []int64 + for _, address := range addresses { + id, err := loader.GetNow(address) + assert.NoError(t, err) + ids = append(ids, id) + } + + loader = NewAccountLoader(testCase.mode) + for _, address := range addresses { + loader.GetFuture(address) + } + + assert.NoError(t, s1.Begin(context.Background())) + assert.NoError(t, loader.Exec(context.Background(), s1)) + + assert.NoError(t, s2.Begin(context.Background())) + q2 := &Q{s2} + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + <-time.After(time.Second * 3) + + q1 := &Q{s1} + for _, address := range addresses { + id, err := loader.GetNow(address) + assert.NoError(t, err) + + var account Account + err = q1.AccountByAddress(context.Background(), &account, address) + if testCase.pass { + assert.NoError(t, err) + assert.Equal(t, account.ID, id) + assert.Equal(t, account.Address, address) + } else { + assert.ErrorContains(t, err, sql.ErrNoRows.Error()) + } + } + assert.NoError(t, s1.Commit()) + }() + + // the reaper should block until s1 has been committed because s1 has locked + // the orphaned rows + deletedCount, err := q2.reapLookupTable(context.Background(), "history_accounts", ids, 1000) + assert.NoError(t, err) + assert.Equal(t, int64(len(addresses)), deletedCount) + assert.NoError(t, s2.Commit()) + + wg.Wait() + }) + } +} diff --git a/services/horizon/internal/db2/history/main.go b/services/horizon/internal/db2/history/main.go index f49d1129c5..7bb4516ff0 100644 --- a/services/horizon/internal/db2/history/main.go +++ b/services/horizon/internal/db2/history/main.go @@ -9,6 +9,7 @@ import ( "database/sql/driver" "encoding/json" "fmt" + "strconv" "strings" "sync" "time" @@ -282,7 +283,8 @@ type IngestionQ interface { NewTradeBatchInsertBuilder() TradeBatchInsertBuilder RebuildTradeAggregationTimes(ctx context.Context, from, to strtime.Millis, roundingSlippageFilter int) error RebuildTradeAggregationBuckets(ctx context.Context, fromLedger, toLedger uint32, roundingSlippageFilter int) error - ReapLookupTables(ctx context.Context, batchSize int) (map[string]LookupTableReapResult, error) + ReapLookupTable(ctx context.Context, table string, ids []int64, newOffset int64) (int64, error) + FindLookupTableRowsToReap(ctx context.Context, table string, batchSize int) ([]int64, int64, error) CreateAssets(ctx context.Context, assets []xdr.Asset, batchSize int) (map[string]Asset, error) QTransactions QTrustLines @@ -307,6 +309,7 @@ type IngestionQ interface { GetNextLedgerSequence(context.Context, uint32) (uint32, bool, error) TryStateVerificationLock(context.Context) (bool, error) TryReaperLock(context.Context) (bool, error) + TryLookupTableReaperLock(ctx context.Context) (bool, error) ElderLedger(context.Context, interface{}) error } @@ -783,6 +786,7 @@ type OperationsQ struct { opIdCol string includeFailed bool includeTransactions bool + boundedIdQuery bool } // Q is a helper struct on which to hang common_trades queries against a history @@ -871,11 +875,12 @@ func (t *Transaction) HasPreconditions() bool { // TransactionsQ is a helper struct to aid in configuring queries that loads // slices of transaction structs. type TransactionsQ struct { - Err error - parent *Q - sql sq.SelectBuilder - includeFailed bool - txIdCol string + Err error + parent *Q + sql sq.SelectBuilder + includeFailed bool + txIdCol string + boundedIdQuery bool } // TrustLine is row of data from the `trust_lines` table from horizon DB @@ -973,63 +978,70 @@ type LookupTableReapResult struct { Duration time.Duration } -// ReapLookupTables removes rows from lookup tables like history_claimable_balances -// which aren't used (orphaned), i.e. history entries for them were reaped. -// This method must be executed inside ingestion transaction. Otherwise it may -// create invalid state in lookup and history tables. -func (q *Q) ReapLookupTables(ctx context.Context, batchSize int) ( - map[string]LookupTableReapResult, - error, -) { - if q.GetTx() == nil { - return nil, errors.New("cannot be called outside of an ingestion transaction") +func (q *Q) FindLookupTableRowsToReap(ctx context.Context, table string, batchSize int) ([]int64, int64, error) { + offset, err := q.getLookupTableReapOffset(ctx, table) + if err != nil { + return nil, 0, fmt.Errorf("could not obtain offsets: %w", err) } - offsets, err := q.getLookupTableReapOffsets(ctx) + // Find new offset before removing the rows + var newOffset int64 + err = q.GetRaw( + ctx, + &newOffset, + fmt.Sprintf( + "SELECT id FROM %s WHERE id >= %d ORDER BY id ASC LIMIT 1 OFFSET %d", + table, offset, batchSize, + ), + ) if err != nil { - return nil, fmt.Errorf("could not obtain offsets: %w", err) + if q.NoRows(err) { + newOffset = 0 + } else { + return nil, 0, err + } } - results := map[string]LookupTableReapResult{} - for table, historyTables := range historyLookupTables { - startTime := time.Now() - query := constructReapLookupTablesQuery(table, historyTables, batchSize, offsets[table]) + var ids []int64 + err = q.SelectRaw(ctx, &ids, constructFindReapLookupTablesQuery(table, batchSize, offset)) + if err != nil { + return nil, 0, fmt.Errorf("could not query orphaned rows: %w", err) + } - // Find new offset before removing the rows - var newOffset int64 - err := q.GetRaw(ctx, &newOffset, fmt.Sprintf("SELECT id FROM %s where id >= %d limit 1 offset %d", table, offsets[table], batchSize)) - if err != nil { - if q.NoRows(err) { - newOffset = 0 - } else { - return nil, err - } - } + return ids, newOffset, nil +} - res, err := q.ExecRaw( - context.WithValue(ctx, &db.QueryTypeContextKey, db.DeleteQueryType), - query, - ) - if err != nil { - return nil, errors.Wrapf(err, "error running query: %s", query) - } +func (q *Q) ReapLookupTable(ctx context.Context, table string, ids []int64, newOffset int64) (int64, error) { + if err := q.Begin(ctx); err != nil { + return 0, fmt.Errorf("could not start transaction: %w", err) + } + defer q.Rollback() - if err = q.updateLookupTableReapOffset(ctx, table, newOffset); err != nil { - return nil, fmt.Errorf("error updating offset: %w", err) - } + rowsDeleted, err := q.reapLookupTable(ctx, table, ids, newOffset) + if err != nil { + return 0, err + } - rows, err := res.RowsAffected() - if err != nil { - return nil, errors.Wrapf(err, "error running RowsAffected after query: %s", query) - } + if err := q.Commit(); err != nil { + return 0, fmt.Errorf("could not commit transaction: %w", err) + } + return rowsDeleted, nil +} + +func (q *Q) reapLookupTable(ctx context.Context, table string, ids []int64, newOffset int64) (int64, error) { + if err := q.updateLookupTableReapOffset(ctx, table, newOffset); err != nil { + return 0, fmt.Errorf("error updating offset for table %s: %w ", table, err) + } - results[table] = LookupTableReapResult{ - Offset: newOffset, - RowsDeleted: rows, - Duration: time.Since(startTime), + var rowsDeleted int64 + if len(ids) > 0 { + var err error + rowsDeleted, err = q.deleteLookupTableRows(ctx, table, ids) + if err != nil { + return 0, fmt.Errorf("could not delete orphaned rows: %w", err) } } - return results, nil + return rowsDeleted, nil } var historyLookupTables = map[string][]tableObjectFieldPair{ @@ -1096,54 +1108,100 @@ var historyLookupTables = map[string][]tableObjectFieldPair{ }, } -// constructReapLookupTablesQuery creates a query like (using history_claimable_balances +func (q *Q) deleteLookupTableRows(ctx context.Context, table string, ids []int64) (int64, error) { + deleteQuery := constructDeleteLookupTableRowsQuery(table, ids) + result, err := q.ExecRaw( + context.WithValue(ctx, &db.QueryTypeContextKey, db.DeleteQueryType), + deleteQuery, + ) + if err != nil { + return 0, fmt.Errorf("error running query %s : %w", deleteQuery, err) + } + var deletedCount int64 + deletedCount, err = result.RowsAffected() + if err != nil { + return 0, fmt.Errorf("error getting deleted count: %w", err) + } + return deletedCount, nil +} + +// constructDeleteLookupTableRowsQuery creates a query like (using history_claimable_balances // as an example): // -// delete from history_claimable_balances where id in ( -// -// WITH ha_batch AS ( -// SELECT id -// FROM history_claimable_balances -// WHERE id >= 1000 -// ORDER BY id limit 1000 -// ) SELECT e1.id as id FROM ha_batch e1 +// WITH ha_batch AS ( +// SELECT id +// FROM history_claimable_balances +// WHERE IN ($1, $2, ...) ORDER BY id asc FOR UPDATE +// ) DELETE FROM history_claimable_balances WHERE id IN ( +// SELECT e1.id as id FROM ha_batch e1 // WHERE NOT EXISTS (SELECT 1 FROM history_transaction_claimable_balances WHERE history_transaction_claimable_balances.history_claimable_balance_id = id limit 1) // AND NOT EXISTS (SELECT 1 FROM history_operation_claimable_balances WHERE history_operation_claimable_balances.history_claimable_balance_id = id limit 1) // ) // -// In short it checks the 1000 rows omitting 1000 row of history_claimable_balances +// It checks each of the candidate rows provided in the top level IN clause // and counts occurrences of each row in corresponding history tables. // If there are no history rows for a given id, the row in // history_claimable_balances is removed. // -// The offset param should be increased before each execution. Given that -// some rows will be removed and some will be added by ingestion it's -// possible that rows will be skipped from deletion. But offset is reset -// when it reaches the table size so eventually all orphaned rows are -// deleted. -func constructReapLookupTablesQuery(table string, historyTables []tableObjectFieldPair, batchSize int, offset int64) string { +// Note that the rows are locked using via SELECT FOR UPDATE. The reason +// for that is to maintain safety when ingestion is running concurrently. +// The ingestion loaders will also lock rows from the history lookup tables +// via SELECT FOR KEY SHARE. This will ensure that the reaping transaction +// will block until the ingestion transaction commits (or vice-versa). +func constructDeleteLookupTableRowsQuery(table string, ids []int64) string { + var conditions []string + for _, referencedTable := range historyLookupTables[table] { + conditions = append( + conditions, + fmt.Sprintf( + "NOT EXISTS ( SELECT 1 as row FROM %s WHERE %s.%s = id LIMIT 1)", + referencedTable.name, + referencedTable.name, referencedTable.objectField, + ), + ) + } + + stringIds := make([]string, len(ids)) + for i, id := range ids { + stringIds[i] = strconv.FormatInt(id, 10) + } + innerQuery := fmt.Sprintf( + "SELECT id FROM %s WHERE id IN (%s) ORDER BY id asc FOR UPDATE", + table, + strings.Join(stringIds, ", "), + ) + + deleteQuery := fmt.Sprintf( + "WITH ha_batch AS (%s) DELETE FROM %s WHERE id IN ("+ + "SELECT e1.id as id FROM ha_batch e1 WHERE %s)", + innerQuery, + table, + strings.Join(conditions, " AND "), + ) + return deleteQuery +} + +func constructFindReapLookupTablesQuery(table string, batchSize int, offset int64) string { var conditions []string - for _, historyTable := range historyTables { + for _, referencedTable := range historyLookupTables[table] { conditions = append( conditions, fmt.Sprintf( "NOT EXISTS ( SELECT 1 as row FROM %s WHERE %s.%s = id LIMIT 1)", - historyTable.name, - historyTable.name, historyTable.objectField, + referencedTable.name, + referencedTable.name, referencedTable.objectField, ), ) } return fmt.Sprintf( - "DELETE FROM %s WHERE id IN ("+ - "WITH ha_batch AS (SELECT id FROM %s WHERE id >= %d ORDER BY id limit %d) "+ + "WITH ha_batch AS (SELECT id FROM %s WHERE id >= %d ORDER BY id ASC limit %d) "+ "SELECT e1.id as id FROM ha_batch e1 WHERE ", table, - table, offset, batchSize, - ) + strings.Join(conditions, " AND ") + ")" + ) + strings.Join(conditions, " AND ") } // DeleteRangeAll deletes a range of rows from all history tables between diff --git a/services/horizon/internal/db2/history/main_test.go b/services/horizon/internal/db2/history/main_test.go index 1a28b9e584..a86f4c14a4 100644 --- a/services/horizon/internal/db2/history/main_test.go +++ b/services/horizon/internal/db2/history/main_test.go @@ -69,20 +69,34 @@ func TestElderLedger(t *testing.T) { } } +func TestConstructDeleteLookupTableRowsQuery(t *testing.T) { + query := constructDeleteLookupTableRowsQuery( + "history_accounts", + []int64{100, 20, 30}, + ) + + assert.Equal(t, + "WITH ha_batch AS (SELECT id FROM history_accounts WHERE id IN (100, 20, 30) ORDER BY id asc FOR UPDATE) "+ + "DELETE FROM history_accounts WHERE id IN (SELECT e1.id as id FROM ha_batch e1 "+ + "WHERE NOT EXISTS ( SELECT 1 as row FROM history_transaction_participants WHERE history_transaction_participants.history_account_id = id LIMIT 1) "+ + "AND NOT EXISTS ( SELECT 1 as row FROM history_effects WHERE history_effects.history_account_id = id LIMIT 1) "+ + "AND NOT EXISTS ( SELECT 1 as row FROM history_operation_participants WHERE history_operation_participants.history_account_id = id LIMIT 1) "+ + "AND NOT EXISTS ( SELECT 1 as row FROM history_trades WHERE history_trades.base_account_id = id LIMIT 1) "+ + "AND NOT EXISTS ( SELECT 1 as row FROM history_trades WHERE history_trades.counter_account_id = id LIMIT 1))", query) +} + func TestConstructReapLookupTablesQuery(t *testing.T) { - query := constructReapLookupTablesQuery( + query := constructFindReapLookupTablesQuery( "history_accounts", - historyLookupTables["history_accounts"], 10, 0, ) assert.Equal(t, - "DELETE FROM history_accounts WHERE id IN ("+ - "WITH ha_batch AS (SELECT id FROM history_accounts WHERE id >= 0 ORDER BY id limit 10) SELECT e1.id as id FROM ha_batch e1 "+ + "WITH ha_batch AS (SELECT id FROM history_accounts WHERE id >= 0 ORDER BY id ASC limit 10) SELECT e1.id as id FROM ha_batch e1 "+ "WHERE NOT EXISTS ( SELECT 1 as row FROM history_transaction_participants WHERE history_transaction_participants.history_account_id = id LIMIT 1) "+ "AND NOT EXISTS ( SELECT 1 as row FROM history_effects WHERE history_effects.history_account_id = id LIMIT 1) "+ "AND NOT EXISTS ( SELECT 1 as row FROM history_operation_participants WHERE history_operation_participants.history_account_id = id LIMIT 1) "+ "AND NOT EXISTS ( SELECT 1 as row FROM history_trades WHERE history_trades.base_account_id = id LIMIT 1) "+ - "AND NOT EXISTS ( SELECT 1 as row FROM history_trades WHERE history_trades.counter_account_id = id LIMIT 1))", query) + "AND NOT EXISTS ( SELECT 1 as row FROM history_trades WHERE history_trades.counter_account_id = id LIMIT 1)", query) } diff --git a/services/horizon/internal/db2/history/operation.go b/services/horizon/internal/db2/history/operation.go index 04a6d00f50..02a72add6f 100644 --- a/services/horizon/internal/db2/history/operation.go +++ b/services/horizon/internal/db2/history/operation.go @@ -210,6 +210,7 @@ func (q *OperationsQ) ForLedger(ctx context.Context, seq int32) *OperationsQ { start.ToInt64(), end.ToInt64(), ) + q.boundedIdQuery = true return q } @@ -231,6 +232,7 @@ func (q *OperationsQ) ForTransaction(ctx context.Context, hash string) *Operatio start.ToInt64(), end.ToInt64(), ) + q.boundedIdQuery = true return q } @@ -266,11 +268,15 @@ func (q *OperationsQ) IncludeTransactions() *OperationsQ { } // Page specifies the paging constraints for the query being built by `q`. -func (q *OperationsQ) Page(page db2.PageQuery) *OperationsQ { +func (q *OperationsQ) Page(page db2.PageQuery, oldestLedger int32) *OperationsQ { if q.Err != nil { return q } + if lowerBound := lowestLedgerBound(oldestLedger); !q.boundedIdQuery && lowerBound > 0 && page.Order == "desc" { + q.sql = q.sql. + Where(q.opIdCol+" > ?", lowerBound) + } q.sql, q.Err = page.ApplyTo(q.sql, q.opIdCol) return q } diff --git a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go index 7e823064f2..eb30fe6659 100644 --- a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go +++ b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go @@ -15,7 +15,7 @@ func TestAddOperationParticipants(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - accountLoader := NewAccountLoader() + accountLoader := NewAccountLoader(ConcurrentInserts) address := keypair.MustRandom().Address() tt.Assert.NoError(q.Begin(tt.Ctx)) builder := q.NewOperationParticipantBatchInsertBuilder() diff --git a/services/horizon/internal/db2/history/operation_test.go b/services/horizon/internal/db2/history/operation_test.go index 1d20a9cb10..341751347a 100644 --- a/services/horizon/internal/db2/history/operation_test.go +++ b/services/horizon/internal/db2/history/operation_test.go @@ -125,7 +125,7 @@ func TestOperationByLiquidityPool(t *testing.T) { // Insert Liquidity Pool history liquidityPoolID := "a2f38836a839de008cf1d782c81f45e1253cc5d3dad9110b872965484fec0a49" - lpLoader := NewLiquidityPoolLoader() + lpLoader := NewLiquidityPoolLoader(ConcurrentInserts) lpOperationBuilder := q.NewOperationLiquidityPoolBatchInsertBuilder() tt.Assert.NoError(lpOperationBuilder.Add(opID1, lpLoader.GetFuture(liquidityPoolID))) @@ -141,7 +141,7 @@ func TestOperationByLiquidityPool(t *testing.T) { Order: "asc", Limit: 2, } - ops, _, err := q.Operations().ForLiquidityPool(tt.Ctx, liquidityPoolID).Page(pq).Fetch(tt.Ctx) + ops, _, err := q.Operations().ForLiquidityPool(tt.Ctx, liquidityPoolID).Page(pq, 0).Fetch(tt.Ctx) tt.Assert.NoError(err) tt.Assert.Len(ops, 2) tt.Assert.Equal(ops[0].ID, opID1) @@ -149,7 +149,7 @@ func TestOperationByLiquidityPool(t *testing.T) { // Check descending order pq.Order = "desc" - ops, _, err = q.Operations().ForLiquidityPool(tt.Ctx, liquidityPoolID).Page(pq).Fetch(tt.Ctx) + ops, _, err = q.Operations().ForLiquidityPool(tt.Ctx, liquidityPoolID).Page(pq, 0).Fetch(tt.Ctx) tt.Assert.NoError(err) tt.Assert.Len(ops, 2) tt.Assert.Equal(ops[0].ID, opID2) @@ -162,23 +162,119 @@ func TestOperationQueryBuilder(t *testing.T) { defer tt.Finish() q := &Q{tt.HorizonSession()} - opsQ := q.Operations().ForAccount(tt.Ctx, "GBXGQJWVLWOYHFLVTKWV5FGHA3LNYY2JQKM7OAJAUEQFU6LPCSEFVXON").Page(db2.PageQuery{Cursor: "8589938689", Order: "asc", Limit: 10}) - tt.Assert.NoError(opsQ.Err) - got, _, err := opsQ.sql.ToSql() - tt.Assert.NoError(err) - - // Operations for account queries will use hopp.history_operation_id in their predicates. - want := "SELECT hop.id, hop.transaction_id, hop.application_order, hop.type, hop.details, hop.source_account, hop.source_account_muxed, COALESCE(hop.is_payment, false) as is_payment, ht.transaction_hash, ht.tx_result, COALESCE(ht.successful, true) as transaction_successful FROM history_operations hop LEFT JOIN history_transactions ht ON ht.id = hop.transaction_id JOIN history_operation_participants hopp ON hopp.history_operation_id = hop.id WHERE hopp.history_account_id = ? AND hopp.history_operation_id > ? ORDER BY hopp.history_operation_id asc LIMIT 10" - tt.Assert.EqualValues(want, got) - - opsQ = q.Operations().ForLedger(tt.Ctx, 2).Page(db2.PageQuery{Cursor: "8589938689", Order: "asc", Limit: 10}) - tt.Assert.NoError(opsQ.Err) - got, _, err = opsQ.sql.ToSql() - tt.Assert.NoError(err) - - // Other operation queries will use hop.id in their predicates. - want = "SELECT hop.id, hop.transaction_id, hop.application_order, hop.type, hop.details, hop.source_account, hop.source_account_muxed, COALESCE(hop.is_payment, false) as is_payment, ht.transaction_hash, ht.tx_result, COALESCE(ht.successful, true) as transaction_successful FROM history_operations hop LEFT JOIN history_transactions ht ON ht.id = hop.transaction_id WHERE hop.id >= ? AND hop.id < ? AND hop.id > ? ORDER BY hop.id asc LIMIT 10" - tt.Assert.EqualValues(want, got) + for _, testCase := range []struct { + q *OperationsQ + expectedSQL string + expectedArgs []interface{} + }{ + { + q.Operations().ForAccount(tt.Ctx, "GBXGQJWVLWOYHFLVTKWV5FGHA3LNYY2JQKM7OAJAUEQFU6LPCSEFVXON"). + Page(db2.PageQuery{Cursor: "8589938689", Order: "asc", Limit: 10}, 50), + "SELECT " + + "hop.id, hop.transaction_id, hop.application_order, hop.type, hop.details, hop.source_account, " + + "hop.source_account_muxed, COALESCE(hop.is_payment, false) as is_payment, ht.transaction_hash, " + + "ht.tx_result, COALESCE(ht.successful, true) as transaction_successful " + + "FROM history_operations hop " + + "LEFT JOIN history_transactions ht ON ht.id = hop.transaction_id " + + "JOIN history_operation_participants hopp ON hopp.history_operation_id = hop.id " + + "WHERE hopp.history_account_id = ? AND " + + "hopp.history_operation_id > ? " + + "ORDER BY hopp.history_operation_id asc LIMIT 10", + []interface{}{ + int64(2), + int64(8589938689), + }, + }, + { + q.Operations().ForAccount(tt.Ctx, "GBXGQJWVLWOYHFLVTKWV5FGHA3LNYY2JQKM7OAJAUEQFU6LPCSEFVXON"). + Page(db2.PageQuery{Cursor: "8589938689", Order: "desc", Limit: 10}, 50), + "SELECT " + + "hop.id, hop.transaction_id, hop.application_order, hop.type, hop.details, hop.source_account, " + + "hop.source_account_muxed, COALESCE(hop.is_payment, false) as is_payment, ht.transaction_hash, " + + "ht.tx_result, COALESCE(ht.successful, true) as transaction_successful " + + "FROM history_operations hop " + + "LEFT JOIN history_transactions ht ON ht.id = hop.transaction_id " + + "JOIN history_operation_participants hopp ON hopp.history_operation_id = hop.id " + + "WHERE hopp.history_account_id = ? AND " + + "hopp.history_operation_id > ? AND " + + "hopp.history_operation_id < ? " + + "ORDER BY hopp.history_operation_id desc LIMIT 10", + []interface{}{ + int64(2), + int64(214748364799), + int64(8589938689), + }, + }, + { + q.Operations().ForAccount(tt.Ctx, "GBXGQJWVLWOYHFLVTKWV5FGHA3LNYY2JQKM7OAJAUEQFU6LPCSEFVXON"). + Page(db2.PageQuery{Cursor: "8589938689", Order: "desc", Limit: 10}, 0), + "SELECT " + + "hop.id, hop.transaction_id, hop.application_order, hop.type, hop.details, hop.source_account, " + + "hop.source_account_muxed, COALESCE(hop.is_payment, false) as is_payment, ht.transaction_hash, " + + "ht.tx_result, COALESCE(ht.successful, true) as transaction_successful " + + "FROM history_operations hop " + + "LEFT JOIN history_transactions ht ON ht.id = hop.transaction_id " + + "JOIN history_operation_participants hopp ON hopp.history_operation_id = hop.id " + + "WHERE hopp.history_account_id = ? AND " + + "hopp.history_operation_id < ? " + + "ORDER BY hopp.history_operation_id desc LIMIT 10", + []interface{}{ + int64(2), + int64(8589938689), + }, + }, + { + q.Operations().ForLedger(tt.Ctx, 2). + Page(db2.PageQuery{Cursor: "8589938689", Order: "asc", Limit: 10}, 50), + "SELECT " + + "hop.id, hop.transaction_id, hop.application_order, hop.type, hop.details, hop.source_account, " + + "hop.source_account_muxed, COALESCE(hop.is_payment, false) as is_payment, ht.transaction_hash, " + + "ht.tx_result, COALESCE(ht.successful, true) as transaction_successful FROM history_operations " + + "hop LEFT JOIN history_transactions ht ON ht.id = hop.transaction_id " + + "WHERE hop.id >= ? AND hop.id < ? AND hop.id > ? ORDER BY hop.id asc LIMIT 10", + []interface{}{ + int64(8589934592), + int64(12884901888), + int64(8589938689), + }, + }, + { + q.Operations().ForLedger(tt.Ctx, 2). + Page(db2.PageQuery{Cursor: "8589938689", Order: "desc", Limit: 10}, 50), + "SELECT " + + "hop.id, hop.transaction_id, hop.application_order, hop.type, hop.details, hop.source_account, " + + "hop.source_account_muxed, COALESCE(hop.is_payment, false) as is_payment, ht.transaction_hash, " + + "ht.tx_result, COALESCE(ht.successful, true) as transaction_successful FROM history_operations " + + "hop LEFT JOIN history_transactions ht ON ht.id = hop.transaction_id " + + "WHERE hop.id >= ? AND hop.id < ? AND hop.id < ? ORDER BY hop.id desc LIMIT 10", + []interface{}{ + int64(8589934592), + int64(12884901888), + int64(8589938689), + }, + }, + { + q.Operations().ForLedger(tt.Ctx, 2). + Page(db2.PageQuery{Cursor: "8589938689", Order: "desc", Limit: 10}, 0), + "SELECT " + + "hop.id, hop.transaction_id, hop.application_order, hop.type, hop.details, hop.source_account, " + + "hop.source_account_muxed, COALESCE(hop.is_payment, false) as is_payment, ht.transaction_hash, " + + "ht.tx_result, COALESCE(ht.successful, true) as transaction_successful FROM history_operations " + + "hop LEFT JOIN history_transactions ht ON ht.id = hop.transaction_id " + + "WHERE hop.id >= ? AND hop.id < ? AND hop.id < ? ORDER BY hop.id desc LIMIT 10", + []interface{}{ + int64(8589934592), + int64(12884901888), + int64(8589938689), + }, + }, + } { + tt.Assert.NoError(testCase.q.Err) + got, args, err := testCase.q.sql.ToSql() + tt.Assert.NoError(err) + tt.Assert.Equal(got, testCase.expectedSQL) + tt.Assert.Equal(args, testCase.expectedArgs) + } } // TestOperationSuccessfulOnly tests if default query returns operations in diff --git a/services/horizon/internal/db2/history/participants_test.go b/services/horizon/internal/db2/history/participants_test.go index 37f7654abb..15d09da0ac 100644 --- a/services/horizon/internal/db2/history/participants_test.go +++ b/services/horizon/internal/db2/history/participants_test.go @@ -35,7 +35,7 @@ func TestTransactionParticipantsBatch(t *testing.T) { q := &Q{tt.HorizonSession()} batch := q.NewTransactionParticipantsBatchInsertBuilder() - accountLoader := NewAccountLoader() + accountLoader := NewAccountLoader(ConcurrentInserts) transactionID := int64(1) otherTransactionID := int64(2) diff --git a/services/horizon/internal/db2/history/reap_test.go b/services/horizon/internal/db2/history/reap_test.go index 5601cd19b6..af20fbc976 100644 --- a/services/horizon/internal/db2/history/reap_test.go +++ b/services/horizon/internal/db2/history/reap_test.go @@ -1,13 +1,43 @@ package history_test import ( + "context" "testing" + "github.com/stretchr/testify/assert" + "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/services/horizon/internal/ingest" "github.com/stellar/go/services/horizon/internal/test" ) +type reapResult struct { + Offset int64 + RowsDeleted int64 +} + +func reapLookupTables(t *testing.T, q *history.Q, batchSize int) map[string]reapResult { + results := map[string]reapResult{} + + for _, table := range []string{ + "history_accounts", + "history_assets", + "history_claimable_balances", + "history_liquidity_pools", + } { + ids, offset, err := q.FindLookupTableRowsToReap(context.Background(), table, batchSize) + assert.NoError(t, err) + rowsDeleted, err := q.ReapLookupTable(context.Background(), table, ids, offset) + assert.NoError(t, err) + results[table] = reapResult{ + Offset: offset, + RowsDeleted: rowsDeleted, + } + } + + return results +} + func TestReapLookupTables(t *testing.T) { tt := test.Start(t) defer tt.Finish() @@ -46,14 +76,7 @@ func TestReapLookupTables(t *testing.T) { q := &history.Q{tt.HorizonSession()} - err = q.Begin(tt.Ctx) - tt.Require.NoError(err) - - results, err := q.ReapLookupTables(tt.Ctx, 5) - tt.Require.NoError(err) - - err = q.Commit() - tt.Require.NoError(err) + results := reapLookupTables(t, q, 5) err = db.GetRaw(tt.Ctx, &curLedgers, `SELECT COUNT(*) FROM history_ledgers`) tt.Require.NoError(err) @@ -91,14 +114,7 @@ func TestReapLookupTables(t *testing.T) { tt.Assert.Equal(int64(0), results["history_claimable_balances"].Offset) tt.Assert.Equal(int64(0), results["history_liquidity_pools"].Offset) - err = q.Begin(tt.Ctx) - tt.Require.NoError(err) - - results, err = q.ReapLookupTables(tt.Ctx, 5) - tt.Require.NoError(err) - - err = q.Commit() - tt.Require.NoError(err) + results = reapLookupTables(t, q, 5) err = db.GetRaw(tt.Ctx, &curAccounts, `SELECT COUNT(*) FROM history_accounts`) tt.Require.NoError(err) @@ -121,14 +137,7 @@ func TestReapLookupTables(t *testing.T) { tt.Assert.Equal(int64(0), results["history_claimable_balances"].Offset) tt.Assert.Equal(int64(0), results["history_liquidity_pools"].Offset) - err = q.Begin(tt.Ctx) - tt.Require.NoError(err) - - results, err = q.ReapLookupTables(tt.Ctx, 1000) - tt.Require.NoError(err) - - err = q.Commit() - tt.Require.NoError(err) + results = reapLookupTables(t, q, 1000) err = db.GetRaw(tt.Ctx, &curAccounts, `SELECT COUNT(*) FROM history_accounts`) tt.Require.NoError(err) diff --git a/services/horizon/internal/db2/history/trade.go b/services/horizon/internal/db2/history/trade.go index 6d8a7fca56..07ee5f6cfb 100644 --- a/services/horizon/internal/db2/history/trade.go +++ b/services/horizon/internal/db2/history/trade.go @@ -39,36 +39,36 @@ type tradesQuery struct { } func (q *Q) GetTrades( - ctx context.Context, page db2.PageQuery, account string, tradeType string, + ctx context.Context, page db2.PageQuery, oldestLedger int32, account string, tradeType string, ) ([]Trade, error) { - return q.getTrades(ctx, page, tradesQuery{ + return q.getTrades(ctx, page, oldestLedger, tradesQuery{ account: account, tradeType: tradeType, }) } func (q *Q) GetTradesForOffer( - ctx context.Context, page db2.PageQuery, offerID int64, + ctx context.Context, page db2.PageQuery, oldestLedger int32, offerID int64, ) ([]Trade, error) { - return q.getTrades(ctx, page, tradesQuery{ + return q.getTrades(ctx, page, oldestLedger, tradesQuery{ offer: offerID, tradeType: AllTrades, }) } func (q *Q) GetTradesForLiquidityPool( - ctx context.Context, page db2.PageQuery, poolID string, + ctx context.Context, page db2.PageQuery, oldestLedger int32, poolID string, ) ([]Trade, error) { - return q.getTrades(ctx, page, tradesQuery{ + return q.getTrades(ctx, page, oldestLedger, tradesQuery{ liquidityPool: poolID, tradeType: AllTrades, }) } func (q *Q) GetTradesForAssets( - ctx context.Context, page db2.PageQuery, account, tradeType string, baseAsset, counterAsset xdr.Asset, + ctx context.Context, page db2.PageQuery, oldestLedger int32, account, tradeType string, baseAsset, counterAsset xdr.Asset, ) ([]Trade, error) { - return q.getTrades(ctx, page, tradesQuery{ + return q.getTrades(ctx, page, oldestLedger, tradesQuery{ account: account, baseAsset: &baseAsset, counterAsset: &counterAsset, @@ -86,7 +86,7 @@ type historyTradesQuery struct { tradeType string } -func (q *Q) getTrades(ctx context.Context, page db2.PageQuery, query tradesQuery) ([]Trade, error) { +func (q *Q) getTrades(ctx context.Context, page db2.PageQuery, oldestLedger int32, query tradesQuery) ([]Trade, error) { // Add explicit query type for prometheus metrics, since we use raw sql. ctx = context.WithValue(ctx, &db.QueryTypeContextKey, db.SelectQueryType) @@ -94,7 +94,7 @@ func (q *Q) getTrades(ctx context.Context, page db2.PageQuery, query tradesQuery if err != nil { return nil, errors.Wrap(err, "invalid trade query") } - rawSQL, args, err := createTradesSQL(page, internalTradesQuery) + rawSQL, args, err := createTradesSQL(page, oldestLedger, internalTradesQuery) if err != nil { return nil, errors.Wrap(err, "could not create trades sql query") } @@ -149,7 +149,7 @@ func (q *Q) transformTradesQuery(ctx context.Context, query tradesQuery) (histor return internalQuery, nil } -func createTradesSQL(page db2.PageQuery, query historyTradesQuery) (string, []interface{}, error) { +func createTradesSQL(page db2.PageQuery, oldestLedger int32, query historyTradesQuery) (string, []interface{}, error) { base := selectTradeFields if !query.orderPreserved { base = selectReverseTradeFields @@ -204,8 +204,8 @@ func createTradesSQL(page db2.PageQuery, query historyTradesQuery) (string, []in secondSelect = sql.Where("htrd.counter_liquidity_pool_id = ?", query.poolID) } - firstSelect = appendOrdering(firstSelect, op, idx, page.Order) - secondSelect = appendOrdering(secondSelect, op, idx, page.Order) + firstSelect = appendOrdering(firstSelect, oldestLedger, op, idx, page.Order) + secondSelect = appendOrdering(secondSelect, oldestLedger, op, idx, page.Order) firstSQL, firstArgs, err := firstSelect.ToSql() if err != nil { return "", nil, errors.Wrap(err, "error building a firstSelect query") @@ -229,7 +229,7 @@ func createTradesSQL(page db2.PageQuery, query historyTradesQuery) (string, []in rawSQL = rawSQL + fmt.Sprintf("LIMIT %d", page.Limit) return rawSQL, args, nil } else { - sql = appendOrdering(sql, op, idx, page.Order) + sql = appendOrdering(sql, oldestLedger, op, idx, page.Order) sql = sql.Limit(page.Limit) rawSQL, args, err := sql.ToSql() if err != nil { @@ -239,7 +239,7 @@ func createTradesSQL(page db2.PageQuery, query historyTradesQuery) (string, []in } } -func appendOrdering(sel sq.SelectBuilder, op, idx int64, order string) sq.SelectBuilder { +func appendOrdering(sel sq.SelectBuilder, oldestLedger int32, op, idx int64, order string) sq.SelectBuilder { // NOTE: Remember to test the queries below with EXPLAIN / EXPLAIN ANALYZE // before changing them. // This condition is using multicolumn index and it's easy to write it in a way that @@ -255,6 +255,9 @@ func appendOrdering(sel sq.SelectBuilder, op, idx int64, order string) sq.Select ))`, op, op, op, idx). OrderBy("htrd.history_operation_id asc, htrd.order asc") case "desc": + if lowerBound := lowestLedgerBound(oldestLedger); lowerBound > 0 { + sel = sel.Where("htrd.history_operation_id > ?", lowerBound) + } return sel. Where(`( htrd.history_operation_id <= ? diff --git a/services/horizon/internal/db2/history/trade_test.go b/services/horizon/internal/db2/history/trade_test.go index e91a1f7b4f..157142e296 100644 --- a/services/horizon/internal/db2/history/trade_test.go +++ b/services/horizon/internal/db2/history/trade_test.go @@ -1,9 +1,11 @@ package history import ( - "github.com/stellar/go/xdr" "testing" + "github.com/stellar/go/toid" + "github.com/stellar/go/xdr" + "github.com/stellar/go/services/horizon/internal/db2" "github.com/stellar/go/services/horizon/internal/test" ) @@ -45,16 +47,18 @@ func TestSelectTrades(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} fixtures := TradeScenario(tt, q) + afterTradesSeq := toid.Parse(fixtures.Trades[0].HistoryOperationID).LedgerSequence + 1 + beforeTradesSeq := afterTradesSeq - 2 for _, account := range append([]string{allAccounts}, fixtures.Addresses...) { for _, tradeType := range []string{AllTrades, OrderbookTrades, LiquidityPoolTrades} { expected := filterByAccount(FilterTradesByType(fixtures.Trades, tradeType), account) - rows, err := q.GetTrades(tt.Ctx, ascPQ, account, tradeType) + rows, err := q.GetTrades(tt.Ctx, ascPQ, 0, account, tradeType) tt.Assert.NoError(err) assertTradesAreEqual(tt, expected, rows) - rows, err = q.GetTrades(tt.Ctx, descPQ, account, tradeType) + rows, err = q.GetTrades(tt.Ctx, descPQ, beforeTradesSeq, account, tradeType) tt.Assert.NoError(err) start, end := 0, len(rows)-1 for start < end { @@ -64,6 +68,10 @@ func TestSelectTrades(t *testing.T) { } assertTradesAreEqual(tt, expected, rows) + + rows, err = q.GetTrades(tt.Ctx, descPQ, afterTradesSeq, account, tradeType) + tt.Assert.NoError(err) + tt.Assert.Empty(rows) } } } @@ -85,6 +93,7 @@ func TestSelectTradesCursor(t *testing.T) { rows, err := q.GetTrades( tt.Ctx, db2.MustPageQuery(expected[0].PagingToken(), false, "asc", 100), + 0, account, tradeType, ) @@ -98,6 +107,7 @@ func TestSelectTradesCursor(t *testing.T) { rows, err = q.GetTrades( tt.Ctx, db2.MustPageQuery(expected[1].PagingToken(), false, "asc", 100), + 0, account, tradeType, ) @@ -116,13 +126,14 @@ func TestTradesQueryForOffer(t *testing.T) { tt.Assert.NotEmpty(fixtures.TradesByOffer) for offer, expected := range fixtures.TradesByOffer { - trades, err := q.GetTradesForOffer(tt.Ctx, ascPQ, offer) + trades, err := q.GetTradesForOffer(tt.Ctx, ascPQ, 0, offer) tt.Assert.NoError(err) assertTradesAreEqual(tt, expected, trades) trades, err = q.GetTradesForOffer( tt.Ctx, db2.MustPageQuery(expected[0].PagingToken(), false, "asc", 100), + 0, offer, ) tt.Assert.NoError(err) @@ -139,13 +150,14 @@ func TestTradesQueryForLiquidityPool(t *testing.T) { tt.Assert.NotEmpty(fixtures.TradesByOffer) for poolID, expected := range fixtures.TradesByPool { - trades, err := q.GetTradesForLiquidityPool(tt.Ctx, ascPQ, poolID) + trades, err := q.GetTradesForLiquidityPool(tt.Ctx, ascPQ, 0, poolID) tt.Assert.NoError(err) assertTradesAreEqual(tt, expected, trades) trades, err = q.GetTradesForLiquidityPool( tt.Ctx, db2.MustPageQuery(expected[0].PagingToken(), false, "asc", 100), + 0, poolID, ) tt.Assert.NoError(err) @@ -167,7 +179,7 @@ func TestTradesForAssetPair(t *testing.T) { for _, tradeType := range []string{AllTrades, OrderbookTrades, LiquidityPoolTrades} { expected := filterByAccount(FilterTradesByType(allTrades, tradeType), account) - trades, err := q.GetTradesForAssets(tt.Ctx, ascPQ, account, tradeType, chfAsset, eurAsset) + trades, err := q.GetTradesForAssets(tt.Ctx, ascPQ, 0, account, tradeType, chfAsset, eurAsset) tt.Assert.NoError(err) assertTradesAreEqual(tt, expected, trades) @@ -178,6 +190,7 @@ func TestTradesForAssetPair(t *testing.T) { trades, err = q.GetTradesForAssets( tt.Ctx, db2.MustPageQuery(expected[0].PagingToken(), false, "asc", 100), + 0, account, tradeType, chfAsset, @@ -219,7 +232,7 @@ func TestTradesForReverseAssetPair(t *testing.T) { expected[i] = reverseTrade(expected[i]) } - trades, err := q.GetTradesForAssets(tt.Ctx, ascPQ, account, tradeType, eurAsset, chfAsset) + trades, err := q.GetTradesForAssets(tt.Ctx, ascPQ, 0, account, tradeType, eurAsset, chfAsset) tt.Assert.NoError(err) assertTradesAreEqual(tt, expected, trades) @@ -230,6 +243,7 @@ func TestTradesForReverseAssetPair(t *testing.T) { trades, err = q.GetTradesForAssets( tt.Ctx, db2.MustPageQuery(expected[0].PagingToken(), false, "asc", 100), + 0, account, tradeType, eurAsset, diff --git a/services/horizon/internal/db2/history/transaction.go b/services/horizon/internal/db2/history/transaction.go index 8e349f5f9d..ca0f08bb21 100644 --- a/services/horizon/internal/db2/history/transaction.go +++ b/services/horizon/internal/db2/history/transaction.go @@ -163,6 +163,7 @@ func (q *TransactionsQ) ForLedger(ctx context.Context, seq int32) *TransactionsQ start.ToInt64(), end.ToInt64(), ) + q.boundedIdQuery = true return q } @@ -174,11 +175,15 @@ func (q *TransactionsQ) IncludeFailed() *TransactionsQ { } // Page specifies the paging constraints for the query being built by `q`. -func (q *TransactionsQ) Page(page db2.PageQuery) *TransactionsQ { +func (q *TransactionsQ) Page(page db2.PageQuery, oldestLedger int32) *TransactionsQ { if q.Err != nil { return q } + if lowerBound := lowestLedgerBound(oldestLedger); !q.boundedIdQuery && lowerBound > 0 && page.Order == "desc" { + q.sql = q.sql. + Where(q.txIdCol+" > ?", lowerBound) + } q.sql, q.Err = page.ApplyTo(q.sql, q.txIdCol) return q } diff --git a/services/horizon/internal/db2/history/transaction_test.go b/services/horizon/internal/db2/history/transaction_test.go index 65c6734644..1a6497f41a 100644 --- a/services/horizon/internal/db2/history/transaction_test.go +++ b/services/horizon/internal/db2/history/transaction_test.go @@ -79,7 +79,7 @@ func TestTransactionByLiquidityPool(t *testing.T) { // Insert Liquidity Pool history liquidityPoolID := "a2f38836a839de008cf1d782c81f45e1253cc5d3dad9110b872965484fec0a49" - lpLoader := NewLiquidityPoolLoader() + lpLoader := NewLiquidityPoolLoader(ConcurrentInserts) lpTransactionBuilder := q.NewTransactionLiquidityPoolBatchInsertBuilder() tt.Assert.NoError(lpTransactionBuilder.Add(txID, lpLoader.GetFuture(liquidityPoolID))) tt.Assert.NoError(lpLoader.Exec(tt.Ctx, q)) @@ -940,15 +940,15 @@ func TestTransactionQueryBuilder(t *testing.T) { tt.Assert.NoError(q.Begin(tt.Ctx)) address := "GBXGQJWVLWOYHFLVTKWV5FGHA3LNYY2JQKM7OAJAUEQFU6LPCSEFVXON" - accountLoader := NewAccountLoader() + accountLoader := NewAccountLoader(ConcurrentInserts) accountLoader.GetFuture(address) cbID := "00000000178826fbfe339e1f5c53417c6fedfe2c05e8bec14303143ec46b38981b09c3f9" - cbLoader := NewClaimableBalanceLoader() + cbLoader := NewClaimableBalanceLoader(ConcurrentInserts) cbLoader.GetFuture(cbID) lpID := "0000a8198b5e25994c1ca5b0556faeb27325ac746296944144e0a7406d501e8a" - lpLoader := NewLiquidityPoolLoader() + lpLoader := NewLiquidityPoolLoader(ConcurrentInserts) lpLoader.GetFuture(lpID) tt.Assert.NoError(accountLoader.Exec(tt.Ctx, q)) @@ -957,93 +957,319 @@ func TestTransactionQueryBuilder(t *testing.T) { tt.Assert.NoError(q.Commit()) - txQ := q.Transactions().ForAccount(tt.Ctx, address).Page(db2.PageQuery{Cursor: "8589938689", Order: "asc", Limit: 10}) - tt.Assert.NoError(txQ.Err) - got, _, err := txQ.sql.ToSql() - tt.Assert.NoError(err) - // Transactions for account queries will use - // history_transaction_participants.history_transaction_id in their predicates. - want := "SELECT " + - "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + - "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + - "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + - "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + - "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + - "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + - "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + - "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + - "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + - "FROM history_transactions ht " + - "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + - "JOIN history_transaction_participants htp ON htp.history_transaction_id = ht.id " + - "WHERE htp.history_account_id = ? AND htp.history_transaction_id > ? " + - "ORDER BY htp.history_transaction_id asc LIMIT 10" - tt.Assert.EqualValues(want, got) - - txQ = q.Transactions().ForClaimableBalance(tt.Ctx, cbID).Page(db2.PageQuery{Cursor: "8589938689", Order: "asc", Limit: 10}) - tt.Assert.NoError(txQ.Err) - got, _, err = txQ.sql.ToSql() - tt.Assert.NoError(err) - // Transactions for claimable balance queries will use - // history_transaction_claimable_balances.history_transaction_id in their predicates. - want = "SELECT " + - "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + - "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + - "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + - "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + - "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + - "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + - "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + - "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + - "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + - "FROM history_transactions ht " + - "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + - "JOIN history_transaction_claimable_balances htcb ON htcb.history_transaction_id = ht.id " + - "WHERE htcb.history_claimable_balance_id = ? AND htcb.history_transaction_id > ? " + - "ORDER BY htcb.history_transaction_id asc LIMIT 10" - tt.Assert.EqualValues(want, got) - - txQ = q.Transactions().ForLiquidityPool(tt.Ctx, lpID).Page(db2.PageQuery{Cursor: "8589938689", Order: "asc", Limit: 10}) - tt.Assert.NoError(txQ.Err) - got, _, err = txQ.sql.ToSql() - tt.Assert.NoError(err) - // Transactions for liquidity pool queries will use - // history_transaction_liquidity_pools.history_transaction_id in their predicates. - want = "SELECT " + - "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + - "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + - "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + - "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + - "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + - "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + - "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + - "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + - "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + - "FROM history_transactions ht " + - "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + - "JOIN history_transaction_liquidity_pools htlp ON htlp.history_transaction_id = ht.id " + - "WHERE htlp.history_liquidity_pool_id = ? AND htlp.history_transaction_id > ? " + - "ORDER BY htlp.history_transaction_id asc LIMIT 10" - tt.Assert.EqualValues(want, got) - - txQ = q.Transactions().Page(db2.PageQuery{Cursor: "8589938689", Order: "asc", Limit: 10}) - tt.Assert.NoError(txQ.Err) - got, _, err = txQ.sql.ToSql() - tt.Assert.NoError(err) - // Other Transaction queries will use history_transactions.id in their predicates. - want = "SELECT " + - "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + - "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + - "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + - "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + - "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + - "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + - "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + - "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + - "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + - "FROM history_transactions ht " + - "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + - "WHERE ht.id > ? " + - "ORDER BY ht.id asc LIMIT 10" - tt.Assert.EqualValues(want, got) + for _, testCase := range []struct { + q *TransactionsQ + expectedSQL string + expectedArgs []interface{} + }{ + { + q.Transactions().ForAccount(tt.Ctx, address).Page(db2.PageQuery{Cursor: "8589938689", Order: "asc", Limit: 10}, 0), + "SELECT " + + "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + + "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + + "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + + "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + + "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + + "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + + "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + + "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + + "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + + "FROM history_transactions ht " + + "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + + "JOIN history_transaction_participants htp ON htp.history_transaction_id = ht.id " + + "WHERE htp.history_account_id = ? AND htp.history_transaction_id > ? " + + "ORDER BY htp.history_transaction_id asc LIMIT 10", + []interface{}{int64(1), int64(8589938689)}, + }, + { + q.Transactions().ForClaimableBalance(tt.Ctx, cbID).Page(db2.PageQuery{Cursor: "8589938689", Order: "asc", Limit: 10}, 0), + "SELECT " + + "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + + "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + + "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + + "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + + "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + + "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + + "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + + "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + + "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + + "FROM history_transactions ht " + + "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + + "JOIN history_transaction_claimable_balances htcb ON htcb.history_transaction_id = ht.id " + + "WHERE htcb.history_claimable_balance_id = ? AND htcb.history_transaction_id > ? " + + "ORDER BY htcb.history_transaction_id asc LIMIT 10", + []interface{}{int64(1), int64(8589938689)}, + }, + { + q.Transactions().ForLiquidityPool(tt.Ctx, lpID).Page(db2.PageQuery{Cursor: "8589938689", Order: "asc", Limit: 10}, 0), + "SELECT " + + "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + + "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + + "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + + "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + + "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + + "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + + "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + + "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + + "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + + "FROM history_transactions ht " + + "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + + "JOIN history_transaction_liquidity_pools htlp ON htlp.history_transaction_id = ht.id " + + "WHERE htlp.history_liquidity_pool_id = ? AND htlp.history_transaction_id > ? " + + "ORDER BY htlp.history_transaction_id asc LIMIT 10", + []interface{}{int64(1), int64(8589938689)}, + }, + { + q.Transactions().Page(db2.PageQuery{Cursor: "8589938689", Order: "asc", Limit: 10}, 0), + "SELECT " + + "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + + "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + + "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + + "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + + "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + + "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + + "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + + "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + + "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + + "FROM history_transactions ht " + + "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + + "WHERE ht.id > ? " + + "ORDER BY ht.id asc LIMIT 10", + []interface{}{int64(8589938689)}, + }, + { + q.Transactions().ForAccount(tt.Ctx, address).Page(db2.PageQuery{Cursor: "8589938689", Order: "asc", Limit: 10}, 100), + "SELECT " + + "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + + "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + + "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + + "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + + "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + + "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + + "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + + "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + + "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + + "FROM history_transactions ht " + + "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + + "JOIN history_transaction_participants htp ON htp.history_transaction_id = ht.id " + + "WHERE htp.history_account_id = ? AND htp.history_transaction_id > ? " + + "ORDER BY htp.history_transaction_id asc LIMIT 10", + []interface{}{int64(1), int64(8589938689)}, + }, + { + q.Transactions().ForClaimableBalance(tt.Ctx, cbID).Page(db2.PageQuery{Cursor: "8589938689", Order: "asc", Limit: 10}, 100), + "SELECT " + + "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + + "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + + "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + + "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + + "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + + "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + + "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + + "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + + "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + + "FROM history_transactions ht " + + "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + + "JOIN history_transaction_claimable_balances htcb ON htcb.history_transaction_id = ht.id " + + "WHERE htcb.history_claimable_balance_id = ? AND htcb.history_transaction_id > ? " + + "ORDER BY htcb.history_transaction_id asc LIMIT 10", + []interface{}{int64(1), int64(8589938689)}, + }, + { + q.Transactions().ForLiquidityPool(tt.Ctx, lpID).Page(db2.PageQuery{Cursor: "8589938689", Order: "asc", Limit: 10}, 100), + "SELECT " + + "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + + "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + + "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + + "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + + "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + + "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + + "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + + "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + + "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + + "FROM history_transactions ht " + + "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + + "JOIN history_transaction_liquidity_pools htlp ON htlp.history_transaction_id = ht.id " + + "WHERE htlp.history_liquidity_pool_id = ? AND htlp.history_transaction_id > ? " + + "ORDER BY htlp.history_transaction_id asc LIMIT 10", + []interface{}{int64(1), int64(8589938689)}, + }, + { + q.Transactions().Page(db2.PageQuery{Cursor: "8589938689", Order: "asc", Limit: 10}, 100), + "SELECT " + + "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + + "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + + "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + + "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + + "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + + "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + + "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + + "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + + "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + + "FROM history_transactions ht " + + "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + + "WHERE ht.id > ? " + + "ORDER BY ht.id asc LIMIT 10", + []interface{}{int64(8589938689)}, + }, + { + q.Transactions().ForAccount(tt.Ctx, address).Page(db2.PageQuery{Cursor: "8589938689", Order: "desc", Limit: 10}, 0), + "SELECT " + + "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + + "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + + "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + + "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + + "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + + "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + + "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + + "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + + "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + + "FROM history_transactions ht " + + "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + + "JOIN history_transaction_participants htp ON htp.history_transaction_id = ht.id " + + "WHERE htp.history_account_id = ? AND htp.history_transaction_id < ? " + + "ORDER BY htp.history_transaction_id desc LIMIT 10", + []interface{}{int64(1), int64(8589938689)}, + }, + { + q.Transactions().ForClaimableBalance(tt.Ctx, cbID).Page(db2.PageQuery{Cursor: "8589938689", Order: "desc", Limit: 10}, 0), + "SELECT " + + "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + + "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + + "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + + "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + + "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + + "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + + "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + + "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + + "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + + "FROM history_transactions ht " + + "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + + "JOIN history_transaction_claimable_balances htcb ON htcb.history_transaction_id = ht.id " + + "WHERE htcb.history_claimable_balance_id = ? AND htcb.history_transaction_id < ? " + + "ORDER BY htcb.history_transaction_id desc LIMIT 10", + []interface{}{int64(1), int64(8589938689)}, + }, + { + q.Transactions().ForLiquidityPool(tt.Ctx, lpID).Page(db2.PageQuery{Cursor: "8589938689", Order: "desc", Limit: 10}, 0), + "SELECT " + + "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + + "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + + "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + + "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + + "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + + "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + + "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + + "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + + "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + + "FROM history_transactions ht " + + "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + + "JOIN history_transaction_liquidity_pools htlp ON htlp.history_transaction_id = ht.id " + + "WHERE htlp.history_liquidity_pool_id = ? AND htlp.history_transaction_id < ? " + + "ORDER BY htlp.history_transaction_id desc LIMIT 10", + []interface{}{int64(1), int64(8589938689)}, + }, + { + q.Transactions().Page(db2.PageQuery{Cursor: "8589938689", Order: "desc", Limit: 10}, 0), + "SELECT " + + "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + + "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + + "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + + "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + + "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + + "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + + "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + + "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + + "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + + "FROM history_transactions ht " + + "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + + "WHERE ht.id < ? " + + "ORDER BY ht.id desc LIMIT 10", + []interface{}{int64(8589938689)}, + }, + { + q.Transactions().ForAccount(tt.Ctx, address).Page(db2.PageQuery{Cursor: "8589938689", Order: "desc", Limit: 10}, 100), + "SELECT " + + "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + + "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + + "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + + "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + + "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + + "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + + "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + + "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + + "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + + "FROM history_transactions ht " + + "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + + "JOIN history_transaction_participants htp ON htp.history_transaction_id = ht.id " + + "WHERE htp.history_account_id = ? AND htp.history_transaction_id > ? " + + "AND htp.history_transaction_id < ? " + + "ORDER BY htp.history_transaction_id desc LIMIT 10", + []interface{}{int64(1), int64(429496729599), int64(8589938689)}, + }, + { + q.Transactions().ForClaimableBalance(tt.Ctx, cbID).Page(db2.PageQuery{Cursor: "8589938689", Order: "desc", Limit: 10}, 100), + "SELECT " + + "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + + "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + + "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + + "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + + "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + + "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + + "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + + "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + + "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + + "FROM history_transactions ht " + + "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + + "JOIN history_transaction_claimable_balances htcb ON htcb.history_transaction_id = ht.id " + + "WHERE htcb.history_claimable_balance_id = ? AND htcb.history_transaction_id > ? " + + "AND htcb.history_transaction_id < ? " + + "ORDER BY htcb.history_transaction_id desc LIMIT 10", + []interface{}{int64(1), int64(429496729599), int64(8589938689)}, + }, + { + q.Transactions().ForLiquidityPool(tt.Ctx, lpID).Page(db2.PageQuery{Cursor: "8589938689", Order: "desc", Limit: 10}, 100), + "SELECT " + + "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + + "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + + "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + + "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + + "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + + "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + + "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + + "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + + "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + + "FROM history_transactions ht " + + "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + + "JOIN history_transaction_liquidity_pools htlp ON htlp.history_transaction_id = ht.id " + + "WHERE htlp.history_liquidity_pool_id = ? AND htlp.history_transaction_id > ? " + + "AND htlp.history_transaction_id < ? " + + "ORDER BY htlp.history_transaction_id desc LIMIT 10", + []interface{}{int64(1), int64(429496729599), int64(8589938689)}, + }, + { + q.Transactions().Page(db2.PageQuery{Cursor: "8589938689", Order: "desc", Limit: 10}, 100), + "SELECT " + + "ht.id, ht.transaction_hash, ht.ledger_sequence, ht.application_order, " + + "ht.account, ht.account_muxed, ht.account_sequence, ht.max_fee, " + + "COALESCE(ht.fee_charged, ht.max_fee) as fee_charged, ht.operation_count, " + + "ht.tx_envelope, ht.tx_result, ht.tx_meta, ht.tx_fee_meta, ht.created_at, " + + "ht.updated_at, COALESCE(ht.successful, true) as successful, ht.signatures, " + + "ht.memo_type, ht.memo, ht.time_bounds, ht.ledger_bounds, ht.min_account_sequence, " + + "ht.min_account_sequence_age, ht.min_account_sequence_ledger_gap, ht.extra_signers, " + + "hl.closed_at AS ledger_close_time, ht.inner_transaction_hash, ht.fee_account, " + + "ht.fee_account_muxed, ht.new_max_fee, ht.inner_signatures " + + "FROM history_transactions ht " + + "LEFT JOIN history_ledgers hl ON ht.ledger_sequence = hl.sequence " + + "WHERE ht.id > ? AND ht.id < ? " + + "ORDER BY ht.id desc LIMIT 10", + []interface{}{int64(429496729599), int64(8589938689)}, + }, + } { + tt.Assert.NoError(testCase.q.Err) + got, args, err := testCase.q.sql.ToSql() + tt.Assert.NoError(err) + tt.Assert.Equal(got, testCase.expectedSQL) + tt.Assert.Equal(args, testCase.expectedArgs) + } } diff --git a/services/horizon/internal/db2/history/verify_lock.go b/services/horizon/internal/db2/history/verify_lock.go index 29bc11a473..4d7d1fbde7 100644 --- a/services/horizon/internal/db2/history/verify_lock.go +++ b/services/horizon/internal/db2/history/verify_lock.go @@ -13,9 +13,13 @@ const ( // all ingesting nodes use the same value which is why it's hard coded here.`1 stateVerificationLockId = 73897213 // reaperLockId is the objid for the advisory lock acquired during - // reaping. The value is arbitrary. The only requirement is that + // reaping of history tables. The value is arbitrary. The only requirement is that // all ingesting nodes use the same value which is why it's hard coded here. reaperLockId = 944670730 + // lookupTableReaperLockId is the objid for the advisory lock acquired during + // reaping of lookup tables. The value is arbitrary. The only requirement is that + // all ingesting nodes use the same value which is why it's hard coded here. + lookupTableReaperLockId = 329518896 ) // TryStateVerificationLock attempts to acquire the state verification lock @@ -34,6 +38,10 @@ func (q *Q) TryReaperLock(ctx context.Context) (bool, error) { return q.tryAdvisoryLock(ctx, reaperLockId) } +func (q *Q) TryLookupTableReaperLock(ctx context.Context) (bool, error) { + return q.tryAdvisoryLock(ctx, lookupTableReaperLockId) +} + func (q *Q) tryAdvisoryLock(ctx context.Context, lockId int) (bool, error) { if tx := q.GetTx(); tx == nil { return false, errors.New("cannot be called outside of a transaction") diff --git a/services/horizon/internal/ingest/main.go b/services/horizon/internal/ingest/main.go index e471b7c4e4..ef676e5487 100644 --- a/services/horizon/internal/ingest/main.go +++ b/services/horizon/internal/ingest/main.go @@ -70,16 +70,15 @@ const ( // * Ledger ingestion, // * State verifications, // * Metrics updates. - // * Reaping (requires 2 connections, the extra connection is used for holding the advisory lock) - MaxDBConnections = 5 + // * Reaping of history (requires 2 connections, the extra connection is used for holding the advisory lock) + // * Reaping of lookup tables (requires 2 connections, the extra connection is used for holding the advisory lock) + MaxDBConnections = 7 stateVerificationErrorThreshold = 3 // 100 ledgers per flush has shown in stress tests // to be best point on performance curve, default to that. MaxLedgersPerFlush uint32 = 100 - - reapLookupTablesBatchSize = 1000 ) var log = logpkg.DefaultLogger.WithField("service", "ingest") @@ -101,6 +100,16 @@ func (s LedgerBackendType) String() string { return "" } +const ( + HistoryCheckpointLedgerInterval uint = 64 + // MinBatchSize is the minimum batch size for reingestion + MinBatchSize uint = HistoryCheckpointLedgerInterval + // MaxBufferedStorageBackendBatchSize is the maximum batch size for Buffered Storage reingestion + MaxBufferedStorageBackendBatchSize uint = 200 * HistoryCheckpointLedgerInterval + // MaxCaptiveCoreBackendBatchSize is the maximum batch size for Captive Core reingestion + MaxCaptiveCoreBackendBatchSize uint = 20_000 * HistoryCheckpointLedgerInterval +) + type StorageBackendConfig struct { DataStoreConfig datastore.DataStoreConfig `toml:"datastore_config"` BufferedStorageBackendConfig ledgerbackend.BufferedStorageBackendConfig `toml:"buffered_storage_backend_config"` @@ -172,9 +181,6 @@ type Metrics struct { // duration of rebuilding trade aggregation buckets. LedgerIngestionTradeAggregationDuration prometheus.Summary - ReapDurationByLookupTable *prometheus.SummaryVec - RowsReapedByLookupTable *prometheus.SummaryVec - // StateVerifyDuration exposes timing metrics about the rate and // duration of state verification. StateVerifyDuration prometheus.Summary @@ -256,7 +262,8 @@ type system struct { maxLedgerPerFlush uint32 - reaper *Reaper + reaper *Reaper + lookupTableReaper *lookupTableReaper currentStateMutex sync.Mutex currentState State @@ -369,6 +376,7 @@ func NewSystem(config Config) (System, error) { config.ReapConfig, config.HistorySession, ), + lookupTableReaper: newLookupTableReaper(config.HistorySession), } system.initMetrics() @@ -409,18 +417,6 @@ func (s *system) initMetrics() { Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }) - s.metrics.ReapDurationByLookupTable = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Namespace: "horizon", Subsystem: "ingest", Name: "reap_lookup_tables_duration_seconds", - Help: "reap lookup tables durations, sliding window = 10m", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }, []string{"table"}) - - s.metrics.RowsReapedByLookupTable = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Namespace: "horizon", Subsystem: "ingest", Name: "reap_lookup_tables_rows_reaped", - Help: "rows deleted during lookup tables reap, sliding window = 10m", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }, []string{"table"}) - s.metrics.StateVerifyDuration = prometheus.NewSummary(prometheus.SummaryOpts{ Namespace: "horizon", Subsystem: "ingest", Name: "state_verify_duration_seconds", Help: "state verification durations, sliding window = 10m", @@ -538,8 +534,6 @@ func (s *system) RegisterMetrics(registry *prometheus.Registry) { registry.MustRegister(s.metrics.LocalLatestLedger) registry.MustRegister(s.metrics.LedgerIngestionDuration) registry.MustRegister(s.metrics.LedgerIngestionTradeAggregationDuration) - registry.MustRegister(s.metrics.ReapDurationByLookupTable) - registry.MustRegister(s.metrics.RowsReapedByLookupTable) registry.MustRegister(s.metrics.StateVerifyDuration) registry.MustRegister(s.metrics.StateInvalidGauge) registry.MustRegister(s.metrics.LedgerStatsCounter) @@ -552,6 +546,7 @@ func (s *system) RegisterMetrics(registry *prometheus.Registry) { registry.MustRegister(s.metrics.IngestionErrorCounter) s.ledgerBackend = ledgerbackend.WithMetrics(s.ledgerBackend, registry, "horizon") s.reaper.RegisterMetrics(registry) + s.lookupTableReaper.RegisterMetrics(registry) } // Run starts ingestion system. Ingestion system supports distributed ingestion @@ -822,55 +817,11 @@ func (s *system) maybeReapLookupTables(lastIngestedLedger uint32) { return } - err = s.historyQ.Begin(s.ctx) - if err != nil { - log.WithError(err).Error("Error starting a transaction") - return - } - defer s.historyQ.Rollback() - - // If so block ingestion in the cluster to reap tables - _, err = s.historyQ.GetLastLedgerIngest(s.ctx) - if err != nil { - log.WithError(err).Error(getLastIngestedErrMsg) - return - } - - // Make sure reaping will not take more than 5s, which is average ledger - // closing time. - ctx, cancel := context.WithTimeout(s.ctx, 5*time.Second) - defer cancel() - - reapStart := time.Now() - results, err := s.historyQ.ReapLookupTables(ctx, reapLookupTablesBatchSize) - if err != nil { - log.WithError(err).Warn("Error reaping lookup tables") - return - } - - err = s.historyQ.Commit() - if err != nil { - log.WithError(err).Error("Error committing a transaction") - return - } - - totalDeleted := int64(0) - reapLog := log - for table, result := range results { - totalDeleted += result.RowsDeleted - reapLog = reapLog.WithField(table+"_offset", result.Offset) - reapLog = reapLog.WithField(table+"_duration", result.Duration) - reapLog = reapLog.WithField(table+"_rows_deleted", result.RowsDeleted) - s.Metrics().RowsReapedByLookupTable.With(prometheus.Labels{"table": table}).Observe(float64(result.RowsDeleted)) - s.Metrics().ReapDurationByLookupTable.With(prometheus.Labels{"table": table}).Observe(result.Duration.Seconds()) - } - - if totalDeleted > 0 { - reapLog.Info("Reaper deleted rows from lookup tables") - } - - s.Metrics().RowsReapedByLookupTable.With(prometheus.Labels{"table": "total"}).Observe(float64(totalDeleted)) - s.Metrics().ReapDurationByLookupTable.With(prometheus.Labels{"table": "total"}).Observe(time.Since(reapStart).Seconds()) + s.wg.Add(1) + go func() { + defer s.wg.Done() + s.lookupTableReaper.deleteOrphanedRows(s.ctx) + }() } func (s *system) incrementStateVerificationErrors() int { diff --git a/services/horizon/internal/ingest/main_test.go b/services/horizon/internal/ingest/main_test.go index d5733ee5e4..470039fd92 100644 --- a/services/horizon/internal/ingest/main_test.go +++ b/services/horizon/internal/ingest/main_test.go @@ -462,6 +462,11 @@ func (m *mockDBQ) TryReaperLock(ctx context.Context) (bool, error) { return args.Get(0).(bool), args.Error(1) } +func (m *mockDBQ) TryLookupTableReaperLock(ctx context.Context) (bool, error) { + args := m.Called(ctx) + return args.Get(0).(bool), args.Error(1) +} + func (m *mockDBQ) GetNextLedgerSequence(ctx context.Context, start uint32) (uint32, bool, error) { args := m.Called(ctx, start) return args.Get(0).(uint32), args.Get(1).(bool), args.Error(2) @@ -562,13 +567,14 @@ func (m *mockDBQ) NewTradeBatchInsertBuilder() history.TradeBatchInsertBuilder { return args.Get(0).(history.TradeBatchInsertBuilder) } -func (m *mockDBQ) ReapLookupTables(ctx context.Context, batchSize int) (map[string]history.LookupTableReapResult, error) { - args := m.Called(ctx, batchSize) - var r1 map[string]history.LookupTableReapResult - if args.Get(0) != nil { - r1 = args.Get(0).(map[string]history.LookupTableReapResult) - } - return r1, args.Error(2) +func (m *mockDBQ) FindLookupTableRowsToReap(ctx context.Context, table string, batchSize int) ([]int64, int64, error) { + args := m.Called(ctx, table, batchSize) + return args.Get(0).([]int64), args.Get(1).(int64), args.Error(2) +} + +func (m *mockDBQ) ReapLookupTable(ctx context.Context, table string, ids []int64, offset int64) (int64, error) { + args := m.Called(ctx, table, ids, offset) + return args.Get(0).(int64), args.Error(1) } func (m *mockDBQ) RebuildTradeAggregationTimes(ctx context.Context, from, to strtime.Millis, roundingSlippageFilter int) error { diff --git a/services/horizon/internal/ingest/parallel.go b/services/horizon/internal/ingest/parallel.go index 4f07c21cc4..a2a641c5cf 100644 --- a/services/horizon/internal/ingest/parallel.go +++ b/services/horizon/internal/ingest/parallel.go @@ -8,11 +8,7 @@ import ( "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/errors" logpkg "github.com/stellar/go/support/log" -) - -const ( - historyCheckpointLedgerInterval = 64 - minBatchSize = historyCheckpointLedgerInterval + "github.com/stellar/go/support/ordered" ) type rangeError struct { @@ -27,23 +23,32 @@ func (e rangeError) Error() string { type ParallelSystems struct { config Config workerCount uint + minBatchSize uint + maxBatchSize uint systemFactory func(Config) (System, error) } -func NewParallelSystems(config Config, workerCount uint) (*ParallelSystems, error) { +func NewParallelSystems(config Config, workerCount uint, minBatchSize, maxBatchSize uint) (*ParallelSystems, error) { // Leaving this because used in tests, will update after a code review. - return newParallelSystems(config, workerCount, NewSystem) + return newParallelSystems(config, workerCount, minBatchSize, maxBatchSize, NewSystem) } // private version of NewParallel systems, allowing to inject a mock system -func newParallelSystems(config Config, workerCount uint, systemFactory func(Config) (System, error)) (*ParallelSystems, error) { +func newParallelSystems(config Config, workerCount uint, minBatchSize, maxBatchSize uint, systemFactory func(Config) (System, error)) (*ParallelSystems, error) { if workerCount < 1 { return nil, errors.New("workerCount must be > 0") } - + if minBatchSize != 0 && minBatchSize < HistoryCheckpointLedgerInterval { + return nil, fmt.Errorf("minBatchSize must be at least the %d", HistoryCheckpointLedgerInterval) + } + if minBatchSize != 0 && maxBatchSize != 0 && maxBatchSize < minBatchSize { + return nil, errors.New("maxBatchSize cannot be less than minBatchSize") + } return &ParallelSystems{ config: config, workerCount: workerCount, + maxBatchSize: maxBatchSize, + minBatchSize: minBatchSize, systemFactory: systemFactory, }, nil } @@ -112,20 +117,27 @@ func enqueueReingestTasks(ledgerRanges []history.LedgerRange, batchSize uint32, } return lowestLedger } +func (ps *ParallelSystems) calculateParallelLedgerBatchSize(rangeSize uint32) uint32 { + // calculate the initial batch size based on available workers + batchSize := rangeSize / uint32(ps.workerCount) + + // ensure the batch size meets min threshold + if ps.minBatchSize > 0 { + batchSize = ordered.Max(batchSize, uint32(ps.minBatchSize)) + } -func calculateParallelLedgerBatchSize(rangeSize uint32, batchSizeSuggestion uint32, workerCount uint) uint32 { - batchSize := batchSizeSuggestion - if batchSize == 0 || rangeSize/batchSize < uint32(workerCount) { - // let's try to make use of all the workers - batchSize = rangeSize / uint32(workerCount) + // ensure the batch size does not exceed max threshold + if ps.maxBatchSize > 0 { + batchSize = ordered.Min(batchSize, uint32(ps.maxBatchSize)) } - // Use a minimum batch size to make it worth it in terms of overhead - if batchSize < minBatchSize { - batchSize = minBatchSize + + // round down to the nearest multiple of HistoryCheckpointLedgerInterval + if batchSize > uint32(HistoryCheckpointLedgerInterval) { + return batchSize / uint32(HistoryCheckpointLedgerInterval) * uint32(HistoryCheckpointLedgerInterval) } - // Also, round the batch size to the closest, lower or equal 64 multiple - return (batchSize / historyCheckpointLedgerInterval) * historyCheckpointLedgerInterval + // HistoryCheckpointLedgerInterval is the minimum batch size. + return uint32(HistoryCheckpointLedgerInterval) } func totalRangeSize(ledgerRanges []history.LedgerRange) uint32 { @@ -136,9 +148,9 @@ func totalRangeSize(ledgerRanges []history.LedgerRange) uint32 { return sum } -func (ps *ParallelSystems) ReingestRange(ledgerRanges []history.LedgerRange, batchSizeSuggestion uint32) error { +func (ps *ParallelSystems) ReingestRange(ledgerRanges []history.LedgerRange) error { var ( - batchSize = calculateParallelLedgerBatchSize(totalRangeSize(ledgerRanges), batchSizeSuggestion, ps.workerCount) + batchSize = ps.calculateParallelLedgerBatchSize(totalRangeSize(ledgerRanges)) reingestJobQueue = make(chan history.LedgerRange) wg sync.WaitGroup diff --git a/services/horizon/internal/ingest/parallel_test.go b/services/horizon/internal/ingest/parallel_test.go index 8004a4048c..f011f51021 100644 --- a/services/horizon/internal/ingest/parallel_test.go +++ b/services/horizon/internal/ingest/parallel_test.go @@ -1,6 +1,8 @@ package ingest import ( + "fmt" + "math" "math/rand" "sort" "sync" @@ -15,13 +17,88 @@ import ( ) func TestCalculateParallelLedgerBatchSize(t *testing.T) { - assert.Equal(t, uint32(6656), calculateParallelLedgerBatchSize(20096, 20096, 3)) - assert.Equal(t, uint32(4992), calculateParallelLedgerBatchSize(20096, 20096, 4)) - assert.Equal(t, uint32(4992), calculateParallelLedgerBatchSize(20096, 0, 4)) - assert.Equal(t, uint32(64), calculateParallelLedgerBatchSize(64, 256, 4)) - assert.Equal(t, uint32(64), calculateParallelLedgerBatchSize(64, 32, 4)) - assert.Equal(t, uint32(64), calculateParallelLedgerBatchSize(2, 256, 4)) - assert.Equal(t, uint32(64), calculateParallelLedgerBatchSize(20096, 64, 1)) + config := Config{} + result := &mockSystem{} + factory := func(c Config) (System, error) { + return result, nil + } + + // worker count 0 + system, err := newParallelSystems(config, 0, MinBatchSize, MaxCaptiveCoreBackendBatchSize, factory) + assert.EqualError(t, err, "workerCount must be > 0") + + // worker count 1, range smaller than HistoryCheckpointLedgerInterval + system, err = newParallelSystems(config, 1, 50, 200, factory) + assert.EqualError(t, err, fmt.Sprintf("minBatchSize must be at least the %d", HistoryCheckpointLedgerInterval)) + + // worker count 1, max batch size smaller than min batch size + system, err = newParallelSystems(config, 1, 5000, 200, factory) + assert.EqualError(t, err, "maxBatchSize cannot be less than minBatchSize") + + // worker count 1, captive core batch size + system, _ = newParallelSystems(config, 1, MinBatchSize, MaxCaptiveCoreBackendBatchSize, factory) + assert.Equal(t, uint32(MaxCaptiveCoreBackendBatchSize), system.calculateParallelLedgerBatchSize(uint32(MaxCaptiveCoreBackendBatchSize)+10)) + assert.Equal(t, uint32(MinBatchSize), system.calculateParallelLedgerBatchSize(0)) + assert.Equal(t, uint32(10048), system.calculateParallelLedgerBatchSize(10048)) // exact multiple + assert.Equal(t, uint32(10048), system.calculateParallelLedgerBatchSize(10090)) // round down + + // worker count 1, buffered storage batch size + system, _ = newParallelSystems(config, 1, MinBatchSize, MaxBufferedStorageBackendBatchSize, factory) + assert.Equal(t, uint32(MaxBufferedStorageBackendBatchSize), system.calculateParallelLedgerBatchSize(uint32(MaxBufferedStorageBackendBatchSize)+10)) + assert.Equal(t, uint32(MinBatchSize), system.calculateParallelLedgerBatchSize(0)) + assert.Equal(t, uint32(10048), system.calculateParallelLedgerBatchSize(10048)) // exact multiple + assert.Equal(t, uint32(10048), system.calculateParallelLedgerBatchSize(10090)) // round down + + // worker count 1, no min/max batch size + system, _ = newParallelSystems(config, 1, 0, 0, factory) + assert.Equal(t, uint32(20096), system.calculateParallelLedgerBatchSize(20096)) // exact multiple + assert.Equal(t, uint32(20032), system.calculateParallelLedgerBatchSize(20090)) // round down + + // worker count 1, min/max batch size + system, _ = newParallelSystems(config, 1, 64, 20000, factory) + assert.Equal(t, uint32(19968), system.calculateParallelLedgerBatchSize(20096)) // round down + system, _ = newParallelSystems(config, 1, 64, 30000, factory) + assert.Equal(t, uint32(20096), system.calculateParallelLedgerBatchSize(20096)) // exact multiple + + // Tests for worker count 2 + + // no min/max batch size + system, _ = newParallelSystems(config, 2, 0, 0, factory) + assert.Equal(t, uint32(64), system.calculateParallelLedgerBatchSize(60)) // range smaller than 64 + assert.Equal(t, uint32(64), system.calculateParallelLedgerBatchSize(128)) // exact multiple + assert.Equal(t, uint32(10048), system.calculateParallelLedgerBatchSize(20096)) + + // range larger than max batch size + system, _ = newParallelSystems(config, 2, 64, 10000, factory) + assert.Equal(t, uint32(9984), system.calculateParallelLedgerBatchSize(20096)) // round down + + // range smaller than min batch size + system, _ = newParallelSystems(config, 2, 64, 0, factory) + assert.Equal(t, uint32(64), system.calculateParallelLedgerBatchSize(50)) // min batch size + assert.Equal(t, uint32(10048), system.calculateParallelLedgerBatchSize(20096)) // exact multiple + assert.Equal(t, uint32(64), system.calculateParallelLedgerBatchSize(100)) // min batch size + + // batch size equal to min + system, _ = newParallelSystems(config, 2, 100, 0, factory) + assert.Equal(t, uint32(64), system.calculateParallelLedgerBatchSize(100)) // round down + + // equal min/max batch size + system, _ = newParallelSystems(config, 2, 5000, 5000, factory) + assert.Equal(t, uint32(4992), system.calculateParallelLedgerBatchSize(20096)) // round down + + // worker count 3 + system, _ = newParallelSystems(config, 3, 64, 7000, factory) + assert.Equal(t, uint32(6656), system.calculateParallelLedgerBatchSize(20096)) + + // worker count 4 + system, _ = newParallelSystems(config, 4, 64, 20000, factory) + assert.Equal(t, uint32(4992), system.calculateParallelLedgerBatchSize(20096)) //round down + assert.Equal(t, uint32(64), system.calculateParallelLedgerBatchSize(64)) + assert.Equal(t, uint32(64), system.calculateParallelLedgerBatchSize(2)) + + // max possible workers + system, _ = newParallelSystems(config, math.MaxUint32, 0, 0, factory) + assert.Equal(t, uint32(64), system.calculateParallelLedgerBatchSize(math.MaxUint32)) } func TestParallelReingestRange(t *testing.T) { @@ -43,31 +120,27 @@ func TestParallelReingestRange(t *testing.T) { factory := func(c Config) (System, error) { return result, nil } - system, err := newParallelSystems(config, 3, factory) + system, err := newParallelSystems(config, 3, MinBatchSize, MaxCaptiveCoreBackendBatchSize, factory) assert.NoError(t, err) - err = system.ReingestRange([]history.LedgerRange{{1, 2050}}, 258) + err = system.ReingestRange([]history.LedgerRange{{1, 2050}}) assert.NoError(t, err) sort.Slice(rangesCalled, func(i, j int) bool { return rangesCalled[i].StartSequence < rangesCalled[j].StartSequence }) expected := []history.LedgerRange{ - {StartSequence: 1, EndSequence: 256}, {StartSequence: 257, EndSequence: 512}, {StartSequence: 513, EndSequence: 768}, {StartSequence: 769, EndSequence: 1024}, {StartSequence: 1025, EndSequence: 1280}, - {StartSequence: 1281, EndSequence: 1536}, {StartSequence: 1537, EndSequence: 1792}, {StartSequence: 1793, EndSequence: 2048}, {StartSequence: 2049, EndSequence: 2050}, + {StartSequence: 1, EndSequence: 640}, {StartSequence: 641, EndSequence: 1280}, {StartSequence: 1281, EndSequence: 1920}, {StartSequence: 1921, EndSequence: 2050}, } assert.Equal(t, expected, rangesCalled) rangesCalled = nil - system, err = newParallelSystems(config, 1, factory) + system, err = newParallelSystems(config, 1, 0, 0, factory) assert.NoError(t, err) result.On("RebuildTradeAggregationBuckets", uint32(1), uint32(1024)).Return(nil).Once() - err = system.ReingestRange([]history.LedgerRange{{1, 1024}}, 64) + err = system.ReingestRange([]history.LedgerRange{{1, 1024}}) result.AssertExpectations(t) expected = []history.LedgerRange{ - {StartSequence: 1, EndSequence: 64}, {StartSequence: 65, EndSequence: 128}, {StartSequence: 129, EndSequence: 192}, {StartSequence: 193, EndSequence: 256}, {StartSequence: 257, EndSequence: 320}, - {StartSequence: 321, EndSequence: 384}, {StartSequence: 385, EndSequence: 448}, {StartSequence: 449, EndSequence: 512}, {StartSequence: 513, EndSequence: 576}, {StartSequence: 577, EndSequence: 640}, - {StartSequence: 641, EndSequence: 704}, {StartSequence: 705, EndSequence: 768}, {StartSequence: 769, EndSequence: 832}, {StartSequence: 833, EndSequence: 896}, {StartSequence: 897, EndSequence: 960}, - {StartSequence: 961, EndSequence: 1024}, + {StartSequence: 1, EndSequence: 1024}, } assert.NoError(t, err) assert.Equal(t, expected, rangesCalled) @@ -77,19 +150,19 @@ func TestParallelReingestRangeError(t *testing.T) { config := Config{} result := &mockSystem{} // Fail on the second range - result.On("ReingestRange", []history.LedgerRange{{1537, 1792}}, false, false).Return(errors.New("failed because of foo")).Once() + result.On("ReingestRange", []history.LedgerRange{{641, 1280}}, false, false).Return(errors.New("failed because of foo")).Once() result.On("ReingestRange", mock.AnythingOfType("[]history.LedgerRange"), false, false).Return(nil) - result.On("RebuildTradeAggregationBuckets", uint32(1), uint32(1537)).Return(nil).Once() + result.On("RebuildTradeAggregationBuckets", uint32(1), uint32(641)).Return(nil).Once() factory := func(c Config) (System, error) { return result, nil } - system, err := newParallelSystems(config, 3, factory) + system, err := newParallelSystems(config, 3, MinBatchSize, MaxCaptiveCoreBackendBatchSize, factory) assert.NoError(t, err) - err = system.ReingestRange([]history.LedgerRange{{1, 2050}}, 258) + err = system.ReingestRange([]history.LedgerRange{{1, 2050}}) result.AssertExpectations(t) assert.Error(t, err) - assert.Equal(t, "job failed, recommended restart range: [1537, 2050]: error when processing [1537, 1792] range: failed because of foo", err.Error()) + assert.Equal(t, "job failed, recommended restart range: [641, 2050]: error when processing [641, 1280] range: failed because of foo", err.Error()) } func TestParallelReingestRangeErrorInEarlierJob(t *testing.T) { @@ -98,27 +171,27 @@ func TestParallelReingestRangeErrorInEarlierJob(t *testing.T) { wg.Add(1) result := &mockSystem{} // Fail on an lower subrange after the first error - result.On("ReingestRange", []history.LedgerRange{{1025, 1280}}, false, false).Run(func(mock.Arguments) { + result.On("ReingestRange", []history.LedgerRange{{641, 1280}}, false, false).Run(func(mock.Arguments) { // Wait for a more recent range to error wg.Wait() // This sleep should help making sure the result of this range is processed later than the one below // (there are no guarantees without instrumenting ReingestRange(), but that's too complicated) time.Sleep(50 * time.Millisecond) }).Return(errors.New("failed because of foo")).Once() - result.On("ReingestRange", []history.LedgerRange{{1537, 1792}}, false, false).Run(func(mock.Arguments) { + result.On("ReingestRange", []history.LedgerRange{{1281, 1920}}, false, false).Run(func(mock.Arguments) { wg.Done() }).Return(errors.New("failed because of bar")).Once() result.On("ReingestRange", mock.AnythingOfType("[]history.LedgerRange"), false, false).Return(error(nil)) - result.On("RebuildTradeAggregationBuckets", uint32(1), uint32(1025)).Return(nil).Once() + result.On("RebuildTradeAggregationBuckets", uint32(1), uint32(641)).Return(nil).Once() factory := func(c Config) (System, error) { return result, nil } - system, err := newParallelSystems(config, 3, factory) + system, err := newParallelSystems(config, 3, 0, 0, factory) assert.NoError(t, err) - err = system.ReingestRange([]history.LedgerRange{{1, 2050}}, 258) + err = system.ReingestRange([]history.LedgerRange{{1, 2050}}) result.AssertExpectations(t) assert.Error(t, err) - assert.Equal(t, "job failed, recommended restart range: [1025, 2050]: error when processing [1025, 1280] range: failed because of foo", err.Error()) + assert.Equal(t, "job failed, recommended restart range: [641, 2050]: error when processing [641, 1280] range: failed because of foo", err.Error()) } diff --git a/services/horizon/internal/ingest/processor_runner.go b/services/horizon/internal/ingest/processor_runner.go index e6f0e0cf74..75b8645953 100644 --- a/services/horizon/internal/ingest/processor_runner.go +++ b/services/horizon/internal/ingest/processor_runner.go @@ -133,11 +133,11 @@ func buildChangeProcessor( }) } -func (s *ProcessorRunner) buildTransactionProcessor(ledgersProcessor *processors.LedgersProcessor) (groupLoaders, *groupTransactionProcessors) { - accountLoader := history.NewAccountLoader() - assetLoader := history.NewAssetLoader() - lpLoader := history.NewLiquidityPoolLoader() - cbLoader := history.NewClaimableBalanceLoader() +func (s *ProcessorRunner) buildTransactionProcessor(ledgersProcessor *processors.LedgersProcessor, concurrencyMode history.ConcurrencyMode) (groupLoaders, *groupTransactionProcessors) { + accountLoader := history.NewAccountLoader(concurrencyMode) + assetLoader := history.NewAssetLoader(concurrencyMode) + lpLoader := history.NewLiquidityPoolLoader(concurrencyMode) + cbLoader := history.NewClaimableBalanceLoader(concurrencyMode) loaders := newGroupLoaders([]horizonLazyLoader{accountLoader, assetLoader, lpLoader, cbLoader}) statsLedgerTransactionProcessor := processors.NewStatsLedgerTransactionProcessor() @@ -366,7 +366,7 @@ func (s *ProcessorRunner) streamLedger(ledger xdr.LedgerCloseMeta, return nil } -func (s *ProcessorRunner) runTransactionProcessorsOnLedger(registry nameRegistry, ledger xdr.LedgerCloseMeta) ( +func (s *ProcessorRunner) runTransactionProcessorsOnLedger(registry nameRegistry, ledger xdr.LedgerCloseMeta, concurrencyMode history.ConcurrencyMode) ( transactionStats processors.StatsLedgerTransactionProcessorResults, transactionDurations runDurations, tradeStats processors.TradeStats, @@ -381,7 +381,7 @@ func (s *ProcessorRunner) runTransactionProcessorsOnLedger(registry nameRegistry groupTransactionFilterers := s.buildTransactionFilterer() // when in online mode, the submission result processor must always run (regardless of whether filter rules exist or not) groupFilteredOutProcessors := s.buildFilteredOutProcessor() - loaders, groupTransactionProcessors := s.buildTransactionProcessor(ledgersProcessor) + loaders, groupTransactionProcessors := s.buildTransactionProcessor(ledgersProcessor, concurrencyMode) if err = registerTransactionProcessors( registry, @@ -494,7 +494,7 @@ func (s *ProcessorRunner) RunTransactionProcessorsOnLedgers(ledgers []xdr.Ledger groupTransactionFilterers := s.buildTransactionFilterer() // intentionally skip filtered out processor groupFilteredOutProcessors := newGroupTransactionProcessors(nil, nil, nil) - loaders, groupTransactionProcessors := s.buildTransactionProcessor(ledgersProcessor) + loaders, groupTransactionProcessors := s.buildTransactionProcessor(ledgersProcessor, history.ConcurrentInserts) startTime := time.Now() curHeap, sysHeap := getMemStats() @@ -611,7 +611,7 @@ func (s *ProcessorRunner) RunAllProcessorsOnLedger(ledger xdr.LedgerCloseMeta) ( return } - transactionStats, transactionDurations, tradeStats, loaderDurations, loaderStats, err := s.runTransactionProcessorsOnLedger(registry, ledger) + transactionStats, transactionDurations, tradeStats, loaderDurations, loaderStats, err := s.runTransactionProcessorsOnLedger(registry, ledger, history.ConcurrentDeletes) stats.changeStats = changeStatsProcessor.GetResults() stats.changeDurations = groupChangeProcessors.processorsRunDurations diff --git a/services/horizon/internal/ingest/processor_runner_test.go b/services/horizon/internal/ingest/processor_runner_test.go index e6ce6b512c..82c712b737 100644 --- a/services/horizon/internal/ingest/processor_runner_test.go +++ b/services/horizon/internal/ingest/processor_runner_test.go @@ -248,7 +248,7 @@ func TestProcessorRunnerBuildTransactionProcessor(t *testing.T) { ledgersProcessor := &processors.LedgersProcessor{} - _, processor := runner.buildTransactionProcessor(ledgersProcessor) + _, processor := runner.buildTransactionProcessor(ledgersProcessor, history.ConcurrentInserts) assert.IsType(t, &groupTransactionProcessors{}, processor) assert.IsType(t, &processors.StatsLedgerTransactionProcessor{}, processor.processors[0]) assert.IsType(t, &processors.EffectProcessor{}, processor.processors[1]) diff --git a/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor_test.go b/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor_test.go index ca918e08ea..5967ef41b1 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor_test.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor_test.go @@ -44,7 +44,7 @@ func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) SetupTest() { }, }, } - s.cbLoader = history.NewClaimableBalanceLoader() + s.cbLoader = history.NewClaimableBalanceLoader(history.ConcurrentInserts) s.processor = NewClaimableBalancesTransactionProcessor( s.cbLoader, diff --git a/services/horizon/internal/ingest/processors/effects_processor_test.go b/services/horizon/internal/ingest/processors/effects_processor_test.go index 0243768fde..276f6fcb03 100644 --- a/services/horizon/internal/ingest/processors/effects_processor_test.go +++ b/services/horizon/internal/ingest/processors/effects_processor_test.go @@ -7,11 +7,11 @@ import ( "crypto/rand" "encoding/hex" "encoding/json" + "math/big" + "strings" "testing" "github.com/guregu/null" - "math/big" - "strings" "github.com/stellar/go/keypair" "github.com/stellar/go/protocols/horizon/base" @@ -62,7 +62,7 @@ func TestEffectsProcessorTestSuiteLedger(t *testing.T) { func (s *EffectsProcessorTestSuiteLedger) SetupTest() { s.ctx = context.Background() - s.accountLoader = history.NewAccountLoader() + s.accountLoader = history.NewAccountLoader(history.ConcurrentInserts) s.mockBatchInsertBuilder = &history.MockEffectBatchInsertBuilder{} s.lcm = xdr.LedgerCloseMeta{ @@ -446,7 +446,7 @@ func TestEffectsCoversAllOperationTypes(t *testing.T) { } assert.True(t, err2 != nil || err == nil, s) }() - err = operation.ingestEffects(history.NewAccountLoader(), &history.MockEffectBatchInsertBuilder{}) + err = operation.ingestEffects(history.NewAccountLoader(history.ConcurrentInserts), &history.MockEffectBatchInsertBuilder{}) }() } @@ -468,7 +468,7 @@ func TestEffectsCoversAllOperationTypes(t *testing.T) { ledgerSequence: 1, } // calling effects should error due to the unknown operation - err := operation.ingestEffects(history.NewAccountLoader(), &history.MockEffectBatchInsertBuilder{}) + err := operation.ingestEffects(history.NewAccountLoader(history.ConcurrentInserts), &history.MockEffectBatchInsertBuilder{}) assert.Contains(t, err.Error(), "Unknown operation type") } @@ -2558,7 +2558,7 @@ type effect struct { } func assertIngestEffects(t *testing.T, operation transactionOperationWrapper, expected []effect) { - accountLoader := history.NewAccountLoader() + accountLoader := history.NewAccountLoader(history.ConcurrentInserts) mockBatchInsertBuilder := &history.MockEffectBatchInsertBuilder{} for _, expectedEffect := range expected { diff --git a/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor_test.go b/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor_test.go index 8d08e44d44..cdafc5bcc3 100644 --- a/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor_test.go +++ b/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor_test.go @@ -44,7 +44,7 @@ func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) SetupTest() { }, }, } - s.lpLoader = history.NewLiquidityPoolLoader() + s.lpLoader = history.NewLiquidityPoolLoader(history.ConcurrentInserts) s.processor = NewLiquidityPoolsTransactionProcessor( s.lpLoader, diff --git a/services/horizon/internal/ingest/processors/participants_processor_test.go b/services/horizon/internal/ingest/processors/participants_processor_test.go index b81bd22f67..f6154b2b39 100644 --- a/services/horizon/internal/ingest/processors/participants_processor_test.go +++ b/services/horizon/internal/ingest/processors/participants_processor_test.go @@ -86,7 +86,7 @@ func (s *ParticipantsProcessorTestSuiteLedger) SetupTest() { s.thirdTx.Envelope.V1.Tx.SourceAccount = aid.ToMuxedAccount() s.thirdTxID = toid.New(int32(sequence), 3, 0).ToInt64() - s.accountLoader = history.NewAccountLoader() + s.accountLoader = history.NewAccountLoader(history.ConcurrentInserts) s.addressToFuture = map[string]history.FutureAccountID{} for _, address := range s.addresses { s.addressToFuture[address] = s.accountLoader.GetFuture(address) diff --git a/services/horizon/internal/ingest/reap.go b/services/horizon/internal/ingest/reap.go index 07a61a4cde..63b56de993 100644 --- a/services/horizon/internal/ingest/reap.go +++ b/services/horizon/internal/ingest/reap.go @@ -16,6 +16,8 @@ import ( "github.com/stellar/go/toid" ) +const reapLookupTablesBatchSize = 1000 + // Reaper represents the history reaping subsystem of horizon. type Reaper struct { historyQ history.IngestionQ @@ -243,3 +245,117 @@ func (r *Reaper) deleteBatch(ctx context.Context, batchStartSeq, batchEndSeq uin r.deleteBatchDuration.Observe(elapsedSeconds) return count, nil } + +type lookupTableReaper struct { + historyQ history.IngestionQ + reapLockQ history.IngestionQ + pending atomic.Bool + logger *logpkg.Entry + + reapDurationByLookupTable *prometheus.SummaryVec + rowsReapedByLookupTable *prometheus.SummaryVec +} + +func newLookupTableReaper(dbSession db.SessionInterface) *lookupTableReaper { + return &lookupTableReaper{ + historyQ: &history.Q{dbSession.Clone()}, + reapLockQ: &history.Q{dbSession.Clone()}, + pending: atomic.Bool{}, + logger: log.WithField("subservice", "lookuptable-reaper"), + reapDurationByLookupTable: prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Namespace: "horizon", Subsystem: "ingest", Name: "reap_lookup_tables_duration_seconds", + Help: "reap lookup tables durations, sliding window = 10m", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, []string{"table", "type"}), + rowsReapedByLookupTable: prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Namespace: "horizon", Subsystem: "ingest", Name: "reap_lookup_tables_rows_reaped", + Help: "rows deleted during lookup tables reap, sliding window = 10m", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, []string{"table"}), + } +} + +func (r *lookupTableReaper) RegisterMetrics(registry *prometheus.Registry) { + registry.MustRegister( + r.reapDurationByLookupTable, + r.rowsReapedByLookupTable, + ) +} + +func (r *lookupTableReaper) deleteOrphanedRows(ctx context.Context) error { + // check if reap is already in progress on this horizon node + if !r.pending.CompareAndSwap(false, true) { + r.logger.Infof("existing reap already in progress, skipping request to start a new one") + return nil + } + defer r.pending.Store(false) + + if err := r.reapLockQ.Begin(ctx); err != nil { + return errors.Wrap(err, "error while starting reaper lock transaction") + } + defer func() { + if err := r.reapLockQ.Rollback(); err != nil { + r.logger.WithField("error", err).Error("failed to release reaper lock") + } + }() + // check if reap is already in progress on another horizon node + if acquired, err := r.reapLockQ.TryLookupTableReaperLock(ctx); err != nil { + return errors.Wrap(err, "error while acquiring reaper database lock") + } else if !acquired { + r.logger.Info("reap already in progress on another node") + return nil + } + + reapStart := time.Now() + var totalQueryDuration, totalDeleteDuration time.Duration + var totalDeleted int64 + for _, table := range []string{ + "history_accounts", "history_claimable_balances", + "history_assets", "history_liquidity_pools", + } { + startTime := time.Now() + ids, offset, err := r.historyQ.FindLookupTableRowsToReap(ctx, table, reapLookupTablesBatchSize) + if err != nil { + r.logger.WithField("table", table).WithError(err).Warn("Error finding orphaned rows") + return err + } + queryDuration := time.Since(startTime) + totalQueryDuration += queryDuration + + deleteStartTime := time.Now() + var rowsDeleted int64 + rowsDeleted, err = r.historyQ.ReapLookupTable(ctx, table, ids, offset) + if err != nil { + r.logger.WithField("table", table).WithError(err).Warn("Error deleting orphaned rows") + return err + } + deleteDuration := time.Since(deleteStartTime) + totalDeleteDuration += deleteDuration + + r.rowsReapedByLookupTable.With(prometheus.Labels{"table": table}). + Observe(float64(rowsDeleted)) + r.reapDurationByLookupTable.With(prometheus.Labels{"table": table, "type": "query"}). + Observe(float64(queryDuration.Seconds())) + r.reapDurationByLookupTable.With(prometheus.Labels{"table": table, "type": "delete"}). + Observe(float64(deleteDuration.Seconds())) + r.reapDurationByLookupTable.With(prometheus.Labels{"table": table, "type": "total"}). + Observe(float64((queryDuration + deleteDuration).Seconds())) + + r.logger.WithField("table", table). + WithField("offset", offset). + WithField("rows_deleted", rowsDeleted). + WithField("query_duration", queryDuration.Seconds()). + WithField("delete_duration", deleteDuration.Seconds()). + Info("Reaper deleted rows from lookup tables") + } + + r.rowsReapedByLookupTable.With(prometheus.Labels{"table": "total"}). + Observe(float64(totalDeleted)) + r.reapDurationByLookupTable.With(prometheus.Labels{"table": "total", "type": "query"}). + Observe(float64(totalQueryDuration.Seconds())) + r.reapDurationByLookupTable.With(prometheus.Labels{"table": "total", "type": "delete"}). + Observe(float64(totalDeleteDuration.Seconds())) + r.reapDurationByLookupTable.With(prometheus.Labels{"table": "total", "type": "total"}). + Observe(time.Since(reapStart).Seconds()) + return nil +} diff --git a/services/horizon/internal/ingest/resume_state_test.go b/services/horizon/internal/ingest/resume_state_test.go index feb5e13bb0..534ec555f6 100644 --- a/services/horizon/internal/ingest/resume_state_test.go +++ b/services/horizon/internal/ingest/resume_state_test.go @@ -402,10 +402,6 @@ func (s *ResumeTestTestSuite) TestReapingObjectsDisabled() { } func (s *ResumeTestTestSuite) TestErrorReapingObjectsIgnored() { - s.system.config.ReapLookupTables = true - defer func() { - s.system.config.ReapLookupTables = false - }() s.historyQ.On("Begin", s.ctx).Return(nil).Once() s.historyQ.On("GetLastLedgerIngest", s.ctx).Return(uint32(100), nil).Once() s.historyQ.On("GetIngestVersion", s.ctx).Return(CurrentVersion, nil).Once() @@ -425,12 +421,6 @@ func (s *ResumeTestTestSuite) TestErrorReapingObjectsIgnored() { s.historyQ.On("GetExpStateInvalid", s.ctx).Return(false, nil).Once() s.historyQ.On("RebuildTradeAggregationBuckets", s.ctx, uint32(101), uint32(101), 0).Return(nil).Once() - // Reap lookup tables: - s.ledgerBackend.On("GetLatestLedgerSequence", s.ctx).Return(uint32(101), nil) - s.historyQ.On("Begin", s.ctx).Return(nil).Once() - s.historyQ.On("GetLastLedgerIngest", s.ctx).Return(uint32(100), nil).Once() - s.historyQ.On("ReapLookupTables", mock.AnythingOfType("*context.timerCtx"), mock.Anything).Return(nil, nil, errors.New("error reaping objects")).Once() - s.historyQ.On("Rollback").Return(nil).Once() mockStats := &historyarchive.MockArchiveStats{} mockStats.On("GetBackendName").Return("name") mockStats.On("GetDownloads").Return(uint32(0)) diff --git a/support/datastore/mocks.go b/support/datastore/mocks.go index 2fa39a4712..a8c10438ab 100644 --- a/support/datastore/mocks.go +++ b/support/datastore/mocks.go @@ -29,7 +29,11 @@ func (m *MockDataStore) GetFileMetadata(ctx context.Context, path string) (map[s func (m *MockDataStore) GetFile(ctx context.Context, path string) (io.ReadCloser, error) { args := m.Called(ctx, path) - return args.Get(0).(io.ReadCloser), args.Error(1) + closer := (io.ReadCloser)(nil) + if args.Get(0) != nil { + closer = args.Get(0).(io.ReadCloser) + } + return closer, args.Error(1) } func (m *MockDataStore) PutFile(ctx context.Context, path string, in io.WriterTo, metadata map[string]string) error { diff --git a/support/db/dbtest/db.go b/support/db/dbtest/db.go index 18de073224..42f252b6f8 100644 --- a/support/db/dbtest/db.go +++ b/support/db/dbtest/db.go @@ -128,9 +128,11 @@ func checkReadOnly(t testing.TB, DSN string) { if !rows.Next() { _, err = tx.Exec("CREATE ROLE user_ro WITH LOGIN PASSWORD 'user_ro';") if err != nil { - // Handle race condition by ignoring the error if it's a duplicate key violation - if pqErr, ok := err.(*pq.Error); ok && pqErr.Code == "23505" { + // Handle race condition by ignoring the error if it's a duplicate key violation or duplicate object error + if pqErr, ok := err.(*pq.Error); ok && (pqErr.Code == "23505" || pqErr.Code == "42710") { return + } else if ok { + t.Logf("pq error code: %s", pqErr.Code) } } require.NoError(t, err)