diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index 493a1947af..514ac3278d 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -6,9 +6,9 @@ file. This project adheres to [Semantic Versioning](http://semver.org/). ## v2.6.0 -**Upgrading to this version from <= v2.1.1 will trigger a state rebuild. During this process (which can take up to 20 minutes), Horizon will not ingest new ledgers.** +**Upgrading to this version from <= v2.5.2 will trigger a state rebuild. During this process (which will take at least 10 minutes), Horizon will not ingest new ledgers.** -* Precompute trade aggregations during ingestion to improve performance. Requires a reingestion ([3641](https://github.com/stellar/go/pull/3641)). +* Precompute trade aggregations during ingestion to improve performance. Will rebuild the aggregations as part of the database migrations. ([3641](https://github.com/stellar/go/pull/3641) & [3760](https://github.com/stellar/go/pull/3760)). * Require `COUNT` param when running `horizon db migrate down COUNT` to prevent accidentally running all downwards migrations. Add `horizon db migrate status` command. ([#3737](https://github.com/stellar/go/pull/3737)) * Fix a bug in `fee_account_muxed` and `fee_account_muxed_id` fields (the fields were incorrectly populated with the source account details). ([3735](https://github.com/stellar/go/pull/3735)) * Validate ledger range when calling `horizon db reingest range` so that we respond with an error when attempting to ingest ledgers which are not available in the history archives. ([3738](https://github.com/stellar/go/pull/3738)) diff --git a/services/horizon/internal/db2/schema/bindata.go b/services/horizon/internal/db2/schema/bindata.go index 0a00af58e8..39cce4346c 100644 --- a/services/horizon/internal/db2/schema/bindata.go +++ b/services/horizon/internal/db2/schema/bindata.go @@ -42,6 +42,7 @@ // migrations/45_add_claimable_balances_history.sql (2.163kB) // migrations/46_add_muxed_accounts.sql (465B) // migrations/47_precompute_trade_aggregations.sql (1.687kB) +// migrations/48_rebuild_trade_aggregations.sql (1.243kB) // migrations/4_add_protocol_version.sql (188B) // migrations/5_create_trades_table.sql (1.1kB) // migrations/6_create_assets_table.sql (366B) @@ -957,6 +958,26 @@ func migrations47_precompute_trade_aggregationsSql() (*asset, error) { return a, nil } +var _migrations48_rebuild_trade_aggregationsSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\x94\x5f\x6f\xda\x30\x14\xc5\xdf\xfd\x29\xae\xfa\x94\x74\xd0\x31\x1e\x26\xad\x7d\xa2\x6d\xb6\x55\xea\x60\x0a\x41\x53\x55\x4d\x96\x49\x2e\xc4\xaa\xff\x20\xfb\xa6\xb4\xdf\x7e\xc2\x21\x09\xd0\xac\x7d\x41\x8e\xfd\xf3\xb9\xbe\x87\x63\x0f\x87\xf0\x49\xcb\xb5\x13\x84\xb0\xd8\x30\x36\x1c\xc2\xb5\xc8\x9f\x56\x52\x29\xa0\x12\x81\xc4\x52\x21\x6c\x25\x95\x80\x2f\xd2\x93\x34\x6b\x28\x04\x89\x0b\xc8\x4a\xe9\x81\xc4\x13\x7a\x10\x4b\x5b\x11\x7c\x03\x2d\x4d\x45\xe8\x2f\xd8\x9f\xbb\xec\x27\x90\x13\x05\x7a\x98\xcc\x21\x62\x00\xf3\xe4\x3e\xb9\xc9\x18\x00\x00\x59\xae\xa5\x52\xd2\x47\x0a\x8b\x35\x3a\x9e\x2b\xeb\xb1\xe0\x82\x06\xf0\x75\x34\x1a\x8d\x62\x10\x1e\x48\x6a\xf4\x24\xf4\x66\x10\x36\x95\xd2\x93\x75\xaf\xdc\x6e\xd0\x09\x92\xd6\x70\x59\xd4\x2b\x67\xd6\x15\xe8\xce\xea\x8f\xa5\xf0\xc8\x85\xf7\x48\xed\x7a\x3d\xa5\x6d\x65\xa8\x9e\xc8\x77\x43\x74\x27\x58\x3b\x7b\x40\x4e\xd2\x74\xf2\xf0\xb8\x71\x32\x47\x6e\x06\x50\x0f\x8a\xbf\xbb\xe3\x85\x31\x03\xf8\x9e\xce\x7e\xb5\x87\xab\x5b\x66\x00\xb3\xf4\x36\x49\xe1\xfa\xe1\xe4\x3c\x6f\x4b\xf7\xf7\xd5\xf4\xc4\xe2\x01\x38\x5c\x56\x52\xd1\xae\xe6\xa9\x8f\xc7\x0e\xf5\xb4\xfe\x4e\xa7\xd1\x79\x70\x39\xef\x9a\xf5\x95\x8e\x0e\xbc\x0a\xcb\xe1\xfb\xd9\xaa\x4a\x63\x07\x1d\x3b\xd5\xc9\xa0\xfb\x00\xbd\xbc\x34\x95\x46\x27\xf3\xf8\xf3\x49\xb1\x6e\x65\xa7\x26\x9e\xd7\xb5\x44\xa4\xc5\x0b\x0f\x4e\x47\xe1\x37\x8e\x1f\xbf\x04\xf7\x4b\xb9\x2e\xb9\xf9\x2f\x34\xee\xa0\xa2\x81\xa4\xe9\x55\x52\x76\xdb\x09\xbd\x61\xc6\x2d\xb3\xd7\x59\x49\xe7\x29\xea\xfb\xd3\xc2\xc9\xed\x06\x0d\xdf\xa7\x9a\x6c\x63\x78\x54\xef\x3a\x2e\x1c\x50\xd3\x0b\x8c\x3b\x60\xaf\xa0\xc4\x7b\x55\xc3\xfd\xe9\x29\x1b\x76\x1d\x57\xad\x51\xd3\x07\x8c\x0f\x80\xa2\x49\x76\x9b\xe8\x1f\xe9\x6c\xf1\x1b\x96\xaf\x1f\x27\xba\x8d\x25\x8b\x19\xc0\xdd\x74\x9e\xa4\x19\xdc\x4d\xb3\xd9\xc9\x35\xe1\xe1\xae\x87\x50\x37\xb1\x86\x73\x58\x39\xab\x9b\xcc\x33\x80\xf8\x2a\xbc\x49\xed\x1b\x75\x6b\xb7\x86\xb1\x2c\x5d\x4c\x6f\x26\x59\x02\xd9\xe4\xfa\x3e\xe9\xd5\xbd\x62\xff\x02\x00\x00\xff\xff\x0e\xad\x7c\x8e\xdb\x04\x00\x00") + +func migrations48_rebuild_trade_aggregationsSqlBytes() ([]byte, error) { + return bindataRead( + _migrations48_rebuild_trade_aggregationsSql, + "migrations/48_rebuild_trade_aggregations.sql", + ) +} + +func migrations48_rebuild_trade_aggregationsSql() (*asset, error) { + bytes, err := migrations48_rebuild_trade_aggregationsSqlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "migrations/48_rebuild_trade_aggregations.sql", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xed, 0xea, 0xa6, 0x74, 0x5b, 0x14, 0xa9, 0x38, 0x4d, 0x4f, 0x90, 0xe7, 0xeb, 0x65, 0x8b, 0xe3, 0xbf, 0x89, 0x12, 0x2f, 0x4b, 0x35, 0xa5, 0x6e, 0x9f, 0xb1, 0x60, 0x92, 0xc7, 0x7, 0x39, 0x7d}} + return a, nil +} + var _migrations4_add_protocol_versionSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\xcd\xb1\x0a\xc2\x30\x10\x06\xe0\x3d\x4f\xf1\xef\x52\x70\xef\x14\x4d\x9d\xce\x44\x4a\x32\x38\x15\xd1\xa3\x06\x6a\xae\x5c\x82\xe2\xdb\xbb\xba\x88\x4f\xf0\x75\x1d\x36\x8f\x3c\xeb\xa5\x31\xd2\x6a\x2c\xc5\x61\x44\xb4\x3b\x1a\x10\x3c\x9d\x71\xcf\xb5\x89\xbe\xa7\x85\x6f\x33\x6b\x85\x01\xac\x73\xd8\x07\x4a\x47\x8f\x55\xa5\xc9\x55\x96\xe9\xc9\x5a\xb3\x14\xe4\xd2\x78\x66\x85\x1b\x0e\x36\x51\xc4\x16\x3e\x44\xf8\x44\xd4\x1b\xf3\x6d\x39\x79\x95\xff\x9a\x1b\xc3\xe9\x97\xd5\x9b\x4f\x00\x00\x00\xff\xff\x83\xbb\x30\x2e\xbc\x00\x00\x00") func migrations4_add_protocol_versionSqlBytes() ([]byte, error) { @@ -1230,6 +1251,7 @@ var _bindata = map[string]func() (*asset, error){ "migrations/45_add_claimable_balances_history.sql": migrations45_add_claimable_balances_historySql, "migrations/46_add_muxed_accounts.sql": migrations46_add_muxed_accountsSql, "migrations/47_precompute_trade_aggregations.sql": migrations47_precompute_trade_aggregationsSql, + "migrations/48_rebuild_trade_aggregations.sql": migrations48_rebuild_trade_aggregationsSql, "migrations/4_add_protocol_version.sql": migrations4_add_protocol_versionSql, "migrations/5_create_trades_table.sql": migrations5_create_trades_tableSql, "migrations/6_create_assets_table.sql": migrations6_create_assets_tableSql, @@ -1323,6 +1345,7 @@ var _bintree = &bintree{nil, map[string]*bintree{ "45_add_claimable_balances_history.sql": &bintree{migrations45_add_claimable_balances_historySql, map[string]*bintree{}}, "46_add_muxed_accounts.sql": &bintree{migrations46_add_muxed_accountsSql, map[string]*bintree{}}, "47_precompute_trade_aggregations.sql": &bintree{migrations47_precompute_trade_aggregationsSql, map[string]*bintree{}}, + "48_rebuild_trade_aggregations.sql": &bintree{migrations48_rebuild_trade_aggregationsSql, map[string]*bintree{}}, "4_add_protocol_version.sql": &bintree{migrations4_add_protocol_versionSql, map[string]*bintree{}}, "5_create_trades_table.sql": &bintree{migrations5_create_trades_tableSql, map[string]*bintree{}}, "6_create_assets_table.sql": &bintree{migrations6_create_assets_tableSql, map[string]*bintree{}}, diff --git a/services/horizon/internal/db2/schema/migrations/48_rebuild_trade_aggregations.sql b/services/horizon/internal/db2/schema/migrations/48_rebuild_trade_aggregations.sql new file mode 100644 index 0000000000..4ef6b6b6fb --- /dev/null +++ b/services/horizon/internal/db2/schema/migrations/48_rebuild_trade_aggregations.sql @@ -0,0 +1,44 @@ +-- +migrate Up + +-- Backfill the table with existing data. This takes about 9 minutes. +WITH trades AS ( + SELECT + to_millis(ledger_closed_at, 60000) as timestamp, + history_operation_id, + "order", + base_asset_id, + base_amount, + counter_asset_id, + counter_amount, + ARRAY[price_n, price_d] as price + FROM history_trades + ORDER BY base_asset_id, counter_asset_id, history_operation_id, "order" +), rebuilt as ( + SELECT + timestamp, + base_asset_id, + counter_asset_id, + count(*) as count, + sum(base_amount) as base_volume, + sum(counter_amount) as counter_volume, + sum(counter_amount::numeric)/sum(base_amount::numeric) as avg, + (max_price(price))[1] as high_n, + (max_price(price))[2] as high_d, + (min_price(price))[1] as low_n, + (min_price(price))[2] as low_d, + first(history_operation_id) as open_ledger_toid, + (first(price))[1] as open_n, + (first(price))[2] as open_d, + last(history_operation_id) as close_ledger_toid, + (last(price))[1] as close_n, + (last(price))[2] as close_d + FROM trades + GROUP by base_asset_id, counter_asset_id, timestamp +) + INSERT INTO history_trades_60000 ( + SELECT * from rebuilt + ); + +-- +migrate Down + +TRUNCATE TABLE history_trades_60000;