diff --git a/README.md b/README.md index 6083abd2..7ac77d51 100644 --- a/README.md +++ b/README.md @@ -4,4 +4,4 @@ ## Required Disclosure -`explored` uses the IP2Location LITE database for IP geolocation. \ No newline at end of file +This product includes GeoLite2 data created by MaxMind, available at https://www.maxmind.com. It is provided under a [Creative Commons Corporation Attribution-ShareAlike 4.0 International License](https://www.maxmind.com/en/geolite2/eula). diff --git a/explorer/scan.go b/explorer/scan.go index 5573095e..e23341ab 100644 --- a/explorer/scan.go +++ b/explorer/scan.go @@ -10,7 +10,7 @@ import ( crhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/core/types" crhpv4 "go.sia.tech/coreutils/rhp/v4" - "go.sia.tech/explored/internal/geoip" + "go.sia.tech/explored/geoip" rhpv2 "go.sia.tech/explored/internal/rhp/v2" rhpv3 "go.sia.tech/explored/internal/rhp/v3" "go.uber.org/zap" @@ -85,10 +85,9 @@ func (e *Explorer) scanV1Host(locator geoip.Locator, host UnscannedHost) (HostSc return HostScan{}, fmt.Errorf("scanHost: failed to resolve host address: %w", err) } - countryCode, err := locator.CountryCode(resolved) + location, err := locator.Locate(resolved) if err != nil { e.log.Debug("Failed to resolve IP geolocation, not setting country code", zap.String("addr", host.NetAddress)) - countryCode = "" } v3Addr := net.JoinHostPort(hostIP, settings.SiaMuxPort) @@ -103,10 +102,10 @@ func (e *Explorer) scanV1Host(locator geoip.Locator, host UnscannedHost) (HostSc } return HostScan{ - PublicKey: host.PublicKey, - CountryCode: countryCode, - Success: true, - Timestamp: types.CurrentTimestamp(), + PublicKey: host.PublicKey, + Location: location, + Success: true, + Timestamp: types.CurrentTimestamp(), Settings: settings, PriceTable: table, @@ -143,17 +142,16 @@ func (e *Explorer) scanV2Host(locator geoip.Locator, host UnscannedHost) (HostSc return HostScan{}, fmt.Errorf("scanHost: failed to resolve host address: %w", err) } - countryCode, err := locator.CountryCode(resolved) + location, err := locator.Locate(resolved) if err != nil { e.log.Debug("Failed to resolve IP geolocation, not setting country code", zap.String("addr", host.NetAddress)) - countryCode = "" } return HostScan{ - PublicKey: host.PublicKey, - CountryCode: countryCode, - Success: true, - Timestamp: types.CurrentTimestamp(), + PublicKey: host.PublicKey, + Location: location, + Success: true, + Timestamp: types.CurrentTimestamp(), RHPV4Settings: settings, }, nil @@ -177,7 +175,7 @@ func (e *Explorer) scanHosts() { } e.log.Info("Syncing complete, will begin scanning hosts") - locator, err := geoip.NewIP2LocationLocator("") + locator, err := geoip.NewMaxMindLocator("") if err != nil { e.log.Info("failed to create geoip database:", zap.Error(err)) return diff --git a/explorer/types.go b/explorer/types.go index 30fc2f18..e2eab6ea 100644 --- a/explorer/types.go +++ b/explorer/types.go @@ -13,6 +13,7 @@ import ( "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" crhpv4 "go.sia.tech/coreutils/rhp/v4" + "go.sia.tech/explored/geoip" ) // A Source represents where a siacoin output came from. @@ -326,11 +327,11 @@ type Metrics struct { // HostScan represents the results of a host scan. type HostScan struct { - PublicKey types.PublicKey `json:"publicKey"` - CountryCode string `json:"countryCode"` - Success bool `json:"success"` - Timestamp time.Time `json:"timestamp"` - NextScan time.Time `json:"nextScan"` + PublicKey types.PublicKey `json:"publicKey"` + Location geoip.Location `json:"location"` + Success bool `json:"success"` + Timestamp time.Time `json:"timestamp"` + NextScan time.Time `json:"nextScan"` Settings rhpv2.HostSettings `json:"settings"` PriceTable rhpv3.HostPriceTable `json:"priceTable"` @@ -371,7 +372,7 @@ type Host struct { NetAddress string `json:"netAddress"` V2NetAddresses []chain.NetAddress `json:"v2NetAddresses,omitempty"` - CountryCode string `json:"countryCode"` + Location geoip.Location `json:"location"` KnownSince time.Time `json:"knownSince"` LastScan time.Time `json:"lastScan"` diff --git a/geoip/geoip.go b/geoip/geoip.go new file mode 100644 index 00000000..0517389c --- /dev/null +++ b/geoip/geoip.go @@ -0,0 +1,81 @@ +package geoip + +import ( + _ "embed" // needed for geolocation database + "errors" + "net" + "sync" + + "github.com/oschwald/geoip2-golang" +) + +//go:embed GeoLite2-City.mmdb +var maxMindCityDB []byte + +// A Location represents an ISO 3166-1 A-2 country codes and an approximate +// latitude/longitude. +type Location struct { + CountryCode string `json:"countryCode"` + + Longitude float64 `json:"longitude"` + Latitude float64 `json:"latitude"` +} + +// A Locator maps IP addresses to their location. +// It is assumed that it implementations are thread-safe. +type Locator interface { + // Close closes the Locator. + Close() error + // Locate maps IP addresses to a Location. + Locate(ip *net.IPAddr) (Location, error) +} + +type maxMindLocator struct { + mu sync.Mutex + + db *geoip2.Reader +} + +// Locate implements Locator. +func (m *maxMindLocator) Locate(addr *net.IPAddr) (Location, error) { + if addr == nil { + return Location{}, errors.New("nil IP") + } + m.mu.Lock() + defer m.mu.Unlock() + + record, err := m.db.City(addr.IP) + if err != nil { + return Location{}, err + } + return Location{ + CountryCode: record.Country.IsoCode, + Latitude: record.Location.Latitude, + Longitude: record.Location.Longitude, + }, nil +} + +// Close implements Locator. +func (m *maxMindLocator) Close() error { + m.mu.Lock() + defer m.mu.Unlock() + return m.db.Close() +} + +// NewMaxMindLocator returns a Locator that uses an underlying MaxMind +// database. If no path is provided, a default embedded GeoLite2-City database +// is used. +func NewMaxMindLocator(path string) (Locator, error) { + var db *geoip2.Reader + var err error + if path == "" { + db, err = geoip2.FromBytes(maxMindCityDB) + } else { + db, err = geoip2.Open(path) + } + if err != nil { + return nil, err + } + + return &maxMindLocator{db: db}, nil +} diff --git a/go.mod b/go.mod index d8b580c2..cdfd7823 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.23.2 require ( github.com/google/go-cmp v0.6.0 - github.com/ip2location/ip2location-go v8.3.0+incompatible github.com/mattn/go-sqlite3 v1.14.24 + github.com/oschwald/geoip2-golang v1.11.0 go.sia.tech/core v0.10.2-0.20250211180922-261f960c1315 go.sia.tech/coreutils v0.11.1 go.sia.tech/jape v0.12.1 @@ -22,6 +22,7 @@ require ( github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/onsi/ginkgo/v2 v2.12.0 // indirect + github.com/oschwald/maxminddb-golang v1.13.0 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.49.0 // indirect github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect diff --git a/go.sum b/go.sum index 2e322031..b7a2bdc8 100644 --- a/go.sum +++ b/go.sum @@ -13,8 +13,6 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ= github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= -github.com/ip2location/ip2location-go v8.3.0+incompatible h1:QwUE+FlSbo6bjOWZpv2Grb57vJhWYFNPyBj2KCvfWaM= -github.com/ip2location/ip2location-go v8.3.0+incompatible/go.mod h1:3JUY1TBjTx1GdA7oRT7Zeqfc0bg3lMMuU5lXmzdpuME= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -27,6 +25,10 @@ github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/oschwald/geoip2-golang v1.11.0 h1:hNENhCn1Uyzhf9PTmquXENiWS6AlxAEnBII6r8krA3w= +github.com/oschwald/geoip2-golang v1.11.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo= +github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU= +github.com/oschwald/maxminddb-golang v1.13.0/go.mod h1:BU0z8BfFVhi1LQaonTwwGQlsHUEu9pWNdMfmq4ztm0o= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= @@ -43,8 +45,6 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.sia.tech/core v0.10.1 h1:96lmgO50oKPiQU46H14Ga+6NYo6IB++VQ4DI3QCc6/o= -go.sia.tech/core v0.10.1/go.mod h1:FRg3rOIM8oSvf5wJoAJEgqqbTtKBDNeqL5/bH1lRuDk= go.sia.tech/core v0.10.2-0.20250211180922-261f960c1315 h1:lWYVzooojpbm5r+6vHziUALL+gvV5k3dIs/rlwd3bMk= go.sia.tech/core v0.10.2-0.20250211180922-261f960c1315/go.mod h1:FRg3rOIM8oSvf5wJoAJEgqqbTtKBDNeqL5/bH1lRuDk= go.sia.tech/coreutils v0.11.1 h1:rpR2a5oB/TRScPK9d0nBM5k2jL5/f0oy5ZgVzfyS4oo= diff --git a/internal/geoip/IP2LOCATION-LITE-DB1.BIN b/internal/geoip/IP2LOCATION-LITE-DB1.BIN deleted file mode 100755 index 4574d4bf..00000000 Binary files a/internal/geoip/IP2LOCATION-LITE-DB1.BIN and /dev/null differ diff --git a/internal/geoip/geoip.go b/internal/geoip/geoip.go deleted file mode 100644 index efccd470..00000000 --- a/internal/geoip/geoip.go +++ /dev/null @@ -1,81 +0,0 @@ -package geoip - -import ( - _ "embed" // needed for geolocation database - "errors" - "net" - "os" - "sync" - - "github.com/ip2location/ip2location-go" -) - -//go:embed IP2LOCATION-LITE-DB1.BIN -var ip2LocationDB []byte - -// A Locator maps IP addresses to their location. -// It is assumed that it implementations are thread-safe. -type Locator interface { - // Close closes the Locator. - Close() error - // CountryCode maps IP addresses to ISO 3166-1 A-2 country codes. - CountryCode(ip *net.IPAddr) (string, error) -} - -type ip2Location struct { - mu sync.Mutex - - path string - db *ip2location.DB -} - -// Close implements Locator. -func (ip *ip2Location) Close() error { - ip.mu.Lock() - defer ip.mu.Unlock() - - ip.db.Close() - return os.Remove(ip.path) -} - -// CountryCode implements Locator. -func (ip *ip2Location) CountryCode(addr *net.IPAddr) (string, error) { - if ip == nil { - return "", errors.New("nil IP") - } - ip.mu.Lock() - defer ip.mu.Unlock() - - loc, err := ip.db.Get_country_short(addr.String()) - if err != nil { - return "", err - } - return loc.Country_short, nil -} - -// NewIP2LocationLocator returns a Locator that uses an underlying IP2Location -// database. If no path is provided, a default embedded LITE database is used. -func NewIP2LocationLocator(path string) (Locator, error) { - // Unfortunately, ip2location.OpenDB only accepts a file path. So we need - // to write the embedded file to a temporary file on disk, and use that - // instead. - if path == "" { - f, err := os.CreateTemp("", "geoip") - if err != nil { - return nil, err - } else if _, err := f.Write(ip2LocationDB); err != nil { - return nil, err - } else if err := f.Sync(); err != nil { - return nil, err - } else if err := f.Close(); err != nil { - return nil, err - } - path = f.Name() - } - - db, err := ip2location.OpenDB(path) - if err != nil { - return nil, err - } - return &ip2Location{path: path, db: db}, nil -} diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index a3484414..121df5cd 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -1106,7 +1106,7 @@ func addHosts(tx *txn, hosts []explorer.Host) error { return nil } - stmt, err := tx.Prepare(`INSERT INTO host_info(public_key, v2, net_address, country_code, known_since, last_scan, last_scan_successful, next_scan, failed_interactions_streak, last_announcement, total_scans, successful_interactions, failed_interactions, settings_accepting_contracts, settings_max_download_batch_size, settings_max_duration, settings_max_revise_batch_size, settings_net_address, settings_remaining_storage, settings_sector_size, settings_total_storage, settings_used_storage, settings_address, settings_window_size, settings_collateral, settings_max_collateral, settings_base_rpc_price, settings_contract_price, settings_download_bandwidth_price, settings_sector_access_price, settings_storage_price, settings_upload_bandwidth_price, settings_ephemeral_account_expiry, settings_max_ephemeral_account_balance, settings_revision_number, settings_version, settings_release, settings_sia_mux_port, price_table_uid, price_table_validity, price_table_host_block_height, price_table_update_price_table_cost, price_table_account_balance_cost, price_table_fund_account_cost, price_table_latest_revision_cost, price_table_subscription_memory_cost, price_table_subscription_notification_cost, price_table_init_base_cost, price_table_memory_time_cost, price_table_download_bandwidth_cost, price_table_upload_bandwidth_cost, price_table_drop_sectors_base_cost, price_table_drop_sectors_unit_cost, price_table_has_sector_base_cost, price_table_read_base_cost, price_table_read_length_cost, price_table_renew_contract_cost, price_table_revision_base_cost, price_table_swap_sector_base_cost, price_table_write_base_cost, price_table_write_length_cost, price_table_write_store_cost, price_table_txn_fee_min_recommended, price_table_txn_fee_max_recommended, price_table_contract_price, price_table_collateral_cost, price_table_max_collateral, price_table_max_duration, price_table_window_size, price_table_registry_entries_left, price_table_registry_entries_total, rhp4_settings_protocol_version, rhp4_settings_release, rhp4_settings_wallet_address, rhp4_settings_accepting_contracts, rhp4_settings_max_collateral, rhp4_settings_max_contract_duration, rhp4_settings_remaining_storage, rhp4_settings_total_storage, rhp4_settings_used_storage, rhp4_prices_contract_price, rhp4_prices_collateral_price, rhp4_prices_storage_price, rhp4_prices_ingress_price, rhp4_prices_egress_price, rhp4_prices_free_sector_price, rhp4_prices_tip_height, rhp4_prices_valid_until, rhp4_prices_signature) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27,$28,$29,$30,$31,$32,$33,$34,$35,$36,$37,$38,$39,$40,$41,$42,$43,$44,$45,$46,$47,$48,$49,$50,$51,$52,$53,$54,$55,$56,$57,$58,$59,$60,$61,$62,$63,$64,$65,$66,$67,$68,$69,$70,$71,$72,$73,$74,$75,$76,$77,$78,$79,$80,$81,$82,$83,$84,$85,$86,$87,$88,$89) ON CONFLICT (public_key) DO UPDATE SET v2 = EXCLUDED.v2, net_address = EXCLUDED.net_address, last_announcement = EXCLUDED.last_announcement, next_scan = EXCLUDED.last_announcement`) + stmt, err := tx.Prepare(`INSERT INTO host_info(public_key, v2, net_address, country_code, latitude, longitude, known_since, last_scan, last_scan_successful, next_scan, failed_interactions_streak, last_announcement, total_scans, successful_interactions, failed_interactions, settings_accepting_contracts, settings_max_download_batch_size, settings_max_duration, settings_max_revise_batch_size, settings_net_address, settings_remaining_storage, settings_sector_size, settings_total_storage, settings_used_storage, settings_address, settings_window_size, settings_collateral, settings_max_collateral, settings_base_rpc_price, settings_contract_price, settings_download_bandwidth_price, settings_sector_access_price, settings_storage_price, settings_upload_bandwidth_price, settings_ephemeral_account_expiry, settings_max_ephemeral_account_balance, settings_revision_number, settings_version, settings_release, settings_sia_mux_port, price_table_uid, price_table_validity, price_table_host_block_height, price_table_update_price_table_cost, price_table_account_balance_cost, price_table_fund_account_cost, price_table_latest_revision_cost, price_table_subscription_memory_cost, price_table_subscription_notification_cost, price_table_init_base_cost, price_table_memory_time_cost, price_table_download_bandwidth_cost, price_table_upload_bandwidth_cost, price_table_drop_sectors_base_cost, price_table_drop_sectors_unit_cost, price_table_has_sector_base_cost, price_table_read_base_cost, price_table_read_length_cost, price_table_renew_contract_cost, price_table_revision_base_cost, price_table_swap_sector_base_cost, price_table_write_base_cost, price_table_write_length_cost, price_table_write_store_cost, price_table_txn_fee_min_recommended, price_table_txn_fee_max_recommended, price_table_contract_price, price_table_collateral_cost, price_table_max_collateral, price_table_max_duration, price_table_window_size, price_table_registry_entries_left, price_table_registry_entries_total, rhp4_settings_protocol_version, rhp4_settings_release, rhp4_settings_wallet_address, rhp4_settings_accepting_contracts, rhp4_settings_max_collateral, rhp4_settings_max_contract_duration, rhp4_settings_remaining_storage, rhp4_settings_total_storage, rhp4_settings_used_storage, rhp4_prices_contract_price, rhp4_prices_collateral_price, rhp4_prices_storage_price, rhp4_prices_ingress_price, rhp4_prices_egress_price, rhp4_prices_free_sector_price, rhp4_prices_tip_height, rhp4_prices_valid_until, rhp4_prices_signature) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27,$28,$29,$30,$31,$32,$33,$34,$35,$36,$37,$38,$39,$40,$41,$42,$43,$44,$45,$46,$47,$48,$49,$50,$51,$52,$53,$54,$55,$56,$57,$58,$59,$60,$61,$62,$63,$64,$65,$66,$67,$68,$69,$70,$71,$72,$73,$74,$75,$76,$77,$78,$79,$80,$81,$82,$83,$84,$85,$86,$87,$88,$89,$90,$91) ON CONFLICT (public_key) DO UPDATE SET v2 = EXCLUDED.v2, net_address = EXCLUDED.net_address, last_announcement = EXCLUDED.last_announcement, next_scan = EXCLUDED.last_announcement`) if err != nil { return fmt.Errorf("failed to prepare host_info stmt: %w", err) } @@ -1129,7 +1129,7 @@ func addHosts(tx *txn, hosts []explorer.Host) error { sV4, pV4 := host.RHPV4Settings, host.RHPV4Settings.Prices isV2 := len(host.V2NetAddresses) > 0 - if _, err := stmt.Exec(encode(host.PublicKey), isV2, host.NetAddress, host.CountryCode, encode(host.KnownSince), encode(host.LastScan), host.LastScanSuccessful, encode(host.LastAnnouncement), 0, encode(host.LastAnnouncement), host.TotalScans, host.SuccessfulInteractions, host.FailedInteractions, s.AcceptingContracts, encode(s.MaxDownloadBatchSize), encode(s.MaxDuration), encode(s.MaxReviseBatchSize), s.NetAddress, encode(s.RemainingStorage), encode(s.SectorSize), encode(s.TotalStorage), encode(s.TotalStorage-s.RemainingStorage), encode(s.Address), encode(s.WindowSize), encode(s.Collateral), encode(s.MaxCollateral), encode(s.BaseRPCPrice), encode(s.ContractPrice), encode(s.DownloadBandwidthPrice), encode(s.SectorAccessPrice), encode(s.StoragePrice), encode(s.UploadBandwidthPrice), s.EphemeralAccountExpiry, encode(s.MaxEphemeralAccountBalance), encode(s.RevisionNumber), s.Version, s.Release, s.SiaMuxPort, encode(p.UID), p.Validity, encode(p.HostBlockHeight), encode(p.UpdatePriceTableCost), encode(p.AccountBalanceCost), encode(p.FundAccountCost), encode(p.LatestRevisionCost), encode(p.SubscriptionMemoryCost), encode(p.SubscriptionNotificationCost), encode(p.InitBaseCost), encode(p.MemoryTimeCost), encode(p.DownloadBandwidthCost), encode(p.UploadBandwidthCost), encode(p.DropSectorsBaseCost), encode(p.DropSectorsUnitCost), encode(p.HasSectorBaseCost), encode(p.ReadBaseCost), encode(p.ReadLengthCost), encode(p.RenewContractCost), encode(p.RevisionBaseCost), encode(p.SwapSectorBaseCost), encode(p.WriteBaseCost), encode(p.WriteLengthCost), encode(p.WriteStoreCost), encode(p.TxnFeeMinRecommended), encode(p.TxnFeeMaxRecommended), encode(p.ContractPrice), encode(p.CollateralCost), encode(p.MaxCollateral), encode(p.MaxDuration), encode(p.WindowSize), encode(p.RegistryEntriesLeft), encode(p.RegistryEntriesTotal), sV4.ProtocolVersion[:], sV4.Release, encode(sV4.WalletAddress), sV4.AcceptingContracts, encode(sV4.MaxCollateral), encode(sV4.MaxContractDuration), encode(sV4.RemainingStorage), encode(sV4.TotalStorage), encode(sV4.TotalStorage-sV4.RemainingStorage), encode(pV4.ContractPrice), encode(pV4.Collateral), encode(pV4.StoragePrice), encode(pV4.IngressPrice), encode(pV4.EgressPrice), encode(pV4.FreeSectorPrice), encode(pV4.TipHeight), encode(pV4.ValidUntil), encode(pV4.Signature)); err != nil { + if _, err := stmt.Exec(encode(host.PublicKey), isV2, host.NetAddress, host.Location.CountryCode, host.Location.Latitude, host.Location.Longitude, encode(host.KnownSince), encode(host.LastScan), host.LastScanSuccessful, encode(host.LastAnnouncement), 0, encode(host.LastAnnouncement), host.TotalScans, host.SuccessfulInteractions, host.FailedInteractions, s.AcceptingContracts, encode(s.MaxDownloadBatchSize), encode(s.MaxDuration), encode(s.MaxReviseBatchSize), s.NetAddress, encode(s.RemainingStorage), encode(s.SectorSize), encode(s.TotalStorage), encode(s.TotalStorage-s.RemainingStorage), encode(s.Address), encode(s.WindowSize), encode(s.Collateral), encode(s.MaxCollateral), encode(s.BaseRPCPrice), encode(s.ContractPrice), encode(s.DownloadBandwidthPrice), encode(s.SectorAccessPrice), encode(s.StoragePrice), encode(s.UploadBandwidthPrice), s.EphemeralAccountExpiry, encode(s.MaxEphemeralAccountBalance), encode(s.RevisionNumber), s.Version, s.Release, s.SiaMuxPort, encode(p.UID), p.Validity, encode(p.HostBlockHeight), encode(p.UpdatePriceTableCost), encode(p.AccountBalanceCost), encode(p.FundAccountCost), encode(p.LatestRevisionCost), encode(p.SubscriptionMemoryCost), encode(p.SubscriptionNotificationCost), encode(p.InitBaseCost), encode(p.MemoryTimeCost), encode(p.DownloadBandwidthCost), encode(p.UploadBandwidthCost), encode(p.DropSectorsBaseCost), encode(p.DropSectorsUnitCost), encode(p.HasSectorBaseCost), encode(p.ReadBaseCost), encode(p.ReadLengthCost), encode(p.RenewContractCost), encode(p.RevisionBaseCost), encode(p.SwapSectorBaseCost), encode(p.WriteBaseCost), encode(p.WriteLengthCost), encode(p.WriteStoreCost), encode(p.TxnFeeMinRecommended), encode(p.TxnFeeMaxRecommended), encode(p.ContractPrice), encode(p.CollateralCost), encode(p.MaxCollateral), encode(p.MaxDuration), encode(p.WindowSize), encode(p.RegistryEntriesLeft), encode(p.RegistryEntriesTotal), sV4.ProtocolVersion[:], sV4.Release, encode(sV4.WalletAddress), sV4.AcceptingContracts, encode(sV4.MaxCollateral), encode(sV4.MaxContractDuration), encode(sV4.RemainingStorage), encode(sV4.TotalStorage), encode(sV4.TotalStorage-sV4.RemainingStorage), encode(pV4.ContractPrice), encode(pV4.Collateral), encode(pV4.StoragePrice), encode(pV4.IngressPrice), encode(pV4.EgressPrice), encode(pV4.FreeSectorPrice), encode(pV4.TipHeight), encode(pV4.ValidUntil), encode(pV4.Signature)); err != nil { return fmt.Errorf("failed to execute host_info stmt: %w", err) } @@ -1156,7 +1156,7 @@ func (s *Store) AddHostScans(scans []explorer.HostScan) error { } defer unsuccessfulStmt.Close() - successfulStmt, err := tx.Prepare(`UPDATE host_info SET country_code = ?, last_scan = ?, last_scan_successful = 1, next_scan = ?, total_scans = total_scans + 1, successful_interactions = successful_interactions + 1, failed_interactions_streak = 0, settings_accepting_contracts = ?, settings_max_download_batch_size = ?, settings_max_duration = ?, settings_max_revise_batch_size = ?, settings_net_address = ?, settings_remaining_storage = ?, settings_sector_size = ?, settings_total_storage = ?, settings_used_storage = ?, settings_address = ?, settings_window_size = ?, settings_collateral = ?, settings_max_collateral = ?, settings_base_rpc_price = ?, settings_contract_price = ?, settings_download_bandwidth_price = ?, settings_sector_access_price = ?, settings_storage_price = ?, settings_upload_bandwidth_price = ?, settings_ephemeral_account_expiry = ?, settings_max_ephemeral_account_balance = ?, settings_revision_number = ?, settings_version = ?, settings_release = ?, settings_sia_mux_port = ?, price_table_uid = ?, price_table_validity = ?, price_table_host_block_height = ?, price_table_update_price_table_cost = ?, price_table_account_balance_cost = ?, price_table_fund_account_cost = ?, price_table_latest_revision_cost = ?, price_table_subscription_memory_cost = ?, price_table_subscription_notification_cost = ?, price_table_init_base_cost = ?, price_table_memory_time_cost = ?, price_table_download_bandwidth_cost = ?, price_table_upload_bandwidth_cost = ?, price_table_drop_sectors_base_cost = ?, price_table_drop_sectors_unit_cost = ?, price_table_has_sector_base_cost = ?, price_table_read_base_cost = ?, price_table_read_length_cost = ?, price_table_renew_contract_cost = ?, price_table_revision_base_cost = ?, price_table_swap_sector_base_cost = ?, price_table_write_base_cost = ?, price_table_write_length_cost = ?, price_table_write_store_cost = ?, price_table_txn_fee_min_recommended = ?, price_table_txn_fee_max_recommended = ?, price_table_contract_price = ?, price_table_collateral_cost = ?, price_table_max_collateral = ?, price_table_max_duration = ?, price_table_window_size = ?, price_table_registry_entries_left = ?, price_table_registry_entries_total = ?, rhp4_settings_protocol_version = ?, rhp4_settings_release = ?, rhp4_settings_wallet_address = ?, rhp4_settings_accepting_contracts = ?, rhp4_settings_max_collateral = ?, rhp4_settings_max_contract_duration = ?, rhp4_settings_remaining_storage = ?, rhp4_settings_total_storage = ?, rhp4_settings_used_storage = ?, rhp4_prices_contract_price = ?, rhp4_prices_collateral_price = ?, rhp4_prices_storage_price = ?, rhp4_prices_ingress_price = ?, rhp4_prices_egress_price = ?, rhp4_prices_free_sector_price = ?, rhp4_prices_tip_height = ?, rhp4_prices_valid_until = ?, rhp4_prices_signature = ? WHERE public_key = ?`) + successfulStmt, err := tx.Prepare(`UPDATE host_info SET country_code = ?, latitude = ?, longitude = ?, last_scan = ?, last_scan_successful = 1, next_scan = ?, total_scans = total_scans + 1, successful_interactions = successful_interactions + 1, failed_interactions_streak = 0, settings_accepting_contracts = ?, settings_max_download_batch_size = ?, settings_max_duration = ?, settings_max_revise_batch_size = ?, settings_net_address = ?, settings_remaining_storage = ?, settings_sector_size = ?, settings_total_storage = ?, settings_used_storage = ?, settings_address = ?, settings_window_size = ?, settings_collateral = ?, settings_max_collateral = ?, settings_base_rpc_price = ?, settings_contract_price = ?, settings_download_bandwidth_price = ?, settings_sector_access_price = ?, settings_storage_price = ?, settings_upload_bandwidth_price = ?, settings_ephemeral_account_expiry = ?, settings_max_ephemeral_account_balance = ?, settings_revision_number = ?, settings_version = ?, settings_release = ?, settings_sia_mux_port = ?, price_table_uid = ?, price_table_validity = ?, price_table_host_block_height = ?, price_table_update_price_table_cost = ?, price_table_account_balance_cost = ?, price_table_fund_account_cost = ?, price_table_latest_revision_cost = ?, price_table_subscription_memory_cost = ?, price_table_subscription_notification_cost = ?, price_table_init_base_cost = ?, price_table_memory_time_cost = ?, price_table_download_bandwidth_cost = ?, price_table_upload_bandwidth_cost = ?, price_table_drop_sectors_base_cost = ?, price_table_drop_sectors_unit_cost = ?, price_table_has_sector_base_cost = ?, price_table_read_base_cost = ?, price_table_read_length_cost = ?, price_table_renew_contract_cost = ?, price_table_revision_base_cost = ?, price_table_swap_sector_base_cost = ?, price_table_write_base_cost = ?, price_table_write_length_cost = ?, price_table_write_store_cost = ?, price_table_txn_fee_min_recommended = ?, price_table_txn_fee_max_recommended = ?, price_table_contract_price = ?, price_table_collateral_cost = ?, price_table_max_collateral = ?, price_table_max_duration = ?, price_table_window_size = ?, price_table_registry_entries_left = ?, price_table_registry_entries_total = ?, rhp4_settings_protocol_version = ?, rhp4_settings_release = ?, rhp4_settings_wallet_address = ?, rhp4_settings_accepting_contracts = ?, rhp4_settings_max_collateral = ?, rhp4_settings_max_contract_duration = ?, rhp4_settings_remaining_storage = ?, rhp4_settings_total_storage = ?, rhp4_settings_used_storage = ?, rhp4_prices_contract_price = ?, rhp4_prices_collateral_price = ?, rhp4_prices_storage_price = ?, rhp4_prices_ingress_price = ?, rhp4_prices_egress_price = ?, rhp4_prices_free_sector_price = ?, rhp4_prices_tip_height = ?, rhp4_prices_valid_until = ?, rhp4_prices_signature = ? WHERE public_key = ?`) if err != nil { return fmt.Errorf("addHostScans: failed to prepare successful statement: %w", err) } @@ -1166,7 +1166,7 @@ func (s *Store) AddHostScans(scans []explorer.HostScan) error { s, p := scan.Settings, scan.PriceTable sV4, pV4 := scan.RHPV4Settings, scan.RHPV4Settings.Prices if scan.Success { - if _, err := successfulStmt.Exec(scan.CountryCode, encode(scan.Timestamp), encode(scan.NextScan), s.AcceptingContracts, encode(s.MaxDownloadBatchSize), encode(s.MaxDuration), encode(s.MaxReviseBatchSize), s.NetAddress, encode(s.RemainingStorage), encode(s.SectorSize), encode(s.TotalStorage), encode(s.TotalStorage-s.RemainingStorage), encode(s.Address), encode(s.WindowSize), encode(s.Collateral), encode(s.MaxCollateral), encode(s.BaseRPCPrice), encode(s.ContractPrice), encode(s.DownloadBandwidthPrice), encode(s.SectorAccessPrice), encode(s.StoragePrice), encode(s.UploadBandwidthPrice), s.EphemeralAccountExpiry, encode(s.MaxEphemeralAccountBalance), encode(s.RevisionNumber), s.Version, s.Release, s.SiaMuxPort, encode(p.UID), p.Validity, encode(p.HostBlockHeight), encode(p.UpdatePriceTableCost), encode(p.AccountBalanceCost), encode(p.FundAccountCost), encode(p.LatestRevisionCost), encode(p.SubscriptionMemoryCost), encode(p.SubscriptionNotificationCost), encode(p.InitBaseCost), encode(p.MemoryTimeCost), encode(p.DownloadBandwidthCost), encode(p.UploadBandwidthCost), encode(p.DropSectorsBaseCost), encode(p.DropSectorsUnitCost), encode(p.HasSectorBaseCost), encode(p.ReadBaseCost), encode(p.ReadLengthCost), encode(p.RenewContractCost), encode(p.RevisionBaseCost), encode(p.SwapSectorBaseCost), encode(p.WriteBaseCost), encode(p.WriteLengthCost), encode(p.WriteStoreCost), encode(p.TxnFeeMinRecommended), encode(p.TxnFeeMaxRecommended), encode(p.ContractPrice), encode(p.CollateralCost), encode(p.MaxCollateral), encode(p.MaxDuration), encode(p.WindowSize), encode(p.RegistryEntriesLeft), encode(p.RegistryEntriesTotal), sV4.ProtocolVersion[:], sV4.Release, encode(sV4.WalletAddress), sV4.AcceptingContracts, encode(sV4.MaxCollateral), encode(sV4.MaxContractDuration), encode(sV4.RemainingStorage), encode(sV4.TotalStorage), encode(sV4.TotalStorage-sV4.RemainingStorage), encode(pV4.ContractPrice), encode(pV4.Collateral), encode(pV4.StoragePrice), encode(pV4.IngressPrice), encode(pV4.EgressPrice), encode(pV4.FreeSectorPrice), encode(pV4.TipHeight), encode(pV4.ValidUntil), encode(pV4.Signature), encode(scan.PublicKey)); err != nil { + if _, err := successfulStmt.Exec(scan.Location.CountryCode, scan.Location.Latitude, scan.Location.Longitude, encode(scan.Timestamp), encode(scan.NextScan), s.AcceptingContracts, encode(s.MaxDownloadBatchSize), encode(s.MaxDuration), encode(s.MaxReviseBatchSize), s.NetAddress, encode(s.RemainingStorage), encode(s.SectorSize), encode(s.TotalStorage), encode(s.TotalStorage-s.RemainingStorage), encode(s.Address), encode(s.WindowSize), encode(s.Collateral), encode(s.MaxCollateral), encode(s.BaseRPCPrice), encode(s.ContractPrice), encode(s.DownloadBandwidthPrice), encode(s.SectorAccessPrice), encode(s.StoragePrice), encode(s.UploadBandwidthPrice), s.EphemeralAccountExpiry, encode(s.MaxEphemeralAccountBalance), encode(s.RevisionNumber), s.Version, s.Release, s.SiaMuxPort, encode(p.UID), p.Validity, encode(p.HostBlockHeight), encode(p.UpdatePriceTableCost), encode(p.AccountBalanceCost), encode(p.FundAccountCost), encode(p.LatestRevisionCost), encode(p.SubscriptionMemoryCost), encode(p.SubscriptionNotificationCost), encode(p.InitBaseCost), encode(p.MemoryTimeCost), encode(p.DownloadBandwidthCost), encode(p.UploadBandwidthCost), encode(p.DropSectorsBaseCost), encode(p.DropSectorsUnitCost), encode(p.HasSectorBaseCost), encode(p.ReadBaseCost), encode(p.ReadLengthCost), encode(p.RenewContractCost), encode(p.RevisionBaseCost), encode(p.SwapSectorBaseCost), encode(p.WriteBaseCost), encode(p.WriteLengthCost), encode(p.WriteStoreCost), encode(p.TxnFeeMinRecommended), encode(p.TxnFeeMaxRecommended), encode(p.ContractPrice), encode(p.CollateralCost), encode(p.MaxCollateral), encode(p.MaxDuration), encode(p.WindowSize), encode(p.RegistryEntriesLeft), encode(p.RegistryEntriesTotal), sV4.ProtocolVersion[:], sV4.Release, encode(sV4.WalletAddress), sV4.AcceptingContracts, encode(sV4.MaxCollateral), encode(sV4.MaxContractDuration), encode(sV4.RemainingStorage), encode(sV4.TotalStorage), encode(sV4.TotalStorage-sV4.RemainingStorage), encode(pV4.ContractPrice), encode(pV4.Collateral), encode(pV4.StoragePrice), encode(pV4.IngressPrice), encode(pV4.EgressPrice), encode(pV4.FreeSectorPrice), encode(pV4.TipHeight), encode(pV4.ValidUntil), encode(pV4.Signature), encode(scan.PublicKey)); err != nil { return fmt.Errorf("addHostScans: failed to execute successful statement: %w", err) } } else { diff --git a/persist/sqlite/hosts.go b/persist/sqlite/hosts.go index ae404d70..100ea672 100644 --- a/persist/sqlite/hosts.go +++ b/persist/sqlite/hosts.go @@ -166,7 +166,7 @@ func (st *Store) QueryHosts(params explorer.HostQuery, sortBy explorer.HostSortC whereClause = "WHERE " + strings.Join(filters, " AND ") } query := fmt.Sprintf(` - SELECT public_key,v2,net_address,country_code,known_since,last_scan,last_scan_successful,last_announcement,total_scans,successful_interactions,failed_interactions,settings_accepting_contracts,settings_max_download_batch_size,settings_max_duration,settings_max_revise_batch_size,settings_net_address,settings_remaining_storage,settings_sector_size,settings_total_storage,settings_address,settings_window_size,settings_collateral,settings_max_collateral,settings_base_rpc_price,settings_contract_price,settings_download_bandwidth_price,settings_sector_access_price,settings_storage_price,settings_upload_bandwidth_price,settings_ephemeral_account_expiry,settings_max_ephemeral_account_balance,settings_revision_number,settings_version,settings_release,settings_sia_mux_port,price_table_uid,price_table_validity,price_table_host_block_height,price_table_update_price_table_cost,price_table_account_balance_cost,price_table_fund_account_cost,price_table_latest_revision_cost,price_table_subscription_memory_cost,price_table_subscription_notification_cost,price_table_init_base_cost,price_table_memory_time_cost,price_table_download_bandwidth_cost,price_table_upload_bandwidth_cost,price_table_drop_sectors_base_cost,price_table_drop_sectors_unit_cost,price_table_has_sector_base_cost,price_table_read_base_cost,price_table_read_length_cost,price_table_renew_contract_cost,price_table_revision_base_cost,price_table_swap_sector_base_cost,price_table_write_base_cost,price_table_write_length_cost,price_table_write_store_cost,price_table_txn_fee_min_recommended,price_table_txn_fee_max_recommended,price_table_contract_price,price_table_collateral_cost,price_table_max_collateral,price_table_max_duration,price_table_window_size,price_table_registry_entries_left,price_table_registry_entries_total,rhp4_settings_protocol_version,rhp4_settings_release,rhp4_settings_wallet_address,rhp4_settings_accepting_contracts,rhp4_settings_max_collateral,rhp4_settings_max_contract_duration,rhp4_settings_remaining_storage,rhp4_settings_total_storage,rhp4_prices_contract_price,rhp4_prices_collateral_price,rhp4_prices_storage_price,rhp4_prices_ingress_price,rhp4_prices_egress_price,rhp4_prices_free_sector_price,rhp4_prices_tip_height,rhp4_prices_valid_until,rhp4_prices_signature FROM host_info + SELECT public_key,v2,net_address,country_code,latitude,longitude,known_since,last_scan,last_scan_successful,last_announcement,total_scans,successful_interactions,failed_interactions,settings_accepting_contracts,settings_max_download_batch_size,settings_max_duration,settings_max_revise_batch_size,settings_net_address,settings_remaining_storage,settings_sector_size,settings_total_storage,settings_address,settings_window_size,settings_collateral,settings_max_collateral,settings_base_rpc_price,settings_contract_price,settings_download_bandwidth_price,settings_sector_access_price,settings_storage_price,settings_upload_bandwidth_price,settings_ephemeral_account_expiry,settings_max_ephemeral_account_balance,settings_revision_number,settings_version,settings_release,settings_sia_mux_port,price_table_uid,price_table_validity,price_table_host_block_height,price_table_update_price_table_cost,price_table_account_balance_cost,price_table_fund_account_cost,price_table_latest_revision_cost,price_table_subscription_memory_cost,price_table_subscription_notification_cost,price_table_init_base_cost,price_table_memory_time_cost,price_table_download_bandwidth_cost,price_table_upload_bandwidth_cost,price_table_drop_sectors_base_cost,price_table_drop_sectors_unit_cost,price_table_has_sector_base_cost,price_table_read_base_cost,price_table_read_length_cost,price_table_renew_contract_cost,price_table_revision_base_cost,price_table_swap_sector_base_cost,price_table_write_base_cost,price_table_write_length_cost,price_table_write_store_cost,price_table_txn_fee_min_recommended,price_table_txn_fee_max_recommended,price_table_contract_price,price_table_collateral_cost,price_table_max_collateral,price_table_max_duration,price_table_window_size,price_table_registry_entries_left,price_table_registry_entries_total,rhp4_settings_protocol_version,rhp4_settings_release,rhp4_settings_wallet_address,rhp4_settings_accepting_contracts,rhp4_settings_max_collateral,rhp4_settings_max_contract_duration,rhp4_settings_remaining_storage,rhp4_settings_total_storage,rhp4_prices_contract_price,rhp4_prices_collateral_price,rhp4_prices_storage_price,rhp4_prices_ingress_price,rhp4_prices_egress_price,rhp4_prices_free_sector_price,rhp4_prices_tip_height,rhp4_prices_valid_until,rhp4_prices_signature FROM host_info %s ORDER BY (%s) %s LIMIT ? OFFSET ?`, @@ -193,7 +193,7 @@ func (st *Store) QueryHosts(params explorer.HostQuery, sortBy explorer.HostSortC s, p := &host.Settings, &host.PriceTable sV4, pV4 := &host.RHPV4Settings, &host.RHPV4Settings.Prices - if err := rows.Scan(decode(&host.PublicKey), &host.V2, &host.NetAddress, &host.CountryCode, decode(&host.KnownSince), decode(&host.LastScan), &host.LastScanSuccessful, decode(&host.LastAnnouncement), &host.TotalScans, &host.SuccessfulInteractions, &host.FailedInteractions, &s.AcceptingContracts, decode(&s.MaxDownloadBatchSize), decode(&s.MaxDuration), decode(&s.MaxReviseBatchSize), &s.NetAddress, decode(&s.RemainingStorage), decode(&s.SectorSize), decode(&s.TotalStorage), decode(&s.Address), decode(&s.WindowSize), decode(&s.Collateral), decode(&s.MaxCollateral), decode(&s.BaseRPCPrice), decode(&s.ContractPrice), decode(&s.DownloadBandwidthPrice), decode(&s.SectorAccessPrice), decode(&s.StoragePrice), decode(&s.UploadBandwidthPrice), &s.EphemeralAccountExpiry, decode(&s.MaxEphemeralAccountBalance), decode(&s.RevisionNumber), &s.Version, &s.Release, &s.SiaMuxPort, decode(&p.UID), &p.Validity, decode(&p.HostBlockHeight), decode(&p.UpdatePriceTableCost), decode(&p.AccountBalanceCost), decode(&p.FundAccountCost), decode(&p.LatestRevisionCost), decode(&p.SubscriptionMemoryCost), decode(&p.SubscriptionNotificationCost), decode(&p.InitBaseCost), decode(&p.MemoryTimeCost), decode(&p.DownloadBandwidthCost), decode(&p.UploadBandwidthCost), decode(&p.DropSectorsBaseCost), decode(&p.DropSectorsUnitCost), decode(&p.HasSectorBaseCost), decode(&p.ReadBaseCost), decode(&p.ReadLengthCost), decode(&p.RenewContractCost), decode(&p.RevisionBaseCost), decode(&p.SwapSectorBaseCost), decode(&p.WriteBaseCost), decode(&p.WriteLengthCost), decode(&p.WriteStoreCost), decode(&p.TxnFeeMinRecommended), decode(&p.TxnFeeMaxRecommended), decode(&p.ContractPrice), decode(&p.CollateralCost), decode(&p.MaxCollateral), decode(&p.MaxDuration), decode(&p.WindowSize), decode(&p.RegistryEntriesLeft), decode(&p.RegistryEntriesTotal), &protocolVersion, &sV4.Release, decode(&sV4.WalletAddress), &sV4.AcceptingContracts, decode(&sV4.MaxCollateral), decode(&sV4.MaxContractDuration), decode(&sV4.RemainingStorage), decode(&sV4.TotalStorage), decode(&pV4.ContractPrice), decode(&pV4.Collateral), decode(&pV4.StoragePrice), decode(&pV4.IngressPrice), decode(&pV4.EgressPrice), decode(&pV4.FreeSectorPrice), decode(&pV4.TipHeight), decode(&pV4.ValidUntil), decode(&pV4.Signature)); err != nil { + if err := rows.Scan(decode(&host.PublicKey), &host.V2, &host.NetAddress, &host.Location.CountryCode, &host.Location.Latitude, &host.Location.Longitude, decode(&host.KnownSince), decode(&host.LastScan), &host.LastScanSuccessful, decode(&host.LastAnnouncement), &host.TotalScans, &host.SuccessfulInteractions, &host.FailedInteractions, &s.AcceptingContracts, decode(&s.MaxDownloadBatchSize), decode(&s.MaxDuration), decode(&s.MaxReviseBatchSize), &s.NetAddress, decode(&s.RemainingStorage), decode(&s.SectorSize), decode(&s.TotalStorage), decode(&s.Address), decode(&s.WindowSize), decode(&s.Collateral), decode(&s.MaxCollateral), decode(&s.BaseRPCPrice), decode(&s.ContractPrice), decode(&s.DownloadBandwidthPrice), decode(&s.SectorAccessPrice), decode(&s.StoragePrice), decode(&s.UploadBandwidthPrice), &s.EphemeralAccountExpiry, decode(&s.MaxEphemeralAccountBalance), decode(&s.RevisionNumber), &s.Version, &s.Release, &s.SiaMuxPort, decode(&p.UID), &p.Validity, decode(&p.HostBlockHeight), decode(&p.UpdatePriceTableCost), decode(&p.AccountBalanceCost), decode(&p.FundAccountCost), decode(&p.LatestRevisionCost), decode(&p.SubscriptionMemoryCost), decode(&p.SubscriptionNotificationCost), decode(&p.InitBaseCost), decode(&p.MemoryTimeCost), decode(&p.DownloadBandwidthCost), decode(&p.UploadBandwidthCost), decode(&p.DropSectorsBaseCost), decode(&p.DropSectorsUnitCost), decode(&p.HasSectorBaseCost), decode(&p.ReadBaseCost), decode(&p.ReadLengthCost), decode(&p.RenewContractCost), decode(&p.RevisionBaseCost), decode(&p.SwapSectorBaseCost), decode(&p.WriteBaseCost), decode(&p.WriteLengthCost), decode(&p.WriteStoreCost), decode(&p.TxnFeeMinRecommended), decode(&p.TxnFeeMaxRecommended), decode(&p.ContractPrice), decode(&p.CollateralCost), decode(&p.MaxCollateral), decode(&p.MaxDuration), decode(&p.WindowSize), decode(&p.RegistryEntriesLeft), decode(&p.RegistryEntriesTotal), &protocolVersion, &sV4.Release, decode(&sV4.WalletAddress), &sV4.AcceptingContracts, decode(&sV4.MaxCollateral), decode(&sV4.MaxContractDuration), decode(&sV4.RemainingStorage), decode(&sV4.TotalStorage), decode(&pV4.ContractPrice), decode(&pV4.Collateral), decode(&pV4.StoragePrice), decode(&pV4.IngressPrice), decode(&pV4.EgressPrice), decode(&pV4.FreeSectorPrice), decode(&pV4.TipHeight), decode(&pV4.ValidUntil), decode(&pV4.Signature)); err != nil { return err } sV4.ProtocolVersion = [3]uint8(protocolVersion) diff --git a/persist/sqlite/hosts_test.go b/persist/sqlite/hosts_test.go index e993e788..407167e6 100644 --- a/persist/sqlite/hosts_test.go +++ b/persist/sqlite/hosts_test.go @@ -13,6 +13,7 @@ import ( "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" "go.sia.tech/explored/explorer" + "go.sia.tech/explored/geoip" "go.uber.org/zap/zaptest" ) @@ -35,10 +36,14 @@ func TestQueryHosts(t *testing.T) { tm := time.Now() hosts := []explorer.Host{ { - PublicKey: pk1, - V2: false, - NetAddress: "host1.com:9982", - CountryCode: "US", + PublicKey: pk1, + V2: false, + NetAddress: "host1.com:9982", + Location: geoip.Location{ + CountryCode: "US", + Latitude: 0.01, + Longitude: -0.02, + }, KnownSince: tm.Add(-4 * time.Hour), LastScan: tm, LastScanSuccessful: true, @@ -58,10 +63,14 @@ func TestQueryHosts(t *testing.T) { }, }, { - PublicKey: pk2, - V2: false, - NetAddress: "host2.com:9982", - CountryCode: "US", + PublicKey: pk2, + V2: false, + NetAddress: "host2.com:9982", + Location: geoip.Location{ + CountryCode: "US", + Latitude: 0.01, + Longitude: -0.02, + }, KnownSince: tm.Add(-3 * time.Hour), LastScan: tm, LastScanSuccessful: true, @@ -81,10 +90,14 @@ func TestQueryHosts(t *testing.T) { }, }, { - PublicKey: pk3, - V2: true, - V2NetAddresses: []chain.NetAddress{{Protocol: crhpv4.ProtocolTCPSiaMux, Address: "host3.com:9982"}}, - CountryCode: "DE", + PublicKey: pk3, + V2: true, + V2NetAddresses: []chain.NetAddress{{Protocol: crhpv4.ProtocolTCPSiaMux, Address: "host3.com:9982"}}, + Location: geoip.Location{ + CountryCode: "DE", + Latitude: 0.05, + Longitude: -0.10, + }, KnownSince: tm.Add(-2 * time.Hour), LastScan: tm, LastScanSuccessful: false, @@ -104,10 +117,14 @@ func TestQueryHosts(t *testing.T) { }, }, { - PublicKey: pk4, - V2: true, - V2NetAddresses: []chain.NetAddress{{Protocol: crhpv4.ProtocolTCPSiaMux, Address: "host4.com:9982"}}, - CountryCode: "DE", + PublicKey: pk4, + V2: true, + V2NetAddresses: []chain.NetAddress{{Protocol: crhpv4.ProtocolTCPSiaMux, Address: "host4.com:9982"}}, + Location: geoip.Location{ + CountryCode: "DE", + Latitude: 0.05, + Longitude: -0.10, + }, KnownSince: tm.Add(-1 * time.Hour), LastScan: tm, LastScanSuccessful: false, diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 7cca9cfd..b828c820 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -478,6 +478,8 @@ CREATE TABLE host_info ( v2 INTEGER NOT NULL, net_address TEXT NOT NULL, country_code TEXT NOT NULL, + latitude REAL NOT NULL, + longitude REAL NOT NULL, known_since INTEGER NOT NULL, last_scan INTEGER NOT NULL, last_scan_successful INTEGER NOT NULL, diff --git a/persist/sqlite/scan_test.go b/persist/sqlite/scan_test.go index 887ece4c..933bb372 100644 --- a/persist/sqlite/scan_test.go +++ b/persist/sqlite/scan_test.go @@ -269,7 +269,7 @@ func TestScan(t *testing.T) { testutil.Equal(t, "host2.KnownSince", b2.Timestamp, host1.KnownSince) testutil.Equal(t, "host1.LastAnnouncement", b2.Timestamp, host1.LastAnnouncement) if !host1.RHPV4Settings.AcceptingContracts { - log.Fatal("AcceptingContracts = false on host that's supposed to be active") + t.Fatal("AcceptingContracts = false on host that's supposed to be active") } host2 := dbHosts[1] @@ -285,7 +285,10 @@ func TestScan(t *testing.T) { host3 := dbHosts[2] testutil.Equal(t, "host3.NetAddress", "sia1.euregiohosting.nl:9982", host3.NetAddress) testutil.Equal(t, "host3.PublicKey", pubkey1, host3.PublicKey) - testutil.Equal(t, "host3.CountryCode", "NL", host3.CountryCode) + testutil.Equal(t, "host3.Location.CountryCode", "NL", host3.Location.CountryCode) + if host3.Location.Latitude == 0 || host3.Location.Longitude == 0 { + t.Fatalf("Unset latitude/longitude: %v", host3.Location) + } testutil.Equal(t, "host3.TotalScans", 1, host3.TotalScans) testutil.Equal(t, "host3.SuccessfulInteractions", 1, host3.SuccessfulInteractions) testutil.Equal(t, "host3.FailedInteractions", 0, host3.FailedInteractions) @@ -293,7 +296,7 @@ func TestScan(t *testing.T) { testutil.Equal(t, "host3.KnownSince", b1.Timestamp, host2.KnownSince) testutil.Equal(t, "host3.LastAnnouncement", b1.Timestamp, host3.LastAnnouncement) if host3.Settings.SectorSize <= 0 { - log.Fatal("SectorSize = 0 on host that's supposed to be active") + t.Fatal("SectorSize = 0 on host that's supposed to be active") } } @@ -326,7 +329,7 @@ func TestScan(t *testing.T) { testutil.Equal(t, "host1.LastAnnouncement", b4.Timestamp, host1.LastAnnouncement) // settings should not be overwritten if there was not a successful scan if !host1.RHPV4Settings.AcceptingContracts { - log.Fatal("AcceptingContracts = false on host that's supposed to be active") + t.Fatal("AcceptingContracts = false on host that's supposed to be active") } host2 := dbHosts[1] @@ -338,7 +341,7 @@ func TestScan(t *testing.T) { testutil.Equal(t, "host3.LastAnnouncement", b3.Timestamp, host3.LastAnnouncement) // settings should not be overwritten if there was not a successful scan if host3.Settings.SectorSize <= 0 { - log.Fatal("SectorSize = 0 on host that's supposed to be active") + t.Fatal("SectorSize = 0 on host that's supposed to be active") } }