Skip to content

Commit

Permalink
Merge pull request #171 from SiaFoundation/christopher/add-lat-lon
Browse files Browse the repository at this point in the history
Use MaxMind instead of IP2Location for geolocation, include coordinates
  • Loading branch information
n8maninger authored Feb 18, 2025
2 parents 28219be + 79644e9 commit b118cd4
Show file tree
Hide file tree
Showing 14 changed files with 156 additions and 134 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@

## Required Disclosure

`explored` uses the IP2Location LITE database for <a href="https://lite.ip2location.com">IP geolocation</a>.
This product includes GeoLite2 data created by MaxMind, available at https://www.maxmind.com. It is provided under a [Creative Commons Corporation Attribution-ShareAlike 4.0 International License](https://www.maxmind.com/en/geolite2/eula).
26 changes: 12 additions & 14 deletions explorer/scan.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
crhpv2 "go.sia.tech/core/rhp/v2"
"go.sia.tech/core/types"
crhpv4 "go.sia.tech/coreutils/rhp/v4"
"go.sia.tech/explored/internal/geoip"
"go.sia.tech/explored/geoip"
rhpv2 "go.sia.tech/explored/internal/rhp/v2"
rhpv3 "go.sia.tech/explored/internal/rhp/v3"
"go.uber.org/zap"
Expand Down Expand Up @@ -85,10 +85,9 @@ func (e *Explorer) scanV1Host(locator geoip.Locator, host UnscannedHost) (HostSc
return HostScan{}, fmt.Errorf("scanHost: failed to resolve host address: %w", err)
}

countryCode, err := locator.CountryCode(resolved)
location, err := locator.Locate(resolved)
if err != nil {
e.log.Debug("Failed to resolve IP geolocation, not setting country code", zap.String("addr", host.NetAddress))
countryCode = ""
}

v3Addr := net.JoinHostPort(hostIP, settings.SiaMuxPort)
Expand All @@ -103,10 +102,10 @@ func (e *Explorer) scanV1Host(locator geoip.Locator, host UnscannedHost) (HostSc
}

return HostScan{
PublicKey: host.PublicKey,
CountryCode: countryCode,
Success: true,
Timestamp: types.CurrentTimestamp(),
PublicKey: host.PublicKey,
Location: location,
Success: true,
Timestamp: types.CurrentTimestamp(),

Settings: settings,
PriceTable: table,
Expand Down Expand Up @@ -143,17 +142,16 @@ func (e *Explorer) scanV2Host(locator geoip.Locator, host UnscannedHost) (HostSc
return HostScan{}, fmt.Errorf("scanHost: failed to resolve host address: %w", err)
}

countryCode, err := locator.CountryCode(resolved)
location, err := locator.Locate(resolved)
if err != nil {
e.log.Debug("Failed to resolve IP geolocation, not setting country code", zap.String("addr", host.NetAddress))
countryCode = ""
}

return HostScan{
PublicKey: host.PublicKey,
CountryCode: countryCode,
Success: true,
Timestamp: types.CurrentTimestamp(),
PublicKey: host.PublicKey,
Location: location,
Success: true,
Timestamp: types.CurrentTimestamp(),

RHPV4Settings: settings,
}, nil
Expand All @@ -177,7 +175,7 @@ func (e *Explorer) scanHosts() {
}
e.log.Info("Syncing complete, will begin scanning hosts")

locator, err := geoip.NewIP2LocationLocator("")
locator, err := geoip.NewMaxMindLocator("")
if err != nil {
e.log.Info("failed to create geoip database:", zap.Error(err))
return
Expand Down
13 changes: 7 additions & 6 deletions explorer/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"go.sia.tech/core/types"
"go.sia.tech/coreutils/chain"
crhpv4 "go.sia.tech/coreutils/rhp/v4"
"go.sia.tech/explored/geoip"
)

// A Source represents where a siacoin output came from.
Expand Down Expand Up @@ -326,11 +327,11 @@ type Metrics struct {

// HostScan represents the results of a host scan.
type HostScan struct {
PublicKey types.PublicKey `json:"publicKey"`
CountryCode string `json:"countryCode"`
Success bool `json:"success"`
Timestamp time.Time `json:"timestamp"`
NextScan time.Time `json:"nextScan"`
PublicKey types.PublicKey `json:"publicKey"`
Location geoip.Location `json:"location"`
Success bool `json:"success"`
Timestamp time.Time `json:"timestamp"`
NextScan time.Time `json:"nextScan"`

Settings rhpv2.HostSettings `json:"settings"`
PriceTable rhpv3.HostPriceTable `json:"priceTable"`
Expand Down Expand Up @@ -371,7 +372,7 @@ type Host struct {
NetAddress string `json:"netAddress"`
V2NetAddresses []chain.NetAddress `json:"v2NetAddresses,omitempty"`

CountryCode string `json:"countryCode"`
Location geoip.Location `json:"location"`

KnownSince time.Time `json:"knownSince"`
LastScan time.Time `json:"lastScan"`
Expand Down
Binary file added geoip/GeoLite2-City.mmdb
Binary file not shown.
81 changes: 81 additions & 0 deletions geoip/geoip.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
package geoip

import (
_ "embed" // needed for geolocation database
"errors"
"net"
"sync"

"github.com/oschwald/geoip2-golang"
)

//go:embed GeoLite2-City.mmdb
var maxMindCityDB []byte

// A Location represents an ISO 3166-1 A-2 country codes and an approximate
// latitude/longitude.
type Location struct {
CountryCode string `json:"countryCode"`

Latitude float64 `json:"latitude"`
Longitude float64 `json:"longitude"`
}

// A Locator maps IP addresses to their location.
// It is assumed that it implementations are thread-safe.
type Locator interface {
// Close closes the Locator.
Close() error
// Locate maps IP addresses to a Location.
Locate(ip *net.IPAddr) (Location, error)
}

type maxMindLocator struct {
mu sync.Mutex

db *geoip2.Reader
}

// Locate implements Locator.
func (m *maxMindLocator) Locate(addr *net.IPAddr) (Location, error) {
if addr == nil {
return Location{}, errors.New("nil IP")
}
m.mu.Lock()
defer m.mu.Unlock()

record, err := m.db.City(addr.IP)
if err != nil {
return Location{}, err
}
return Location{
CountryCode: record.Country.IsoCode,
Latitude: record.Location.Latitude,
Longitude: record.Location.Longitude,
}, nil
}

// Close implements Locator.
func (m *maxMindLocator) Close() error {
m.mu.Lock()
defer m.mu.Unlock()
return m.db.Close()
}

// NewMaxMindLocator returns a Locator that uses an underlying MaxMind
// database. If no path is provided, a default embedded GeoLite2-City database
// is used.
func NewMaxMindLocator(path string) (Locator, error) {
var db *geoip2.Reader
var err error
if path == "" {
db, err = geoip2.FromBytes(maxMindCityDB)
} else {
db, err = geoip2.Open(path)
}
if err != nil {
return nil, err
}

return &maxMindLocator{db: db}, nil
}
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ toolchain go1.23.2

require (
github.com/google/go-cmp v0.6.0
github.com/ip2location/ip2location-go v8.3.0+incompatible
github.com/mattn/go-sqlite3 v1.14.24
github.com/oschwald/geoip2-golang v1.11.0
go.sia.tech/core v0.10.2-0.20250211180922-261f960c1315
go.sia.tech/coreutils v0.11.1
go.sia.tech/jape v0.12.1
Expand All @@ -22,6 +22,7 @@ require (
github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect
github.com/julienschmidt/httprouter v1.3.0 // indirect
github.com/onsi/ginkgo/v2 v2.12.0 // indirect
github.com/oschwald/maxminddb-golang v1.13.0 // indirect
github.com/quic-go/qpack v0.5.1 // indirect
github.com/quic-go/quic-go v0.49.0 // indirect
github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect
Expand Down
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ=
github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
github.com/ip2location/ip2location-go v8.3.0+incompatible h1:QwUE+FlSbo6bjOWZpv2Grb57vJhWYFNPyBj2KCvfWaM=
github.com/ip2location/ip2location-go v8.3.0+incompatible/go.mod h1:3JUY1TBjTx1GdA7oRT7Zeqfc0bg3lMMuU5lXmzdpuME=
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
Expand All @@ -27,6 +25,10 @@ github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI
github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ=
github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
github.com/oschwald/geoip2-golang v1.11.0 h1:hNENhCn1Uyzhf9PTmquXENiWS6AlxAEnBII6r8krA3w=
github.com/oschwald/geoip2-golang v1.11.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo=
github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU=
github.com/oschwald/maxminddb-golang v1.13.0/go.mod h1:BU0z8BfFVhi1LQaonTwwGQlsHUEu9pWNdMfmq4ztm0o=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
Expand All @@ -43,8 +45,6 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk=
go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk=
go.sia.tech/core v0.10.1 h1:96lmgO50oKPiQU46H14Ga+6NYo6IB++VQ4DI3QCc6/o=
go.sia.tech/core v0.10.1/go.mod h1:FRg3rOIM8oSvf5wJoAJEgqqbTtKBDNeqL5/bH1lRuDk=
go.sia.tech/core v0.10.2-0.20250211180922-261f960c1315 h1:lWYVzooojpbm5r+6vHziUALL+gvV5k3dIs/rlwd3bMk=
go.sia.tech/core v0.10.2-0.20250211180922-261f960c1315/go.mod h1:FRg3rOIM8oSvf5wJoAJEgqqbTtKBDNeqL5/bH1lRuDk=
go.sia.tech/coreutils v0.11.1 h1:rpR2a5oB/TRScPK9d0nBM5k2jL5/f0oy5ZgVzfyS4oo=
Expand Down
Binary file removed internal/geoip/IP2LOCATION-LITE-DB1.BIN
Binary file not shown.
81 changes: 0 additions & 81 deletions internal/geoip/geoip.go

This file was deleted.

Loading

0 comments on commit b118cd4

Please sign in to comment.