diff --git a/.gitignore b/.gitignore index 1c7755a..1624ee6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ *~ *# -*.dat \ No newline at end of file +*.dat +/.idea/ +/udger.iml diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..c95b9aa --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module udger + +go 1.15 + +require ( + github.com/glenn-brown/golang-pkg-pcre v0.0.0-20120522223659-48bb82a8b8ce + github.com/mattn/go-sqlite3 v1.14.4 + github.com/smartystreets/goconvey v1.6.4 + github.com/udger/udger v0.0.0-20170323112903-41a723f8095d +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..d26ad89 --- /dev/null +++ b/go.sum @@ -0,0 +1,19 @@ +github.com/glenn-brown/golang-pkg-pcre v0.0.0-20120522223659-48bb82a8b8ce h1:MS/JOOAHf4U2iKl8+1+vzUcG9t9ru1hnZJ9NEBDvMnY= +github.com/glenn-brown/golang-pkg-pcre v0.0.0-20120522223659-48bb82a8b8ce/go.mod h1:5385NDJ+Gt5loLrAlc8Rr5lKA1L5BE5O94jfdwEX9kg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/mattn/go-sqlite3 v1.14.4 h1:4rQjbDxdu9fSgI/r3KN72G3c2goxknAqHHgPWWs8UlI= +github.com/mattn/go-sqlite3 v1.14.4/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/udger/udger v0.0.0-20170323112903-41a723f8095d h1:JaY8FwQAhxM+gyoLCVMIbQbdVYCt7c+/CCNWZ2XaGRc= +github.com/udger/udger v0.0.0-20170323112903-41a723f8095d/go.mod h1:etMsQwePZP/2ZZTQhbGddwzcLSIxtpR1Hc5c2vDlkCc= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= diff --git a/types.go b/types.go index fa8debc..35d3c7a 100644 --- a/types.go +++ b/types.go @@ -3,21 +3,28 @@ package udger import ( "database/sql" - _ "github.com/mattn/go-sqlite3" "github.com/glenn-brown/golang-pkg-pcre/src/pkg/pcre" + _ "github.com/mattn/go-sqlite3" ) // Udger contains the data and exposes the Lookup(ua string) function type Udger struct { - db *sql.DB - rexBrowsers []rexData - rexDevices []rexData - rexOS []rexData - browserTypes map[int]string - browserOS map[int]int - Browsers map[int]Browser - OS map[int]OS - Devices map[int]Device + db *sql.DB + rexBrowsers []rexData + rexDevices []rexData + rexOS []rexData + browserTypes map[int]string + browserOS map[int]int + Browsers map[int]Browser + OS map[int]OS + Devices map[int]Device + IP map[string]IP + IPClass map[int]IPClass + Crawler map[int]Crawler + CrawlerClass map[int]CrawlerClass + DataCenter map[int]DataCenter + DataCenterRange []DataCenterRange + DataCenterRange6 []DataCenterRange6 } // Info is the struct returned by the Lookup(ua string) function, contains everything about the UA @@ -58,3 +65,92 @@ type Device struct { Name string `json:"name"` Icon string `json:"icon"` } + +type IPInfo struct { + IP IP `json:"ip"` + IPClass IPClass `json:"ip_class"` + Crawler Crawler `json:"crawler"` + CrawlerClass CrawlerClass `json:"crawler_class"` + DataCenter DataCenter `json:"data_center"` + DataCenterRange DataCenterRange `json:"data_center_range"` + DataCenterRange6 DataCenterRange6 `json:"data_center_range6"` +} + +// Device contains all the information about the device type +type IP struct { + IP string `json:"ip"` + ClassID int `json:"class_id"` + CrawlerID int `json:"crawler_id"` + IPLastSeen string `json:"ip_last_seen"` + IPHostname string `json:"ip_hostname"` + IPCountry string `json:"ip_country"` + IPCity string `json:"ip_city"` + IPCountryCode string `json:"ip_country_code"` +} + +type Crawler struct { + ID int `json:"id"` + UA string `json:"ua_string"` + Ver string `json:"ver"` + VerMajor string `json:"ver_major"` + ClassID int `json:"class_id"` + LastSeen string `json:"last_seen"` + RespectRobotstxt string `json:"respect_robotstxt"` + Family string `json:"family"` + FamilyCode string `json:"family_code"` + FamilyHomepage string `json:"family_homepage"` + FamilyIcon string `json:"family_icon"` + Vendor string `json:"vendor"` + VendorCode string `json:"vendor_code"` + VendorHomepage string `json:"vendor_homepage"` + Name string `json:"name"` +} + +type IPClass struct { + ID int `json:"id"` + IPClassification string `json:"ip_classification"` + IPClassificationCode string `json:"ip_classification_code"` +} + +type CrawlerClass struct { + ID int `json:"id"` + CrawlerClassification string `json:"crawler_classification"` + CrawlerClassificationCode string `json:"crawler_classification_code"` +} + +type DataCenter struct { + ID int `json:"id"` + Name string `json:"name"` + NameCode string `json:"name_code"` + Homepage string `json:"homepage"` +} + +type DataCenterRange struct { + DatacenterID int `json:"datacenter_id"` + IPFrom string `json:"ip_from"` + IPTo string `json:"ip_to"` + IPLongFrom int `json:"iplong_from"` + IPLongTo int `json:"iplong_to"` +} + +type DataCenterRange6 struct { + DatacenterID int `json:"datacenter_id"` + IPFrom string `json:"ip_from"` + IPTo string `json:"ip_to"` + IPLongFrom0 int `json:"iplong_from0"` + IPLongFrom1 int `json:"iplong_from1"` + IPLongFrom2 int `json:"iplong_from2"` + IPLongFrom3 int `json:"iplong_from3"` + IPLongFrom4 int `json:"iplong_from4"` + IPLongFrom5 int `json:"iplong_from5"` + IPLongFrom6 int `json:"iplong_from6"` + IPLongFrom7 int `json:"iplong_from7"` + IPLongTo0 int `json:"iplong_to0"` + IPLongTo1 int `json:"iplong_to1"` + IPLongTo2 int `json:"iplong_to2"` + IPLongTo3 int `json:"iplong_to3"` + IPLongTo4 int `json:"iplong_to4"` + IPLongTo5 int `json:"iplong_to5"` + IPLongTo6 int `json:"iplong_to6"` + IPLongTo7 int `json:"iplong_to7"` +} diff --git a/udger.go b/udger.go index fbc1241..39c2a6e 100644 --- a/udger.go +++ b/udger.go @@ -2,8 +2,11 @@ package udger import ( + "bytes" "database/sql" + "encoding/binary" "errors" + "net" "os" "strings" @@ -14,11 +17,18 @@ import ( // you need to pass the sqlite database in parameter func New(dbPath string) (*Udger, error) { u := &Udger{ - Browsers: make(map[int]Browser), - OS: make(map[int]OS), - Devices: make(map[int]Device), - browserTypes: make(map[int]string), - browserOS: make(map[int]int), + Browsers: make(map[int]Browser), + OS: make(map[int]OS), + Devices: make(map[int]Device), + IP: make(map[string]IP), + IPClass: make(map[int]IPClass), + Crawler: make(map[int]Crawler), + CrawlerClass: make(map[int]CrawlerClass), + DataCenter: make(map[int]DataCenter), + DataCenterRange: make([]DataCenterRange, 0), + DataCenterRange6: make([]DataCenterRange6, 0), + browserTypes: make(map[int]string), + browserOS: make(map[int]int), } var err error @@ -92,6 +102,74 @@ func (udger *Udger) Lookup(ua string) (*Info, error) { return info, nil } +func (udger *Udger) LookupIP(ip net.IP) (*IPInfo, error) { + info := &IPInfo{} + + var ipVersion byte + if ip.To4() == nil { + ipVersion = 6 + } else { + ipVersion = 4 + } + + uIP, ok := udger.IP[ip.String()] + if ok { + info.IP = uIP + uIPClass, classok := udger.IPClass[uIP.ClassID] + if classok { + info.IPClass = uIPClass + } + uCrawler, crawlerok := udger.Crawler[uIP.CrawlerID] + if crawlerok { + info.Crawler = uCrawler + uCrawlerClass, crawlerclassok := udger.CrawlerClass[uCrawler.ClassID] + if crawlerclassok { + info.CrawlerClass = uCrawlerClass + } + } + } + + if ipVersion == 4 { + ip4 := ip.To4() + + if ip4 != nil { + ipInt := int(binary.BigEndian.Uint32(ip4)) + for _, dcr := range udger.DataCenterRange { + if ipInt >= dcr.IPLongFrom && ipInt <= dcr.IPLongTo { + info.DataCenterRange = dcr + dc, ok := udger.DataCenter[dcr.DatacenterID] + if ok { + info.DataCenter = dc + } + break + } + } + } + } else { + ip16 := ip.To16() + + if ip16 != nil { + for _, dcr := range udger.DataCenterRange6 { + from16 := net.ParseIP(dcr.IPFrom).To16() + to16 := net.ParseIP(dcr.IPTo).To16() + if from16 != nil && to16 != nil { + if bytes.Compare(ip16, from16) >= 0 && bytes.Compare(ip16, to16) <= 0 { + info.DataCenterRange6 = dcr + dc, ok := udger.DataCenter[dcr.DatacenterID] + if ok { + info.DataCenter = dc + } + break + } + } + } + } + } + + return info, nil +} + + func (udger *Udger) cleanRegex(r string) string { if strings.HasSuffix(r, "/si") { r = r[:len(r)-3] @@ -245,5 +323,82 @@ func (udger *Udger) init() error { } rows.Close() + rows, err = udger.db.Query("SELECT ip, class_id, crawler_id, ip_last_seen, ip_hostname, ip_country, ip_city, ip_country_code FROM udger_ip_list") + if err != nil { + return err + } + for rows.Next() { + var ip IP + rows.Scan(&ip.IP, &ip.ClassID, &ip.CrawlerID, &ip.IPLastSeen, &ip.IPHostname, &ip.IPCountry, &ip.IPCity, &ip.IPCountryCode) + udger.IP[ip.IP] = ip + } + rows.Close() + + rows, err = udger.db.Query("SELECT id, ua_string, ver, ver_major, class_id, last_seen, respect_robotstxt, family, family_code, family_homepage, family_icon, vendor, vendor_code, vendor_homepage, name FROM udger_crawler_list") + if err != nil { + return err + } + for rows.Next() { + var c Crawler + rows.Scan(&c.ID, &c.UA, &c.Ver, &c.VerMajor, &c.ClassID, &c.LastSeen, &c.RespectRobotstxt, &c.Family, &c.FamilyCode, &c.FamilyHomepage, &c.FamilyIcon, &c.Vendor, &c.VendorCode, &c.VendorHomepage, &c.Name) + udger.Crawler[c.ID] = c + } + rows.Close() + + rows, err = udger.db.Query("SELECT id, ip_classification, ip_classification_code FROM udger_ip_class") + if err != nil { + return err + } + for rows.Next() { + var ip IPClass + rows.Scan(&ip.ID, &ip.IPClassification, &ip.IPClassificationCode) + udger.IPClass[ip.ID] = ip + } + rows.Close() + + rows, err = udger.db.Query("SELECT id, crawler_classification, crawler_classification_code FROM udger_crawler_class") + if err != nil { + return err + } + for rows.Next() { + var c CrawlerClass + rows.Scan(&c.ID, &c.CrawlerClassification, &c.CrawlerClassificationCode) + udger.CrawlerClass[c.ID] = c + } + rows.Close() + + rows, err = udger.db.Query("SELECT id, name, name_code, homepage FROM udger_datacenter_list") + if err != nil { + return err + } + for rows.Next() { + var d DataCenter + rows.Scan(&d.ID, &d.Name, &d.NameCode, &d.Homepage) + udger.DataCenter[d.ID] = d + } + rows.Close() + + rows, err = udger.db.Query("SELECT datacenter_id, ip_from, ip_to, iplong_from, iplong_to FROM udger_datacenter_range") + if err != nil { + return err + } + for rows.Next() { + var d DataCenterRange + rows.Scan(&d.DatacenterID, &d.IPFrom, &d.IPTo, &d.IPLongFrom, &d.IPLongTo) + udger.DataCenterRange = append(udger.DataCenterRange, d) + } + rows.Close() + + rows, err = udger.db.Query("SELECT datacenter_id, ip_from, ip_to, iplong_from0, iplong_from1, iplong_from2, iplong_from3, iplong_from4, iplong_from5, iplong_from6, iplong_from7, iplong_to0, iplong_to1, iplong_to2, iplong_to3, iplong_to4, iplong_to5, iplong_to6, iplong_to7 FROM udger_datacenter_range6") + if err != nil { + return err + } + for rows.Next() { + var d DataCenterRange6 + rows.Scan(&d.DatacenterID, &d.IPFrom, &d.IPTo, &d.IPLongFrom0, &d.IPLongFrom1, &d.IPLongFrom2, &d.IPLongFrom3, &d.IPLongFrom4, &d.IPLongFrom5, &d.IPLongFrom6, &d.IPLongFrom7, &d.IPLongTo0, &d.IPLongTo1, &d.IPLongTo2, &d.IPLongTo3, &d.IPLongTo4, &d.IPLongTo5, &d.IPLongTo6, &d.IPLongTo7) + udger.DataCenterRange6 = append(udger.DataCenterRange6, d) + } + rows.Close() + return nil } diff --git a/udger_test.go b/udger_test.go index 65e1094..0a6b465 100644 --- a/udger_test.go +++ b/udger_test.go @@ -1,11 +1,12 @@ package udger_test import ( + "fmt" + "net" "testing" + "udger" . "github.com/smartystreets/goconvey/convey" - - "github.com/udger/udger" ) func TestInvalidDbName(t *testing.T) { @@ -16,6 +17,19 @@ func TestInvalidDbName(t *testing.T) { }) } +func TestIP(t *testing.T) { + u, err := udger.New("./udgerdb_v3.dat") + if err != nil { + fmt.Errorf("ERROR %v", err) + t.Fail() + } + + // ip := net.ParseIP("1.0.186.186") + ip := net.ParseIP("2001:16b0:1003:ffff:ffff:ffff:ffff:ffff") + lookup, err := u.LookupIP(ip) + fmt.Println(lookup) +} + func TestValidDbName(t *testing.T) { Convey("load valid path", t, func() { udger, err := udger.New("./udgerdb_v3.dat")