Skip to content

Commit

Permalink
fixed unable to save default data source issue
Browse files Browse the repository at this point in the history
  • Loading branch information
tikazyq committed Aug 12, 2020
1 parent 37a3196 commit b502a35
Show file tree
Hide file tree
Showing 4 changed files with 50 additions and 1 deletion.
2 changes: 1 addition & 1 deletion database/mongo.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func GetMongoCol(ds entity.DataSource) (*mgo.Session, *mgo.Collection, error) {
os.Getenv("CRAWLAB_MONGO_USERNAME"),
os.Getenv("CRAWLAB_MONGO_PASSWORD"),
os.Getenv("CRAWLAB_MONGO_AUTHSOURCE"),
os.Getenv("CRAWLAB_MONGO_DATABASE"),
os.Getenv("CRAWLAB_MONGO_DB"),
os.Getenv("CRAWLAB_COLLECTION"),
)
}
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,5 @@ require (
github.com/jmoiron/sqlx v1.2.0
github.com/olivere/elastic/v7 v7.0.19
github.com/segmentio/kafka-go v0.4.1
github.com/smartystreets/goconvey v1.6.4
)
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
Expand Down Expand Up @@ -122,6 +123,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
Expand Down Expand Up @@ -199,8 +201,10 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/assertions v1.1.1 h1:T/YLemO5Yp7KPzS+lVtu+WsHn8yoSwTfItdAd1r3cck=
github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
github.com/smartystreets/gunit v1.3.4/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
Expand Down
44 changes: 44 additions & 0 deletions utils/data_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package utils

import (
"github.com/crawlab-team/crawlab-go-sdk/database"
"github.com/crawlab-team/crawlab-go-sdk/entity"
"github.com/globalsign/mgo/bson"
"github.com/smartystreets/goconvey/convey"
"os"
"testing"
)

func TestSaveItem(t *testing.T) {
convey.Convey("Test SaveItem", t, func() {
taskId := "test_task_id"
url := "http://example.com"
title := "test"
_ = os.Setenv("CRAWLAB_TASK_ID", taskId)
_ = os.Setenv("CRAWLAB_MONGO_HOST", "localhost")
_ = os.Setenv("CRAWLAB_MONGO_PORT", "27017")
_ = os.Setenv("CRAWLAB_MONGO_DATABASE", "crawlab_test")
_ = os.Setenv("CRAWLAB_COLLECTION", "result_test")
item := entity.Item{
"url": url,
"title": title,
}
if err := SaveItem(item); err != nil {
t.Fatal("save item failed")
}
_, c, _ := database.GetMongoCol(entity.DataSource{
Type: "",
})
//defer s.Close()
var itemDb entity.Item
_ = c.Find(bson.M{"url": url}).One(&itemDb)
convey.Convey("title should be 'test'", func() {
convey.So(itemDb["title"], convey.ShouldEqual, title)
})
count, _ := c.Find(bson.M{"url": url}).Count()
convey.Convey("count should be greater than 0", func() {
convey.So(count, convey.ShouldBeGreaterThan, 0)
})
_, _ = c.RemoveAll(bson.M{"url": url})
})
}

0 comments on commit b502a35

Please sign in to comment.