From b285a900de37d355878871473a768cf40f938860 Mon Sep 17 00:00:00 2001 From: Haohan Yang Date: Tue, 8 Oct 2024 22:53:25 +0000 Subject: [PATCH] Add back datasource test --- .github/workflows/ci.yml | 5 + pkg/models/column.go | 22 ++-- pkg/plugin/datasource_test.go | 197 +++++++++++++++++++--------------- 3 files changed, 124 insertions(+), 100 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index adbad75..fe3b2e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,11 @@ jobs: has-backend: ${{ steps.check-for-backend.outputs.has-backend }} env: GRAFANA_ACCESS_POLICY_TOKEN: ${{ secrets.GRAFANA_ACCESS_POLICY_TOKEN }} + services: + mongo: + image: mongo + ports: + - 27018:27017 steps: - uses: actions/checkout@v4 - name: Setup Node.js environment diff --git a/pkg/models/column.go b/pkg/models/column.go index 902e694..f403a8d 100644 --- a/pkg/models/column.go +++ b/pkg/models/column.go @@ -24,8 +24,8 @@ func (c *Column) AppendValue(rv bson.RawValue) error { c.Field.Append(nil) case bson.TypeBoolean: - if c.Type() != data.FieldTypeBool { - return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String()) + if c.Type() != data.FieldTypeNullableBool { + return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String()) } v := new(bool) @@ -44,7 +44,7 @@ func (c *Column) AppendValue(rv bson.RawValue) error { c.Field.Append(pointer(float64(v))) } else { - return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String()) + return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String()) } case bson.TypeInt64: v := rv.Int64() @@ -68,7 +68,7 @@ func (c *Column) AppendValue(rv bson.RawValue) error { c.Field.Append(pointer(float64(v))) } else { - return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String()) + return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String()) } case bson.TypeDouble: @@ -102,45 +102,45 @@ func (c *Column) AppendValue(rv bson.RawValue) error { float64Values[c.Field.Len()] = pointer(v) c.Field = data.NewField(c.Name, nil, float64Values) } else { - return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String()) + return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String()) } case bson.TypeString: if c.Type() != data.FieldTypeNullableString { - return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String()) + return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String()) } c.Field.Append(pointer(rv.StringValue())) case bson.TypeDateTime: if c.Type() != data.FieldTypeNullableTime { - return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String()) + return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String()) } c.Field.Append(pointer(rv.Time())) case bson.TypeObjectID: if c.Type() != data.FieldTypeNullableString { - return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String()) + return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String()) } c.Field.Append(pointer(rv.ObjectID().String())) case bson.TypeEmbeddedDocument: if c.Type() != data.FieldTypeNullableString { - return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String()) + return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String()) } c.Field.Append(pointer(rv.Document().String())) case bson.TypeArray: if c.Type() != data.FieldTypeNullableString { - return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String()) + return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String()) } c.Field.Append(pointer(rv.Array().String())) default: if c.Type() != data.FieldTypeNullableString { - return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String()) + return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String()) } c.Field.Append(pointer(UNSUPPORTED_TYPE)) diff --git a/pkg/plugin/datasource_test.go b/pkg/plugin/datasource_test.go index 2a2e33c..c6855f6 100644 --- a/pkg/plugin/datasource_test.go +++ b/pkg/plugin/datasource_test.go @@ -2,92 +2,111 @@ package plugin // TODO -// import ( -// "context" -// "encoding/json" -// "fmt" -// "path/filepath" -// "strings" -// "testing" - -// "github.com/grafana/grafana-plugin-sdk-go/backend" -// "go.mongodb.org/mongo-driver/bson" -// "go.mongodb.org/mongo-driver/mongo" -// "go.mongodb.org/mongo-driver/mongo/options" -// ) - -// var uri = "mongodb://localhost:27018" - -// var datasetLinks = []string{} - -// func TestQueryData(t *testing.T) { - -// ctx := context.Background() -// opts := options.Client().ApplyURI(uri) - -// client, err := mongo.Connect(ctx, opts) - -// if err != nil { -// t.Fatal(err) -// } - -// ds := Datasource{ -// client: client, -// host: "localhost", -// port: 27018, -// database: "test", -// } - -// t.Cleanup(func() { -// ds.client.Database(ds.database).Drop(ctx) -// }) - -// for _, url := range datasetLinks { -// fileName := filepath.Base(url) -// t.Run(fmt.Sprintf("test table query of %s", fileName), func(t *testing.T) { -// temDir := t.TempDir() -// err := downloadAndImportMongoData(url, temDir) -// if err != nil { -// t.Fatal(err) -// } - -// aggregate, err := json.Marshal(bson.A{bson.M{"$limit": 50000}}) -// if err != nil { -// t.Fatal(err) -// } - -// qm := queryModel{ -// QueryText: string(aggregate), -// Collection: strings.Split(fileName, ".")[0], -// QueryType: "table", -// QueryLanguage: "json", -// } - -// rawJson, err := json.Marshal(qm) -// if err != nil { -// t.Fatal(err) -// } - -// resp, err := ds.QueryData( -// ctx, -// &backend.QueryDataRequest{ -// Queries: []backend.DataQuery{ -// {RefID: "A", JSON: rawJson}, -// }, -// }, -// ) -// if err != nil { -// t.Fatal(err) -// } - -// if len(resp.Responses["A"].Frames) != 1 { -// t.Fatal("The number of Frame should be 1") -// } - -// _, err = resp.Responses["A"].Frames[0].RowLen() -// if err != nil { -// t.Fatal(err) -// } -// }) -// } -// } +import ( + "context" + "encoding/json" + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +var uri = "mongodb://localhost:27018" + +var datasetLinks = []string{ + "https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_airbnb/listingsAndReviews.json", + "https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_analytics/accounts.json", + "https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_analytics/customers.json", + "https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_analytics/transactions.json", + "https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_geospatial/shipwrecks.json", + "https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_mflix/comments.json", + "https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_mflix/theaters.json", + "https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_mflix/users.json", + "https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_supplies/sales.json", + "https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_training/grades.json", + "https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_training/posts.json", + "https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_training/stories.json", + "https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_training/tweets.json", + "https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_weatherdata/data.json", +} + +func TestQueryTableData(t *testing.T) { + + ctx := context.Background() + opts := options.Client().ApplyURI(uri) + + client, err := mongo.Connect(ctx, opts) + + if err != nil { + t.Fatal(err) + } + + ds := Datasource{ + client: client, + host: "localhost", + port: 27018, + database: "test", + } + + t.Cleanup(func() { + ds.client.Database(ds.database).Drop(ctx) + }) + + for _, url := range datasetLinks { + fileName := filepath.Base(url) + t.Run(fmt.Sprintf("test table query of %s", fileName), func(t *testing.T) { + temDir := t.TempDir() + err := downloadAndImportMongoData(url, temDir) + if err != nil { + t.Fatal(err) + } + + aggregate, err := json.Marshal(bson.A{}) + if err != nil { + t.Fatal(err) + } + + qm := queryModel{ + QueryText: string(aggregate), + Collection: strings.Split(fileName, ".")[0], + QueryType: "table", + QueryLanguage: "json", + } + + rawJson, err := json.Marshal(qm) + if err != nil { + t.Fatal(err) + } + + resp, err := ds.QueryData( + ctx, + &backend.QueryDataRequest{ + Queries: []backend.DataQuery{ + {RefID: "A", JSON: rawJson}, + }, + }, + ) + if err != nil { + t.Fatal(err) + } + + if len(resp.Responses["A"].Frames) != 1 { + t.Fatal("The number of Frame should be 1") + } + + rowCount, err := resp.Responses["A"].Frames[0].RowLen() + if err != nil { + t.Fatal(err) + } + t.Logf("%d rows in total", rowCount) + }) + + time.Sleep(time.Second) + } +}