Skip to content

Commit

Permalink
Add back datasource test
Browse files Browse the repository at this point in the history
  • Loading branch information
@haohanyang
haohanyang committed Oct 8, 2024
1 parent 26090da commit b285a90
Show file tree
Hide file tree
Showing 3 changed files with 124 additions and 100 deletions.
5 changes: 5 additions & 0 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ jobs:
has-backend: ${{ steps.check-for-backend.outputs.has-backend }}
env:
GRAFANA_ACCESS_POLICY_TOKEN: ${{ secrets.GRAFANA_ACCESS_POLICY_TOKEN }}
services:
mongo:
image: mongo
ports:
- 27018:27017
steps:
- uses: actions/checkout@v4
- name: Setup Node.js environment
Expand Down
22 changes: 11 additions & 11 deletions pkg/models/column.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ func (c *Column) AppendValue(rv bson.RawValue) error {
c.Field.Append(nil)

case bson.TypeBoolean:
if c.Type() != data.FieldTypeBool {
return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String())
if c.Type() != data.FieldTypeNullableBool {
return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String())
}

v := new(bool)
Expand All @@ -44,7 +44,7 @@ func (c *Column) AppendValue(rv bson.RawValue) error {
c.Field.Append(pointer(float64(v)))

} else {
return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String())
return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String())
}
case bson.TypeInt64:
v := rv.Int64()
Expand All @@ -68,7 +68,7 @@ func (c *Column) AppendValue(rv bson.RawValue) error {
c.Field.Append(pointer(float64(v)))

} else {
return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String())
return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String())
}

case bson.TypeDouble:
Expand Down Expand Up @@ -102,45 +102,45 @@ func (c *Column) AppendValue(rv bson.RawValue) error {
float64Values[c.Field.Len()] = pointer(v)
c.Field = data.NewField(c.Name, nil, float64Values)
} else {
return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String())
return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String())
}

case bson.TypeString:
if c.Type() != data.FieldTypeNullableString {
return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String())
return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String())
}

c.Field.Append(pointer(rv.StringValue()))

case bson.TypeDateTime:
if c.Type() != data.FieldTypeNullableTime {
return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String())
return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String())
}

c.Field.Append(pointer(rv.Time()))

case bson.TypeObjectID:
if c.Type() != data.FieldTypeNullableString {
return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String())
return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String())
}
c.Field.Append(pointer(rv.ObjectID().String()))

case bson.TypeEmbeddedDocument:
if c.Type() != data.FieldTypeNullableString {
return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String())
return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String())
}

c.Field.Append(pointer(rv.Document().String()))
case bson.TypeArray:
if c.Type() != data.FieldTypeNullableString {
return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String())
return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String())
}

c.Field.Append(pointer(rv.Array().String()))

default:
if c.Type() != data.FieldTypeNullableString {
return fmt.Errorf("field %s should have type %s", c.Name, c.Type().String())
return fmt.Errorf("field %s should have type %s, but got %s", c.Name, c.Type().ItemTypeString(), rv.Type.String())
}

c.Field.Append(pointer(UNSUPPORTED_TYPE))
Expand Down
197 changes: 108 additions & 89 deletions pkg/plugin/datasource_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,92 +2,111 @@ package plugin

// TODO

// import (
// "context"
// "encoding/json"
// "fmt"
// "path/filepath"
// "strings"
// "testing"

// "github.com/grafana/grafana-plugin-sdk-go/backend"
// "go.mongodb.org/mongo-driver/bson"
// "go.mongodb.org/mongo-driver/mongo"
// "go.mongodb.org/mongo-driver/mongo/options"
// )

// var uri = "mongodb://localhost:27018"

// var datasetLinks = []string{}

// func TestQueryData(t *testing.T) {

// ctx := context.Background()
// opts := options.Client().ApplyURI(uri)

// client, err := mongo.Connect(ctx, opts)

// if err != nil {
// t.Fatal(err)
// }

// ds := Datasource{
// client: client,
// host: "localhost",
// port: 27018,
// database: "test",
// }

// t.Cleanup(func() {
// ds.client.Database(ds.database).Drop(ctx)
// })

// for _, url := range datasetLinks {
// fileName := filepath.Base(url)
// t.Run(fmt.Sprintf("test table query of %s", fileName), func(t *testing.T) {
// temDir := t.TempDir()
// err := downloadAndImportMongoData(url, temDir)
// if err != nil {
// t.Fatal(err)
// }

// aggregate, err := json.Marshal(bson.A{bson.M{"$limit": 50000}})
// if err != nil {
// t.Fatal(err)
// }

// qm := queryModel{
// QueryText: string(aggregate),
// Collection: strings.Split(fileName, ".")[0],
// QueryType: "table",
// QueryLanguage: "json",
// }

// rawJson, err := json.Marshal(qm)
// if err != nil {
// t.Fatal(err)
// }

// resp, err := ds.QueryData(
// ctx,
// &backend.QueryDataRequest{
// Queries: []backend.DataQuery{
// {RefID: "A", JSON: rawJson},
// },
// },
// )
// if err != nil {
// t.Fatal(err)
// }

// if len(resp.Responses["A"].Frames) != 1 {
// t.Fatal("The number of Frame should be 1")
// }

// _, err = resp.Responses["A"].Frames[0].RowLen()
// if err != nil {
// t.Fatal(err)
// }
// })
// }
// }
import (
"context"
"encoding/json"
"fmt"
"path/filepath"
"strings"
"testing"
"time"

"github.com/grafana/grafana-plugin-sdk-go/backend"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)

var uri = "mongodb://localhost:27018"

var datasetLinks = []string{
"https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_airbnb/listingsAndReviews.json",
"https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_analytics/accounts.json",
"https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_analytics/customers.json",
"https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_analytics/transactions.json",
"https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_geospatial/shipwrecks.json",
"https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_mflix/comments.json",
"https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_mflix/theaters.json",
"https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_mflix/users.json",
"https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_supplies/sales.json",
"https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_training/grades.json",
"https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_training/posts.json",
"https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_training/stories.json",
"https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_training/tweets.json",
"https://github.com/neelabalan/mongodb-sample-dataset/raw/refs/heads/main/sample_weatherdata/data.json",
}

func TestQueryTableData(t *testing.T) {

ctx := context.Background()
opts := options.Client().ApplyURI(uri)

client, err := mongo.Connect(ctx, opts)

if err != nil {
t.Fatal(err)
}

ds := Datasource{
client: client,
host: "localhost",
port: 27018,
database: "test",
}

t.Cleanup(func() {
ds.client.Database(ds.database).Drop(ctx)
})

for _, url := range datasetLinks {
fileName := filepath.Base(url)
t.Run(fmt.Sprintf("test table query of %s", fileName), func(t *testing.T) {
temDir := t.TempDir()
err := downloadAndImportMongoData(url, temDir)
if err != nil {
t.Fatal(err)
}

aggregate, err := json.Marshal(bson.A{})
if err != nil {
t.Fatal(err)
}

qm := queryModel{
QueryText: string(aggregate),
Collection: strings.Split(fileName, ".")[0],
QueryType: "table",
QueryLanguage: "json",
}

rawJson, err := json.Marshal(qm)
if err != nil {
t.Fatal(err)
}

resp, err := ds.QueryData(
ctx,
&backend.QueryDataRequest{
Queries: []backend.DataQuery{
{RefID: "A", JSON: rawJson},
},
},
)
if err != nil {
t.Fatal(err)
}

if len(resp.Responses["A"].Frames) != 1 {
t.Fatal("The number of Frame should be 1")
}

rowCount, err := resp.Responses["A"].Frames[0].RowLen()
if err != nil {
t.Fatal(err)
}
t.Logf("%d rows in total", rowCount)
})

time.Sleep(time.Second)
}
}

0 comments on commit b285a90

Please sign in to comment.