BQ: resolve issues 56, 57, 58, 61 (#62)

* add logic handling for empty table, add logs of ensuring logic and update README to include full table path

* update README to include full table paths

* Update README.md

* Update README.md

* Update README.md

* Update README.md

* Update README.md

* Update main.go

* Update main.go

pr fixes

* pr fixes

* pr fixes

* Update main.go

* add tests for empty table and table with initialized schema

* Update README.md

* address pr comments:

* address pr fixes

bigquery/README.md

@@ -4,6 +4,18 @@ This notifier pushes build data to a BigQuery instance.
 
 *Alpha Feature - Under Development*
 
+## Dataset and table setup
+
+The BQ notifier initializes datasets and tables automatically.
+Dataset identifiers in the BigQuery Notifier config can refer to existing or nonexistent datasets. 
+
+Table identifiers in the BigQuery Notifier config can refer to either:
+1. A nonexistent table (will be created upon deployment of the notifier)
+2. An empty table not yet initialized with a schema.
+3. An existing table with a schema that matches the bq notifier schema specifications.
+
+References to already existing tables with differing schemas will throw errors upon writing.
+
 ## Accessing build insights with SQL queries through the BigQuery CLI:
 
 To access BQ data through queries, run the following command below.
@@ -12,6 +24,8 @@ To access BQ data through queries, run the following command below.
 ```bash
 $ bq query '<SQL QUERY>'
 ```
+Adding the `--format=prettyjson` flag allows for more readable output.
+
 More detailed information can be found here: [BQ CLI Reference](https://cloud.google.com/bigquery/docs/bq-command-line-tool)
 
 Legacy SQL dialect is set on default for the BigQuery CLI and must be disabled for the example queries to work.
@@ -34,12 +48,12 @@ More information can be found here: [Switching SQL Dialects](https://cloud.googl
 ```sql
 # Listing overall build history
 
-SELECT * FROM `dataset.table`
+SELECT * FROM `projectID.datasetName.tableName`
 
 # Aggregating build counts by status
 
 SELECT STATUS, COUNT(*) 
-FROM `dataset.table` 
+FROM `projectID.datasetName.tableName`
 GROUP BY STATUS
 
 # Getting daily deployment frequency for current week
@@ -48,22 +62,22 @@ SELECT DAY, COUNT(STATUS) AS Deployments
 FROM (SELECT DATETIME_TRUNC(CreateTime, WEEK) AS WEEK, 
       DATETIME_TRUNC(CreateTime, DAY) AS DAY, 
       STATUS 
-      FROM `dataset.table` 
+      FROM `projectID.datasetName.tableName` 
       WHERE STATUS="SUCCESS") 
 WHERE WEEK = DATETIME_TRUNC(CURRENT_DATETIME(), WEEK) 
 GROUP BY DAY
 
 # Calculating build times
 
 SELECT CreateTime, DATETIME_DIFF(FinishTime, StartTime, SECOND) as BuildTime 
-FROM `dataset.table`  
+FROM `projectID.datasetName.tableName`  
 WHERE STATUS = "SUCCESS" 
 ORDER BY BuildTime
 
 # Getting build statuses for the current day
 
 SELECT DAY, STATUS 
 FROM (SELECT DATETIME_TRUNC(CreateTime, DAY) AS DAY, 
-      STATUS FROM `dataset.table`) 
+      STATUS FROM `projectID.datasetName.tableName`) 
 WHERE DAY = DATETIME_TRUNC(CURRENT_DATETIME(), DAY)
 ```

bigquery/main.go

@@ -108,7 +108,7 @@ func (bqf *actualBQFactory) Make(ctx context.Context) (bq, error) {
 	}
 	bqClient, err := bigquery.NewClient(ctx, projectID)
 	if err != nil {
-		return nil, fmt.Errorf("error intializing bigquery client: %v", err)
+		return nil, fmt.Errorf("error initializing bigquery client: %v", err)
 	}
 	newClient := &actualBQ{client: bqClient}
 	return newClient, nil
@@ -192,7 +192,7 @@ func (n *bqNotifier) SendNotification(ctx context.Context, build *cbpb.Build) er
 		return nil
 	}
 	if build.BuildTriggerId == "" {
-		log.Warningf("build passes filter but does not have trigger ID, no build data will be captured: %v, status: %v", n.filter, build.GetStatus())
+		log.Warningf("build passes filter but does not have a trigger ID. Build id: %q, status: %v", build.Id, build.GetStatus())
 	}
 	log.Infof("sending Big Query write for build %q (status: %q)", build.Id, build.Status)
 	if build.ProjectId == "" {
@@ -281,7 +281,7 @@ func (bq *actualBQ) EnsureDataset(ctx context.Context, datasetName string) error
 	bq.dataset = bq.client.Dataset(datasetName)
 	_, err := bq.client.Dataset(datasetName).Metadata(ctx)
 	if err != nil {
-		log.Warningf("error obtaining dataset metadata: %v", err)
+		log.Warningf("error obtaining dataset metadata: %v;Creating new BigQuery dataset: %q", err, datasetName)
 		if err := bq.dataset.Create(ctx, &bigquery.DatasetMetadata{
 			Name: datasetName, Description: "BigQuery Notifier Build Data",
 		}); err != nil {
@@ -294,18 +294,27 @@ func (bq *actualBQ) EnsureDataset(ctx context.Context, datasetName string) error
 func (bq *actualBQ) EnsureTable(ctx context.Context, tableName string) error {
 	// Check for existence of table, create if false
 	bq.table = bq.dataset.Table(tableName)
-	_, err := bq.dataset.Table(tableName).Metadata(ctx)
+	schema, err := bigquery.InferSchema(bqRow{})
 	if err != nil {
-		log.Warningf("Error obtaining table metadata: %v", err)
-		schema, err := bigquery.InferSchema(bqRow{})
-		if err != nil {
-			return fmt.Errorf("Failed to infer schema: %v", err)
-		}
+		return fmt.Errorf("Failed to infer schema: %v", err)
+	}
+	metadata, err := bq.dataset.Table(tableName).Metadata(ctx)
+	if err != nil {
+		log.Warningf("Error obtaining table metadata: %q;Creating new BigQuery table: %q", err, tableName)
 		// Create table if it does not exist.
 		if err := bq.table.Create(ctx, &bigquery.TableMetadata{Name: tableName, Description: "BigQuery Notifier Build Data Table", Schema: schema}); err != nil {
 			return fmt.Errorf("Failed to initialize table %v: ", err)
 		}
+	} else if len(metadata.Schema) == 0 {
+		log.Warningf("No schema found for table, writing new schema for table: %v", tableName)
+		update := bigquery.TableMetadataToUpdate{
+			Schema: schema,
+		}
+		if _, err := bq.table.Update(ctx, update, metadata.ETag); err != nil {
+			return fmt.Errorf("Error: unable to update schema of table: %v", err)
+		}
 	}
+
 	return nil
 }