updated schema & null behavior for hashtags & urls to biqquery

stanfordio · Nov 3, 2024 · 134e704 · 134e704
1 parent 8f09c7d
commit 134e704
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 0 deletions.
diff --git a/pkg/hydrator/hydrator.go b/pkg/hydrator/hydrator.go
@@ -251,6 +251,8 @@ func (h *Hydrator) flattenFullProfile(profile *bsky.ActorDefs_ProfileViewDetaile
 }
 
 func (h *Hydrator) flattenFacets(facets []*bsky.RichtextFacet) (hashtags []string, urls []string) {
+	hashtags = []string{}
+	urls = []string{}
 	if facets != nil {
 			for _, facet := range facets {
 					if facet != nil {

diff --git a/pkg/output/bq/schema/schema.go b/pkg/output/bq/schema/schema.go
@@ -276,6 +276,16 @@ func GetSchema() bigquery.Schema {
 				"name": "RepostCount",
 				"type": "INTEGER"
 			},
+			{
+				"mode": "REPEATED",
+				"name": "Hashtags",
+				"type": "STRING"
+			},
+			{
+				"mode": "REPEATED",
+				"name": "URLs",
+				"type": "STRING"
+			},
 			{
 				"name": "Text",
 				"type": "STRING"
@@ -399,6 +409,16 @@ func GetSchema() bigquery.Schema {
 				"name": "ReplyParentCID",
 				"type": "STRING"
 			},
+			{
+				"mode": "REPEATED",
+				"name": "Hashtags",
+				"type": "STRING"
+			},
+			{
+				"mode": "REPEATED",
+				"name": "URLs",
+				"type": "STRING"
+			},
 			{
 				"name": "Text",
 				"type": "STRING"
@@ -566,6 +586,16 @@ func GetSchema() bigquery.Schema {
 				"name": "RepostCount",
 				"type": "INTEGER"
 			},
+			{
+				"mode": "REPEATED",
+				"name": "Hashtags",
+				"type": "STRING"
+			},
+			{
+				"mode": "REPEATED",
+				"name": "URLs",
+				"type": "STRING"
+			},
 			{
 				"name": "Text",
 				"type": "STRING"