Patch Exploding Components ETL changes into production

moped-database/metadata/tables.yaml

@@ -301,6 +301,35 @@
           - phase_name_simple
         filter: {}
         allow_aggregations: true
+- table:
+    name: exploded_component_arcgis_online_view
+    schema: public
+  select_permissions:
+    - role: moped-admin
+      permission:
+        columns:
+          - project_component_id
+          - exploded_geometry
+          - project_updated_at
+        filter: {}
+      comment: ""
+    - role: moped-editor
+      permission:
+        columns:
+          - project_component_id
+          - exploded_geometry
+          - project_updated_at
+        filter: {}
+        allow_aggregations: true
+      comment: ""
+    - role: moped-viewer
+      permission:
+        columns:
+          - project_component_id
+          - exploded_geometry
+          - project_updated_at
+        filter: {}
+      comment: ""
 - table:
     name: feature_drawn_lines
     schema: public

moped-database/migrations/1725649291445_add_exploded_moped_geometry_view_for_agol/down.sql

@@ -0,0 +1 @@
+DROP VIEW IF EXISTS exploded_component_arcgis_online_view;

moped-database/migrations/1725649291445_add_exploded_moped_geometry_view_for_agol/up.sql

@@ -0,0 +1,14 @@
+CREATE VIEW exploded_component_arcgis_online_view AS
+SELECT
+    component_arcgis_online_view.project_id,
+    component_arcgis_online_view.project_component_id,
+    ST_GEOMETRYTYPE(dump.geom) AS geometry_type,
+    dump.path[1] AS point_index, -- ordinal value of the point in the MultiPoint geometry
+    component_arcgis_online_view.geometry AS original_geometry,
+    ST_ASGEOJSON(dump.geom) AS exploded_geometry, -- noqa: RF04
+    component_arcgis_online_view.project_updated_at
+FROM
+    component_arcgis_online_view,
+    LATERAL ST_DUMP(ST_GEOMFROMGEOJSON(component_arcgis_online_view.geometry)) AS dump -- noqa: RF04
+WHERE
+    ST_GEOMETRYTYPE(ST_GEOMFROMGEOJSON(component_arcgis_online_view.geometry)) = 'ST_MultiPoint';

moped-database/views/exploded_component_arcgis_online_view.sql

@@ -0,0 +1,13 @@
+-- Most recent migration: moped-database/migrations/1725649291445_add_exploded_moped_geometry_view_for_agol/up.sql
+
+CREATE OR REPLACE VIEW exploded_component_arcgis_online_view AS SELECT
+    component_arcgis_online_view.project_id,
+    component_arcgis_online_view.project_component_id,
+    st_geometrytype(dump.geom) AS geometry_type,
+    dump.path[1] AS point_index,
+    component_arcgis_online_view.geometry AS original_geometry,
+    st_asgeojson(dump.geom) AS exploded_geometry,
+    component_arcgis_online_view.project_updated_at
+FROM component_arcgis_online_view,
+    LATERAL st_dump(st_geomfromgeojson(component_arcgis_online_view.geometry)) dump (path, geom)
+WHERE st_geometrytype(st_geomfromgeojson(component_arcgis_online_view.geometry)) = 'ST_MultiPoint'::text;

moped-etl/arcgis/.dockerignore

@@ -1,3 +1,4 @@
 env_file
 .git%
 __pycache__
+*.json

moped-etl/arcgis/.gitignore

@@ -0,0 +1 @@
+*json

moped-etl/arcgis/README.md

@@ -1,38 +1,33 @@
-# ArcGIS ETLs
+# Moped → ArcGIS Online ETL
 
-Scripts which integrate Moped data with Esri ArcGIS
+Python script integration pushing Moped data to ESRI's ArcGIS Online (AGOL) platform
 
-## Publish components to ArcGIS Online (AGOL)
+## Publish components to AGOL
 
-The script `components_to_agol.py` is used to publish component record data to ArcGIS Online (AGOL). It replaces all records in the AGOL feature services with the latest component data in Moped.
+The script `components_to_agol.py` is used to publish component record data to AGOL. It has two primary modes of operation:
 
-The data is sourced from a view, `component_arcgis_online_view` which defines all columns which are available to be processed.
+- Full refresh: This mode will delete all existing records in the AGOL feature layer and replace them with the current data from the Moped database.
+- Incremental refresh: This mode will only update records that have been modified since a given timestamp.
 
-The AGOL layers can be found here:
+The script is responsible for maintaining four layers in the AGOL in the [Moped Project Components](https://austin.maps.arcgis.com/home/item.html?id=1c084c8756a84e6db7e2796c98c850a2) feature service:
 
-- [Project component points](https://austin.maps.arcgis.com/home/item.html?id=997555f6e0904aa88eafe73f19ee65c0)
-- [Project component lines](https://austin.maps.arcgis.com/home/item.html?id=e8f03d2cec154cacae539b630bcaa70b)
+- [Moped Points](https://austin.maps.arcgis.com/home/item.html?id=1c084c8756a84e6db7e2796c98c850a2&sublayer=0): Components best represented as points, utilizing MultiPoint geometries
+- [Moped Lines](https://austin.maps.arcgis.com/home/item.html?id=1c084c8756a84e6db7e2796c98c850a2&sublayer=1): Components best represented as lines, using Line geometries
+- [MOPED CombinedGeometries](https://austin.maps.arcgis.com/home/item.html?id=1c084c8756a84e6db7e2796c98c850a2&sublayer=2) (SIC): All components, where points are transformed into a line ringing the location
+- [Moped Feature Points](https://austin.maps.arcgis.com/home/item.html?id=1c084c8756a84e6db7e2796c98c850a2&sublayer=3): Components best represented as points, but where MultiPoints are exploded into individual points. Note, the same component can be represented as multiple features, one for each point in the MultiPoint.
 
-### Get it running
+The data for the first three layers listed above is sourced from a view, `component_arcgis_online_view` which defines all columns which are available to be processed.
 
-1. Configure an `env_file` according to the `env_template` example. You can find the AGOL Scripts Publisher username and password in the API Secrets vault in the team password store.
-
-2. Create and activate a Python environment that meets the requirments in `requirements.txt`. Alternatively, you can use the provided Dockerfile.
-
-3. Run the script
+The fourth layer is sourced from a derivative view, `exploded_component_arcgis_online_view`, which takes the previous view and explodes MultiPoint geometries into individual points.
 
-If you want to fully replace the dataset:
+## Running the Script
 
-```shell
-$ python components_to_agol.py -f
-```
-Or, if you want to replace only data updated since a timestamp with time zone offset:
-```shell
-$ python components_to_agol.py -d <timestamptz>
-```
+1. Configure an `env_file` according to the `env_template` example. You can find the AGOL Scripts Publisher username and password in the API Secrets vault in the team password store.
 
-or, to mount your local copy to a Docker container
+1. `docker compose build` to build the container.
 
-```shell
-docker run -it --rm  --network host --env-file env_file -v ${PWD}:/app  atddocker/atd-moped-etl-arcgis:production python components_to_agol.py
-```
+1. Run the script via one or more of the following:
+   - `docker compose run arcgis -d` to start the script with the default interval of changes over the last week.
+   - `docker compose run arcgis -f` to start the script with a full refresh.
+   - `docker compose run arcgis -d <timestamptz>` to start the script with a refresh since the given timestamp.
+   - `docker compose run --entrypoint /bin/bash arcgis` to start a shell inside the container.

moped-etl/arcgis/components_to_agol.py

@@ -1,13 +1,15 @@
 #!/usr/bin/env python
 """Copies all Moped component records to ArcGIS Online (AGOL)"""
-# docker run -it --rm  --network host --env-file env_file -v ${PWD}:/app  moped-agol /bin/bash
+# docker compose run arcgis;
 import argparse
 import logging
-from datetime import datetime, timezone
+import json
+from datetime import datetime, timezone, timedelta
 
 from process.logging import get_logger
 from settings import (
     COMPONENTS_QUERY_BY_LAST_UPDATE_DATE,
+    EXPLODED_COMPONENTS_QUERY_BY_LAST_UPDATE_DATE,
     UPLOAD_CHUNK_SIZE,
 )
 from utils import (
@@ -42,7 +44,7 @@ def make_esri_feature(*, esri_geometry_key, geometry, attributes):
     See: https://developers.arcgis.com/documentation/common-data-types/feature-object.htm
 
     Args:
-        esri_geometry_key (str): `paths` or `points`: see the `get_esri_geometry_key` docstring
+        esri_geometry_key (str): `paths` or `points`, `point`: see the `get_esri_geometry_key` docstring
         geometry (dict): A geojson geometry object, such as one returned from our
             component view in Moped
         attribute (dict): Any additional properties to be included as feature attributes
@@ -59,27 +61,41 @@ def make_esri_feature(*, esri_geometry_key, geometry, attributes):
             "spatialReference": {"wkid": 4326},
         },
     }
-    feature["geometry"][esri_geometry_key] = geometry["coordinates"]
+    if (esri_geometry_key == "points") or (esri_geometry_key == "paths"):
+        feature["geometry"][esri_geometry_key] = geometry["coordinates"]
+    elif esri_geometry_key == "point":
+        geometry = json.loads(geometry)
+        feature["geometry"]["y"] = geometry["coordinates"][1]
+        feature["geometry"]["x"] = geometry["coordinates"][0]
+    else:
+        feature["geometry"]["y"] = geometry["coordinates"][1]
+        feature["geometry"]["x"] = geometry["coordinates"][0]
+
     return feature
 
 
-def make_all_features(data):
+def make_all_features(data, exploded_geometry):
     """Take a list of component feature records and create Esri feature objects for lines, points, and combined layers in AGOL.
 
     Args:
         data (dict): a list of component feature records
+        exploded_geometry (dict): a dictionary of exploded geometry data from the component_arcgis_online_view. This is created
+            by taking the multi-point geometry from the component_arcgis_online_view and "exploding" it into individual points.
 
     Returns:
         dict: An object with lists of Esri feature objects for lines, points, and combined layers
     """
-    all_features = {"lines": [], "points": [], "combined": []}
+
+    all_features = {"lines": [], "points": [], "combined": [], "exploded": []}
 
     logger.info("Building Esri feature objects...")
     for component in data:
-        # extract geometry and line geometry from component data
-        # for line features, the line geometry is redundant/identical to geometry
-        # for point features, it is the buffered ring around the point as defined
-        # in the Moped component view
+
+        # Extract geometry and line geometry from component data.
+        # For line features, the line geometry is redundant/identical to geometry.
+        # For point features, it is the buffered ring around the point as defined
+        # in the Moped component view.
+
         geometry = component.pop("geometry")
         line_geometry = component.pop("line_geometry")
 
@@ -115,6 +131,27 @@ def make_all_features(data):
                 attributes=component,
             )
             all_features["combined"].append(line_feature)
+
+            project_component_id = feature["attributes"]["project_component_id"]
+            # Filter exploded_geometry to only include dicts with matching project_component_id
+            matching_exploded_geometry_records = [
+                feature
+                for feature in exploded_geometry
+                if feature.get("project_component_id") == project_component_id
+            ]
+            for record in matching_exploded_geometry_records:
+                geometry = record.pop("geometry")
+                esri_geometry_key = "point"
+
+                feature = make_esri_feature(
+                    esri_geometry_key="point",
+                    geometry=geometry,
+                    attributes=component,
+                )
+
+                feature["attributes"]["source_geometry_type"] = "point"
+                all_features["exploded"].append(feature)
+
         else:
             all_features["lines"].append(feature)
             all_features["combined"].append(feature)
@@ -140,10 +177,15 @@ def main(args):
         variables=variables,
     )["component_arcgis_online_view"]
 
-    all_features = make_all_features(data)
+    exploded_data = make_hasura_request(
+        query=EXPLODED_COMPONENTS_QUERY_BY_LAST_UPDATE_DATE,
+        variables=variables,
+    )["exploded_component_arcgis_online_view"]
+
+    all_features = make_all_features(data, exploded_data)
 
     if args.full:
-        for feature_type in ["points", "lines", "combined"]:
+        for feature_type in ["points", "lines", "combined", "exploded"]:
             logger.info(f"Processing {feature_type} features...")
             features = all_features[feature_type]
 
@@ -166,7 +208,7 @@ def main(args):
         project_ids_for_feature_delete = list(set(project_ids))
 
         # Delete outdated feature from AGOL and add updated features
-        for feature_type in ["points", "lines", "combined"]:
+        for feature_type in ["points", "lines", "combined", "exploded"]:
             logger.info(f"Processing {feature_type} features...")
             logger.info(
                 f"Deleting all {len(all_features[feature_type])} existing features in {feature_type} layer for updated projects in chunks of {UPLOAD_CHUNK_SIZE}..."
@@ -195,15 +237,17 @@ def main(args):
         "-d",
         "--date",
         type=str,
+        nargs="?",
+        const=(datetime.now(timezone.utc) - timedelta(days=7)).isoformat(),
         default=None,
-        help=f"ISO date string with TZ offset (ex. 2024-06-28T00:06:16.360805+00:00) of latest updated_at value to find project records to update.",
+        help="ISO date string with TZ offset (ex. 2024-06-28T00:06:16.360805+00:00) of latest updated_at value to find project records to update. Defaults to 7 days ago if -d is used without a value.",
     )
 
     parser.add_argument(
         "-f",
         "--full",
         action="store_true",
-        help=f"Delete and replace all project components.",
+        help="Delete and replace all project components.",
     )
 
     args = parser.parse_args()
@@ -214,6 +258,11 @@ def main(args):
             "Please provide either the -d flag with ISO date string with TZ offset or the -f flag and not both."
         )
 
+    if not args.date and not args.full:
+        raise Exception(
+            "Please provide either the -d flag with optional ISO date string with TZ offset or the -f flag."
+        )
+
     if args.full:
         logger.info(f"Starting sync. Replacing all projects' components data...")
     else:

moped-etl/arcgis/docker-compose.yaml

@@ -0,0 +1,10 @@
+services:
+  arcgis:
+    build:
+      context: .
+    volumes:
+      - .:/app
+    entrypoint: python /app/components_to_agol.py
+    command: -d
+    env_file:
+      - env_file

moped-etl/arcgis/env_template

@@ -1,4 +1,4 @@
 AGOL_USERNAME=
 AGOL_PASSWORD=
-HASURA_ENDPOINT=http://localhost:8080/v1/graphql
+HASURA_ENDPOINT=http://host.docker.internal:8080/v1/graphql
 HASURA_ADMIN_SECRET=hasurapassword

moped-etl/arcgis/settings.py

@@ -1,6 +1,6 @@
 UPLOAD_CHUNK_SIZE = 100
 
-LAYER_IDS = {"points": 0, "lines": 1, "combined": 2}
+LAYER_IDS = {"points": 0, "lines": 1, "combined": 2, "exploded": 3}
 
 COMPONENTS_QUERY_BY_LAST_UPDATE_DATE = """
 query GetProjectsComponents($where: component_arcgis_online_view_bool_exp!) {
@@ -84,3 +84,13 @@
   }
 }
 """
+
+# line_geometry
+EXPLODED_COMPONENTS_QUERY_BY_LAST_UPDATE_DATE = """
+query GetExplodedProjectsComponents($where: exploded_component_arcgis_online_view_bool_exp!) {
+  exploded_component_arcgis_online_view(where: $where) {
+    geometry: exploded_geometry
+    project_component_id
+  }
+}
+"""