diff --git a/moped-database/metadata/tables.yaml b/moped-database/metadata/tables.yaml index 0639d4ced0..0a588ccbdf 100644 --- a/moped-database/metadata/tables.yaml +++ b/moped-database/metadata/tables.yaml @@ -301,6 +301,35 @@ - phase_name_simple filter: {} allow_aggregations: true +- table: + name: exploded_component_arcgis_online_view + schema: public + select_permissions: + - role: moped-admin + permission: + columns: + - project_component_id + - exploded_geometry + - project_updated_at + filter: {} + comment: "" + - role: moped-editor + permission: + columns: + - project_component_id + - exploded_geometry + - project_updated_at + filter: {} + allow_aggregations: true + comment: "" + - role: moped-viewer + permission: + columns: + - project_component_id + - exploded_geometry + - project_updated_at + filter: {} + comment: "" - table: name: feature_drawn_lines schema: public diff --git a/moped-database/migrations/1725649291445_add_exploded_moped_geometry_view_for_agol/down.sql b/moped-database/migrations/1725649291445_add_exploded_moped_geometry_view_for_agol/down.sql new file mode 100644 index 0000000000..efe9386b0f --- /dev/null +++ b/moped-database/migrations/1725649291445_add_exploded_moped_geometry_view_for_agol/down.sql @@ -0,0 +1 @@ +DROP VIEW IF EXISTS exploded_component_arcgis_online_view; diff --git a/moped-database/migrations/1725649291445_add_exploded_moped_geometry_view_for_agol/up.sql b/moped-database/migrations/1725649291445_add_exploded_moped_geometry_view_for_agol/up.sql new file mode 100644 index 0000000000..c2599aef5a --- /dev/null +++ b/moped-database/migrations/1725649291445_add_exploded_moped_geometry_view_for_agol/up.sql @@ -0,0 +1,14 @@ +CREATE VIEW exploded_component_arcgis_online_view AS +SELECT + component_arcgis_online_view.project_id, + component_arcgis_online_view.project_component_id, + ST_GEOMETRYTYPE(dump.geom) AS geometry_type, + dump.path[1] AS point_index, -- ordinal value of the point in the MultiPoint geometry + component_arcgis_online_view.geometry AS original_geometry, + ST_ASGEOJSON(dump.geom) AS exploded_geometry, -- noqa: RF04 + component_arcgis_online_view.project_updated_at +FROM + component_arcgis_online_view, + LATERAL ST_DUMP(ST_GEOMFROMGEOJSON(component_arcgis_online_view.geometry)) AS dump -- noqa: RF04 +WHERE + ST_GEOMETRYTYPE(ST_GEOMFROMGEOJSON(component_arcgis_online_view.geometry)) = 'ST_MultiPoint'; diff --git a/moped-database/views/exploded_component_arcgis_online_view.sql b/moped-database/views/exploded_component_arcgis_online_view.sql new file mode 100644 index 0000000000..cddba03fdb --- /dev/null +++ b/moped-database/views/exploded_component_arcgis_online_view.sql @@ -0,0 +1,13 @@ +-- Most recent migration: moped-database/migrations/1725649291445_add_exploded_moped_geometry_view_for_agol/up.sql + +CREATE OR REPLACE VIEW exploded_component_arcgis_online_view AS SELECT + component_arcgis_online_view.project_id, + component_arcgis_online_view.project_component_id, + st_geometrytype(dump.geom) AS geometry_type, + dump.path[1] AS point_index, + component_arcgis_online_view.geometry AS original_geometry, + st_asgeojson(dump.geom) AS exploded_geometry, + component_arcgis_online_view.project_updated_at +FROM component_arcgis_online_view, + LATERAL st_dump(st_geomfromgeojson(component_arcgis_online_view.geometry)) dump (path, geom) +WHERE st_geometrytype(st_geomfromgeojson(component_arcgis_online_view.geometry)) = 'ST_MultiPoint'::text; diff --git a/moped-etl/arcgis/.dockerignore b/moped-etl/arcgis/.dockerignore index ebfcfbe643..9303574d2e 100644 --- a/moped-etl/arcgis/.dockerignore +++ b/moped-etl/arcgis/.dockerignore @@ -1,3 +1,4 @@ env_file .git% __pycache__ +*.json diff --git a/moped-etl/arcgis/.gitignore b/moped-etl/arcgis/.gitignore new file mode 100644 index 0000000000..8d080e88eb --- /dev/null +++ b/moped-etl/arcgis/.gitignore @@ -0,0 +1 @@ +*json diff --git a/moped-etl/arcgis/README.md b/moped-etl/arcgis/README.md index e09acc0453..b161fb356f 100644 --- a/moped-etl/arcgis/README.md +++ b/moped-etl/arcgis/README.md @@ -1,38 +1,33 @@ -# ArcGIS ETLs +# Moped → ArcGIS Online ETL -Scripts which integrate Moped data with Esri ArcGIS +Python script integration pushing Moped data to ESRI's ArcGIS Online (AGOL) platform -## Publish components to ArcGIS Online (AGOL) +## Publish components to AGOL -The script `components_to_agol.py` is used to publish component record data to ArcGIS Online (AGOL). It replaces all records in the AGOL feature services with the latest component data in Moped. +The script `components_to_agol.py` is used to publish component record data to AGOL. It has two primary modes of operation: -The data is sourced from a view, `component_arcgis_online_view` which defines all columns which are available to be processed. +- Full refresh: This mode will delete all existing records in the AGOL feature layer and replace them with the current data from the Moped database. +- Incremental refresh: This mode will only update records that have been modified since a given timestamp. -The AGOL layers can be found here: +The script is responsible for maintaining four layers in the AGOL in the [Moped Project Components](https://austin.maps.arcgis.com/home/item.html?id=1c084c8756a84e6db7e2796c98c850a2) feature service: -- [Project component points](https://austin.maps.arcgis.com/home/item.html?id=997555f6e0904aa88eafe73f19ee65c0) -- [Project component lines](https://austin.maps.arcgis.com/home/item.html?id=e8f03d2cec154cacae539b630bcaa70b) +- [Moped Points](https://austin.maps.arcgis.com/home/item.html?id=1c084c8756a84e6db7e2796c98c850a2&sublayer=0): Components best represented as points, utilizing MultiPoint geometries +- [Moped Lines](https://austin.maps.arcgis.com/home/item.html?id=1c084c8756a84e6db7e2796c98c850a2&sublayer=1): Components best represented as lines, using Line geometries +- [MOPED CombinedGeometries](https://austin.maps.arcgis.com/home/item.html?id=1c084c8756a84e6db7e2796c98c850a2&sublayer=2) (SIC): All components, where points are transformed into a line ringing the location +- [Moped Feature Points](https://austin.maps.arcgis.com/home/item.html?id=1c084c8756a84e6db7e2796c98c850a2&sublayer=3): Components best represented as points, but where MultiPoints are exploded into individual points. Note, the same component can be represented as multiple features, one for each point in the MultiPoint. -### Get it running +The data for the first three layers listed above is sourced from a view, `component_arcgis_online_view` which defines all columns which are available to be processed. -1. Configure an `env_file` according to the `env_template` example. You can find the AGOL Scripts Publisher username and password in the API Secrets vault in the team password store. - -2. Create and activate a Python environment that meets the requirments in `requirements.txt`. Alternatively, you can use the provided Dockerfile. - -3. Run the script +The fourth layer is sourced from a derivative view, `exploded_component_arcgis_online_view`, which takes the previous view and explodes MultiPoint geometries into individual points. -If you want to fully replace the dataset: +## Running the Script -```shell -$ python components_to_agol.py -f -``` -Or, if you want to replace only data updated since a timestamp with time zone offset: -```shell -$ python components_to_agol.py -d -``` +1. Configure an `env_file` according to the `env_template` example. You can find the AGOL Scripts Publisher username and password in the API Secrets vault in the team password store. -or, to mount your local copy to a Docker container +1. `docker compose build` to build the container. -```shell -docker run -it --rm --network host --env-file env_file -v ${PWD}:/app atddocker/atd-moped-etl-arcgis:production python components_to_agol.py -``` +1. Run the script via one or more of the following: + - `docker compose run arcgis -d` to start the script with the default interval of changes over the last week. + - `docker compose run arcgis -f` to start the script with a full refresh. + - `docker compose run arcgis -d ` to start the script with a refresh since the given timestamp. + - `docker compose run --entrypoint /bin/bash arcgis` to start a shell inside the container. diff --git a/moped-etl/arcgis/components_to_agol.py b/moped-etl/arcgis/components_to_agol.py index 864d691a1f..64ac5e932f 100644 --- a/moped-etl/arcgis/components_to_agol.py +++ b/moped-etl/arcgis/components_to_agol.py @@ -1,13 +1,15 @@ #!/usr/bin/env python """Copies all Moped component records to ArcGIS Online (AGOL)""" -# docker run -it --rm --network host --env-file env_file -v ${PWD}:/app moped-agol /bin/bash +# docker compose run arcgis; import argparse import logging -from datetime import datetime, timezone +import json +from datetime import datetime, timezone, timedelta from process.logging import get_logger from settings import ( COMPONENTS_QUERY_BY_LAST_UPDATE_DATE, + EXPLODED_COMPONENTS_QUERY_BY_LAST_UPDATE_DATE, UPLOAD_CHUNK_SIZE, ) from utils import ( @@ -42,7 +44,7 @@ def make_esri_feature(*, esri_geometry_key, geometry, attributes): See: https://developers.arcgis.com/documentation/common-data-types/feature-object.htm Args: - esri_geometry_key (str): `paths` or `points`: see the `get_esri_geometry_key` docstring + esri_geometry_key (str): `paths` or `points`, `point`: see the `get_esri_geometry_key` docstring geometry (dict): A geojson geometry object, such as one returned from our component view in Moped attribute (dict): Any additional properties to be included as feature attributes @@ -59,27 +61,41 @@ def make_esri_feature(*, esri_geometry_key, geometry, attributes): "spatialReference": {"wkid": 4326}, }, } - feature["geometry"][esri_geometry_key] = geometry["coordinates"] + if (esri_geometry_key == "points") or (esri_geometry_key == "paths"): + feature["geometry"][esri_geometry_key] = geometry["coordinates"] + elif esri_geometry_key == "point": + geometry = json.loads(geometry) + feature["geometry"]["y"] = geometry["coordinates"][1] + feature["geometry"]["x"] = geometry["coordinates"][0] + else: + feature["geometry"]["y"] = geometry["coordinates"][1] + feature["geometry"]["x"] = geometry["coordinates"][0] + return feature -def make_all_features(data): +def make_all_features(data, exploded_geometry): """Take a list of component feature records and create Esri feature objects for lines, points, and combined layers in AGOL. Args: data (dict): a list of component feature records + exploded_geometry (dict): a dictionary of exploded geometry data from the component_arcgis_online_view. This is created + by taking the multi-point geometry from the component_arcgis_online_view and "exploding" it into individual points. Returns: dict: An object with lists of Esri feature objects for lines, points, and combined layers """ - all_features = {"lines": [], "points": [], "combined": []} + + all_features = {"lines": [], "points": [], "combined": [], "exploded": []} logger.info("Building Esri feature objects...") for component in data: - # extract geometry and line geometry from component data - # for line features, the line geometry is redundant/identical to geometry - # for point features, it is the buffered ring around the point as defined - # in the Moped component view + + # Extract geometry and line geometry from component data. + # For line features, the line geometry is redundant/identical to geometry. + # For point features, it is the buffered ring around the point as defined + # in the Moped component view. + geometry = component.pop("geometry") line_geometry = component.pop("line_geometry") @@ -115,6 +131,27 @@ def make_all_features(data): attributes=component, ) all_features["combined"].append(line_feature) + + project_component_id = feature["attributes"]["project_component_id"] + # Filter exploded_geometry to only include dicts with matching project_component_id + matching_exploded_geometry_records = [ + feature + for feature in exploded_geometry + if feature.get("project_component_id") == project_component_id + ] + for record in matching_exploded_geometry_records: + geometry = record.pop("geometry") + esri_geometry_key = "point" + + feature = make_esri_feature( + esri_geometry_key="point", + geometry=geometry, + attributes=component, + ) + + feature["attributes"]["source_geometry_type"] = "point" + all_features["exploded"].append(feature) + else: all_features["lines"].append(feature) all_features["combined"].append(feature) @@ -140,10 +177,15 @@ def main(args): variables=variables, )["component_arcgis_online_view"] - all_features = make_all_features(data) + exploded_data = make_hasura_request( + query=EXPLODED_COMPONENTS_QUERY_BY_LAST_UPDATE_DATE, + variables=variables, + )["exploded_component_arcgis_online_view"] + + all_features = make_all_features(data, exploded_data) if args.full: - for feature_type in ["points", "lines", "combined"]: + for feature_type in ["points", "lines", "combined", "exploded"]: logger.info(f"Processing {feature_type} features...") features = all_features[feature_type] @@ -166,7 +208,7 @@ def main(args): project_ids_for_feature_delete = list(set(project_ids)) # Delete outdated feature from AGOL and add updated features - for feature_type in ["points", "lines", "combined"]: + for feature_type in ["points", "lines", "combined", "exploded"]: logger.info(f"Processing {feature_type} features...") logger.info( f"Deleting all {len(all_features[feature_type])} existing features in {feature_type} layer for updated projects in chunks of {UPLOAD_CHUNK_SIZE}..." @@ -195,15 +237,17 @@ def main(args): "-d", "--date", type=str, + nargs="?", + const=(datetime.now(timezone.utc) - timedelta(days=7)).isoformat(), default=None, - help=f"ISO date string with TZ offset (ex. 2024-06-28T00:06:16.360805+00:00) of latest updated_at value to find project records to update.", + help="ISO date string with TZ offset (ex. 2024-06-28T00:06:16.360805+00:00) of latest updated_at value to find project records to update. Defaults to 7 days ago if -d is used without a value.", ) parser.add_argument( "-f", "--full", action="store_true", - help=f"Delete and replace all project components.", + help="Delete and replace all project components.", ) args = parser.parse_args() @@ -214,6 +258,11 @@ def main(args): "Please provide either the -d flag with ISO date string with TZ offset or the -f flag and not both." ) + if not args.date and not args.full: + raise Exception( + "Please provide either the -d flag with optional ISO date string with TZ offset or the -f flag." + ) + if args.full: logger.info(f"Starting sync. Replacing all projects' components data...") else: diff --git a/moped-etl/arcgis/docker-compose.yaml b/moped-etl/arcgis/docker-compose.yaml new file mode 100644 index 0000000000..b1f5673700 --- /dev/null +++ b/moped-etl/arcgis/docker-compose.yaml @@ -0,0 +1,10 @@ +services: + arcgis: + build: + context: . + volumes: + - .:/app + entrypoint: python /app/components_to_agol.py + command: -d + env_file: + - env_file diff --git a/moped-etl/arcgis/env_template b/moped-etl/arcgis/env_template index 6362550760..eaf86a73b7 100644 --- a/moped-etl/arcgis/env_template +++ b/moped-etl/arcgis/env_template @@ -1,4 +1,4 @@ AGOL_USERNAME= AGOL_PASSWORD= -HASURA_ENDPOINT=http://localhost:8080/v1/graphql +HASURA_ENDPOINT=http://host.docker.internal:8080/v1/graphql HASURA_ADMIN_SECRET=hasurapassword diff --git a/moped-etl/arcgis/settings.py b/moped-etl/arcgis/settings.py index 604eca2a80..3a23279a44 100644 --- a/moped-etl/arcgis/settings.py +++ b/moped-etl/arcgis/settings.py @@ -1,6 +1,6 @@ UPLOAD_CHUNK_SIZE = 100 -LAYER_IDS = {"points": 0, "lines": 1, "combined": 2} +LAYER_IDS = {"points": 0, "lines": 1, "combined": 2, "exploded": 3} COMPONENTS_QUERY_BY_LAST_UPDATE_DATE = """ query GetProjectsComponents($where: component_arcgis_online_view_bool_exp!) { @@ -84,3 +84,13 @@ } } """ + +# line_geometry +EXPLODED_COMPONENTS_QUERY_BY_LAST_UPDATE_DATE = """ +query GetExplodedProjectsComponents($where: exploded_component_arcgis_online_view_bool_exp!) { + exploded_component_arcgis_online_view(where: $where) { + geometry: exploded_geometry + project_component_id + } +} +"""