#1149 switch to formatting geojson in postgres

dags/ecocounter_open_data.py

@@ -35,6 +35,12 @@
 README_PATH = os.path.join(repo_path, 'volumes/ecocounter/readme.md')
 DOC_MD = get_readme_docmd(README_PATH, DAG_NAME)
 EXPORT_PATH = '/home/airflow/open_data/permanent-bike-counters' #'/data/open_data/permanent-bike-counters'
+BASH_ENV = {
+    'HOST': '{{ conn.ecocounter_bot.host }}',
+    'LOGIN': '{{ conn.ecocounter_bot.login }}',
+    'PGPASSWORD': '{{ conn.ecocounter_bot.password }}',
+    'EXPORT_PATH': EXPORT_PATH
+}
 
 default_args = {
     'owner': ','.join(DAG_OWNERS),
@@ -132,14 +138,7 @@ def insert_15min(yr):
             )
             return t.execute(context=context)
 
-        @task.bash(
-            env = {
-                'HOST': '{{ conn.ecocounter_bot.host }}',
-                'LOGIN': '{{ conn.ecocounter_bot.login }}',
-                'PGPASSWORD': '{{ conn.ecocounter_bot.password }}',
-                'EXPORT_PATH': EXPORT_PATH
-            }
-        )
+        @task.bash(env = BASH_ENV)
         def download_daily_open_data()->str:
             return '''/usr/bin/psql -h $HOST -U $LOGIN -d bigdata -c \
                 "SELECT
@@ -151,12 +150,7 @@ def download_daily_open_data()->str:
                 --csv -o "$EXPORT_PATH/cycling_permanent_counts_daily.csv"'''
 
         @task.bash(
-            env = {
-                'HOST': '{{ conn.ecocounter_bot.host }}',
-                'LOGIN': '{{ conn.ecocounter_bot.login }}',
-                'PGPASSWORD': '{{ conn.ecocounter_bot.password }}',
-                'EXPORT_PATH': EXPORT_PATH
-            },
+            env = BASH_ENV,
             map_index_template="{{ yr }}"
         )
         def download_15min_open_data(yr)->str:
@@ -175,14 +169,7 @@ def download_15min_open_data(yr)->str:
         insert_daily(yr) >> download_daily_open_data()
         insert_15min(yr) >> download_15min_open_data(yr)
 
-    @task.bash(
-        env = {
-            'HOST': '{{ conn.ecocounter_bot.host }}',
-            'LOGIN': '{{ conn.ecocounter_bot.login }}',
-            'PGPASSWORD': '{{ conn.ecocounter_bot.password }}',
-            'EXPORT_PATH': EXPORT_PATH
-        }
-    )
+    @task.bash(env = BASH_ENV)
     def download_locations_open_data()->str:
         return '''/usr/bin/psql -h $HOST -U $LOGIN -d bigdata -c \
                 "SELECT location_dir_id, location_name, direction, linear_name_full, side_street,
@@ -192,24 +179,11 @@ def download_locations_open_data()->str:
                 ORDER BY location_dir_id;" \
                 --csv -o "$EXPORT_PATH/cycling_permanent_counts_locations.csv"'''
 
-    @task.bash(
-        env = {
-            'HOST': '{{ conn.ecocounter_bot.host }}',
-            'LOGIN': '{{ conn.ecocounter_bot.login }}',
-            'PGPASSWORD': '{{ conn.ecocounter_bot.password }}',
-            'EXPORT_PATH': EXPORT_PATH
-        }
-    )
+    @task.bash(env = BASH_ENV)
     def download_locations_open_data_geojson()->str:
-        return '''/usr/bin/ogr2ogr -f "GeoJSON" \
-            "$EXPORT_PATH/cycling_permanent_counts_locations.geojson" \
-            PG:"host=$HOST dbname=bigdata" \
-            -sql "SELECT location_dir_id, location_name, direction, linear_name_full, side_street,
-                            longitude, latitude, centreline_id, bin_size, latest_calibration_study,
-                            first_active, last_active, date_decommissioned, technology
-                        FROM open_data.cycling_permanent_counts_locations
-                        ORDER BY location_dir_id;" \
-            -nln cycling_permanent_counts_locations'''
+        return '''/usr/bin/psql -h $HOST -U $LOGIN -d bigdata -c \
+    "SELECT featurecollection FROM open_data.cycling_permanent_counts_locations_geojson;" \
+    --tuples-only -o "$EXPORT_PATH/cycling_permanent_counts_locations.geojson"'''
 
     @task.bash()
     def output_readme()->str:

volumes/open_data/sql/create-view-cycling_permanent_counts_locations_geojson.sql

@@ -0,0 +1,37 @@
+CREATE OR REPLACE VIEW open_data.cycling_permanent_counts_locations_geojson AS
+WITH features AS (
+    SELECT json_build_object(
+        'type', 'Feature',
+        'id', location_dir_id,
+        'geometry', st_asgeojson(st_setsrid(st_makepoint(longitude, latitude), 4326))::jsonb,
+        'attributes', json_build_object(
+            'location_dir_id', location_dir_id,
+            'location_name', location_name,
+            'direction', direction,
+            'linear_name_full', linear_name_full,
+            'side_street', side_street,
+            'centreline_id', centreline_id,
+            'bin_size', bin_size,
+            'latest_calibration_study', latest_calibration_study,
+            'first_active', first_active,
+            'last_active', last_active,
+            'date_decommissioned', date_decommissioned,
+            'technology', technology
+        )
+    ) AS features
+    FROM open_data.cycling_permanent_counts_locations
+)
+
+--aggregate features into FeatureCollection
+SELECT json_build_object(
+    'type',
+    'FeatureCollection',
+    'features',
+    json_agg(features.features)
+) AS featurecollection
+FROM features;
+
+ALTER VIEW open_data.cycling_permanent_counts_locations_geojson OWNER TO od_admins;
+
+GRANT SELECT ON open_data.cycling_permanent_counts_locations_geojson TO ecocounter_bot;
+GRANT SELECT ON open_data.cycling_permanent_counts_locations_geojson TO bdit_humans;