merge country and centre-id (#562)

wmo-im · Nov 14, 2023 · 1d7e83a · 1d7e83a
1 parent 35b1663
commit 1d7e83a
Show file tree

Hide file tree

Showing 19 changed files with 70 additions and 81 deletions.
diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml
@@ -49,9 +49,9 @@ jobs:
         python3 wis2box-ctl.py execute wis2box metadata station publish-collection
     - name: add Malawi data 🇲🇼
       env:
-        TOPIC_HIERARCHY: mwi.mwi_met_centre.data.core.weather.surface-based-observations.synop
-        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/mwi-surface-weather-observations.yml
-        DISCOVERY_METADATA_ID: urn:x-wmo:md:mwi:mwi_met_centre:surface-weather-observations
+        TOPIC_HIERARCHY: mw-mw_met_centre.data.core.weather.surface-based-observations.synop
+        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/mw-surface-weather-observations.yml
+        DISCOVERY_METADATA_ID: urn:x-wmo:md:mw-mw_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/malawi
       run: |
         python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA
@@ -61,9 +61,9 @@ jobs:
         check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
     - name: add Italy data 🇮🇹
       env:
-        TOPIC_HIERARCHY: ita.roma_met_centre.data.core.weather.surface-based-observations.synop
-        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/ita-surface-weather-observations.yml
-        DISCOVERY_METADATA_ID: urn:x-wmo:md:ita:roma_met_centre:surface-weather-observations
+        TOPIC_HIERARCHY: it-roma_met_centre.data.core.weather.surface-based-observations.synop
+        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/it-surface-weather-observations.yml
+        DISCOVERY_METADATA_ID: urn:x-wmo:md:it-roma_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/italy
       run: |
         python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA
@@ -73,9 +73,9 @@ jobs:
         check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
     - name: add Algeria data 🇩🇿
       env:
-        TOPIC_HIERARCHY: dza.alger_met_centre.data.core.weather.surface-based-observations.synop
-        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/dza-surface-weather-observations.yml
-        DISCOVERY_METADATA_ID: urn:x-wmo:md:dza:dza_met_centre:surface-weather-observations
+        TOPIC_HIERARCHY: dz-alger_met_centre.data.core.weather.surface-based-observations.synop
+        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/dz-surface-weather-observations.yml
+        DISCOVERY_METADATA_ID: urn:x-wmo:md:dz-alger_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/algeria
       run: |
         python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA
@@ -85,9 +85,9 @@ jobs:
         check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
     - name: add Romania data 🇷🇴
       env:
-        TOPIC_HIERARCHY: rou.rnimh.data.core.weather.surface-based-observations.synop
-        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/rou-synoptic-weather-observations.yml
-        DISCOVERY_METADATA_ID: urn:x-wmo:md:rou:rnimh:synoptic-weather-observations
+        TOPIC_HIERARCHY: ro-rnimh.data.core.weather.surface-based-observations.synop
+        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/ro-synoptic-weather-observations.yml
+        DISCOVERY_METADATA_ID: urn:x-wmo:md:ro-rnimh:synoptic-weather-observations
         TEST_DATA: /data/wis2box/observations/romania
       run: |
         python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA
@@ -97,9 +97,9 @@ jobs:
         check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
     - name: add Congo data 🇨🇩
       env:
-        TOPIC_HIERARCHY: cog.brazza_met_centre.data.core.weather.surface-based-observations.synop
-        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cog-surface-weather-observations.yml
-        DISCOVERY_METADATA_ID: urn:x-wmo:md:cog:brazza_met_centre:surface-weather-observations
+        TOPIC_HIERARCHY: cd-brazza_met_centre.data.core.weather.surface-based-observations.synop
+        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cd-surface-weather-observations.yml
+        DISCOVERY_METADATA_ID: urn:x-wmo:md:cd-brazza_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/congo
       run: |
         python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA

diff --git a/config-templates/data-mappings.yml.tmpl b/config-templates/data-mappings.yml.tmpl
@@ -1,5 +1,5 @@
 data:
-    $COUNTRY_CODE.$CENTRE_ID.data.core.weather.surface-based-observations.synop:
+    $CENTRE_ID.data.core.weather.surface-based-observations.synop:
         plugins:
             txt:
                 - plugin: wis2box.data.synop2bufr.ObservationDataSYNOP2BUFR
@@ -37,7 +37,7 @@ data:
                   buckets:
                     - $${WIS2BOX_STORAGE_PUBLIC}
                   file-pattern: '^WIGOS_(\d-\d+-\d+-\w+)_.*\.bufr4$$'
-    $COUNTRY_CODE.$CENTRE_ID.data.core.weather.surface-based-observations.temp:
+    $CENTRE_ID.data.core.weather.surface-based-observations.temp:
         plugins:
             b:
                 - plugin: wis2box.data.bufr4.ObservationDataBUFR

diff --git a/config-templates/metadata-synop.yml.tmpl b/config-templates/metadata-synop.yml.tmpl
@@ -1,14 +1,13 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: $COUNTRY_CODE.$CENTRE_ID.data.core.weather.surface-based-observations.synop
-    country: $COUNTRY_CODE
+    topic_hierarchy: $CENTRE_ID.data.core.weather.surface-based-observations.synop
     centre_id: $CENTRE_ID
 
 mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:$COUNTRY_CODE:$CENTRE_ID:surface-based-observations.synop
+    identifier: urn:x-wmo:md:$CENTRE_ID:surface-based-observations.synop
     language: en
     charset: utf8
     hierarchylevel: dataset

diff --git a/config-templates/metadata-temp.yml.tmpl b/config-templates/metadata-temp.yml.tmpl
@@ -1,14 +1,13 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: $COUNTRY_CODE.$CENTRE_ID.data.core.weather.surface-based-observations.temp
-    country: $COUNTRY_CODE
+    topic_hierarchy: $CENTRE_ID.data.core.weather.surface-based-observations.temp
     centre_id: $CENTRE_ID
 
 mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:$COUNTRY_CODE:$CENTRE_ID:surface-based-observations.temp
+    identifier: urn:x-wmo:md:$CENTRE_ID:surface-based-observations.temp
     language: en
     charset: utf8
     hierarchylevel: dataset

diff --git a/docs/architecture/c4-container.png b/docs/architecture/c4-container.png
diff --git a/examples/config/surface-weather-observations.yml b/examples/config/surface-weather-observations.yml
@@ -1,23 +1,23 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: country.centre_id.data.core.weather.surface-based-observations.synop
+    topic_hierarchy: centre_id.data.core.weather.surface-based-observations.synop
     country: country
     centre_id: centre_id
 
 mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:country:centre_id:surface-weather-observations
+    identifier: urn:x-wmo:md:centre_id:surface-weather-observations
     language: en
     charset: utf8
     hierarchylevel: dataset
 
 identification:
     language: en
     charset: utf8
-    title: Surface weather observations from country.centre_id
-    abstract: Surface weather observations from country.centre_id
+    title: Surface weather observations from centre_id
+    abstract: Surface weather observations from centre_id
     dates:
         creation: 2021-11-29
         publication: 2021-11-29

diff --git a/examples/config/synop-bin-mappings.yml b/examples/config/synop-bin-mappings.yml
@@ -1,5 +1,5 @@
 data:
-    [country].[centre_id].data.core.weather.surface-based-observations.synop:
+    [centre_id].data.core.weather.surface-based-observations.synop:
         plugins:
             bin:
                 - plugin: wis2box.data.bufr4.ObservationDataBUFR

diff --git a/examples/config/synop-bufr-mappings.yml b/examples/config/synop-bufr-mappings.yml
@@ -1,5 +1,5 @@
 data:
-    [country].[centre_id].data.core.weather.surface-based-observations.synop:
+    [centre_id].data.core.weather.surface-based-observations.synop:
         plugins:
             bufr:
                 - plugin: wis2box.data.bufr4.ObservationDataBUFR

diff --git a/examples/config/synop-csv-mappings.yml b/examples/config/synop-csv-mappings.yml
@@ -1,5 +1,5 @@
 data:
-    [country].[centre_id].data.core.weather.surface-based-observations.synop:
+    [centre_id].data.core.weather.surface-based-observations.synop:
         plugins:
             csv:
                 - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR

diff --git a/examples/config/synop-synop-mappings.yml b/examples/config/synop-synop-mappings.yml
@@ -1,5 +1,5 @@
 data:
-    [country].[centre_id].data.core.weather.surface-based-observations.synop:
+    [centre_id].data.core.weather.surface-based-observations.synop:
         plugins:
             txt:
                 - plugin: wis2box.data.synop2bufr.ObservationDataSYNOP2BUFR

diff --git a/tests/data/data-mappings.yml b/tests/data/data-mappings.yml
@@ -1,5 +1,5 @@
 data:
-    cog.brazza_met_centre.data.core.weather.surface-based-observations.synop:
+    cd-brazza_met_centre.data.core.weather.surface-based-observations.synop:
         plugins:
             txt:
                 - plugin: wis2box.data.synop2bufr.ObservationDataSYNOP2BUFR
@@ -10,7 +10,7 @@ data:
                   buckets:
                     - ${WIS2BOX_STORAGE_PUBLIC}
                   file-pattern: '^.*\.bufr4$' 
-    rou.rnimh.data.core.weather.surface-based-observations.synop:
+    ro-rnimh.data.core.weather.surface-based-observations.synop:
         plugins:
             txt:
                 - plugin: wis2box.data.synop2bufr.ObservationDataSYNOP2BUFR
@@ -24,7 +24,7 @@ data:
             bufr4:
                 - plugin: wis2box.data.bufr2geojson.ObservationDataBUFR2GeoJSON
                   file-pattern: '^A_SMR.*EDZW_(\d{4})(\d{2}).*.bufr4$'
-    mwi.mwi_met_centre.data.core.weather.surface-based-observations.synop:
+    mw-mw_met_centre.data.core.weather.surface-based-observations.synop:
         plugins:
             csv:
                 - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR
@@ -34,7 +34,7 @@ data:
             bufr4:
                 - plugin: wis2box.data.bufr2geojson.ObservationDataBUFR2GeoJSON
                   file-pattern: '^WIGOS_(\d-\d+-\d+-\w+)_.*\.bufr4$'
-    ita.roma_met_centre.data.core.weather.surface-based-observations.synop:
+    it-roma_met_centre.data.core.weather.surface-based-observations.synop:
         plugins:
             bin:
                 - plugin: wis2box.data.bufr4.ObservationDataBUFR
@@ -43,7 +43,7 @@ data:
             bufr4:
                 - plugin: wis2box.data.bufr2geojson.ObservationDataBUFR2GeoJSON
                   file-pattern: '^WIGOS_(\d-\d+-\d+-\w+)_.*\.bufr4$'
-    dza.alger_met_centre.data.core.weather.surface-based-observations.synop:
+    dz-alger_met_centre.data.core.weather.surface-based-observations.synop:
         plugins:
             bufr4:
                 - plugin: wis2box.data.bufr4.ObservationDataBUFR

diff --git a/...very/cog-surface-weather-observations.yml → ...overy/cd-surface-weather-observations.yml b/...very/cog-surface-weather-observations.yml → ...overy/cd-surface-weather-observations.yml
@@ -1,14 +1,14 @@
 wis2box:
     retention: P180D
-    topic_hierarchy: cog.brazza_met_centre.data.core.weather.surface-based-observations.synop
+    topic_hierarchy: cd-brazza_met_centre.data.core.weather.surface-based-observations.synop
     country: cog
-    centre_id: brazza_met_centre
+    centre_id: cd-brazza_met_centre
 
 mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:cog:brazza_met_centre:surface-weather-observations
+    identifier: urn:x-wmo:md:cd-brazza_met_centre:surface-weather-observations
     hierarchylevel: dataset
 
 identification:

diff --git a/...very/dza-surface-weather-observations.yml → ...overy/dz-surface-weather-observations.yml b/...very/dza-surface-weather-observations.yml → ...overy/dz-surface-weather-observations.yml
@@ -1,14 +1,14 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: dza.alger_met_centre.data.core.weather.surface-based-observations.synop
+    topic_hierarchy: dz-alger_met_centre.data.core.weather.surface-based-observations.synop
     country: dza
-    centre_id: alger_met_centre
+    centre_id: dz-alger_met_centre
 
 mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:dza:dza_met_centre:surface-weather-observations
+    identifier: urn:x-wmo:md:dz-alger_met_centre:surface-weather-observations
     hierarchylevel: dataset
 
 identification:

diff --git a/...very/ita-surface-weather-observations.yml → ...overy/it-surface-weather-observations.yml b/...very/ita-surface-weather-observations.yml → ...overy/it-surface-weather-observations.yml
@@ -1,14 +1,14 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: ita.roma_met_centre.data.core.weather.surface-based-observations.synop
+    topic_hierarchy: it-roma_met_centre.data.core.weather.surface-based-observations.synop
     country: ita
-    centre_id: roma_met_centre
+    centre_id: it-roma_met_centre
 
 mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:ita:roma_met_centre:surface-weather-observations
+    identifier: urn:x-wmo:md:it-roma_met_centre:surface-weather-observations
     hierarchylevel: dataset
 
 identification:

diff --git a/...very/mwi-surface-weather-observations.yml → ...overy/mw-surface-weather-observations.yml b/...very/mwi-surface-weather-observations.yml → ...overy/mw-surface-weather-observations.yml
@@ -1,14 +1,14 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: mwi.mwi_met_centre.data.core.weather.surface-based-observations.synop
+    topic_hierarchy: mw-mw_met_centre.data.core.weather.surface-based-observations.synop
     country: mwi
-    centre_id: mwi_met_centre
+    centre_id: mw-mw_met_centre
 
 mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:mwi:mwi_met_centre:surface-weather-observations
+    identifier: urn:x-wmo:md:mw-mw_met_centre:surface-weather-observations
     hierarchylevel: dataset
 
 identification:

diff --git a/...ery/rou-synoptic-weather-observations.yml → ...very/ro-synoptic-weather-observations.yml b/...ery/rou-synoptic-weather-observations.yml → ...very/ro-synoptic-weather-observations.yml
@@ -1,14 +1,14 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: rou.rnimh.data.core.weather.surface-based-observations.synop
+    topic_hierarchy: ro-rnimh.data.core.weather.surface-based-observations.synop
     country: rou
-    centre_id: rnimh
+    centre_id: ro-rnimh
 
 mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:rou:rnimh:synoptic-weather-observations
+    identifier: urn:x-wmo:md:ro-rnimh:synoptic-weather-observations
     hierarchylevel: dataset
 
 identification:

diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py
@@ -33,7 +33,7 @@
 
 URL = 'http://localhost'
 API_URL = f'{URL}/oapi'
-ID = 'urn:x-wmo:md:mwi:mwi_met_centre:surface-weather-observations'
+ID = 'urn:x-wmo:md:mw-mw_met_centre:surface-weather-observations'
 SESSION = Session()
 SESSION.hooks = {
    'response': lambda r, *args, **kwargs: r.raise_for_status()
@@ -101,7 +101,7 @@ def test_metadata_discovery_publish():
     mqtt_link = [d for d in r['links'] if d['type'] == 'MQTT'][0]
 
     assert 'everyone:everyone' in mqtt_link['href']
-    assert mqtt_link['channel'] == 'origin/a/wis2/mwi/mwi_met_centre/data/core/weather/surface-based-observations/synop'  # noqa
+    assert mqtt_link['channel'] == 'origin/a/wis2/mw-mw_met_centre/data/core/weather/surface-based-observations/synop'  # noqa
 
     params = {
         'q': 'temperature'
@@ -114,17 +114,17 @@ def test_metadata_discovery_publish():
 
     # test access of discovery metadata from notification message
 
-    countries_and_centre_ids = [
-        ('mwi', 'mwi_met_centre'),
-        ('ita', 'roma_met_centre'),
-        ('dza', 'alger_met_centre'),
-        ('rou', 'rnimh'),
-        ('cog', 'brazza_met_centre')
+    centre_ids = [
+        'mw-mw_met_centre',
+        'it-roma_met_centre',
+        'dz-alger_met_centre',
+        'ro-rnimh',
+        'cd-brazza_met_centre'
     ]
 
-    for cacid in countries_and_centre_ids:
+    for centre_id in centre_ids:
         params = {
-            'q': f'{cacid[0]} AND {cacid[1]} AND metadata'
+            'q': f'{centre_id} AND metadata'
         }
 
         r = SESSION.get(f'{API_URL}/collections/messages/items',
@@ -133,8 +133,7 @@ def test_metadata_discovery_publish():
         assert r['numberMatched'] == 1
 
         feature = r['features'][0]
-        assert feature['properties']['data_id'].startswith(
-            f'{cacid[0]}/{cacid[1]}')
+        assert feature['properties']['data_id'].startswith(centre_id)
 
         link = feature['links'][0]
 
@@ -157,7 +156,7 @@ def test_data_ingest():
 
     assert item_api['reportId'] == 'WIGOS_0-454-2-AWSNAMITAMBO_20210707T145500'
     assert item_api['properties']['resultTime'] == '2021-07-07T14:55:00Z'  # noqa
-    item_source = f'2021-07-07/wis/mwi/mwi_met_centre/data/core/weather/surface-based-observations/synop/{item_api["reportId"]}.bufr4' # noqa
+    item_source = f'2021-07-07/wis/mw-mw_met_centre/data/core/weather/surface-based-observations/synop/{item_api["reportId"]}.bufr4' # noqa
     r = SESSION.get(f'{URL}/data/{item_source}')  # noqa
     assert r.status_code == codes.ok
 
@@ -228,7 +227,7 @@ def test_message_api():
 
     # test messages per test dataset
     counts = {
-        'mwi_met_centre': 24,
+        'mw_met_centre': 24,
         'roma_met_centre': 33,
         'alger_met_centre': 28,
         'rnimh': 188,
@@ -257,7 +256,7 @@ def test_message_api():
     assert props['integrity']['method'] == 'sha512'
     assert not props['data_id'].startswith('wis2')
     assert not props['data_id'].startswith('origin/a/wis2')
-    assert props['data_id'].startswith('cog')
+    assert props['data_id'].startswith('cd')
 
     link_rel = msg['links'][0]