WCMP2 and internal message bus fixes

.github/workflows/tests-docker.yml

@@ -51,7 +51,7 @@ jobs:
       env:
         TOPIC_HIERARCHY: mw-mw_met_centre.data.core.weather.surface-based-observations.synop
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/mw-surface-weather-observations.yml
-        DISCOVERY_METADATA_ID: urn:x-wmo:md:mw-mw_met_centre:surface-weather-observations
+        DISCOVERY_METADATA_ID: urn:wmo:md:mw-mw_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/malawi
         TEST_DATA_UPDATE: /data/wis2box/observations/malawi_update
       run: |
@@ -64,7 +64,7 @@ jobs:
       env:
         TOPIC_HIERARCHY: it-roma_met_centre.data.core.weather.surface-based-observations.synop
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/it-surface-weather-observations.yml
-        DISCOVERY_METADATA_ID: urn:x-wmo:md:it-roma_met_centre:surface-weather-observations
+        DISCOVERY_METADATA_ID: urn:wmo:md:it-roma_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/italy
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
@@ -75,7 +75,7 @@ jobs:
       env:
         TOPIC_HIERARCHY: dz-alger_met_centre.data.core.weather.surface-based-observations.synop
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/dz-surface-weather-observations.yml
-        DISCOVERY_METADATA_ID: urn:x-wmo:md:dz-alger_met_centre:surface-weather-observations
+        DISCOVERY_METADATA_ID: urn:wmo:md:dz-alger_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/algeria
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
@@ -86,7 +86,7 @@ jobs:
       env:
         TOPIC_HIERARCHY: ro-rnimh.data.core.weather.surface-based-observations.synop
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/ro-synoptic-weather-observations.yml
-        DISCOVERY_METADATA_ID: urn:x-wmo:md:ro-rnimh:synoptic-weather-observations
+        DISCOVERY_METADATA_ID: urn:wmo:md:ro-rnimh:synoptic-weather-observations
         TEST_DATA: /data/wis2box/observations/romania
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
@@ -97,7 +97,7 @@ jobs:
       env:
         TOPIC_HIERARCHY: cd-brazza_met_centre.data.core.weather.surface-based-observations.synop
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cd-surface-weather-observations.yml
-        DISCOVERY_METADATA_ID: urn:x-wmo:md:cd-brazza_met_centre:surface-weather-observations
+        DISCOVERY_METADATA_ID: urn:wmo:md:cd-brazza_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/congo
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
@@ -108,7 +108,7 @@ jobs:
       env:
         TOPIC_HIERARCHY: cn-cma.data.core.weather.prediction.forecast.medium-range.probabilistic.global
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cn-grapes-geps-global.yml
-        DISCOVERY_METADATA_ID: urn:x-wmo:md:cn-cma:grapes-geps-global
+        DISCOVERY_METADATA_ID: urn:wmo:md:cn-cma:grapes-geps-global
         TEST_DATA: /data/wis2box/observations/china
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA

config-templates/metadata-synop.yml.tmpl

@@ -7,7 +7,7 @@ mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:$CENTRE_ID:surface-based-observations.synop
+    identifier: urn:wmo:md:$CENTRE_ID:surface-based-observations.synop
     language: en
     charset: utf8
     hierarchylevel: dataset
@@ -32,7 +32,7 @@ identification:
             keywords_type: theme
             vocabulary:
                 name: WMO WIS2 Topic Hierarchy
-                url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline/index.csv
+                url: https://codes.wmo.int/wis/topic-hierarchy/earth-system-discipline
     extents:
         spatial:
             - bbox: [$BOUNDING_BOX]

config-templates/metadata-temp.yml.tmpl

@@ -7,7 +7,7 @@ mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:$CENTRE_ID:surface-based-observations.temp
+    identifier: urn:wmo:md:$CENTRE_ID:surface-based-observations.temp
     language: en
     charset: utf8
     hierarchylevel: dataset
@@ -33,7 +33,7 @@ identification:
             keywords_type: theme
             vocabulary:
                 name: WMO WIS2 Topic Hierarchy
-                url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline/index.csv
+                url: https://codes.wmo.int/wis/topic-hierarchy/earth-system-discipline
     extents:
         spatial:
             - bbox: [$BOUNDING_BOX]

examples/config/surface-weather-observations.yml

@@ -8,7 +8,7 @@ mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:centre_id:surface-weather-observations
+    identifier: urn:wmo:md:centre_id:surface-weather-observations
     language: en
     charset: utf8
     hierarchylevel: dataset
@@ -33,7 +33,7 @@ identification:
             keywords_type: theme
             vocabulary:
                 name: WMO WIS2 Topic Hierarchy
-                url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline/index.csv
+                url: https://codes.wmo.int/topic-hierarchy/earth-system-discipline
     extents:
         spatial:
             - bbox: [-180, -90, 190, 90]

tests/data/metadata/discovery/cd-surface-weather-observations.yml

@@ -19,7 +19,7 @@ mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:cd-brazza_met_centre:surface-weather-observations
+    identifier: urn:wmo:md:cd-brazza_met_centre:surface-weather-observations
     hierarchylevel: dataset
 
 identification:
@@ -39,7 +39,7 @@ identification:
             keywords_type: theme
             vocabulary:
                 name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1.
-                url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline/index.csv
+                url: https://codes.wmo.int/topic-hierarchy/earth-system-discipline
     extents:
         spatial:
             - bbox: [11.0937728207,-5.03798674888,18.4530652198,3.72819651938]

tests/data/metadata/discovery/cn-grapes-geps-global.yml

@@ -16,7 +16,7 @@ mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:cn-cma:grapes-geps-global
+    identifier: urn:wmo:md:cn-cma:grapes-geps-global
     hierarchylevel: dataset
 
 identification:
@@ -42,7 +42,7 @@ identification:
             keywords_type: theme
             vocabulary:
                 name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1.
-                url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline/index.csv
+                url: https://codes.wmo.int/wis/topic-hierarchy/earth-system-discipline
     extents:
         spatial:
             - bbox: [73.66000, 4.00000, 135.08000, 53.52000]
@@ -51,7 +51,7 @@ identification:
             - begin: 2021-11-29
               end: null
               resolution: P12H
-    url: http://gisc.wis.cma.cn/wis/portal.pub?M_PID=urn:x-wmo:md:int.wmo.wis::CMA_GEPS
+    url: http://gisc.wis.cma.cn/wis/portal.pub?M_PID=urn:wmo:md:int.wmo.wis::CMA_GEPS
     wmo_data_policy: core
 
 contact:

tests/data/metadata/discovery/dz-surface-weather-observations.yml

@@ -20,7 +20,7 @@ mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:dz-alger_met_centre:surface-weather-observations
+    identifier: urn:wmo:md:dz-alger_met_centre:surface-weather-observations
     hierarchylevel: dataset
 
 identification:
@@ -40,7 +40,7 @@ identification:
             keywords_type: theme
             vocabulary:
                 name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1.
-                url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline/index.csv
+                url: https://codes.wmo.int/wis/topic-hierarchy/earth-system-discipline
     extents:
         spatial:
             - bbox: [-8.68439978681, 19.0573642034, 11.9995056495, 37.1183806422]

tests/data/metadata/discovery/it-surface-weather-observations.yml

@@ -17,7 +17,7 @@ mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:it-roma_met_centre:surface-weather-observations
+    identifier: urn:wmo:md:it-roma_met_centre:surface-weather-observations
     hierarchylevel: dataset
 
 identification:
@@ -37,7 +37,7 @@ identification:
             keywords_type: theme
             vocabulary:
                 name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1.
-                url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline/index.csv
+                url: https://codes.wmo.int/wis/topic-hierarchy/earth-system-discipline
     extents:
         spatial:
             - bbox: [6.74995, 36.61998, 18.48024, 47.11539]

tests/data/metadata/discovery/mw-surface-weather-observations.yml

@@ -18,7 +18,7 @@ mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:mw-mw_met_centre:surface-weather-observations
+    identifier: urn:wmo:md:mw-mw_met_centre:surface-weather-observations
     hierarchylevel: dataset
 
 identification:
@@ -38,7 +38,7 @@ identification:
             keywords_type: theme
             vocabulary:
                 name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1.
-                url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline/index.csv
+                url: https://codes.wmo.int/wis/topic-hierarchy/earth-system-discipline
     extents:
         spatial:
             - bbox: [32.6881653175,-16.8012997372,35.7719047381,-9.23059905359]

tests/data/metadata/discovery/ro-synoptic-weather-observations.yml

@@ -22,7 +22,7 @@ mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:ro-rnimh:synoptic-weather-observations
+    identifier: urn:wmo:md:ro-rnimh:synoptic-weather-observations
     hierarchylevel: dataset
 
 identification:
@@ -42,7 +42,7 @@ identification:
             keywords_type: theme
             vocabulary:
                 name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1.
-                url:  https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline/index.csv
+                url:  https://codes.wmo.int/wis/topic-hierarchy/earth-system-discipline
     extents:
         spatial:
             - bbox: [20.2201924985,43.6884447292,29.62654341,48.2208812526]

tests/integration/test_workflow.py

@@ -33,7 +33,7 @@
 
 URL = 'http://localhost'
 API_URL = f'{URL}/oapi'
-ID = 'urn:x-wmo:md:mw-mw_met_centre:surface-weather-observations'
+ID = 'urn:wmo:md:mw-mw_met_centre:surface-weather-observations'
 SESSION = Session()
 SESSION.hooks = {
    'response': lambda r, *args, **kwargs: r.raise_for_status()

wis2box-management/Dockerfile

@@ -54,7 +54,14 @@ RUN apt-get update -y && apt-get install -y ${DEBIAN_PACKAGES} \
     # cleanup
     && apt autoremove -y  \
     && apt-get -q clean \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/* \
+    # cache WMDR codelists
+    && mkdir -p /home/wis2box/wmdr-codelists \
+    && curl -s "https://codes.wmo.int/wmdr/FacilityType?_format=csv&status=valid" > /home/wis2box/wmdr-codelists/FacilityType.csv \
+    && curl -s "https://codes.wmo.int/wmdr/InstrumentOperatingStatus?_format=csv&status=valid" > /home/wis2box/wmdr-codelists/InstrumentOperatingStatus.csv \
+    && curl -s "https://codes.wmo.int/wmdr/TerritoryName?_format=csv&status=valid" > /home/wis2box/wmdr-codelists/TerritoryName.csv \
+    && curl -s "https://codes.wmo.int/wmdr/WMORegion?_format=csv&status=valid" > /home/wis2box/wmdr-codelists/WMORegion.csv
+
 
 # copy the app
 COPY . /app

wis2box-management/wis2box/metadata/station.py

@@ -26,8 +26,8 @@
 import io
 import json
 import logging
+from pathlib import Path
 from typing import Iterator, Tuple, Union
-from urllib.request import urlretrieve
 
 from elasticsearch import Elasticsearch
 from owslib.ogcapi.features import Features
@@ -51,6 +51,7 @@
 
 STATION_METADATA = DATADIR / 'metadata' / 'station'
 STATIONS = STATION_METADATA / 'station_list.csv'
+WMDR_CODELISTS = Path('/home/wis2box/wmdr-codelists')
 
 
 def get_wmdr_codelists() -> dict:
@@ -71,7 +72,7 @@ def get_wmdr_codelists() -> dict:
     }
 
     for key, value in column2codelists.items():
-        codelist_filename = STATION_METADATA / f'{value}.csv'
+        codelist_filename = WMDR_CODELISTS / f'{value}.csv'
         codelists[key] = []
         with codelist_filename.open() as fh:
             reader = csv.reader(fh)
@@ -432,26 +433,6 @@ def get_geometry(wsi: str = '') -> Union[dict, None]:
     return None
 
 
-def cache_wmdr_code_tables() -> None:
-    """
-    Cache WMDR code tables from codes.wmo.int
-
-    :returns: None
-    """
-
-    code_tables = [
-        'FacilityType',
-        'InstrumentOperatingStatus',
-        'TerritoryName',
-        'WMORegion'
-    ]
-
-    for ct in code_tables:
-        url = f'https://codes.wmo.int/wmdr/{ct}?_format=csv&status=valid'
-        filename = STATION_METADATA / f'{ct}.csv'
-        urlretrieve(url, filename)
-
-
 @click.command()
 @click.pass_context
 @click.argument('wsi')
@@ -510,8 +491,6 @@ def setup(ctx, verbosity):
 
     click.echo('Setting up station metadata repository')
     setup_collection(meta=gcm())
-    click.echo('Caching WMDR code tables')
-    cache_wmdr_code_tables()
 
 
 @click.command()

wis2box-management/wis2box/pubsub/subscribe.py

@@ -95,13 +95,13 @@ def on_message_handler(self, client, userdata, msg):
             LOGGER.debug('Refreshing data mappings')
             self.data_mappings = get_data_mappings()
             return
-        elif topic == 'wis2box/dataset_publication':
+        elif topic == 'wis2box/dataset/publication':
             LOGGER.debug('Publishing dataset')
             metadata = message
             discovery_metadata.publish_discovery_metadata(metadata)
             data_.add_collection_data(metadata)
             return
-        elif topic.startswith('wis2box/dataset_unpublication'):
+        elif topic.startswith('wis2box/dataset/unpublication'):
             LOGGER.debug('Unpublishing dataset')
             identifier = topic.split('/')[-1]
             remove_collection(identifier)