diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index dc73c6ab..0b04ca0d 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -184,6 +184,19 @@ jobs: curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA + - name: add org-daycli-test data (csv2bufr) + env: + TOPIC_HIERARCHY: origin/a/wis2/org-daycli-test/data/core/climate/surface-based-observations/daily + DISCOVERY_METADATA: /data/wis2box/metadata/discovery/org-daycli-test.yml + DISCOVERY_METADATA_ID: urn:wmo:md:org-daycli-test:surface-climate-observations:daily + STATION_METADATA: /data/wis2box/metadata/station/daycli-test.csv + TEST_DATA: /data/wis2box/observations/daycli-test + run: | + python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA + python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID + python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - name: add China GRIB2 data (universal pipeline) 🇨🇳 env: DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cn-grapes-geps-global.yml diff --git a/tests/data/metadata/discovery/org-daycli-test.yml b/tests/data/metadata/discovery/org-daycli-test.yml new file mode 100644 index 00000000..9ba6c21d --- /dev/null +++ b/tests/data/metadata/discovery/org-daycli-test.yml @@ -0,0 +1,71 @@ +wis2box: + retention: P30D + topic_hierarchy: org-daycli-test/data/core/climate/surface-based-observations/daily + country: int + centre_id: org-daycli-test + data_mappings: + plugins: + csv: + - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR + template: daycli-template + notify: true + file-pattern: '^.*\.csv$' + bufr4: + - plugin: wis2box.data.bufr2geojson.ObservationDataBUFR2GeoJSON + file-pattern: '^WIGOS_(\d-\d+-\d+-\w+)_.*\.bufr4$' + +mcf: + version: 1.0 + +metadata: + identifier: urn:wmo:md:org-daycli-test:surface-climate-observations:daily + hierarchylevel: dataset + +identification: + title: Daily values from land stations (DAYCLI) + abstract: Daily values from land stations (DAYCLI) + dates: + creation: 2024-03-04 + keywords: + default: + keywords: + - temperature + - snow depth + - precipitation + - climate + - daily + - daycli + wmo: + keywords: + - weather + keywords_type: theme + vocabulary: + name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1. + url: https://codes.wmo.int/wis/topic-hierarchy/earth-system-discipline + extents: + spatial: + - bbox: [-180.0, -90.0, 180.0, 90.0] + crs: 4326 + temporal: + - begin: 2024-02-09 + end: null + resolution: P1H + rights: WMO Unified Policy for the International Exchange of Earth System Data + url: https://example.org/fixme + +contact: + host: &contact_poc + organization: World Meteorological Organisation + url: https://www.wmo.int/ + individualname: Firstname Lastname + positionname: Position Name + phone: null + fax: null + address: null + city: null + administrativearea: null + postalcode: null + country: null + email: you@example.org + hoursofservice: 0800h - 1600h UTC + contactinstructions: email diff --git a/tests/data/metadata/station/daycli-test.csv b/tests/data/metadata/station/daycli-test.csv new file mode 100644 index 00000000..6779c016 --- /dev/null +++ b/tests/data/metadata/station/daycli-test.csv @@ -0,0 +1,2 @@ +station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,barometer_height,territory_name,wmo_region +"DENVER INTERNATIONAL AIRPORT, CO.",0-20000-0-72565,72565,landFixed,39.8666666667,-104.6666666667,1656.0,1656.0,USA,northCentralAmericaCaribbean \ No newline at end of file diff --git a/tests/data/observations/daycli-test/daycli.csv b/tests/data/observations/daycli-test/daycli.csv new file mode 100644 index 00000000..c131f92a --- /dev/null +++ b/tests/data/observations/daycli-test/daycli.csv @@ -0,0 +1,31 @@ +wsi_series,wsi_issuer,wsi_issue_number,wsi_local,wmo_block_number,wmo_station_number,latitude,longitude,station_height_above_msl,temperature_siting_classification,precipitation_siting_classification,averaging_method,year,month,day,precipitation_day_offset,precipitation_hour,precipitation_minute,precipitation_second,precipitation,precipitation_flag,fresh_snow_day_offset,fresh_snow_hour,fresh_snow_minute,fresh_snow_second,fresh_snow_depth,fresh_snow_depth_flag,total_snow_day_offset,total_snow_hour,total_snow_minute,total_snow_second,total_snow_depth,total_snow_depth_flag,thermometer_height,maximum_temperature_day_offset,maximum_temperature_hour,maximum_temperature_minute,maximum_temperature_second,maximum_temperature,maximum_temperature_flag,minimum_temperature_day_offset,minimum_temperature_hour,minimum_temperature_minute,minimum_temperature_second,minimum_temperature,minimum_temperature_flag,average_temperature_day_offset,average_temperature_hour,average_temperature_minute,average_temperature_second,average_temperature,average_temperature_flag +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,1,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,278.15,0,0,7,0,1,273.75,0,0,0,0,1,276.25,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,2,0,7,0,1,1.5,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,279.25,0,0,7,0,1,273.75,0,0,0,0,1,275.75,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,3,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,288.75,0,0,7,0,1,274.25,0,0,0,0,1,280.05,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,4,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,293.75,0,0,7,0,1,274.85,0,0,0,0,1,281.75,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,5,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,296.45,0,0,7,0,1,279.85,0,0,0,0,1,287.25,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,6,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,299.85,0,0,7,0,1,277.55,0,0,0,0,1,286.95,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,7,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,298.15,0,0,7,0,1,280.95,0,0,0,0,1,287.5,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,8,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,283.75,0,0,7,0,1,271.55,0,0,0,0,1,281.05,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,9,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,287.05,0,0,7,0,1,272.15,0,0,0,0,1,278.05,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,10,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,287.55,0,0,7,0,1,273.75,0,0,0,0,1,281.55,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,11,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,286.45,0,0,7,0,1,268.85,0,0,0,0,1,277.85,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,12,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,285.35,0,0,7,0,1,266.55,0,0,0,0,1,276.55,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,13,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,292.55,0,0,7,0,1,274.25,0,0,0,0,1,284.05,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,14,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,293.15,0,0,7,0,1,268.85,0,0,0,0,1,281.25,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,15,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,296.45,0,0,7,0,1,280.95,0,0,0,0,1,290.25,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,16,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,294.85,0,0,7,0,1,274.85,0,0,0,0,1,288.55,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,17,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,275.95,0,0,7,0,1,266.05,0,0,0,0,1,276.25,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,18,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,287.55,0,0,7,0,1,262.15,0,0,0,0,1,271.75,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,19,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,292.05,0,0,7,0,1,275.35,0,0,0,0,1,280.05,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,20,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,283.75,0,0,7,0,1,274.85,0,0,0,0,1,281.45,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,21,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,285.95,0,0,7,0,1,269.35,0,0,0,0,1,277.65,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,22,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,293.15,0,0,7,0,1,272.65,0,0,0,0,1,281.95,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,23,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,294.85,0,0,7,0,1,274.25,0,0,0,0,1,283.95,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,24,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,285.95,0,0,7,0,1,264.95,0,0,0,0,1,281.05,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,25,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,288.15,0,0,7,0,1,264.35,0,0,0,0,1,273.75,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,26,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,293.15,0,0,7,0,1,278.15,0,0,0,0,1,283.15,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,27,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,285.35,0,0,7,0,1,271.05,0,0,0,0,1,281.85,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,28,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,292.55,0,0,7,0,1,271.05,0,0,0,0,1,278.75,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,29,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,295.35,0,0,7,0,1,277.05,0,0,0,0,1,286.15,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,30,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,289.25,0,0,7,0,1,274.85,0,0,0,0,1,281.65,0 diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py index c27ad856..4250ccee 100644 --- a/tests/integration/test_workflow.py +++ b/tests/integration/test_workflow.py @@ -58,7 +58,8 @@ def test_wis2downloader(): 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind-profile': 1, # noqa 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship': 5, # noqa 'origin/a/wis2/it-meteoam/data/core/weather/surface-based-observations/synop': 31, # noqa - 'origin/a/wis2/int-wmo-test/data/core/weather/advisories-warnings': 1 # noqa + 'origin/a/wis2/int-wmo-test/data/core/weather/advisories-warnings': 1, # noqa + 'origin/a/wis2/org-daycli-test/data/core/climate/surface-based-observations/daily': 30 # noqa } topic_nfiles_dict_found = {} @@ -118,15 +119,15 @@ def test_metadata_station_publish(): stations = r.json() - assert stations['numberReturned'] == 103 - assert stations['numberMatched'] == 103 + assert stations['numberReturned'] == 104 + assert stations['numberMatched'] == 104 def test_metadata_discovery_publish(): """Test discovery metadata publishing""" r = SESSION.get(f'{API_URL}/collections/discovery-metadata/items').json() - assert r['numberMatched'] == 10 + assert r['numberMatched'] == 11 r = SESSION.get(f'{API_URL}/collections/discovery-metadata/items/{ID}').json() # noqa @@ -160,7 +161,7 @@ def test_metadata_discovery_publish(): r = SESSION.get(f'{API_URL}/collections/discovery-metadata/items', params=params).json() - assert r['numberMatched'] == 8 + assert r['numberMatched'] == 9 # test access of discovery metadata from notification message @@ -170,7 +171,8 @@ def test_metadata_discovery_publish(): 'dz-meteoalgerie', 'ro-rnimh-test', 'cg-met', - 'int-wmo-test' + 'int-wmo-test', + 'org-daycli-test' ] for centre_id in centre_ids: @@ -304,7 +306,8 @@ def test_message_api(): 'ro-rnimh': 50, 'cg-met': 15, 'int-wmo': 13, - 'cn-cma': 11 + 'cn-cma': 11, + 'org-daycli': 31 } for key, value in counts.items(): url = f'{API_URL}/collections/messages/items?sortby=-datetime&q={key}&limit=1' # noqa