-
Notifications
You must be signed in to change notification settings - Fork 17
138 lines (135 loc) · 7.44 KB
/
tests-docker.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
name: test docker deployment ⚙️
on: [ push, pull_request ]
jobs:
main:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
name: setup Python
with:
python-version: 3.8
- name: install testing requirements 📦
working-directory: tests
run: |
pip3 install -r requirements.txt
pip3 install check-jsonschema
- name: cache schemas 📦
run: |
pywis-pubsub schema sync
curl -s https://raw.githubusercontent.com/wmo-im/wcmp2/main/schemas/wcmp2-bundled.json --output /tmp/wcmp2-bundled.json
- name: display Docker and Python versions 📦
run: |
docker version
docker compose version
python3 -V
- name: setup wis2box configuration
run: |
cp tests/test.env wis2box.env
cat wis2box.env
python3 wis2box-ctl.py config
- name: build wis2box
run: |
python3 wis2box-ctl.py build
python3 wis2box-ctl.py update
- name: start containers ⚙️
run: |
python3 wis2box-ctl.py start
python3 wis2box-ctl.py status -a
docker logs wis2box-management
- name: setup wis2box-management ⚙️
run: |
sleep 30
python3 wis2box-ctl.py execute wis2box environment show
- name: populate stations from CSV 📡
run: |
python3 wis2box-ctl.py execute wis2box metadata station publish-collection
- name: add Malawi synop data (csv2bufr synop_bufr template) 🇲🇼
env:
TOPIC_HIERARCHY: mw-mw_met_centre.data.core.weather.surface-based-observations.synop
DISCOVERY_METADATA: /data/wis2box/metadata/discovery/mw-surface-weather-observations.yml
DISCOVERY_METADATA_ID: urn:x-wmo:md:mw-mw_met_centre:surface-weather-observations
TEST_DATA: /data/wis2box/observations/malawi
TEST_DATA_UPDATE: /data/wis2box/observations/malawi_update
run: |
python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA
python3 wis2box-ctl.py execute wis2box data add-collection $DISCOVERY_METADATA
python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA_UPDATE
- name: add Italy synop data (bufr2bufr) 🇮🇹
env:
TOPIC_HIERARCHY: it-roma_met_centre.data.core.weather.surface-based-observations.synop
DISCOVERY_METADATA: /data/wis2box/metadata/discovery/it-surface-weather-observations.yml
DISCOVERY_METADATA_ID: urn:x-wmo:md:it-roma_met_centre:surface-weather-observations
TEST_DATA: /data/wis2box/observations/italy
run: |
python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA
python3 wis2box-ctl.py execute wis2box data add-collection $DISCOVERY_METADATA
python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
- name: add Algeria synop data (bufr2bufr) 🇩🇿
env:
TOPIC_HIERARCHY: dz-alger_met_centre.data.core.weather.surface-based-observations.synop
DISCOVERY_METADATA: /data/wis2box/metadata/discovery/dz-surface-weather-observations.yml
DISCOVERY_METADATA_ID: urn:x-wmo:md:dz-alger_met_centre:surface-weather-observations
TEST_DATA: /data/wis2box/observations/algeria
run: |
python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA
python3 wis2box-ctl.py execute wis2box data add-collection $DISCOVERY_METADATA
python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
- name: add Romania synop data (synop2bufr and csv2bufr aws-template) 🇷🇴
env:
TOPIC_HIERARCHY: ro-rnimh.data.core.weather.surface-based-observations.synop
DISCOVERY_METADATA: /data/wis2box/metadata/discovery/ro-synoptic-weather-observations.yml
DISCOVERY_METADATA_ID: urn:x-wmo:md:ro-rnimh:synoptic-weather-observations
TEST_DATA: /data/wis2box/observations/romania
run: |
python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA
python3 wis2box-ctl.py execute wis2box data add-collection $DISCOVERY_METADATA
python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
- name: add Congo synop data (synop2bufr) 🇨🇩
env:
TOPIC_HIERARCHY: cd-brazza_met_centre.data.core.weather.surface-based-observations.synop
DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cd-surface-weather-observations.yml
DISCOVERY_METADATA_ID: urn:x-wmo:md:cd-brazza_met_centre:surface-weather-observations
TEST_DATA: /data/wis2box/observations/congo
run: |
python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA
python3 wis2box-ctl.py execute wis2box data add-collection $DISCOVERY_METADATA
python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
- name: add China GRIB data (universal pipeline) 🇨🇳
env:
TOPIC_HIERARCHY: cn-cma-babj.data.core.weather.prediction.forecast.short-range.probabilistic.global
DISCOVERY_METADATA: /data/wis2box/metadata/discovery/GRAPES-GEPS-GLB.yml
DISCOVERY_METADATA_ID: urn:x-wmo:md:cn-cma-babj:data.core.weather.prediction.forecast.short-range.probabilistic.global
TEST_DATA: /data/wis2box/observations/china
run: |
python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA
python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
- name: sync stations 🔄
run: |
sleep 30
python3 wis2box-ctl.py execute wis2box metadata station publish-collection
- name: run integration tests ⚙️
run: |
sleep 30
pytest -s tests/integration
- name: run flake8 ⚙️
run: |
find . -type f -name "*.py" | xargs flake8
- name: failed tests 🚩
if: ${{ failure() }}
run: |
docker logs -n1000 wis2box-management
docker logs -n1000 wis2box-api