diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index e10959e4..08f9e4e2 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -26,9 +26,11 @@ jobs: docker version docker compose version python3 -V - - name: setup wis2box configuration + - name: setup wis2box configuration, replace localhost with IP on host 📦 run: | + export IP=$(hostname -I | awk '{print $1}') cp tests/test.env wis2box.env + sed -i "s/localhost/$IP/g" wis2box.env cat wis2box.env python3 wis2box-ctl.py config - name: build wis2box @@ -39,11 +41,18 @@ jobs: run: | python3 wis2box-ctl.py start python3 wis2box-ctl.py status -a - docker logs wis2box-management - - name: setup wis2box-management ⚙️ + - name: show environment and check collections exist ⚙️ run: | sleep 30 python3 wis2box-ctl.py execute wis2box environment show + curl http://localhost/oapi/collections/stations + curl http://localhost/oapi/collections/discovery-metadata + curl http://localhost/oapi/collections/messages + - name: downloader subscribe to topics + run: | + python3 wis2box-ctl.py execute wis2box auth add-token -p wis2downloader github123 -y + echo '{"topic": "origin/a/wis2/+/data/core/#"}' > test.json + curl -X POST http://localhost/wis2downloader/subscriptions -H "Content-Type: application/json" -H "Authorization: Bearer github123" -d @test.json - name: populate stations from CSV 📡 run: | python3 wis2box-ctl.py execute wis2box metadata station publish-collection @@ -176,6 +185,7 @@ jobs: - name: sleep 30 seconds then run integration tests ⚙️ run: | sleep 30 + docker logs wis2box-management pytest -s tests/integration - name: run flake8 ⚙️ run: | diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml index c7f0dc64..2a25f4ee 100644 --- a/docker-compose.monitoring.yml +++ b/docker-compose.monitoring.yml @@ -114,7 +114,7 @@ services: <<: *logging web-proxy: <<: *logging - wis2-downloader: + wis2downloader: <<: *logging volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 4afb92b6..dce6636a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -145,9 +145,9 @@ services: depends_on: - wis2box-management - wis2-downloader: - container_name: wis2-downloader - build: ./wis2-downloader + wis2downloader: + container_name: wis2downloader + build: ./wis2downloader env_file: - wis2box.env volumes: diff --git a/docs/source/index.rst b/docs/source/index.rst index c721268c..742c22dd 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -33,6 +33,7 @@ The user guide helps you setup your own wis2box instance. user/getting-started user/setup user/data-ingest + user/gts-headers-in-wis2 user/public-services-setup user/downloading-data diff --git a/docs/source/user/downloading-data.rst b/docs/source/user/downloading-data.rst index e3cadcde..abc52a57 100644 --- a/docs/source/user/downloading-data.rst +++ b/docs/source/user/downloading-data.rst @@ -11,86 +11,93 @@ This section provides guidance how to download data from WIS2 Global Services. WIS2 Global Services include a Global Broker that provides users the ability to subscribe to data (via topics) and download to their local environment / workstation / decision support system from the WIS2 Global Cache. -wis2-downloader ---------------- +wis2downloader +-------------- -wis2box enables subscribe and data download workflow the WIS2 network, by using the ``wis2-downloader`` container, inside of which runs the `wis2-downloader tool + +This will add a subscription to the topic you specify and return the JSON object with the current subscriptions. -```bash -curl http://localhost/wis2-downloader/list -H "Authorization: Bearer " -``` +Deleting a subscription +~~~~~~~~~~~~~~~~~~~~~~~ -The list of the currently active subscriptions should be returned as a JSON object. +To delete a subscription, you can use the following command: -Adding subscriptions -~~~~~~~~~~~~~~~~~~~~ +.. code-block:: console -Subscriptions can be added via a GET request to the `./add` endpoint that is proxied on /wis2-downloader on the wis2box host, with the following form: + wis2downloader delete-subscription --topic -```bash -curl http://localhost/wis2-downloader/add?topic=&target= -H "Authorization: Bearer " -``` +This will delete the subscription to the topic you specify and return the JSON object with the current subscriptions. -- `topic` specifies the topic to subscribe to. *Special characters (+, #) must be URL encoded, i.e. `+` = `%2B`, `#` = `%23`.* -- `target` specifies the directory to save the downloads to, relative to `download_dir` from `config.json`. *If this is not provided, the directory will default to that of the topic hierarchy.* -For example: -```bash -curl http://localhost/wis2-downloader/add?topic=cache/a/wis2/%2B/data/core/weather/%23&target=example_data -H "Authorization: Bearer " -``` +Managing subscriptions from outside the wis2box +----------------------------------------------- -The list of active subscriptions after addition should be returned as a JSON object. +The wis2downloader API-endpoint is proxied on the path `/wis2downloader` on the wis2box host-url, allowing you to interact with it using curl or other HTTP clients from any machine that can reach the wis2box host. -Deleting subscriptions -~~~~~~~~~~~~~~~~~~~~~~ +The wis2box-proxy by default secures the path `/wis2downloader` with a bearer token, which can be generated using the `wis2box auth` command as follows: -Subscriptions are deleted similarly via a GET request to the `./delete` endpoint, with the following form: -```bash -curl http://:/delete?topic= -H "Authorization: Bearer " -``` +.. code-block:: console -For example: -```bash -curl http://localhost:8080/delete?topic=cache/a/wis2/%2B/data/core/weather/%23 -H "Authorization: Bearer " -``` + python3 wis2box.ctl.py execute wis2box auth add-token --path wis2downloader -y -The list of active subscriptions after deletion should be returned as a JSON object. diff --git a/docs/source/user/gts-headers-in-wis2.rst b/docs/source/user/gts-headers-in-wis2.rst new file mode 100644 index 00000000..a5a9f12f --- /dev/null +++ b/docs/source/user/gts-headers-in-wis2.rst @@ -0,0 +1,57 @@ +.. _gts-headers-in-wis2: + +Optional: adding GTS headers to WIS2 notifications during the transition period +=============================================================================== + +Overview +-------- + +This section provides guidance how to add GTS headers to your WIS2 notifications. + +By adding GTS headers to your WIS2 notifications, you can stop your MSS and still have your data available on the GTS during the transition period. + +To enable the WIS2 to GTS Gateway to correctly identify the data to be republished on the GTS, you need to include the GTS property in the WIS2 Notification Message as follows: + +.. code-block:: json + + "properties": { + "gts": { + "ttaaii": "FTAE31", + "cccc": "VTBB" + } + } + +wis2box can add these to WIS2 Notifications automatically, provided you specify the additional file `gts_headers_mappings.json` that contains the required information to map the GTS headers to the incoming filenames. + +Note that this is optional and only required if you want to turn off the existing system responsible for sending data to the GTS during the transition period. + +gts_headers_mapping.csv +----------------------- + +If you want to add GTS headers to your WIS2 notifications, a CSV file is required that maps GTS headers to incoming filenames. + +The CSV file should be named (exactly) `gts_headers_mapping.csv` and should be placed in the directory defined using the `WIS2BOX_HOST_DATADIR` environment variable. + +The CSV should contain the following columns: `string_in_filepath`, `TTAAii`, and `CCCC`. + +Example content for `gts_headers_mapping.csv`: + +.. code-block:: csv + + string_in_filepath,TTAAii,CCCC + ISMD01LIIB,ISMD01,LIBB + ISMD02LIIB,ISMD02,LIBB + +In this example, whenever `ISMD01LIIB` or `ISMD02LIIB` is contained in the filepath of the incoming file, +the corresponding GTS headers will be added to the WIS2 Notification Message as a dictionary in the `properties` field: + +.. code-block:: json + + "properties": { + "gts": { + "ttaaii": "ISMD01", + "cccc": "LIBB" + } + } + +If the `gts_headers_mapping.csv` file is not present in the directory you defined using the `WIS2BOX_HOST_DATADIR` environment variable, wis2box will not add any GTS headers to the WIS2 Notification Message. diff --git a/grafana/dashboards/cadvisor.json b/grafana/dashboards/cadvisor.json deleted file mode 100644 index b5b88121..00000000 --- a/grafana/dashboards/cadvisor.json +++ /dev/null @@ -1,593 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 6, - "interval": "1m", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "code", - "expr": "avg(container_memory_usage_bytes{name!=\"\"}/1024/1024) by (name)", - "interval": "1m", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Memory: used MB", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 2, - "interval": "1m", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "code", - "expr": "sum(rate(container_network_transmit_bytes_total{name!=\"\"}[1m])) by (name)", - "interval": "1m", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Network: Sent rate bytes/s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "code", - "expr": "sum(rate(container_network_receive_errors_total{name!=\"\"} [1m])) by (name)", - "interval": "1m", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Network: Received error rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 9 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "code", - "expr": "sum(rate(container_cpu_usage_seconds_total{name!=\"\"}[$__rate_interval])) by (name)", - "interval": "1m", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "CPU load average", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 9 - }, - "id": 4, - "interval": "1m", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "code", - "expr": "sum(rate(container_network_receive_bytes_total{name!=\"\"}[1m])) by (name)", - "interval": "1m", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Network: Received rate bytes/s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 9 - }, - "id": 11, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "code", - "expr": "sum(rate(container_network_transmit_errors_total{name!=\"\"} [1m])) by (name)", - "interval": "1m", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Network: Transmit error rate", - "type": "timeseries" - } - ], - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "cAdvisor dashboard", - "uid": "81cZrlS4z", - "version": 1, - "weekStart": "" -} \ No newline at end of file diff --git a/grafana/dashboards/home.json b/grafana/dashboards/home.json index 51afba30..2bada33c 100644 --- a/grafana/dashboards/home.json +++ b/grafana/dashboards/home.json @@ -640,7 +640,7 @@ }, "timepicker": {}, "timezone": "", - "title": "wis2box workflow monitoring", + "title": "wis2box data publication dashboard", "uid": "KkBocEA4k", "version": 4, "weekStart": "" diff --git a/grafana/dashboards/storage_dashboard.json b/grafana/dashboards/storage_dashboard.json deleted file mode 100644 index 84920ddf..00000000 --- a/grafana/dashboards/storage_dashboard.json +++ /dev/null @@ -1,362 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 3, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 18, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.0.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "minio_bucket_usage_object_total", - "legendFormat": "{{bucket}}", - "range": true, - "refId": "A" - } - ], - "title": "Total objects stored per bucket", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "fixed" - }, - "decimals": 1, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 18, - "w": 4, - "x": 4, - "y": 0 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.0.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "minio_bucket_usage_total_bytes", - "legendFormat": "{{bucket}}", - "range": true, - "refId": "A" - } - ], - "title": "Total MB stored per bucket", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 11, - "x": 8, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "rate(minio_bucket_traffic_sent_bytes[$__rate_interval])", - "legendFormat": "{{bucket}}", - "range": true, - "refId": "A" - } - ], - "title": "Traffic sent bytes rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 11, - "x": 8, - "y": 10 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "rate(minio_bucket_traffic_received_bytes[$__rate_interval])", - "legendFormat": "{{bucket}}", - "range": true, - "refId": "A" - } - ], - "title": "Traffic received bytes rate", - "type": "timeseries" - } - ], - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Storage statistics", - "uid": "XCoSx0g4z", - "version": 1, - "weekStart": "" - } \ No newline at end of file diff --git a/grafana/dashboards/wis2-downloader.json b/grafana/dashboards/wis2-downloader.json deleted file mode 100644 index a6136bcf..00000000 --- a/grafana/dashboards/wis2-downloader.json +++ /dev/null @@ -1,870 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 2, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 0, - "y": 0 - }, - "id": 14, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Topic" - } - ] - }, - "pluginVersion": "", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "max by(topic) (topic_subscription_status == 1)", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "WIS2-downloader subscriptions", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Value": true - }, - "indexByName": {}, - "renameByName": { - "topic": "Topic" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 25 - }, - { - "color": "red", - "value": 50 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 5, - "y": 0 - }, - "id": 18, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "expr": "queue_size", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Download queue size", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "light-green", - "mode": "fixed" - }, - "mappings": [], - "max": 1000, - "min": 0, - "noValue": "No data", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 100 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 8, - "y": 0 - }, - "id": 11, - "interval": "1h", - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": false - }, - "pluginVersion": "", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum(increase(downloaded_files_total[1h]))", - "format": "time_series", - "instant": true, - "interval": "1h", - "legendFormat": "{{label_name}}", - "range": false, - "refId": "A" - } - ], - "title": "Files downloaded last hour", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "light-blue", - "mode": "fixed" - }, - "decimals": 1, - "mappings": [], - "max": 1000, - "min": 0, - "noValue": "No data", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 100 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 11, - "y": 0 - }, - "id": 12, - "interval": "1h", - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": false - }, - "pluginVersion": "", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum(increase(downloaded_bytes_total[1h]))", - "format": "time_series", - "instant": true, - "interval": "1h", - "legendFormat": "{{label_name}}", - "range": false, - "refId": "A" - } - ], - "title": "Data downloaded last hour", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "light-green", - "mode": "fixed" - }, - "mappings": [], - "max": 9997, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 14, - "y": 0 - }, - "id": 16, - "interval": "1h", - "options": { - "displayMode": "lcd", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "text": { - "titleSize": 17, - "valueSize": 20 - } - }, - "pluginVersion": "", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum(increase(downloaded_files_total{file_type=\"bufr\"}[1h]))", - "format": "time_series", - "instant": true, - "interval": "1h", - "legendFormat": "BUFR", - "range": false, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum(increase(downloaded_files_total{file_type=\"grib\"}[1h]))", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "1h", - "legendFormat": "GRIB", - "range": false, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(downloaded_files_total{file_type!~\"bufr|grib\"}[1h]))", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "1h", - "legendFormat": "Other", - "range": false, - "refId": "C" - } - ], - "title": "Files downloaded by file-type", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "light-blue", - "mode": "fixed" - }, - "decimals": 1, - "mappings": [], - "max": 1000000000, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 19, - "y": 0 - }, - "id": 19, - "interval": "1h", - "options": { - "displayMode": "lcd", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "text": { - "titleSize": 17, - "valueSize": 20 - } - }, - "pluginVersion": "", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum(increase(downloaded_bytes_total{file_type=\"bufr\"}[1h]))", - "format": "time_series", - "instant": true, - "interval": "1h", - "legendFormat": "BUFR", - "range": false, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum(increase(downloaded_bytes_total{file_type=\"grib\"}[1h]))", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "1h", - "legendFormat": "GRIB", - "range": false, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(downloaded_bytes_total{file_type!~\"grib|bufr\"}[1h]))", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "1h", - "legendFormat": "Other", - "range": false, - "refId": "C" - } - ], - "title": "Data downloaded by file-type", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 19, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "light-green", - "value": null - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Data downloaded per hour" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-blue", - "mode": "fixed" - } - } - ] - }, - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Data downloaded per minute" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Data downloaded per minute" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-blue", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 14, - "x": 0, - "y": 4 - }, - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "expr": "sum(increase(downloaded_bytes_total[1m]))", - "hide": false, - "legendFormat": "Data downloaded per minute", - "range": true, - "refId": "A" - } - ], - "title": "Data downloaded per minute", - "type": "timeseries" - }, - { - "datasource": { - "type": "loki", - "uid": "P982945308D3682D1" - }, - "gridPos": { - "h": 18, - "w": 10, - "x": 14, - "y": 4 - }, - "id": 6, - "options": { - "dedupStrategy": "none", - "enableLogDetails": false, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": false, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "P982945308D3682D1" - }, - "editorMode": "builder", - "expr": "{container_name=\"wis2-downloader\"} != `Queue` != `GET` != `CLEAN`", - "queryType": "range", - "refId": "A" - } - ], - "title": "WIS2 downloader logs", - "transparent": true, - "type": "logs" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "download per minute" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Files downloaded per hour" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 14, - "x": 0, - "y": 13 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "expr": "sum(increase(downloaded_files_total[1m]))", - "legendFormat": "Files downloaded per minute", - "range": true, - "refId": "A" - } - ], - "title": "Files downloaded per minute", - "type": "timeseries" - } - ], - "refresh": "5s", - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "wis2-downloader monitoring dashboard", - "uid": "zB4GjVaIk", - "version": 5, - "weekStart": "" - } \ No newline at end of file diff --git a/grafana/dashboards/wis2box_broker.json b/grafana/dashboards/wis2box_broker.json deleted file mode 100644 index 9673553b..00000000 --- a/grafana/dashboards/wis2box_broker.json +++ /dev/null @@ -1,405 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 4, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 11, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "increase(wis2box_broker_msg_sent[1m])", - "legendFormat": "messages sent rate", - "range": true, - "refId": "A" - } - ], - "title": "wis2box-broker: messages sent per minute", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 11, - "x": 11, - "y": 0 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "wis2box_broker_msg_dropped", - "legendFormat": "messages dropped", - "range": true, - "refId": "A" - } - ], - "title": "wis2box-broker: messages dropped", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 11, - "x": 0, - "y": 11 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "increase(wis2box_broker_msg_received[1m])", - "legendFormat": "messages received", - "range": true, - "refId": "A" - } - ], - "title": "wis2box-broker: messages received per minute", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 11, - "x": 11, - "y": 11 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "wis2box_broker_msg_stored", - "legendFormat": "messages stored", - "range": true, - "refId": "A" - } - ], - "title": "wis2box-broker: messages stored", - "type": "timeseries" - } - ], - "refresh": "5s", - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "wis2box broker statistics $SYS/broker/messages", - "uid": "LNKi9EA4z", - "version": 1, - "weekStart": "" - } \ No newline at end of file diff --git a/grafana/dashboards/wis2downloader.json b/grafana/dashboards/wis2downloader.json new file mode 100644 index 00000000..690c2b5b --- /dev/null +++ b/grafana/dashboards/wis2downloader.json @@ -0,0 +1,769 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 0, + "y": 0 + }, + "id": 14, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Topic" + } + ] + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "max by(topic) (topic_subscription_status == 1)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "WIS2-downloader subscriptions", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true + }, + "indexByName": {}, + "renameByName": { + "topic": "Topic" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 25 + }, + { + "color": "red", + "value": 50 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 0 + }, + "id": 18, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "builder", + "expr": "queue_size", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Download queue size", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-green", + "mode": "fixed" + }, + "decimals": 0, + "mappings": [], + "max": 9997, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 16, + "interval": "1h", + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": { + "titleSize": 17, + "valueSize": 20 + } + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(sum(downloaded_files_total{file_type=\"bufr\"} or vector(0))[1h:])", + "format": "time_series", + "instant": true, + "interval": "1h", + "legendFormat": "BUFR", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(sum(downloaded_files_total{file_type=\"grib\"} or vector(0))[1h:])", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "1h", + "legendFormat": "GRIB", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(sum(downloaded_files_total{file_type!~\"bufr|grib\"} or vector(0))[1h:])", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "1h", + "legendFormat": "Other", + "range": false, + "refId": "C" + } + ], + "title": "Files downloaded by file-type, last hour", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-blue", + "mode": "fixed" + }, + "decimals": 1, + "mappings": [], + "max": 1000000000, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 19, + "interval": "1h", + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": { + "titleSize": 17, + "valueSize": 20 + } + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(sum(downloaded_bytes_total{file_type=\"bufr\"} or vector(0))[1h:])", + "format": "time_series", + "instant": true, + "interval": "1h", + "legendFormat": "BUFR", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(sum(downloaded_bytes_total{file_type=\"grib\"} or vector(0))[1h:])", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "1h", + "legendFormat": "GRIB", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(sum(downloaded_bytes_total{file_type!~\"grib|bufr\"} or vector(0))[1h:])", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "1h", + "legendFormat": "Other", + "range": false, + "refId": "C" + } + ], + "title": "Data downloaded by file-type, last hour", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "download per minute" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Files downloaded per hour" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Failded downloads per minute" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Failed downloads per minute" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 15, + "x": 9, + "y": 4 + }, + "id": 2, + "interval": "60s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "code", + "expr": "increase(sum(downloaded_files_total or vector(0))[1m:])", + "legendFormat": "Successful downloads per minute", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "code", + "expr": "increase(sum(failed_downloads_total or vector(0))[1m:])", + "hide": false, + "legendFormat": "Failed downloads per minute", + "range": true, + "refId": "B" + } + ], + "title": "Successful / Failed downloads per minute", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "P55348B596EBB51C3" + }, + "gridPos": { + "h": 16, + "w": 9, + "x": 0, + "y": 6 + }, + "id": 6, + "options": { + "dedupStrategy": "none", + "enableLogDetails": false, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P55348B596EBB51C3" + }, + "editorMode": "builder", + "expr": "{container_name=\"wis2downloader\"} != `Queue` != `GET` != `CLEAN` |~ `ERROR|WARNING`", + "queryType": "range", + "refId": "A" + } + ], + "title": "WIS2 downloader logs: ERROR and WARNING", + "transparent": true, + "type": "logs" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 19, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Data downloaded per hour" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Data downloaded per minute" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Data downloaded per minute" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 15, + "x": 9, + "y": 13 + }, + "id": 9, + "interval": "60s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "builder", + "expr": "increase(sum(downloaded_bytes_total or vector(0))[1m:])", + "hide": false, + "legendFormat": "Data downloaded per minute", + "range": true, + "refId": "A" + } + ], + "title": "Data downloaded per minute", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "wis2downloader dashboard", + "uid": "zB4GjVaIk", + "version": 5, + "weekStart": "" +} \ No newline at end of file diff --git a/nginx/nginx-ssl.conf b/nginx/nginx-ssl.conf index 64f9a044..81a38d16 100644 --- a/nginx/nginx-ssl.conf +++ b/nginx/nginx-ssl.conf @@ -59,12 +59,12 @@ rewrite ^/data(/.*)$ /wis2box-public$1 break; proxy_pass http://minio:9000; } - location /wis2-downloader { + location /wis2downloader { set $x_api_http_method $request_method; auth_request /auth; auth_request_set $auth_status $upstream_status; - rewrite ^/wis2-downloader/(.*) /$1 break; - proxy_pass http://wis2-downloader:5000; + rewrite ^/wis2downloader/(.*) /$1 break; + proxy_pass http://wis2downloader:5000; } location /oapi { set $x_api_http_method $request_method; diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 362e5887..fd19673f 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -62,12 +62,12 @@ auth_request_set $auth_status $upstream_status; proxy_pass http://wis2box-api:80; } - location /wis2-downloader { + location /wis2downloader { set $x_api_http_method $request_method; auth_request /auth; auth_request_set $auth_status $upstream_status; - rewrite ^/wis2-downloader/(.*) /$1 break; - proxy_pass http://wis2-downloader:5000; + rewrite ^/wis2downloader/(.*) /$1 break; + proxy_pass http://wis2downloader:5000; } location /wis2box-webapp/ { auth_basic "Restricted Access"; diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 16d7d5c9..83bcc38f 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -26,8 +26,8 @@ scrape_configs: - targets: ['cadvisor:8080'] labels: alias: 'cadvisor' -- job_name: wis2-downloader +- job_name: wis2downloader metrics_path: /metrics scheme: http static_configs: - - targets: ['wis2-downloader:5000'] + - targets: ['wis2downloader:5000'] diff --git a/tests/data/gts_headers_mapping.csv b/tests/data/gts_headers_mapping.csv new file mode 100644 index 00000000..e2a24005 --- /dev/null +++ b/tests/data/gts_headers_mapping.csv @@ -0,0 +1,8 @@ +string_in_filepath,ttaaii,cccc +ISIN21DAMM,ISIN21,DAMM +ISMD01LIIB,ISMD01,LIBB +ISMD02LIIB,ISMD02,LIBB +SMRO01YRBK,SMRO01,YRBK +IOBH01LFPW,IOBH01,LFPW +ISSA05LFPW,ISSA05,LFPW +SICG20FCBB,SICG20,FCBB \ No newline at end of file diff --git a/tests/data/observations/congo/synop_202308.txt b/tests/data/observations/congo/SICG20FCBB_202308.txt similarity index 100% rename from tests/data/observations/congo/synop_202308.txt rename to tests/data/observations/congo/SICG20FCBB_202308.txt diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py index 9661b538..5d7d02df 100644 --- a/tests/integration/test_workflow.py +++ b/tests/integration/test_workflow.py @@ -24,6 +24,8 @@ # .github/workflows/tests-docker.yml has been executed import csv +import os + from pathlib import Path from pywis_pubsub.validation import validate_message @@ -40,6 +42,43 @@ } +def test_wis2downloader(): + """Test if the wis2downloader has downloaded + the expected number of files in the download directory""" + + DOWNLOAD_DIR = DATADIR / 'downloads' + + topic_nfiles_dict = { + 'origin/a/wis2/mw-mw_met_centre/data/core/weather/surface-based-observations/synop': 23, # noqa + 'origin/a/wis2/dz-alger_met_centre/data/core/weather/surface-based-observations/synop': 28, # noqa + 'origin/a/wis2/cn-cma/data/core/weather/prediction/forecast/medium-range/probabilistic/global': 10, # noqa + 'origin/a/wis2/ro-rnimh/data/core/weather/surface-based-observations/synop': 49, # noqa + 'origin/a/wis2/cd-brazza_met_centre/data/core/weather/surface-based-observations/synop': 14, # noqa + 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/buoy': 2, # noqa + 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind_profiler': 1, # noqa + 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship': 5, # noqa + 'origin/a/wis2/it-roma_met_centre/data/core/weather/surface-based-observations/synop': 31 # noqa + } + + topic_nfiles_dict_found = {} + for key in topic_nfiles_dict.keys(): + topic_nfiles_dict_found[key] = 0 + + # count the number of files received in the download directory + # over all subdirectories + total_files = 0 + for root, dirs, files in os.walk(DOWNLOAD_DIR): + total_files += len(files) + for key in topic_nfiles_dict.keys(): + if key in root: + topic_nfiles_dict_found[key] += len(files) + + # check if the number of files downloaded for each topic + # matches the expected number + for key in topic_nfiles_dict.keys(): + assert topic_nfiles_dict[key] == topic_nfiles_dict_found[key] + + def test_metadata_station_cache(): """Test station metadata caching""" @@ -292,6 +331,9 @@ def test_message_api(): assert props['content']['size'] == 253 assert props['content']['encoding'] == 'base64' assert props['content']['value'] is not None + assert 'gts' in props + assert props['gts']['ttaaii'] == 'SICG20' + assert props['gts']['cccc'] == 'FCBB' link_rel = msg['links'][0] diff --git a/tests/test.env b/tests/test.env index 65b65037..82ae0dff 100644 --- a/tests/test.env +++ b/tests/test.env @@ -60,4 +60,12 @@ MINIO_NOTIFY_MQTT_QOS_WIS2BOX=1 # wis2box webapp credentials WIS2BOX_WEBAPP_USERNAME=wis2box-user -WIS2BOX_WEBAPP_PASSWORD=wis2boxtest123 \ No newline at end of file +WIS2BOX_WEBAPP_PASSWORD=wis2boxtest123 + +# downloader settings, subscribe to local broker for test +DOWNLOAD_BROKER_HOST=mosquitto +DOWNLOAD_BROKER_PORT=1883 +DOWNLOAD_BROKER_USERNAME=everyone +DOWNLOAD_BROKER_PASSWORD=everyone +DOWNLOAD_BROKER_TRANSPORT=tcp +DOWNLOAD_MIN_FREE_SPACE_GB=0 \ No newline at end of file diff --git a/wis2-downloader/config.template b/wis2-downloader/config.template deleted file mode 100644 index c5de79ac..00000000 --- a/wis2-downloader/config.template +++ /dev/null @@ -1,15 +0,0 @@ -{ - "broker_url": "${DOWNLOAD_BROKER_HOST}", - "broker_port": $DOWNLOAD_BROKER_PORT, - "username": "${DOWNLOAD_BROKER_USERNAME}", - "password": "${DOWNLOAD_BROKER_PASSWORD}", - "protocol": "${DOWNLOAD_BROKER_PROTOCOL}", - "topics": {}, - "download_dir": "${DOWNLOAD_DIR}", - "retention_period_hours": "${DOWNLOAD_RETENTION_PERIOD_HOURS}", - "flask_host": "${FLASK_HOST}", - "flask_port": "${FLASK_PORT}", - "download_workers": $DOWNLOAD_WORKERS, - "save_logs": false, - "log_dir": "logs" -} diff --git a/wis2box-create-config.py b/wis2box-create-config.py index 32be22df..11a5ff6d 100644 --- a/wis2box-create-config.py +++ b/wis2box-create-config.py @@ -358,6 +358,17 @@ def create_wis2box_env(config_dir: str) -> None: fh.write('MINIO_NOTIFY_MQTT_TOPIC_WIS2BOX=wis2box/storage\n') fh.write('MINIO_NOTIFY_MQTT_QOS_WIS2BOX=1\n') fh.write('\n') + # downloader settings + fh.write('# downloader settings\n') + fh.write('DOWNLOAD_BROKER_HOST=globalbroker.meteo.fr\n') + fh.write('DOWNLOAD_BROKER_PORT=443\n') + fh.write('DOWNLOAD_BROKER_USERNAME=everyone\n') + fh.write('DOWNLOAD_BROKER_PASSWORD=everyone\n') + fh.write('# download transport mechanism (tcp or websockets)\n') + fh.write('DOWNLOAD_BROKER_TRANSPORT=websockets\n') + fh.write('# maximum MB in download directory\n') + fh.write('DOWNLOAD_MIN_FREE_SPACE_GB=1\n') + fh.write('\n') print('*' * 80) print('The file wis2box.env has been created in the current directory.') diff --git a/wis2box-management/docker/entrypoint.sh b/wis2box-management/docker/entrypoint.sh index 5d065f8b..8baf4c22 100755 --- a/wis2box-management/docker/entrypoint.sh +++ b/wis2box-management/docker/entrypoint.sh @@ -91,14 +91,14 @@ else # Add the token wis2box auth add-token --path collections/stations -y fi -# repeat for wis2-downloader -is_restricted=$(wis2box auth is-restricted-path --path wis2-downloader) +# repeat for wis2downloader +is_restricted=$(wis2box auth is-restricted-path --path wis2downloader) if [ "$is_restricted" = "True" ]; then - echo "wis2-downloader is restricted" + echo "wis2downloader is restricted" else - echo "restricting wis2-downloader" + echo "restricting wis2downloader" # Add the token - wis2box auth add-token --path wis2-downloader -y + wis2box auth add-token --path wis2downloader -y fi echo "END /entrypoint.sh" diff --git a/wis2box-management/wis2box/data/base.py b/wis2box-management/wis2box/data/base.py index 340cf56b..e5ed363d 100644 --- a/wis2box-management/wis2box/data/base.py +++ b/wis2box-management/wis2box/data/base.py @@ -60,6 +60,15 @@ def __init__(self, defs: dict) -> None: self.buckets = defs.get('buckets', ()) self.output_data = {} self.discovery_metadata = {} + self.gts = None + gts_ttaaii = defs.get('gts_ttaaii') + gts_cccc = defs.get('gts_cccc') + if None not in [gts_ttaaii, gts_cccc]: + self.gts = { + 'ttaaii': gts_ttaaii, + 'cccc': gts_cccc + } + # if discovery_metadata: # self.setup_discovery_metadata(discovery_metadata) @@ -152,7 +161,8 @@ def notify(self, identifier: str, storage_path: str, wis_message = WISNotificationMessage( f"{metadata_id.replace('urn:wmo:md:','')}/{identifier}", metadata_id, storage_path, datetime_, geometry, - wigos_station_identifier, operation) + wigos_station_identifier, self.gts, + operation) # load plugin for public broker defs = { diff --git a/wis2box-management/wis2box/data_mappings.py b/wis2box-management/wis2box/data_mappings.py index 37300d93..757f9d54 100644 --- a/wis2box-management/wis2box/data_mappings.py +++ b/wis2box-management/wis2box/data_mappings.py @@ -82,6 +82,7 @@ def get_data_mappings() -> dict: def validate_and_load(path: str, data_mappings: dict = None, + gts_mappings: dict = None, file_type: str = None ) -> Tuple[str, Tuple[Any]]: """ @@ -135,7 +136,8 @@ def validate_and_load(path: str, LOGGER.debug(f'Adding plugin definition for {file_type}') def data_defs(plugin): - return { + notify = plugin.get('notify', False) + data_defs = { 'metadata_id': metadata_id, 'íncoming_filepath': path, 'topic_hierarchy': topic_hierarchy, @@ -143,9 +145,17 @@ def data_defs(plugin): 'pattern': plugin['file-pattern'], 'template': plugin.get('template'), 'buckets': plugin.get('buckets', ()), - 'notify': plugin.get('notify', False), + 'notify': notify, 'format': file_type } + if notify and gts_mappings: + for key in gts_mappings.keys(): + # check if string defined by key is contained in path + if key in path: + data_defs['gts_ttaaii'] = gts_mappings[key]['ttaaii'] + data_defs['gts_cccc'] = gts_mappings[key]['cccc'] + break + return data_defs plugins_ = [load_plugin('data', data_defs(p), data_mappings) for p in plugins[file_type]] diff --git a/wis2box-management/wis2box/handler.py b/wis2box-management/wis2box/handler.py index 1230b3af..b0edbe21 100644 --- a/wis2box-management/wis2box/handler.py +++ b/wis2box-management/wis2box/handler.py @@ -37,8 +37,8 @@ class Handler: def __init__(self, filepath: str, - metadata_id: str = None, - data_mappings: dict = None) -> None: + data_mappings: dict = None, + gts_mappings: dict = None) -> None: self.filepath = filepath self.plugins = () self.input_bytes = None @@ -61,7 +61,7 @@ def __init__(self, filepath: str, raise NotHandledError(msg) try: self.metadata_id, self.plugins = validate_and_load( - self.filepath, data_mappings, self.filetype) + self.filepath, data_mappings, gts_mappings, self.filetype) except Exception as err: msg = f'Path validation error: {err}' # errors in public storage are not handled diff --git a/wis2box-management/wis2box/pubsub/message.py b/wis2box-management/wis2box/pubsub/message.py index d036a630..edba090c 100644 --- a/wis2box-management/wis2box/pubsub/message.py +++ b/wis2box-management/wis2box/pubsub/message.py @@ -126,7 +126,8 @@ def _generate_checksum(self, bytes, algorithm: SecureHashAlgorithms) -> str: # class WISNotificationMessage(PubSubMessage): def __init__(self, identifier: str, metadata_id: str, filepath: str, - datetime_: str, geometry=None, wigos_station_identifier=None, + datetime_: str, geometry=None, + wigos_station_identifier=None, gts: dict = None, operation: str = 'create') -> None: super().__init__('wis2-notification-message', identifier, @@ -186,6 +187,10 @@ def __init__(self, identifier: str, metadata_id: str, filepath: str, if metadata_id is not None: self.message['properties']['metadata_id'] = metadata_id + # if gts-dictionary is provided, include it in the message properties + if gts is not None: + self.message['properties']['gts'] = gts + if self.length < 4096: LOGGER.debug('Including data inline via properties.content') content_value = base64.b64encode(self.filebytes) diff --git a/wis2box-management/wis2box/pubsub/subscribe.py b/wis2box-management/wis2box/pubsub/subscribe.py index 8595f5e1..8b3b5472 100644 --- a/wis2box-management/wis2box/pubsub/subscribe.py +++ b/wis2box-management/wis2box/pubsub/subscribe.py @@ -36,7 +36,8 @@ from wis2box.data_mappings import get_data_mappings from wis2box.data.message import MessageData -from wis2box.env import (DOCKER_BROKER, STORAGE_SOURCE, STORAGE_ARCHIVE) +from wis2box.env import (DATADIR, DOCKER_BROKER, + STORAGE_SOURCE, STORAGE_ARCHIVE) from wis2box.handler import Handler, NotHandledError import wis2box.metadata.discovery as discovery_metadata from wis2box.plugin import load_plugin, PLUGINS @@ -46,10 +47,33 @@ LOGGER = logging.getLogger(__name__) +def get_gts_mappings(): + # read gts mappings from CSV file in DATADIR + gts_mappings = {} + mapping_file = 'gts_headers_mapping.csv' + try: + with open(f'{DATADIR}/{mapping_file}', 'r') as f: + for line in f: + key = line.strip().split(',')[0] + if key == 'string_in_filepath': + continue + ttaaii = line.strip().split(',')[1] + cccc = line.strip().split(',')[2] + value = {'ttaaii': ttaaii, 'cccc': cccc} + gts_mappings[key] = value + LOGGER.info(f'GTS mapping: string_in_filepath={key}, {value}') + except FileNotFoundError: + LOGGER.warning(f'To add GTS headers, please create {mapping_file} in WIS2BOX_HOST_DATADIR') # noqa + except Exception as err: + LOGGER.error(f'Error reading GTS mappings: {err}') + return gts_mappings + + class WIS2BoxSubscriber: def __init__(self, broker): self.data_mappings = get_data_mappings() + self.gts_mappings = get_gts_mappings() self.broker = broker self.broker.bind('on_message', self.on_message_handler) self.broker.sub('wis2box/#') @@ -59,7 +83,8 @@ def handle(self, filepath): LOGGER.info(f'Processing {filepath}') # load handler handler = Handler(filepath=filepath, - data_mappings=self.data_mappings) + data_mappings=self.data_mappings, + gts_mappings=self.gts_mappings) if handler.handle(): LOGGER.debug('Data processed') for plugin in handler.plugins: @@ -76,10 +101,21 @@ def handle(self, filepath): def handle_publish(self, message): LOGGER.debug('Loading MessageData plugin to publish data from message') # noqa + topic_hierarchy = message['channel'] + metadata_id = message.get('metadata_id') + # if metadata_id not provided determine from topic_hierarchy + if metadata_id is None: + for key, value in self.data_mappings.items(): + if topic_hierarchy in (value['topic_hierarchy']): + topic_hierarchy = (value['topic_hierarchy']) + metadata_id = key + break + defs = { - 'topic_hierarchy': message['channel'].replace('origin/a/wis2/', ''), # noqa + 'topic_hierarchy': topic_hierarchy, '_meta': message['_meta'], - 'notify': True + 'notify': True, + 'metadata_id': metadata_id } MessageData(defs=defs) plugin = MessageData(defs=defs) diff --git a/wis2box.env.example b/wis2box.env.example index 7194e3cd..cbfdb6f9 100644 --- a/wis2box.env.example +++ b/wis2box.env.example @@ -59,3 +59,13 @@ MINIO_NOTIFY_MQTT_BROKER_WIS2BOX=tcp://${WIS2BOX_BROKER_HOST}:${WIS2BOX_BROKER_P MINIO_NOTIFY_MQTT_TOPIC_WIS2BOX=wis2box/storage MINIO_NOTIFY_MQTT_QOS_WIS2BOX=1 +# downloader settings +DOWNLOAD_BROKER_HOST=globalbroker.meteo.fr +DOWNLOAD_BROKER_PORT=443 +DOWNLOAD_BROKER_USERNAME=everyone +DOWNLOAD_BROKER_PASSWORD=everyone +# download transport mechanism (tcp or websockets) +DOWNLOAD_BROKER_TRANSPORT=websockets +# maximum MB in download directory +DOWNLOAD_MIN_FREE_SPACE_GB=1 + diff --git a/wis2-downloader/Dockerfile b/wis2downloader/Dockerfile similarity index 63% rename from wis2-downloader/Dockerfile rename to wis2downloader/Dockerfile index a5ee6fd1..c52784ec 100644 --- a/wis2-downloader/Dockerfile +++ b/wis2downloader/Dockerfile @@ -10,17 +10,17 @@ ENV DOWNLOAD_BROKER_HOST "globalbroker.meteo.fr" ENV DOWNLOAD_BROKER_PORT 443 ENV DOWNLOAD_BROKER_USERNAME "everyone" ENV DOWNLOAD_BROKER_PASSWORD "everyone" -ENV DOWNLOAD_BROKER_PROTOCOL "websockets" +ENV DOWNLOAD_BROKER_TRANSPORT "websockets" ENV DOWNLOAD_WORKERS 8 -ENV FLASK_HOST "0.0.0.0" -ENV FLASK_PORT 5000 -ENV SAVE_LOGS false -ENV LOGS_DIR /app/logs +ENV DOWNLOAD_MIN_FREE_SPACE_GB 1 +ENV DOWNLOAD_VALIDATE_TOPICS "false" +ENV LOG_PATH /app/logs +ENV WIS2DOWNLOADER_CONFIG "/app/config.json" # update pyopenssl and pin requests and urllib3 to avoid SSL error RUN pip install pyopenssl --upgrade && pip install requests==2.26.0 urllib3==1.26.0 -# install cron and envsubst -RUN apt-get update && apt-get install -y cron gettext-base +# install cron and envsubst and guincorn +RUN apt-get update && apt-get install -y cron gettext-base gunicorn # copy all the code to the Docker image COPY . /app @@ -28,8 +28,8 @@ COPY . /app # Set the working directory to /app WORKDIR /app -# install wis2-downloader -RUN pip install https://github.com/wmo-im/wis2-downloader/archive/main.zip +# install wis2downloader from pypi +RUN pip install wis2downloader # add wis2box.cron to crontab COPY ./clean.cron /etc/cron.d/clean.cron @@ -42,5 +42,5 @@ RUN chmod +x /app/entrypoint.sh ENTRYPOINT [ "/app/entrypoint.sh" ] -# Run wis2-downloader when the container launches -CMD ["wis2downloader","--config","/app/config.json"] +# Run wis2downloader when the container launches +CMD ["gunicorn","--bind","0.0.0.0:5000","--pythonpath","/usr/local/lib/python3.9/site-packages/","--workers", "1", "wis2downloader.app:app"] diff --git a/wis2-downloader/clean.cron b/wis2downloader/clean.cron similarity index 100% rename from wis2-downloader/clean.cron rename to wis2downloader/clean.cron diff --git a/wis2-downloader/clean.py b/wis2downloader/clean.py similarity index 100% rename from wis2-downloader/clean.py rename to wis2downloader/clean.py diff --git a/wis2downloader/config.template b/wis2downloader/config.template new file mode 100644 index 00000000..7f11111e --- /dev/null +++ b/wis2downloader/config.template @@ -0,0 +1,18 @@ +{ + "broker_hostname": "${DOWNLOAD_BROKER_HOST}", + "broker_port": ${DOWNLOAD_BROKER_PORT}, + "broker_username": "${DOWNLOAD_BROKER_USERNAME}", + "broker_password": "${DOWNLOAD_BROKER_PASSWORD}", + "broker_protocol": "${DOWNLOAD_BROKER_TRANSPORT}", + "download_workers": ${DOWNLOAD_WORKERS}, + "min_free_space": ${DOWNLOAD_MIN_FREE_SPACE_GB}, + "validate_topics": ${DOWNLOAD_VALIDATE_TOPICS}, + "mqtt_session_info": "${DOWNLOAD_DIR}/.session-info.json", + "download_dir": "${DOWNLOAD_DIR}", + "log_level": "INFO", + "base_url": "http://localhost:5000", + "flask_host": "0.0.0.0", + "flask_port": 5000, + "save_logs": false, + "log_path": "logs" +} diff --git a/wis2-downloader/entrypoint.sh b/wis2downloader/entrypoint.sh similarity index 50% rename from wis2-downloader/entrypoint.sh rename to wis2downloader/entrypoint.sh index ff9a19d1..61386e13 100644 --- a/wis2-downloader/entrypoint.sh +++ b/wis2downloader/entrypoint.sh @@ -14,8 +14,6 @@ echo "END /entrypoint.sh" echo "Download directory in container: $DOWNLOAD_DIR" # print the retention period hours echo "Retention period in hours: $RETENTION_PERIOD_HOURS" -# print the maximum MB allowed in the download directory -echo "Maximum MB allowed in the download directory: $MAX_MB_DOWNLOAD_DIR" # ensure DOWNLOAD_DIR exists if [ ! -d $DOWNLOAD_DIR ]; then @@ -24,6 +22,17 @@ if [ ! -d $DOWNLOAD_DIR ]; then fi envsubst < config.template > config.json +# if session-info.json does not exists in $DOWNLOAD_DIR, create it +if [ ! -f $DOWNLOAD_DIR/.session-info.json ]; then + echo "Creating .session-info.json" + echo "{" > $DOWNLOAD_DIR/.session-info.json + echo ' "topics": {},' >> $DOWNLOAD_DIR/.session-info.json + # generate a random string for client_id + echo "Generating random client_id" + echo ' "client_id": "wis2box-wis2downloader-'$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)'"' >> $DOWNLOAD_DIR/.session-info.json + echo "}" >> $DOWNLOAD_DIR/.session-info.json +fi + # print the config echo "Config:" cat /app/config.json