diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index e10959e4..67048e11 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -26,9 +26,11 @@ jobs: docker version docker compose version python3 -V - - name: setup wis2box configuration + - name: setup wis2box configuration, replace localhost with IP on host 📦 run: | + export IP=$(hostname -I | awk '{print $1}') cp tests/test.env wis2box.env + sed -i "s/localhost/$IP/g" wis2box.env cat wis2box.env python3 wis2box-ctl.py config - name: build wis2box @@ -39,11 +41,18 @@ jobs: run: | python3 wis2box-ctl.py start python3 wis2box-ctl.py status -a - docker logs wis2box-management - - name: setup wis2box-management ⚙️ + - name: show environment and check collections exist ⚙️ run: | sleep 30 python3 wis2box-ctl.py execute wis2box environment show + curl http://localhost/oapi/collections/stations + curl http://localhost/oapi/collections/discovery-metadata + curl http://localhost/oapi/collections/messages + - name: downloader subscribe to topics + env: + SUBSCRIPTION_TOPIC: origin/a/wis2/+/data/core/# + run: | + python3 wis2box-ctl.py execute wis2box downloader add-subscription --topic $SUBSCRIPTION_TOPIC - name: populate stations from CSV 📡 run: | python3 wis2box-ctl.py execute wis2box metadata station publish-collection @@ -176,6 +185,7 @@ jobs: - name: sleep 30 seconds then run integration tests ⚙️ run: | sleep 30 + docker logs wis2box-management pytest -s tests/integration - name: run flake8 ⚙️ run: | diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml index c7f0dc64..2a25f4ee 100644 --- a/docker-compose.monitoring.yml +++ b/docker-compose.monitoring.yml @@ -114,7 +114,7 @@ services: <<: *logging web-proxy: <<: *logging - wis2-downloader: + wis2downloader: <<: *logging volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 4afb92b6..dce6636a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -145,9 +145,9 @@ services: depends_on: - wis2box-management - wis2-downloader: - container_name: wis2-downloader - build: ./wis2-downloader + wis2downloader: + container_name: wis2downloader + build: ./wis2downloader env_file: - wis2box.env volumes: diff --git a/docs/source/index.rst b/docs/source/index.rst index c721268c..742c22dd 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -33,6 +33,7 @@ The user guide helps you setup your own wis2box instance. user/getting-started user/setup user/data-ingest + user/gts-headers-in-wis2 user/public-services-setup user/downloading-data diff --git a/docs/source/user/downloading-data.rst b/docs/source/user/downloading-data.rst index e3cadcde..dff74efb 100644 --- a/docs/source/user/downloading-data.rst +++ b/docs/source/user/downloading-data.rst @@ -11,86 +11,82 @@ This section provides guidance how to download data from WIS2 Global Services. WIS2 Global Services include a Global Broker that provides users the ability to subscribe to data (via topics) and download to their local environment / workstation / decision support system from the WIS2 Global Cache. -wis2-downloader +wis2downloader --------------- -wis2box enables subscribe and data download workflow the WIS2 network, by using the ``wis2-downloader`` container, inside of which runs the `wis2-downloader tool " +``` +python3 wis2box.ctl.py execute wis2box downloader list-subscriptions ``` -The list of the currently active subscriptions should be returned as a JSON object. +This will return a JSON object with the current subscriptions. -Adding subscriptions -~~~~~~~~~~~~~~~~~~~~ +Adding a subscription +~~~~~~~~~~~~~~~~~~~~~ -Subscriptions can be added via a GET request to the `./add` endpoint that is proxied on /wis2-downloader on the wis2box host, with the following form: +To add a subscription, you can use the following command: -```bash -curl http://localhost/wis2-downloader/add?topic=&target= -H "Authorization: Bearer " ``` +python3 wis2box.ctl.py execute wis2box downloader add-subscription --topic +``` + +This will add a subscription to the topic you specify and return the JSON object with the current subscriptions. -- `topic` specifies the topic to subscribe to. *Special characters (+, #) must be URL encoded, i.e. `+` = `%2B`, `#` = `%23`.* -- `target` specifies the directory to save the downloads to, relative to `download_dir` from `config.json`. *If this is not provided, the directory will default to that of the topic hierarchy.* +Deleting a subscription +~~~~~~~~~~~~~~~~~~~~~~~ -For example: -```bash -curl http://localhost/wis2-downloader/add?topic=cache/a/wis2/%2B/data/core/weather/%23&target=example_data -H "Authorization: Bearer " +To delete a subscription, you can use the following command: + +``` +python3 wis2box.ctl.py execute wis2box downloader delete-subscription --topic ``` -The list of active subscriptions after addition should be returned as a JSON object. +This will delete the subscription to the topic you specify and return the JSON object with the current subscriptions. -Deleting subscriptions -~~~~~~~~~~~~~~~~~~~~~~ -Subscriptions are deleted similarly via a GET request to the `./delete` endpoint, with the following form: -```bash -curl http://:/delete?topic= -H "Authorization: Bearer " -``` +Managing subscriptions from outside the wis2box +---------------------------------------------- -For example: -```bash -curl http://localhost:8080/delete?topic=cache/a/wis2/%2B/data/core/weather/%23 -H "Authorization: Bearer " -``` +The wis2downloader API-endpoint is proxied on the path `/wis2downloader` on the wis2box host-url, allowing you to interact with it using curl or other HTTP clients from any machine that can reach the wis2box host. -The list of active subscriptions after deletion should be returned as a JSON object. +The wis2box-proxy by default secures the path `/wis2downloader` with a bearer token, which can be generated using the `wis2box auth` command as follows: +``` +python3 wis2box.ctl.py execute wis2box auth add-token --path wis2downloader -y +``` diff --git a/docs/source/user/gts-headers-in-wis2.rst b/docs/source/user/gts-headers-in-wis2.rst new file mode 100644 index 00000000..a5a9f12f --- /dev/null +++ b/docs/source/user/gts-headers-in-wis2.rst @@ -0,0 +1,57 @@ +.. _gts-headers-in-wis2: + +Optional: adding GTS headers to WIS2 notifications during the transition period +=============================================================================== + +Overview +-------- + +This section provides guidance how to add GTS headers to your WIS2 notifications. + +By adding GTS headers to your WIS2 notifications, you can stop your MSS and still have your data available on the GTS during the transition period. + +To enable the WIS2 to GTS Gateway to correctly identify the data to be republished on the GTS, you need to include the GTS property in the WIS2 Notification Message as follows: + +.. code-block:: json + + "properties": { + "gts": { + "ttaaii": "FTAE31", + "cccc": "VTBB" + } + } + +wis2box can add these to WIS2 Notifications automatically, provided you specify the additional file `gts_headers_mappings.json` that contains the required information to map the GTS headers to the incoming filenames. + +Note that this is optional and only required if you want to turn off the existing system responsible for sending data to the GTS during the transition period. + +gts_headers_mapping.csv +----------------------- + +If you want to add GTS headers to your WIS2 notifications, a CSV file is required that maps GTS headers to incoming filenames. + +The CSV file should be named (exactly) `gts_headers_mapping.csv` and should be placed in the directory defined using the `WIS2BOX_HOST_DATADIR` environment variable. + +The CSV should contain the following columns: `string_in_filepath`, `TTAAii`, and `CCCC`. + +Example content for `gts_headers_mapping.csv`: + +.. code-block:: csv + + string_in_filepath,TTAAii,CCCC + ISMD01LIIB,ISMD01,LIBB + ISMD02LIIB,ISMD02,LIBB + +In this example, whenever `ISMD01LIIB` or `ISMD02LIIB` is contained in the filepath of the incoming file, +the corresponding GTS headers will be added to the WIS2 Notification Message as a dictionary in the `properties` field: + +.. code-block:: json + + "properties": { + "gts": { + "ttaaii": "ISMD01", + "cccc": "LIBB" + } + } + +If the `gts_headers_mapping.csv` file is not present in the directory you defined using the `WIS2BOX_HOST_DATADIR` environment variable, wis2box will not add any GTS headers to the WIS2 Notification Message. diff --git a/grafana/dashboards/cadvisor.json b/grafana/dashboards/cadvisor.json deleted file mode 100644 index b5b88121..00000000 --- a/grafana/dashboards/cadvisor.json +++ /dev/null @@ -1,593 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 6, - "interval": "1m", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "code", - "expr": "avg(container_memory_usage_bytes{name!=\"\"}/1024/1024) by (name)", - "interval": "1m", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Memory: used MB", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 2, - "interval": "1m", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "code", - "expr": "sum(rate(container_network_transmit_bytes_total{name!=\"\"}[1m])) by (name)", - "interval": "1m", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Network: Sent rate bytes/s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "code", - "expr": "sum(rate(container_network_receive_errors_total{name!=\"\"} [1m])) by (name)", - "interval": "1m", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Network: Received error rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 9 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "code", - "expr": "sum(rate(container_cpu_usage_seconds_total{name!=\"\"}[$__rate_interval])) by (name)", - "interval": "1m", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "CPU load average", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 9 - }, - "id": 4, - "interval": "1m", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "code", - "expr": "sum(rate(container_network_receive_bytes_total{name!=\"\"}[1m])) by (name)", - "interval": "1m", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Network: Received rate bytes/s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 9 - }, - "id": 11, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "code", - "expr": "sum(rate(container_network_transmit_errors_total{name!=\"\"} [1m])) by (name)", - "interval": "1m", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Network: Transmit error rate", - "type": "timeseries" - } - ], - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "cAdvisor dashboard", - "uid": "81cZrlS4z", - "version": 1, - "weekStart": "" -} \ No newline at end of file diff --git a/grafana/dashboards/home.json b/grafana/dashboards/home.json index 51afba30..2bada33c 100644 --- a/grafana/dashboards/home.json +++ b/grafana/dashboards/home.json @@ -640,7 +640,7 @@ }, "timepicker": {}, "timezone": "", - "title": "wis2box workflow monitoring", + "title": "wis2box data publication dashboard", "uid": "KkBocEA4k", "version": 4, "weekStart": "" diff --git a/grafana/dashboards/storage_dashboard.json b/grafana/dashboards/storage_dashboard.json deleted file mode 100644 index 84920ddf..00000000 --- a/grafana/dashboards/storage_dashboard.json +++ /dev/null @@ -1,362 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 3, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 18, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.0.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "minio_bucket_usage_object_total", - "legendFormat": "{{bucket}}", - "range": true, - "refId": "A" - } - ], - "title": "Total objects stored per bucket", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "fixed" - }, - "decimals": 1, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 18, - "w": 4, - "x": 4, - "y": 0 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.0.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "minio_bucket_usage_total_bytes", - "legendFormat": "{{bucket}}", - "range": true, - "refId": "A" - } - ], - "title": "Total MB stored per bucket", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 11, - "x": 8, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "rate(minio_bucket_traffic_sent_bytes[$__rate_interval])", - "legendFormat": "{{bucket}}", - "range": true, - "refId": "A" - } - ], - "title": "Traffic sent bytes rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 11, - "x": 8, - "y": 10 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "rate(minio_bucket_traffic_received_bytes[$__rate_interval])", - "legendFormat": "{{bucket}}", - "range": true, - "refId": "A" - } - ], - "title": "Traffic received bytes rate", - "type": "timeseries" - } - ], - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Storage statistics", - "uid": "XCoSx0g4z", - "version": 1, - "weekStart": "" - } \ No newline at end of file diff --git a/grafana/dashboards/wis2-downloader.json b/grafana/dashboards/wis2-downloader.json deleted file mode 100644 index a6136bcf..00000000 --- a/grafana/dashboards/wis2-downloader.json +++ /dev/null @@ -1,870 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 2, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 0, - "y": 0 - }, - "id": 14, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Topic" - } - ] - }, - "pluginVersion": "", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "max by(topic) (topic_subscription_status == 1)", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "WIS2-downloader subscriptions", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Value": true - }, - "indexByName": {}, - "renameByName": { - "topic": "Topic" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 25 - }, - { - "color": "red", - "value": 50 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 5, - "y": 0 - }, - "id": 18, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "expr": "queue_size", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Download queue size", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "light-green", - "mode": "fixed" - }, - "mappings": [], - "max": 1000, - "min": 0, - "noValue": "No data", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 100 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 8, - "y": 0 - }, - "id": 11, - "interval": "1h", - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": false - }, - "pluginVersion": "", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum(increase(downloaded_files_total[1h]))", - "format": "time_series", - "instant": true, - "interval": "1h", - "legendFormat": "{{label_name}}", - "range": false, - "refId": "A" - } - ], - "title": "Files downloaded last hour", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "light-blue", - "mode": "fixed" - }, - "decimals": 1, - "mappings": [], - "max": 1000, - "min": 0, - "noValue": "No data", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 100 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 11, - "y": 0 - }, - "id": 12, - "interval": "1h", - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": false - }, - "pluginVersion": "", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum(increase(downloaded_bytes_total[1h]))", - "format": "time_series", - "instant": true, - "interval": "1h", - "legendFormat": "{{label_name}}", - "range": false, - "refId": "A" - } - ], - "title": "Data downloaded last hour", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "light-green", - "mode": "fixed" - }, - "mappings": [], - "max": 9997, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 14, - "y": 0 - }, - "id": 16, - "interval": "1h", - "options": { - "displayMode": "lcd", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "text": { - "titleSize": 17, - "valueSize": 20 - } - }, - "pluginVersion": "", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum(increase(downloaded_files_total{file_type=\"bufr\"}[1h]))", - "format": "time_series", - "instant": true, - "interval": "1h", - "legendFormat": "BUFR", - "range": false, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum(increase(downloaded_files_total{file_type=\"grib\"}[1h]))", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "1h", - "legendFormat": "GRIB", - "range": false, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(downloaded_files_total{file_type!~\"bufr|grib\"}[1h]))", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "1h", - "legendFormat": "Other", - "range": false, - "refId": "C" - } - ], - "title": "Files downloaded by file-type", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "light-blue", - "mode": "fixed" - }, - "decimals": 1, - "mappings": [], - "max": 1000000000, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 19, - "y": 0 - }, - "id": 19, - "interval": "1h", - "options": { - "displayMode": "lcd", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "text": { - "titleSize": 17, - "valueSize": 20 - } - }, - "pluginVersion": "", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum(increase(downloaded_bytes_total{file_type=\"bufr\"}[1h]))", - "format": "time_series", - "instant": true, - "interval": "1h", - "legendFormat": "BUFR", - "range": false, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum(increase(downloaded_bytes_total{file_type=\"grib\"}[1h]))", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "1h", - "legendFormat": "GRIB", - "range": false, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(downloaded_bytes_total{file_type!~\"grib|bufr\"}[1h]))", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "1h", - "legendFormat": "Other", - "range": false, - "refId": "C" - } - ], - "title": "Data downloaded by file-type", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 19, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "light-green", - "value": null - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Data downloaded per hour" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-blue", - "mode": "fixed" - } - } - ] - }, - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Data downloaded per minute" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Data downloaded per minute" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-blue", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 14, - "x": 0, - "y": 4 - }, - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "expr": "sum(increase(downloaded_bytes_total[1m]))", - "hide": false, - "legendFormat": "Data downloaded per minute", - "range": true, - "refId": "A" - } - ], - "title": "Data downloaded per minute", - "type": "timeseries" - }, - { - "datasource": { - "type": "loki", - "uid": "P982945308D3682D1" - }, - "gridPos": { - "h": 18, - "w": 10, - "x": 14, - "y": 4 - }, - "id": 6, - "options": { - "dedupStrategy": "none", - "enableLogDetails": false, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": false, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "P982945308D3682D1" - }, - "editorMode": "builder", - "expr": "{container_name=\"wis2-downloader\"} != `Queue` != `GET` != `CLEAN`", - "queryType": "range", - "refId": "A" - } - ], - "title": "WIS2 downloader logs", - "transparent": true, - "type": "logs" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "download per minute" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Files downloaded per hour" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 14, - "x": 0, - "y": 13 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "editorMode": "builder", - "expr": "sum(increase(downloaded_files_total[1m]))", - "legendFormat": "Files downloaded per minute", - "range": true, - "refId": "A" - } - ], - "title": "Files downloaded per minute", - "type": "timeseries" - } - ], - "refresh": "5s", - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "wis2-downloader monitoring dashboard", - "uid": "zB4GjVaIk", - "version": 5, - "weekStart": "" - } \ No newline at end of file diff --git a/grafana/dashboards/wis2box_broker.json b/grafana/dashboards/wis2box_broker.json deleted file mode 100644 index 9673553b..00000000 --- a/grafana/dashboards/wis2box_broker.json +++ /dev/null @@ -1,405 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 4, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 11, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "increase(wis2box_broker_msg_sent[1m])", - "legendFormat": "messages sent rate", - "range": true, - "refId": "A" - } - ], - "title": "wis2box-broker: messages sent per minute", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 11, - "x": 11, - "y": 0 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "wis2box_broker_msg_dropped", - "legendFormat": "messages dropped", - "range": true, - "refId": "A" - } - ], - "title": "wis2box-broker: messages dropped", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 11, - "x": 0, - "y": 11 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "increase(wis2box_broker_msg_received[1m])", - "legendFormat": "messages received", - "range": true, - "refId": "A" - } - ], - "title": "wis2box-broker: messages received per minute", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 11, - "x": 11, - "y": 11 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "wis2box_broker_msg_stored", - "legendFormat": "messages stored", - "range": true, - "refId": "A" - } - ], - "title": "wis2box-broker: messages stored", - "type": "timeseries" - } - ], - "refresh": "5s", - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "wis2box broker statistics $SYS/broker/messages", - "uid": "LNKi9EA4z", - "version": 1, - "weekStart": "" - } \ No newline at end of file diff --git a/grafana/dashboards/wis2downloader.json b/grafana/dashboards/wis2downloader.json new file mode 100644 index 00000000..690c2b5b --- /dev/null +++ b/grafana/dashboards/wis2downloader.json @@ -0,0 +1,769 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 0, + "y": 0 + }, + "id": 14, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Topic" + } + ] + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "max by(topic) (topic_subscription_status == 1)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "WIS2-downloader subscriptions", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true + }, + "indexByName": {}, + "renameByName": { + "topic": "Topic" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 25 + }, + { + "color": "red", + "value": 50 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 0 + }, + "id": 18, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "builder", + "expr": "queue_size", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Download queue size", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-green", + "mode": "fixed" + }, + "decimals": 0, + "mappings": [], + "max": 9997, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 16, + "interval": "1h", + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": { + "titleSize": 17, + "valueSize": 20 + } + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(sum(downloaded_files_total{file_type=\"bufr\"} or vector(0))[1h:])", + "format": "time_series", + "instant": true, + "interval": "1h", + "legendFormat": "BUFR", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(sum(downloaded_files_total{file_type=\"grib\"} or vector(0))[1h:])", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "1h", + "legendFormat": "GRIB", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(sum(downloaded_files_total{file_type!~\"bufr|grib\"} or vector(0))[1h:])", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "1h", + "legendFormat": "Other", + "range": false, + "refId": "C" + } + ], + "title": "Files downloaded by file-type, last hour", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-blue", + "mode": "fixed" + }, + "decimals": 1, + "mappings": [], + "max": 1000000000, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 19, + "interval": "1h", + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": { + "titleSize": 17, + "valueSize": 20 + } + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(sum(downloaded_bytes_total{file_type=\"bufr\"} or vector(0))[1h:])", + "format": "time_series", + "instant": true, + "interval": "1h", + "legendFormat": "BUFR", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(sum(downloaded_bytes_total{file_type=\"grib\"} or vector(0))[1h:])", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "1h", + "legendFormat": "GRIB", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(sum(downloaded_bytes_total{file_type!~\"grib|bufr\"} or vector(0))[1h:])", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "1h", + "legendFormat": "Other", + "range": false, + "refId": "C" + } + ], + "title": "Data downloaded by file-type, last hour", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "download per minute" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Files downloaded per hour" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Failded downloads per minute" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Failed downloads per minute" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 15, + "x": 9, + "y": 4 + }, + "id": 2, + "interval": "60s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "code", + "expr": "increase(sum(downloaded_files_total or vector(0))[1m:])", + "legendFormat": "Successful downloads per minute", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "code", + "expr": "increase(sum(failed_downloads_total or vector(0))[1m:])", + "hide": false, + "legendFormat": "Failed downloads per minute", + "range": true, + "refId": "B" + } + ], + "title": "Successful / Failed downloads per minute", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "P55348B596EBB51C3" + }, + "gridPos": { + "h": 16, + "w": 9, + "x": 0, + "y": 6 + }, + "id": 6, + "options": { + "dedupStrategy": "none", + "enableLogDetails": false, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P55348B596EBB51C3" + }, + "editorMode": "builder", + "expr": "{container_name=\"wis2downloader\"} != `Queue` != `GET` != `CLEAN` |~ `ERROR|WARNING`", + "queryType": "range", + "refId": "A" + } + ], + "title": "WIS2 downloader logs: ERROR and WARNING", + "transparent": true, + "type": "logs" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 19, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Data downloaded per hour" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Data downloaded per minute" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Data downloaded per minute" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 15, + "x": 9, + "y": 13 + }, + "id": 9, + "interval": "60s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "builder", + "expr": "increase(sum(downloaded_bytes_total or vector(0))[1m:])", + "hide": false, + "legendFormat": "Data downloaded per minute", + "range": true, + "refId": "A" + } + ], + "title": "Data downloaded per minute", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "wis2downloader dashboard", + "uid": "zB4GjVaIk", + "version": 5, + "weekStart": "" +} \ No newline at end of file diff --git a/nginx/nginx-ssl.conf b/nginx/nginx-ssl.conf index 64f9a044..81a38d16 100644 --- a/nginx/nginx-ssl.conf +++ b/nginx/nginx-ssl.conf @@ -59,12 +59,12 @@ rewrite ^/data(/.*)$ /wis2box-public$1 break; proxy_pass http://minio:9000; } - location /wis2-downloader { + location /wis2downloader { set $x_api_http_method $request_method; auth_request /auth; auth_request_set $auth_status $upstream_status; - rewrite ^/wis2-downloader/(.*) /$1 break; - proxy_pass http://wis2-downloader:5000; + rewrite ^/wis2downloader/(.*) /$1 break; + proxy_pass http://wis2downloader:5000; } location /oapi { set $x_api_http_method $request_method; diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 362e5887..fd19673f 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -62,12 +62,12 @@ auth_request_set $auth_status $upstream_status; proxy_pass http://wis2box-api:80; } - location /wis2-downloader { + location /wis2downloader { set $x_api_http_method $request_method; auth_request /auth; auth_request_set $auth_status $upstream_status; - rewrite ^/wis2-downloader/(.*) /$1 break; - proxy_pass http://wis2-downloader:5000; + rewrite ^/wis2downloader/(.*) /$1 break; + proxy_pass http://wis2downloader:5000; } location /wis2box-webapp/ { auth_basic "Restricted Access"; diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 16d7d5c9..83bcc38f 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -26,8 +26,8 @@ scrape_configs: - targets: ['cadvisor:8080'] labels: alias: 'cadvisor' -- job_name: wis2-downloader +- job_name: wis2downloader metrics_path: /metrics scheme: http static_configs: - - targets: ['wis2-downloader:5000'] + - targets: ['wis2downloader:5000'] diff --git a/tests/data/gts_headers_mapping.csv b/tests/data/gts_headers_mapping.csv new file mode 100644 index 00000000..e2a24005 --- /dev/null +++ b/tests/data/gts_headers_mapping.csv @@ -0,0 +1,8 @@ +string_in_filepath,ttaaii,cccc +ISIN21DAMM,ISIN21,DAMM +ISMD01LIIB,ISMD01,LIBB +ISMD02LIIB,ISMD02,LIBB +SMRO01YRBK,SMRO01,YRBK +IOBH01LFPW,IOBH01,LFPW +ISSA05LFPW,ISSA05,LFPW +SICG20FCBB,SICG20,FCBB \ No newline at end of file diff --git a/tests/data/observations/congo/synop_202308.txt b/tests/data/observations/congo/SICG20FCBB_202308.txt similarity index 100% rename from tests/data/observations/congo/synop_202308.txt rename to tests/data/observations/congo/SICG20FCBB_202308.txt diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py index 9661b538..5d7d02df 100644 --- a/tests/integration/test_workflow.py +++ b/tests/integration/test_workflow.py @@ -24,6 +24,8 @@ # .github/workflows/tests-docker.yml has been executed import csv +import os + from pathlib import Path from pywis_pubsub.validation import validate_message @@ -40,6 +42,43 @@ } +def test_wis2downloader(): + """Test if the wis2downloader has downloaded + the expected number of files in the download directory""" + + DOWNLOAD_DIR = DATADIR / 'downloads' + + topic_nfiles_dict = { + 'origin/a/wis2/mw-mw_met_centre/data/core/weather/surface-based-observations/synop': 23, # noqa + 'origin/a/wis2/dz-alger_met_centre/data/core/weather/surface-based-observations/synop': 28, # noqa + 'origin/a/wis2/cn-cma/data/core/weather/prediction/forecast/medium-range/probabilistic/global': 10, # noqa + 'origin/a/wis2/ro-rnimh/data/core/weather/surface-based-observations/synop': 49, # noqa + 'origin/a/wis2/cd-brazza_met_centre/data/core/weather/surface-based-observations/synop': 14, # noqa + 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/buoy': 2, # noqa + 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind_profiler': 1, # noqa + 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship': 5, # noqa + 'origin/a/wis2/it-roma_met_centre/data/core/weather/surface-based-observations/synop': 31 # noqa + } + + topic_nfiles_dict_found = {} + for key in topic_nfiles_dict.keys(): + topic_nfiles_dict_found[key] = 0 + + # count the number of files received in the download directory + # over all subdirectories + total_files = 0 + for root, dirs, files in os.walk(DOWNLOAD_DIR): + total_files += len(files) + for key in topic_nfiles_dict.keys(): + if key in root: + topic_nfiles_dict_found[key] += len(files) + + # check if the number of files downloaded for each topic + # matches the expected number + for key in topic_nfiles_dict.keys(): + assert topic_nfiles_dict[key] == topic_nfiles_dict_found[key] + + def test_metadata_station_cache(): """Test station metadata caching""" @@ -292,6 +331,9 @@ def test_message_api(): assert props['content']['size'] == 253 assert props['content']['encoding'] == 'base64' assert props['content']['value'] is not None + assert 'gts' in props + assert props['gts']['ttaaii'] == 'SICG20' + assert props['gts']['cccc'] == 'FCBB' link_rel = msg['links'][0] diff --git a/tests/test.env b/tests/test.env index 65b65037..82ae0dff 100644 --- a/tests/test.env +++ b/tests/test.env @@ -60,4 +60,12 @@ MINIO_NOTIFY_MQTT_QOS_WIS2BOX=1 # wis2box webapp credentials WIS2BOX_WEBAPP_USERNAME=wis2box-user -WIS2BOX_WEBAPP_PASSWORD=wis2boxtest123 \ No newline at end of file +WIS2BOX_WEBAPP_PASSWORD=wis2boxtest123 + +# downloader settings, subscribe to local broker for test +DOWNLOAD_BROKER_HOST=mosquitto +DOWNLOAD_BROKER_PORT=1883 +DOWNLOAD_BROKER_USERNAME=everyone +DOWNLOAD_BROKER_PASSWORD=everyone +DOWNLOAD_BROKER_TRANSPORT=tcp +DOWNLOAD_MIN_FREE_SPACE_GB=0 \ No newline at end of file diff --git a/wis2-downloader/config.template b/wis2-downloader/config.template deleted file mode 100644 index c5de79ac..00000000 --- a/wis2-downloader/config.template +++ /dev/null @@ -1,15 +0,0 @@ -{ - "broker_url": "${DOWNLOAD_BROKER_HOST}", - "broker_port": $DOWNLOAD_BROKER_PORT, - "username": "${DOWNLOAD_BROKER_USERNAME}", - "password": "${DOWNLOAD_BROKER_PASSWORD}", - "protocol": "${DOWNLOAD_BROKER_PROTOCOL}", - "topics": {}, - "download_dir": "${DOWNLOAD_DIR}", - "retention_period_hours": "${DOWNLOAD_RETENTION_PERIOD_HOURS}", - "flask_host": "${FLASK_HOST}", - "flask_port": "${FLASK_PORT}", - "download_workers": $DOWNLOAD_WORKERS, - "save_logs": false, - "log_dir": "logs" -} diff --git a/wis2box-create-config.py b/wis2box-create-config.py index 32be22df..1e5d6226 100644 --- a/wis2box-create-config.py +++ b/wis2box-create-config.py @@ -358,6 +358,17 @@ def create_wis2box_env(config_dir: str) -> None: fh.write('MINIO_NOTIFY_MQTT_TOPIC_WIS2BOX=wis2box/storage\n') fh.write('MINIO_NOTIFY_MQTT_QOS_WIS2BOX=1\n') fh.write('\n') + # downloader settings + fh.write('# downloader settings\n') + fh.write('DOWNLOAD_BROKER_HOST=globalbroker.meteo.fr\n') + fh.write('DOWNLOAD_BROKER_PORT=8883\n') + fh.write('DOWNLOAD_BROKER_USERNAME=everyone\n') + fh.write('DOWNLOAD_BROKER_PASSWORD=everyone\n') + fh.write('# download transport mechanism (tcp or websockets)\n') + fh.write('DOWNLOAD_BROKER_TRANSPORT=tcp\n') + fh.write('# maximum MB in download directory\n') + fh.write('DOWNLOAD_MIN_FREE_SPACE_GB=1\n') + fh.write('\n') print('*' * 80) print('The file wis2box.env has been created in the current directory.') diff --git a/wis2box-management/docker/entrypoint.sh b/wis2box-management/docker/entrypoint.sh index 5d065f8b..8baf4c22 100755 --- a/wis2box-management/docker/entrypoint.sh +++ b/wis2box-management/docker/entrypoint.sh @@ -91,14 +91,14 @@ else # Add the token wis2box auth add-token --path collections/stations -y fi -# repeat for wis2-downloader -is_restricted=$(wis2box auth is-restricted-path --path wis2-downloader) +# repeat for wis2downloader +is_restricted=$(wis2box auth is-restricted-path --path wis2downloader) if [ "$is_restricted" = "True" ]; then - echo "wis2-downloader is restricted" + echo "wis2downloader is restricted" else - echo "restricting wis2-downloader" + echo "restricting wis2downloader" # Add the token - wis2box auth add-token --path wis2-downloader -y + wis2box auth add-token --path wis2downloader -y fi echo "END /entrypoint.sh" diff --git a/wis2box-management/wis2box/__init__.py b/wis2box-management/wis2box/__init__.py index fa8a2f53..481951b1 100644 --- a/wis2box-management/wis2box/__init__.py +++ b/wis2box-management/wis2box/__init__.py @@ -30,6 +30,7 @@ from wis2box.metadata import metadata from wis2box.pubsub import pubsub from wis2box.auth import auth +from wis2box.downloader import downloader @click.group() @@ -46,3 +47,4 @@ def cli(): cli.add_command(metadata) cli.add_command(auth) cli.add_command(pubsub) +cli.add_command(downloader) diff --git a/wis2box-management/wis2box/data/base.py b/wis2box-management/wis2box/data/base.py index 340cf56b..e5ed363d 100644 --- a/wis2box-management/wis2box/data/base.py +++ b/wis2box-management/wis2box/data/base.py @@ -60,6 +60,15 @@ def __init__(self, defs: dict) -> None: self.buckets = defs.get('buckets', ()) self.output_data = {} self.discovery_metadata = {} + self.gts = None + gts_ttaaii = defs.get('gts_ttaaii') + gts_cccc = defs.get('gts_cccc') + if None not in [gts_ttaaii, gts_cccc]: + self.gts = { + 'ttaaii': gts_ttaaii, + 'cccc': gts_cccc + } + # if discovery_metadata: # self.setup_discovery_metadata(discovery_metadata) @@ -152,7 +161,8 @@ def notify(self, identifier: str, storage_path: str, wis_message = WISNotificationMessage( f"{metadata_id.replace('urn:wmo:md:','')}/{identifier}", metadata_id, storage_path, datetime_, geometry, - wigos_station_identifier, operation) + wigos_station_identifier, self.gts, + operation) # load plugin for public broker defs = { diff --git a/wis2box-management/wis2box/data_mappings.py b/wis2box-management/wis2box/data_mappings.py index 37300d93..757f9d54 100644 --- a/wis2box-management/wis2box/data_mappings.py +++ b/wis2box-management/wis2box/data_mappings.py @@ -82,6 +82,7 @@ def get_data_mappings() -> dict: def validate_and_load(path: str, data_mappings: dict = None, + gts_mappings: dict = None, file_type: str = None ) -> Tuple[str, Tuple[Any]]: """ @@ -135,7 +136,8 @@ def validate_and_load(path: str, LOGGER.debug(f'Adding plugin definition for {file_type}') def data_defs(plugin): - return { + notify = plugin.get('notify', False) + data_defs = { 'metadata_id': metadata_id, 'íncoming_filepath': path, 'topic_hierarchy': topic_hierarchy, @@ -143,9 +145,17 @@ def data_defs(plugin): 'pattern': plugin['file-pattern'], 'template': plugin.get('template'), 'buckets': plugin.get('buckets', ()), - 'notify': plugin.get('notify', False), + 'notify': notify, 'format': file_type } + if notify and gts_mappings: + for key in gts_mappings.keys(): + # check if string defined by key is contained in path + if key in path: + data_defs['gts_ttaaii'] = gts_mappings[key]['ttaaii'] + data_defs['gts_cccc'] = gts_mappings[key]['cccc'] + break + return data_defs plugins_ = [load_plugin('data', data_defs(p), data_mappings) for p in plugins[file_type]] diff --git a/wis2box-management/wis2box/downloader.py b/wis2box-management/wis2box/downloader.py new file mode 100644 index 00000000..a673bf71 --- /dev/null +++ b/wis2box-management/wis2box/downloader.py @@ -0,0 +1,112 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +import logging + +import click +import requests + +from wis2box import cli_helpers + +LOGGER = logging.getLogger(__name__) + +DOWNLOAD_URL = 'http://wis2downloader:5000' + + +@click.group() +def downloader(): + """Interact with the wis2downloader""" + pass + + +@click.command('list-subscriptions') +@click.pass_context +@cli_helpers.OPTION_VERBOSITY +def list_subscriptions(ctx, verbosity): + """list all subscriptions""" + + # make a GET requests to http://{DOWNLOAD_URL}/subscriptions + try: + response = requests.get(f'{DOWNLOAD_URL}/subscriptions') + # check response status + if response.status_code == 200: + click.echo('Current subscriptions:') + click.echo(response.text) + else: + click.echo(f'Error: {response.status_code}') + click.echo(response.text) + except requests.exceptions.ConnectionError: + click.echo('Error: Connection refused') + click.echo(f'Is the wis2downloader running on {DOWNLOAD_URL}?') + + +@click.command('add-subscription') +@click.pass_context +@click.option('--topic', '-t', help='The topic to subscribe to', required=True) +def add_subscription(ctx, topic): + """add a subscription""" + + # make a POST request to http://{DOWNLOAD_URL}/subscriptions + try: + response = requests.post(f'{DOWNLOAD_URL}/subscriptions', + json={'topic': topic}) + # check response status + if response.status_code == 201: + click.echo('Subscription added') + click.echo('Added subscription:') + click.echo(response.text) + else: + click.echo('Subscription not added') + click.echo(f'Error: {response.status_code}') + click.echo(response.text) + except requests.exceptions.ConnectionError: + click.echo('Error: Connection refused') + click.echo(f'Is the wis2downloader running on {DOWNLOAD_URL}?') + + +@click.command('remove-subscription') +@click.pass_context +@click.option('--topic', '-t', help='The topic to subscribe to', required=True) +def remove_subscription(ctx, topic): + """remove a subscription""" + + topic = topic.replace('%23', '#') + topic = topic.replace('%2B', '+') + # make a DELETE request to http://{DOWNLOAD_URL}/subscriptions + try: + response = requests.delete(f'{DOWNLOAD_URL}/subscriptions/{topic}') # noqa + # check response status + if response.status_code == 200: + click.echo('Subscription deleted') + click.echo('Current subscriptions:') + click.echo(response.text) + else: + click.echo('Subscription not deleted') + click.echo(f'Error: {response.status_code}') + click.echo(response.text) + except requests.exceptions.ConnectionError: + click.echo('Error: Connection refused') + click.echo(f'Is the wis2downloader running on {DOWNLOAD_URL}?') + + +downloader.add_command(list_subscriptions) +downloader.add_command(add_subscription) +downloader.add_command(remove_subscription) diff --git a/wis2box-management/wis2box/handler.py b/wis2box-management/wis2box/handler.py index 1230b3af..b0edbe21 100644 --- a/wis2box-management/wis2box/handler.py +++ b/wis2box-management/wis2box/handler.py @@ -37,8 +37,8 @@ class Handler: def __init__(self, filepath: str, - metadata_id: str = None, - data_mappings: dict = None) -> None: + data_mappings: dict = None, + gts_mappings: dict = None) -> None: self.filepath = filepath self.plugins = () self.input_bytes = None @@ -61,7 +61,7 @@ def __init__(self, filepath: str, raise NotHandledError(msg) try: self.metadata_id, self.plugins = validate_and_load( - self.filepath, data_mappings, self.filetype) + self.filepath, data_mappings, gts_mappings, self.filetype) except Exception as err: msg = f'Path validation error: {err}' # errors in public storage are not handled diff --git a/wis2box-management/wis2box/pubsub/message.py b/wis2box-management/wis2box/pubsub/message.py index d036a630..edba090c 100644 --- a/wis2box-management/wis2box/pubsub/message.py +++ b/wis2box-management/wis2box/pubsub/message.py @@ -126,7 +126,8 @@ def _generate_checksum(self, bytes, algorithm: SecureHashAlgorithms) -> str: # class WISNotificationMessage(PubSubMessage): def __init__(self, identifier: str, metadata_id: str, filepath: str, - datetime_: str, geometry=None, wigos_station_identifier=None, + datetime_: str, geometry=None, + wigos_station_identifier=None, gts: dict = None, operation: str = 'create') -> None: super().__init__('wis2-notification-message', identifier, @@ -186,6 +187,10 @@ def __init__(self, identifier: str, metadata_id: str, filepath: str, if metadata_id is not None: self.message['properties']['metadata_id'] = metadata_id + # if gts-dictionary is provided, include it in the message properties + if gts is not None: + self.message['properties']['gts'] = gts + if self.length < 4096: LOGGER.debug('Including data inline via properties.content') content_value = base64.b64encode(self.filebytes) diff --git a/wis2box-management/wis2box/pubsub/subscribe.py b/wis2box-management/wis2box/pubsub/subscribe.py index 8595f5e1..dd3724ea 100644 --- a/wis2box-management/wis2box/pubsub/subscribe.py +++ b/wis2box-management/wis2box/pubsub/subscribe.py @@ -36,7 +36,8 @@ from wis2box.data_mappings import get_data_mappings from wis2box.data.message import MessageData -from wis2box.env import (DOCKER_BROKER, STORAGE_SOURCE, STORAGE_ARCHIVE) +from wis2box.env import (DATADIR, DOCKER_BROKER, + STORAGE_SOURCE, STORAGE_ARCHIVE) from wis2box.handler import Handler, NotHandledError import wis2box.metadata.discovery as discovery_metadata from wis2box.plugin import load_plugin, PLUGINS @@ -46,10 +47,33 @@ LOGGER = logging.getLogger(__name__) +def get_gts_mappings(): + # read gts mappings from CSV file in DATADIR + gts_mappings = {} + mapping_file = 'gts_headers_mapping.csv' + try: + with open(f'{DATADIR}/{mapping_file}', 'r') as f: + for line in f: + key = line.strip().split(',')[0] + if key == 'string_in_filepath': + continue + ttaaii = line.strip().split(',')[1] + cccc = line.strip().split(',')[2] + value = {'ttaaii': ttaaii, 'cccc': cccc} + gts_mappings[key] = value + LOGGER.info(f'GTS mapping: string_in_filepath={key}, {value}') + except FileNotFoundError: + LOGGER.warning(f'To add GTS headers, please create {mapping_file} in WIS2BOX_HOST_DATADIR') # noqa + except Exception as err: + LOGGER.error(f'Error reading GTS mappings: {err}') + return gts_mappings + + class WIS2BoxSubscriber: def __init__(self, broker): self.data_mappings = get_data_mappings() + self.gts_mappings = get_gts_mappings() self.broker = broker self.broker.bind('on_message', self.on_message_handler) self.broker.sub('wis2box/#') @@ -59,7 +83,8 @@ def handle(self, filepath): LOGGER.info(f'Processing {filepath}') # load handler handler = Handler(filepath=filepath, - data_mappings=self.data_mappings) + data_mappings=self.data_mappings, + gts_mappings=self.gts_mappings) if handler.handle(): LOGGER.debug('Data processed') for plugin in handler.plugins: diff --git a/wis2box.env.example b/wis2box.env.example index 7194e3cd..cbfdb6f9 100644 --- a/wis2box.env.example +++ b/wis2box.env.example @@ -59,3 +59,13 @@ MINIO_NOTIFY_MQTT_BROKER_WIS2BOX=tcp://${WIS2BOX_BROKER_HOST}:${WIS2BOX_BROKER_P MINIO_NOTIFY_MQTT_TOPIC_WIS2BOX=wis2box/storage MINIO_NOTIFY_MQTT_QOS_WIS2BOX=1 +# downloader settings +DOWNLOAD_BROKER_HOST=globalbroker.meteo.fr +DOWNLOAD_BROKER_PORT=443 +DOWNLOAD_BROKER_USERNAME=everyone +DOWNLOAD_BROKER_PASSWORD=everyone +# download transport mechanism (tcp or websockets) +DOWNLOAD_BROKER_TRANSPORT=websockets +# maximum MB in download directory +DOWNLOAD_MIN_FREE_SPACE_GB=1 + diff --git a/wis2-downloader/Dockerfile b/wis2downloader/Dockerfile similarity index 61% rename from wis2-downloader/Dockerfile rename to wis2downloader/Dockerfile index a5ee6fd1..3b20c407 100644 --- a/wis2-downloader/Dockerfile +++ b/wis2downloader/Dockerfile @@ -10,17 +10,17 @@ ENV DOWNLOAD_BROKER_HOST "globalbroker.meteo.fr" ENV DOWNLOAD_BROKER_PORT 443 ENV DOWNLOAD_BROKER_USERNAME "everyone" ENV DOWNLOAD_BROKER_PASSWORD "everyone" -ENV DOWNLOAD_BROKER_PROTOCOL "websockets" +ENV DOWNLOAD_BROKER_TRANSPORT "websockets" ENV DOWNLOAD_WORKERS 8 -ENV FLASK_HOST "0.0.0.0" -ENV FLASK_PORT 5000 -ENV SAVE_LOGS false -ENV LOGS_DIR /app/logs +ENV DOWNLOAD_MIN_FREE_SPACE_GB 1 +ENV DOWNLOAD_VALIDATE_TOPICS "false" +ENV LOG_PATH /app/logs +ENV WIS2DOWNLOADER_CONFIG "/app/config.json" # update pyopenssl and pin requests and urllib3 to avoid SSL error RUN pip install pyopenssl --upgrade && pip install requests==2.26.0 urllib3==1.26.0 -# install cron and envsubst -RUN apt-get update && apt-get install -y cron gettext-base +# install cron and envsubst and guincorn +RUN apt-get update && apt-get install -y cron gettext-base gunicorn # copy all the code to the Docker image COPY . /app @@ -28,8 +28,8 @@ COPY . /app # Set the working directory to /app WORKDIR /app -# install wis2-downloader -RUN pip install https://github.com/wmo-im/wis2-downloader/archive/main.zip +# install wis2downloader from github main branch +RUN pip3 install https://github.com/wmo-im/wis2downloader/archive/main.zip # add wis2box.cron to crontab COPY ./clean.cron /etc/cron.d/clean.cron @@ -42,5 +42,5 @@ RUN chmod +x /app/entrypoint.sh ENTRYPOINT [ "/app/entrypoint.sh" ] -# Run wis2-downloader when the container launches -CMD ["wis2downloader","--config","/app/config.json"] +# Run wis2downloader when the container launches +CMD ["gunicorn","--bind","0.0.0.0:5000","--pythonpath","/usr/local/lib/python3.9/site-packages/","--workers", "1", "wis2downloader.app:app"] \ No newline at end of file diff --git a/wis2-downloader/clean.cron b/wis2downloader/clean.cron similarity index 100% rename from wis2-downloader/clean.cron rename to wis2downloader/clean.cron diff --git a/wis2-downloader/clean.py b/wis2downloader/clean.py similarity index 100% rename from wis2-downloader/clean.py rename to wis2downloader/clean.py diff --git a/wis2downloader/config.template b/wis2downloader/config.template new file mode 100644 index 00000000..7f11111e --- /dev/null +++ b/wis2downloader/config.template @@ -0,0 +1,18 @@ +{ + "broker_hostname": "${DOWNLOAD_BROKER_HOST}", + "broker_port": ${DOWNLOAD_BROKER_PORT}, + "broker_username": "${DOWNLOAD_BROKER_USERNAME}", + "broker_password": "${DOWNLOAD_BROKER_PASSWORD}", + "broker_protocol": "${DOWNLOAD_BROKER_TRANSPORT}", + "download_workers": ${DOWNLOAD_WORKERS}, + "min_free_space": ${DOWNLOAD_MIN_FREE_SPACE_GB}, + "validate_topics": ${DOWNLOAD_VALIDATE_TOPICS}, + "mqtt_session_info": "${DOWNLOAD_DIR}/.session-info.json", + "download_dir": "${DOWNLOAD_DIR}", + "log_level": "INFO", + "base_url": "http://localhost:5000", + "flask_host": "0.0.0.0", + "flask_port": 5000, + "save_logs": false, + "log_path": "logs" +} diff --git a/wis2-downloader/entrypoint.sh b/wis2downloader/entrypoint.sh similarity index 50% rename from wis2-downloader/entrypoint.sh rename to wis2downloader/entrypoint.sh index ff9a19d1..61386e13 100644 --- a/wis2-downloader/entrypoint.sh +++ b/wis2downloader/entrypoint.sh @@ -14,8 +14,6 @@ echo "END /entrypoint.sh" echo "Download directory in container: $DOWNLOAD_DIR" # print the retention period hours echo "Retention period in hours: $RETENTION_PERIOD_HOURS" -# print the maximum MB allowed in the download directory -echo "Maximum MB allowed in the download directory: $MAX_MB_DOWNLOAD_DIR" # ensure DOWNLOAD_DIR exists if [ ! -d $DOWNLOAD_DIR ]; then @@ -24,6 +22,17 @@ if [ ! -d $DOWNLOAD_DIR ]; then fi envsubst < config.template > config.json +# if session-info.json does not exists in $DOWNLOAD_DIR, create it +if [ ! -f $DOWNLOAD_DIR/.session-info.json ]; then + echo "Creating .session-info.json" + echo "{" > $DOWNLOAD_DIR/.session-info.json + echo ' "topics": {},' >> $DOWNLOAD_DIR/.session-info.json + # generate a random string for client_id + echo "Generating random client_id" + echo ' "client_id": "wis2box-wis2downloader-'$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)'"' >> $DOWNLOAD_DIR/.session-info.json + echo "}" >> $DOWNLOAD_DIR/.session-info.json +fi + # print the config echo "Config:" cat /app/config.json