From ffdba2c255a56fc6f04f47bb4d6fd743f9c88b1b Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 18 Mar 2024 15:55:26 -0400 Subject: [PATCH] Containerize (#6) * containerize application * add metric collector --- Makefile | 4 +- README.md | 44 ++++- ...-compose.dev.yml => docker-compose.dev.yml | 0 ...verride.yml => docker-compose.override.yml | 5 +- .../docker-compose.yml => docker-compose.yml | 22 ++- docker/README.md | 18 -- docker/wis2-gdc.env | 18 -- pywis-pubsub.yml | 11 -- .../wis2-gdc-api => wis2-gdc-api}/Dockerfile | 0 {docker/wis2-gdc-api => wis2-gdc-api}/app.py | 0 .../entrypoint.sh | 0 .../wis2-gdc-api.yml | 0 .../mosquitto => wis2-gdc-broker}/Dockerfile | 0 .../mosquitto => wis2-gdc-broker}/acl.conf | 0 .../entrypoint.sh | 0 .../mosquitto.conf | 0 Dockerfile => wis2-gdc-management/Dockerfile | 0 .../MANIFEST.in | 0 wis2-gdc-management/README.md | 3 + .../docker}/entrypoint.sh | 0 .../docker}/pywis-pubsub.yml | 0 .../docker}/wis2-gdc-management.cron | 0 .../requirements-backend.txt | 0 .../requirements-dev.txt | 0 .../requirements.txt | 0 setup.py => wis2-gdc-management/setup.py | 0 .../wis2_gdc}/__init__.py | 0 .../wis2_gdc}/archive.py | 0 .../wis2_gdc}/backend/__init__.py | 0 .../wis2_gdc}/backend/base.py | 0 .../wis2_gdc}/backend/elastic.py | 0 .../wis2_gdc}/backend/ogcapi_records.py | 0 .../wis2_gdc}/env.py | 6 +- .../wis2_gdc}/harvester/__init__.py | 0 .../wis2_gdc}/harvester/base.py | 0 .../wis2_gdc}/harvester/wis2box.py | 0 .../wis2_gdc}/hook.py | 0 .../wis2_gdc}/monitor/__init__.py | 0 .../wis2_gdc}/registrar.py | 19 +- .../wis2_gdc}/sync.py | 0 wis2-gdc-metrics-collector/Dockerfile | 29 +++ .../metrics_collector.py | 165 ++++++++++++++++++ wis2-gdc-metrics-collector/requirements.txt | 3 + wis2-gdc.env | 10 +- wis2_gdc/monitor/metrics.py | 160 ----------------- 45 files changed, 279 insertions(+), 238 deletions(-) rename docker/docker-compose.dev.yml => docker-compose.dev.yml (100%) rename docker/docker-compose.override.yml => docker-compose.override.yml (93%) rename docker/docker-compose.yml => docker-compose.yml (87%) delete mode 100644 docker/README.md delete mode 100644 docker/wis2-gdc.env delete mode 100644 pywis-pubsub.yml rename {docker/wis2-gdc-api => wis2-gdc-api}/Dockerfile (100%) rename {docker/wis2-gdc-api => wis2-gdc-api}/app.py (100%) rename {docker/wis2-gdc-api => wis2-gdc-api}/entrypoint.sh (100%) rename {docker/wis2-gdc-api => wis2-gdc-api}/wis2-gdc-api.yml (100%) rename {docker/mosquitto => wis2-gdc-broker}/Dockerfile (100%) rename {docker/mosquitto => wis2-gdc-broker}/acl.conf (100%) rename {docker/mosquitto => wis2-gdc-broker}/entrypoint.sh (100%) rename {docker/mosquitto => wis2-gdc-broker}/mosquitto.conf (100%) rename Dockerfile => wis2-gdc-management/Dockerfile (100%) rename MANIFEST.in => wis2-gdc-management/MANIFEST.in (100%) create mode 100644 wis2-gdc-management/README.md rename {docker => wis2-gdc-management/docker}/entrypoint.sh (100%) rename {docker => wis2-gdc-management/docker}/pywis-pubsub.yml (100%) rename {docker => wis2-gdc-management/docker}/wis2-gdc-management.cron (100%) rename requirements-backend.txt => wis2-gdc-management/requirements-backend.txt (100%) rename requirements-dev.txt => wis2-gdc-management/requirements-dev.txt (100%) rename requirements.txt => wis2-gdc-management/requirements.txt (100%) rename setup.py => wis2-gdc-management/setup.py (100%) rename {wis2_gdc => wis2-gdc-management/wis2_gdc}/__init__.py (100%) rename {wis2_gdc => wis2-gdc-management/wis2_gdc}/archive.py (100%) rename {wis2_gdc => wis2-gdc-management/wis2_gdc}/backend/__init__.py (100%) rename {wis2_gdc => wis2-gdc-management/wis2_gdc}/backend/base.py (100%) rename {wis2_gdc => wis2-gdc-management/wis2_gdc}/backend/elastic.py (100%) rename {wis2_gdc => wis2-gdc-management/wis2_gdc}/backend/ogcapi_records.py (100%) rename {wis2_gdc => wis2-gdc-management/wis2_gdc}/env.py (91%) rename {wis2_gdc => wis2-gdc-management/wis2_gdc}/harvester/__init__.py (100%) rename {wis2_gdc => wis2-gdc-management/wis2_gdc}/harvester/base.py (100%) rename {wis2_gdc => wis2-gdc-management/wis2_gdc}/harvester/wis2box.py (100%) rename {wis2_gdc => wis2-gdc-management/wis2_gdc}/hook.py (100%) rename {wis2_gdc => wis2-gdc-management/wis2_gdc}/monitor/__init__.py (100%) rename {wis2_gdc => wis2-gdc-management/wis2_gdc}/registrar.py (93%) rename {wis2_gdc => wis2-gdc-management/wis2_gdc}/sync.py (100%) create mode 100644 wis2-gdc-metrics-collector/Dockerfile create mode 100644 wis2-gdc-metrics-collector/metrics_collector.py create mode 100644 wis2-gdc-metrics-collector/requirements.txt delete mode 100644 wis2_gdc/monitor/metrics.py diff --git a/Makefile b/Makefile index 7dd862c..73c69dd 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ # ############################################################################### -DOCKER_COMPOSE_ARGS=--project-name wis2-gdc --file docker/docker-compose.yml --file docker/docker-compose.override.yml +DOCKER_COMPOSE_ARGS=--project-name wis2-gdc --file docker-compose.yml --file docker-compose.override.yml build: docker compose $(DOCKER_COMPOSE_ARGS) build @@ -28,7 +28,7 @@ up: docker compose $(DOCKER_COMPOSE_ARGS) up --detach dev: - docker compose $(DOCKER_COMPOSE_ARGS) --file docker/docker-compose.dev.yml up --detach + docker compose $(DOCKER_COMPOSE_ARGS) --file docker-compose.dev.yml up --detach login: docker exec -it wis2-gdc-management /bin/bash diff --git a/README.md b/README.md index bb8dc55..d5239ae 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ source bin/activate # clone codebase and install git clone https://github.com/wmo-im/wis2-gdc.git -cd wis2-gdc +cd wis2-gdc-management python3 setup.py install ``` @@ -84,7 +84,47 @@ wis2-gdc archive foo.zip ### Docker -Instructions to run wis2-gdc via Docker can be found the [`docker`](docker) directory. +The Docker setup uses Docker and Docker Compose to manage the following services: + +- **wis2-gdc-api**: GDC API powered by [pygeoapi](https://pygeoapi.io) +- **Elasticsearch**: GDC search engine backend +- **wis2-gdc-management**: management service to ingest, validate and publish discovery metadata published from a WIS2 Global Broker instance + - the default Global Broker connection is to Météo-France. This can be modified in `pywis-pubsub.yml` to point to an alternate Global Broker + +See [`wis2-gdc.env`](wis2-gdc.env) for default environment variable settings. + +To adjust service ports, edit [`docker-compose.override.yml`](docker-compose.override.yml) accordingly. + +The [`Makefile`](Makefile) in the root directory provides options to manage the Docker Compose setup. + +```bash +# build all images +make build + +# build all images (no cache) +make force-build + +# start all containers +make up + +# start all containers in dev mode +make dev + +# view all container logs in realtime +make logs + +# login to the wis2-gdc-management container +make login + +# restart all containers +make restart + +# shutdown all containers +make down + +# remove all volumes +make rm +``` ## Development diff --git a/docker/docker-compose.dev.yml b/docker-compose.dev.yml similarity index 100% rename from docker/docker-compose.dev.yml rename to docker-compose.dev.yml diff --git a/docker/docker-compose.override.yml b/docker-compose.override.yml similarity index 93% rename from docker/docker-compose.override.yml rename to docker-compose.override.yml index 3cc7c9f..2765537 100644 --- a/docker/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -20,10 +20,13 @@ ############################################################################### services: - mosquitto: + wis2-gdc-broker: ports: - 1883:1883 # default - 1884:1884 # websockets + wis2-gdc-metrics-collector: + ports: + - 8006:8006 wis2-gdc-api: ports: - 80:80 diff --git a/docker/docker-compose.yml b/docker-compose.yml similarity index 87% rename from docker/docker-compose.yml rename to docker-compose.yml index 104bb39..6fc3cde 100644 --- a/docker/docker-compose.yml +++ b/docker-compose.yml @@ -50,20 +50,32 @@ services: networks: - wis2-gdc-net - mosquitto: - container_name: mosquitto + wis2-gdc-broker: + container_name: wis2-gdc-broker restart: always build: - context: ./mosquitto + context: ./wis2-gdc-broker/ env_file: - wis2-gdc.env networks: - wis2-gdc-net + wis2-gdc-metrics-collector: + container_name: wis2-gdc-metrics-collector + restart: unless-stopped + build: + context: ./wis2-gdc-metrics-collector/ + env_file: + - wis2-gdc.env + networks: + - wis2-gdc-net + depends_on: + - wis2-gdc-broker + wis2-gdc-management: container_name: wis2-gdc-management build: - context: .. + context: ./wis2-gdc-management/ env_file: - wis2-gdc.env environment: @@ -85,7 +97,7 @@ services: wis2-gdc-api: container_name: wis2-gdc-api build: - context: wis2-gdc-api/ + context: ./wis2-gdc-api/ image: geopython/pygeoapi:latest depends_on: wis2-gdc-management: diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index ddb2d30..0000000 --- a/docker/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Docker - -## Overview - -This Docker setup uses Docker and Docker Compose to manage the following services: - -- **pygeoapi**: OGC API - Records metadata catalogue -- **Elasticsearch**: GDC search engine backend -- **wis2-gdc-management**: management service to ingest, validate and publish discovery metadata published from a WIS2 Global Broker instance - - the default Global Broker connection is to Météo-France. This can be modified in `pywis-pubsub.yml` to point to an alternate Global Broker - -See [`wis2-gdc.env`](wis2-gdc.env) for default environment variable settings. - -To adjust service ports, edit [`docker-compose.override.yml`](docker-compose.override.yml) accordingly. - -## Running - -The [`Makefile`](../Makefile) in the root directory provides options to manage the Docker Compose setup. diff --git a/docker/wis2-gdc.env b/docker/wis2-gdc.env deleted file mode 100644 index 8829540..0000000 --- a/docker/wis2-gdc.env +++ /dev/null @@ -1,18 +0,0 @@ -export WIS2_GDC_API_URL=http://localhost -export WIS2_GDC_API_URL_DOCKER=http://wis2-gdc-api -export WIS2_GDC_BACKEND_TYPE=Elasticsearch -export WIS2_GDC_BACKEND_CONNECTION=http://elasticsearch:9200/wis2-discovery-metadata -export WIS2_GDC_BROKER_URL=mqtt://wis2-gdc:wis2-gdc@mosquitto:1883 -export WIS2_GDC_CENTRE_ID=ca-eccc-msc-global-discovery-catalogue -export WIS2_GDC_GB=mqtts://everyone:everyone@globalbroker.meteo.fr:8883 -export WIS2_GDC_GB_TOPIC=origin/a/wis2/+/metadata/# -export WIS2_GDC_OPENMETRICS_FILE=/data/wis2-gdc-metrics.txt -export WIS2_GDC_METADATA_ARCHIVE_ZIPFILE=/data/wis2-gdc-archive.zip -export WIS2_GDC_PUBLISH_REPORTS=true -export WIS2_GDC_REJECT_ON_FAILING_ETS=true -export WIS2_GDC_RUN_KPI=true - -# global broker links -export WIS2_GDC_GB_LINK_METEOFRANCE="Météo-France,mqtts://everyone:everyone@globalbroker.meteo.fr:8883" -export WIS2_GDC_GB_LINK_CMA="China Meteorological Agency,mqtts://everyone:everyone@gb.wis.cma.cn:8883" -export WIS2_GDC_GB_LINK_NOAA="National Oceanic and Atmospheric Administration, National Weather Service, Global Broker Service,mqtts://everyone:everyone@wis2globalbroker.nws.noaa.gov:8883" diff --git a/pywis-pubsub.yml b/pywis-pubsub.yml deleted file mode 100644 index 182af29..0000000 --- a/pywis-pubsub.yml +++ /dev/null @@ -1,11 +0,0 @@ - -broker: ${WIS2_GDC_GB} - -subscribe_topics: - - ${WIS2_GDC_GB_TOPIC} - -verify_data: true - -validate_message: true - -hook: wis2_gdc.hook.DiscoveryMetadataHook diff --git a/docker/wis2-gdc-api/Dockerfile b/wis2-gdc-api/Dockerfile similarity index 100% rename from docker/wis2-gdc-api/Dockerfile rename to wis2-gdc-api/Dockerfile diff --git a/docker/wis2-gdc-api/app.py b/wis2-gdc-api/app.py similarity index 100% rename from docker/wis2-gdc-api/app.py rename to wis2-gdc-api/app.py diff --git a/docker/wis2-gdc-api/entrypoint.sh b/wis2-gdc-api/entrypoint.sh similarity index 100% rename from docker/wis2-gdc-api/entrypoint.sh rename to wis2-gdc-api/entrypoint.sh diff --git a/docker/wis2-gdc-api/wis2-gdc-api.yml b/wis2-gdc-api/wis2-gdc-api.yml similarity index 100% rename from docker/wis2-gdc-api/wis2-gdc-api.yml rename to wis2-gdc-api/wis2-gdc-api.yml diff --git a/docker/mosquitto/Dockerfile b/wis2-gdc-broker/Dockerfile similarity index 100% rename from docker/mosquitto/Dockerfile rename to wis2-gdc-broker/Dockerfile diff --git a/docker/mosquitto/acl.conf b/wis2-gdc-broker/acl.conf similarity index 100% rename from docker/mosquitto/acl.conf rename to wis2-gdc-broker/acl.conf diff --git a/docker/mosquitto/entrypoint.sh b/wis2-gdc-broker/entrypoint.sh similarity index 100% rename from docker/mosquitto/entrypoint.sh rename to wis2-gdc-broker/entrypoint.sh diff --git a/docker/mosquitto/mosquitto.conf b/wis2-gdc-broker/mosquitto.conf similarity index 100% rename from docker/mosquitto/mosquitto.conf rename to wis2-gdc-broker/mosquitto.conf diff --git a/Dockerfile b/wis2-gdc-management/Dockerfile similarity index 100% rename from Dockerfile rename to wis2-gdc-management/Dockerfile diff --git a/MANIFEST.in b/wis2-gdc-management/MANIFEST.in similarity index 100% rename from MANIFEST.in rename to wis2-gdc-management/MANIFEST.in diff --git a/wis2-gdc-management/README.md b/wis2-gdc-management/README.md new file mode 100644 index 0000000..3aded3a --- /dev/null +++ b/wis2-gdc-management/README.md @@ -0,0 +1,3 @@ +# wis2-gdc-management + +Python package to perform WIS2 GDC management functions. diff --git a/docker/entrypoint.sh b/wis2-gdc-management/docker/entrypoint.sh similarity index 100% rename from docker/entrypoint.sh rename to wis2-gdc-management/docker/entrypoint.sh diff --git a/docker/pywis-pubsub.yml b/wis2-gdc-management/docker/pywis-pubsub.yml similarity index 100% rename from docker/pywis-pubsub.yml rename to wis2-gdc-management/docker/pywis-pubsub.yml diff --git a/docker/wis2-gdc-management.cron b/wis2-gdc-management/docker/wis2-gdc-management.cron similarity index 100% rename from docker/wis2-gdc-management.cron rename to wis2-gdc-management/docker/wis2-gdc-management.cron diff --git a/requirements-backend.txt b/wis2-gdc-management/requirements-backend.txt similarity index 100% rename from requirements-backend.txt rename to wis2-gdc-management/requirements-backend.txt diff --git a/requirements-dev.txt b/wis2-gdc-management/requirements-dev.txt similarity index 100% rename from requirements-dev.txt rename to wis2-gdc-management/requirements-dev.txt diff --git a/requirements.txt b/wis2-gdc-management/requirements.txt similarity index 100% rename from requirements.txt rename to wis2-gdc-management/requirements.txt diff --git a/setup.py b/wis2-gdc-management/setup.py similarity index 100% rename from setup.py rename to wis2-gdc-management/setup.py diff --git a/wis2_gdc/__init__.py b/wis2-gdc-management/wis2_gdc/__init__.py similarity index 100% rename from wis2_gdc/__init__.py rename to wis2-gdc-management/wis2_gdc/__init__.py diff --git a/wis2_gdc/archive.py b/wis2-gdc-management/wis2_gdc/archive.py similarity index 100% rename from wis2_gdc/archive.py rename to wis2-gdc-management/wis2_gdc/archive.py diff --git a/wis2_gdc/backend/__init__.py b/wis2-gdc-management/wis2_gdc/backend/__init__.py similarity index 100% rename from wis2_gdc/backend/__init__.py rename to wis2-gdc-management/wis2_gdc/backend/__init__.py diff --git a/wis2_gdc/backend/base.py b/wis2-gdc-management/wis2_gdc/backend/base.py similarity index 100% rename from wis2_gdc/backend/base.py rename to wis2-gdc-management/wis2_gdc/backend/base.py diff --git a/wis2_gdc/backend/elastic.py b/wis2-gdc-management/wis2_gdc/backend/elastic.py similarity index 100% rename from wis2_gdc/backend/elastic.py rename to wis2-gdc-management/wis2_gdc/backend/elastic.py diff --git a/wis2_gdc/backend/ogcapi_records.py b/wis2-gdc-management/wis2_gdc/backend/ogcapi_records.py similarity index 100% rename from wis2_gdc/backend/ogcapi_records.py rename to wis2-gdc-management/wis2_gdc/backend/ogcapi_records.py diff --git a/wis2_gdc/env.py b/wis2-gdc-management/wis2_gdc/env.py similarity index 91% rename from wis2_gdc/env.py rename to wis2-gdc-management/wis2_gdc/env.py index 9a2c3bf..24214f1 100644 --- a/wis2_gdc/env.py +++ b/wis2-gdc-management/wis2_gdc/env.py @@ -51,16 +51,14 @@ def str2bool(value: Any) -> bool: CENTRE_ID = os.environ.get('WIS2_GDC_CENTRE_ID') GB = os.environ.get('WIS2_GDC_GB') GB_TOPIC = os.environ.get('WIS2_GDC_GB_TOPIC') -OPENMETRICS_FILE = os.environ.get('WIS2_GDC_OPENMETRICS_FILE') PUBLISH_REPORTS = str2bool(os.environ.get('WIS2_GDC_PUBLISH_REPORTS', 'false')) REJECT_ON_FAILING_ETS = str2bool(os.environ.get('WIS2_GDC_REJECT_ON_FAILING_ETS', 'true')) # noqa RUN_KPI = str2bool(os.environ.get('WIS2_GDC_RUN_KPI', 'false')) GB_LINKS = [] -if None in [API_URL, API_URL_DOCKER, BACKEND_TYPE, - BACKEND_CONNECTION, BROKER_URL, CENTRE_ID, - GB, GB_TOPIC, OPENMETRICS_FILE]: +if None in [API_URL, API_URL_DOCKER, BACKEND_TYPE, BACKEND_CONNECTION, + BROKER_URL, CENTRE_ID, GB, GB_TOPIC]: raise EnvironmentError('Environment variables not set!') for key, value in os.environ.items(): diff --git a/wis2_gdc/harvester/__init__.py b/wis2-gdc-management/wis2_gdc/harvester/__init__.py similarity index 100% rename from wis2_gdc/harvester/__init__.py rename to wis2-gdc-management/wis2_gdc/harvester/__init__.py diff --git a/wis2_gdc/harvester/base.py b/wis2-gdc-management/wis2_gdc/harvester/base.py similarity index 100% rename from wis2_gdc/harvester/base.py rename to wis2-gdc-management/wis2_gdc/harvester/base.py diff --git a/wis2_gdc/harvester/wis2box.py b/wis2-gdc-management/wis2_gdc/harvester/wis2box.py similarity index 100% rename from wis2_gdc/harvester/wis2box.py rename to wis2-gdc-management/wis2_gdc/harvester/wis2box.py diff --git a/wis2_gdc/hook.py b/wis2-gdc-management/wis2_gdc/hook.py similarity index 100% rename from wis2_gdc/hook.py rename to wis2-gdc-management/wis2_gdc/hook.py diff --git a/wis2_gdc/monitor/__init__.py b/wis2-gdc-management/wis2_gdc/monitor/__init__.py similarity index 100% rename from wis2_gdc/monitor/__init__.py rename to wis2-gdc-management/wis2_gdc/monitor/__init__.py diff --git a/wis2_gdc/registrar.py b/wis2-gdc-management/wis2_gdc/registrar.py similarity index 93% rename from wis2_gdc/registrar.py rename to wis2-gdc-management/wis2_gdc/registrar.py index 7204bba..8a2a53f 100644 --- a/wis2_gdc/registrar.py +++ b/wis2-gdc-management/wis2_gdc/registrar.py @@ -36,7 +36,6 @@ from wis2_gdc.env import (BACKEND_TYPE, BACKEND_CONNECTION, BROKER_URL, CENTRE_ID, GB_LINKS, PUBLISH_REPORTS, REJECT_ON_FAILING_ETS, RUN_KPI) -from wis2_gdc.monitor.metrics import Metrics LOGGER = logging.getLogger(__name__) @@ -51,7 +50,6 @@ def __init__(self): self.broker = None self.metadata = None - self.metrics = Metrics() if PUBLISH_REPORTS: self.broker = MQTTPubSubClient(BROKER_URL) @@ -108,19 +106,18 @@ def register(self, metadata: dict) -> None: LOGGER.warning('ETS errors; metadata not published') return except KeyError: # validation error - self.metrics.failed_total.labels(*centre_id_labels).inc() LOGGER.debug('Validation errors; metadata not published') - self.metrics.write() + self.broker.pub('wis2-gdc/metrics/failed_total', + json.dumps(centre_id_labels)) return - self.metrics.passed_total.labels(*centre_id_labels).inc() + self.broker.pub('wis2-gdc/metrics/passed_total', + json.dumps(centre_id_labels)) - # TODO: remove following wis2box b7 updates data_policy = self.metadata['properties']['wmo:dataPolicy'] - if data_policy == 'core': - self.metrics.core_total.labels(*centre_id_labels).inc() - elif data_policy == 'recommended': - self.metrics.recommended_total.labels(*centre_id_labels).inc() + + self.broker.pub(f'wis2-gdc/metrics/{data_policy}_total', + json.dumps(centre_id_labels)) LOGGER.info('Updating links') self.update_record_links() @@ -138,8 +135,6 @@ def register(self, metadata: dict) -> None: LOGGER.info('Publishing KPI report to broker') self.broker.pub(topic, json.dumps(kpi_results)) - self.metrics.write() - def _run_ets(self) -> dict: """ Helper function to run ETS diff --git a/wis2_gdc/sync.py b/wis2-gdc-management/wis2_gdc/sync.py similarity index 100% rename from wis2_gdc/sync.py rename to wis2-gdc-management/wis2_gdc/sync.py diff --git a/wis2-gdc-metrics-collector/Dockerfile b/wis2-gdc-metrics-collector/Dockerfile new file mode 100644 index 0000000..1919fb6 --- /dev/null +++ b/wis2-gdc-metrics-collector/Dockerfile @@ -0,0 +1,29 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +FROM python:3-alpine + +COPY . /app +RUN pip install -r /app/requirements.txt + +ENV PYTHONUNBUFFERED="true" + +ENTRYPOINT [ "python3","-u","/app/metrics_collector.py" ] diff --git a/wis2-gdc-metrics-collector/metrics_collector.py b/wis2-gdc-metrics-collector/metrics_collector.py new file mode 100644 index 0000000..a31969b --- /dev/null +++ b/wis2-gdc-metrics-collector/metrics_collector.py @@ -0,0 +1,165 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +import json +import logging +import os +import sys +from urllib.parse import urlparse + +import paho.mqtt.client as mqtt_client +from prometheus_client import ( + Counter, Gauge, Info, start_http_server, REGISTRY, GC_COLLECTOR, + PLATFORM_COLLECTOR, PROCESS_COLLECTOR +) + +REGISTRY.unregister(GC_COLLECTOR) +REGISTRY.unregister(PLATFORM_COLLECTOR) +REGISTRY.unregister(PROCESS_COLLECTOR) + +API_URL = os.environ['WIS2_GDC_API_URL'] +BROKER_URL = os.environ['WIS2_GDC_BROKER_URL'] +CENTRE_ID = os.environ['WIS2_GDC_CENTRE_ID'] +GB = urlparse(os.environ['WIS2_GDC_GB']) +GB_TOPIC = os.environ['WIS2_GDC_GB_TOPIC'] +HTTP_PORT = 8006 +LOGGING_LEVEL = os.environ['WIS2_GDC_LOGGING_LEVEL'] + +logging.basicConfig(stream=sys.stdout) +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(LOGGING_LEVEL) + +# sets metrics as per https://github.com/wmo-im/wis2-metric-hierarchy/blob/main/metric-hierarchy/gdc.csv # noqa + +METRIC_INFO = Info( + 'wis2_gdc_metrics', + 'WIS2 GDC metrics' +) + +METRIC_PASSED_TOTAL = Counter( + 'wmo_wis2_gdc_passed_total', + 'Number of metadata records passed validation', + ['centre_id', 'report_by'] +) + +METRIC_FAILED_TOTAL = Counter( + 'wmo_wis2_gdc_failed_total', + 'Number of metadata records failed validation', + ['centre_id', 'report_by'] +) + +METRIC_CORE_TOTAL = Counter( + 'wmo_wis2_gdc_core_total', + 'Number of core metadata records', + ['centre_id', 'report_by'] +) + +METRIC_RECOMMENDED_TOTAL = Counter( + 'wmo_wis2_gdc_recommendedcore_total', + 'Number of recommended metadata records', + ['centre_id', 'report_by'] +) + +METRIC_KPI_PERCENTAGE_TOTAL = Counter( + 'wmo_wis2_gdc_kpi_percentage_total', + 'KPI percentage for a single metadata record (metadata_id equals WCMP2 id)', # noqa + ['metadata_id', 'centre_id', 'report_by'] +) + +METRIC_KPI_PERCENTAGE_AVERAGE = Gauge( + 'wmo_wis2_gdc_kpi_percentage_average', + 'Average KPI percentage', + ['centre_id', 'report_by'] +) + +METRIC_KPI_PERCENTAGE_OVER80_TOTAL = Counter( + 'wmo_wis2_gdc_kpi_percentage_over80_total', + 'Number of metadata records with KPI percentage over 80', + ['centre_id', 'report_by'] +) + +METRIC_SEARCH_TOTAL = Counter( + 'wmo_wis2_gdc_search_total', + 'Number of search requests (during last monitoring period)', + ['centre_id', 'report_by'] +) + +METRIC_SEARCH_TERMS = Gauge( + 'wmo_wis2_gdc_search_terms', + 'Most popular search terms (e.g. top=1 to top=5)', + ['top', 'centre_id', 'report_by'] +) + +METRIC_INFO.info({ + 'centre-id': CENTRE_ID, + 'url': API_URL, + 'subscribed-to': f'{GB.scheme}://{GB.hostname}:{GB.port} (topic: {GB_TOPIC})' # noqa +}) + + +def collect_metrics() -> None: + """ + Subscribe to MQTT wis2-gdc/metrics and collect metrics + + :returns: `None` + """ + + def _sub_connect(client, userdata, flags, rc): + LOGGER.info('Subscribing to topic wis2-gdc/metrics/#') + client.subscribe('wis2-gdc/metrics/#', qos=0) + + def _sub_message(client, userdata, msg): + LOGGER.debug('Processing message') + topic = msg.topic + labels = json.loads(msg.payload) + LOGGER.debug(f'Topic: {topic}') + LOGGER.debug(f'Labels: {labels}') + + if topic == 'wis2-gdc/metrics/passed_total': + METRIC_PASSED_TOTAL.labels(*labels).inc() + if topic == 'wis2-gdc/metrics/failed_total': + METRIC_FAILED_TOTAL.labels(*labels).inc() + elif topic == 'wis2-gdc/metrics/core_total': + METRIC_CORE_TOTAL.labels(*labels).inc() + elif topic == 'wis2-gdc/metrics/recommended_total': + METRIC_RECOMMENDED_TOTAL.labels(*labels).inc() + + url = urlparse(BROKER_URL) + + client_id = 'wis2-gdc metrics collector' + + try: + LOGGER.info('Setting up MQTT client') + client = mqtt_client.Client(client_id) + client.on_connect = _sub_connect + client.on_message = _sub_message + client.username_pw_set(url.username, url.password) + LOGGER.info(f'Connecting to {url.hostname}') + client.connect(url.hostname, url.port) + client.loop_forever() + except Exception as err: + LOGGER.error(err) + + +if __name__ == '__main__': + LOGGER.info(f'Starting metrics collector server on port {HTTP_PORT}') + start_http_server(HTTP_PORT) + collect_metrics() diff --git a/wis2-gdc-metrics-collector/requirements.txt b/wis2-gdc-metrics-collector/requirements.txt new file mode 100644 index 0000000..205a71f --- /dev/null +++ b/wis2-gdc-metrics-collector/requirements.txt @@ -0,0 +1,3 @@ +paho-mqtt<2 +prometheus-client +requests diff --git a/wis2-gdc.env b/wis2-gdc.env index fbf4cb0..0c8dc1f 100644 --- a/wis2-gdc.env +++ b/wis2-gdc.env @@ -1,15 +1,15 @@ +export WIS2_GDC_LOGGING_LEVEL=DEBUG export WIS2_GDC_API_URL=http://localhost export WIS2_GDC_API_URL_DOCKER=http://wis2-gdc-api export WIS2_GDC_BACKEND_TYPE=Elasticsearch export WIS2_GDC_BACKEND_CONNECTION=http://elasticsearch:9200/wis2-discovery-metadata -export WIS2_GDC_BROKER_URL=mqtt://wis2-gdc:wis2-gdc@mosquitto:1883 +export WIS2_GDC_BROKER_URL=mqtt://wis2-gdc:wis2-gdc@wis2-gdc-broker:1883 export WIS2_GDC_CENTRE_ID=ca-eccc-msc-global-discovery-catalogue -export WIS2_GDC_GB=mqtt://everyone:everyone@localhost:1883 +export WIS2_GDC_GB=mqtts://everyone:everyone@globalbroker.meteo.fr:8883 export WIS2_GDC_GB_TOPIC=origin/a/wis2/+/metadata/# -export WIS2_GDC_OPENMETRICS_FILE=/tmp/wis2-gdc-metrics.txt -export WIS2_GDC_METADATA_ARCHIVE_ZIPFILE=/tmp/wis2-gdc-archive.zip +export WIS2_GDC_METADATA_ARCHIVE_ZIPFILE=/data/wis2-gdc-archive.zip export WIS2_GDC_PUBLISH_REPORTS=true -export WIS2_GDC_REJECT_ON_FAILING_ETS=true +export WIS2_GDC_REJECT_ON_FAILING_ETS=false export WIS2_GDC_RUN_KPI=true # global broker links diff --git a/wis2_gdc/monitor/metrics.py b/wis2_gdc/monitor/metrics.py deleted file mode 100644 index 5146e15..0000000 --- a/wis2_gdc/monitor/metrics.py +++ /dev/null @@ -1,160 +0,0 @@ -############################################################################### -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -############################################################################### - -import logging -from time import time -from typing import Union - -import prometheus_client -from prometheus_client import CollectorRegistry, Gauge, Info, write_to_textfile - -from wis2_gdc.env import API_URL, CENTRE_ID, GB, GB_TOPIC, OPENMETRICS_FILE - -prometheus_client.REGISTRY.unregister(prometheus_client.GC_COLLECTOR) -prometheus_client.REGISTRY.unregister(prometheus_client.PLATFORM_COLLECTOR) -prometheus_client.REGISTRY.unregister(prometheus_client.PROCESS_COLLECTOR) - -LOGGER = logging.getLogger(__name__) - -# sets metrics as per https://github.com/wmo-im/wis2-metric-hierarchy/blob/main/metric-hierarchy/gdc.csv # noqa - - -class Metrics: - def __init__(self): - self.registry = CollectorRegistry() - - self.info = Info( - 'wis2_gdc_metrics', - 'WIS2 GDC metrics', - registry=self.registry - ) - - self.passed_total = Gauge( - 'wmo_wis2_gdc_passed_total', - 'Number of metadata records passed validation', - ['centre_id', 'report_by'], - registry=self.registry - ) - - self.failed_total = Gauge( - 'wmo_wis2_gdc_failed_total', - 'Number of metadata records failed validation', - ['centre_id', 'report_by'], - registry=self.registry - ) - - self.core_total = Gauge( - 'wmo_wis2_gdc_core_total', - 'Number of core metadata records', - ['centre_id', 'report_by'], - registry=self.registry - ) - - self.recommended_total = Gauge( - 'wmo_wis2_gdc_recommendedcore_total', - 'Number of recommended metadata records', - ['centre_id', 'report_by'], - registry=self.registry - ) - - self.kpi_percentage_total = Gauge( - 'wmo_wis2_gdc_kpi_percentage_total', - 'KPI percentage for a single metadata record (metadata_id equals WCMP2 id)', # noqa - ['metadata_id', 'centre_id', 'report_by'], - registry=self.registry - ) - - self.kpi_percentage_average = Gauge( - 'wmo_wis2_gdc_kpi_percentage_average', - 'Average KPI percentage', - ['centre_id', 'report_by'], - registry=self.registry - ) - - self.kpi_percentage_over80_total = Gauge( - 'wmo_wis2_gdc_kpi_percentage_over80_total', - 'Number of metadata records with KPI percentage over 80', - ['centre_id', 'report_by'], - registry=self.registry - ) - - self.search_total = Gauge( - 'wmo_wis2_gdc_search_total', - 'Number of search requests (during last monitoring period)', - ['centre_id', 'report_by'], - registry=self.registry - ) - - self.search_terms = Gauge( - 'wmo_wis2_gdc_search_terms', - 'Most popular search terms (e.g. top=1 to top=5)', - ['top', 'centre_id', 'report_by'], - registry=self.registry - ) - - self.info.info({ - 'centre-id': CENTRE_ID, - 'url': API_URL, - 'subscribed-to': f'{GB}/{GB_TOPIC}' - }) - - def inc(self, metric: str, labels: list) -> None: - """ - Convenience function to increment a value for a given label set - - :param metric: `str` of metric name - :param labels: `list` of label set - - :returns: `None` - """ - - getattr(self, metric).labels(*labels).inc() - getattr(self, metric).labels(*labels)._timestamp = time() - - def set(self, metric: str, labels: list, - value: Union[int, float, str]) -> None: - """ - Convenience function to set a value for a given label set - - :param metric: `str` of metric name - :param labels: `list` of label set - :param value: literal of value - - :returns: `None` - """ - - getattr(self, metric).labels(*labels).set(value) - getattr(self, metric).labels(*labels)._timestamp = time() - - def write(self) -> None: - """ - Writes OpenMetrics to file - - :returns: `None` - """ - - write_to_textfile(OPENMETRICS_FILE, self.registry) - - def __exit__(self, exc_type, exc_value, traceback): - LOGGER.debug('Exiting') - - def __repr__(self): - return ''