diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index 36f926c9..dcdaf6eb 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -36,16 +36,16 @@ jobs: mkdir -p tests/data/downloads chown -R $(whoami):docker tests/data/downloads chmod -R 775 tests/data/downloads - - name: setup wis2box configuration, replace localhost with IP on host 📦 + - name: setup wis2box configuration, replace localhost with IP on host, use LOCAL_BUILD as wis2box.version 📦 run: | export IP=$(hostname -I | awk '{print $1}') cp tests/test.env wis2box.env + echo "LOCAL_BUILD" > wis2box.version sed -i "s/localhost/$IP/g" wis2box.env cat wis2box.env python3 wis2box-ctl.py config - - name: build wis2box + - name: build wis2box locally ⚙️ run: | - python3 wis2box-ctl.py build python3 wis2box-ctl.py update - name: start containers ⚙️ run: | diff --git a/.github/workflows/zaproxy.yml b/.github/workflows/zaproxy.yml index a8796082..a016435f 100644 --- a/.github/workflows/zaproxy.yml +++ b/.github/workflows/zaproxy.yml @@ -11,7 +11,7 @@ jobs: - name: build and start containers using tests/test.env ⚙️ run: | cp tests/test.env wis2box.env - python3 wis2box-ctl.py build + python3 wis2box-ctl.py update python3 wis2box-ctl.py start python3 wis2box-ctl.py status -a sleep 30 diff --git a/.gitignore b/.gitignore index 20bf19c9..e24b1508 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,8 @@ wis2box.env docker/.env .ipynb_checkpoints +tests/data/.ssh/id_rsa +tests/data/.ssh/id_rsa.pub + +docker-compose.yml +docker-compose.yml.bak diff --git a/.zap/rules.tsv b/.zap/rules.tsv index fcdad969..dc50cafa 100644 --- a/.zap/rules.tsv +++ b/.zap/rules.tsv @@ -22,3 +22,4 @@ 10110 IGNORE Dangerous JS Functions Low 10105 IGNORE Authentication Credentials Captured Medium 10003 IGNORE Vulnerable JS Library Medium +90004 IGNORE Insufficient Site Isolation Against Spectre Vulnerability Low diff --git a/docker-compose.yml b/docker-compose.base.yml similarity index 90% rename from docker-compose.yml rename to docker-compose.base.yml index f45a6684..8951accb 100644 --- a/docker-compose.yml +++ b/docker-compose.base.yml @@ -104,10 +104,8 @@ services: mosquitto: container_name: mosquitto - #image: ghcr.io/wmo-im/wis2box-broker:latest + image: ghcr.io/wmo-im/wis2box-broker:latest restart: always - build: - context: ./wis2box-broker env_file: - wis2box.env volumes: @@ -115,13 +113,10 @@ services: wis2box-management: container_name: wis2box-management + image: ghcr.io/wmo-im/wis2box-management:latest mem_limit: 1g memswap_limit: 1g restart: always - #image: ghcr.io/wmo-im/wis2box-management:latest - build: - context: ./wis2box-management - #user: wis2box:wis2box env_file: - wis2box.env volumes: @@ -134,6 +129,19 @@ services: condition: service_healthy command: ["wis2box", "pubsub" , "subscribe"] + # mqtt_metrics_collector, listens to mqtt-broker + mqtt_metrics_collector: + container_name: mqtt_metrics_collector + restart: unless-stopped + env_file: + - wis2box.env + image: ghcr.io/wmo-im/wis2box-mqtt-metrics-collector:latest + depends_on: + - mosquitto + - wis2box-management + ports: + - 8001:8001 + wis2box-auth: container_name: wis2box-auth image: ghcr.io/wmo-im/wis2box-auth:latest diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml index d4291023..eade2ea8 100644 --- a/docker-compose.monitoring.yml +++ b/docker-compose.monitoring.yml @@ -34,22 +34,6 @@ services: vpcbr: # this is the place where we assign the static ipv4 address ipv4_address: 10.5.0.2 default: - - # mqtt_metrics_collector, listens to mqtt-broker - mqtt_metrics_collector: - <<: *logging - container_name: mqtt_metrics_collector - restart: unless-stopped - env_file: - - wis2box.env - #image: ghcr.io/wmo-im/wis2box-mqtt-metrics-collector:latest - build: - context: ./wis2box-mqtt-metrics-collector - depends_on: - - mosquitto - - wis2box-management - ports: - - 8001:8001 # prometheus to collect metrics prometheus: @@ -129,6 +113,8 @@ services: <<: *logging wis2box-auth: <<: *logging + mqtt_metrics_collector: + <<: *logging minio: <<: *logging web-proxy: diff --git a/docs/source/reference/quickstart.rst b/docs/source/reference/quickstart.rst index da6c6c05..d9236571 100644 --- a/docs/source/reference/quickstart.rst +++ b/docs/source/reference/quickstart.rst @@ -24,12 +24,11 @@ To run with the 'quickstart' configuration, copy this file to ``wis2box.env`` in cp tests/test.env wis2box.env -Build and update wis2box: +Build and update wis2box from the source code: .. code-block:: bash - python3 wis2box-ctl.py build - python3 wis2box-ctl.py update + python3 wis2box-ctl.py update-local-build Start wis2box and login to the wis2box-management container: diff --git a/wis2box-ctl.py b/wis2box-ctl.py index 070a8d19..1405bfd2 100755 --- a/wis2box-ctl.py +++ b/wis2box-ctl.py @@ -22,7 +22,17 @@ import argparse import os +import requests import subprocess +import shutil + +from packaging.version import Version + +# read wis2box.version file if it exists +WIS2BOX_VERSION = 'LOCAL_BUILD' +if os.path.exists('wis2box.version'): + with open('wis2box.version', 'r') as f: + WIS2BOX_VERSION = f.read().strip() if subprocess.call(['docker', 'compose'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) > 0: DOCKER_COMPOSE_COMMAND = 'docker-compose' @@ -38,25 +48,17 @@ """ parser = argparse.ArgumentParser( - description='manage a compposition of docker containers to implement a wis 2 box', - formatter_class=argparse.RawTextHelpFormatter) - -parser.add_argument( - '--ssl', - dest='ssl', - action='store_true', - help='run wis2box with SSL enabled') + description='Manage a composition of Docker containers to implement wis2box', + formatter_class=argparse.RawTextHelpFormatter +) -parser.add_argument( - '--simulate', - dest='simulate', - action='store_true', - help='simulate execution by printing action rather than executing') +parser.add_argument('--simulate', + action='store_true', + help='Simulate execution by printing action rather than executing') commands = [ 'build', 'config', - 'down', 'execute', 'lint', 'logs', @@ -67,31 +69,181 @@ 'start-dev', 'status', 'stop', - 'up', - 'update', + 'update' ] parser.add_argument('command', choices=commands, - help=""" + help="""The command to execute: - config: validate and view Docker configuration - - build [containers]: build all services - - start [containers]: start system - - start-dev [containers]: start system in local development mode - - login [container]: login to the container (default: wis2box-management) - - login-root [container]: login to the container as root - - stop: stop [container] system + - build: build all services + - start: start system + - start-dev: start system in local development mode + - login: login to the container (default: wis2box-management) + - stop: stop system - update: update Docker images - prune: cleanup dangling containers and images - - restart [containers]: restart one or all containers - - status [containers|-a]: view status of wis2box containers + - restart: restart containers + - status: view status of wis2box containers - lint: run PEP8 checks against local Python code """) -parser.add_argument('args', nargs=argparse.REMAINDER) +parser.add_argument('args', nargs=argparse.REMAINDER, help='Additional arguments for the command') args = parser.parse_args() +LOCAL_IMAGES = [ + 'ghcr.io/wmo-im/wis2box-management', + 'ghcr.io/wmo-im/wis2box-broker', + 'ghcr.io/wmo-im/wis2box-mqtt-metrics-collector' +] + +def remove_docker_images(filter: str) -> None: + # Get the IDs of images matching the filter + result = subprocess.run( + ['docker', 'images', '--filter', f'reference={filter}', '-q', '--no-trunc'], + capture_output=True, + text=True + ) + + image_ids = result.stdout.strip() + if image_ids: # If there are images to remove + for image_id in image_ids.splitlines(): + try: + subprocess.run(['docker', 'rmi', image_id], check=True, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) + except subprocess.CalledProcessError as e: + # do nothing + pass + +def build_local_images() -> None: + """ + Build local images + + :returns: None. + """ + + for image in LOCAL_IMAGES: + print(f'Building {image}') + context = image.split('/')[-1] + run(split(f'docker build -t {image}:local {context}')) + return None + +def get_resolved_version(base_version: str) -> str: + """ + Fetches the latest release tag for the wis2box-images repository. + + :rbase_version: required, string. The major release version. + + :return: The latest release tag or an error message if not found. + """ + + # NOTE using maaikelimper/wis2box-images for demo purposes, should be wmo-im/wis2box-images + url = f'https://api.github.com/repos/maaikelimper/wis2box-images/releases' + headers = {'Accept': 'application/vnd.github.v3+json'} + + options = [] + try: + response = requests.get(url, headers=headers) + if response.status_code == 200: + releases = response.json() + for release in releases: + if base_version in release['tag_name']: + options.append(release['tag_name']) + else: + print(f'Error fetching latest release tag for {base_version}: {response.status_code}') + except requests.exceptions.RequestException as e: + print(f'Error fetching latest release tag for {base_version}: {e}') + + # throw error if options is empty + if not options: + raise ValueError(f'No wis2box-release found matching wis2box-version={base_version}') + + # Use semantic versioning for sorting + sorted_versions = sorted(options, key=lambda v: Version(v), reverse=True) + + resolved_version = sorted_versions[0] + return resolved_version + +def get_latest_release_tag(image: str, resolved_version: str = '') -> str: + """ + + Fetches the latest release tag for a GitHub repository. + + :param image: required, string. The name of the image repository. + :param base_version: required, string. The major release version. + + :return: The latest release tag or an error message if not found. + """ + + # look up the image tag for the release by downloading the wis2box-images.json file + # NOTE using maaikelimper/wis2box-images for demo purposes, should be wmo-im/wis2box-images + github_repo = 'maaikelimper/wis2box-images' + + url = f'https://github.com/{github_repo}/releases/download/{resolved_version}/wis2box-images.json' + if resolved_version == 'LOCAL_BUILD': + # use the version of images in main branch + url = f'https://raw.githubusercontent.com/{github_repo}/refs/heads/main/wis2box-images.json' + try: + response = requests.get(url) + if response.status_code == 200: + images = response.json() + if image in images: + return images[image] + else: + print(f'Error fetching image tag for {image} from {url}: {response.status_code}') + except requests.exceptions.RequestException as e: + print(f'Error fetching image tag for {image} from {url}: {e}') + + # throw error if image tag is not found + raise ValueError(f'No image tag found for {image} in {url}') + +def update_docker_images(base_version: str) -> None: + """ + Write docker-compose.yml using docker-compose.base.yml as base + + + :param base_version: required, string. The version of wis2box to use. + + :returns: None. + """ + + if os.path.exists('docker-compose.yml'): + print('Backing up current docker-compose.yml to docker-compose.yml.bak') + shutil.copy('docker-compose.yml', 'docker-compose.yml.bak') + + if base_version == 'LOCAL_BUILD': + print('Building local images') + build_local_images() + + resolved_version = base_version + if base_version != 'LOCAL_BUILD': + resolved_version = get_resolved_version(base_version) + + print(f'Updating docker-compose.yml, using base_version={resolved_version}') + + with open('docker-compose.base.yml', 'r') as f: + lines = f.readlines() + with open('docker-compose.yml', 'w') as f: + for line in lines: + if 'image: ' in line: + image = line.split('image: ')[1].split(':')[0] + tag = '' + if image in LOCAL_IMAGES and base_version == 'LOCAL_BUILD': + tag = 'local' + else: + tag = get_latest_release_tag(image, resolved_version) + # pull the image if it is not local + if tag != 'local': + print(f'Pulling {image}:{tag}') + # pull the latest tag for the image + run(split(f'docker pull {image}:{tag}')) + # update the image tag in the docker-compose.yml + print(f'Set {image} to {tag}') + f.write(f' image: {image}:{tag}\n') + else: + f.write(line) + print('docker-compose.yml updated') + return None def split(value: str) -> list: """ @@ -155,11 +307,8 @@ def make(args) -> None: if 'WIS2BOX_SSL_CERT' in line: ssl_cert = line.split('=')[1].strip() docker_compose_args = DOCKER_COMPOSE_ARGS - if args.ssl or (ssl_key and ssl_cert): + if (ssl_key and ssl_cert): docker_compose_args +=" --file docker-compose.ssl.yml" - if args.ssl and not (ssl_key and ssl_cert): - print("ERROR: SSL is enabled but WIS2BOX_SSL_KEY and WIS2BOX_SSL_CERT are not set in wis2box.env") - exit(1) # if you selected a bunch of them, default to all containers = "" if not args.args else ' '.join(args.args) @@ -168,10 +317,9 @@ def make(args) -> None: if args.command == "config": run(split(f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} config')) - elif args.command == "build": - run(split( - f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} build {containers}')) elif args.command in ["up", "start", "start-dev"]: + if not os.path.exists('docker-compose.yml'): + update_docker_images() run(split( 'docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions'), silence_stderr=True) @@ -183,6 +331,17 @@ def make(args) -> None: run(split(f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} --file docker-compose.dev.yml up -d')) else: run(split(f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} up -d')) + elif args.command in ["update"]: + update_docker_images(WIS2BOX_VERSION) + # restart all containers + run(split( + f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} down --remove-orphans')) + run(split( + f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} up -d')) + # perform cleanup of images after update, unless updating local build + if not WIS2BOX_VERSION == 'LOCAL_BUILD': + remove_docker_images('wmoim/wis2box*') + remove_docker_images('ghcr.io/wmo-im/wis2box*') elif args.command == "execute": run(['docker', 'exec', '-i', 'wis2box-management', 'sh', '-c', containers]) elif args.command == "login": @@ -198,16 +357,14 @@ def make(args) -> None: else: run(split( f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} down --remove-orphans {containers}')) - elif args.command == "update": - run(split(f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} pull')) elif args.command == "prune": run(split('docker builder prune -f')) run(split('docker container prune -f')) run( split('docker volume prune -f')) - _ = run(split('docker images --filter dangling=true -q --no-trunc')) - run(split(f'docker rmi {_}')) - _ = run(split('docker ps -a -q')) - run(split(f'docker rm {_}')) + # prune any unused images starting with wmoim/wis2box + remove_docker_images('wmoim/wis2box*') + # prune any unused images starting with ghcr.io/wmo-im/wis2box + remove_docker_images('ghcr.io/wmo-im/wis2box*') elif args.command == "restart": if containers: run(split( diff --git a/wis2box.version b/wis2box.version new file mode 100644 index 00000000..452a44d4 --- /dev/null +++ b/wis2box.version @@ -0,0 +1 @@ +LOCAL_BUILD \ No newline at end of file