diff --git a/.ci/tests/examples/configure.sh b/.ci/tests/examples/configure.sh index 0400c8847..ff36dfe36 100755 --- a/.ci/tests/examples/configure.sh +++ b/.ci/tests/examples/configure.sh @@ -14,15 +14,8 @@ pushd "examples/$example" # If example equals "mnist-keras" if [ "$example" == "mnist-keras" ]; then - bin/init_venv.sh - >&2 echo "Download and prepare data" bin/get_data bin/split_data fi -popd - -pushd "examples/$example" ->&2 echo "Build compute package and seed" -bin/build.sh popd \ No newline at end of file diff --git a/.ci/tests/examples/run.sh b/.ci/tests/examples/run.sh index 38bd4bf1c..a9cd4c89b 100755 --- a/.ci/tests/examples/run.sh +++ b/.ci/tests/examples/run.sh @@ -16,6 +16,8 @@ pip install ./fedn/ fire >&2 echo "Start FEDn" pushd "examples/$example" +fedn package create --path client + docker compose \ -f ../../docker-compose.yaml \ -f docker-compose.override.yaml \ diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 73469da36..ffc5821e1 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -29,11 +29,6 @@ jobs: - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python_version }} - - - name: configure ${{ matrix.to_test }} - run: | - echo 'BASE_IMG=python:${{ matrix.python_version }}-slim' > .env - .ci/tests/examples/configure.sh ${{ matrix.to_test }} - name: run ${{ matrix.to_test }} run: .ci/tests/examples/run.sh ${{ matrix.to_test }} diff --git a/examples/mnist-keras/bin/build.sh b/examples/mnist-keras/bin/build.sh deleted file mode 100755 index 44eda61df..000000000 --- a/examples/mnist-keras/bin/build.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -e - -# Init seed -client/entrypoint init_seed - -# Make compute package -tar -czvf package.tgz client diff --git a/examples/mnist-keras/bin/get_data b/examples/mnist-keras/bin/get_data deleted file mode 100755 index 4c449d03e..000000000 --- a/examples/mnist-keras/bin/get_data +++ /dev/null @@ -1,21 +0,0 @@ -#!./.mnist-keras/bin/python -import os - -import fire -import numpy as np -import tensorflow as tf - - -def get_data(out_dir='data'): - # Make dir if necessary - if not os.path.exists(out_dir): - os.mkdir(out_dir) - - # Download data - (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() - np.savez(f'{out_dir}/mnist.npz', x_train=x_train, - y_train=y_train, x_test=x_test, y_test=y_test) - - -if __name__ == '__main__': - fire.Fire(get_data) diff --git a/examples/mnist-keras/bin/init_venv.sh b/examples/mnist-keras/bin/init_venv.sh deleted file mode 100755 index 6856700a9..000000000 --- a/examples/mnist-keras/bin/init_venv.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -e - -# Init venv -python3 -m venv .mnist-keras - -# Pip deps -.mnist-keras/bin/pip install --upgrade pip -.mnist-keras/bin/pip install -e ../../fedn -.mnist-keras/bin/pip install -r requirements.txt \ No newline at end of file diff --git a/examples/mnist-keras/bin/init_venv_macm1.sh b/examples/mnist-keras/bin/init_venv_macm1.sh deleted file mode 100755 index d60f602d5..000000000 --- a/examples/mnist-keras/bin/init_venv_macm1.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -e - -# Init venv -python3 -m venv .mnist-keras - -# Pip deps -.mnist-keras/bin/pip install --upgrade pip -.mnist-keras/bin/pip install -e ../../fedn -.mnist-keras/bin/pip install -r requirements-macos.txt diff --git a/examples/mnist-keras/client/entrypoint b/examples/mnist-keras/client/entrypoint.py similarity index 93% rename from examples/mnist-keras/client/entrypoint rename to examples/mnist-keras/client/entrypoint.py index f0149fad6..37b66ebfc 100755 --- a/examples/mnist-keras/client/entrypoint +++ b/examples/mnist-keras/client/entrypoint.py @@ -1,8 +1,6 @@ -#!./.mnist-keras/bin/python import json import os -import docker import fire import numpy as np import tensorflow as tf @@ -16,12 +14,9 @@ def _get_data_path(): - # Figure out FEDn client number from container name - client = docker.from_env() - container = client.containers.get(os.environ['HOSTNAME']) - number = container.name[-1] - # Return data path - return f"/var/data/clients/{number}/mnist.npz" + data_path = os.environ.get('FEDN_DATA_PATH', '/var/data/clients/1/mnist.npz') + + return data_path def compile_model(img_rows=28, img_cols=28): @@ -74,7 +69,7 @@ def load_data(data_path, is_train=True): return X, y -def init_seed(out_path='seed.npz'): +def init_seed(out_path='../seed.npz'): """ Initialize seed model and save it to file. :param out_path: The path to save the seed model to. diff --git a/examples/mnist-keras/client/fedn.yaml b/examples/mnist-keras/client/fedn.yaml index 91ec40c2a..403adfb48 100644 --- a/examples/mnist-keras/client/fedn.yaml +++ b/examples/mnist-keras/client/fedn.yaml @@ -1,7 +1,12 @@ +python_env: python_env.yaml entry_points: + build: + command: python entrypoint.py init_seed + startup: + command: python get_data.py train: - command: /venv/bin/python entrypoint train $ENTRYPOINT_OPTS + command: python entrypoint.py train $ENTRYPOINT_OPTS validate: - command: /venv/bin/python entrypoint validate $ENTRYPOINT_OPTS + command: python entrypoint.py validate $ENTRYPOINT_OPTS infer: - command: /venv/bin/python entrypoint infer $ENTRYPOINT_OPTS + command: python entrypoint.py infer $ENTRYPOINT_OPTS diff --git a/examples/mnist-keras/bin/split_data b/examples/mnist-keras/client/get_data.py similarity index 72% rename from examples/mnist-keras/bin/split_data rename to examples/mnist-keras/client/get_data.py index bb583b6d7..28a12bd20 100755 --- a/examples/mnist-keras/bin/split_data +++ b/examples/mnist-keras/client/get_data.py @@ -1,9 +1,8 @@ -#!./.mnist-keras/bin/python import os from math import floor -import fire import numpy as np +import tensorflow as tf def splitset(dataset, parts): @@ -38,5 +37,17 @@ def split(dataset='data/mnist.npz', outdir='data', n_splits=2): y_test=data['y_test'][i]) +def get_data(out_dir='data'): + # Make dir if necessary + if not os.path.exists(out_dir): + os.mkdir(out_dir) + + # Download data + (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() + np.savez(f'{out_dir}/mnist.npz', x_train=x_train, + y_train=y_train, x_test=x_test, y_test=y_test) + + if __name__ == '__main__': - fire.Fire(split) + get_data() + split() diff --git a/examples/mnist-keras/client/python_env.yaml b/examples/mnist-keras/client/python_env.yaml new file mode 100644 index 000000000..4910e56dd --- /dev/null +++ b/examples/mnist-keras/client/python_env.yaml @@ -0,0 +1,9 @@ +name: mnist-keras +build_dependencies: + - pip + - setuptools + - wheel==0.37.1 +dependencies: + - tensorflow==2.13.1 + - fire==0.3.1 + - fedn==0.8.0 diff --git a/examples/mnist-keras/docker-compose.override.yaml b/examples/mnist-keras/docker-compose.override.yaml index 466a68d7b..a7bcaa5f4 100644 --- a/examples/mnist-keras/docker-compose.override.yaml +++ b/examples/mnist-keras/docker-compose.override.yaml @@ -4,9 +4,6 @@ version: '3.3' # Overriding requirements services: client: - build: - args: - REQUIREMENTS: examples/mnist-keras/requirements.txt deploy: replicas: 2 volumes: diff --git a/examples/mnist-pytorch/client/python_env.yaml b/examples/mnist-pytorch/client/python_env.yaml index 695e18de1..6ac97259a 100644 --- a/examples/mnist-pytorch/client/python_env.yaml +++ b/examples/mnist-pytorch/client/python_env.yaml @@ -6,5 +6,4 @@ build_dependencies: dependencies: - torch==1.13.1 - torchvision==0.14.1 - - fire==0.3.1 - fedn==0.8.0