Merge branch 'feature/SK-746' of https://github.com/scaleoutsystems/fedn

 into feature/SK-746

.ci/tests/examples/configure.sh

@@ -14,15 +14,8 @@ pushd "examples/$example"
 # If example equals "mnist-keras"
 if [ "$example" == "mnist-keras" ]; then
 
-    bin/init_venv.sh
-
     >&2 echo "Download and prepare data"
     bin/get_data
     bin/split_data
 fi
-popd
-
-pushd "examples/$example"
->&2 echo "Build compute package and seed"
-bin/build.sh
 popd

.ci/tests/examples/run.sh

@@ -16,6 +16,8 @@ pip install ./fedn/ fire
 >&2 echo "Start FEDn"
 pushd "examples/$example"
 
+fedn package create --path client
+
 docker compose \
     -f ../../docker-compose.yaml \
     -f docker-compose.override.yaml \

.github/workflows/integration-tests.yaml

@@ -29,11 +29,6 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python_version }}
-
-      - name: configure ${{ matrix.to_test }}
-        run: |
-          echo 'BASE_IMG=python:${{ matrix.python_version }}-slim' > .env 
-          .ci/tests/examples/configure.sh ${{ matrix.to_test }}
 
       - name: run ${{ matrix.to_test }}
         run: .ci/tests/examples/run.sh ${{ matrix.to_test }}

examples/mnist-keras/client/entrypoint → examples/mnist-keras/client/entrypoint.py

@@ -1,8 +1,6 @@
-#!./.mnist-keras/bin/python
 import json
 import os
 
-import docker
 import fire
 import numpy as np
 import tensorflow as tf
@@ -16,12 +14,9 @@
 
 
 def _get_data_path():
-    # Figure out FEDn client number from container name
-    client = docker.from_env()
-    container = client.containers.get(os.environ['HOSTNAME'])
-    number = container.name[-1]
-    # Return data path
-    return f"/var/data/clients/{number}/mnist.npz"
+    data_path = os.environ.get('FEDN_DATA_PATH', '/var/data/clients/1/mnist.npz')
+
+    return data_path
 
 
 def compile_model(img_rows=28, img_cols=28):
@@ -74,7 +69,7 @@ def load_data(data_path, is_train=True):
     return X, y
 
 
-def init_seed(out_path='seed.npz'):
+def init_seed(out_path='../seed.npz'):
     """ Initialize seed model and save it to file.
 
     :param out_path: The path to save the seed model to.

examples/mnist-keras/client/fedn.yaml

@@ -1,7 +1,12 @@
+python_env: python_env.yaml
 entry_points:
+  build:
+    command: python entrypoint.py init_seed
+  startup:
+    command: python get_data.py
   train:
-    command: /venv/bin/python entrypoint train $ENTRYPOINT_OPTS
+    command: python entrypoint.py train $ENTRYPOINT_OPTS
   validate:
-    command: /venv/bin/python entrypoint validate $ENTRYPOINT_OPTS
+    command: python entrypoint.py validate $ENTRYPOINT_OPTS
   infer:
-    command: /venv/bin/python entrypoint infer $ENTRYPOINT_OPTS
+    command: python entrypoint.py infer $ENTRYPOINT_OPTS

examples/mnist-keras/bin/split_data → examples/mnist-keras/client/get_data.py

@@ -1,9 +1,8 @@
-#!./.mnist-keras/bin/python
 import os
 from math import floor
 
-import fire
 import numpy as np
+import tensorflow as tf
 
 
 def splitset(dataset, parts):
@@ -38,5 +37,17 @@ def split(dataset='data/mnist.npz', outdir='data', n_splits=2):
                  y_test=data['y_test'][i])
 
 
+def get_data(out_dir='data'):
+    # Make dir if necessary
+    if not os.path.exists(out_dir):
+        os.mkdir(out_dir)
+
+    # Download data
+    (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()
+    np.savez(f'{out_dir}/mnist.npz', x_train=x_train,
+             y_train=y_train, x_test=x_test, y_test=y_test)
+
+
 if __name__ == '__main__':
-    fire.Fire(split)
+    get_data()
+    split()

examples/mnist-keras/client/python_env.yaml

@@ -0,0 +1,9 @@
+name: mnist-keras
+build_dependencies:
+  - pip
+  - setuptools
+  - wheel==0.37.1
+dependencies:
+  - tensorflow==2.13.1
+  - fire==0.3.1
+  - fedn==0.8.0

examples/mnist-keras/docker-compose.override.yaml

@@ -4,9 +4,6 @@ version: '3.3'
 # Overriding requirements
 services:
   client:
-    build:
-      args:
-        REQUIREMENTS: examples/mnist-keras/requirements.txt
     deploy:
       replicas: 2
     volumes:

examples/mnist-pytorch/client/python_env.yaml

@@ -6,5 +6,4 @@ build_dependencies:
 dependencies:
   - torch==1.13.1
   - torchvision==0.14.1
-  - fire==0.3.1
   - fedn==0.8.0