Merged master

.ci/tests/examples/run.sh

@@ -11,7 +11,7 @@ helper="$2"
 
 python -m venv ".$example"
 source ".$example/bin/activate"
-".$example/bin/pip" install ./fedn/ fire
+".$example/bin/pip" install . fire
 
 >&2 echo "Start FEDn"
 pushd "examples/$example"

.devcontainer/bin/init_venv.sh

@@ -10,8 +10,5 @@ python -m venv .venv
     sphinx==4.4.0 \
     sphinx_press_theme==0.8.0 \
     sphinx-autobuild==2021.3.14 \
-    autopep8==1.5.7 \
-    isort==5.10.1 \
-    flake8==4.0.1 \
     sphinx_rtd_theme==0.5.2
-.venv/bin/pip install -e fedn
+.venv/bin/pip install -e .

.devcontainer/devcontainer.json.tpl

@@ -5,23 +5,26 @@
   "remoteUser": "default",
   // "workspaceFolder": "/fedn",
   // "workspaceMount": "source=/path/to/fedn,target=/fedn,type=bind,consistency=default",
-  "extensions": [
-    "ms-azuretools.vscode-docker",
-    "ms-python.python",
-    "exiasr.hadolint",
-    "yzhang.markdown-all-in-one",
-    "ms-python.isort"
-  ],
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-azuretools.vscode-docker",
+        "ms-python.python",
+        "exiasr.hadolint",
+        "yzhang.markdown-all-in-one",
+        "charliermarsh.ruff"
+      ]
+    }
+  },
   "mounts": [
-    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind,consistency=default",
+    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind,consistency=default"
   ],
   "runArgs": [
     "--net=host"
   ],
   "build": {
     "args": {
-      "BASE_IMG": "python:3.9"
+      "BASE_IMG": "python:3.11"
     }
   }
-}
-
+}

.dockerignore

@@ -9,4 +9,5 @@ docs/*
 **/.mnist-pytorch
 **/*.npz
 **/data
-**/*.tgz
+**/*.tgz
+dist

.github/workflows/code-checks.yaml

@@ -11,30 +11,9 @@ jobs:
 
       - name: init venv
         run: .devcontainer/bin/init_venv.sh
-
-      - name: check Python imports
-        run: >
-          .venv/bin/isort . --check --diff 
-          --skip .venv
-          --skip .mnist-keras
-          --skip .mnist-pytorch
-          --skip fedn_pb2.py
-          --skip fedn_pb2_grpc.py
-
-      - name: check Python formatting
-        run: >
-          .venv/bin/autopep8 --recursive --diff
-          --exclude .venv
-          --exclude .mnist-keras
-          --exclude .mnist-pytorch
-          --exclude fedn_pb2.py
-          --exclude fedn_pb2_grpc.py
-          .
 
-      - name: run Python linter
-        run: >
-          .venv/bin/flake8 . 
-          --exclude ".venv,.mnist-keras,.mnist-pytorch,fedn_pb2.py,fedn_pb2_grpc.py"
+      - name: Ruff Linting
+        uses: chartboost/ruff-action@v1
 
       - name: check for floating imports
         run: >

.github/workflows/push-to-pypi.yaml

@@ -16,15 +16,15 @@ jobs:
 
       - name: Install pypa/build
         run: python -m pip install build
-        working-directory: ./fedn
+        working-directory: ./
 
       - name: Build package
         run: python -m build
-        working-directory: ./fedn
+        working-directory: ./
 
       - name: Publish to Test PyPI
         uses: pypa/gh-action-pypi-publish@v1.8.14
         with:
           user: __token__
           password: ${{ secrets.PYPI_TOKEN }}
-          packages_dir: fedn/dist
+          packages_dir: ./dist

.gitignore

@@ -21,8 +21,8 @@ __pycache__/
 # Distribution / packaging
 .Python
 build/
+package/
 develop-eggs/
-#dist/
 downloads/
 eggs/
 .eggs/
@@ -38,6 +38,7 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
+dist
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

Dockerfile

@@ -8,7 +8,7 @@ ARG GRPC_HEALTH_PROBE_VERSION=""
 ARG REQUIREMENTS=""
 
 # Add FEDn and default configs
-COPY fedn /app/fedn
+COPY . /app
 COPY config/settings-client.yaml.template /app/config/settings-client.yaml
 COPY config/settings-combiner.yaml.template /app/config/settings-combiner.yaml
 COPY config/settings-reducer.yaml.template /app/config/settings-reducer.yaml
@@ -27,6 +27,8 @@ RUN if [ ! -z "$GRPC_HEALTH_PROBE_VERSION" ]; then \
   echo "No grpc_health_probe version specified, skipping installation"; \
   fi
 
+# Setup working directory
+WORKDIR /app
 
 # Create FEDn app directory
 SHELL ["/bin/bash", "-c"]
@@ -39,14 +41,13 @@ RUN mkdir -p /app \
   # Install FEDn and requirements
   && python -m venv /venv \
   && /venv/bin/pip install --upgrade pip \
-  && /venv/bin/pip install --no-cache-dir -e /app/fedn \
+  && /venv/bin/pip install --no-cache-dir setuptools>=65 \
+  && /venv/bin/pip install --no-cache-dir -e . \
   && if [[ ! -z "$REQUIREMENTS" ]]; then \
   /venv/bin/pip install --no-cache-dir -r /app/config/requirements.txt; \
   fi \
   #
   # Clean up
   && rm -r /app/config/requirements.txt
 
-# Setup working directory
-WORKDIR /app
 ENTRYPOINT [ "/venv/bin/fedn" ]

README.rst

@@ -30,61 +30,66 @@ We develop the FEDn framework following these core design principles:
 Features
 =========
 
-Federated machine learning: 
+Core FL framework (this repository): 
 
-- Support for any ML framework (e.g. PyTorch, Tensforflow/Keras and Scikit-learn)
+- Tiered federated learning architecture enabling massive scalability and resilience. 
+- Support for any ML framework (examples for PyTorch, Tensforflow/Keras and Scikit-learn)
 - Extendable via a plug-in architecture (aggregators, load balancers, object storage backends, databases  etc.)
 - Built-in federated algorithms (FedAvg, FedAdam, FedYogi, FedAdaGrad, etc.)
-- CLI and Python API client for running FEDn networks and coordinating experiments. 
+- CLI and Python API.
 - Implement clients in any language (Python, C++, Kotlin etc.)
 - No open ports needed client-side.
+- Flexible deployment of server-side components using Docker / docker compose.   
 
 
 FEDn Studio - From development to FL in production: 
 
--  Leverage Scaleout's free managed service for development and testing in real-world scenarios (SaaS).      
--  Token-based authentication (JWT) and role-based access control (RBAC) for FL clients.  
--  REST API and UI. 
--  Data science dashboard for orchestrating experiments and visualizing results.
--  Admin dashboard for managing the FEDn network and users/clients.
--  View extensive logging and tracing information. 
--  Collaborate with other data-scientists on the project specification in a shared workspace. 
--  Cloud or on-premise deployment (cloud-native design, deploy to any Kubernetes cluster)
+-  Secure deployment of server-side / control-plane on Kubernetes.
+-  UI with dashboards for orchestrating experiments and visualizing results
+-  Team features - collaborate with other users in shared project workspaces. 
+-  Features for the trusted-third party: Manage access to the FL network, FL clients and training progress.
+-  REST API for handling experiments/jobs. 
+-  View and export logging and tracing information. 
+-  Public cloud, dedicated cloud and on-premise deployment options. 
 
 
 Getting started
 ============================
 
-The best way to get started is to take the quickstart tutorial: 
+Get started with FEDn in two steps:  
 
-- `Quickstart <https://fedn.readthedocs.io/en/stable/quickstart.html>`__
+1. Sign up for a `Free FEDn Studio account <https://fedn.scaleoutsystems.com/signup>`__
+2. Take the `Quickstart tutorial <https://fedn.readthedocs.io/en/stable/quickstart.html>`__
+
+FEDn Studio (SaaS) is free for academic use and personal development / small-scale testing and exploration. For users and teams requiring
+additional project resources, dedicated support or other hosting options, `explore our plans <https://www.scaleoutsystems.com/start#pricing>`__.  
 
 Documentation
 =============
 
-More details about the architecture, deployment, and how to develop your own application and framework extensions (such as custom aggregators) are found in the documentation:
+More details about the architecture, deployment, and how to develop your own application and framework extensions are found in the documentation:
 
 -  `Documentation <https://fedn.readthedocs.io>`__
 
 
-Running your project in FEDn Studio (SaaS or on-premise)
-========================================================
-
-The FEDn Studio SaaS is free for development, testing and research (one project per user, backend compute resources sized for dev/test):   
+FEDn Studio Deployment options
+==============================
 
-- `Register for a free account in FEDn Studio <https://fedn.scaleoutsystems.com/signup/>`__
-- `Take the tutorial to deploy your project on FEDn Studio <https://fedn.readthedocs.io/en/stable/studio.html>`__  
+Several hosting options are available to suit different project settings.
 
-Scaleout can also support users to scale up experiments and demonstrators on Studio, by granting custom resource quotas. Additonally, charts are available for self-managed deployment on-premise or in your cloud VPC (all major cloud providers). Contact the Scaleout team for more information.
+-  `Public cloud (multi-tenant) <https://fedn.scaleoutsystems.com>`__: Managed multi-tenant deployment in public cloud. 
+-   Dedicated cloud (single-tenant): Managed, dedicated deployment in a cloud region of your choice (AWS, GCP, Azure, managed Kubernetes) 
+-   Self-managed: Set up a self-managed deployment in your VPC or on-premise Kubernets cluster using Helm Chart and container images provided by Scaleout. 
 
+Contact the Scaleout team for information.
 
 Support
 =================
 
-Community support in available in our `Discord
+Community support is available in our `Discord
 server <https://discord.gg/KMg4VwszAd>`__.
 
-Options are available for `Enterprise support <https://www.scaleoutsystems.com/start#pricing>`__.
+Options are available for `Dedicated/custom support <https://www.scaleoutsystems.com/start#pricing>`__.
 
 Making contributions
 ====================
@@ -114,4 +119,4 @@ License
 FEDn is licensed under Apache-2.0 (see `LICENSE <LICENSE>`__ file for
 full information).
 
-Use of FEDn Studio (SaaS) is subject to the `Terms of Use <https://www.scaleoutsystems.com/terms>`__.
+Use of FEDn Studio is subject to the `Terms of Use <https://www.scaleoutsystems.com/terms>`__.

config/combiner-settings.override.yaml

@@ -5,5 +5,5 @@ version: '3.3'
 services:
   combiner:
     volumes:
-      - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
+      - ${HOST_REPO_DIR:-.}:/app
       - ${HOST_REPO_DIR:-.}/config/settings-combiner.yaml:/app/config/settings-combiner.yaml

config/reducer-settings.override.yaml

@@ -5,5 +5,5 @@ version: '3.3'
 services:
   reducer:
     volumes:
-      - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
+      - ${HOST_REPO_DIR:-.}:/app
       - ${HOST_REPO_DIR:-.}/config/settings-reducer.yaml:/app/config/settings-reducer.yaml

docker-compose.yaml

@@ -78,7 +78,7 @@ services:
       - mongo
     entrypoint: [ "sh", "-c" ]
     command:
-      - "/venv/bin/pip install --no-cache-dir -e /app/fedn && /venv/bin/python fedn/fedn/network/api/server.py"
+      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/python fedn/network/api/server.py"
     ports:
       - 8092:8092
 
@@ -97,7 +97,7 @@ services:
       - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
     entrypoint: [ "sh", "-c" ]
     command:
-      - "/venv/bin/pip install --no-cache-dir -e /app/fedn && /venv/bin/fedn run combiner --init config/settings-combiner.yaml"
+      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn combiner start --init config/settings-combiner.yaml"
     ports:
       - 12080:12080
     healthcheck:
@@ -127,7 +127,7 @@ services:
       - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
     entrypoint: [ "sh", "-c" ]
     command:
-      - "/venv/bin/pip install --no-cache-dir -e /app/fedn && /venv/bin/fedn run client --init config/settings-client.yaml"
+      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn client start --init config/settings-client.yaml"
     deploy:
       replicas: 0
     depends_on: