Added load test based on shuffling numpy arrays

Author: Andreas Hellander

examples/async-simulation/docker-compose.override.yaml

@@ -6,10 +6,10 @@ services:
   client:
     build:
       args:
-        REQUIREMENTS: examples/async-simulation/requirements.txt
+        REQUIREMENTS: examples/load-test/requirements.txt
     deploy:
       replicas: 2
     volumes:
       - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
-      - ${HOST_REPO_DIR:-.}/examples/async-simulation/data:/var/data
+      - ${HOST_REPO_DIR:-.}/examples/load-test/data:/var/data
       - /var/run/docker.sock:/var/run/docker.sock

examples/load-test/.gitignore

@@ -0,0 +1,6 @@
+data
+*.npz
+*.tgz
+*.tar.gz
+.async-simulation
+client.yaml

examples/load-test/README.md

@@ -0,0 +1,56 @@
+# LOAD TEST 
+This example can be used as a load test for FEDn.  
+
+No actual machine learning is being done - the clients generate a 
+random array of a configurable size. In this way a developer can
+test the performance / scalability of a given FEDn network in a flexible
+way simply by shuffling around and aggregating numeric arrays. 
+
+## Prerequisites
+- [Python 3.8, 3.9 or 3.10](https://www.python.org/downloads)
+- [Docker](https://docs.docker.com/get-docker)
+- [Docker Compose](https://docs.docker.com/compose/install)
+
+## Running the example (pseudo-distributed, single host)
+
+Clone FEDn and locate into this directory.
+```sh
+git clone https://github.com/scaleoutsystems/fedn.git
+cd fedn/examples/load-test
+```
+
+### Preparing the environment, the local data, the compute package and seed model
+
+Install FEDn: 
+```
+pip install fedn
+```
+
+Standing in examples/load-test
+```
+pip install -r requirements.txt
+```
+
+Create the compute package and a seed model that you will be asked to upload in the next step.
+```
+tar -czvf package.tgz client
+```
+
+```
+python client/entrypoint init_seed
+```
+
+### Deploy FEDn and two clients
+docker-compose -f ../../docker-compose.yaml -f docker-compose.override.yaml up 
+
+### Initialize the FEDn network 
+Edit 'init_fedn.py' to configure the FEDn host (controller) to connect to, then
+```
+python init_fedn.py
+```
+
+Launch clients
+> **Note**: run with `--scale client=N` to start *N* clients.
+
+## Clean up
+You can clean up by running `docker-compose down -v`.

examples/load-test/client/entrypoint

@@ -0,0 +1,98 @@
+# /bin/python
+import time
+
+import fire
+import numpy as np
+
+from fedn.utils.helpers.helpers import get_helper, save_metadata, save_metrics
+
+HELPER_MODULE = 'numpyhelper'
+ARRAY_SIZE = 1000000
+
+
+def save_model(weights, out_path):
+    """ Save model to disk.
+
+    :param model: The model to save.
+    :type model: torch.nn.Module
+    :param out_path: The path to save to.
+    :type out_path: str
+    """
+    helper = get_helper(HELPER_MODULE)
+    helper.save(weights, out_path)
+
+
+def load_model(model_path):
+    """ Load model from disk.
+
+    param model_path: The path to load from.
+    :type model_path: str
+    :return: The loaded model.
+    :rtype: torch.nn.Module
+    """
+    helper = get_helper(HELPER_MODULE)
+    weights = helper.load(model_path)
+    return weights
+
+
+def init_seed(out_path='seed.npz'):
+    """ Initialize seed model.
+
+    :param out_path: The path to save the seed model to.
+    :type out_path: str
+    """
+    # Init and save
+    weights = [np.random.rand(1, ARRAY_SIZE)]
+    save_model(weights, out_path)
+
+
+def train(in_model_path, out_model_path):
+    """ Train model.
+
+    """
+
+    # Load model
+    weights = load_model(in_model_path)
+
+    # Train
+    time.sleep(np.random.randint(4, 15))
+
+    # Metadata needed for aggregation server side
+    metadata = {
+        'num_examples': ARRAY_SIZE,
+    }
+
+    # Save JSON metadata file
+    save_metadata(metadata, out_model_path)
+
+    # Save model update
+    save_model(weights, out_model_path)
+
+
+def validate(in_model_path, out_json_path):
+    """ Validate model.
+
+    :param in_model_path: The path to the input model.
+    :type in_model_path: str
+    :param out_json_path: The path to save the output JSON to.
+    :type out_json_path: str
+    :param data_path: The path to the data file.
+    :type data_path: str
+    """
+    weights = load_model(in_model_path)
+
+    # JSON schema
+    report = {
+        "mean": np.mean(weights),
+    }
+
+    # Save JSON
+    save_metrics(report, out_json_path)
+
+
+if __name__ == '__main__':
+    fire.Fire({
+        'init_seed': init_seed,
+        'train': train,
+        'validate': validate
+    })

examples/load-test/client/fedn.yaml

@@ -0,0 +1,5 @@
+entry_points:
+  train:
+    command: /venv/bin/python entrypoint train $ENTRYPOINT_OPTS
+  validate: 
+    command: /venv/bin/python entrypoint validate $ENTRYPOINT_OPTS

examples/load-test/docker-compose.override.yaml

@@ -0,0 +1,15 @@
+# Compose schema version
+version: '3.3'
+
+# Overriding requirements
+services:
+  client:
+    build:
+      args:
+        REQUIREMENTS: examples/async-simulation/requirements.txt
+    deploy:
+      replicas: 2
+    volumes:
+      - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
+      - ${HOST_REPO_DIR:-.}/examples/async-simulation/data:/var/data
+      - /var/run/docker.sock:/var/run/docker.sock

examples/load-test/init_fedn.py

@@ -0,0 +1,8 @@
+from fedn import APIClient
+
+DISCOVER_HOST = '127.0.0.1'
+DISCOVER_PORT = 8092
+
+client = APIClient(DISCOVER_HOST, DISCOVER_PORT)
+client.set_package('package.tgz', 'numpyhelper')
+client.set_initial_model('seed.npz')

examples/load-test/requirements.txt

@@ -0,0 +1 @@
+fire==0.3.1

examples/load-test/run_clients.py

@@ -0,0 +1,59 @@
+"""This scripts starts N_CLIENTS using the SDK.
+
+If you are running with a local deploy of FEDn
+using docker compose, you need to make sure that clients
+are able to resolve the name "combiner" to 127.0.0.1
+
+One way to accomplish this is to edit your /etc/host,
+adding the line:
+
+combiner    127.0.0.1
+
+"""
+
+
+import copy
+import time
+
+from fedn import APIClient
+from fedn.network.clients.client import Client
+
+DISCOVER_HOST = '127.0.0.1'
+DISCOVER_PORT = 8092
+N_CLIENTS = 5
+CLIENTS_AVAILABLE_TIME = 120
+
+config = {'discover_host': DISCOVER_HOST, 'discover_port': DISCOVER_PORT, 'token': None, 'name': 'testclient',
+          'client_id': 1, 'remote_compute_context': True, 'force_ssl': False, 'dry_run': False, 'secure': False,
+          'preshared_cert': False, 'verify': False, 'preferred_combiner': False,
+          'validator': True, 'trainer': True, 'init': None, 'logfile': 'test.log', 'heartbeat_interval': 2,
+          'reconnect_after_missed_heartbeat': 30}
+
+if __name__ == '__main__':
+
+    # Start up N_CLIENTS clients
+    clients = []
+    for i in range(N_CLIENTS):
+        config_i = copy.deepcopy(config)
+        config['name'] = 'client{}'.format(i)
+        clients.append(Client(config))
+
+    # Run a session
+    client = APIClient(DISCOVER_HOST, DISCOVER_PORT)
+
+    session_config_fedavg = {
+        "helper": "numpyhelper",
+        "session_id": str(uuid.uuid4()),
+        "aggregator": "fedavg",
+        "round_timeout": 30,
+        "rounds": 5,
+    }
+
+    result_fedavg = client.start_session(**session_config_fedavg)
+    while not client.session_is_finished(session_id):
+        time.sleep(2)
+
+    # Disconnect clients
+    time.sleep(CLIENTS_AVAILABLE_TIME)
+    for client in clients:
+        client.detach()