From 7b26c5a489fb58135dd2e461ecd816ad4af7c050 Mon Sep 17 00:00:00 2001 From: mattiasakesson Date: Thu, 4 Apr 2024 16:56:54 +0200 Subject: [PATCH 1/6] fed prox example --- examples/mnist-pytorch-fedprox/bin/build.sh | 8 + examples/mnist-pytorch-fedprox/bin/get_data | 21 ++ .../mnist-pytorch-fedprox/bin/init_venv.sh | 10 + examples/mnist-pytorch-fedprox/bin/split_data | 51 ++++ .../mnist-pytorch-fedprox/client/entrypoint | 245 ++++++++++++++++++ .../mnist-pytorch-fedprox/client/fedn.yaml | 5 + 6 files changed, 340 insertions(+) create mode 100755 examples/mnist-pytorch-fedprox/bin/build.sh create mode 100755 examples/mnist-pytorch-fedprox/bin/get_data create mode 100755 examples/mnist-pytorch-fedprox/bin/init_venv.sh create mode 100755 examples/mnist-pytorch-fedprox/bin/split_data create mode 100755 examples/mnist-pytorch-fedprox/client/entrypoint create mode 100644 examples/mnist-pytorch-fedprox/client/fedn.yaml diff --git a/examples/mnist-pytorch-fedprox/bin/build.sh b/examples/mnist-pytorch-fedprox/bin/build.sh new file mode 100755 index 000000000..18cdb5128 --- /dev/null +++ b/examples/mnist-pytorch-fedprox/bin/build.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +# Init seed +client/entrypoint init_seed + +# Make compute package +tar -czvf package.tgz client \ No newline at end of file diff --git a/examples/mnist-pytorch-fedprox/bin/get_data b/examples/mnist-pytorch-fedprox/bin/get_data new file mode 100755 index 000000000..24be1a6ea --- /dev/null +++ b/examples/mnist-pytorch-fedprox/bin/get_data @@ -0,0 +1,21 @@ +#!./.mnist-pytorch/bin/python +import os + +import fire +import torchvision + + +def get_data(out_dir='data'): + # Make dir if necessary + if not os.path.exists(out_dir): + os.mkdir(out_dir) + + # Download data + torchvision.datasets.MNIST( + root=f'{out_dir}/train', transform=torchvision.transforms.ToTensor, train=True, download=True) + torchvision.datasets.MNIST( + root=f'{out_dir}/test', transform=torchvision.transforms.ToTensor, train=False, download=True) + + +if __name__ == '__main__': + fire.Fire(get_data) diff --git a/examples/mnist-pytorch-fedprox/bin/init_venv.sh b/examples/mnist-pytorch-fedprox/bin/init_venv.sh new file mode 100755 index 000000000..930f300ea --- /dev/null +++ b/examples/mnist-pytorch-fedprox/bin/init_venv.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e + +# Init venv +python3.10 -m venv .mnist-pytorch + +# Pip deps +.mnist-pytorch/bin/pip install --upgrade pip +.mnist-pytorch/bin/pip install -e ../../fedn +.mnist-pytorch/bin/pip install -r requirements.txt \ No newline at end of file diff --git a/examples/mnist-pytorch-fedprox/bin/split_data b/examples/mnist-pytorch-fedprox/bin/split_data new file mode 100755 index 000000000..51e38b4d4 --- /dev/null +++ b/examples/mnist-pytorch-fedprox/bin/split_data @@ -0,0 +1,51 @@ +#!./.mnist-pytorch/bin/python +import os +from math import floor + +import fire +import torch +import torchvision + + +def splitset(dataset, parts): + n = dataset.shape[0] + local_n = floor(n/parts) + result = [] + for i in range(parts): + result.append(dataset[i*local_n: (i+1)*local_n]) + return result + + +def split(out_dir='data', n_splits=2): + # Make dir + if not os.path.exists(f'{out_dir}/clients'): + os.mkdir(f'{out_dir}/clients') + + # Load and convert to dict + train_data = torchvision.datasets.MNIST( + root=f'{out_dir}/train', transform=torchvision.transforms.ToTensor, train=True) + test_data = torchvision.datasets.MNIST( + root=f'{out_dir}/test', transform=torchvision.transforms.ToTensor, train=False) + data = { + 'x_train': splitset(train_data.data, n_splits), + 'y_train': splitset(train_data.targets, n_splits), + 'x_test': splitset(test_data.data, n_splits), + 'y_test': splitset(test_data.targets, n_splits), + } + + # Make splits + for i in range(n_splits): + subdir = f'{out_dir}/clients/{str(i+1)}' + if not os.path.exists(subdir): + os.mkdir(subdir) + torch.save({ + 'x_train': data['x_train'][i], + 'y_train': data['y_train'][i], + 'x_test': data['x_test'][i], + 'y_test': data['y_test'][i], + }, + f'{subdir}/mnist.pt') + + +if __name__ == '__main__': + fire.Fire(split) diff --git a/examples/mnist-pytorch-fedprox/client/entrypoint b/examples/mnist-pytorch-fedprox/client/entrypoint new file mode 100755 index 000000000..c37f99fa5 --- /dev/null +++ b/examples/mnist-pytorch-fedprox/client/entrypoint @@ -0,0 +1,245 @@ +#!./.mnist-pytorch/bin/python +import collections +import math +import os + +import docker +import fire +import torch + +from fedn.utils.helpers.helpers import get_helper, save_metadata, save_metrics +import copy +HELPER_MODULE = 'numpyhelper' +helper = get_helper(HELPER_MODULE) + +NUM_CLASSES = 10 + + +def _get_data_path(): + """ For test automation using docker-compose. """ + # Figure out FEDn client number from container name + client = docker.from_env() + container = client.containers.get(os.environ['HOSTNAME']) + number = container.name[-1] + + # Return data path + return f"/var/data/clients/{number}/mnist.pt" + + +def compile_model(): + """ Compile the pytorch model. + + :return: The compiled model. + :rtype: torch.nn.Module + """ + class Net(torch.nn.Module): + def __init__(self): + super(Net, self).__init__() + self.fc1 = torch.nn.Linear(784, 64) + self.fc2 = torch.nn.Linear(64, 32) + self.fc3 = torch.nn.Linear(32, 10) + + def forward(self, x): + x = torch.nn.functional.relu(self.fc1(x.reshape(x.size(0), 784))) + x = torch.nn.functional.dropout(x, p=0.5, training=self.training) + x = torch.nn.functional.relu(self.fc2(x)) + x = torch.nn.functional.log_softmax(self.fc3(x), dim=1) + return x + + return Net() + + +def load_data(data_path, is_train=True): + """ Load data from disk. + + :param data_path: Path to data file. + :type data_path: str + :param is_train: Whether to load training or test data. + :type is_train: bool + :return: Tuple of data and labels. + :rtype: tuple + """ + if data_path is None: + data = torch.load(_get_data_path()) + else: + data = torch.load(data_path) + + if is_train: + X = data['x_train'] + y = data['y_train'] + else: + X = data['x_test'] + y = data['y_test'] + + # Normalize + X = X / 255 + + return X, y + + +def save_parameters(model, out_path): + """ Save model paramters to file. + + :param model: The model to serialize. + :type model: torch.nn.Module + :param out_path: The path to save to. + :type out_path: str + """ + parameters_np = [val.cpu().numpy() for _, val in model.state_dict().items()] + helper.save(parameters_np, out_path) + + +def load_parameters(model_path): + """ Load model parameters from file and populate model. + + param model_path: The path to load from. + :type model_path: str + :return: The loaded model. + :rtype: torch.nn.Module + """ + model = compile_model() + parameters_np = helper.load(model_path) + + params_dict = zip(model.state_dict().keys(), parameters_np) + state_dict = collections.OrderedDict({key: torch.tensor(x) for key, x in params_dict}) + model.load_state_dict(state_dict, strict=True) + return model + + +def init_seed(out_path='seed.npz'): + """ Initialize seed model and save it to file. + + :param out_path: The path to save the seed model to. + :type out_path: str + """ + # Init and save + model = compile_model() + save_parameters(model, out_path) + + +def train(in_model_path, out_model_path, data_path=None, batch_size=32, epochs=1, lr=0.01, mu=3): + """ Complete a model update. + + Load model paramters from in_model_path (managed by the FEDn client), + perform a model update, and write updated paramters + to out_model_path (picked up by the FEDn client). + + :param in_model_path: The path to the input model. + :type in_model_path: str + :param out_model_path: The path to save the output model to. + :type out_model_path: str + :param data_path: The path to the data file. + :type data_path: str + :param batch_size: The batch size to use. + :type batch_size: int + :param epochs: The number of epochs to train. + :type epochs: int + :param lr: The learning rate to use. + :type lr: float + """ + print("data_path: ", data_path) + print(os.getcwd()) + print("list data path: ", os.listdir('/var/data')) + + print("list data/clients path: ", os.listdir('/var/data/clients')) + # Load data + x_train, y_train = load_data(data_path) + + # Load parmeters and initialize model + model = load_parameters(in_model_path) + global_model = copy.deepcopy(model) + + # Train + optimizer = torch.optim.SGD(model.parameters(), lr=lr) + n_batches = int(math.ceil(len(x_train) / batch_size)) + criterion = torch.nn.NLLLoss() + for e in range(epochs): # epoch loop + for b in range(n_batches): # batch loop + # Retrieve current batch + batch_x = x_train[b * batch_size:(b + 1) * batch_size] + batch_y = y_train[b * batch_size:(b + 1) * batch_size] + # Train on batch + optimizer.zero_grad() + outputs = model(batch_x) + + proximal_term = 0.0 + for w, w_t in zip(model.parameters(), global_model.parameters()): + proximal_term += (w - w_t).norm(2) + #print("proximal_term: ", proximal_term) + + # loss = criterion(outputs, batch_y) # <-- old + + # loss = loss_function(y_pred, label) + (args.mu / 2) * proximal_term <-- fed prox term + loss = criterion(outputs, batch_y) + (mu / 2) * proximal_term # <-- new + + loss.backward() + optimizer.step() + # Log + if b % 100 == 0: + print( + f"Epoch {e}/{epochs-1} | Batch: {b}/{n_batches-1} | Loss: {loss.item()}") + + # Metadata needed for aggregation server side + metadata = { + # num_examples are mandatory + 'num_examples': len(x_train), + 'batch_size': batch_size, + 'epochs': epochs, + 'lr': lr + } + + # Save JSON metadata file (mandatory) + save_metadata(metadata, out_model_path) + + # Save model update (mandatory) + save_parameters(model, out_model_path) + + +def validate(in_model_path, out_json_path, data_path=None): + """ Validate model. + + :param in_model_path: The path to the input model. + :type in_model_path: str + :param out_json_path: The path to save the output JSON to. + :type out_json_path: str + :param data_path: The path to the data file. + :type data_path: str + """ + # Load data + x_train, y_train = load_data(data_path) + x_test, y_test = load_data(data_path, is_train=False) + + # Load model + model = load_parameters(in_model_path) + model.eval() + + # Evaluate + criterion = torch.nn.NLLLoss() + with torch.no_grad(): + train_out = model(x_train) + training_loss = criterion(train_out, y_train) + training_accuracy = torch.sum(torch.argmax( + train_out, dim=1) == y_train) / len(train_out) + test_out = model(x_test) + test_loss = criterion(test_out, y_test) + test_accuracy = torch.sum(torch.argmax( + test_out, dim=1) == y_test) / len(test_out) + + # JSON schema + report = { + "training_loss": training_loss.item(), + "training_accuracy": training_accuracy.item(), + "test_loss": test_loss.item(), + "test_accuracy": test_accuracy.item(), + } + print("validation data: ", report) + # Save JSON + save_metrics(report, out_json_path) + + +if __name__ == '__main__': + fire.Fire({ + 'init_seed': init_seed, + 'train': train, + 'validate': validate, + }) diff --git a/examples/mnist-pytorch-fedprox/client/fedn.yaml b/examples/mnist-pytorch-fedprox/client/fedn.yaml new file mode 100644 index 000000000..29c475270 --- /dev/null +++ b/examples/mnist-pytorch-fedprox/client/fedn.yaml @@ -0,0 +1,5 @@ +entry_points: + train: + command: /venv/bin/python entrypoint train $ENTRYPOINT_OPTS + validate: + command: /venv/bin/python entrypoint validate $ENTRYPOINT_OPTS \ No newline at end of file From 2bdf6f1ebac80e24a7deabd077cb0a436a42b274 Mon Sep 17 00:00:00 2001 From: mattiasakesson Date: Tue, 9 Apr 2024 17:30:16 +0200 Subject: [PATCH 2/6] added base files --- .../mnist-pytorch-fedprox/API_Example.ipynb | 249 ++++++++++++++++++ examples/mnist-pytorch-fedprox/README.rst | 149 +++++++++++ .../mnist-pytorch-fedprox/client/entrypoint | 15 +- .../client_settings.yaml | 1 + .../docker-compose.override.yaml | 16 ++ .../mnist-pytorch-fedprox/requirements.txt | 4 + 6 files changed, 432 insertions(+), 2 deletions(-) create mode 100644 examples/mnist-pytorch-fedprox/API_Example.ipynb create mode 100644 examples/mnist-pytorch-fedprox/README.rst create mode 100644 examples/mnist-pytorch-fedprox/client_settings.yaml create mode 100644 examples/mnist-pytorch-fedprox/docker-compose.override.yaml create mode 100644 examples/mnist-pytorch-fedprox/requirements.txt diff --git a/examples/mnist-pytorch-fedprox/API_Example.ipynb b/examples/mnist-pytorch-fedprox/API_Example.ipynb new file mode 100644 index 000000000..6bfe28a10 --- /dev/null +++ b/examples/mnist-pytorch-fedprox/API_Example.ipynb @@ -0,0 +1,249 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "622f7047", + "metadata": {}, + "source": [ + "## FEDn Quickstart (PyTorch)\n", + "\n", + "This notebook provides an example of how to use the FEDn API to organize experiments and to analyze validation results. We will here run one training session (a collection of global rounds) using FedAvg, then retrive and visualize the results.\n", + "\n", + "When you start this tutorial you should have a deployed FEDn Network up and running, and you should have created the compute package and the initial model, see the example README for instructions." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "743dfe47", + "metadata": {}, + "outputs": [], + "source": [ + "from fedn import APIClient\n", + "import time\n", + "import uuid\n", + "import json\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import collections" + ] + }, + { + "cell_type": "markdown", + "id": "1046a4e5", + "metadata": {}, + "source": [ + "We make a client connection to the FEDn API service. Here we assume that FEDn is deployed locally in pseudo-distributed mode with default ports." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "1061722d", + "metadata": {}, + "outputs": [], + "source": [ + "DISCOVER_HOST = '127.0.0.1'\n", + "DISCOVER_PORT = 8092\n", + "client = APIClient(DISCOVER_HOST, DISCOVER_PORT)" + ] + }, + { + "cell_type": "markdown", + "id": "07f69f5f", + "metadata": {}, + "source": [ + "Initialize FEDn with the compute package and seed model. Note that these files needs to be created separately by follwing instructions in the README." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "5107f6f9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'committed_at': 'Tue, 09 Apr 2024 15:00:15 GMT', 'id': '661557ff541a813d4965e726', 'key': 'models', 'model': '926a9a59-f1ac-498e-985f-591b1adc0449', 'parent_model': None, 'session_id': None}\n" + ] + } + ], + "source": [ + "client.set_active_package('package.tgz', 'numpyhelper')\n", + "client.set_active_model('seed.npz')\n", + "seed_model = client.get_active_model()\n", + "print(seed_model)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "77949714-bf80-4d9b-8fa7-7cba3fb8b086", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'926a9a59-f1ac-498e-985f-591b1adc0449'" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "client.get_active_model()['model']" + ] + }, + { + "cell_type": "markdown", + "id": "4e26c50b", + "metadata": {}, + "source": [ + "Next we start a training session using FedAvg and wait until it has finished:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f0380d35", + "metadata": {}, + "outputs": [], + "source": [ + "session_id = \"experiment33\"\n", + "\n", + "session_config = {\n", + " \"helper\": \"numpyhelper\",\n", + " \"id\": session_id,\n", + " \"aggregator\": \"fedavg\",\n", + " \"model_id\": client.get_active_model()['model'],\n", + " \"rounds\": 5\n", + " }\n", + "\n", + "result_fedavg = client.start_session(**session_config)\n", + "\n", + "# We wait for the session to finish\n", + "while not client.session_is_finished(session_config['id']):\n", + " time.sleep(2)" + ] + }, + { + "cell_type": "markdown", + "id": "16874cec", + "metadata": {}, + "source": [ + "Next, we get the model trail, retrieve all model validations from all clients, extract the training accuracy metric, and compute its mean value accross all clients." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "4e8044b7", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/mattias/Documents/projects/temp/fedprox/fedn/examples/mnist-pytorch-fedprox/.mnist-pytorch/lib/python3.10/site-packages/numpy/core/fromnumeric.py:3504: RuntimeWarning: Mean of empty slice.\n", + " return _methods._mean(a, axis=axis, dtype=dtype,\n", + "/home/mattias/Documents/projects/temp/fedprox/fedn/examples/mnist-pytorch-fedprox/.mnist-pytorch/lib/python3.10/site-packages/numpy/core/_methods.py:129: RuntimeWarning: invalid value encountered in scalar divide\n", + " ret = ret.dtype.type(ret / rcount)\n" + ] + } + ], + "source": [ + "session_id = \"experiment32\"\n", + "models = client.get_model_trail()\n", + "\n", + "acc = []\n", + "for model in models:\n", + " \n", + " model_id = model[\"model\"]\n", + " validations = client.get_validations(model_id=model_id)\n", + "\n", + " a = []\n", + " for validation in validations['result']: \n", + " metrics = json.loads(validation['data'])\n", + " a.append(metrics['training_accuracy'])\n", + " \n", + " acc.append(a)\n", + "\n", + "mean_acc = [np.mean(x) for x in acc]" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "42425c43", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABOmUlEQVR4nO3deVxU5f4H8M/MwMywDiA6KKCguODC4sbVFr3KTcvbtdJUrKt5b93qlySRpXZdKktQyVDwl2Wm3t91K9PyWppGaWmmJuCCiruCyqLisG8zz+8Pr5OjbIPAmeXzfr3O6+Wcec6Z7+PhMB/OPPMcmRBCgIiIiMiCyaUugIiIiKg+DCxERERk8RhYiIiIyOIxsBAREZHFY2AhIiIii8fAQkRERBaPgYWIiIgsHgMLERERWTwHqQtoKgaDAVeuXIGbmxtkMpnU5RAREVEDCCFQVFSEdu3aQS6v/TqKzQSWK1euwN/fX+oyiIiIqBGysrLg5+dX6/M2E1jc3NwA3Oqwu7u7xNUQERFRQxQWFsLf39/4Pl4bmwkstz8Gcnd3Z2AhIiKyMvUN5+CgWyIiIrJ4DCxERERk8RhYiIiIyOLZzBiWhtDr9aiqqpK6DCKqh0KhgIODA6coICKjRgWWpUuXYuHChcjJyUFoaCiSkpLQv3//GtsuX74c//rXv3Ds2DEAQJ8+fTBv3jxj+6qqKsycORPffvstzp07B41Gg8jISMTHx6Ndu3aN7Na9iouLkZ2dDSFEk+2TiJqPs7Mz2rZtC6VSKXUpRGQBzA4sGzZsQGxsLJYtW4aIiAgkJiZi2LBhyMzMRJs2be5pv2vXLkRFRWHgwIFQq9WYP38+HnnkEWRkZMDX1xelpaVITU3FrFmzEBoaioKCAkyZMgV/+ctf8NtvvzVJJ/V6PbKzs+Hs7IzWrVvzrzYiCyaEQGVlJfLz83H+/Hl07ty5zsmkiMg+yISZlxwiIiLQr18/JCcnA7g1w6y/vz+io6Mxffr0erfX6/Xw9PREcnIyJkyYUGObgwcPon///rh48SLat2/foLoKCwuh0Wig0+nu+VpzeXk5zp8/j4CAADg5OTVof0QkrdLSUly8eBGBgYFQq9VSl0NEzaSu9+87mfVnS2VlJQ4dOoTIyMjfdyCXIzIyEvv27WvQPkpLS1FVVQUvL69a2+h0OshkMnh4eNTapqKiAoWFhSZLfXhlhch68KoKEd3JrN8I165dg16vh1arNVmv1WqRk5PToH1MmzYN7dq1Mwk9dyovL8e0adMQFRVVZ9KKi4uDRqMxLpyWn4iIyHa16J8w8fHxWL9+PTZv3lzjJd6qqiqMGTMGQgh89NFHde5rxowZ0Ol0xiUrK6u5yrYpAQEBSExMbHD7Xbt2QSaT4ebNm81WE9Xu4Ycfxtq1a5tt/1Id32XLluHxxx9v0dckIutmVmDx9vaGQqFAbm6uyfrc3Fz4+PjUuW1CQgLi4+OxY8cOhISE3PP87bBy8eJF7Ny5s97p9VUqlXEaflucjl8mk9W5vP32243a78GDB/GPf/yjwe0HDhyIq1evQqPRNOr1GqNbt25QqVQNvmpnq7Zs2YLc3FyMGzfOuC4gIOCen4W6bhZ2P5rzOPztb39Damoqfv755ybfNxHZJrMCi1KpRJ8+fZCSkmJcZzAYkJKSggEDBtS63YIFCzB37lxs374dffv2vef522Hl9OnT+P7779GqVStzyrJJV69eNS6JiYlwd3c3WTd16lRjWyEEqqurG7Tf1q1bw9nZucF1KJVK+Pj4tNj4nz179qCsrAyjR4/G6tWrW+Q16yLlvD1LlizBpEmT7hnL8e6775r8LKSlpTX5azf3cVAqlRg/fjyWLFnS5PsmIttk9kdCsbGxWL58OVavXo0TJ07g5ZdfRklJCSZNmgQAmDBhAmbMmGFsP3/+fMyaNQufffYZAgICkJOTg5ycHBQXFwO49YYwevRo/Pbbb1izZg30er2xTWVlZRN10/r4+PgYF41GA5lMZnx88uRJuLm5Ydu2bejTpw9UKhX27NmDs2fPYuTIkdBqtXB1dUW/fv3w/fffm+z37o+EZDIZPv30Uzz55JNwdnZG586dsWXLFuPzd39ksGrVKnh4eOC7775DcHAwXF1dMXz4cFy9etW4TXV1NV599VV4eHigVatWmDZtGiZOnIgnnnii3n6vWLEC48ePx1//+ld89tln9zyfnZ2NqKgoeHl5wcXFBX379sX+/fuNz//nP/9Bv379oFar4e3tjSeffNKkr1999ZXJ/jw8PLBq1SoAwIULFyCTybBhwwYMGjQIarUaa9aswfXr1xEVFQVfX184OzujV69eWLduncl+DAYDFixYgKCgIKhUKrRv3x7vv/8+AGDIkCGYPHmySfv8/HwolUqT8H/38z/88EONH5u4ubmZ/Hy0bt3aWENcXBwCAwPh5OSE0NBQbNy40WTbb7/9Fl26dIGTkxP++Mc/4sKFCzW+fm3HYceOHVCr1fd8hDRlyhQMGTLE+Hj58uXw9/eHs7MznnzySSxatOieQfSPP/44tmzZgrKyshprICLL8cGOTCxJOQ29QcK5zEQjJCUlifbt2wulUin69+8vfv31V+NzgwYNEhMnTjQ+7tChgwBwzzJnzhwhhBDnz5+v8XkA4scff2xwTTqdTgAQOp3unufKysrE8ePHRVlZmRBCCIPBIEoqqiRZDAaD2f/fK1euFBqNxvj4xx9/FABESEiI2LFjhzhz5oy4fv26SE9PF8uWLRNHjx4Vp06dEjNnzhRqtVpcvHjR5Hh8+OGHxscAhJ+fn1i7dq04ffq0ePXVV4Wrq6u4fv26yWsVFBQYa3F0dBSRkZHi4MGD4tChQyI4OFiMHz/euM/33ntPeHl5iU2bNokTJ06Il156Sbi7u4uRI0fW2c/CwkLh4uIijh07Jqqrq4VWqxU//fST8fmioiLRsWNH8dBDD4mff/5ZnD59WmzYsEH88ssvQgghtm7dKhQKhZg9e7Y4fvy4SE9PF/PmzTPp6+bNm01eU6PRiJUrVwohfv9ZDAgIEF9++aU4d+6cuHLlisjOzhYLFy4UaWlp4uzZs2LJkiVCoVCI/fv3G/fz5ptvCk9PT7Fq1Spx5swZ8fPPP4vly5cLIYRYs2aN8PT0FOXl5cb2ixYtEgEBAbX+PGzatEm4uLgIvV5vsv7u43en9957T3Tr1k1s375dnD17VqxcuVKoVCqxa9cuIYQQly5dEiqVSsTGxoqTJ0+Kf//730Kr1Zoc3/qOw+3Hn376qbH93ev27Nkj5HK5WLhwocjMzBRLly4VXl5eJj/DQghRUlIi5HJ5ref53ectEUnjp1N5ImD6VtFh2lax53R+k++/rvfvOzUqsFgicwJLSUWV6DBtqyRLSUWV2X2rLbB89dVX9W7bo0cPkZSUZHxcU2CZOXOm8XFxcbEAILZt22byWncGFgDizJkzxm2WLl0qtFqt8bFWqxULFy40Pq6urhbt27evN7B88sknIiwszPh4ypQpJuH3448/Fm5ubsYwdbcBAwaIZ555ptb9NzSwJCYm1lmnEEKMGDFCvP7660KIW2/wKpXKGFDuVlZWJjw9PcWGDRuM60JCQsTbb79d6/4//PBD0bFjx3vWd+jQQSiVSuHi4mJcFi9eLMrLy4Wzs7MxvN3297//XURFRQkhhJgxY4bo3r27yfPTpk27J7DUdxymTJkihgwZYnz83XffCZVKZdzH2LFjxYgRI0xe55lnnrknsAghjCGvJgwsRNLLKywXfebuFB2mbRVvbTrSLK/R0MDCiQ6s2N3jgYqLizF16lQEBwfDw8MDrq6uOHHiBC5dulTnfu4cBO3i4gJ3d3fk5eXV2t7Z2RmdOnUyPm7btq2xvU6nQ25ursmtGhQKBfr06VNvfz777DM8++yzxsfPPvssvvjiCxQVFQEA0tPTER4eXuscPunp6Rg6dGi9r1Ofu/9f9Xo95s6di169esHLywuurq747rvvjP+vJ06cQEVFRa2vrVarTT5aSU1NxbFjx/Dcc8/VWkNZWVmtk6W98cYbSE9PNy4TJkzAmTNnUFpaij/96U9wdXU1Lv/6179w9uxZY50REREm+6pp7Fl9x+GZZ57Brl27cOXKFQDAmjVrMGLECONHPpmZmffcqqO2W3c4OTmhtLS01v8HIpKOwSAQ+3k6rhVXoKvWDbP+3F3Seuzq5oe3OTkqcPzdYZK9dlNxcXExeTx16lTs3LkTCQkJCAoKgpOTE0aPHl3vWCBHR0eTxzKZDAaDwaz24j7v0XT8+HH8+uuvOHDgAKZNm2Zcr9frsX79erzwwgv1zlJc3/M11VnToNq7/18XLlyIxYsXIzExEb169YKLiwtiYmKM/68NmT35+eefR1hYGLKzs7Fy5UoMGTIEHTp0qLW9t7c3CgoKan0uKCjIZF1mZiYA4JtvvoGvr6/JcyqVqt76bmvIcejXrx86deqE9evX4+WXX8bmzZuN44DMdePGDeMYHCKyLJ/8fA4/n74GtaMcyePDoW7C96/GsMvAIpPJ4Ky0va7v3bsXzz33nHGgaXFxca2DKpuLRqOBVqvFwYMH8fDDDwO49WaXmpqKsLCwWrdbsWIFHn74YSxdutRk/cqVK7FixQq88MILCAkJwaeffoobN27UeJUlJCQEKSkpxgHgd2vdurXJ4ODTp0836K/7vXv3YuTIkcarDgaDAadOnUL37rf+2ujcuTOcnJyQkpKC559/vsZ99OrVC3379sXy5cuxdu1a460tahMeHo6cnBwUFBTA09Oz3hq7d+8OlUqFS5cuYdCgQTW2CQ4ONhlQDQC//vqryeOGHAfg1lWWNWvWwM/PD3K5HCNGjDC27dq1Kw4ePGiy/d2PAeDs2bMoLy9HeHh4vf0jopaVeqkACd/d+kPonb/0QGetm8QVtfDEcdS8OnfujE2bNiE9PR2HDx/G+PHj67xS0lyio6MRFxeHr7/+GpmZmZgyZQoKCgpq/Wp0VVUV/u///g9RUVHo2bOnyfL8889j//79yMjIQFRUFHx8fPDEE09g7969OHfuHL788kvjbSHmzJmDdevWYc6cOThx4gSOHj2K+fPnG19nyJAhSE5ORlpaGn777Te89NJL91wtqknnzp2xc+dO/PLLLzhx4gRefPFFk7mI1Go1pk2bhjfffNP4Ecyvv/6KFStWmOzn+eefR3x8PIQQJt9eqkl4eDi8vb2xd+/eeusDbn1zaOrUqXjttdewevVqnD17FqmpqUhKSjJ+Lfmll17C6dOn8cYbbyAzMxNr1641uTLS0OMA3AosqampeP/99zF69GiTqzjR0dH49ttvsWjRIpw+fRoff/wxtm3bds/x//nnn9GxY0eTjxeJSHq6sipEr01DtUHg8dB2GNPXMmaSZ2CxIYsWLYKnpycGDhyIxx9/HMOGDUPv3r1bvI7bt1aYMGECBgwYAFdXVwwbNqzWMRlbtmzB9evXa3wTDw4ORnBwMFasWAGlUokdO3agTZs2eOyxx9CrVy/Ex8dDobh1mXLw4MH44osvsGXLFoSFhWHIkCE4cOCAcV8ffPAB/P398dBDD2H8+PGYOnVqg+akmTlzJnr37o1hw4Zh8ODBxtB0p1mzZuH111/H7NmzERwcjLFjx94zDigqKgoODg6Iioqq92Z+CoUCkyZNwpo1a+qt77a5c+di1qxZiIuLQ3BwMIYPH45vvvkGgYGBAID27dvjyy+/xFdffYXQ0FAsW7YM8+bNM27f0OMAAEFBQejfvz+OHDmCZ555xqTtAw88gGXLlmHRokUIDQ3F9u3b8dprr93T53Xr1hmv2BCRZRBCYPqXR3D5Zhnaeznj/Sd7Wsx9+My+W7OlasjdmnnXV2kYDAYEBwdjzJgxmDt3rtTlSObChQvo1KkTDh482KAgmZOTgx49eiA1NbXO8S7W4IUXXsDJkyeNM9tmZGRgyJAhOHXqVK2zKPO8JWp5a/ZfxD83H4ODXIYvXx6IUH+PZn/Nht6t2fYGcpDkLl68iB07dmDQoEGoqKhAcnIyzp8/j/Hjx0tdmiSqqqpw/fp1zJw5E3/4wx8afNXLx8cHK1aswKVLl6wusCQkJOBPf/oTXFxcsG3bNqxevRr/+7//a3z+6tWr+Ne//tWit3wgorqdzCnEu/85DgCYNrxbi4QVczCwUJOTy+VYtWoVpk6dCiEEevbsie+//x7BwcFSlyaJvXv34o9//CO6dOlyz8yz9WnI7MCW6MCBA1iwYAGKiorQsWNHLFmyxGRAcm13ayciaZRWVmPy2jRUVBswuGtr/P3BQKlLugcDCzU5f3//Bg8WtQeDBw++7699W5vPP/9c6hKIyAzv/uc4zuQVo42bCglPh0Iut4xxK3fioFsiIiI7tuXwFaw/mAWZDEgcGwZv14bP3dSSGFiIiIjs1KXrpXhr01EAQPQfgzAwyFviimpnV4HF3i7LE1kznq9Ezauy2oDodakorqhGvwBPvDq0s9Ql1ckuAsvteTrqm6KeiCzH7VmIGzK5HxGZL2FHJg5n66BxcsTiceFwUFh2JLCLQbcODg5wdnZGfn4+HB0dIZdb9kEhsmdCCJSWliIvLw8eHh7GPziIqOn8mJmHT346BwBYODoE7Tzqvyea1OwisMhkMrRt2xbnz5/HxYsXpS6HiBrAw8MDPj4+UpdBZHNyC8vx+ueHAQDPDQzAIz2s4zyzi8ACAEqlEp07d+bHQkRWwNHRkVdWiJqB3iDw2oZ03CipRPe27pj+aDepS2owuwkswK0JzTjFNxER2auPdp3BL2evw1mpQNL4cKgdrecPAw7mICIisgMHL9zAop2nAABzR/ZEp9auEldkHgYWIiIiG3eztBKvrkuDQQBPhftiVB8/qUsyGwMLERGRDRNC4I2NR3BVV45Abxe8+0RPqUtqFAYWIiIiG/avfRex83gulAo5kqLC4aqyzuGrDCxEREQ2KuOKDu9/cwIAMOOxbujpq5G4osZjYCEiIrJBJRXViF6bhkq9AZHBbfDcwACpS7ovDCxEREQ2aPbXGTh3rQQ+7mosHB0KmUwmdUn3hYGFiIjIxmxKzcaXqdmQy4AlUeHwdFFKXdJ9Y2AhIiKyIefyizHzq2MAgJjILugf6CVxRU2DgYWIiMhGVFTrEb0uDaWVevyhoxde+WOQ1CU1GQYWIiIiGxG/7SQyrhTCy0WJxePCoZBb97iVOzGwEBER2YCdx3Oxcu8FAEDC0yHQutvWvfMYWIiIiKzcVV0Z3th4GADw/IOBGNJNK3FFTY+BhYiIyIpV6w2Ysi4dN0ur0MtXgzeHd5O6pGbBwEJERGTFkn44gwMXbsBV5YCkqHAoHWzzrd02e0VERGQH9p29jqQfTgMA3n+yJwK8XSSuqPkwsBAREVmh68UViNmQBoMAxvT1w8gwX6lLalYMLERERFZGCIGpXxxGbmEFOrV2wdt/6SF1Sc2OgYWIiMjKrNhzHj9m5kPpIEfy+N5wVjpIXVKzY2AhIiKyIkeyb2L+9pMAgFl/7o7gtu4SV9QyGFiIiIisRFF5FaLXpaFKLzC8hw+ejWgvdUkthoGFiIjICggh8M/Nx3Dxeil8PZwwf1QIZDLbmXq/PgwsREREVuCLQ9nYcvgKFHIZlkSFQ+PsKHVJLYqBhYiIyMKdySvCnK8zAACvP9IFfTp4SlxRy2NgISIismDlVXpMXpuGsio9HgzyxksPd5K6JEkwsBAREVmw9785gZM5RfB2VWLR2FDI5fYzbuVODCxEREQWavuxq/i/Xy8CABaNCUMbN7XEFUmHgYWIiMgCZReU4s2NRwAALw3qhIe7tJa4ImkxsBAREVmYKr0Br65LQ2F5NcL8PfD6I12kLklyDCxEREQW5sOdp5B66Sbc1A5IigqHo4Jv1436H1i6dCkCAgKgVqsRERGBAwcO1Np2+fLleOihh+Dp6QlPT09ERkbe014IgdmzZ6Nt27ZwcnJCZGQkTp8+3ZjSiIiIrNrPp/Px0e6zAID4p0Lg7+UscUWWwezAsmHDBsTGxmLOnDlITU1FaGgohg0bhry8vBrb79q1C1FRUfjxxx+xb98++Pv745FHHsHly5eNbRYsWIAlS5Zg2bJl2L9/P1xcXDBs2DCUl5c3vmdERERWJr+oAq9tOAwhgPER7TEipK3UJVkMmRBCmLNBREQE+vXrh+TkZACAwWCAv78/oqOjMX369Hq31+v18PT0RHJyMiZMmAAhBNq1a4fXX38dU6dOBQDodDpotVqsWrUK48aNa1BdhYWF0Gg00Ol0cHe3jxtBERGR7TAYBCauPICfT19DV60bvp78ANSOCqnLanYNff826wpLZWUlDh06hMjIyN93IJcjMjIS+/bta9A+SktLUVVVBS8vLwDA+fPnkZOTY7JPjUaDiIiIBu+TiIjI2n3y8zn8fPoa1I5yJI0Pt4uwYg4Hcxpfu3YNer0eWq3WZL1Wq8XJkycbtI9p06ahXbt2xoCSk5Nj3Mfd+7z9XE0qKipQUVFhfFxYWNig1yciIrI0qZcKkPBdJgDg7cd7oIvWTeKKLE+LDjuOj4/H+vXrsXnzZqjV9zf5TVxcHDQajXHx9/dvoiqJiIhajq6sCq+uS0O1QeDPIW0xth/fz2piVmDx9vaGQqFAbm6uyfrc3Fz4+PjUuW1CQgLi4+OxY8cOhISEGNff3s7cfc6YMQM6nc64ZGVlmdMVIiIiyQkh8Namo8guKEN7L2fMe6oXZDL7nHq/PmYFFqVSiT59+iAlJcW4zmAwICUlBQMGDKh1uwULFmDu3LnYvn07+vbta/JcYGAgfHx8TPZZWFiI/fv317lPlUoFd3d3k4WIiMiarDuQhW+OXoWDXIakqHC4qx2lLslimTWGBQBiY2MxceJE9O3bF/3790diYiJKSkowadIkAMCECRPg6+uLuLg4AMD8+fMxe/ZsrF27FgEBAcZxKa6urnB1dYVMJkNMTAzee+89dO7cGYGBgZg1axbatWuHJ554oul6SkREZEEyc4rwzn8yAADThndDqL+HtAVZOLMDy9ixY5Gfn4/Zs2cjJycHYWFh2L59u3HQ7KVLlyCX/37h5qOPPkJlZSVGjx5tsp85c+bg7bffBgC8+eabKCkpwT/+8Q/cvHkTDz74ILZv337f41yIiIgsUVmlHpPXpqKi2oDBXVvj7w8GSl2SxTN7HhZLxXlYiIjIWszYdATrDmShjZsK3055CN6uKqlLkkyzzMNCRERE9+c/h69g3YEsyGRA4tgwuw4r5mBgISIiaiGXrpfirU1HAQCT/xiEgUHeEldkPRhYiIiIWkBltQHR69NQVFGNvh08MWVoZ6lLsioMLERERC0gYUcmDmfdhMbJEYujwuGg4FuwOfi/RURE1Mx+zMzDJz+dAwAsGB0CXw8niSuyPgwsREREzSi3sByvf34YADBxQAcM61H3zPBUMwYWIiKiZqI3CLy2IR03SioR3NYdMx4Llrokq8XAQkRE1Ew+2nUGv5y9DmelAsnjw6F2VEhdktViYCEiImoGBy/cwIffnwYAvDuyJzq1dpW4IuvGwEJERNTEbpZWYsq6NOgNAk+G+2JUb1+pS7J6DCxERERNSAiBNzcewRVdOQK9XTD3iZ6QyWRSl2X1GFiIiIia0P/9ehE7judCqZAjKSocriqz7zNMNWBgISIiaiIZV3R4b+sJAMCMx7qhp69G4opsBwMLERFREyipqEb0ujRU6g2IDG6D5wYGSF2STWFgISIiagJztmTgXH4JfNzVWDg6lONWmhgDCxER0X3anJaNjYeyIZcBi8eFwdNFKXVJNoeBhYiI6D6cv1aCmZuPAQCmDO2CiI6tJK7INjGwEBERNVJFtR7R61JRUqlHRKAXJg8Jkrokm8XAQkRE1Ejx207i2OVCeDo7YvG4cCjkHLfSXBhYiIiIGmHn8Vys3HsBAPDBmFD4aNTSFmTjGFiIiIjMdFVXhjc2HgYA/P3BQAzpppW4ItvHwEJERGSGar0BU9al42ZpFXr5avDm8K5Sl2QXGFiIiIjMkPTDGRy4cAMuSgWSosKhclBIXZJdYGAhIiJqoH1nryPph9MAgHlP9UKAt4vEFdkPBhYiIqIGuFFSiZgNaTAI4Ok+fhgZ5it1SXaFgYWIiKgeQghM/eIwcgsr0Km1C94Z2UPqkuwOAwsREVE9Ptt7AT+czIPSQY7k8b3hrHSQuiS7w8BCRERUh6PZOsRvOwEAmPXn7ghu6y5xRfaJgYWIiKgWReVVmLwuFVV6geE9fPBsRHupS7JbDCxEREQ1EEJg5lfHcPF6KXw9nDB/VAhkMk69LxUGFiIiohpsPJSNr9OvQCGXYUlUGDTOjlKXZNcYWIiIiO5yJq8Ys7/OAADE/qkL+nTwkrgiYmAhIiK6Q3mVHpPXpqKsSo8Hg7zx8qBOUpdEYGAhIiIy8f43J3AypwjerkosGhsKuZzjViwBAwsREdF/bT92Ff/360UAwAdjwtDGTS1xRXQbAwsRERGA7IJSvLnxCADgxUEdMahLa4krojsxsBARkd2r0hvw6ro0FJZXI8zfA1Mf6Sp1SXQXBhYiIrJ7id+fQuqlm3BTOSApKhyOCr49WhoeESIismt7Tl/D/+46CwCIHxUCfy9niSuimjCwEBGR3covqsBrn6dDCCCqf3uMCGkrdUlUCwYWIiKySwaDwOtfHEZ+UQW6aF0x+8/dpS6J6sDAQkREdmn5z+fw06l8qB3lSB7fG05KhdQlUR0YWIiIyO6kXSrAwu8yAQBvP94DXbRuEldE9WFgISIiu6Irq0L0ujRUGwT+HNIWY/v5S10SNQADCxER2Q0hBN7adBTZBWXw93LCvKd6QSbj1PvWgIGFiIjsxvqDWfjm6FU4yGVIiuoNd7Wj1CVRAzGwEBGRXTiVW4S3t2QAAN4c3hVh/h7SFkRmYWAhIiKbV1apxytrUlFRbcCgLq3x/IMdpS6JzNSowLJ06VIEBARArVYjIiICBw4cqLVtRkYGRo0ahYCAAMhkMiQmJt7TRq/XY9asWQgMDISTkxM6deqEuXPnQgjRmPKIiIhMvLs1A6fzitHaTYUPxoRCLue4FWtjdmDZsGEDYmNjMWfOHKSmpiI0NBTDhg1DXl5eje1LS0vRsWNHxMfHw8fHp8Y28+fPx0cffYTk5GScOHEC8+fPx4IFC5CUlGRueURERCb+c/gK1h3IgkwGJI4Ng7erSuqSqBFkwszLGBEREejXrx+Sk5MBAAaDAf7+/oiOjsb06dPr3DYgIAAxMTGIiYkxWf/nP/8ZWq0WK1asMK4bNWoUnJyc8O9//7tBdRUWFkKj0UCn08Hd3d2cLhERkY26dL0UI5b8jKKKakz+YxCmDuNdmC1NQ9+/zbrCUllZiUOHDiEyMvL3HcjliIyMxL59+xpd7MCBA5GSkoJTp04BAA4fPow9e/bg0UcfrXWbiooKFBYWmixERES3VVYbEL0+DUUV1ejbwRMxkZ2lLonug4M5ja9duwa9Xg+tVmuyXqvV4uTJk40uYvr06SgsLES3bt2gUCig1+vx/vvv45lnnql1m7i4OLzzzjuNfk0iIrJtH+zIxOGsm3BXO2BxVDgcFPyeiTWziKP3+eefY82aNVi7di1SU1OxevVqJCQkYPXq1bVuM2PGDOh0OuOSlZXVghUTEZEl25WZh49/OgcAWDA6FL4eThJXRPfLrCss3t7eUCgUyM3NNVmfm5tb64DahnjjjTcwffp0jBs3DgDQq1cvXLx4EXFxcZg4cWKN26hUKqhUHDhFRESm8grL8frnhwEAEwZ0wPCejX9/Isth1hUWpVKJPn36ICUlxbjOYDAgJSUFAwYMaHQRpaWlkMtNS1EoFDAYDI3eJxER2R+9QSBmQzqul1QiuK073nosWOqSqImYdYUFAGJjYzFx4kT07dsX/fv3R2JiIkpKSjBp0iQAwIQJE+Dr64u4uDgAtwbqHj9+3Pjvy5cvIz09Ha6urggKCgIAPP7443j//ffRvn179OjRA2lpaVi0aBH+9re/NVU/iYjIDizbfRa/nL0OZ6UCyePDoXZUSF0SNRGzA8vYsWORn5+P2bNnIycnB2FhYdi+fbtxIO6lS5dMrpZcuXIF4eHhxscJCQlISEjAoEGDsGvXLgBAUlISZs2ahf/5n/9BXl4e2rVrhxdffBGzZ8++z+4REZG9+O3CDSzaeevbpu+O7IlOrV0lroiaktnzsFgqzsNCRGS/bpZWYsSSPbh8swxPhvti0ZhQ3oXZSjTLPCxERESWRgiBaV8eweWbZQho5Yy5T/RkWLFBDCxERGTV/v3rRXyXkQtHhQzJ43vDVWX2aAeyAgwsRERktY5fKcTcb04AAGY8GoyevhqJK6LmwsBCRERWqaSiGpPXpaKy2oCh3dpg0gMBUpdEzYiBhYiIrNKcLRk4l18CH3c1Fj7NQba2joGFiIiszua0bGw8lA25DEgcFwYvF6XUJVEzY2AhIiKrcv5aCWZuPgYAeHVoZ/yhYyuJK6KWwMBCRERWo6Jaj+h1qSip1CMi0AvRQzpLXRK1EAYWIiKyGvO3ZeLY5UJ4OjsicVwYFHKOW7EXDCxERGQVvj+ei8/2ngcAJDwdirYaJ4kropbEwEJERBbvqq4Mb2w8DAD42wOBGBqslbgiamkMLEREZNH0BoEp69NRUFqFnr7umPZoV6lLIgkwsBARkUVL+uE0Dpy/ARelAslRvaFyUEhdEkmAgYWIiCzWr+euY0nKaQDAvKd6IcDbReKKSCoMLEREZJFulFQiZn06DAJ4uo8fRob5Sl0SSYiBhYiILI4QAm98cRg5heXo2NoF74zsIXVJJDEGFiIisjgr915Aysk8KB3kSI7qDWelg9QlkcQYWIiIyKIczdYhbtsJAMCsEcHo3s5d4orIEjCwEBGRxSgqr8Lkdamo0gsM66HFs3/oIHVJZCEYWIiIyCIIITDzq2O4eL0Uvh5OWDAqFDIZp96nWxhYiIjIImw8lI2v069AIZdh8bgwaJwdpS6JLAgDCxERSe5MXjFmf50BAIj9Uxf0DfCSuCKyNAwsREQkqfIqPSavTUVZlR4PBLXCS4M6SV0SWSAGFiIiktS8b0/gZE4RWrko8eGYMCjkHLdC92JgISIiyWw/loN/7bsIAPhgTCjauKslrogsFQMLERFJIrugFG9uPAwAePHhjhjctY3EFZElY2AhIqIWV603YMr6dBSWVyPU3wOvP9JV6pLIwjGwEBFRi0v8/jQOXSyAm8oByVHhUDrw7Yjqxp8QIiJqUXvPXMPSXWcAAPGjQuDv5SxxRWQNGFiIiKjFXCuuQMyGdAgBRPVvjxEhbaUuiawEAwsREbUIg0Hg9c8PI7+oAl20rpj95+5Sl0RWhIGFiIhaxKd7zmH3qXyoHeVIHt8bTkqF1CWRFWFgISKiZpd2qQALtmcCAOY83gNdtG4SV0TWhoGFiIiala6sCtHr0lBtEBgR0hbj+vlLXRJZIQYWIiJqNkIIvLXpKLILyuDn6YS4p3pBJuPU+2Q+BhYiImo26w9m4ZujV+EglyEpKhzuakepSyIrxcBCRETN4lRuEd7ekgEAeGNYV4S395S4IrJmDCxERNTkyir1mLw2FRXVBjzcpTVeeKij1CWRlWNgISKiJvfu1uM4lVuM1m4qLBoTCrmc41bo/jCwEBFRk9p65ArWHbgEmQz4cEwYvF1VUpdENoCBhYiImkzWjVLM+PIoAOB/BnfCg529Ja6IbAUDCxERNYkqvQGT16WhqKIafTp44rXILlKXRDaEgYWIiJpEwo5MHM66CXe1A5ZEhcNBwbcYajr8aSIiovu2+1Q+Pt59DgCwYHQofD2cJK6IbA0DCxER3Ze8wnLEbkgHAEwY0AHDe/pIWxDZJAYWIiJqNINB4LXP03G9pBLdfNzw1mPBUpdENoqBhYiIGu2j3Wex98x1ODkqkDy+N9SOCqlLIhvVqMCydOlSBAQEQK1WIyIiAgcOHKi1bUZGBkaNGoWAgADIZDIkJibW2O7y5ct49tln0apVKzg5OaFXr1747bffGlMeERG1gN8u3MCinacAAO+O7IGgNq4SV0S2zOzAsmHDBsTGxmLOnDlITU1FaGgohg0bhry8vBrbl5aWomPHjoiPj4ePT82faxYUFOCBBx6Ao6Mjtm3bhuPHj+ODDz6ApyfvO0FEZIlullZiyvp06A0CT4S1w+g+flKXRDZOJoQQ5mwQERGBfv36ITk5GQBgMBjg7++P6OhoTJ8+vc5tAwICEBMTg5iYGJP106dPx969e/Hzzz+bV/0dCgsLodFooNPp4O7u3uj9EBFR3YQQeOnfh/BdRi4CWjlj66sPwVXlIHVZZKUa+v5t1hWWyspKHDp0CJGRkb/vQC5HZGQk9u3b1+hit2zZgr59++Lpp59GmzZtEB4ejuXLl9e5TUVFBQoLC00WIiJqfv/+9SK+y8iFo0KGpKjeDCvUIswKLNeuXYNer4dWqzVZr9VqkZOT0+gizp07h48++gidO3fGd999h5dffhmvvvoqVq9eXes2cXFx0Gg0xsXf37/Rr09ERA1z/Eoh5n5zAgAw/dFg9PLTSFwR2QuL+JaQwWBA7969MW/ePISHh+Mf//gHXnjhBSxbtqzWbWbMmAGdTmdcsrKyWrBiIiL7U1pZjcnrUlFZbcDQbm3wtwcCpC6J7IhZgcXb2xsKhQK5ubkm63Nzc2sdUNsQbdu2Rffu3U3WBQcH49KlS7Vuo1Kp4O7ubrIQEVHzmfN1Bs7ll0DrrsLCp0Mhk8mkLonsiFmBRalUok+fPkhJSTGuMxgMSElJwYABAxpdxAMPPIDMzEyTdadOnUKHDh0avU8iImo6X6VdxheHsiGXAYvHhcPLRSl1SWRnzB4pFRsbi4kTJ6Jv377o378/EhMTUVJSgkmTJgEAJkyYAF9fX8TFxQG4NVD3+PHjxn9fvnwZ6enpcHV1RVBQEADgtddew8CBAzFv3jyMGTMGBw4cwCeffIJPPvmkqfpJRESNdOFaCf65+SgAIHpIZ/yhYyuJKyJ7ZPbXmgEgOTkZCxcuRE5ODsLCwrBkyRJEREQAAAYPHoyAgACsWrUKAHDhwgUEBgbes49BgwZh165dxsdbt27FjBkzcPr0aQQGBiI2NhYvvPBCg2vi15qJiJpeRbUeoz76BccuFyIi0AtrX/gDFHJ+FERNp6Hv340KLJaIgYWIqOnN3XocK/ach6ezI76d8hDaangXZmpazTIPCxER2Y+UE7lYsec8ACDh6VCGFZIUAwsREd0jR1eOqV8cBgD87YFADA3W1rMFUfNiYCEiIhN6g8CU9WkoKK1CT193THu0q9QlETGwEBGRqaQfTmP/+RtwUSqQFNUbKgeF1CURMbAQEdHvfj13HUtSTgMA3n+yFwK9XSSuiOgWBhYiIgIA3CipRMz6dBgEMLqPH54I95W6JCIjBhYiIoIQAm98cRg5heXo2NoF7/ylh9QlEZlgYCEiIqzcewEpJ/OgdJAjKSocLiqzJ0InalYMLEREdu5otg5x204AAGaOCEaPdhqJKyK6FwMLEZEdK66oRvS6VFTpBR7prsVf/8CbzpJlYmAhIrJTQgjM3HwUF66Xop1GjQWjQyCT8T5BZJkYWIiI7NSXqZfxVfoVKOQyLIkKh4ezUuqSiGrFwEJEZIfO5hdj1lfHAACvRXZG3wAviSsiqhsDCxGRnSmv0mPy2jSUVenxQFArvDw4SOqSiOrFwEJEZGfivj2BE1cL0cpFiQ/HhEEh57gVsnwMLEREduS7jBys3ncRAPDBmFC0cVdLXBFRwzCwEBHZics3y/DmxiMAgBcf7ojBXdtIXBFRwzGwEBHZgWq9AVPWpUFXVoVQfw+8/khXqUsiMgsDCxGRHUj8/jR+u1gAN5UDksaFQ+nAX/9kXfgTS0Rk4/aeuYalu84AAOY91QvtWzlLXBGR+RhYiIhs2LXiCsRsSIcQQFR/fzwe2k7qkogahYGFiMhGGQwCr39+GPlFFejcxhWz/9xD6pKIGo2BhYjIRn265xx2n8qHykGO5PG94aRUSF0SUaMxsBAR2aD0rJtYsD0TADDn8R7o6uMmcUVE94eBhYjIxhSWVyF6XSqqDQIjerVFVH9/qUsium8MLERENkQIgRmbjiLrRhn8PJ0w76lekMk49T5ZPwYWIiIbsuFgFr45chUOchmWRIVD4+QodUlETYKBhYjIRpzKLcLb/8kAAEwd1hW923tKXBFR02FgISKyAeVVekxem4ryKgMe7tIa/3ioo9QlETUpBhYiIhvw7tbjOJVbjNZuKiwaEwq5nONWyLYwsBARWblvjlzF2v2XIJMBH44Jg7erSuqSiJocAwsRkRXLulGK6ZuOAAD+Z3AnPNjZW+KKiJoHAwsRkZWq0hsQvS4NReXV6NPBEzGRXaQuiajZMLAQEVmphB2ZSM+6CXe1AxaPC4Ojgr/SyXbxp5uIyArtPpWPj3efAwAsGB0CP09niSsial4MLEREViavsByxG9IBAH/9QwcM79lW2oKIWgADCxGRFTEYBF77PB3XSyrRzccN/xwRLHVJRC2CgYWIyIp8tPss9p65DidHBZLHh0PtqJC6JKIWwcBCRGQlDl28gUU7TwEA3hnZA0Ft3CSuiKjlMLAQEVkBXWkVXl2XDr1BYGRYOzzdx0/qkohaFAMLEZGFE0Jg2pdHcPlmGTq0csZ7T/SETMap98m+MLAQEVm4f++/hO0ZOXBUyJAUFQ43taPUJRG1OAYWIiILduJqIeZuPQ4AmDa8G0L8PKQtiEgiDCxERBaqtLIak9emorLagKHd2uDvDwZKXRKRZBhYiIgs1NtbMnA2vwRadxUWPh3KcStk1xhYiIgs0Nfpl/H5b9mQy4DF48Lh5aKUuiQiSTGwEBFZmAvXSvDPzccAANFDOuMPHVtJXBGR9BoVWJYuXYqAgACo1WpERETgwIEDtbbNyMjAqFGjEBAQAJlMhsTExDr3HR8fD5lMhpiYmMaURkRk1Sqq9Yhel4biimr0D/RC9JAgqUsisghmB5YNGzYgNjYWc+bMQWpqKkJDQzFs2DDk5eXV2L60tBQdO3ZEfHw8fHx86tz3wYMH8fHHHyMkJMTcsoiIbMKC7Zk4elkHD2dHLB4XBgcFL4QTAY0ILIsWLcILL7yASZMmoXv37li2bBmcnZ3x2Wef1di+X79+WLhwIcaNGweVSlXrfouLi/HMM89g+fLl8PT0NLcsIiKrl3IiFyv2nAcAJIwORVuNk8QVEVkOswJLZWUlDh06hMjIyN93IJcjMjIS+/btu69CXnnlFYwYMcJk33WpqKhAYWGhyUJEZK1ydOWY+sVhAMCkBwIQ2V0rcUVElsWswHLt2jXo9XpotaYnklarRU5OTqOLWL9+PVJTUxEXF9fgbeLi4qDRaIyLv79/o1+fiEhKeoPAlPVpKCitQo927pj+aDepSyKyOJJ/OJqVlYUpU6ZgzZo1UKvVDd5uxowZ0Ol0xiUrK6sZqyQiaj7JP5zB/vM34KJUIHl8b6gcFFKXRGRxHMxp7O3tDYVCgdzcXJP1ubm59Q6orc2hQ4eQl5eH3r17G9fp9Xr89NNPSE5ORkVFBRSKe09elUpV55gYIiJrsP/cdSxOOQUAeO/Jngj0dpG4IiLLZNYVFqVSiT59+iAlJcW4zmAwICUlBQMGDGhUAUOHDsXRo0eRnp5uXPr27YtnnnkG6enpNYYVIiJbUFBSiSnr02EQwKjefngy3E/qkogslllXWAAgNjYWEydORN++fdG/f38kJiaipKQEkyZNAgBMmDABvr6+xvEolZWVOH78uPHfly9fRnp6OlxdXREUFAQ3Nzf07NnT5DVcXFzQqlWre9YTEdkKIQTe2HgYOYXl6OjtgndH9pC6JCKLZnZgGTt2LPLz8zF79mzk5OQgLCwM27dvNw7EvXTpEuTy3y/cXLlyBeHh4cbHCQkJSEhIwKBBg7Br16777wERkRVa9csFfH8iD0qFHEnjw+GiMvvXMZFdkQkhhNRFNIXCwkJoNBrodDq4u7tLXQ4RUa2OXdbhqf/9BZV6A975Sw9MHBggdUlEkmno+7fk3xIiIrInxRXViF6Xhkq9AY9012LCgA5Sl0RkFRhYiIha0OyvjuH8tRK006ixYHQIZDKZ1CURWQUGFiKiFvLloWxsSrsMhVyGJVHh8HBWSl0SkdVgYCEiagFn84sx6+tjAIDXIjujb4CXxBURWRcGFiKiZlZepcfktWkordRjYKdWeHlwkNQlEVkdBhYiomYW9+0JnLhaiFYuSnw4NgwKOcetEJmLgYWIqBl9l5GD1fsuAgASxoRC697we6YR0e8YWIiImsnlm2V4c+MRAMA/Hu6IP3ZtI3FFRNaLgYWIqBlU6w2Ysi4NurIqhPppMPWRrlKXRGTVGFiIiJrB4pTT+O1iAdxUDkiK6g2lA3/dEt0PnkFERE3slzPXkPzjGQDAvKd6oX0rZ4krIrJ+DCxERE3oWnEFpmxIhxDAuH7+eDy0ndQlEdkEBhYioiZiMAhM/eIw8osqENTGFXMe7yF1SUQ2g4GFiKiJrNhzHrsy86FykCN5fDiclAqpSyKyGQwsRERN4HDWTczffhIAMPvx7ujm4y5xRUS2hYGFiOg+FZZXIXpdGqoNAiN6tcX4/u2lLonI5jCwEBHdByEE3tp0FJdulMLP0wnznuoFmYxT7xM1NQYWIqL78PlvWdh65Coc5DIsiQqHxslR6pKIbBIDCxFRI53OLcKcLRkAgKnDuqJ3e0+JKyKyXQwsRESNUF6lx+S1aSivMuChzt74x0MdpS6JyKYxsBARNcK7W48jM7cI3q4qLBoTBrmc41aImhMDCxGRmb45chVr91+CTAYkjg1DazeV1CUR2TwGFiIiM2TdKMX0TUcAAC8P6oQHO3tLXBGRfWBgISJqoCq9AdHr0lBUXo3e7T3w2p+6SF0Skd1gYCEiaqAPdpxCetZNuKsdsHhcOBwV/BVK1FJ4thERNcBPp/KxbPdZAMD8USHw93KWuCIi+8LAQkRUj7yicsR+ng4AePYP7fFor7bSFkRkhxhYiIjqYDAIxG44jGvFlejm44aZI7pLXRKRXWJgISKqw7KfzmLPmWtwclQgeXw41I4KqUsisksMLEREtTh0sQAf7DgFAHjnLz0Q1MZN4oqI7BcDCxFRDXSlVXh1XRr0BoGRYe3wdF8/qUsismsMLEREdxFCYPqmI7h8swwdWjnjvSd6Qibj1PtEUmJgISK6y5r9l7DtWA4cFTIkRYXDTe0odUlEdo+BhYjoDieuFuLdrccBANOGd0OIn4e0BRERAAYWIiKj0spqTF6bispqA4Z0a4O/PxgodUlE9F8MLERE//X2lgyczS+B1l2FhaNDOG6FyIIwsBARAfg6/TI+/y0bMhmQODYcrVxVUpdERHdgYCEiu3fhWgn+ufkYACB6SGcM6NRK4oqI6G4MLERk1yqrDYhel4biimr0D/DCq0OCpC6JiGrAwEJEdm3B9pM4elkHD2dHJI4Lg4OCvxaJLBHPTCKyWz+czMWne84DABaODkU7DyeJKyKi2jCwEJFdytGVY+oXRwAAzw0MwJ+6ayWuiIjqwsBCRHZHbxCI2ZCGGyWV6NHOHTMe6yZ1SURUDwYWIrI7S388g1/P3YCzUoGkqHCoHBRSl0RE9WBgISK7cuD8DSR+fwoA8N4TPdGxtavEFRFRQzCwEJHdKCipxJT1aTAIYFRvPzzV20/qkoiogRhYiMguCCHwxsYjuKorR0dvF7w7sofUJRGRGRoVWJYuXYqAgACo1WpERETgwIEDtbbNyMjAqFGjEBAQAJlMhsTExHvaxMXFoV+/fnBzc0ObNm3wxBNPIDMzszGlERHVaPUvF/D9iVwoFXIkjQ+Hi8pB6pKIyAxmB5YNGzYgNjYWc+bMQWpqKkJDQzFs2DDk5eXV2L60tBQdO3ZEfHw8fHx8amyze/duvPLKK/j111+xc+dOVFVV4ZFHHkFJSYm55RER3ePYZR3mfXsSAPDPEcHo0U4jcUVEZC6ZEEKYs0FERAT69euH5ORkAIDBYIC/vz+io6Mxffr0OrcNCAhATEwMYmJi6myXn5+PNm3aYPfu3Xj44YcbVFdhYSE0Gg10Oh3c3d0btA0R2b7iimo8nrQH56+V4E/dtfjkr314F2YiC9LQ92+zrrBUVlbi0KFDiIyM/H0HcjkiIyOxb9++xld7F51OBwDw8vJqsn0SkX2a/dUxnL9WgnYaNRaODmFYIbJSZn2Ie+3aNej1emi1pjNCarVanDx5skkKMhgMiImJwQMPPICePXvW2q6iogIVFRXGx4WFhU3y+kRkO748lI1NaZchlwGLo8Lh4ayUuiQiaiSL+5bQK6+8gmPHjmH9+vV1touLi4NGozEu/v7+LVQhEVmDs/nFmPX1MQDAa5Fd0C+AV2yJrJlZgcXb2xsKhQK5ubkm63Nzc2sdUGuOyZMnY+vWrfjxxx/h51f3/AgzZsyATqczLllZWff9+kRkG8qr9Ihem4bSSj0GdGyF//ljkNQlEdF9MiuwKJVK9OnTBykpKcZ1BoMBKSkpGDBgQKOLEEJg8uTJ2Lx5M3744QcEBgbWu41KpYK7u7vJQkQEAPHbTuL41UJ4uSiROC4MCjnHrRBZO7MnIoiNjcXEiRPRt29f9O/fH4mJiSgpKcGkSZMAABMmTICvry/i4uIA3Bqoe/z4ceO/L1++jPT0dLi6uiIo6NZfPa+88grWrl2Lr7/+Gm5ubsjJyQEAaDQaODnxdu9E1HA7MnKw6pcLAIAPng6F1l0tbUFE1CTM/lozACQnJ2PhwoXIyclBWFgYlixZgoiICADA4MGDERAQgFWrVgEALly4UOMVk0GDBmHXrl23iqhl1P7KlSvx3HPPNagmfq2ZiK7cLMOji3+GrqwKLzwUiH+O6C51SURUj4a+fzcqsFgiBhYi+1atNyBq+a84eKEAIX4abHxpIJQOFve9AiK6S7PMw0JEZKmWpJzGwQsFcFU5ICkqnGGFyMbwjCYiq/fL2WtI+vEMAGDeU73QoZWLxBURUVNjYCEiq3a9uAIx69MhBDCunz/+EtpO6pKIqBkwsBCR1TIYBF7/4jDyiioQ1MYVcx7vIXVJRNRMGFiIyGp9tvc8dmXmQ+UgR/L4cDgpFVKXRETNhIGFiKzS4aybmL/91j3MZj/eHd18+O1AIlvGwEJEVqewvArR69JQpRd4rJcPxvdvL3VJRNTMGFiIyKoIIfDWpqO4dKMUvh5OiHsqpNbJJ4nIdjCwEJFV+fy3LGw9chUKuQxJ48OhcXKUuiQiagEMLERkNU7nFmHOlgwAwNRHuqJ3e0+JKyKilsLAQkRWobxKj8lr01BeZcBDnb3x4sMdpS6JiFoQAwsRWYW5W48jM7cI3q4qLBoTBrmc41aI7AkDCxFZvG+PXsWa/ZcAAB+ODUVrN5XEFRFRS3OQugAiojuVV+mRXVCKrBtlyCooRdaNUqw/mAUAeHlwJzzUubXEFRKRFBhYiKhFVesNuKorR1ZBKbLvCCVZBWXIulGKvKKKGrcLb++B2D91aeFqichSMLAQUZMSQiC/uAJZN8r+e6XkjqslBaW4erMc1QZR5z5cVQ7w83SCv5cz/D2dEejtjCd7+8FRwU+xiewVAwsRma2wvMoYRLILSnHpxu9XSbILSlFeZahze6VCDj9PJ/h5OcP/jmDi7+UEf09neDg7cjI4IjLBwEJE97g1jqTsvx/b/P5xTdZ/x5boyqrq3F4mA9q6q+Hn5Yz2d4aR//67jZuK3/IhIrMwsBDZIb1B4Kqu7NZHNcYgUv84kju1clHWeoWknYcTlA78+IaImg4DC5ENEkLgWnGlMYhk33WF5MrNsnrHkbgoFbeCyF1hxN/LGX6eTnBR8dcHEbUc/sYhslJ3jyO58wpJdkEZyqr0dW6vVMjh6+lkMrj1zlDiyXEkRGRBGFiILFR5lR6Xb5b9Ppj1jiskWQWluFna8HEkd4cRfy8naN3UHEdCRFaDgYVIIibjSGoY3JpbWP84Ei8XJfyN37YxDSXtPNRQOShaoCdERM2PgYWomQghcL2k8q6Pam5/Bdi8cSR+NVwh8fN0hivHkRCRneBvO6L7UFReZTKF/N2DW+sbR+KokMHXw8kklPz+NWCOIyEiuo2BhagOFdV6XC4oQ1ZBGS7dKG3UOBIfdzX8PZ3hd+cVkv8OdNW6q6HgOBIionoxsJBd0xsEcgrL//ttm3sHt+YWlUPU/akNPJ0da/36L8eREBE1DQYWsmm1jSO5fYXkys0yVOnrTiTOSoUxiPjddYXE34vjSIiIWgJ/05LVK66oNrlCcncoKa00fxzJnR/deLkoOY6EiEhiDCxk8e4cR3J7QGv2HQNdCxowjkTrpjYGEb+7rpD4cBwJEZHFY2AhyekNArm3x5HcDiWNHUdSw+BWX08njiMhIrJyDCzU7IQQuFFSec8df29PJ3+5AeNInBwV99zLxv+/dwL283SCm9qxhXpDRERSYGChJlHXOJLsglKU1DOOxEEug6+nU62DW1txHAkRkV1jYKEGqaw23HFfm98HtN6eTv5GSWW9+9C6q+6Zh4TjSIiIqCEYWAhALeNI7hjcmlNY/zgSD2dHk3lI7hzc6uvhBLUjx5EQEVHjMLDYCSEECkqr7rlCcvtjnPsZR3I7pHAcCRERNRcGFhtSUlH9exipYXBrQ8aRtPNwqjWUeLtyHAkREUmDgcWKVFYbcOVmmckVkt/vb9O4cSR+d1wh8XFXw0Ehb4GeEBERmYeBxYIYDAK5ReX3XCG5Pbg1p7AchnrGkWicHO+ZqdXvv1//5TgSIiKyVgwsLUgIgZulVcYrI3eOI8kuKMPlgjJU6g117kPtKL/nmzbG6eS9nOHOcSRERGSDGFiaWGlldY1XSG6HkuKK6jq3V8hlaOehvhVK7ggiHEdCRET2jIHFTDWNI7n9VeDsG6W43oBxJG3cVKZzkdwxnXxbDceREBER3Y2BpQ5CCCT9cOa/H9/cukJyVVdW7zgSd7WDMYi0b2U6uNXPk+NIiIiIzMXAUgeZTIZ/7buAa8WmV03UjvJb40bumofk9nTyGieOIyEiImpKDCz1eG5gAACYDG5t7ariOBIiIqIWxMBSj8lDOktdAhERkd3j6E4iIiKyeAwsREREZPEaFViWLl2KgIAAqNVqRERE4MCBA7W2zcjIwKhRoxAQEACZTIbExMT73icRERHZF7MDy4YNGxAbG4s5c+YgNTUVoaGhGDZsGPLy8mpsX1paio4dOyI+Ph4+Pj5Nsk8iIiKyLzIhRD2zipiKiIhAv379kJycDAAwGAzw9/dHdHQ0pk+fXue2AQEBiImJQUxMTJPt87bCwkJoNBrodDq4u7ub0yUiIiKSSEPfv826wlJZWYlDhw4hMjLy9x3I5YiMjMS+ffsaVWhj91lRUYHCwkKThYiIiGyTWYHl2rVr0Ov10Gq1Juu1Wi1ycnIaVUBj9xkXFweNRmNc/P39G/X6REREZPms9ltCM2bMgE6nMy5ZWVlSl0RERETNxKyJ47y9vaFQKJCbm2uyPjc3t9YBtc21T5VKBZVK1ajXJCIiIuti1hUWpVKJPn36ICUlxbjOYDAgJSUFAwYMaFQBzbFPIiIisi1mT80fGxuLiRMnom/fvujfvz8SExNRUlKCSZMmAQAmTJgAX19fxMXFAbg1qPb48ePGf1++fBnp6elwdXVFUFBQg/ZJRERE9s3swDJ27Fjk5+dj9uzZyMnJQVhYGLZv324cNHvp0iXI5b9fuLly5QrCw8ONjxMSEpCQkIBBgwZh165dDdonERER2Tez52GxVJyHhYiIyPo09P3bZu7WfDt3cT4WIiIi63H7fbu+6yc2E1iKiooAgPOxEBERWaGioiJoNJpan7eZj4QMBgOuXLkCNzc3yGSyJttvYWEh/P39kZWVZbMfNdl6H9k/62frfWT/rJ+t97E5+yeEQFFREdq1a2cyBvZuNnOFRS6Xw8/Pr9n27+7ubpM/hHey9T6yf9bP1vvI/lk/W+9jc/Wvrisrt1ntTLdERERkPxhYiIiIyOIxsNRDpVJhzpw5Nn0bAFvvI/tn/Wy9j+yf9bP1PlpC/2xm0C0RERHZLl5hISIiIovHwEJEREQWj4GFiIiILB4DCxEREVk8uwoscXFx6NevH9zc3NCmTRs88cQTyMzMrHe7L774At26dYNarUavXr3w7bffmjwvhMDs2bPRtm1bODk5ITIyEqdPn26ubtSqMf1bvnw5HnroIXh6esLT0xORkZE4cOCASZvnnnsOMpnMZBk+fHhzdqVWjenjqlWr7qlfrVabtLHmYzh48OB7+ieTyTBixAhjG0s5hh999BFCQkKMk08NGDAA27Ztq3Mbazn/bjO3j9Z2DprbP2s6/wDz+2dN519N4uPjIZPJEBMTU2c7izgPhR0ZNmyYWLlypTh27JhIT08Xjz32mGjfvr0oLi6udZu9e/cKhUIhFixYII4fPy5mzpwpHB0dxdGjR41t4uPjhUajEV999ZU4fPiw+Mtf/iICAwNFWVlZS3TLqDH9Gz9+vFi6dKlIS0sTJ06cEM8995zQaDQiOzvb2GbixIli+PDh4urVq8blxo0bLdGlezSmjytXrhTu7u4m9efk5Ji0seZjeP36dZO+HTt2TCgUCrFy5UpjG0s5hlu2bBHffPONOHXqlMjMzBRvvfWWcHR0FMeOHauxvTWdf7eZ20drOwfN7Z81nX9CmN8/azr/7nbgwAEREBAgQkJCxJQpU2ptZynnoV0Flrvl5eUJAGL37t21thkzZowYMWKEybqIiAjx4osvCiGEMBgMwsfHRyxcuND4/M2bN4VKpRLr1q1rnsIbqCH9u1t1dbVwc3MTq1evNq6bOHGiGDlyZDNUeP8a0seVK1cKjUZT6/O2dgw//PBD4ebmZhJyLPkYenp6ik8//bTG56z5/LtTXX28m7Wdg0LU3T9rPv9uM+f4Wcv5V1RUJDp37ix27twpBg0aVGdgsZTz0K4+ErqbTqcDAHh5edXaZt++fYiMjDRZN2zYMOzbtw8AcP78eeTk5Ji00Wg0iIiIMLaRSkP6d7fS0lJUVVXds82uXbvQpk0bdO3aFS+//DKuX7/epLU2VkP7WFxcjA4dOsDf3x8jR45ERkaG8TlbO4YrVqzAuHHj4OLiYrLe0o6hXq/H+vXrUVJSggEDBtTYxprPP6BhfbybNZ2DDe2ftZ5/jTl+1nL+vfLKKxgxYsQ951dNLOU8tJmbH5rLYDAgJiYGDzzwAHr27Flru5ycHGi1WpN1Wq0WOTk5xudvr6utjRQa2r+7TZs2De3atTP5wRs+fDieeuopBAYG4uzZs3jrrbfw6KOPYt++fVAoFM1RfoM0tI9du3bFZ599hpCQEOh0OiQkJGDgwIHIyMiAn5+fTR3DAwcO4NixY1ixYoXJeks6hkePHsWAAQNQXl4OV1dXbN68Gd27d6+xrbWef+b08W7WcA6a0z9rPP8ae/ys4fwDgPXr1yM1NRUHDx5sUHtLOQ/tNrC88sorOHbsGPbs2SN1Kc2iMf2Lj4/H+vXrsWvXLpNBcePGjTP+u1evXggJCUGnTp2wa9cuDB06tEnrNkdD+zhgwACTv44GDhyI4OBgfPzxx5g7d25zl9lojTmGK1asQK9evdC/f3+T9ZZ0DLt27Yr09HTodDps3LgREydOxO7duxv8hm4NGttHazkHzemfNZ5/jT1+1nD+ZWVlYcqUKdi5c+c9g58tnV1+JDR58mRs3boVP/74I/z8/Ops6+Pjg9zcXJN1ubm58PHxMT5/e11tbVqaOf27LSEhAfHx8dixYwdCQkLqbNuxY0d4e3vjzJkzTVFuozSmj7c5OjoiPDzcWL+tHMOSkhKsX78ef//73+ttK+UxVCqVCAoKQp8+fRAXF4fQ0FAsXry4xrbWeP4B5vXxNms6BxvTv9us4fxrTP+s5fw7dOgQ8vLy0Lt3bzg4OMDBwQG7d+/GkiVL4ODgAL1ef882lnIe2lVgEUJg8uTJ2Lx5M3744QcEBgbWu82AAQOQkpJism7nzp3GvxgCAwPh4+Nj0qawsBD79+9v8GeeTaUx/QOABQsWYO7cudi+fTv69u1bb/vs7Gxcv34dbdu2vd+SzdbYPt5Jr9fj6NGjxvpt4RgCt752WFFRgWeffbbetlIew7sZDAZUVFTU+Jw1nX91qauPgHWdgzWpr393suTzrzYN6Z+1nH9Dhw7F0aNHkZ6eblz69u2LZ555Bunp6TV+RGUx52GTDd+1Ai+//LLQaDRi165dJl8vKy0tNbb561//KqZPn258vHfvXuHg4CASEhLEiRMnxJw5c2r8OpeHh4f4+uuvxZEjR8TIkSMl+UpeY/oXHx8vlEql2Lhxo8k2RUVFQohbI8mnTp0q9u3bJ86fPy++//570bt3b9G5c2dRXl7eov1rbB/feecd8d1334mzZ8+KQ4cOiXHjxgm1Wi0yMjKMbaz5GN724IMPirFjx96z3pKO4fTp08Xu3bvF+fPnxZEjR8T06dOFTCYTO3bsEEJY9/l3m7l9tLZz0Nz+WdP515j+3WYN519t7v6WkKWeh3YVWADUuNz5fflBgwaJiRMnmmz3+eefiy5dugilUil69OghvvnmG5PnDQaDmDVrltBqtUKlUomhQ4eKzMzMFuiRqcb0r0OHDjVuM2fOHCGEEKWlpeKRRx4RrVu3Fo6OjqJDhw7ihRdeuGcehZbSmD7GxMSI9u3bC6VSKbRarXjsscdEamqqyX6t+RgKIcTJkycFAOMv1TtZ0jH829/+Jjp06CCUSqVo3bq1GDp0qEnN1nz+3WZuH63tHDS3f9Z0/gnRuJ9Razn/anN3YLHU81AmhBBNd72GiIiIqOnZ1RgWIiIisk4MLERERGTxGFiIiIjI4jGwEBERkcVjYCEiIiKLx8BCREREFo+BhYiIiCweAwsRERFZPAYWIiIisngMLERERGTxGFiIiIjI4jGwEBERkcX7f94rmxx25rbMAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "x = range(1,len(mean_acc)+1)\n", + "plt.plot(x, mean_acc)\n", + "plt.legend(['Training Accuracy (FedAvg)'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7b21d109", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d95def48-a06f-48ca-8094-361d523a97b7", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "fedn", + "language": "python", + "name": "fedn" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/mnist-pytorch-fedprox/README.rst b/examples/mnist-pytorch-fedprox/README.rst new file mode 100644 index 000000000..18d688dc1 --- /dev/null +++ b/examples/mnist-pytorch-fedprox/README.rst @@ -0,0 +1,149 @@ +Quickstart Tutorial PyTorch (MNIST) +------------- + +This classic example of hand-written text recognition is well suited as a lightweight test when developing on FEDn in pseudo-distributed mode. +A normal high-end laptop or a workstation should be able to sustain a few clients. +The example automates the partitioning of data and deployment of a variable number of clients on a single host. +We here assume working experience with containers, Docker and docker-compose. + +Prerequisites +------------- + +- `Python 3.8, 3.9 or 3.10 `__ +- `Docker `__ +- `Docker Compose `__ + +Quick start +----------- + +Clone this repository, locate into this directory: + +.. code-block:: + + git clone https://github.com/scaleoutsystems/fedn.git + cd fedn/examples/mnist-pytorch + +Start a pseudo-distributed FEDn network using docker-compose: + +.. code-block:: + + docker-compose -f ../../docker-compose.yaml up + +This starts up the needed backend services MongoDB and Minio, the API Server and one Combiner. +You can verify the deployment using these urls: + +- API Server: http://localhost:8092/get_controller_status +- Minio: http://localhost:9000 +- Mongo Express: http://localhost:8081 + +Next, we will prepare the client. A key concept in FEDn is the compute package - +a code bundle that contains entrypoints for training and (optionally) validating a model update on the client. + +Locate into 'examples/mnist-pytorch' and familiarize yourself with the project structure. The entrypoints +are defined in 'client/entrypoint'. The dependencies needed in the client environment are specified in +'requirements.txt'. For convenience, we have provided utility scripts to set up a virtual environment. + +Start by initializing a virtual enviroment with all of the required dependencies for this project. + +.. code-block:: + + bin/init_venv.sh + +Next create the compute package and a seed model: + +.. code-block:: + + bin/build.sh + +You should now have a file 'package.tgz' and 'seed.npz' in the project folder. + +Next we prepare the local dataset. For this we download MNIST data and make data partitions: + +Download the data: + +.. code-block:: + + bin/get_data + + +Split the data in 10 partitions: + +.. code-block:: + + bin/split_data --n_splits=10 + +Data partitions will be generated in the folder 'data/clients'. + +FEDn relies on a configuration file for the client to connect to the server. Create a file called 'client.yaml' with the follwing content: + +.. code-block:: + + network_id: fedn-network + discover_host: api-server + discover_port: 8092 + +Make sure to move the file ``client.yaml`` to the root of the examples/mnist-pytorch folder. +To connect a client that uses the data partition ``data/clients/1/mnist.pt`` and the config file ``client.yaml`` to the network, run the following docker command: + +.. code-block:: + + docker run \ + -v $PWD/client.yaml:/app/client.yaml \ + -v $PWD/data/clients/1:/var/data \ + -e ENTRYPOINT_OPTS=--data_path=/var/data/mnist.pt \ + --network=fedn_default \ + ghcr.io/scaleoutsystems/fedn/fedn:master-mnist-pytorch run client -in client.yaml --name client1 + +Observe the API Server logs and combiner logs, you should see the client connecting and entering into a state asking for a compute package. + +In a separate terminal, start a second client using the data partition 'data/clients/2/mnist.pt': + +.. code-block:: + + docker run \ + -v $PWD/client.yaml:/app/client.yaml \ + -v $PWD/data/clients/2:/var/data \ + -e ENTRYPOINT_OPTS=--data_path=/var/data/mnist.pt \ + --network=fedn_default \ + ghcr.io/scaleoutsystems/fedn/fedn:master-mnist-pytorch run client -in client.yaml --name client2 + +You are now ready to use the API to initialize the system with the compute package and seed model, and to start federated training. + +- Follow the example in the `Jupyter Notebook `__ + + +Automate experimentation with several clients: +----------- + +Now that you have an understanding of the main components of FEDn, you can use the provided docker-compose templates to automate deployment of FEDn and clients. +To start the network and attach 4 clients: + +.. code-block:: + + docker-compose -f ../../docker-compose.yaml -f docker-compose.override.yaml up --scale client=4 + + +Access logs and validation data from MongoDB +----------- +You can access and download event logs and validation data via the API, and you can also as a developer obtain +the MongoDB backend data using pymongo or via the MongoExpress interface: + +- http://localhost:8081/db/fedn-network/ + +The credentials are as set in docker-compose.yaml in the root of the repository. + +Access model updates +----------- + +You can obtain model updates from the 'fedn-models' bucket in Minio: + +- http://localhost:9000 + + +Clean up +----------- +You can clean up by running + +.. code-block:: + + docker-compose down diff --git a/examples/mnist-pytorch-fedprox/client/entrypoint b/examples/mnist-pytorch-fedprox/client/entrypoint index c37f99fa5..f8d00378e 100755 --- a/examples/mnist-pytorch-fedprox/client/entrypoint +++ b/examples/mnist-pytorch-fedprox/client/entrypoint @@ -6,7 +6,7 @@ import os import docker import fire import torch - +import yaml from fedn.utils.helpers.helpers import get_helper, save_metadata, save_metrics import copy HELPER_MODULE = 'numpyhelper' @@ -117,7 +117,7 @@ def init_seed(out_path='seed.npz'): save_parameters(model, out_path) -def train(in_model_path, out_model_path, data_path=None, batch_size=32, epochs=1, lr=0.01, mu=3): +def train(in_model_path, out_model_path, data_path=None, batch_size=32, epochs=1, lr=0.01, client_settings_path ='/var/client_settings.yaml'): """ Complete a model update. Load model paramters from in_model_path (managed by the FEDn client), @@ -137,6 +137,17 @@ def train(in_model_path, out_model_path, data_path=None, batch_size=32, epochs=1 :param lr: The learning rate to use. :type lr: float """ + + with open(client_settings_path, 'r') as fh: + try: + client_settings = dict(yaml.safe_load(fh)) + except yaml.YAMLError as e: + raise + + mu = client_settings['mu'] + + print("mu: ", mu) + print("data_path: ", data_path) print(os.getcwd()) print("list data path: ", os.listdir('/var/data')) diff --git a/examples/mnist-pytorch-fedprox/client_settings.yaml b/examples/mnist-pytorch-fedprox/client_settings.yaml new file mode 100644 index 000000000..841373543 --- /dev/null +++ b/examples/mnist-pytorch-fedprox/client_settings.yaml @@ -0,0 +1 @@ +mu: 0.1 \ No newline at end of file diff --git a/examples/mnist-pytorch-fedprox/docker-compose.override.yaml b/examples/mnist-pytorch-fedprox/docker-compose.override.yaml new file mode 100644 index 000000000..c9d037449 --- /dev/null +++ b/examples/mnist-pytorch-fedprox/docker-compose.override.yaml @@ -0,0 +1,16 @@ +# Compose schema version +version: '3.3' + +# Overriding requirements +services: + client: + build: + args: + REQUIREMENTS: examples/mnist-pytorch-fedprox/requirements.txt + deploy: + replicas: 2 + volumes: + - ${HOST_REPO_DIR:-.}/fedn:/app/fedn + - ${HOST_REPO_DIR:-.}/examples/mnist-pytorch-fedprox/data:/var/data + - ${HOST_REPO_DIR:-.}/examples/mnist-pytorch-fedprox/client_settings.yaml:/var/client_settings.yaml + - /var/run/docker.sock:/var/run/docker.sock diff --git a/examples/mnist-pytorch-fedprox/requirements.txt b/examples/mnist-pytorch-fedprox/requirements.txt new file mode 100644 index 000000000..0bf7a6e78 --- /dev/null +++ b/examples/mnist-pytorch-fedprox/requirements.txt @@ -0,0 +1,4 @@ +torch==1.13.1 +torchvision==0.14.1 +fire==0.3.1 +docker==6.1.1 From 858a62cb05140c80ab92835f89e0a53705716f6f Mon Sep 17 00:00:00 2001 From: mattiasakesson <33224977+mattiasakesson@users.noreply.github.com> Date: Tue, 9 Apr 2024 17:49:41 +0200 Subject: [PATCH 3/6] Update README.rst --- examples/mnist-pytorch-fedprox/README.rst | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/examples/mnist-pytorch-fedprox/README.rst b/examples/mnist-pytorch-fedprox/README.rst index 18d688dc1..ed5a799fb 100644 --- a/examples/mnist-pytorch-fedprox/README.rst +++ b/examples/mnist-pytorch-fedprox/README.rst @@ -1,11 +1,11 @@ Quickstart Tutorial PyTorch (MNIST) -------------- +------------------------------------ This classic example of hand-written text recognition is well suited as a lightweight test when developing on FEDn in pseudo-distributed mode. A normal high-end laptop or a workstation should be able to sustain a few clients. The example automates the partitioning of data and deployment of a variable number of clients on a single host. We here assume working experience with containers, Docker and docker-compose. - + Prerequisites ------------- @@ -112,8 +112,9 @@ You are now ready to use the API to initialize the system with the compute packa - Follow the example in the `Jupyter Notebook `__ + Automate experimentation with several clients: ------------ +----------------------------------------------- Now that you have an understanding of the main components of FEDn, you can use the provided docker-compose templates to automate deployment of FEDn and clients. To start the network and attach 4 clients: @@ -124,7 +125,8 @@ To start the network and attach 4 clients: Access logs and validation data from MongoDB ------------ +--------------------------------------------- + You can access and download event logs and validation data via the API, and you can also as a developer obtain the MongoDB backend data using pymongo or via the MongoExpress interface: @@ -132,8 +134,12 @@ the MongoDB backend data using pymongo or via the MongoExpress interface: The credentials are as set in docker-compose.yaml in the root of the repository. +Adjust fed-Prox parameter μ +-------------------------------- +open file: client_settings.yaml and change mu value. + Access model updates ------------ +----------------------- You can obtain model updates from the 'fedn-models' bucket in Minio: From deb25985bc869d4aa7e8d8e37500d70f8f03daaa Mon Sep 17 00:00:00 2001 From: mattiasakesson Date: Tue, 9 Apr 2024 17:52:25 +0200 Subject: [PATCH 4/6] clean script --- examples/mnist-pytorch-fedprox/client/entrypoint | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/examples/mnist-pytorch-fedprox/client/entrypoint b/examples/mnist-pytorch-fedprox/client/entrypoint index f8d00378e..3ddb5b156 100755 --- a/examples/mnist-pytorch-fedprox/client/entrypoint +++ b/examples/mnist-pytorch-fedprox/client/entrypoint @@ -146,13 +146,8 @@ def train(in_model_path, out_model_path, data_path=None, batch_size=32, epochs=1 mu = client_settings['mu'] - print("mu: ", mu) + print("mu value: ", mu) - print("data_path: ", data_path) - print(os.getcwd()) - print("list data path: ", os.listdir('/var/data')) - - print("list data/clients path: ", os.listdir('/var/data/clients')) # Load data x_train, y_train = load_data(data_path) @@ -176,12 +171,8 @@ def train(in_model_path, out_model_path, data_path=None, batch_size=32, epochs=1 proximal_term = 0.0 for w, w_t in zip(model.parameters(), global_model.parameters()): proximal_term += (w - w_t).norm(2) - #print("proximal_term: ", proximal_term) - - # loss = criterion(outputs, batch_y) # <-- old - # loss = loss_function(y_pred, label) + (args.mu / 2) * proximal_term <-- fed prox term - loss = criterion(outputs, batch_y) + (mu / 2) * proximal_term # <-- new + loss = criterion(outputs, batch_y) + (mu / 2) * proximal_term loss.backward() optimizer.step() From c3cd6eca67960ae6760121b927a5f98467217290 Mon Sep 17 00:00:00 2001 From: mattiasakesson <33224977+mattiasakesson@users.noreply.github.com> Date: Tue, 9 Apr 2024 18:29:24 +0200 Subject: [PATCH 5/6] Update README.rst --- examples/mnist-pytorch-fedprox/README.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/examples/mnist-pytorch-fedprox/README.rst b/examples/mnist-pytorch-fedprox/README.rst index ed5a799fb..0fe4b078c 100644 --- a/examples/mnist-pytorch-fedprox/README.rst +++ b/examples/mnist-pytorch-fedprox/README.rst @@ -1,5 +1,10 @@ -Quickstart Tutorial PyTorch (MNIST) ------------------------------------- +[link text itself]: http://www.reddit.com + + +Quickstart Tutorial PyTorch with FEDProx (MNIST) +------------------------------------------------- +This is an enhanced version of our Pytorch MNIST example that let you use the FedProx algorithm [paper](/guides/content/editing-an-existing-page) +eg. [click here](www.google.com) This classic example of hand-written text recognition is well suited as a lightweight test when developing on FEDn in pseudo-distributed mode. A normal high-end laptop or a workstation should be able to sustain a few clients. @@ -137,6 +142,7 @@ The credentials are as set in docker-compose.yaml in the root of the repository. Adjust fed-Prox parameter μ -------------------------------- open file: client_settings.yaml and change mu value. +If mu is set to 0 it is vanilla fedavg. Access model updates ----------------------- From b1f2c14755bab291923708ad7801a9112f22efb4 Mon Sep 17 00:00:00 2001 From: mattiasakesson <33224977+mattiasakesson@users.noreply.github.com> Date: Tue, 9 Apr 2024 18:34:01 +0200 Subject: [PATCH 6/6] Update README.rst --- examples/mnist-pytorch-fedprox/README.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/mnist-pytorch-fedprox/README.rst b/examples/mnist-pytorch-fedprox/README.rst index 0fe4b078c..ee407f142 100644 --- a/examples/mnist-pytorch-fedprox/README.rst +++ b/examples/mnist-pytorch-fedprox/README.rst @@ -1,10 +1,9 @@ -[link text itself]: http://www.reddit.com Quickstart Tutorial PyTorch with FEDProx (MNIST) ------------------------------------------------- -This is an enhanced version of our Pytorch MNIST example that let you use the FedProx algorithm [paper](/guides/content/editing-an-existing-page) -eg. [click here](www.google.com) +This is an enhanced version of our Pytorch MNIST example that let you use the FedProx algorithm [paper](https://arxiv.org/abs/1812.06127). + This classic example of hand-written text recognition is well suited as a lightweight test when developing on FEDn in pseudo-distributed mode. A normal high-end laptop or a workstation should be able to sustain a few clients.