From c4ada0cec56d40467a6b59ff113d6fac68ba88cb Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Mon, 8 Jul 2024 12:03:15 +0200 Subject: [PATCH 1/9] start the controller using cli command --- fedn/cli/__init__.py | 1 + fedn/cli/controller_cmd.py | 19 +++++++++++++++++++ fedn/network/api/server.py | 6 ------ 3 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 fedn/cli/controller_cmd.py diff --git a/fedn/cli/__init__.py b/fedn/cli/__init__.py index 137fc9b9c..bcd27dc53 100644 --- a/fedn/cli/__init__.py +++ b/fedn/cli/__init__.py @@ -9,3 +9,4 @@ from .session_cmd import session_cmd # noqa: F401 from .status_cmd import status_cmd # noqa: F401 from .validation_cmd import validation_cmd # noqa: F401 +from .controller_cmd import controller_cmd # noqa: F401 diff --git a/fedn/cli/controller_cmd.py b/fedn/cli/controller_cmd.py new file mode 100644 index 000000000..da579359b --- /dev/null +++ b/fedn/cli/controller_cmd.py @@ -0,0 +1,19 @@ +import click +from .main import main +from fedn.network.api.server import app +from fedn.common.config import get_controller_config + +@main.group("controller") +@click.pass_context +def controller_cmd(ctx): + """:param ctx: + """ + pass +@controller_cmd.command("start") +@click.option("-H", "--host", required=False, default="0.0.0.0", help="Host address of controller") +@click.pass_context +def controller_cmd(ctx,host): + config = get_controller_config() + port = config["port"] + debug = config["debug"] + app.run(debug=debug, port=port,host=host) diff --git a/fedn/network/api/server.py b/fedn/network/api/server.py index c9e54ff87..53b2fbb2b 100644 --- a/fedn/network/api/server.py +++ b/fedn/network/api/server.py @@ -2,7 +2,6 @@ from flask import Flask, jsonify, request -from fedn.common.config import get_controller_config from fedn.network.api.auth import jwt_auth_required from fedn.network.api.interface import API from fedn.network.api.shared import control, statestore @@ -625,8 +624,3 @@ def list_combiners_data(): if custom_url_prefix: app.add_url_rule(f"{custom_url_prefix}/list_combiners_data", view_func=list_combiners_data, methods=["POST"]) -if __name__ == "__main__": - config = get_controller_config() - port = config["port"] - debug = config["debug"] - app.run(debug=debug, port=port, host="0.0.0.0") From 015486a68c380dbdccc16c6746a46adb4858ae2b Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Mon, 8 Jul 2024 12:37:50 +0200 Subject: [PATCH 2/9] fixed the api-server command in docker compose file --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 85386c6da..9560eb652 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -78,7 +78,7 @@ services: - mongo entrypoint: [ "sh", "-c" ] command: - - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/python fedn/network/api/server.py" + - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/python fedn controller start" ports: - 8092:8092 From da72cabe2fbc1084a04628d3896228afb58cb9c1 Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Mon, 8 Jul 2024 12:39:59 +0200 Subject: [PATCH 3/9] command update for api-server --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 9560eb652..34f9ee6d0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -78,7 +78,7 @@ services: - mongo entrypoint: [ "sh", "-c" ] command: - - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/python fedn controller start" + - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn fedn controller start" ports: - 8092:8092 From dd9a0f622e49bde362bca121678fb9c59a0d57ad Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Mon, 8 Jul 2024 14:52:24 +0200 Subject: [PATCH 4/9] import sequence changed --- fedn/cli/controller_cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedn/cli/controller_cmd.py b/fedn/cli/controller_cmd.py index da579359b..f2e72e9f8 100644 --- a/fedn/cli/controller_cmd.py +++ b/fedn/cli/controller_cmd.py @@ -1,7 +1,7 @@ import click from .main import main -from fedn.network.api.server import app from fedn.common.config import get_controller_config +from fedn.network.api.server import app @main.group("controller") @click.pass_context From 5dab5e4abb0c9a80bf8cdddcc7a933612cae5a79 Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Wed, 10 Jul 2024 11:15:51 +0200 Subject: [PATCH 5/9] fedn controller start cli implemented version and to get absolute path of config is moved to dist under utils wrapped the code under function to avoid execution during import --- fedn/cli/controller_cmd.py | 11 +++-------- fedn/cli/main.py | 9 ++------- fedn/common/config.py | 9 +++++---- fedn/network/api/server.py | 10 +++++++++- fedn/network/api/shared.py | 25 ++++++++++++++++++------- fedn/network/api/v1/shared.py | 3 +-- 6 files changed, 38 insertions(+), 29 deletions(-) diff --git a/fedn/cli/controller_cmd.py b/fedn/cli/controller_cmd.py index f2e72e9f8..3e1135fca 100644 --- a/fedn/cli/controller_cmd.py +++ b/fedn/cli/controller_cmd.py @@ -1,7 +1,6 @@ import click from .main import main -from fedn.common.config import get_controller_config -from fedn.network.api.server import app +from fedn.network.api.server import start_server_api @main.group("controller") @click.pass_context @@ -10,10 +9,6 @@ def controller_cmd(ctx): """ pass @controller_cmd.command("start") -@click.option("-H", "--host", required=False, default="0.0.0.0", help="Host address of controller") @click.pass_context -def controller_cmd(ctx,host): - config = get_controller_config() - port = config["port"] - debug = config["debug"] - app.run(debug=debug, port=port,host=host) +def controller_cmd(ctx): + start_server_api() diff --git a/fedn/cli/main.py b/fedn/cli/main.py index 0d5660c0b..ab1dd448e 100644 --- a/fedn/cli/main.py +++ b/fedn/cli/main.py @@ -1,5 +1,4 @@ -import importlib.metadata - +from fedn.utils.dist import get_version import click CONTEXT_SETTINGS = dict( @@ -7,11 +6,7 @@ help_option_names=["-h", "--help"], ) -# Dynamically get the version of the package -try: - version = importlib.metadata.version("fedn") -except importlib.metadata.PackageNotFoundError: - version = "unknown" +version=get_version("fedn") @click.group(context_settings=CONTEXT_SETTINGS) diff --git a/fedn/common/config.py b/fedn/common/config.py index 94b346d65..962f31c29 100644 --- a/fedn/common/config.py +++ b/fedn/common/config.py @@ -1,7 +1,7 @@ import os - import yaml +from fedn.utils.dist import get_absolute_path SECRET_KEY = os.environ.get("FEDN_JWT_SECRET_KEY", False) FEDN_JWT_CUSTOM_CLAIM_KEY = os.environ.get("FEDN_JWT_CUSTOM_CLAIM_KEY", False) FEDN_JWT_CUSTOM_CLAIM_VALUE = os.environ.get("FEDN_JWT_CUSTOM_CLAIM_VALUE", False) @@ -20,9 +20,10 @@ def get_environment_config(): """Get the configuration from environment variables.""" global STATESTORE_CONFIG global MODELSTORAGE_CONFIG - - STATESTORE_CONFIG = os.environ.get("STATESTORE_CONFIG", "/workspaces/fedn/config/settings-reducer.yaml.template") - MODELSTORAGE_CONFIG = os.environ.get("MODELSTORAGE_CONFIG", "/workspaces/fedn/config/settings-reducer.yaml.template") + Statestore_absolute_path=get_absolute_path("STATESTORE_CONFIG") + modelstorage_absolute_path=get_absolute_path("MODELSTORAGE_CONFIG") + STATESTORE_CONFIG = os.environ.get("STATESTORE_CONFIG", Statestore_absolute_path) + MODELSTORAGE_CONFIG = os.environ.get("MODELSTORAGE_CONFIG", modelstorage_absolute_path) def get_statestore_config(file=None): diff --git a/fedn/network/api/server.py b/fedn/network/api/server.py index 53b2fbb2b..bb4df7369 100644 --- a/fedn/network/api/server.py +++ b/fedn/network/api/server.py @@ -4,10 +4,12 @@ from fedn.network.api.auth import jwt_auth_required from fedn.network.api.interface import API -from fedn.network.api.shared import control, statestore +from fedn.network.api.shared import statestore,control from fedn.network.api.v1 import _routes +from fedn.common.config import get_controller_config custom_url_prefix = os.environ.get("FEDN_CUSTOM_URL_PREFIX", False) +# statestore_config,modelstorage_config,network_id,control=set_statestore_config() api = API(statestore, control) app = Flask(__name__) for bp in _routes: @@ -624,3 +626,9 @@ def list_combiners_data(): if custom_url_prefix: app.add_url_rule(f"{custom_url_prefix}/list_combiners_data", view_func=list_combiners_data, methods=["POST"]) +def start_server_api(): + config = get_controller_config() + port = config["port"] + debug = config["debug"] + host="0.0.0.0" + app.run(debug=debug, port=port,host=host) diff --git a/fedn/network/api/shared.py b/fedn/network/api/shared.py index fc8d4ae57..7a50e0b01 100644 --- a/fedn/network/api/shared.py +++ b/fedn/network/api/shared.py @@ -1,11 +1,22 @@ from fedn.common.config import get_modelstorage_config, get_network_config, get_statestore_config from fedn.network.controller.control import Control from fedn.network.storage.statestore.mongostatestore import MongoStateStore +statestore=None +statestore_config=None +modelstorage_config=None +network_id=None +control=None +def set_statestore_config(): + global statestore + global modelstorage_config + global network_id + global control + global statestore_config + statestore_config = get_statestore_config() + modelstorage_config = get_modelstorage_config() + network_id = get_network_config() + statestore = MongoStateStore(network_id, statestore_config["mongo_config"]) + statestore.set_storage_backend(modelstorage_config) + control = Control(statestore=statestore) -statestore_config = get_statestore_config() -modelstorage_config = get_modelstorage_config() -network_id = get_network_config() - -statestore = MongoStateStore(network_id, statestore_config["mongo_config"]) -statestore.set_storage_backend(modelstorage_config) -control = Control(statestore=statestore) +set_statestore_config() diff --git a/fedn/network/api/v1/shared.py b/fedn/network/api/v1/shared.py index b7ae170af..a27a6f637 100644 --- a/fedn/network/api/v1/shared.py +++ b/fedn/network/api/v1/shared.py @@ -3,10 +3,9 @@ import pymongo from pymongo.database import Database -from fedn.network.api.shared import statestore_config, network_id +from fedn.network.api.shared import statestore_config,network_id api_version = "v1" - mc = pymongo.MongoClient(**statestore_config["mongo_config"]) mc.server_info() mdb: Database = mc[network_id] From 31ac7f4c1ef81dd9e96dbd594327a9a9e236c9ae Mon Sep 17 00:00:00 2001 From: sowmyasri Date: Wed, 10 Jul 2024 12:04:08 +0200 Subject: [PATCH 6/9] dist file in utils --- fedn/utils/dist.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 fedn/utils/dist.py diff --git a/fedn/utils/dist.py b/fedn/utils/dist.py new file mode 100644 index 000000000..e419085d7 --- /dev/null +++ b/fedn/utils/dist.py @@ -0,0 +1,17 @@ +import importlib.metadata +import os +from pathlib import Path + +def get_version(pacakge): + # Dynamically get the version of the package + try: + version = importlib.metadata.version("fedn") + except importlib.metadata.PackageNotFoundError: + version = "unknown" + return version +def get_absolute_path(env_var): + default_path = ".././fedn/config/settings-reducer.yaml.template" + file_path = os.environ.get(env_var, default_path) + # Resolve the absolute path + absolute_file_path = Path(file_path).resolve() + return absolute_file_path From 828aeeae4de5c5a61adba7c97ef7b3cba7d51746 Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Wed, 10 Jul 2024 12:25:08 +0000 Subject: [PATCH 7/9] fixes --- docker-compose.yaml | 2 +- fedn/cli/controller_cmd.py | 10 +++++--- fedn/common/config.py | 17 +++++++++---- fedn/common/settings-controller.yaml.template | 24 +++++++++++++++++++ fedn/network/api/shared.py | 24 +++++-------------- fedn/utils/dist.py | 16 ++++++------- 6 files changed, 58 insertions(+), 35 deletions(-) create mode 100644 fedn/common/settings-controller.yaml.template diff --git a/docker-compose.yaml b/docker-compose.yaml index 34f9ee6d0..c3620e79d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -78,7 +78,7 @@ services: - mongo entrypoint: [ "sh", "-c" ] command: - - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn fedn controller start" + - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn controller start" ports: - 8092:8092 diff --git a/fedn/cli/controller_cmd.py b/fedn/cli/controller_cmd.py index 3e1135fca..ab8727b27 100644 --- a/fedn/cli/controller_cmd.py +++ b/fedn/cli/controller_cmd.py @@ -1,14 +1,18 @@ import click + from .main import main -from fedn.network.api.server import start_server_api + @main.group("controller") @click.pass_context def controller_cmd(ctx): - """:param ctx: - """ + """:param ctx:""" pass + + @controller_cmd.command("start") @click.pass_context def controller_cmd(ctx): + from fedn.network.api.server import start_server_api + start_server_api() diff --git a/fedn/common/config.py b/fedn/common/config.py index 962f31c29..23d873ff7 100644 --- a/fedn/common/config.py +++ b/fedn/common/config.py @@ -1,7 +1,9 @@ import os + import yaml -from fedn.utils.dist import get_absolute_path +from fedn.utils.dist import get_package_path + SECRET_KEY = os.environ.get("FEDN_JWT_SECRET_KEY", False) FEDN_JWT_CUSTOM_CLAIM_KEY = os.environ.get("FEDN_JWT_CUSTOM_CLAIM_KEY", False) FEDN_JWT_CUSTOM_CLAIM_VALUE = os.environ.get("FEDN_JWT_CUSTOM_CLAIM_VALUE", False) @@ -20,10 +22,15 @@ def get_environment_config(): """Get the configuration from environment variables.""" global STATESTORE_CONFIG global MODELSTORAGE_CONFIG - Statestore_absolute_path=get_absolute_path("STATESTORE_CONFIG") - modelstorage_absolute_path=get_absolute_path("MODELSTORAGE_CONFIG") - STATESTORE_CONFIG = os.environ.get("STATESTORE_CONFIG", Statestore_absolute_path) - MODELSTORAGE_CONFIG = os.environ.get("MODELSTORAGE_CONFIG", modelstorage_absolute_path) + if not os.environ.get("STATESTORE_CONFIG", False): + STATESTORE_CONFIG = get_package_path() + "/common/settings-controller.yaml.template" + else: + STATESTORE_CONFIG = os.environ.get("STATESTORE_CONFIG") + + if not os.environ.get("MODELSTORAGE_CONFIG", False): + MODELSTORAGE_CONFIG = get_package_path() + "/common/settings-controller.yaml.template" + else: + MODELSTORAGE_CONFIG = os.environ.get("MODELSTORAGE_CONFIG") def get_statestore_config(file=None): diff --git a/fedn/common/settings-controller.yaml.template b/fedn/common/settings-controller.yaml.template new file mode 100644 index 000000000..a5266a38b --- /dev/null +++ b/fedn/common/settings-controller.yaml.template @@ -0,0 +1,24 @@ +network_id: fedn-network +controller: + host: localhost + port: 8092 + debug: True + +statestore: + type: MongoDB + mongo_config: + username: fedn_admin + password: password + host: localhost + port: 6534 + +storage: + storage_type: S3 + storage_config: + storage_hostname: localhost + storage_port: 9000 + storage_access_key: fedn_admin + storage_secret_key: password + storage_bucket: fedn-models + context_bucket: fedn-context + storage_secure_mode: False diff --git a/fedn/network/api/shared.py b/fedn/network/api/shared.py index 7a50e0b01..9e0e5acbd 100644 --- a/fedn/network/api/shared.py +++ b/fedn/network/api/shared.py @@ -1,22 +1,10 @@ from fedn.common.config import get_modelstorage_config, get_network_config, get_statestore_config from fedn.network.controller.control import Control from fedn.network.storage.statestore.mongostatestore import MongoStateStore -statestore=None -statestore_config=None -modelstorage_config=None -network_id=None -control=None -def set_statestore_config(): - global statestore - global modelstorage_config - global network_id - global control - global statestore_config - statestore_config = get_statestore_config() - modelstorage_config = get_modelstorage_config() - network_id = get_network_config() - statestore = MongoStateStore(network_id, statestore_config["mongo_config"]) - statestore.set_storage_backend(modelstorage_config) - control = Control(statestore=statestore) -set_statestore_config() +statestore_config = get_statestore_config() +modelstorage_config = get_modelstorage_config() +network_id = get_network_config() +statestore = MongoStateStore(network_id, statestore_config["mongo_config"]) +statestore.set_storage_backend(modelstorage_config) +control = Control(statestore=statestore) diff --git a/fedn/utils/dist.py b/fedn/utils/dist.py index e419085d7..e5fa7192b 100644 --- a/fedn/utils/dist.py +++ b/fedn/utils/dist.py @@ -1,6 +1,7 @@ import importlib.metadata -import os -from pathlib import Path + +import fedn + def get_version(pacakge): # Dynamically get the version of the package @@ -9,9 +10,8 @@ def get_version(pacakge): except importlib.metadata.PackageNotFoundError: version = "unknown" return version -def get_absolute_path(env_var): - default_path = ".././fedn/config/settings-reducer.yaml.template" - file_path = os.environ.get(env_var, default_path) - # Resolve the absolute path - absolute_file_path = Path(file_path).resolve() - return absolute_file_path + + +def get_package_path(): + # Get the path of the package + return fedn.__path__[0] From 4ce2eb061a41e0193c8ecf0d7acdd3c7c5f7a8bf Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Wed, 10 Jul 2024 12:41:42 +0000 Subject: [PATCH 8/9] add controller_cmd to exclude --- .github/workflows/code-checks.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/code-checks.yaml b/.github/workflows/code-checks.yaml index dbe378c63..8c48a3015 100644 --- a/.github/workflows/code-checks.yaml +++ b/.github/workflows/code-checks.yaml @@ -25,6 +25,7 @@ jobs: --exclude-dir='docs' --exclude-dir='flower-client' --exclude='tests.py' + --exclude='controller_cmd.py' --exclude='README.rst' '^[ \t]+(import|from) ' -I . From 1e739c3016efe47503168ba7a1a314e6170bd148 Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Thu, 11 Jul 2024 07:34:54 +0000 Subject: [PATCH 9/9] add __main__ --- fedn/network/api/server.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/fedn/network/api/server.py b/fedn/network/api/server.py index bb4df7369..d56c3ab0b 100644 --- a/fedn/network/api/server.py +++ b/fedn/network/api/server.py @@ -2,11 +2,11 @@ from flask import Flask, jsonify, request +from fedn.common.config import get_controller_config from fedn.network.api.auth import jwt_auth_required from fedn.network.api.interface import API -from fedn.network.api.shared import statestore,control +from fedn.network.api.shared import control, statestore from fedn.network.api.v1 import _routes -from fedn.common.config import get_controller_config custom_url_prefix = os.environ.get("FEDN_CUSTOM_URL_PREFIX", False) # statestore_config,modelstorage_config,network_id,control=set_statestore_config() @@ -626,9 +626,14 @@ def list_combiners_data(): if custom_url_prefix: app.add_url_rule(f"{custom_url_prefix}/list_combiners_data", view_func=list_combiners_data, methods=["POST"]) + def start_server_api(): config = get_controller_config() port = config["port"] debug = config["debug"] - host="0.0.0.0" - app.run(debug=debug, port=port,host=host) + host = "0.0.0.0" + app.run(debug=debug, port=port, host=host) + + +if __name__ == "__main__": + start_server_api()