Optional Downloader Data Provider (#424)

* feat: optional download data provider

* feat: test to download data provider with different params

* feat: installed data provider list

* feat: general tests

* refactor: separate data-provider from data-feed

* fix: validation of amount of request on api

* feat: remove default value for data-provider-historical

* feat: test case with local historical provider

* remove: repeat test case
refactor: assertion in test with different version of py

* fix: missed rename data_feed to data_provider

* feat: additional test case

* revert: data_provider -> data_feed
commit: e52b3a4

* revert: repeat code of finding data-provider

* rename: helper method more fit name

* rename: missed
PR: #416

Author: Romazes

lean/commands/live/deploy.py

@@ -79,7 +79,7 @@ def _install_modules(modules: List[LeanConfigConfigurer], user_kwargs: Dict[str,
     :param modules: the modules to check
     """
     for module in modules:
-        if not module._installs:
+        if module is None or not module._installs:
             continue
         organization_id = container.organization_manager.try_get_working_organization_id()
         module.ensure_module_installed(organization_id)
@@ -192,6 +192,15 @@ def _configure_lean_config_interactively(lean_config: Dict[str, Any],
 
 _cached_lean_config = None
 
+def _try_get_data_historical_name(data_provider_historical_name: str, data_provider_live_name: str) -> str:
+    """ Get name for historical data provider based on data provider live (if exist)
+
+    :param data_provider_historical_name: the current (default) data provider historical
+    :param data_provider_live_name: the current data provider live name
+    """
+    return next((live_data_historical.get_name() for live_data_historical in all_data_providers
+                                             if live_data_historical.get_name() in data_provider_live_name), data_provider_historical_name)
+
 
 # being used by lean.models.click_options.get_the_correct_type_default_value()
 def _get_default_value(key: str) -> Optional[Any]:
@@ -235,7 +244,6 @@ def _get_default_value(key: str) -> Optional[Any]:
               help="The live data provider to use")
 @option("--data-provider-historical",
               type=Choice([dp.get_name() for dp in all_data_providers if dp._id != "TerminalLinkBrokerage"], case_sensitive=False),
-              default="Local",
               help="Update the Lean configuration file to retrieve data from the given historical provider")
 @options_from_json(get_configs_for_options("live-local"))
 @option("--release",
@@ -387,7 +395,7 @@ def deploy(project: Path,
         [update_essential_properties_available(data_feed_configurers, kwargs)]
 
     elif brokerage is not None or len(data_provider_live) > 0:
-        ensure_options(["brokerage", "data_feed"])
+        ensure_options(["brokerage", "data_provider_live"])
 
         environment_name = "lean-cli"
         lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None)
@@ -408,9 +416,10 @@ def deploy(project: Path,
         lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None)
         _configure_lean_config_interactively(lean_config, environment_name, kwargs, show_secrets=show_secrets)
 
-    if data_provider_historical is not None:
-        [data_provider_configurer] = [get_and_build_module(data_provider_historical, all_data_providers, kwargs, logger)]
-        data_provider_configurer.configure(lean_config, environment_name)
+    if data_provider_historical is None:
+        data_provider_historical = _try_get_data_historical_name("Local", data_provider_live)
+    [data_provider_configurer] = [get_and_build_module(data_provider_historical, all_data_providers, kwargs, logger)]
+    data_provider_configurer.configure(lean_config, environment_name)
 
     if "environments" not in lean_config or environment_name not in lean_config["environments"]:
         lean_config_path = lean_config_manager.get_lean_config_path()
@@ -422,7 +431,7 @@ def deploy(project: Path,
                             "https://www.lean.io/docs/v2/lean-cli/live-trading/brokerages/quantconnect-paper-trading")
 
     env_brokerage, env_data_queue_handlers = _get_configurable_modules_from_environment(lean_config, environment_name)
-    _install_modules([env_brokerage] + env_data_queue_handlers, kwargs)
+    _install_modules([env_brokerage] + env_data_queue_handlers + [data_provider_configurer], kwargs)
 
     _raise_for_missing_properties(lean_config, environment_name, lean_config_manager.get_lean_config_path())
 
@@ -497,4 +506,4 @@ def deploy(project: Path,
             raise RuntimeError(f"InteractiveBrokers is currently not supported for ARM hosts")
 
     lean_runner = container.lean_runner
-    lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach, loads(extra_docker_config))
+    lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach, loads(extra_docker_config))

lean/models/data_providers/__init__.py

@@ -23,4 +23,4 @@
 
 # QuantConnect DataProvider
 [QuantConnectDataProvider] = [
-    data_provider for data_provider in all_data_providers if data_provider._id == "QuantConnect"]
+    data_provider for data_provider in all_data_providers if data_provider._id == "QuantConnect"]

tests/commands/test_live.py

@@ -25,8 +25,9 @@
 from lean.constants import DEFAULT_ENGINE_IMAGE
 from lean.container import container
 from lean.models.docker import DockerImage
-from tests.test_helpers import create_fake_lean_cli_directory
+from tests.test_helpers import create_fake_lean_cli_directory, reset_state_installed_modules
 from tests.conftest import initialize_container
+from click.testing import Result
 
 ENGINE_IMAGE = DockerImage.parse(DEFAULT_ENGINE_IMAGE)
 
@@ -413,7 +414,21 @@ def test_live_sets_dependent_configurations_from_modules_json_based_on_environme
     "Terminal Link": terminal_link_required_options,
     "Kraken": brokerage_required_options["Kraken"],
     "TDAmeritrade": brokerage_required_options["TDAmeritrade"],
-    "Bybit": brokerage_required_options["Bybit"]
+    "Bybit": brokerage_required_options["Bybit"],
+}
+
+data_provider_required_options = {
+    "IEX": {
+        "iex-cloud-api-key": "123",
+        "iex-price-plan": "Launch",
+    },
+    "Polygon": {
+        "polygon-api-key": "123",
+    },
+    "AlphaVantage": {
+        "alpha-vantage-api-key": "111",
+        "alpha-vantage-price-plan": "Free"
+    }
 }
 
 
@@ -530,7 +545,7 @@ def test_live_non_interactive_raise_error_when_missing_data_provider_live_option
 
     error_msg = str(result.exc_info[1]).split()
 
-    assert "data-provider-live" in error_msg
+    assert "--data-provider-live" in error_msg
     assert "data-queue-handler" not in error_msg 
 
     assert result.exit_code != 0
@@ -1026,3 +1041,191 @@ def test_live_passes_live_holdings_to_lean_runner_when_given_as_option(brokerage
         return
 
     assert args[0]["live-holdings"] == holding_list
+
+def test_live_non_interactive_deploy_with_live_and_historical_provider_missed_historical_not_optional_config() -> None:
+    create_fake_lean_cli_directory()
+    create_fake_environment("live-paper", True)
+
+    container.initialize(docker_manager=mock.Mock(), lean_runner=mock.Mock(), api_client = mock.MagicMock())
+
+    provider_live_option = ["--data-provider-live", "IEX",
+                            "--iex-cloud-api-key", "123",
+                            "--iex-price-plan", "Launch"]
+
+    provider_history_option = ["--data-provider-historical", "Polygon"]
+                               # "--polygon-api-key", "123"]
+
+    result = CliRunner().invoke(lean, ["live", "deploy" , "--brokerage", "Paper Trading",
+                                       *provider_live_option,
+                                       *provider_history_option,
+                                       "Python Project",
+                                       ])
+    error_msg = str(result.exc_info[1]).split()
+
+    assert "--polygon-api-key" in error_msg
+    assert "--iex-cloud-api-key" not in error_msg
+
+    assert result.exit_code == 1
+
+def test_live_non_interactive_deploy_with_live_and_historical_provider_missed_live_not_optional_config() -> None:
+    create_fake_lean_cli_directory()
+    create_fake_environment("live-paper", True)
+
+    container.initialize(docker_manager=mock.Mock(), lean_runner=mock.Mock(), api_client = mock.MagicMock())
+
+    provider_live_option = ["--data-provider-live", "IEX",
+                            "--iex-cloud-api-key", "123"]
+                            #"--iex-price-plan", "Launch"]
+
+    provider_history_option = ["--data-provider-historical", "Polygon", "--polygon-api-key", "123"]
+
+    result = CliRunner().invoke(lean, ["live", "deploy" , "--brokerage", "Paper Trading",
+                                       *provider_live_option,
+                                       *provider_history_option,
+                                       "Python Project",
+                                       ])
+
+    error_msg = str(result.exc_info[1]).split()
+
+    assert "--iex-price-plan" in error_msg
+    assert "--polygon-api-key" not in error_msg
+
+    assert result.exit_code == 1
+
+def test_live_non_interactive_deploy_with_real_brokerage_without_credentials() -> None:
+    create_fake_lean_cli_directory()
+    create_fake_environment("live-paper", True)
+
+    container.initialize(docker_manager=mock.Mock(), lean_runner=mock.Mock(), api_client = mock.MagicMock())
+
+    # create fake environment has IB configs already
+    brokerage = ["--brokerage", "OANDA"]
+
+    provider_live_option = ["--data-provider-live", "IEX",
+                            "--iex-cloud-api-key", "123",
+                            "--iex-price-plan", "Launch"]
+
+    result = CliRunner().invoke(lean, ["live", "deploy" ,
+                                       *brokerage,
+                                       *provider_live_option,
+                                       "Python Project",
+                                       ])
+    assert result.exit_code == 1
+
+    error_msg = str(result.exc_info[1]).split()
+
+    assert "--oanda-account-id" in error_msg
+    assert "--oanda-access-token" in error_msg
+    assert "--oanda-environment" in error_msg
+    assert "--iex-price-plan" not in error_msg    
+
+def create_lean_option(brokerage_name: str, data_provider_live_name: str, data_provider_historical_name: str, api_client: any) -> Result:
+    reset_state_installed_modules()
+    create_fake_lean_cli_directory()
+    create_fake_environment("live-paper", True)
+
+    initialize_container(api_client_to_use=api_client)
+
+    option = ["--brokerage", brokerage_name]
+    for key, value in brokerage_required_options[brokerage_name].items():
+        option.extend([f"--{key}", value])
+
+    data_feed_required_options.update(data_provider_required_options)
+
+    option.extend(["--data-provider-live", data_provider_live_name])
+    for key, value in data_feed_required_options[data_provider_live_name].items():
+        if f"--{key}" not in option:
+            option.extend([f"--{key}", value])
+
+    if data_provider_historical_name is not None:
+        option.extend(["--data-provider-historical", data_provider_historical_name])
+        if data_provider_historical_name is not "Local": 
+            for key, value in data_feed_required_options[data_provider_historical_name].items():
+                if f"--{key}" not in option:
+                    option.extend([f"--{key}", value])
+
+    result = CliRunner().invoke(lean, ["live", "deploy",
+                                       *option,
+                                       "Python Project",
+                                       ])
+    assert result.exit_code == 0
+    return result
+
+@pytest.mark.parametrize("brokerage_name,data_provider_live_name,data_provider_historical_name,brokerage_product_id,data_provider_live_product_id,data_provider_historical_id",
+                         [("Interactive Brokers", "IEX", "Polygon", "181", "333", "306"),
+                          ("Paper Trading", "IEX", "Polygon", None, "333", "306"),
+                          ("Tradier", "IEX", "AlphaVantage", "185", "333", "334"),
+                          ("Paper Trading", "IEX", "Local", None, "333", "222")])
+def test_live_deploy_with_different_brokerage_and_different_live_data_provider_and_historical_data_provider(brokerage_name: str, data_provider_live_name: str, data_provider_historical_name: str, brokerage_product_id: str, data_provider_live_product_id: str, data_provider_historical_id: str) -> None:
+    api_client = mock.MagicMock()
+    create_lean_option(brokerage_name, data_provider_live_name, data_provider_historical_name, api_client)
+    
+    is_exists = []
+    if brokerage_product_id is None and data_provider_historical_name != "Local":
+        assert len(api_client.method_calls) == 3
+        for m_c, id in zip(api_client.method_calls, [data_provider_live_product_id, data_provider_historical_id]):
+            if id in m_c[1]:
+                is_exists.append(True)
+        assert is_exists
+        assert len(is_exists) == 2
+    elif brokerage_product_id is None and data_provider_historical_name == "Local":
+        assert len(api_client.method_calls) == 2
+        if data_provider_live_product_id in api_client.method_calls[0][1]:
+            is_exists.append(True)
+        assert is_exists
+        assert len(is_exists) == 1
+    else:
+        assert len(api_client.method_calls) == 3
+        for m_c, id in zip(api_client.method_calls, [brokerage_product_id, data_provider_live_product_id, data_provider_historical_id]):
+            if id in f"{m_c[1]}":
+                is_exists.append(True)
+        assert is_exists
+        assert len(is_exists) == 3
+
+@pytest.mark.parametrize("brokerage_name,data_provider_live_name,brokerage_product_id,data_provider_live_product_id",
+                         [("Interactive Brokers", "IEX", "181", "333"),
+                          ("Tradier", "IEX", "185", "333")])
+def test_live_non_interactive_deploy_with_different_brokerage_and_different_live_data_provider(brokerage_name: str, data_provider_live_name: str, brokerage_product_id: str, data_provider_live_product_id: str) -> None:
+    api_client = mock.MagicMock()
+    create_lean_option(brokerage_name, data_provider_live_name, None, api_client)
+    
+    assert len(api_client.method_calls) == 2
+    is_exists = []
+    for m_c, id in zip(api_client.method_calls, [brokerage_product_id, data_provider_live_product_id]):
+        if id in m_c[1]:
+            is_exists.append(True)
+    
+    assert is_exists
+    assert len(is_exists) == 2
+
+@pytest.mark.parametrize("brokerage_name,data_provider_live_name,brokerage_product_id",
+                         [("Bybit", "Bybit", "305"),
+                          ("Coinbase Advanced Trade", "Coinbase Advanced Trade", "183"),
+                          ("Interactive Brokers", "Interactive Brokers", "181"),
+                          ("Tradier", "Tradier", "185")])
+def test_live_non_interactive_deploy_with_different_brokerage_with_the_same_live_data_provider(brokerage_name: str, data_provider_live_name: str, brokerage_product_id: str) -> None:
+    api_client = mock.MagicMock()
+    create_lean_option(brokerage_name, data_provider_live_name, None, api_client)
+
+    print(api_client.call_args_list)
+    print(api_client.call_args)
+
+    for m_c in api_client.method_calls:
+        if brokerage_product_id in m_c[1]:
+            is_exist = True
+
+    assert is_exist
+
+@pytest.mark.parametrize("brokerage_name,data_provider_live_name,data_provider_live_product_id",
+                         [("Paper Trading", "IEX", "333"),
+                          ("Paper Trading", "Polygon", "306")])
+def test_live_non_interactive_deploy_paper_brokerage_different_live_data_provider(brokerage_name: str, data_provider_live_name: str, data_provider_live_product_id: str) -> None:
+    api_client = mock.MagicMock()
+    create_lean_option(brokerage_name, data_provider_live_name, None, api_client)
+
+    assert len(api_client.method_calls) == 2
+    for m_c in api_client.method_calls:
+        if data_provider_live_product_id in m_c[1]:
+            is_exist = True
+
+    assert is_exist

tests/test_helpers.py

@@ -15,6 +15,8 @@
 from datetime import datetime
 from pathlib import Path
 from typing import List
+from lean.models.data_providers import all_data_providers
+from lean.models.brokerages.local import all_local_brokerages, all_local_data_feeds
 
 from lean.commands.create_project import (DEFAULT_CSHARP_MAIN, DEFAULT_CSHARP_NOTEBOOK, DEFAULT_PYTHON_MAIN,
                                           DEFAULT_PYTHON_NOTEBOOK, LIBRARY_PYTHON_MAIN, LIBRARY_CSHARP_MAIN)
@@ -224,3 +226,11 @@ def create_lean_environments() -> List[QCLeanEnvironment]:
                           description="",
                           public=True)
     ]
+
+def reset_state_installed_modules() -> None:
+    for data_provider in all_data_providers:
+        data_provider.__setattr__("_is_module_installed", False)
+    for local_brokerage in all_local_brokerages:
+        local_brokerage.__setattr__("_is_module_installed", False)
+    for local_data_feed in all_local_data_feeds:
+        local_data_feed.__setattr__("_is_module_installed", False)