Merge pull request freqtrade#11548 from freqtrade/feat/config_to_btresults

Save config and Strategy to backtest result file

Author: xmatthias

docs/backtesting.md

@@ -435,6 +435,20 @@ To save time, by default backtest will reuse a cached result from within the las
 To further analyze your backtest results, freqtrade will export the trades to file by default.
 You can then load the trades to perform further analysis as shown in the [data analysis](strategy_analysis_example.md#load-backtest-results-to-pandas-dataframe) backtesting section.
 
+### Backtest output file
+
+The output file freqtrade produces is a zip file containing the following files:
+
+- The backtest report in json format
+- the market change data in feather format
+- a copy of the strategy file
+- a copy of the strategy parameters (if a parameter file was used)
+- a sanitized copy of the config file
+
+This will ensure results are reproducible - under the assumption that the same data is available.
+
+Only the strategy file and the config file are included in the zip file, eventual dependencies are not included.
+
 ## Assumptions made by backtesting
 
 Since backtesting lacks some detailed information about what happens within a candle, it needs to take a few assumptions:

freqtrade/ft_types/backtest_result_type.py

@@ -1,4 +1,5 @@
-from typing import Any
+from copy import deepcopy
+from typing import Any, cast
 
 from typing_extensions import TypedDict
 
@@ -15,11 +16,16 @@ class BacktestResultType(TypedDict):
 
 
 def get_BacktestResultType_default() -> BacktestResultType:
-    return {
-        "metadata": {},
-        "strategy": {},
-        "strategy_comparison": [],
-    }
+    return cast(
+        BacktestResultType,
+        deepcopy(
+            {
+                "metadata": {},
+                "strategy": {},
+                "strategy_comparison": [],
+            }
+        ),
+    )
 
 
 class BacktestHistoryEntryType(BacktestMetadataType):

freqtrade/optimize/backtesting.py

@@ -1792,6 +1792,7 @@ def start(self) -> None:
                     dt_appendix,
                     market_change_data=combined_res,
                     analysis_results=self.analysis_results,
+                    strategy_files={s.get_strategy_name(): s.__file__ for s in self.strategylist},
                 )
 
         # Results may be mixed up now. Sort them so they follow --strategy-list order.

freqtrade/optimize/optimize_reports/bt_storage.py

@@ -6,6 +6,7 @@
 
 from pandas import DataFrame
 
+from freqtrade.configuration import sanitize_config
 from freqtrade.constants import LAST_BT_RESULT_FN
 from freqtrade.enums.runmode import RunMode
 from freqtrade.ft_types import BacktestResultType
@@ -52,6 +53,7 @@ def store_backtest_results(
     *,
     market_change_data: DataFrame | None = None,
     analysis_results: dict[str, dict[str, DataFrame]] | None = None,
+    strategy_files: dict[str, str] | None = None,
 ) -> Path:
     """
     Stores backtest results and analysis data in a zip file, with metadata stored separately
@@ -85,6 +87,32 @@ def store_backtest_results(
         dump_json_to_file(stats_buf, stats_copy)
         zipf.writestr(json_filename.name, stats_buf.getvalue())
 
+        config_buf = StringIO()
+        dump_json_to_file(config_buf, sanitize_config(config["original_config"]))
+        zipf.writestr(f"{base_filename.stem}_config.json", config_buf.getvalue())
+
+        for strategy_name, strategy_file in (strategy_files or {}).items():
+            # Store the strategy file and its parameters
+            strategy_buf = BytesIO()
+            strategy_path = Path(strategy_file)
+            if not strategy_path.is_file():
+                logger.warning(f"Strategy file '{strategy_path}' does not exist. Skipping.")
+                continue
+            with strategy_path.open("rb") as strategy_file_obj:
+                strategy_buf.write(strategy_file_obj.read())
+            strategy_buf.seek(0)
+            zipf.writestr(f"{base_filename.stem}_{strategy_name}.py", strategy_buf.getvalue())
+            strategy_params = strategy_path.with_suffix(".json")
+            if strategy_params.is_file():
+                strategy_params_buf = BytesIO()
+                with strategy_params.open("rb") as strategy_params_obj:
+                    strategy_params_buf.write(strategy_params_obj.read())
+                strategy_params_buf.seek(0)
+                zipf.writestr(
+                    f"{base_filename.stem}_{strategy_name}.json",
+                    strategy_params_buf.getvalue(),
+                )
+
         # Add market change data if present
         if market_change_data is not None:
             market_change_name = f"{base_filename.stem}_market_change.feather"

freqtrade/optimize/optimize_reports/optimize_reports.py

@@ -18,7 +18,7 @@
     calculate_sortino,
     calculate_sqn,
 )
-from freqtrade.ft_types import BacktestResultType
+from freqtrade.ft_types import BacktestResultType, get_BacktestResultType_default
 from freqtrade.util import decimals_per_coin, fmt_coin, get_dry_run_wallet
 
 
@@ -587,11 +587,7 @@ def generate_backtest_stats(
     :param max_date: Backtest end date
     :return: Dictionary containing results per strategy and a strategy summary.
     """
-    result: BacktestResultType = {
-        "metadata": {},
-        "strategy": {},
-        "strategy_comparison": [],
-    }
+    result: BacktestResultType = get_BacktestResultType_default()
     market_change = calculate_market_change(btdata, "close")
     metadata = {}
     pairlist = list(btdata.keys())

freqtrade/rpc/api_server/api_backtest.py

@@ -108,6 +108,9 @@ def __run_backtest_bg(btconfig: Config):
                     ApiBG.bt["bt"].results,
                     datetime.now().strftime("%Y-%m-%d_%H-%M-%S"),
                     market_change_data=combined_res,
+                    strategy_files={
+                        s.get_strategy_name(): s.__file__ for s in ApiBG.bt["bt"].strategylist
+                    },
                 )
                 ApiBG.bt["bt"].results["metadata"][strategy_name]["filename"] = str(fn.stem)
                 ApiBG.bt["bt"].results["metadata"][strategy_name]["strategy"] = strategy_name

freqtrade/strategy/interface.py

@@ -132,6 +132,7 @@ class IStrategy(ABC, HyperStrategyMixin):
     stake_currency: str
     # container variable for strategy source code
     __source__: str = ""
+    __file__: str = ""
 
     # Definition of plot_config. See plotting documentation for more details.
     plot_config: dict = {}

tests/conftest.py

@@ -652,6 +652,7 @@ def get_default_conf(testdatadir):
         "trading_mode": "spot",
         "margin_mode": "",
         "candle_type_def": CandleType.SPOT,
+        "original_config": {},
     }
     return configuration
 

tests/optimize/test_optimize_reports.py

@@ -1,5 +1,6 @@
 import json
 import re
+import shutil
 from datetime import timedelta
 from pathlib import Path
 from shutil import copyfile
@@ -41,7 +42,7 @@
 from freqtrade.resolvers.strategy_resolver import StrategyResolver
 from freqtrade.util import dt_ts
 from freqtrade.util.datetime_helpers import dt_from_ts, dt_utc
-from tests.conftest import CURRENT_TEST_STRATEGY
+from tests.conftest import CURRENT_TEST_STRATEGY, log_has_re
 from tests.data.test_history import _clean_test_file
 
 
@@ -253,8 +254,9 @@ def test_store_backtest_results(testdatadir, mocker):
     dump_mock = mocker.patch("freqtrade.optimize.optimize_reports.bt_storage.file_dump_json")
     zip_mock = mocker.patch("freqtrade.optimize.optimize_reports.bt_storage.ZipFile")
     data = {"metadata": {}, "strategy": {}, "strategy_comparison": []}
-
-    store_backtest_results({"exportfilename": testdatadir}, data, "2022_01_01_15_05_13")
+    store_backtest_results(
+        {"exportfilename": testdatadir, "original_config": {}}, data, "2022_01_01_15_05_13"
+    )
 
     assert dump_mock.call_count == 2
     assert zip_mock.call_count == 1
@@ -264,17 +266,26 @@ def test_store_backtest_results(testdatadir, mocker):
     dump_mock.reset_mock()
     zip_mock.reset_mock()
     filename = testdatadir / "testresult.json"
-    store_backtest_results({"exportfilename": filename}, data, "2022_01_01_15_05_13")
+    store_backtest_results(
+        {"exportfilename": filename, "original_config": {}}, data, "2022_01_01_15_05_13"
+    )
     assert dump_mock.call_count == 2
     assert zip_mock.call_count == 1
     assert isinstance(dump_mock.call_args_list[0][0][0], Path)
     # result will be testdatadir / testresult-<timestamp>.json
     assert str(dump_mock.call_args_list[0][0][0]).startswith(str(testdatadir / "testresult"))
 
 
-def test_store_backtest_results_real(tmp_path):
+def test_store_backtest_results_real(tmp_path, caplog):
     data = {"metadata": {}, "strategy": {}, "strategy_comparison": []}
-    store_backtest_results({"exportfilename": tmp_path}, data, "2022_01_01_15_05_13")
+    config = {
+        "exportfilename": tmp_path,
+        "original_config": {},
+    }
+    store_backtest_results(
+        config, data, "2022_01_01_15_05_13", strategy_files={"DefStrat": "NoFile"}
+    )
+    assert log_has_re(r"Strategy file .* does not exist\. Skipping\.", caplog)
 
     zip_file = tmp_path / "backtest-result-2022_01_01_15_05_13.zip"
     assert zip_file.is_file()
@@ -287,8 +298,19 @@ def test_store_backtest_results_real(tmp_path):
     fn = get_latest_backtest_filename(tmp_path)
     assert fn == "backtest-result-2022_01_01_15_05_13.zip"
 
+    strategy_test_dir = Path(__file__).parent.parent / "strategy" / "strats"
+
+    shutil.copy(strategy_test_dir / "strategy_test_v3.py", tmp_path)
+    params_file = tmp_path / "strategy_test_v3.json"
+    with params_file.open("w") as f:
+        f.write("""{"strategy_name": "TurtleStrategyX5","params":{}}""")
+
     store_backtest_results(
-        {"exportfilename": tmp_path}, data, "2024_01_01_15_05_25", market_change_data=pd.DataFrame()
+        config,
+        data,
+        "2024_01_01_15_05_25",
+        market_change_data=pd.DataFrame(),
+        strategy_files={"DefStrat": str(tmp_path / "strategy_test_v3.py")},
     )
     zip_file = tmp_path / "backtest-result-2024_01_01_15_05_25.zip"
     assert zip_file.is_file()
@@ -298,6 +320,22 @@ def test_store_backtest_results_real(tmp_path):
     with ZipFile(zip_file, "r") as zipf:
         assert "backtest-result-2024_01_01_15_05_25.json" in zipf.namelist()
         assert "backtest-result-2024_01_01_15_05_25_market_change.feather" in zipf.namelist()
+        assert "backtest-result-2024_01_01_15_05_25_config.json" in zipf.namelist()
+        # strategy file is copied to the zip file
+        assert "backtest-result-2024_01_01_15_05_25_DefStrat.py" in zipf.namelist()
+        # compare the content of the strategy file
+        with zipf.open("backtest-result-2024_01_01_15_05_25_DefStrat.py") as strategy_file:
+            strategy_content = strategy_file.read()
+            with (strategy_test_dir / "strategy_test_v3.py").open("rb") as original_file:
+                original_content = original_file.read()
+                assert strategy_content == original_content
+        assert "backtest-result-2024_01_01_15_05_25_DefStrat.py" in zipf.namelist()
+        with zipf.open("backtest-result-2024_01_01_15_05_25_DefStrat.json") as pf:
+            params_content = pf.read()
+            with params_file.open("rb") as original_file:
+                original_content = original_file.read()
+                assert params_content == original_content
+
     assert (tmp_path / LAST_BT_RESULT_FN).is_file()
 
     # Last file reference should be updated
@@ -313,6 +351,7 @@ def test_write_read_backtest_candles(tmp_path):
         "exportfilename": tmp_path,
         "export": "signals",
         "runmode": "backtest",
+        "original_config": {},
     }
     # test directory exporting
     sample_date = "2022_01_01_15_05_13"