remove follow mode in favor of producer consumer

robcaulk · robcaulk · commit ccb4efbe889e · 2023-02-02T11:40:23.000+01:00
diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py
@@ -59,7 +59,7 @@ class FreqaiDataDrawer:
     Juha Nykänen @suikula, Wagner Costa @wagnercosta, Johan Vlugt @Jooopieeert
     """
 
-    def __init__(self, full_path: Path, config: Config, follow_mode: bool = False):
+    def __init__(self, full_path: Path, config: Config):
 
         self.config = config
         self.freqai_info = config.get("freqai", {})
@@ -84,9 +84,6 @@ def __init__(self, full_path: Path, config: Config, follow_mode: bool = False):
         self.pair_dictionary_path = Path(self.full_path / "pair_dictionary.json")
         self.global_metadata_path = Path(self.full_path / "global_metadata.json")
         self.metric_tracker_path = Path(self.full_path / "metric_tracker.json")
-        self.follow_mode = follow_mode
-        if follow_mode:
-            self.create_follower_dict()
         self.load_drawer_from_disk()
         self.load_historic_predictions_from_disk()
         self.metric_tracker: Dict[str, Dict[str, Dict[str, list]]] = {}
@@ -149,13 +146,8 @@ def load_drawer_from_disk(self):
         if exists:
             with open(self.pair_dictionary_path, "r") as fp:
                 self.pair_dict = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
-        elif not self.follow_mode:
-            logger.info("Could not find existing datadrawer, starting from scratch")
         else:
-            logger.warning(
-                f"Follower could not find pair_dictionary at {self.full_path} "
-                "sending null values back to strategy"
-            )
+            logger.info("Could not find existing datadrawer, starting from scratch")
 
     def load_metric_tracker_from_disk(self):
         """
@@ -193,13 +185,8 @@ def load_historic_predictions_from_disk(self):
                     self.historic_predictions = cloudpickle.load(fp)
                 logger.warning('FreqAI successfully loaded the backup historical predictions file.')
 
-        elif not self.follow_mode:
-            logger.info("Could not find existing historic_predictions, starting from scratch")
         else:
-            logger.warning(
-                f"Follower could not find historic predictions at {self.full_path} "
-                "sending null values back to strategy"
-            )
+            logger.info("Could not find existing historic_predictions, starting from scratch")
 
         return exists
 
@@ -248,23 +235,6 @@ def save_global_metadata_to_disk(self, metadata: Dict[str, Any]):
                 rapidjson.dump(metadata, fp, default=self.np_encoder,
                                number_mode=rapidjson.NM_NATIVE)
 
-    def create_follower_dict(self):
-        """
-        Create or dictionary for each follower to maintain unique persistent prediction targets
-        """
-
-        whitelist_pairs = self.config.get("exchange", {}).get("pair_whitelist")
-
-        exists = self.follower_dict_path.is_file()
-
-        if exists:
-            logger.info("Found an existing follower dictionary")
-
-        for pair in whitelist_pairs:
-            self.follower_dict[pair] = {}
-
-        self.save_follower_dict_to_disk()
-
     def np_encoder(self, object):
         if isinstance(object, np.generic):
             return object.item()
@@ -282,27 +252,17 @@ def get_pair_dict_info(self, pair: str) -> Tuple[str, int, bool]:
         """
 
         pair_dict = self.pair_dict.get(pair)
-        data_path_set = self.pair_dict.get(pair, self.empty_pair_dict).get("data_path", "")
+        # data_path_set = self.pair_dict.get(pair, self.empty_pair_dict).get("data_path", "")
         return_null_array = False
 
         if pair_dict:
             model_filename = pair_dict["model_filename"]
             trained_timestamp = pair_dict["trained_timestamp"]
-        elif not self.follow_mode:
+        else:
             self.pair_dict[pair] = self.empty_pair_dict.copy()
             model_filename = ""
             trained_timestamp = 0
 
-        if not data_path_set and self.follow_mode:
-            logger.warning(
-                f"Follower could not find current pair {pair} in "
-                f"pair_dictionary at path {self.full_path}, sending null values "
-                "back to strategy."
-            )
-            trained_timestamp = 0
-            model_filename = ''
-            return_null_array = True
-
         return model_filename, trained_timestamp, return_null_array
 
     def set_pair_dict_info(self, metadata: dict) -> None:
@@ -311,7 +271,6 @@ def set_pair_dict_info(self, metadata: dict) -> None:
             return
         else:
             self.pair_dict[metadata["pair"]] = self.empty_pair_dict.copy()
-
             return
 
     def set_initial_return_values(self, pair: str, pred_df: DataFrame) -> None:
diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py
@@ -66,12 +66,11 @@ def __init__(self, config: Config) -> None:
         self.retrain = False
         self.first = True
         self.set_full_path()
-        self.follow_mode: bool = self.freqai_info.get("follow_mode", False)
         self.save_backtest_models: bool = self.freqai_info.get("save_backtest_models", True)
         if self.save_backtest_models:
             logger.info('Backtesting module configured to save all models.')
 
-        self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode)
+        self.dd = FreqaiDataDrawer(Path(self.full_path), self.config)
         # set current candle to arbitrary historical date
         self.current_candle: datetime = datetime.fromtimestamp(637887600, tz=timezone.utc)
         self.dd.current_candle = self.current_candle
@@ -153,7 +152,7 @@ def start(self, dataframe: DataFrame, metadata: dict, strategy: IStrategy) -> Da
         # (backtest window, i.e. window immediately following the training window).
         # FreqAI slides the window and sequentially builds the backtesting results before returning
         # the concatenated results for the full backtesting period back to the strategy.
-        elif not self.follow_mode:
+        else:
             self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"])
             if not self.config.get("freqai_backtest_live_models", False):
                 logger.info(f"Training {len(self.dk.training_timeranges)} timeranges")
@@ -379,46 +378,28 @@ def start_live(
         :returns:
         dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only
         """
-        # update follower
-        if self.follow_mode:
-            self.dd.update_follower_metadata()
 
         # get the model metadata associated with the current pair
         (_, trained_timestamp, return_null_array) = self.dd.get_pair_dict_info(metadata["pair"])
 
-        # if the metadata doesn't exist, the follower returns null arrays to strategy
-        if self.follow_mode and return_null_array:
-            logger.info("Returning null array from follower to strategy")
-            self.dd.return_null_values_to_strategy(dataframe, dk)
-            return dk
-
         # append the historic data once per round
         if self.dd.historic_data:
             self.dd.update_historic_data(strategy, dk)
             logger.debug(f'Updating historic data on pair {metadata["pair"]}')
             self.track_current_candle()
 
-        if not self.follow_mode:
-
-            (_, new_trained_timerange, data_load_timerange) = dk.check_if_new_training_required(
-                trained_timestamp
-            )
-            dk.set_paths(metadata["pair"], new_trained_timerange.stopts)
-
-            # load candle history into memory if it is not yet.
-            if not self.dd.historic_data:
-                self.dd.load_all_pair_histories(data_load_timerange, dk)
+        (_, new_trained_timerange, data_load_timerange) = dk.check_if_new_training_required(
+            trained_timestamp
+        )
+        dk.set_paths(metadata["pair"], new_trained_timerange.stopts)
 
-            if not self.scanning:
-                self.scanning = True
-                self.start_scanning(strategy)
+        # load candle history into memory if it is not yet.
+        if not self.dd.historic_data:
+            self.dd.load_all_pair_histories(data_load_timerange, dk)
 
-        elif self.follow_mode:
-            dk.set_paths(metadata["pair"], trained_timestamp)
-            logger.info(
-                "FreqAI instance set to follow_mode, finding existing pair "
-                f"using { self.identifier }"
-            )
+        if not self.scanning:
+            self.scanning = True
+            self.start_scanning(strategy)
 
         # load the model and associated data into the data kitchen
         self.model = self.dd.load_data(metadata["pair"], dk)
diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py
@@ -376,57 +376,6 @@ def test_backtesting_fit_live_predictions(mocker, freqai_conf, caplog):
     shutil.rmtree(Path(freqai.dk.full_path))
 
 
-def test_follow_mode(mocker, freqai_conf):
-    freqai_conf.update({"timerange": "20180110-20180130"})
-
-    strategy = get_patched_freqai_strategy(mocker, freqai_conf)
-    exchange = get_patched_exchange(mocker, freqai_conf)
-    strategy.dp = DataProvider(freqai_conf, exchange)
-    strategy.freqai_info = freqai_conf.get("freqai", {})
-    freqai = strategy.freqai
-    freqai.live = True
-    freqai.dk = FreqaiDataKitchen(freqai_conf)
-    timerange = TimeRange.parse_timerange("20180110-20180130")
-    freqai.dd.load_all_pair_histories(timerange, freqai.dk)
-
-    metadata = {"pair": "ADA/BTC"}
-    freqai.dd.set_pair_dict_info(metadata)
-
-    data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
-    new_timerange = TimeRange.parse_timerange("20180120-20180130")
-
-    freqai.extract_data_and_train_model(
-        new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange)
-
-    assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").is_file()
-    assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").is_file()
-    assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").is_file()
-    assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").is_file()
-
-    # start the follower and ask it to predict on existing files
-
-    freqai_conf.get("freqai", {}).update({"follow_mode": "true"})
-
-    strategy = get_patched_freqai_strategy(mocker, freqai_conf)
-    exchange = get_patched_exchange(mocker, freqai_conf)
-    strategy.dp = DataProvider(freqai_conf, exchange)
-    strategy.freqai_info = freqai_conf.get("freqai", {})
-    freqai = strategy.freqai
-    freqai.live = True
-    freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.live)
-    timerange = TimeRange.parse_timerange("20180110-20180130")
-    freqai.dd.load_all_pair_histories(timerange, freqai.dk)
-
-    df = strategy.dp.get_pair_dataframe('ADA/BTC', '5m')
-
-    freqai.dk.pair = "ADA/BTC"
-    freqai.start_live(df, metadata, strategy, freqai.dk)
-
-    assert len(freqai.dk.return_dataframe.index) == 5702
-
-    shutil.rmtree(Path(freqai.dk.full_path))
-
-
 def test_principal_component_analysis(mocker, freqai_conf):
     freqai_conf.update({"timerange": "20180110-20180130"})
     freqai_conf.get("freqai", {}).get("feature_parameters", {}).update(
