Merge branch 'develop' into backtest_live_models

Author: wagnercosta

.github/workflows/ci.yml

@@ -258,7 +258,7 @@ jobs:
           webhookUrl: ${{ secrets.DISCORD_WEBHOOK }}
 
   mypy_version_check:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v3
 
@@ -283,7 +283,7 @@ jobs:
     - uses: pre-commit/action@v3.0.0
 
   docs_check:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v3
 
@@ -313,7 +313,7 @@ jobs:
   # Notify only once - when CI completes (and after deploy) in case it's successfull
   notify-complete:
     needs: [ build_linux, build_macos, build_windows, docs_check, mypy_version_check, pre-commit ]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     # Discord notification can't handle schedule events
     if: (github.event_name != 'schedule')
     permissions:
@@ -338,7 +338,7 @@ jobs:
 
   deploy:
     needs: [ build_linux, build_macos, build_windows, docs_check, mypy_version_check, pre-commit ]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade'
 

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
           - types-filelock==3.2.7
           - types-requests==2.28.11.2
           - types-tabulate==0.9.0.0
-          - types-python-dateutil==2.8.19.1
+          - types-python-dateutil==2.8.19.2
         # stages: [push]
 
   - repo: https://github.com/pycqa/isort

build_helpers/pyarrow-10.0.0-cp39-cp39-linux_armv7l.whl

@@ -53,7 +53,7 @@
             "XTZ/BTC"
         ],
         "pair_blacklist": [
-            "BNB/BTC"
+            "BNB/.*"
         ]
     },
     "pairlists": [

config_examples/config_binance.example.json

@@ -18,13 +18,8 @@
         "name": "binance",
         "key": "",
         "secret": "",
-        "ccxt_config": {
-            "enableRateLimit": true
-        },
-        "ccxt_async_config": {
-            "enableRateLimit": true,
-            "rateLimit": 200
-        },
+        "ccxt_config": {},
+        "ccxt_async_config": {},
         "pair_whitelist": [
             "1INCH/USDT",
             "ALGO/USDT"

config_examples/config_freqai.example.json

@@ -11,7 +11,7 @@ ENV FT_APP_ENV="docker"
 # Prepare environment
 RUN mkdir /freqtrade \
   && apt-get update \
-  && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-dev \
+  && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-dev libutf8proc-dev libsnappy-dev \
   && apt-get clean \
   && useradd -u 1000 -G sudo -U -m ftuser \
   && chown ftuser:ftuser /freqtrade \
@@ -37,6 +37,7 @@ ENV LD_LIBRARY_PATH /usr/local/lib
 COPY --chown=ftuser:ftuser requirements.txt /freqtrade/
 USER ftuser
 RUN  pip install --user --no-cache-dir numpy \
+  && pip install --user /tmp/pyarrow-*.whl \
   && pip install --user --no-cache-dir -r requirements.txt
 
 # Copy dependencies to runtime-image

docker/Dockerfile.armhf

@@ -78,6 +78,8 @@ This function needs to return a floating point number (`float`). Smaller numbers
 To override a pre-defined space (`roi_space`, `generate_roi_table`, `stoploss_space`, `trailing_space`), define a nested class called Hyperopt and define the required spaces as follows:
 
 ```python
+from freqtrade.optimize.space import Categorical, Dimension, Integer, SKDecimal
+
 class MyAwesomeStrategy(IStrategy):
     class HyperOpt:
         # Define a custom stoploss space.
@@ -94,6 +96,33 @@ class MyAwesomeStrategy(IStrategy):
                 SKDecimal(0.01, 0.07, decimals=3, name='roi_p2'),
                 SKDecimal(0.01, 0.20, decimals=3, name='roi_p3'),
             ]
+
+        def generate_roi_table(params: Dict) -> Dict[int, float]:
+
+            roi_table = {}
+            roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3']
+            roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2']
+            roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1']
+            roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0
+
+            return roi_table
+
+        def trailing_space() -> List[Dimension]:
+            # All parameters here are mandatory, you can only modify their type or the range.
+            return [
+                # Fixed to true, if optimizing trailing_stop we assume to use trailing stop at all times.
+                Categorical([True], name='trailing_stop'),
+
+                SKDecimal(0.01, 0.35, decimals=3, name='trailing_stop_positive'),
+                # 'trailing_stop_positive_offset' should be greater than 'trailing_stop_positive',
+                # so this intermediate parameter is used as the value of the difference between
+                # them. The value of the 'trailing_stop_positive_offset' is constructed in the
+                # generate_trailing_params() method.
+                # This is similar to the hyperspace dimensions used for constructing the ROI tables.
+                SKDecimal(0.001, 0.1, decimals=3, name='trailing_stop_positive_offset_p1'),
+
+                Categorical([True, False], name='trailing_only_offset_is_reached'),
+        ]
 ```
 
 !!! Note

docs/advanced-hyperopt.md

@@ -522,13 +522,13 @@ Since backtesting lacks some detailed information about what happens within a ca
 - ROI
   - exits are compared to high - but the ROI value is used (e.g. ROI = 2%, high=5% - so the exit will be at 2%)
   - exits are never "below the candle", so a ROI of 2% may result in a exit at 2.4% if low was at 2.4% profit
-  - Forceexits caused by `<N>=-1` ROI entries use low as exit value, unless N falls on the candle open (e.g. `120: -1` for 1h candles)
+  - Force-exits caused by `<N>=-1` ROI entries use low as exit value, unless N falls on the candle open (e.g. `120: -1` for 1h candles)
 - Stoploss exits happen exactly at stoploss price, even if low was lower, but the loss will be `2 * fees` higher than the stoploss price
 - Stoploss is evaluated before ROI within one candle. So you can often see more trades with the `stoploss` exit reason comparing to the results obtained with the same strategy in the Dry Run/Live Trade modes
 - Low happens before high for stoploss, protecting capital first
 - Trailing stoploss
   - Trailing Stoploss is only adjusted if it's below the candle's low (otherwise it would be triggered)
-  - On trade entry candles that trigger trailing stoploss, the "minimum offset" (`stop_positive_offset`) is assumed (instead of high) - and the stop is calculated from this point
+  - On trade entry candles that trigger trailing stoploss, the "minimum offset" (`stop_positive_offset`) is assumed (instead of high) - and the stop is calculated from this point. This rule is NOT applicable to custom-stoploss scenarios, since there's no information about the stoploss logic available.
   - High happens first - adjusting stoploss
   - Low uses the adjusted stoploss (so exits with large high-low difference are backtested correctly)
   - ROI applies before trailing-stop, ensuring profits are "top-capped" at ROI if both ROI and trailing stop applies
@@ -546,8 +546,8 @@ In addition to the above assumptions, strategy authors should carefully read the
 
 ### Trading limits in backtesting
 
-Exchanges have certain trading limits, like minimum base currency, or minimum stake (quote) currency.
-These limits are usually listed in the exchange documentation as "trading rules" or similar.
+Exchanges have certain trading limits, like minimum (and maximum) base currency, or minimum/maximum stake (quote) currency.
+These limits are usually listed in the exchange documentation as "trading rules" or similar and can be quite different between different pairs.
 
 Backtesting (as well as live and dry-run) does honor these limits, and will ensure that a stoploss can be placed below this value - so the value will be slightly higher than what the exchange specifies.
 Freqtrade has however no information about historic limits.

docs/backtesting.md

@@ -102,6 +102,12 @@ If this happens for all pairs in the pairlist, this might indicate a recent exch
 
 Irrespectively of the reason, Freqtrade will fill up these candles with "empty" candles, where open, high, low and close are set to the previous candle close - and volume is empty. In a chart, this will look like a `_` - and is aligned with how exchanges usually represent 0 volume candles.
 
+### I'm getting "Price jump between 2 candles detected"
+
+This message is a warning that the candles had a price jump of > 30%.
+This might be a sign that the pair stopped trading, and some token exchange took place (e.g. COCOS in 2021 - where price jumped from 0.0000154 to 0.01621).
+This message is often accompanied by ["Missing data fillup"](#im-getting-missing-data-fillup-messages-in-the-log) - as trading on such pairs is often stopped for some time.
+
 ### I'm getting "Outdated history for pair xxx" in the log
 
 The bot is trying to tell you that it got an outdated last candle (not the last complete candle).

docs/faq.md

@@ -61,28 +61,25 @@ The FreqAI strategy requires including the following lines of code in the standa
         """
         Function designed to automatically generate, name and merge features
         from user indicated timeframes in the configuration file. User controls the indicators
-        passed to the training/prediction by prepending indicators with `'%-' + coin `
+        passed to the training/prediction by prepending indicators with `'%-' + pair `
         (see convention below). I.e. user should not prepend any supporting metrics
         (e.g. bb_lowerband below) with % unless they explicitly want to pass that metric to the
         model.
         :param pair: pair to be used as informative
         :param df: strategy dataframe which will receive merges from informatives
         :param tf: timeframe of the dataframe which will modify the feature names
         :param informative: the dataframe associated with the informative pair
-        :param coin: the name of the coin which will modify the feature names.
         """
 
-        coin = pair.split('/')[0]
-
         if informative is None:
             informative = self.dp.get_pair_dataframe(pair, tf)
 
         # first loop is automatically duplicating indicators for time periods
         for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
             t = int(t)
-            informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
-            informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
-            informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t)
+            informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
+            informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
+            informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, window=t)
 
         indicators = [col for col in informative if col.startswith("%")]
         # This loop duplicates and shifts all indicators to add a sense of recency to data
@@ -134,7 +131,7 @@ Notice also the location of the labels under `if set_generalized_indicators:` at
     (as exemplified in `freqtrade/templates/FreqaiExampleStrategy.py`):
 
     ```python
-        def populate_any_indicators(self, metadata, pair, df, tf, informative=None, coin="", set_generalized_indicators=False):
+        def populate_any_indicators(self, pair, df, tf, informative=None, set_generalized_indicators=False):
 
             ...
 
@@ -192,11 +189,11 @@ dataframe["target_roi"] = dataframe["&-s_close_mean"] + dataframe["&-s_close_std
 dataframe["sell_roi"] = dataframe["&-s_close_mean"] - dataframe["&-s_close_std"] * 1.25
 ```
 
-To consider the population of *historical predictions* for creating the dynamic target instead of information from the training as discussed above, you would set `fit_live_prediction_candles` in the config to the number of historical prediction candles you wish to use to generate target statistics.
+To consider the population of *historical predictions* for creating the dynamic target instead of information from the training as discussed above, you would set `fit_live_predictions_candles` in the config to the number of historical prediction candles you wish to use to generate target statistics.
 
 ```json
     "freqai": {
-        "fit_live_prediction_candles": 300,
+        "fit_live_predictions_candles": 300,
     }
 ```
 

docs/freqai-configuration.md

@@ -2,7 +2,10 @@
 
 ## Defining the features
 
-Low level feature engineering is performed in the user strategy within a function called `populate_any_indicators()`. That function sets the `base features` such as, `RSI`, `MFI`, `EMA`, `SMA`, time of day, volume, etc. The `base features` can be custom indicators or they can be imported from any technical-analysis library that you can find. One important syntax rule is that all `base features` string names are prepended with `%`, while labels/targets are prepended with `&`.
+Low level feature engineering is performed in the user strategy within a function called `populate_any_indicators()`. That function sets the `base features` such as, `RSI`, `MFI`, `EMA`, `SMA`, time of day, volume, etc. The `base features` can be custom indicators or they can be imported from any technical-analysis library that you can find. One important syntax rule is that all `base features` string names are prepended with `%-{pair}`, while labels/targets are prepended with `&`.
+
+!!! Note
+    Adding the full pair string, e.g. XYZ/USD, in the feature name enables improved performance for dataframe caching on the backend. If you decide *not* to add the full pair string in the feature string, FreqAI will operate in a reduced performance mode.
 
 Meanwhile, high level feature engineering is handled within `"feature_parameters":{}` in the FreqAI config. Within this file, it is possible to decide large scale feature expansions on top of the `base_features` such as "including correlated pairs" or "including informative timeframes" or even "including recent candles."
 
@@ -15,45 +18,42 @@ It is advisable to start from the template `populate_any_indicators()` in the so
         """
         Function designed to automatically generate, name, and merge features
         from user-indicated timeframes in the configuration file. The user controls the indicators
-        passed to the training/prediction by prepending indicators with `'%-' + coin `
+        passed to the training/prediction by prepending indicators with `'%-' + pair `
         (see convention below). I.e., the user should not prepend any supporting metrics
         (e.g., bb_lowerband below) with % unless they explicitly want to pass that metric to the
         model.
         :param pair: pair to be used as informative
         :param df: strategy dataframe which will receive merges from informatives
         :param tf: timeframe of the dataframe which will modify the feature names
         :param informative: the dataframe associated with the informative pair
-        :param coin: the name of the coin which will modify the feature names.
         """
 
-        coin = pair.split('/')[0]
-
         if informative is None:
             informative = self.dp.get_pair_dataframe(pair, tf)
 
         # first loop is automatically duplicating indicators for time periods
         for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
             t = int(t)
-            informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
-            informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
-            informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t)
+            informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
+            informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
+            informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, window=t)
 
             bollinger = qtpylib.bollinger_bands(
                 qtpylib.typical_price(informative), window=t, stds=2.2
             )
-            informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"]
-            informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"]
-            informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"]
-
-            informative[f"%-{coin}bb_width-period_{t}"] = (
-                informative[f"{coin}bb_upperband-period_{t}"]
-                - informative[f"{coin}bb_lowerband-period_{t}"]
-            ) / informative[f"{coin}bb_middleband-period_{t}"]
-            informative[f"%-{coin}close-bb_lower-period_{t}"] = (
-                informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"]
+            informative[f"{pair}bb_lowerband-period_{t}"] = bollinger["lower"]
+            informative[f"{pair}bb_middleband-period_{t}"] = bollinger["mid"]
+            informative[f"{pair}bb_upperband-period_{t}"] = bollinger["upper"]
+
+            informative[f"%-{pair}bb_width-period_{t}"] = (
+                informative[f"{pair}bb_upperband-period_{t}"]
+                - informative[f"{pair}bb_lowerband-period_{t}"]
+            ) / informative[f"{pair}bb_middleband-period_{t}"]
+            informative[f"%-{pair}close-bb_lower-period_{t}"] = (
+                informative["close"] / informative[f"{pair}bb_lowerband-period_{t}"]
             )
 
-            informative[f"%-{coin}relative_volume-period_{t}"] = (
+            informative[f"%-{pair}relative_volume-period_{t}"] = (
                 informative["volume"] / informative["volume"].rolling(t).mean()
             )
 

docs/freqai-feature-engineering.md

@@ -173,9 +173,13 @@ You can indicate to the bot that it should not train models, but instead should
 
 ```json
     "freqai": {
+        "enabled": true,
         "follow_mode": true,
-        "identifier": "example"
+        "identifier": "example",
+        "feature_parameters": {
+        // leader bots feature_parameters inserted here 
+        },
     }
 ```
 
-In this example, the user has a leader bot with the `"identifier": "example"`. The leader bot is already running or is launched simultaneously with the follower. The follower will load models created by the leader and inference them to obtain predictions instead of training its own models.
+In this example, the user has a leader bot with the `"identifier": "example"`. The leader bot is already running or is launched simultaneously with the follower. The follower will load models created by the leader and inference them to obtain predictions instead of training its own models. The user will also need to duplicate the `feature_parameters` parameters from from the leaders freqai configuration file into the freqai section of the followers config.