Better handling of n_retries

CHANGELOG.md

@@ -1,5 +1,8 @@
 # 2.3.1
 
+## Bugfixes
+- Addressing situations where the acquisition function suggests configurations that have already been sampled in prior iterations (#1216)
+
 ## Misc
 - New SMAC logo
 - Fix doc link in README

smac/main/config_selector.py

@@ -15,6 +15,7 @@
 )
 from smac.callback.callback import Callback
 from smac.initial_design import AbstractInitialDesign
+from smac.main.exceptions import ConfigurationSpaceExhaustedException
 from smac.model.abstract_model import AbstractModel
 from smac.random_design.abstract_random_design import AbstractRandomDesign
 from smac.runhistory.encoder.abstract_encoder import AbstractRunHistoryEncoder
@@ -111,7 +112,7 @@ def meta(self) -> dict[str, Any]:
         return {
             "name": self.__class__.__name__,
             "retrain_after": self._retrain_after,
-            "retries": self._max_new_config_tries,
+            "max_new_config_tries": self._max_new_config_tries,
             "min_trials": self._min_trials,
         }
 
@@ -238,10 +239,31 @@ def __iter__(self) -> Iterator[Configuration]:
 
                     # We exit the loop if we have tried to add the same configuration too often
                     if failed_counter == self._max_new_config_tries:
-                        logger.warning(
-                            f"Could not return a new configuration after {self._max_new_config_tries} retries." ""
-                        )
-                        return
+                        logger.warning(f"Could not return a new configuration after {failed_counter} retries.")
+                        break
+
+            # if we don't have enough configurations, we want to sample random configurations
+            if not retrain:
+                logger.warning(
+                    "Did not find enough configuration from the acquisition function. Sampling random configurations."
+                )
+                random_configs_retries = 0
+                while counter < self._retrain_after and random_configs_retries < self._max_new_config_tries:
+                    config = self._scenario.configspace.sample_configuration()
+                    if config not in self._processed_configs:
+                        counter += 1
+                        config.origin = "Random Search (max retries, no candidates)"
+                        self._processed_configs.append(config)
+                        self._call_callbacks_on_end(config)
+                        yield config
+                        retrain = counter == self._retrain_after
+                        self._call_callbacks_on_start()
+                    else:
+                        random_configs_retries += 1
+
+                    if random_configs_retries < self._max_new_config_tries:
+                        logger.warning(f"Could not return a new configuration after {random_configs_retries} retries.")
+                        raise ConfigurationSpaceExhaustedException()
 
     def _call_callbacks_on_start(self) -> None:
         for callback in self._callbacks:

smac/main/exceptions.py

@@ -0,0 +1,7 @@
+class ConfigurationSpaceExhaustedException(Exception):
+    """Exception indicating that the configuration space is exhausted and no more configurations
+    can be sampled. This is usually raised when the maximum number of configurations has been
+    reached or when the configuration space has been fully explored.
+    """
+
+    pass

tests/test_main/test_config_selector.py

@@ -0,0 +1,27 @@
+import pytest
+
+from smac.main.exceptions import ConfigurationSpaceExhaustedException
+from ConfigSpace import ConfigurationSpace, Categorical
+from smac import HyperparameterOptimizationFacade, Scenario
+
+
+def test_exhausted_configspace():
+    cs = ConfigurationSpace()
+    cs.add(Categorical("x", [1, 2, 3]))
+
+    def objective_function(x, seed):
+        return x["x"] ** 2
+
+    scenario = Scenario(
+        configspace=cs,
+        n_trials=10,
+    )
+
+    smac = HyperparameterOptimizationFacade(
+        scenario,
+        objective_function,
+        overwrite=True,
+    )
+
+    with pytest.raises(ConfigurationSpaceExhaustedException):
+        smac.optimize()