Merge pull request #1212 from automl/feature/1206_ask_and_tell_with_initial_design

ask and tell without initial design

Author: timruhkopf

CHANGELOG.md

@@ -14,6 +14,12 @@
 ## Improvements
 - `target_function` becomes optional in Facade when using ask and tell exclusively (#946)
 
+## Documentation
+- Ask and tell without initial design and warmstarting
+
+## Bugfixes 
+- Ask and tell without initial design may no longer return a config from the initial design - if it is not "removed".  
+
 # 2.3.0
 
 ## Documentation

examples/1_basics/3_ask_and_tell.py

@@ -2,6 +2,10 @@
 # Flags: doc-Runnable
 
 This examples show how to use the Ask-and-Tell interface.
+
+Notice, that the ask-and-tell interface will still use the initial design specified in the facade.
+Should you wish to add your own evaluated configurations instead or deactivate the initial
+design all together, please refer to the warmstarting example in conjunction with this one.
 """
 
 from ConfigSpace import Configuration, ConfigurationSpace, Float
@@ -52,7 +56,7 @@ def train(self, config: Configuration, seed: int = 0) -> float:
     # Now we use SMAC to find the best hyperparameters
     smac = HyperparameterOptimizationFacade(
         scenario,
-        model.train,
+        target_function=model.train,
         intensifier=intensifier,
         overwrite=True,
     )
@@ -68,7 +72,14 @@ def train(self, config: Configuration, seed: int = 0) -> float:
         smac.tell(info, value)
 
     # After calling ask+tell, we can still optimize
-    # Note: SMAC will optimize the next 90 trials because 10 trials already have been evaluated
+    # Note: SMAC will optimize the next 90 trials because 10 trials already have been evaluated.
+    # If we however choose not to call optimize; e.g. because we want to manage heavy
+    # computation of model.train completely outside smac, but still use it to suggest new
+    # configurations, then n_trials will only be relevant for the initial design in combination
+    # with initial design max_ratio! In fact in an only ask+tell case, we could even set
+    # target_function=None in the constructor, because smac wouldn't even need to know
+    # what the target function is. But that will prevent us from calling optimize and validate later
+    # on.
     incumbent = smac.optimize()
 
     # Get cost of default configuration

examples/1_basics/8_warmstart.py

@@ -56,15 +56,19 @@ def evaluate(self, config: Configuration, seed: int = 0) -> float:
     intensifier = HyperparameterOptimizationFacade.get_intensifier(scenario, max_config_calls=1)
     smac = HyperparameterOptimizationFacade(
         scenario,
-        task.evaluate,
+        target_function=task.evaluate,
         intensifier=intensifier,
         overwrite=True,
 
         # Modify the initial design to use our custom initial design
         initial_design=HyperparameterOptimizationFacade.get_initial_design(
             scenario, 
-            n_configs=0,  # Do not use the default initial design
-            additional_configs=configurations  # Use the configurations previously evaluated as initial design
+            n_configs=0,  # Do not use the default initial design at all
+
+            # You can pass the configurations as additional_configs, which will specify their
+            # origin to be the initial design. However, this is not necessary and we can just
+            # smac.tell the configurations.
+            # additional_configs=configurations  # Use the configurations previously evaluated as initial design
                                                # This only passes the configurations but not the cost!
                                                # So in order to actually use the custom, pre-evaluated initial design
                                                # we need to tell those trials, like below.
@@ -80,4 +84,6 @@ def evaluate(self, config: Configuration, seed: int = 0) -> float:
         smac.tell(info, value)
 
     # Optimize as usual
-    smac.optimize()
+    # Notice, that since we added three configurations, n_trials for the remaining optimization
+    # is effectively 27 in optimize().
+    smac.optimize()

smac/initial_design/abstract_initial_design.py

@@ -82,7 +82,11 @@ def __init__(
             )
 
         # If the number of configurations is too large, we reduce it
-        _n_configs = int(max(1, min(self._n_configs, (max_ratio * scenario.n_trials))))
+        if self._n_configs > 1:
+            _n_configs = int(max(1, min(self._n_configs, (max_ratio * scenario.n_trials))))
+        else:
+            _n_configs = self._n_configs
+
         if self._n_configs != _n_configs:
             logger.info(
                 f"Reducing the number of initial configurations from {self._n_configs} to "

smac/main/config_selector.py

@@ -103,7 +103,8 @@ def _set_components(
 
         self._initial_design_configs = initial_design.select_configurations()
         if len(self._initial_design_configs) == 0:
-            raise RuntimeError("SMAC needs initial configurations to work.")
+            # raise RuntimeError("SMAC needs initial configurations to work.")
+            logger.warning("No initial configurations were sampled.")
 
     @property
     def meta(self) -> dict[str, Any]: