Support running Centipede in separate processes.

centipede/BUILD

@@ -734,6 +734,7 @@ cc_library(
     }),
     visibility = PUBLIC_API_VISIBILITY,
     deps = [
+        ":centipede_flags",
         ":environment",
         "@abseil-cpp//absl/flags:flag",
         "@abseil-cpp//absl/log",
@@ -744,6 +745,11 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "centipede_flags",
+    textual_hdrs = ["centipede_flags.inc"],
+)
+
 cc_library(
     name = "centipede_lib",
     srcs = [
@@ -857,11 +863,13 @@ cc_library(
     }),
     visibility = PUBLIC_API_VISIBILITY,
     deps = [
+        ":centipede_flags",
         ":feature",
         ":knobs",
         ":util",
         "@abseil-cpp//absl/base:no_destructor",
         "@abseil-cpp//absl/container:flat_hash_map",
+        "@abseil-cpp//absl/flags:marshalling",
         "@abseil-cpp//absl/log",
         "@abseil-cpp//absl/log:check",
         "@abseil-cpp//absl/strings",

centipede/centipede_callbacks.cc

@@ -361,7 +361,7 @@ MutationResult CentipedeCallbacks::MutateViaExternalBinary(
   inputs_blobseq_.ReleaseSharedMemory();  // Inputs are already consumed.
 
   if (retval != EXIT_SUCCESS) {
-    LOG(WARNING) << "Custom mutator failed with exit code " << retval;
+    LOG(WARNING) << "Custom mutator failed with exit code: " << retval;
   }
   if (env_.print_runner_log || retval != EXIT_SUCCESS) {
     PrintExecutionLog();

centipede/centipede_interface.cc

@@ -38,6 +38,7 @@
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "absl/strings/ascii.h"
+#include "absl/strings/escaping.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
@@ -836,7 +837,9 @@ int CentipedeMain(const Environment &env,
   if (!env.binary.empty() && !env.has_input_wildcards) {
     const auto serialized_target_config = [&]() -> absl::StatusOr<std::string> {
       if (!env.fuzztest_configuration.empty()) {
-        return env.fuzztest_configuration;
+        std::string result;
+        CHECK(absl::WebSafeBase64Unescape(env.fuzztest_configuration, &result));
+        return result;
       }
       ScopedCentipedeCallbacks scoped_callbacks(callbacks_factory, env);
       return scoped_callbacks.callbacks()->GetSerializedTargetConfig();

centipede/config_util_test.cc

@@ -46,8 +46,8 @@ using ::testing::IsSupersetOf;
 TEST(FlagUtilTest, GetFlagsPerSource) {
   constexpr const char* kCentipedeRoot = "centipede/";
   constexpr const char* kThisCc = "centipede/config_util_test.cc";
-  constexpr const char* kEnvironmentFlagsCc =
-      "centipede/environment_flags.cc";
+  constexpr const char* kCentipedeFlagsInc =
+      "././centipede/centipede_flags.inc";
 
   // Change some flag values to non-defaults.
   absl::SetFlag(&FLAGS_foo, "baz");
@@ -61,13 +61,13 @@ TEST(FlagUtilTest, GetFlagsPerSource) {
     const FlagInfosPerSource flags = GetFlagsPerSource(kCentipedeRoot);
     SCOPED_TRACE(FormatFlagfileString(flags));
     ASSERT_EQ(flags.count(kThisCc), 1);
-    ASSERT_EQ(flags.count(kEnvironmentFlagsCc), 1);
+    ASSERT_EQ(flags.count(kCentipedeFlagsInc), 1);
     ASSERT_THAT(flags.at(kThisCc),
                 ElementsAreArray({
                     FlagInfo{"foo", "baz", "bar", "foo help"},
                     FlagInfo{"qux", "true", "false", "qux help"},
                 }));
-    ASSERT_THAT(flags.at(kEnvironmentFlagsCc),
+    ASSERT_THAT(flags.at(kCentipedeFlagsInc),
                 IsSupersetOf({
                     FlagInfo{"binary", "*", "*", "*"},
                     FlagInfo{"workdir", "*", "*", "*"},
@@ -78,7 +78,7 @@ TEST(FlagUtilTest, GetFlagsPerSource) {
     const FlagInfosPerSource flags = GetFlagsPerSource(kThisCc);
     SCOPED_TRACE(FormatFlagfileString(flags));
     ASSERT_EQ(flags.count(kThisCc), 1);
-    ASSERT_EQ(flags.count(kEnvironmentFlagsCc), 0);
+    ASSERT_EQ(flags.count(kCentipedeFlagsInc), 0);
     ASSERT_THAT(flags.at(kThisCc),
                 ElementsAreArray({
                     FlagInfo{"foo", "baz", "bar", "foo help"},
@@ -91,7 +91,7 @@ TEST(FlagUtilTest, GetFlagsPerSource) {
         GetFlagsPerSource(kThisCc, /*exclude_flags=*/{"qux"});
     SCOPED_TRACE(FormatFlagfileString(flags));
     ASSERT_EQ(flags.count(kThisCc), 1);
-    ASSERT_EQ(flags.count(kEnvironmentFlagsCc), 0);
+    ASSERT_EQ(flags.count(kCentipedeFlagsInc), 0);
     ASSERT_THAT(flags.at(kThisCc),
                 ElementsAreArray({
                     FlagInfo{"foo", "baz", "bar", "foo help"},

centipede/environment.cc

@@ -26,8 +26,10 @@
 
 #include "absl/base/no_destructor.h"
 #include "absl/container/flat_hash_map.h"
+#include "absl/flags/marshalling.h"
 #include "absl/log/check.h"
 #include "absl/log/log.h"
+#include "absl/strings/str_cat.h"
 #include "absl/strings/str_split.h"
 #include "absl/strings/string_view.h"
 #include "absl/time/time.h"
@@ -335,4 +337,22 @@ void Environment::UpdateBinaryHashIfEmpty() {
   }
 }
 
+std::vector<std::string> Environment::SerializeToCommandFlags() const {
+  std::vector<std::string> flags;
+#define CENTIPEDE_FLAG(TYPE, NAME, _DEFAULT, _DESC)                         \
+  if (NAME != Default().NAME) {                                             \
+    flags.push_back(absl::StrCat("--" #NAME "=", absl::UnparseFlag(NAME))); \
+  }
+#define CENTIPEDE_FLAG_ALIAS(_TYPE, _ALIAS_NAME, _FLAG_NAME, _DEFAULT)
+#define CENTIPEDE_FUZZTEST_FLAG(TYPE, NAME, _DEFAULT, _DESC)                \
+  if (NAME != Default().NAME) {                                             \
+    flags.push_back(absl::StrCat("--" #NAME "=", absl::UnparseFlag(NAME))); \
+  }
+#include "./centipede/centipede_flags.inc"
+#undef CENTIPEDE_FLAG
+#undef CENTIPEDE_FLAG_ALIAS
+#undef CENTIPEDE_FUZZTEST_FLAG
+  return flags;
+}
+
 }  // namespace fuzztest::internal

centipede/environment.h

@@ -35,127 +35,15 @@ namespace fuzztest::internal {
 // the flags defined in environment_flags.cc, while other users can use
 // CentipedeMain() as a library function without importing the flags.
 struct Environment {
-  // Global params. See environment_flags.cc for help on each parameter. -------
-
-  std::string binary;
-  std::string coverage_binary;
-  std::string clang_coverage_binary;
-  std::vector<std::string> extra_binaries;
-  std::string workdir;
-  std::string merge_from;
-  size_t num_runs = std::numeric_limits<size_t>::max();
-  size_t total_shards = 1;
-  size_t my_shard_index = 0;
-  size_t num_threads = 1;
-  size_t j = 0;
-  size_t max_len = 4000;
-  size_t batch_size = 1000;
-  size_t mutate_batch_size = 2;
-  bool use_legacy_default_mutator = false;
-  size_t load_other_shard_frequency = 10;
-  bool serialize_shard_loads = false;
-  size_t seed = 0;
-  size_t prune_frequency = 100;
-#ifdef __APPLE__
-  // Address space limit is ignored on MacOS.
-  // Reference: https://bugs.chromium.org/p/chromium/issues/detail?id=853873#c2
-  size_t address_space_limit_mb = 0;
-#else   // __APPLE__
-  size_t address_space_limit_mb = 8192;
-#endif  // __APPLE__
-  size_t rss_limit_mb = 4096;
-  size_t stack_limit_kb = 0;
-  size_t timeout_per_input = 60;
-  size_t timeout_per_batch = 0;
-  bool ignore_timeout_reports = false;
-  absl::Time stop_at = absl::InfiniteFuture();
-  bool fork_server = true;
-  bool full_sync = false;
-  bool use_corpus_weights = true;
-  bool use_coverage_frontier = false;
-  size_t max_corpus_size = 100000;
-  size_t crossover_level = 50;
-  bool use_pc_features = true;
-  size_t path_level = 0;
-  bool use_cmp_features = true;
-  size_t callstack_level = 0;
-  bool use_auto_dictionary = true;
-  bool use_dataflow_features = true;
-  bool use_counter_features = false;
-  bool use_pcpair_features = false;
-  uint64_t user_feature_domain_mask = ~0UL;
-  size_t feature_frequency_threshold = 100;
-  bool require_pc_table = true;
-  bool require_seeds = false;
-  int telemetry_frequency = 0;
-  bool print_runner_log = false;
-  bool distill = false;
-  size_t log_features_shards = 0;
-  std::string knobs_file;
-  std::string corpus_to_files;
-  std::string corpus_from_files;
-  std::vector<std::string> corpus_dir;
-  std::string symbolizer_path = "llvm-symbolizer";
-  std::string objdump_path = "objdump";
-  std::string runner_dl_path_suffix;
-  std::string input_filter;
-  std::vector<std::string> dictionary;
-  std::string function_filter;
-  std::string for_each_blob;
-  std::string experiment;
-  bool analyze = false;
-  bool exit_on_crash = false;
-  size_t max_num_crash_reports = 5;
-  std::string minimize_crash_file_path;
-  bool batch_triage_suspect_only = false;
-  size_t shmem_size_mb = 1024;
-#ifdef __APPLE__
-  bool use_posix_shmem = true;
-#else
-  bool use_posix_shmem = false;
-#endif
-  bool dry_run = false;
-  bool save_binary_info = false;
-  bool populate_binary_info = true;
-#ifdef CENTIPEDE_DISABLE_RIEGELI
-  bool riegeli = false;
-#else
-  bool riegeli = true;
-#endif  // CENTIPEDE_DISABLE_RIEGELI
-
-  // Internal settings without global flags ------------------------------------
-
-  // If set, treat the first entry of `corpus_dir` as output-only.
-  bool first_corpus_dir_output_only = false;
-  // If set, load/merge shards without fuzzing new inputs.
-  bool load_shards_only = false;
-  // If set, operate on the corpus database for a single test specified by
-  // FuzzTest instead of all the tests.
-  bool fuzztest_single_test_mode = false;
-  // If set, deserializes the configuration from the value instead of querying
-  // the configuration via runner callbacks.
-  std::string fuzztest_configuration;
-  // If set, lists the crash IDs of a single test of the binary to
-  // the `crash_ids_file` with each crash ID in a single line. If there is no
-  // crash for the test, the empty content will be written to the file.
-  bool list_crash_ids = false;
-  // The path to list the crash IDs for `list_crash_ids`.
-  std::string list_crash_ids_file;
-  // The crash ID used for `replay_crash` or `export_crash`.
-  std::string crash_id;
-  // If set, replay `crash_id` in the corpus database.
-  bool replay_crash = false;
-  // If set, export the input contents of `crash_id` from the corpus database.
-  bool export_crash = false;
-  // The path to export the input contents of `crash_id` for `export_crash`.
-  std::string export_crash_file;
+#define CENTIPEDE_FLAG(TYPE, NAME, DEFAULT, _DESC) TYPE NAME = DEFAULT;
+#include "./centipede/centipede_flags.inc"
+#undef CENTIPEDE_FLAG
 
   // Command line-related fields -----------------------------------------------
 
   std::string exec_name;          // copied from argv[0]
   std::vector<std::string> args;  // copied from argv[1:].
   std::string binary_name;        // Name of `coverage_binary`, w/o directories.
-  std::string binary_hash;        // Hash of the `coverage_binary` file.
   bool has_input_wildcards = false;  // Set to true iff `binary` contains "@@".
 
   // Experiment-related settings -----------------------------------------------
@@ -243,6 +131,8 @@ struct Environment {
   void UpdateTimeoutPerBatchIfEqualTo(size_t val);
   // If `binary_hash` is empty, updates it using the file in `coverage_binary`.
   void UpdateBinaryHashIfEmpty();
+
+  std::vector<std::string> SerializeToCommandFlags() const;
 };
 
 }  // namespace fuzztest::internal