support creating index on update if missing (#5773)

* add support for creating missing index on update

* add create-on-update to cli

* add test

* optionally create source on update

* add documentation

Author: trinity-1686a

docs/reference/cli.md

@@ -199,6 +199,7 @@ quickwit index update
 |-----------------|-------------|
 | `--index` | ID of the target index |
 | `--index-config` | Location of the index config file. |
+| `--create` | Create the index if it doesn't exist. |
 ### index clear
 
 Clears an index: deletes all splits and resets checkpoint.  
@@ -514,6 +515,7 @@ quickwit source update
 | `--index` | ID of the target index |
 | `--source` | ID of the source |
 | `--source-config` | Path to source config file. Please, refer to the documentation for more details. |
+| `--create` | Create the source if it doesn't exist. |
 ### source enable
 
 Enables a source for an index.  

docs/reference/rest-api.md

@@ -327,7 +327,19 @@ The response is the index metadata of the created index, and the content type is
 PUT api/v1/indexes/<index id>
 ```
 
-Updates the configurations of an index. This endpoint follows PUT semantics, which means that all the fields of the current configuration are replaced by the values specified in this request or the associated defaults. In particular, if the field is optional (e.g. `retention_policy`), omitting it will delete the associated configuration. If the new configuration file contains updates that cannot be applied, the request fails, and none of the updates are applied. The API accepts JSON with `content-type: application/json` and YAML with `content-type: application/yaml`.
+#### Path variable
+
+| Variable      | Description   |
+| ------------- | ------------- |
+| `index id`    | The index id  |
+
+#### Query parameters
+
+| Variable  | Type   | Description                                   | Default value |
+|-----------|--------|-----------------------------------------------|---------------|
+| `create`  | `bool` | Create the index if it doesn't already exists | `false`       |
+
+Update the configurations of an index. This endpoint follows PUT semantics, which means that all the fields of the current configuration are replaced by the values specified in this request or the associated defaults. In particular, if the field is optional (e.g. `retention_policy`), omitting it will delete the associated configuration. If the new configuration file contains updates that cannot be applied, the request fails, and none of the updates are applied. The API accepts JSON with `content-type: application/json` and YAML with `content-type: application/yaml`.
 
 - The retention policy update is automatically picked up by the janitor service on its next state refresh.
 - The search settings update is automatically picked up by searcher nodes when the next query is executed.
@@ -645,6 +657,19 @@ The response is the created source config, and the content type is `application/
 PUT api/v1/indexes/<index id>/sources/<source id>
 ```
 
+#### Path variable
+
+| Variable      | Description   |
+| ------------- | ------------- |
+| `index id`    | The index id  |
+| `source id`   | The source id  |
+
+#### Query parameters
+
+| Variable  | Type   | Description                                   | Default value |
+|-----------|--------|-----------------------------------------------|---------------|
+| `create`  | `bool` | Create the index if it doesn't already exists | `false`       |
+
 Update a source by posting a source config JSON payload.
 
 #### PUT payload

quickwit/quickwit-cli/src/index.rs

@@ -76,6 +76,9 @@ pub fn build_index_command() -> Command {
                 arg!(--"index-config" <INDEX_CONFIG> "Location of the index config file.")
                     .display_order(2)
                     .required(true),
+                arg!(--"create" "Create the index if it does not already exists.")
+                    .display_order(3)
+                    .required(false),
             ])
         )
         .subcommand(
@@ -212,6 +215,7 @@ pub struct UpdateIndexArgs {
     pub client_args: ClientArgs,
     pub index_id: IndexId,
     pub index_config_uri: Uri,
+    pub create: bool,
     pub assume_yes: bool,
 }
 
@@ -335,12 +339,14 @@ impl IndexCliCommand {
             .remove_one::<String>("index-config")
             .map(|uri| Uri::from_str(&uri))
             .expect("`index-config` should be a required arg")?;
+        let create = matches.get_flag("create");
         let assume_yes = matches.get_flag("yes");
 
         Ok(Self::Update(UpdateIndexArgs {
             index_id,
             client_args,
             index_config_uri,
+            create,
             assume_yes,
         }))
     }
@@ -548,7 +554,12 @@ pub async fn update_index_cli(args: UpdateIndexArgs) -> anyhow::Result<()> {
     }
     qw_client
         .indexes()
-        .update(&args.index_id, &index_config_str, config_format)
+        .update(
+            &args.index_id,
+            &index_config_str,
+            config_format,
+            args.create,
+        )
         .await?;
     println!("{} Index successfully updated.", "✔".color(GREEN_COLOR));
     Ok(())

quickwit/quickwit-cli/src/source.rs

@@ -57,6 +57,8 @@ pub fn build_source_command() -> Command {
                         .required(true),
                     arg!(--"source-config" <SOURCE_CONFIG> "Path to source config file. Please, refer to the documentation for more details.")
                         .required(true),
+                    arg!(--"create" "Create the index if it does not already exists.")
+                        .required(false),
                 ])
             )
         .subcommand(
@@ -162,6 +164,7 @@ pub struct UpdateSourceArgs {
     pub index_id: IndexId,
     pub source_id: SourceId,
     pub source_config_uri: Uri,
+    pub create: bool,
 }
 
 #[derive(Debug, Eq, PartialEq)]
@@ -276,11 +279,14 @@ impl SourceCliCommand {
             .remove_one::<String>("source-config")
             .map(|uri_str| Uri::from_str(&uri_str))
             .expect("`source-config` should be a required arg.")?;
+        let create = matches.get_flag("create");
+
         Ok(UpdateSourceArgs {
             client_args,
             index_id,
             source_id,
             source_config_uri,
+            create,
         })
     }
 
@@ -393,7 +399,12 @@ async fn update_source_cli(args: UpdateSourceArgs) -> anyhow::Result<()> {
     let qw_client = args.client_args.client();
     qw_client
         .sources(&args.index_id)
-        .update(&args.source_id, source_config_str, config_format)
+        .update(
+            &args.source_id,
+            source_config_str,
+            config_format,
+            args.create,
+        )
         .await?;
     println!("{} Source successfully updated.", "✔".color(GREEN_COLOR));
     Ok(())
@@ -683,6 +694,7 @@ mod tests {
                 index_id: "hdfs-logs".to_string(),
                 source_id: "kafka-foo".to_string(),
                 source_config_uri: Uri::from_str("file:///source-conf.yaml").unwrap(),
+                create: false,
             }));
         assert_eq!(command, expected_command);
     }

quickwit/quickwit-cli/tests/cli.rs

@@ -564,6 +564,7 @@ async fn test_cmd_update_index() {
         client_args: test_env.default_client_args(),
         index_id: index_id.clone(),
         index_config_uri: test_env.resource_files.index_config_with_retention.clone(),
+        create: false,
         assume_yes: true,
     };
     update_index_cli(args).await.unwrap();
@@ -582,6 +583,7 @@ async fn test_cmd_update_index() {
         client_args: test_env.default_client_args(),
         index_id,
         index_config_uri: test_env.resource_files.index_config.clone(),
+        create: false,
         assume_yes: true,
     };
     update_index_cli(args).await.unwrap();

quickwit/quickwit-integration-tests/src/tests/sqs_tests.rs

@@ -349,7 +349,7 @@ async fn test_update_source_multi_node_cluster() {
     sandbox
         .rest_client(QuickwitService::Metastore)
         .sources(index_id)
-        .update(source_id, source_config_input, ConfigFormat::Yaml)
+        .update(source_id, source_config_input, ConfigFormat::Yaml, false)
         .await
         .unwrap();