PI-2526 Separate chunks for date, type, outcome and description (#4601)

ministryofjustice · Jan 30, 2025 · a5c238b · a5c238b
1 parent 8f1cdb4
commit a5c238b
Showing 1 changed file with 12 additions and 21 deletions.
diff --git a/...erson-search-index-from-delius/container/pipelines/contact/index/ingest-pipeline.tpl.json b/...erson-search-index-from-delius/container/pipelines/contact/index/ingest-pipeline.tpl.json
@@ -1,26 +1,6 @@
 {
   "description": "Split text into chunks and generate embeddings",
   "processors": [
-    {
-      "append": {
-        "field": "text",
-        "value": [
-          "{{{date}}}",
-          "{{{startTime}}}",
-          "{{{typeDescription}}}",
-          "{{{typeShortDescription}}}",
-          "{{{outcomeDescription}}}",
-          "{{{description}}}",
-          "{{{notes}}}"
-        ]
-      }
-    },
-    {
-      "join": {
-        "field": "text",
-        "separator": "\n"
-      }
-    },
     {
       "text_chunking": {
         "algorithm": {
@@ -31,10 +11,21 @@
           }
         },
         "field_map": {
-          "text": "textChunks"
+          "notes": "textChunks"
         }
       }
     },
+    {
+      "append": {
+        "field": "textChunks",
+        "value": [
+          "{{{date}}} {{{startTime}}}",
+          "{{{typeDescription}}} {{{typeShortDescription}}}",
+          "{{{outcomeDescription}}}",
+          "{{{description}}}"
+        ]
+      }
+    },
     {
       "text_embedding": {
         "model_id": "${model_id}",