Metadata import: Column selection

app/controllers/concerns/file_import_actions.rb

@@ -34,6 +34,6 @@ def create # rubocop:disable Metrics/AbcSize
   private
 
   def file_import_params
-    params.require(:file_import).permit(:file, :sample_id_column, :ignore_empty_values)
+    params.require(:file_import).permit(:file, :sample_id_column, :ignore_empty_values, metadata_columns: [])
   end
 end

app/javascript/controllers/metadata/file_import_controller.js

@@ -2,11 +2,12 @@ import { Controller } from "@hotwired/stimulus";
 import * as XLSX from "xlsx";
 
 export default class extends Controller {
-  static targets = ["selectInput", "submitButton"];
+  static targets = ["sampleIdColumn", "metadataColumns", "submitButton"];
   static values = {
-    loaded: Boolean
+    loaded: Boolean,
   };
 
+  #headers = [];
   #disabled_classes = [
     "bg-slate-50",
     "border",
@@ -28,12 +29,26 @@ export default class extends Controller {
   ];
 
   connect() {
-    this.#disableSelectInput();
+    this.#disableTarget(this.sampleIdColumnTarget);
+    if (this.hasMetadataColumnsTarget) {
+      this.#disableTarget(this.metadataColumnsTarget);
+    }
     this.submitButtonTarget.disabled = true;
     this.loadedValue = true;
   }
 
-  toggleSubmitButton(event) {
+  changeSampleIDInput() {
+    if (this.hasMetadataColumnsTarget) {
+      this.#removeInputOptions(this.metadataColumnsTarget);
+      this.#addMetadataInputOptions();
+      this.#enableTarget(this.metadataColumnsTarget);
+      this.submitButtonTarget.disabled = true;
+    } else {
+      this.submitButtonTarget.disabled = false;
+    }
+  }
+
+  changeMetadataInput(event) {
     const { value } = event.target;
     this.submitButtonTarget.disabled = !value;
   }
@@ -42,7 +57,7 @@ export default class extends Controller {
     const { files } = event.target;
 
     if (!files.length) {
-      this.#removeSelectOptions();
+      this.#removeInputsOptions();
       return;
     }
 
@@ -53,33 +68,75 @@ export default class extends Controller {
       const workbook = XLSX.read(reader.result, { sheetRows: 1 });
       const worksheetName = workbook.SheetNames[0];
       const worksheet = workbook.Sheets[worksheetName];
-      const headers = XLSX.utils.sheet_to_json(worksheet, { header: 1 })[0];
-      this.#removeSelectOptions();
-      this.#addSelectOptions(headers);
+      this.#headers = XLSX.utils.sheet_to_json(worksheet, { header: 1 })[0];
+      this.#removeInputsOptions();
+      this.#addSampleIDInputOptions();
+      this.#enableTarget(this.sampleIdColumnTarget);
+      if (this.hasMetadataColumnsTarget) {
+        this.#disableTarget(this.metadataColumnsTarget);
+      }
     };
   }
 
-  #removeSelectOptions() {
-    while (this.selectInputTarget.options.length > 1) {
-      this.selectInputTarget.remove(this.selectInputTarget.options.length - 1);
+  #removeInputsOptions() {
+    this.#removeInputOptions(this.sampleIdColumnTarget);
+    this.#disableTarget(this.sampleIdColumnTarget);
+    if (this.hasMetadataColumnsTarget) {
+      this.#removeInputOptions(this.metadataColumnsTarget);
+      this.#disableTarget(this.metadataColumnsTarget);
     }
-    this.#disableSelectInput();
     this.submitButtonTarget.disabled = true;
   }
 
-  #addSelectOptions(headers) {
-    for (let header of headers) {
+  #addSampleIDInputOptions() {
+    for (let header of this.#headers) {
       const option = document.createElement("option");
       option.value = header;
       option.text = header;
-      this.selectInputTarget.append(option);
+      this.sampleIdColumnTarget.append(option);
     }
-    this.selectInputTarget.disabled = false;
-    this.selectInputTarget.classList.remove(...this.#disabled_classes);
   }
 
-  #disableSelectInput() {
-    this.selectInputTarget.disabled = true;
-    this.selectInputTarget.classList.add(...this.#disabled_classes);
+  #addMetadataInputOptions() {
+    const ignoreList = [
+      "sample id",
+      "sample name",
+      "project id",
+      "created_at",
+      "updated_at",
+      "last_updated_at",
+    ];
+
+    let columns = this.#headers.filter(
+      (header) =>
+        !ignoreList.includes(header.toLowerCase()) &&
+        header.toLowerCase() != this.sampleIdColumnTarget.value.toLowerCase(),
+    );
+
+    for (let column of columns) {
+      const option = document.createElement("option");
+      option.value = column;
+      option.text = column;
+      this.metadataColumnsTarget.append(option);
+    }
+  }
+
+  #removeInputOptions(target) {
+    for (let index = target.options.length - 1; index >= 0; index--) {
+      //do not remove the placeholder
+      if (target.options[index].value) {
+        target.remove(index);
+      }
+    }
+  }
+
+  #disableTarget(target) {
+    target.disabled = true;
+    target.classList.add(...this.#disabled_classes);
+  }
+
+  #enableTarget(target) {
+    target.disabled = false;
+    target.classList.remove(...this.#disabled_classes);
   }
 }

app/services/samples/metadata/file_import_service.rb

@@ -8,9 +8,9 @@ module Metadata
     class FileImportService < BaseSpreadsheetImportService
       def initialize(namespace, user = nil, blob_id = nil, params = {})
         @sample_id_column = params[:sample_id_column]
-        required_headers = [@sample_id_column]
+        @selected_headers = params[:metadata_columns] || []
         @ignore_empty_values = params[:ignore_empty_values]
-        super(namespace, user, blob_id, required_headers, 1, params)
+        super(namespace, user, blob_id, [@sample_id_column], 1, params)
       end
 
       def execute
@@ -24,9 +24,14 @@ def execute
 
       protected
 
-      def perform_file_import
+      def perform_file_import # rubocop:disable Metrics/MethodLength
         response = {}
-        parse_settings = @headers.zip(@headers).to_h
+        headers = if Flipper.enabled?(:metadata_import_field_selection)
+                    @selected_headers << @sample_id_column
+                  else
+                    @headers
+                  end
+        parse_settings = headers.zip(headers).to_h
         @spreadsheet.each_with_index(parse_settings) do |metadata, index|
           next unless index.positive?
 

app/views/shared/samples/metadata/file_imports/_dialog.html.erb

@@ -47,10 +47,26 @@
                         { prompt: t(".select_sample_id_column") },
                         required: true,
                         data: {
-                          "metadata--file-import-target": "selectInput",
-                          action: "change->metadata--file-import#toggleSubmitButton",
+                          "metadata--file-import-target": "sampleIdColumn",
+                          action: "change->metadata--file-import#changeSampleIDInput",
                         } %>
           </div>
+          <% if Flipper.enabled?(:metadata_import_field_selection) %>
+            <div class="form-field">
+              <%= form.label :metadata_columns,
+                         t(".metadata_columns"),
+                         class:
+                           "block mb-2 text-sm font-medium text-slate-900 dark:text-white" %>
+              <%= form.select :metadata_columns,
+                          {},
+                          { multiple: true, include_hidden: false },
+                          required: true,
+                          data: {
+                            "metadata--file-import-target": "metadataColumns",
+                            action: "change->metadata--file-import#changeMetadataInput",
+                          } %>
+            </div>
+          <% end %>
           <div class="flex items-center">
             <%= form.check_box :ignore_empty_values,
                            {

config/locales/en.yml

@@ -1819,8 +1819,9 @@ en:
             file_help: CSV, TSV, XLS or XLSX.
             ignore_empty_values:
               description: If selected, any metadata fields without an associated value will be ignored and those metadata keys will not be removed from the sample if present. However, if this not selected, any samples with the metadata key and empty value will be deleted.
+            metadata_columns: Metadata Columns
             namespace:
-              description: 'The spreadsheet is required to have a column that contains a sample identifier. '
+              description: The spreadsheet is required to have a column that contains a sample identifier.
               group:
                 description_html: The identifier is case-sensitive and must contain the <b>sample ID</b>.
               project:

config/locales/fr.yml

@@ -1819,8 +1819,9 @@ fr:
             file_help: CSV, TSV, XLS or XLSX.
             ignore_empty_values:
               description: If selected, any metadata fields without an associated value will be ignored and those metadata keys will not be removed from the sample if present. However, if this not selected, any samples with the metadata key and empty value will be deleted.
+            metadata_columns: Metadata Columns
             namespace:
-              description: 'The spreadsheet is required to have a column that contains a sample identifier. '
+              description: The spreadsheet is required to have a column that contains a sample identifier.
               group:
                 description_html: The identifier is case-sensitive and must contain the <b>sample ID</b>.
               project:

db/seeds.rb

@@ -5,6 +5,7 @@
 Faker::Config.locale = 'en'
 
 Flipper.enable(:workflow_execution_sharing)
+Flipper.enable(:metadata_import_field_selection)
 
 @namespace_group_link_expiry_date = (Time.zone.today + 14).strftime('%Y-%m-%d')
 

test/fixtures/files/metadata/duplicate_headers.csv

@@ -1,3 +1,3 @@
-sample_name,metadatafield1,metadatafield2,sample_name,metadatafield3
-Project 1 Sample 1,10,20,Project 1 Sample 1,30
-Project 1 Sample 2,15,25,Project 1 Sample 2,35
+sample_name,metadatafield1,metadatafield2,metadatafield3,metadatafield3
+Project 1 Sample 1,10,20,30,30
+Project 1 Sample 2,15,25,35,35

test/fixtures/files/metadata/invalid.txt

@@ -1 +1 @@
-invalid file extension
+invalid file extension,header