Merge pull request #513 from digital-preservation/DR2-1969-assisted-f…

…ile-path-substitution

DR2-1969 assisted file path substitution

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-13, windows-latest]
-        jdk: [8, 11, 17]
+        jdk: [11, 17, 21]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4

.gitignore

@@ -15,6 +15,7 @@ project/plugins/project/
 # IntelliJ specific
 .idea/*
 *.iml
+.bsp/
 
 # MacOS specific
 **/.DS_Store

README.md

@@ -20,9 +20,9 @@ The Validation tool and APIs are written in Scala 2.13 and may be used as:
 
 * A library in your Scala project.
 
-* A library in your Java project (We provide a Java 8 interface, to make things simple for Java programmers too).
+* A library in your Java project (We provide a Java 11 interface, to make things simple for Java programmers too).
 
-The Validation Tool and APIs can be used on any Java Virtual Machine which supports Java 8 or better (**NB Java 6 support was removed in version 1.1**). The source code is
+The Validation Tool and APIs can be used on any Java Virtual Machine which supports Java 11 or better (**NB Java 6 support was removed in version 1.1**). The source code is
 built using the [Apache Maven](https://maven.apache.org/) build tool:
 
 1. For use in other Java/Scala Applications, build by executing `mvn clean install`.

csv-validator-cmd/src/main/scala/uk/gov/nationalarchives/csv/validator/cmd/CsvValidatorCmdApp.scala

@@ -20,7 +20,7 @@ import java.nio.charset.Charset
 import java.nio.file.{Files, Path, Paths}
 import java.text.DecimalFormat
 import java.util.jar.{Attributes, Manifest}
-import scala.util.Using
+import scala.util.{Try, Using}
 
 object SystemExitCodes extends Enumeration {
   type ExitCode = Int
@@ -140,6 +140,15 @@ object CsvValidatorCmdApp extends App {
     case _ =>
   }
 
+  def getColumnFromCsv(csvFile: TextFile, csvSchemaFile: TextFile, columnName: String): List[String] = Try {
+    val validator = createValidator(true, Nil, false, false)
+    val csv = validator.loadCsvFile(csvFile, csvSchemaFile)
+    csv.headOption.map(_.indexOf("identifier")).map { identifierIdx =>
+      csv.tail.map(arr => arr(identifierIdx))
+    }.getOrElse(Nil)
+  }.getOrElse(Nil)
+
+
   def validate(
     csvFile: TextFile,
     schemaFile: TextFile,

csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidator.scala

@@ -104,13 +104,7 @@ trait MetaDataValidator {
     validateKnownRows(csv, schema, pf, rowCallback)
   }
 
-  def validateKnownRows(
-    csv: JReader,
-    schema: Schema,
-    progress: Option[ProgressFor],
-    rowCallback: MetaDataValidation[Any] => Unit
-  ): Boolean = {
-
+  def createCsvParser(schema: Schema): CsvParser = {
     val separator: Char = schema.globalDirectives.collectFirst {
       case Separator(sep) =>
         sep
@@ -135,8 +129,20 @@ trait MetaDataValidator {
     //format.setLineSeparator(CSV_RFC1480_LINE_SEPARATOR)  // CRLF
 
     //we need a better CSV Reader!
+    new CsvParser(settings)
+  }
+
+
+  def validateKnownRows(
+    csv: JReader,
+    schema: Schema,
+    progress: Option[ProgressFor],
+    rowCallback: MetaDataValidation[Any] => Unit
+  ): Boolean = {
+
+    val parser = createCsvParser(schema)
+
     val result : Try[Boolean] = Using {
-      val parser = new CsvParser(settings)
       parser.beginParsing(csv)
       parser
     } {

csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidator.scala

@@ -10,12 +10,15 @@ package uk.gov.nationalarchives.csv.validator.api
 
 import cats.data.{Chain, Validated, ValidatedNel}
 import cats.implicits._
+import com.univocity.parsers.csv.{CsvParser, CsvParserSettings}
 import uk.gov.nationalarchives.csv.validator._
-import uk.gov.nationalarchives.csv.validator.schema.{Schema, SchemaParser}
+import uk.gov.nationalarchives.csv.validator.schema.{Quoted, Schema, SchemaParser, Separator}
 
 import java.io.{Reader => JReader}
 import java.nio.charset.{Charset => JCharset}
 import java.nio.file.Path
+import scala.jdk.CollectionConverters._
+import scala.util.Try
 
 object CsvValidator {
 
@@ -71,7 +74,19 @@ trait CsvValidator extends SchemaParser {
       case Some(errors) => Validated.invalid(errors)
     }
   }
-
+
+
+
+  def loadCsvFile(csvFile: TextFile, csvSchemaFile: TextFile): List[Array[String]] = {
+    parseSchema(csvSchemaFile) match {
+      case Validated.Valid(schema) =>
+        withReader(csvFile) { reader =>
+          createCsvParser(schema).parseAll(reader)
+        }.asScala.toList  
+      case Validated.Invalid(_) => Nil
+    }
+  }
+
   def validateCsvFile(
     csvFile: TextFile,
     csvSchema: Schema,

csv-validator-ui/src/main/scala/uk/gov/nationalarchives/csv/validator/ui/CsvValidatorUi.scala

@@ -27,14 +27,17 @@ import java.nio.file.{Files, Path, Paths, StandardOpenOption}
 import java.util
 import java.util.Properties
 import java.util.jar.{Attributes, Manifest}
+import javax.swing.SpringLayout.Constraints
 import javax.swing._
 import javax.swing.filechooser.FileNameExtensionFilter
 import javax.swing.table.DefaultTableModel
 import scala.jdk.CollectionConverters.CollectionHasAsScala
 import scala.language.reflectiveCalls
+import scala.swing.FileChooser.SelectionMode
 import scala.swing.GridBagPanel.Anchor
 import scala.swing.PopupMenuImplicits._
 import scala.swing._
+import scala.swing.event.ButtonClicked
 import scala.util.{Failure, Success, Try, Using}
 
 /**
@@ -53,6 +56,9 @@ object CsvValidatorUi extends SimpleSwingApplication {
     super.startup(args)
   }
 
+  private lazy val txtCsvFile = new JTextField(30)
+  private lazy val txtCsvSchemaFile = new JTextField(30)
+
   def top: SJXFrame = new SJXFrame {
 
     title = "CSV Validator"
@@ -233,8 +239,6 @@ object CsvValidatorUi extends SimpleSwingApplication {
     peer.setTransferHandler(fileHandler)
     private val layout = new DesignGridLayout(peer)
 
-    private val txtCsvFile = new JTextField(30)
-
     private def showErrorDialog(message: String) = {
       JOptionPane.showMessageDialog(parentFrame.peer, message, "Error", JOptionPane.ERROR_MESSAGE)
       false
@@ -302,7 +306,7 @@ object CsvValidatorUi extends SimpleSwingApplication {
     }
 
     private val lblCsvSchemaFile = new Label("CSV Schema file:")
-    private val txtCsvSchemaFile = new JTextField(30)
+
     txtCsvSchemaFile.setTransferHandler(fileHandler)
     private val csvSchemaFileChooser = new FileChooser(loadSettings match {
       case Some(s) =>
@@ -480,6 +484,42 @@ object CsvValidatorUi extends SimpleSwingApplication {
     private val cbEnforceCaseSensitivePathChecks = new CheckBox("Enforce case-sensitive file path checks?")
     cbEnforceCaseSensitivePathChecks.tooltip = "Performs additional checks to ensure that the case of file-paths in the CSV file match those of the filesystem"
 
+    private def tablePathDialog(): Unit = {
+      val csvFile = TextFile(Paths.get(txtCsvFile.getText), csvEncoding, validateUtf8)
+      val schemaFile = TextFile(Paths.get(txtCsvSchemaFile.getText), csvSchemaEncoding)
+      val identifierRows = CsvValidatorCmdApp.getColumnFromCsv(csvFile, schemaFile, "identifier").sorted
+      val fromPath = identifierRows.headOption.getOrElse("")
+
+      val fileTextField = new TextField(30)
+      val fromPathText = new TextField(fromPath, 30)
+
+      def pathToUri(path: Path) = {
+        val uri = path.toUri.toString
+        if (uri.endsWith("/")) uri else s"$uri/"
+      }
+
+      def updateFileText(path: Path): Option[IOException] = {
+        fileTextField.text = pathToUri(path)
+        None
+      }
+
+      val okButton = new Button("OK")
+      val fileButton = new Button("...")
+      fileButton.reactions += {
+        case ev: ButtonClicked =>
+          val startingDir = if(fileTextField.text.isEmpty) userDir.toFile else Path.of(fileTextField.text).toFile
+          val fileChooser = new FileChooser(startingDir)
+          fileChooser.fileSelectionMode = SelectionMode.FilesAndDirectories
+          chooseFile(fileChooser, f => updateFileText(f), fileButton, s"Select the ${fromPath.split("/").last} folder")
+      }
+
+      val rows = List(
+        Row("From", List(fromPathText)),
+        Row("To", List(fileTextField, fileButton))
+      )
+      addToTableDialog(parentFrame, "Add path substitution...", rows, tblPathSubstitutions.addRow)
+    }
+
     private val tblPathSubstitutions = new Table(0, 2) {
       preferredViewportSize = new Dimension(500, 70)
       model = new DefaultTableModel(Array[Object]("From", "To"), 0)
@@ -505,7 +545,8 @@ object CsvValidatorUi extends SimpleSwingApplication {
 
     private val spTblPathSubstitutions = new ScrollPane(tblPathSubstitutions)
     private val btnAddPathSubstitution = new Button("Add Path Substitution...")
-    btnAddPathSubstitution.reactions += onClick(addToTableDialog(parentFrame, "Add Path Substitution...", tblPathSubstitutions, tblPathSubstitutions.addRow))
+
+    btnAddPathSubstitution.reactions += onClick(tablePathDialog())
 
     private val settingsGroupLayout = new GridBagPanel {
       private val c = new Constraints

csv-validator-ui/src/main/scala/uk/gov/nationalarchives/csv/validator/ui/ScalaSwingHelpers.scala

@@ -42,8 +42,8 @@ object ScalaSwingHelpers {
    * @param result A function which takes the chosen file
    * @param locateOver A component over which the FileChooser dialog should be located
    */
-  def chooseFile(fileChooser: FileChooser, result: Path => Option[IOException], locateOver: Component) : Unit = {
-    fileChooser.showSaveDialog(locateOver) match {
+  def chooseFile(fileChooser: FileChooser, result: Path => Option[IOException], locateOver: Component, dialogText: String = "Save") : Unit = {
+    fileChooser.showDialog(locateOver, dialogText) match {
       case Result.Approve =>
         result(fileChooser.selectedFile.toPath) match {
           case Some(ioe) =>
@@ -64,28 +64,33 @@ object ScalaSwingHelpers {
    * @param table The table to create a dialog for
    * @param result A function which takes a row as the result of the dialog box
    */
-  def addToTableDialog(owner: Window, title: String, table: Table, result: Array[String] => Unit) : Unit = {
+  case class Row(label: String, components: List[Component])
+  val c = List()
+  def addToTableDialog(owner: Window, title: String, rows: List[Row], result: Array[String] => Unit) : Unit = {
 
     val btnOk = new Button("Ok")
 
     val optionLayout: GridBagPanel = new GridBagPanel {
       val c = new Constraints
-
-      for(colIdx <- 0 to table.model.getColumnCount - 1) {
-        c.gridx = 0
-        c.gridy = colIdx
-        c.anchor = Anchor.LineStart
-        layout(new Label(table.model.getColumnName(colIdx) + ":")) = c
-
-        c.gridx = 1
-        c.gridy = colIdx
-        c.anchor = Anchor.LineStart
-        layout(new TextField(30)) = c
+      rows.zipWithIndex.map {
+        case (row, colIdx) =>
+          c.gridx = 0
+          c.gridy = colIdx
+          c.anchor = Anchor.LineStart
+          layout(new Label(row.label + ":")) = c
+
+          row.components.zipWithIndex.map {
+            case (component, rowIdx) =>
+              c.gridx = rowIdx + 1
+              c.gridy = colIdx
+              c.anchor = Anchor.LineStart
+              layout(component) = c
+          }
       }
 
       c.gridx = 0
-      c.gridy = table.model.getColumnCount
-      c.gridwidth = 2
+      c.gridy = rows.size
+      c.gridwidth = rows.size + 1
       c.anchor = Anchor.LineEnd
       layout(btnOk) = c
     }