From ce41fca883c596f63aaf4094b6a3102d1db3d2f8 Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Mon, 3 Feb 2025 20:52:09 -0600 Subject: [PATCH 1/5] Add ExecuteCommandImportDatabase WIP --- CHANGELOG.md | 2 + .../ExecuteCommandImportDatabase.cs | 298 ++++++++++++++++++ 2 files changed, 300 insertions(+) create mode 100644 Rdmp.Dicom/CommandExecution/ExecuteCommandImportDatabase.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bf2b281..c13a97c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- Add new command ImportDatabase to generate RDMP metadata for an existing SMI-type DB + ## [7.1.3] 2024-12-02 - Bump RDMP from 8.3.1 to 8.4.0 diff --git a/Rdmp.Dicom/CommandExecution/ExecuteCommandImportDatabase.cs b/Rdmp.Dicom/CommandExecution/ExecuteCommandImportDatabase.cs new file mode 100644 index 00000000..322c31be --- /dev/null +++ b/Rdmp.Dicom/CommandExecution/ExecuteCommandImportDatabase.cs @@ -0,0 +1,298 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using Rdmp.Core.ReusableLibraryCode.Checks; +using FAnsi.Discovery; +using Rdmp.Dicom.Attachers.Routing; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Repositories; +using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.Data.Pipelines; +using Rdmp.Core.DataLoad.Modules.Mutilators; +using Rdmp.Core.Curation.Data.Defaults; +using Rdmp.Core.Curation; +using Rdmp.Core.DataLoad.Engine.Checks; +using Rdmp.Core.DataLoad; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.Repositories.Construction; +using Rdmp.Dicom.PipelineComponents.DicomSources; +using Rdmp.Core.ReusableLibraryCode.Annotations; +using Rdmp.Core.DataExport.Data; +using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming; +using Rdmp.Core.DataLoad.Engine.LoadProcess; + +namespace Rdmp.Dicom.CommandExecution; + +public partial class ExecuteCommandImportDatabase : BasicCommandExecution +{ + private readonly DiscoveredDatabase _databaseToCreateInto; + private readonly DirectoryInfo _projectDirectory; + private readonly IExternalDatabaseServer _loggingServer; + private readonly IRDMPPlatformRepositoryServiceLocator _repositoryLocator; + private readonly ICatalogueRepository _catalogueRepository; + + private List NewCataloguesCreated { get; } + private LoadMetadata NewLoadMetadata { get; set; } + + /// + /// The component of the data load that will handle reading the Dicom files / json and converting it into DataTables (only populated after Execute has been called). + /// Note that this is a PipelineComponent meaning it is the template which gets stamped out into a hydrated instance at runtime. The DicomSourcePipelineComponent.Path Should + /// contain the DicomSourceType.Name and when the DLE is run the DicomSourceType is the Type that will be created from the template + /// + private PipelineComponent DicomSourcePipelineComponent { get; set; } + + /// + /// The DicomSource component Type to use for the Loadmetadata pipeline responsible for loading the dicom metadata into the Catalogues (e.g. DicomDatasetCollectionSource + /// for Json or DicomFileCollectionSource for files) + /// + private Type DicomSourceType { get; set; } + + /// + /// True to create an AutoRoutingAttacherWithPersistentRaw instead of a AutoRoutingAttacher + /// + private bool PersistentRaw { get; set; } + + public ExecuteCommandImportDatabase(IRDMPPlatformRepositoryServiceLocator repositoryLocator, DiscoveredDatabase databaseToCreateInto, DirectoryInfo projectDirectory) + { + _repositoryLocator = repositoryLocator; + _catalogueRepository = repositoryLocator.CatalogueRepository; + _databaseToCreateInto = databaseToCreateInto; + _projectDirectory = projectDirectory; + NewCataloguesCreated = new List(); + + _loggingServer = _catalogueRepository.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID); + + if (_loggingServer == null) + SetImpossible("No default logging server has been configured in your Catalogue database"); + } + + [UsedImplicitly] + [UseWithObjectConstructor] + public ExecuteCommandImportDatabase( + IRDMPPlatformRepositoryServiceLocator repositoryLocator, + DiscoveredDatabase databaseToCreateInto, + DirectoryInfo projectDirectory, + [DemandsInitialization("The pipeline source for reading dicom tags from e.g. from files or from serialized JSON",TypeOf = typeof(DicomSource))] + Type dicomSourceType, + bool persistentRaw + ) : this(repositoryLocator, databaseToCreateInto, projectDirectory) + { + DicomSourceType = dicomSourceType ?? typeof(DicomFileCollectionSource); + PersistentRaw = persistentRaw; + } + + public override void Execute() + { + if (DicomSourceType == null) + { + SetImpossible("You must specify a Type for DicomSourceType"); + throw new ImpossibleCommandException(this, ReasonCommandImpossible); + } + + base.Execute(); + + // create the database if it does not exist + if (!_databaseToCreateInto.Exists() || !_databaseToCreateInto.Server.Exists()) + throw new Exception($"Database '{_databaseToCreateInto.GetRuntimeName()}' did not exist"); + + List tables = []; + foreach (var t in _databaseToCreateInto.DiscoverTables(false)) + { + var tableBits = TableNameRegex().Match(t.GetRuntimeName()); + if (!tableBits.Success) + continue; + + tables.Add(t); + + var importer = new TableInfoImporter(_repositoryLocator.CatalogueRepository, t); + importer.DoImport(out var tis, out var cis); + + // Mark table as primary extraction table for this modality if it is the Study table, or the only SR/OTHER table + if (tableBits.Groups[0].Value.Equals("SR", StringComparison.Ordinal) + || tableBits.Groups[0].Value.Equals("OTHER", StringComparison.Ordinal) + || tableBits.Groups[1].Value.Equals("Study", StringComparison.Ordinal)) + tis.IsPrimaryExtractionTable = true; + + // TODO: Create JoinInfo for Image<->Series<->Study tables + // - newobject joininfo columninfo:*${modality}_*ImageTable\`*Series*UID* columninfo:*${modality}_*SeriesTable\`*Series*UID* right null + // - newobject joininfo columninfo:*${modality}_*SeriesTable\`*Study*UID* columninfo:*${modality}_*StudyTable\`*Study*UID* right null + + var engineer = new ForwardEngineerCatalogue(tis, cis); + engineer.ExecuteForwardEngineering(out var cata, out _, out var eis); + var patientIdentifier = eis.SingleOrDefault(static e => e.GetRuntimeName()?.Equals("PatientID") == true); + + if (patientIdentifier != null) + { + patientIdentifier.IsExtractionIdentifier = true; + patientIdentifier.SaveToDatabase(); + } + + var seriesEi = eis.SingleOrDefault(static e => e.GetRuntimeName()?.Equals("SeriesInstanceUID") == true); + if (seriesEi != null) + { + seriesEi.IsExtractionIdentifier = true; + seriesEi.SaveToDatabase(); + } + + //make it extractable + _ = new ExtractableDataSet(_repositoryLocator.DataExportRepository, cata); + + NewCataloguesCreated.Add(cata); + } + + const string loadName = "SMI Image Loading"; + + NewLoadMetadata = new LoadMetadata(_catalogueRepository, loadName); + + //tell all the catalogues that they are part of this load and where to log under the same task + foreach (var c in NewCataloguesCreated) + { + NewLoadMetadata.LinkToCatalogue(c); + c.LoggingDataTask = loadName; + c.LiveLoggingServer_ID = _loggingServer.ID; + c.SaveToDatabase(); + } + + //create the logging task + new Core.Logging.LogManager(_loggingServer).CreateNewLoggingTaskIfNotExists(loadName); + + var projDir = LoadDirectory.CreateDirectoryStructure(_projectDirectory, "ImageLoading", true); + NewLoadMetadata.LocationOfForLoadingDirectory = projDir.ForLoading.FullName; + NewLoadMetadata.LocationOfForArchivingDirectory = projDir.ForArchiving.FullName; + NewLoadMetadata.LocationOfExecutablesDirectory = projDir.ExecutablesPath.FullName; + NewLoadMetadata.LocationOfCacheDirectory = projDir.Cache.FullName; + NewLoadMetadata.SaveToDatabase(); + + /////////////////////////////////////////////Attacher//////////////////////////// + + + //Create a pipeline for reading from Dicom files and writing to any destination component (which must be fixed) + var name = "Image Loading Pipe"; + name = MakeUniqueName(_catalogueRepository.GetAllObjects().Select(static p => p.Name).ToArray(), name); + + var pipe = new Pipeline(_catalogueRepository, name); + DicomSourcePipelineComponent = new PipelineComponent(_catalogueRepository, pipe, DicomSourceType, 0, DicomSourceType.Name); + DicomSourcePipelineComponent.CreateArgumentsForClassIfNotExists(DicomSourceType); + + // Set the argument for only populating tags who appear in the end tables of the load (no need for source to read all the tags only those we are actually loading) + var arg = DicomSourcePipelineComponent.GetAllArguments().FirstOrDefault(static a => a.Name.Equals(nameof(DicomSource.UseAllTableInfoInLoadAsFieldMap))); + if (arg != null) + { + arg.SetValue(NewLoadMetadata); + arg.SaveToDatabase(); + } + + pipe.SourcePipelineComponent_ID = DicomSourcePipelineComponent.ID; + pipe.SaveToDatabase(); + + + //Create the load process task that uses the pipe to load RAW tables with data from the dicom files + var pt = new ProcessTask(_catalogueRepository, NewLoadMetadata, LoadStage.Mounting) + { + Name = "Auto Routing Attacher", + ProcessTaskType = ProcessTaskType.Attacher, + Path = PersistentRaw + ? typeof(AutoRoutingAttacherWithPersistentRaw).FullName + : typeof(AutoRoutingAttacher).FullName, + Order = 1 + }; + + + pt.SaveToDatabase(); + + var args = PersistentRaw ? pt.CreateArgumentsForClassIfNotExists() : pt.CreateArgumentsForClassIfNotExists(); + SetArgument(args, "LoadPipeline", pipe); + + /////////////////////////////////////// Distinct tables on load ///////////////////////// + + + var distincter = new ProcessTask(_catalogueRepository, NewLoadMetadata, LoadStage.AdjustRaw); + var distincterArgs = distincter.CreateArgumentsForClassIfNotExists(); + + distincter.Name = "Distincter"; + distincter.ProcessTaskType = ProcessTaskType.MutilateDataTable; + distincter.Path = typeof(Distincter).FullName; + distincter.Order = 2; + distincter.SaveToDatabase(); + SetArgument(distincterArgs, "TableRegexPattern", ".*"); + + ///////////////////////////////////////////////////////////////////////////////////// + + if (true) + { + var coalescer = new ProcessTask(_catalogueRepository, NewLoadMetadata, LoadStage.AdjustRaw) + { + Name = "Coalescer", + ProcessTaskType = ProcessTaskType.MutilateDataTable, + Path = typeof(Coalescer).FullName, + Order = 3 + }; + coalescer.SaveToDatabase(); + + var regexPattern = tables + .Where(static tbl => !tbl.DiscoverColumns().Any(static c => + c.GetRuntimeName().Equals("SOPInstanceUID", StringComparison.CurrentCultureIgnoreCase))) + .Select(static tbl => $"({tbl.GetRuntimeName()})"); + + + var coalArgs = coalescer.CreateArgumentsForClassIfNotExists(); + SetArgument(coalArgs, "TableRegexPattern", string.Join('|', regexPattern)); + SetArgument(coalArgs, "CreateIndex", true); + } + + ////////////////////////////////Load Ender (if no rows in load) //////////////////////////// + + var prematureLoadEnder = new ProcessTask(_catalogueRepository, NewLoadMetadata, LoadStage.Mounting) + { + Name = "Premature Load Ender", + ProcessTaskType = ProcessTaskType.MutilateDataTable, + Path = typeof(PrematureLoadEnder).FullName, + Order = 4 + }; + prematureLoadEnder.SaveToDatabase(); + + args = prematureLoadEnder.CreateArgumentsForClassIfNotExists(); + SetArgument(args, "ExitCodeToReturnIfConditionMet", ExitCodeType.OperationNotRequired); + SetArgument(args, "ConditionsToTerminateUnder", PrematureLoadEndCondition.NoRecordsInAnyTablesInDatabase); + + //////////////////////////////////////////////////////////////////////////////////////////////// + + var checker = new CheckEntireDataLoadProcess(BasicActivator, NewLoadMetadata, new HICDatabaseConfiguration(NewLoadMetadata), new HICLoadConfigurationFlags()); + checker.Check(new AcceptAllCheckNotifier()); + } + + private static string MakeUniqueName(string[] existingUsedNames, string candidate) + { + // if name is unique then keep candidate name + if (!existingUsedNames.Any(p => p.Equals(candidate, StringComparison.CurrentCultureIgnoreCase))) + return candidate; + + // otherwise give it a suffix + var suffix = 2; + while (existingUsedNames.Any(p => p.Equals(candidate + suffix, StringComparison.CurrentCultureIgnoreCase))) + { + suffix++; + } + + return candidate + suffix; + } + + private static void SetArgument(IArgument[] args, string property, object value) + { + ArgumentNullException.ThrowIfNull(value); + + var arg = args.Single(a => a.Name.Equals(property)); + + if (MEF.GetType(value.GetType().FullName) == null) + throw new ArgumentException($"No type found for {value.GetType().FullName}"); + + //if this fails, look to see if GetType returned null (indicates that your Type is not loaded by MEF). Look at mef.DescribeBadAssembliesIfAny() to investigate this issue + arg.SetValue(value); + arg.SaveToDatabase(); + } + + [GeneratedRegex("^([A-Z]+)_(Image|Series|Study)Table$")] + private static partial Regex TableNameRegex(); +} From 82fa811743d6b9c35da724da30d31072c9a1dfc0 Mon Sep 17 00:00:00 2001 From: James A Sutherland <> Date: Mon, 3 Feb 2025 21:08:29 -0600 Subject: [PATCH 2/5] Get .Net framework from global.json, autoupdate --- .github/dependabot.yml | 6 ++++++ .github/workflows/codeql.yml | 1 + .github/workflows/dotnet-core.yml | 2 -- Rdmp.Dicom.Tests/Rdmp.Dicom.Tests.csproj | 2 +- Rdmp.Dicom.UI/Rdmp.Dicom.UI.csproj | 2 +- Rdmp.Dicom/Rdmp.Dicom.csproj | 2 +- 6 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1cd891f3..91e2d8e6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,3 +13,9 @@ updates: open-pull-requests-limit: 99 reviewers: - SMI/reviewers +- package-ecosystem: "dotnet-sdk" + directory: "/" + schedule: + interval: weekly + reviewers: + - SMI/reviewers diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 97cfbcf1..c8cf6607 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -31,6 +31,7 @@ jobs: repository: HicServices/RDMP ref: develop path: RDMP + - uses: actions/setup-dotnet@v4 - name: Initialize CodeQL uses: github/codeql-action/init@v3 diff --git a/.github/workflows/dotnet-core.yml b/.github/workflows/dotnet-core.yml index 2f679e10..3e2a0380 100644 --- a/.github/workflows/dotnet-core.yml +++ b/.github/workflows/dotnet-core.yml @@ -20,8 +20,6 @@ jobs: ref: develop path: RDMP - uses: actions/setup-dotnet@v4 - with: - dotnet-version: 8.0.x - name: Get version id: version shell: cmd diff --git a/Rdmp.Dicom.Tests/Rdmp.Dicom.Tests.csproj b/Rdmp.Dicom.Tests/Rdmp.Dicom.Tests.csproj index b835a664..4b4ec9ab 100644 --- a/Rdmp.Dicom.Tests/Rdmp.Dicom.Tests.csproj +++ b/Rdmp.Dicom.Tests/Rdmp.Dicom.Tests.csproj @@ -2,7 +2,7 @@ Rdmp.Dicom.Tests Rdmp.Dicom.Tests - net8.0 + net$(NETCoreAppMaximumVersion) false diff --git a/Rdmp.Dicom.UI/Rdmp.Dicom.UI.csproj b/Rdmp.Dicom.UI/Rdmp.Dicom.UI.csproj index 95fba54d..b234fcec 100644 --- a/Rdmp.Dicom.UI/Rdmp.Dicom.UI.csproj +++ b/Rdmp.Dicom.UI/Rdmp.Dicom.UI.csproj @@ -2,7 +2,7 @@ Rdmp.Dicom.UI Rdmp.Dicom.UI - net8.0-windows + net$(NETCoreAppMaximumVersion)-windows false true true diff --git a/Rdmp.Dicom/Rdmp.Dicom.csproj b/Rdmp.Dicom/Rdmp.Dicom.csproj index 43a41fb0..9a40f142 100644 --- a/Rdmp.Dicom/Rdmp.Dicom.csproj +++ b/Rdmp.Dicom/Rdmp.Dicom.csproj @@ -3,7 +3,7 @@ HIC.Rdmp.Dicom Rdmp.Dicom Rdmp.Dicom - net8.0 + net$(NETCoreAppMaximumVersion) false From ebf2b9c4b07d43f725987e5af675ae37e86b4335 Mon Sep 17 00:00:00 2001 From: James A Sutherland <> Date: Mon, 3 Feb 2025 21:12:14 -0600 Subject: [PATCH 3/5] Add missed global.json file --- global.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 global.json diff --git a/global.json b/global.json new file mode 100644 index 00000000..a4effd96 --- /dev/null +++ b/global.json @@ -0,0 +1,5 @@ +{ + "sdk": { + "version": "9.0.102" + } +} \ No newline at end of file From ed7e5d08462d8bae5ccd059d42b3fb4d9d0c316d Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Mon, 3 Feb 2025 21:17:13 -0600 Subject: [PATCH 4/5] Update dependencies --- Rdmp.Dicom.Tests/Rdmp.Dicom.Tests.csproj | 7 +++---- Rdmp.Dicom/Rdmp.Dicom.csproj | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Rdmp.Dicom.Tests/Rdmp.Dicom.Tests.csproj b/Rdmp.Dicom.Tests/Rdmp.Dicom.Tests.csproj index 4b4ec9ab..9929e814 100644 --- a/Rdmp.Dicom.Tests/Rdmp.Dicom.Tests.csproj +++ b/Rdmp.Dicom.Tests/Rdmp.Dicom.Tests.csproj @@ -42,15 +42,14 @@ - + - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/Rdmp.Dicom/Rdmp.Dicom.csproj b/Rdmp.Dicom/Rdmp.Dicom.csproj index 9a40f142..b54da94d 100644 --- a/Rdmp.Dicom/Rdmp.Dicom.csproj +++ b/Rdmp.Dicom/Rdmp.Dicom.csproj @@ -31,7 +31,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + From 900c0739801d7cbb9ed88a98f0f8bca3e8bc2a18 Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Mon, 3 Feb 2025 21:25:18 -0600 Subject: [PATCH 5/5] Fix WFO1000 --- Rdmp.Dicom.UI/TagColumnAdderUI.cs | 4 ++++ Rdmp.Dicom.UI/TagElevationXmlUI.cs | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Rdmp.Dicom.UI/TagColumnAdderUI.cs b/Rdmp.Dicom.UI/TagColumnAdderUI.cs index ca7755f9..b20af3da 100644 --- a/Rdmp.Dicom.UI/TagColumnAdderUI.cs +++ b/Rdmp.Dicom.UI/TagColumnAdderUI.cs @@ -1,4 +1,5 @@ using System; +using System.ComponentModel; using System.Windows.Forms; using Rdmp.Core.Curation.Data; using Rdmp.Dicom.TagPromotionSchema; @@ -37,7 +38,10 @@ private void cbxTag_SelectedIndexChanged(object sender, EventArgs e) } } + [DesignerSerializationVisibility(DesignerSerializationVisibility.Hidden)] public string ColumnName { get; private set; } + + [DesignerSerializationVisibility(DesignerSerializationVisibility.Hidden)] public string ColumnDataType { get; private set; } private void btnOk_Click(object sender, EventArgs e) diff --git a/Rdmp.Dicom.UI/TagElevationXmlUI.cs b/Rdmp.Dicom.UI/TagElevationXmlUI.cs index 8c323c53..6135ef97 100644 --- a/Rdmp.Dicom.UI/TagElevationXmlUI.cs +++ b/Rdmp.Dicom.UI/TagElevationXmlUI.cs @@ -5,6 +5,7 @@ using Rdmp.Core.ReusableLibraryCode.Checks; using ScintillaNET; using System; +using System.ComponentModel; using System.Runtime.Versioning; using System.Windows.Forms; using Rdmp.UI.ScintillaHelper; @@ -71,7 +72,8 @@ private void RunChecks() } } - public ICatalogueRepository CatalogueRepository { get;set; } + [DesignerSerializationVisibility(DesignerSerializationVisibility.Hidden)] + public ICatalogueRepository CatalogueRepository { get; set; } public ICustomUIDrivenClass GetFinalStateOfUnderlyingObject() {