From 1a1fb4127b5c5210850d0d9c3760ac0e4ed6f150 Mon Sep 17 00:00:00 2001 From: Andre Masella Date: Tue, 6 Dec 2022 21:40:10 -0500 Subject: [PATCH] Redesign workflow engine and provisioner API This replaces the call-back based `WorkMonitor` with the fluent monad `OperationAction` API. This change will prevent recovery of any inflight operations. --- changes/change_operation_api.md | 1 + plugin-guide.md | 107 ++-- .../gsi/vidarr/cli/SingleShotOperation.java | 4 +- .../gsi/vidarr/cli/TargetConfiguration.java | 51 +- .../oicr/gsi/vidarr/core/ActiveWorkflow.java | 1 + .../oicr/gsi/vidarr/core/BaseProcessor.java | 401 +++++++-------- .../gsi/vidarr/core/DelayWorkMonitor.java | 87 ---- .../core/InputProvisioningStateExternal.java | 6 + .../core/InputProvisioningStateInternal.java | 5 + .../on/oicr/gsi/vidarr/core/JsonMutation.java | 21 +- .../oicr/gsi/vidarr/core/MonitorWithType.java | 64 --- .../vidarr/core/OneOfInputProvisioner.java | 63 +-- .../vidarr/core/OneOfOutputProvisioner.java | 55 ++- .../gsi/vidarr/core/OutputProvisionState.java | 14 + .../vidarr/core/PrepareInputProvisioning.java | 122 ++++- .../core/PrepareOutputProvisioning.java | 67 ++- .../vidarr/core/PreparePreflightChecks.java | 5 +- .../oicr/gsi/vidarr/core/ProvisionData.java | 33 +- .../gsi/vidarr/core/RawInputProvisioner.java | 33 +- .../oicr/gsi/vidarr/core/RawInputState.java | 5 + .../on/oicr/gsi/vidarr/core/RecoveryType.java | 35 +- .../vidarr/core/RuntimeProvisionState.java | 9 + .../ca/on/oicr/gsi/vidarr/core/Target.java | 8 +- .../on/oicr/gsi/vidarr/core/TaskStarter.java | 187 ++++++- .../oicr/gsi/vidarr/core/WrappedMonitor.java | 129 ----- .../gsi/vidarr/cromwell/CleanupState.java | 5 + .../vidarr/cromwell/CromwellMetadataURL.java | 2 +- .../cromwell/CromwellOutputProvisioner.java | 418 ++++------------ .../cromwell/CromwellWorkflowEngine.java | 455 ++++-------------- .../oicr/gsi/vidarr/cromwell/EngineState.java | 73 --- .../gsi/vidarr/cromwell/PreflightState.java | 3 + .../gsi/vidarr/cromwell/ProvisionState.java | 100 ++-- .../gsi/vidarr/cromwell/StateStarted.java | 32 ++ .../gsi/vidarr/cromwell/StateUnstarted.java | 106 ++++ .../cromwell/WorkflowMetadataResponse.java | 15 +- .../cromwell/WorkflowStatusResponse.java | 4 + vidarr-pluginapi/pom.xml | 4 + .../on/oicr/gsi/vidarr}/ActiveOperation.java | 42 +- .../gsi/vidarr/BaseJsonInputProvisioner.java | 133 ----- .../gsi/vidarr/BaseJsonOutputProvisioner.java | 171 ------- .../vidarr/BaseJsonRuntimeProvisioner.java | 125 ----- .../gsi/vidarr/BaseJsonWorkflowEngine.java | 191 -------- .../on/oicr/gsi/vidarr/InputProvisioner.java | 35 +- .../gsi/vidarr/InputProvisionerProvider.java | 2 +- .../on/oicr/gsi/vidarr/OperationAction.java | 362 ++++++++++++++ .../gsi/vidarr/OperationActionBranch.java | 98 ++++ .../gsi/vidarr/OperationActionConstant.java | 53 ++ .../vidarr/OperationActionDoStatefulStep.java | 51 ++ .../gsi/vidarr/OperationActionDoStep.java | 78 +++ .../oicr/gsi/vidarr/OperationActionLoad.java | 64 +++ .../gsi/vidarr/OperationActionReload.java | 83 ++++ .../oicr/gsi/vidarr/OperationControlFlow.java | 43 ++ .../gsi/vidarr/OperationStatefulStep.java | 314 ++++++++++++ .../OperationStatefulStepDebugInfo.java | 82 ++++ .../gsi/vidarr/OperationStatefulStepLog.java | 83 ++++ .../vidarr/OperationStatefulStepMapping.java | 79 +++ .../gsi/vidarr/OperationStatefulStepPoll.java | 97 ++++ ...erationStatefulStepRepeatUntilSuccess.java | 91 ++++ .../vidarr/OperationStatefulStepRequire.java | 87 ++++ .../vidarr/OperationStatefulStepStatus.java | 81 ++++ .../vidarr/OperationStatefulStepSubStep.java | 146 ++++++ .../on/oicr/gsi/vidarr}/OperationStatus.java | 6 +- .../ca/on/oicr/gsi/vidarr/OperationStep.java | 295 ++++++++++++ .../OperationStepCompletableFuture.java | 28 ++ .../gsi/vidarr/OperationStepDebugInfo.java | 30 ++ .../on/oicr/gsi/vidarr/OperationStepLog.java | 31 ++ .../oicr/gsi/vidarr/OperationStepMapping.java | 28 ++ .../oicr/gsi/vidarr/OperationStepMonitor.java | 35 ++ .../oicr/gsi/vidarr/OperationStepRequire.java | 36 ++ .../oicr/gsi/vidarr/OperationStepStatus.java | 29 ++ .../on/oicr/gsi/vidarr/OperationStepThen.java | 49 ++ .../on/oicr/gsi/vidarr/OutputProvisioner.java | 54 +-- .../gsi/vidarr/OutputProvisionerProvider.java | 2 +- .../ca/on/oicr/gsi/vidarr/PollResult.java | 65 +++ .../on/oicr/gsi/vidarr/PollResultActive.java | 16 + .../on/oicr/gsi/vidarr/PollResultFailed.java | 16 + .../oicr/gsi/vidarr/PollResultFinished.java | 9 + .../ca/on/oicr/gsi/vidarr/ProcessInput.java | 14 + .../ca/on/oicr/gsi/vidarr/ProcessOutput.java | 20 + .../oicr/gsi/vidarr/ProcessOutputAsJson.java | 34 ++ .../oicr/gsi/vidarr/ProcessOutputHandler.java | 69 +++ .../oicr/gsi/vidarr/ProcessOutputToFile.java | 15 + .../vidarr/ProcessOutputToStandardOutput.java | 12 + .../oicr/gsi/vidarr/RuntimeProvisioner.java | 41 +- .../vidarr/RuntimeProvisionerProvider.java | 2 +- .../ca/on/oicr/gsi/vidarr/WorkMonitor.java | 137 ------ .../ca/on/oicr/gsi/vidarr/WorkflowEngine.java | 122 ++--- .../gsi/vidarr/WorkflowEngineProvider.java | 2 +- .../ca/on/oicr/gsi/vidarr/WorkingStatus.java | 19 + .../src/main/java/module-info.java | 1 + vidarr-server/pom.xml | 2 +- .../server/DatabaseBackedProcessor.java | 4 +- .../gsi/vidarr/server/DatabaseOperation.java | 4 +- .../ca/on/oicr/gsi/vidarr/server/Main.java | 55 ++- .../server/dto/ServerConfiguration.java | 24 +- .../db/migration/V0019__delete_recovery.sql | 6 + .../on/oicr/gsi/vidarr/sh/CleanupState.java | 3 + .../ca/on/oicr/gsi/vidarr/sh/ShellState.java | 23 - .../on/oicr/gsi/vidarr/sh/StateInitial.java | 6 + .../vidarr/sh/UnixShellWorkflowEngine.java | 172 ++----- 100 files changed, 4126 insertions(+), 2711 deletions(-) create mode 100644 changes/change_operation_api.md delete mode 100644 vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/DelayWorkMonitor.java create mode 100644 vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/InputProvisioningStateExternal.java create mode 100644 vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/InputProvisioningStateInternal.java delete mode 100644 vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/MonitorWithType.java create mode 100644 vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/OutputProvisionState.java create mode 100644 vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/RawInputState.java create mode 100644 vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/RuntimeProvisionState.java delete mode 100644 vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/WrappedMonitor.java create mode 100644 vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/CleanupState.java delete mode 100644 vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/EngineState.java create mode 100644 vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/PreflightState.java create mode 100644 vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/StateStarted.java create mode 100644 vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/StateUnstarted.java rename {vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core => vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr}/ActiveOperation.java (58%) delete mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/BaseJsonInputProvisioner.java delete mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/BaseJsonOutputProvisioner.java delete mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/BaseJsonRuntimeProvisioner.java delete mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/BaseJsonWorkflowEngine.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationAction.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionBranch.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionConstant.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionDoStatefulStep.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionDoStep.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionLoad.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionReload.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationControlFlow.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStep.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepDebugInfo.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepLog.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepMapping.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepPoll.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepRepeatUntilSuccess.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepRequire.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepStatus.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepSubStep.java rename {vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core => vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr}/OperationStatus.java (75%) create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStep.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepCompletableFuture.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepDebugInfo.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepLog.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepMapping.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepMonitor.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepRequire.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepStatus.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepThen.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/PollResult.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/PollResultActive.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/PollResultFailed.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/PollResultFinished.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessInput.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutput.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutputAsJson.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutputHandler.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutputToFile.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutputToStandardOutput.java delete mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/WorkMonitor.java create mode 100644 vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/WorkingStatus.java create mode 100644 vidarr-server/src/main/resources/db/migration/V0019__delete_recovery.sql create mode 100644 vidarr-sh/src/main/java/ca/on/oicr/gsi/vidarr/sh/CleanupState.java delete mode 100644 vidarr-sh/src/main/java/ca/on/oicr/gsi/vidarr/sh/ShellState.java create mode 100644 vidarr-sh/src/main/java/ca/on/oicr/gsi/vidarr/sh/StateInitial.java diff --git a/changes/change_operation_api.md b/changes/change_operation_api.md new file mode 100644 index 00000000..5e272017 --- /dev/null +++ b/changes/change_operation_api.md @@ -0,0 +1 @@ +* Redesign `WorkflowEngine` and `Provisioner` APIs to be fluent rather than callbacks diff --git a/plugin-guide.md b/plugin-guide.md index c6c6e298..db3a085b 100644 --- a/plugin-guide.md +++ b/plugin-guide.md @@ -9,7 +9,7 @@ using the `provides` keyword. All plugins need to depend only on the infrastructure. There are several services that a plugin can provide and a plugin is free to -provide multiple. Plugins are loaded from JSON data in the Vidarr configuration +provide multiple. Plugins are loaded from JSON data in the Víðarr configuration file or, in the case of an unload filter, user requests, using Jackson. Each plugin can load whatever Jackson-compatible data from JSON it requires. Each plugin has a small "provider" class which provides type information for @@ -18,7 +18,7 @@ appropriate class instance. The provider class lists what values for `"type"` correspond to what Java objects that Jackson should load. Since objects are instantiated by Jackson, most have a `startup` method that is called after loading is complete where the plugin can do any initialisation required. If it -throws exceptions, the Vidarr server will fail to start, which is probably the +throws exceptions, the Víðarr server will fail to start, which is probably the correct behaviour for a badly misconfigured plugin. As an example of a configuration file: @@ -53,7 +53,7 @@ expected to journal their current state to the database. The `WorkMonitor` provides methods to journal state to the database for crash recovery and to provide status information to users. -Most plugins have a `recover` method. If Vidarr is restarted, the plugin will +Most plugins have a `recover` method. If Víðarr is restarted, the plugin will be asked to recover its state from the last state information in journaled to the database using the `WorkMonitor`. Plugins are expected to be able to pick up where they left off based only on this information. @@ -75,29 +75,29 @@ resources must be available at the start of its run and it holds the resource until the workflow completes (successfully or not), at which point the resource may be reused by another workflow run. Within quota-type, some require information (_e.g._, the amount of RAM), while others are based purely on the -existence of the workflow run (_e.g._, max-in-flight). The priority consumable +existence of the workflow run (_e.g._, max-in-flight). The priority consumable resources operates within the restrictions imposed from quota resources and allows -users to manually set the order in which workflow runs will launch. Other -resource are more "throttling"-type. These include maintenance schedules and -Prometheus alerts which block workflow runs from starting but don't track -anything once the workflow run is underway. +users to manually set the order in which workflow runs will launch. Other +resource are more "throttling"-type. These include maintenance schedules and +Prometheus alerts which block workflow runs from starting but don't track +anything once the workflow run is underway. -Consumable resources are long-running. Whenever Vidarr attempts to run a +Consumable resources are long-running. Whenever Víðarr attempts to run a workflow, it will consult the consumable resources to see if there is capacity to run the workflow (the `request` method). At that point the consumable resource must make a decision as to whether the workflow can proceed. Once the -workflow has finished running (successfully or not), Vidarr will `release` the -resource so that it can be used again. When Vidarr restarts, any running +workflow has finished running (successfully or not), Víðarr will `release` the +resource so that it can be used again. When Víðarr restarts, any running workflows will be called with `recover` to indicate that the resource is being used and the resource cannot stop the workflow even if the resource is -over-capacity. +over-capacity. Consumable resources can request data from the user, if desired. The `inputFromSubmitter` can return an empty optional to indicate that no information is required or can indicate the name and type of information that is required. The `request` and `release` methods will contain a copy of this information, encoded as JSON, if the submitter provided it. The JSON data has been -type-checked by Vidarr, so it should be safe to convert to the expected type +type-checked by Víðarr, so it should be safe to convert to the expected type using Jackson. Sometimes, consumable resources are doing scoring that would be helpful to know @@ -122,9 +122,7 @@ shared directory instead of, say, `/tmp` and ensure the right permissions are set up. These are not the responsibility of the plugin author. -The class `BaseJsonInputProvisioner` is a partial implementation that can store -crash recovery information in a JSON object of the implementor's choosing, -making recovery easier. +This plugin type uses the [operations API](#operations-api). # Output Provisioners Output provisioners implement `ca.on.oicr.gsi.vidarr.OutputProvisionerProvider` @@ -143,9 +141,7 @@ the plugin to validate any configuration metadata provided by the submitter provision out step will be run with the metadata provided by the submitter and the output provided by the workflow. -The class `BaseJsonOutputProvisioner` is a partial implementation that can -store crash recovery information in a JSON object of the implementer's -choosing, making recovery easier. +This plugin type uses the [operations API](#operations-api). # Runtime Provisioners Runtime provisioners implement `ca.on.oicr.gsi.vidarr.RuntimeProvisionerProvider` @@ -159,9 +155,7 @@ This plugin and the workflow plugins must have a mutual understanding of what a workflow engine's identifier means. That is somewhat the responsibility of the system administrator. -The class `BaseJsonRuntimeProvisioner` is a partial implementation that can -store crash recovery information in a JSON object of the implementer's -choosing, making recovery easier. +This plugin type uses the [operations API](#operations-api). # Workflow Engine Workflow engines implement `ca.on.oicr.gsi.vidarr.WorkflowEngineProvider` @@ -174,17 +168,15 @@ which ones are allowed via the `supports` method. The workflow engine will be given the complete input to the workflow (with real paths provided by the input provisioners) and the workflow itself. Once the workflow has completed, it must provide a JSON structure that references the -output of the workflow. Vidarr will identified the output files generated by -the workflow engine and they will be passed to the output provisioners. +output of the workflow. Víðarr will identify the output files generated by the +workflow engine and they will be passed to the output provisioners. After the output provisioners have completed, the workflow engine will be called again to cleanup any output, if this is appropriate. If the workflow engine does not support cleanup, it should gracefully succeed during the clean-up (and clean-up recovery) methods. -The class `BaseJsonWorkflowEngine` is a partial implementation that can store -crash recovery information in a JSON object of the implementer's choosing, -making recovery easier. +This plugin type uses the [operations API](#operations-api). # Unload Filters Unload filters implement `ca.on.oicr.gsi.vidarr.UnloadFilterProvider` and @@ -211,6 +203,67 @@ to filter on runs. A filter could query Pinery and get all the external identifiers associated with that run and then construct a query based on those to match workflow runs that use any of those identifiers. + +# The Operations API +Multiple plugins use an operations API rather than direct method calls. The API +is designed to simplify two messy tasks: asynchronous operations and creating a +recoverable state. The operations API consists of a few classes in +`ca.on.oicr.gsi.vidarr`: + +- `OperationAction`: the core class that describes an operations process +- `OperationStep`: a class that describes an asynchronous operation +- `OperationStatefulStep`: a class that describes an asynchronous operation + which reads or modifies the on-going state + +The process starts with the plugin generating an original state object. This is +a plugin-defined record. State objects should _not_ be mutated and using a +record helps to encourage this. The plugin defines an `OperationAction` that +describes the process, starting with the original state object, that outlines +the steps needed to transform that state into the final output expected by the +plugin. + +As Víðarr runs the operations, it keeps track of two values: the state and the +_current value_. The current value is the output of the previous step and the +input to the next step. In effect, given steps _A_, _B_, and _C_, the +operations API will allow writing this as `load(...).then(A).then(B).then(C)`, +but it will be executing it as `C(B(A(...)))`, where the return value of the +previous step is the only parameter to the next one. This design is preferable +to direct calls because Víðarr can stop executing the task when it needs to +wait and restart it later, removing the burden of asynchronous scheduling from +the plugin author. + +To start, a call to `OperationAction.load` or `OperationAction.value` is +required. This primes the sequence of events by computing a value from the +state information alone which will be the input for the first step. After this, +`then` may be called to manipulate this value. Additionally, there are +convenience methods `map`, to modify the current value, and `reload`, to discard +the current value and load a new one from the state. + +`OperationStep` is technically a subset of `OperationStatefulStep`, but it is +implemented separately because the restricted design of `OperationStep` has +simpler type constraints, producing better errors during development. + +How state is managed along this chain of events is intentionally hidden to +simplify the process. A `OperationStatefulStep` can wrap the state in +additional information. For instance, `repeatUntilSuccess` needs to track the +number of times it attempted an operation, so it wraps the state in +`RepeatCounter`. When the chain is executed, Víðarr takes the original state +and wraps it in classes like `RepeatCounter` to build up a state that tracks +all of the paths required by the chain. It can then write this wrapped state to +the database and, if Víðarr restarts, it can recover the operations +automatically using this state information. + +This means that the steps along the way are automatically wrapping and +unwrapping state along the way so that the correct information is stored in the +database. One caveat is that if the structure of the operation of this code +changes, then so does the state stored in the database. Meaning that +redesigning the operations may mean that Víðarr cannot recover. Having the +plugin programmer manually manage state does allow them to have better control +over this scenario, but for a lot of overhead in the plugin implementation. The +interfaces Víðarr uses intentionally hide the state type behind a wild card +(`?`) generic to simplify writing plugins, but any changes to this type will +cause recovery issues. + # Provided Implementations This core implementation provides several plugins independent of external systems. diff --git a/vidarr-cli/src/main/java/ca/on/oicr/gsi/vidarr/cli/SingleShotOperation.java b/vidarr-cli/src/main/java/ca/on/oicr/gsi/vidarr/cli/SingleShotOperation.java index c4690fd3..eda7781d 100644 --- a/vidarr-cli/src/main/java/ca/on/oicr/gsi/vidarr/cli/SingleShotOperation.java +++ b/vidarr-cli/src/main/java/ca/on/oicr/gsi/vidarr/cli/SingleShotOperation.java @@ -1,7 +1,7 @@ package ca.on.oicr.gsi.vidarr.cli; -import ca.on.oicr.gsi.vidarr.core.ActiveOperation; -import ca.on.oicr.gsi.vidarr.core.OperationStatus; +import ca.on.oicr.gsi.vidarr.ActiveOperation; +import ca.on.oicr.gsi.vidarr.OperationStatus; import com.fasterxml.jackson.databind.JsonNode; import java.lang.System.Logger.Level; diff --git a/vidarr-cli/src/main/java/ca/on/oicr/gsi/vidarr/cli/TargetConfiguration.java b/vidarr-cli/src/main/java/ca/on/oicr/gsi/vidarr/cli/TargetConfiguration.java index bc88c375..e720dfec 100644 --- a/vidarr-cli/src/main/java/ca/on/oicr/gsi/vidarr/cli/TargetConfiguration.java +++ b/vidarr-cli/src/main/java/ca/on/oicr/gsi/vidarr/cli/TargetConfiguration.java @@ -5,7 +5,7 @@ import ca.on.oicr.gsi.vidarr.core.Target; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; @@ -14,51 +14,52 @@ /** Configuration file format for workflow testing */ public final class TargetConfiguration { - private WorkflowEngine engine; - private List inputs; - private List outputs; - private List runtimes; + private WorkflowEngine engine; + private List> inputs; + private List> outputs; + private List> runtimes; @JsonCreator - public TargetConfiguration(@JsonProperty("engine") WorkflowEngine engine, - @JsonProperty("inputs") List inputs, - @JsonProperty("outputs") List outputs, - @JsonProperty("runtimes") List runtimes) { + public TargetConfiguration( + @JsonProperty("engine") WorkflowEngine engine, + @JsonProperty("inputs") List> inputs, + @JsonProperty("outputs") List> outputs, + @JsonProperty("runtimes") List> runtimes) { this.engine = Objects.requireNonNull(engine, "Engine object missing from config"); this.inputs = Objects.requireNonNull(inputs, "Inputs object missing from config"); this.outputs = Objects.requireNonNull(outputs, "Outputs object missing from config"); this.runtimes = Objects.requireNonNull(runtimes, "Runtimes object missing from config"); } - public WorkflowEngine getEngine() { + public WorkflowEngine getEngine() { return engine; } - public List getInputs() { + public List> getInputs() { return inputs; } - public List getOutputs() { + public List> getOutputs() { return outputs; } - public List getRuntimes() { + public List> getRuntimes() { return runtimes; } - public void setEngine(WorkflowEngine engine) { + public void setEngine(WorkflowEngine engine) { this.engine = engine; } - public void setInputs(List inputs) { + public void setInputs(List> inputs) { this.inputs = inputs; } - public void setOutputs(List outputs) { + public void setOutputs(List> outputs) { this.outputs = outputs; } - public void setRuntimes(List runtimes) { + public void setRuntimes(List> runtimes) { this.runtimes = runtimes; } @@ -74,7 +75,7 @@ public Target toTarget() { } engine.startup(); return new Target() { - final Map inputs = + final Map> inputs = TargetConfiguration.this.inputs.stream() .flatMap( p -> @@ -83,7 +84,7 @@ public Target toTarget() { .map(f -> new Pair<>(f, p))) .collect(Collectors.toMap(Pair::first, Pair::second)); - final Map outputs = + final Map> outputs = TargetConfiguration.this.outputs.stream() .flatMap( p -> @@ -92,8 +93,8 @@ public Target toTarget() { .map(f -> new Pair<>(f, p))) .collect(Collectors.toMap(Pair::first, Pair::second)); - final List runtimes = - TargetConfiguration.this.runtimes.stream().collect(Collectors.toList()); + final List> runtimes = + new ArrayList<>(TargetConfiguration.this.runtimes); @Override public Stream> consumableResources() { @@ -101,22 +102,22 @@ public Stream> consumableResources() { } @Override - public WorkflowEngine engine() { + public WorkflowEngine engine() { return engine; } @Override - public InputProvisioner provisionerFor(InputProvisionFormat type) { + public InputProvisioner provisionerFor(InputProvisionFormat type) { return inputs.get(type); } @Override - public OutputProvisioner provisionerFor(OutputProvisionFormat type) { + public OutputProvisioner provisionerFor(OutputProvisionFormat type) { return outputs.get(type); } @Override - public Stream runtimeProvisioners() { + public Stream> runtimeProvisioners() { return runtimes.stream(); } }; diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/ActiveWorkflow.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/ActiveWorkflow.java index 857d00b9..0af45367 100644 --- a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/ActiveWorkflow.java +++ b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/ActiveWorkflow.java @@ -1,6 +1,7 @@ package ca.on.oicr.gsi.vidarr.core; import ca.on.oicr.gsi.Pair; +import ca.on.oicr.gsi.vidarr.ActiveOperation; import ca.on.oicr.gsi.vidarr.api.ExternalId; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/BaseProcessor.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/BaseProcessor.java index f67caeba..925adb9f 100644 --- a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/BaseProcessor.java +++ b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/BaseProcessor.java @@ -1,19 +1,27 @@ package ca.on.oicr.gsi.vidarr.core; -import ca.on.oicr.gsi.Pair; -import ca.on.oicr.gsi.vidarr.*; +import ca.on.oicr.gsi.vidarr.ActiveOperation; +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import ca.on.oicr.gsi.vidarr.InputProvisionFormat; +import ca.on.oicr.gsi.vidarr.OperationControlFlow; +import ca.on.oicr.gsi.vidarr.OperationStatus; +import ca.on.oicr.gsi.vidarr.OutputProvisionFormat; import ca.on.oicr.gsi.vidarr.OutputProvisioner.ResultVisitor; +import ca.on.oicr.gsi.vidarr.WorkflowDefinition; +import ca.on.oicr.gsi.vidarr.WorkflowEngine; import ca.on.oicr.gsi.vidarr.WorkflowEngine.Result; import ca.on.oicr.gsi.vidarr.api.ExternalId; -import ca.on.oicr.gsi.vidarr.core.PrepareOutputProvisioning.ProvisioningOutWorkMonitor; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.NullNode; import com.fasterxml.jackson.databind.node.ObjectNode; import java.lang.System.Logger.Level; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; @@ -27,10 +35,29 @@ public abstract class BaseProcessor< W extends ActiveWorkflow, PO extends ActiveOperation, TX> - implements FileResolver { + implements TransactionManager, FileResolver { + + OperationControlFlow createNext( + PO operation, TerminalHandler handler) { + return new TerminalOperationControlFlow<>(operation, handler); + } + + @Override + public final void inTransaction(Consumer transaction) {} + + @Override + public final void scheduleTask(Runnable task) { + executor.execute(task); + } + + @Override + public final void scheduleTask(long delay, TimeUnit units, Runnable task) { + executor.schedule(task, delay, units); + } + private interface PhaseManager { - WorkMonitor createMonitor(PO operation); + TerminalHandler createTerminal(PO operation); WorkflowDefinition definition(); @@ -41,116 +68,62 @@ private interface PhaseManager { W workflow(); } - private abstract class BaseOperationMonitor implements WorkMonitor, FileResolver { + interface TerminalHandler { + void failed(); + + JsonNode serialize(Output output); + + void succeeded(Output output); + } + + private final class TerminalOperationControlFlow + implements OperationControlFlow { private boolean finished; private final PO operation; + private final TerminalHandler handler; - protected BaseOperationMonitor(PO operation) { + TerminalOperationControlFlow(PO operation, TerminalHandler handler) { this.operation = operation; + this.handler = handler; + } + + @Override + public void cancel() { + // Do nothing. } @Override - public final synchronized void complete(R result) { + public void error(String error) { if (finished) { throw new IllegalStateException("Operation is already complete."); } finished = true; startTransaction( transaction -> { - operation.status(OperationStatus.SUCCEEDED, transaction); - operation.recoveryState(serialize(result), transaction); + operation.status(OperationStatus.FAILED, transaction); + operation.error(error, transaction); + operation.log(Level.ERROR, error); }); - succeeded(result); - } - - protected abstract void failed(); - - @Override - public final void log(System.Logger.Level level, String message) { - operation.log(level, message); + handler.failed(); } @Override - public Optional pathForId(String id) { - return BaseProcessor.this.pathForId(id); - } - - @Override - public final synchronized void permanentFailure(String reason) { + public void next(Output output) { if (finished) { throw new IllegalStateException("Operation is already complete."); } finished = true; startTransaction( transaction -> { - operation.status(OperationStatus.FAILED, transaction); - operation.error(reason, transaction); - operation.log(Level.ERROR, reason); + operation.status(OperationStatus.SUCCEEDED, transaction); + operation.recoveryState(handler.serialize(output), transaction); }); - failed(); - } - - private Runnable safeWrap(Runnable task) { - return () -> { - if (operation.isLive()) { - try { - task.run(); - } catch (Throwable e) { - log(Level.ERROR, "Task threw exception. Failing workflow: " + e.getMessage()); - e.printStackTrace(); - permanentFailure(e.toString()); - } - } else { - log(Level.ERROR, "Task is now dead."); - } - }; - } - - @Override - public final synchronized void scheduleTask(long delay, TimeUnit units, Runnable task) { - if (finished) { - throw new IllegalStateException("Operation is already complete. Cannot schedule task."); - } - executor.schedule(safeWrap(task), delay, units); - } - - @Override - public final synchronized void scheduleTask(Runnable task) { - if (finished) { - throw new IllegalStateException("Operation is already complete. Cannot schedule task."); - } - executor.execute(safeWrap(task)); - } - - protected abstract JsonNode serialize(R result); - - @Override - public void storeDebugInfo(JsonNode information) { - if (finished) { - throw new IllegalStateException( - "Operation is already complete. Cannot store debugging information."); - } - startTransaction(transaction -> operation.debugInfo(information, transaction)); + handler.succeeded(output); } @Override - public final synchronized void storeRecoveryInformation(JsonNode state) { - if (finished) { - throw new IllegalStateException( - "Operation is already complete. Cannot store recovery information."); - } - startTransaction(transaction -> operation.recoveryState(state, transaction)); - } - - protected abstract void succeeded(R result); - - @Override - public final synchronized void updateState(Status status) { - if (finished) { - throw new IllegalStateException( - "Operation is already complete. Cannot store recovery information."); - } - startTransaction(transaction -> operation.status(OperationStatus.of(status), transaction)); + public JsonNode serializeNestedState(State state) { + return mapper().valueToTree(state); } } @@ -167,7 +140,7 @@ public Phase0Initial(Target target, W workflow, WorkflowDefinition definition) { } @Override - public WorkMonitor createMonitor(PO operation) { + public TerminalHandler createTerminal(PO operation) { throw new UnsupportedOperationException("Not available for the initial phase"); } @@ -209,20 +182,20 @@ public Phase1Preflight( } @Override - public WorkMonitor createMonitor(PO operation) { - return new BaseOperationMonitor(operation) { + public TerminalHandler createTerminal(PO operation) { + return new TerminalHandler<>() { @Override - protected void failed() { + public void failed() { release(false); } @Override - protected JsonNode serialize(Boolean result) { + public JsonNode serialize(Boolean result) { return JsonNodeFactory.instance.booleanNode(result); } @Override - protected void succeeded(Boolean result) { + public void succeeded(Boolean result) { release(result); } }; @@ -276,6 +249,7 @@ public void release(Boolean result) { target, activeWorkflow.arguments().get(parameter.name()), Stream.of(JsonPath.object(parameter.name())), + definition.language(), BaseProcessor.this, provisionInTasks::add, retryModifications))); @@ -369,10 +343,10 @@ public Phase2ProvisionIn( } @Override - public WorkMonitor createMonitor(PO operation) { - return new BaseOperationMonitor(operation) { + public TerminalHandler createTerminal(PO operation) { + return new TerminalHandler<>() { @Override - protected void failed() { + public void failed() { if (size.decrementAndGet() == 0) { startTransaction( transaction -> @@ -381,12 +355,12 @@ protected void failed() { } @Override - protected JsonNode serialize(JsonMutation result) { + public JsonNode serialize(JsonMutation result) { return mapper().valueToTree(result); } @Override - protected void succeeded(JsonMutation result) { + public void succeeded(JsonMutation result) { semaphore.acquireUninterruptibly(); startTransaction( transaction -> { @@ -404,19 +378,8 @@ protected void succeeded(JsonMutation result) { startNextPhase( Phase2ProvisionIn.this, List.of( - (workflowLanguage, workflowId, operation) -> - new Pair<>( - "", - target - .engine() - .run( - definition.language(), - definition.contents(), - definition.accessoryFiles(), - activeWorkflow.id(), - inputs.get(0), - activeWorkflow.engineArguments(), - operation))), + TaskStarter.launch( + definition, activeWorkflow, target.engine(), inputs.get(0))), transaction); } }); @@ -447,8 +410,7 @@ public W workflow() { } private class Phase3Run - implements PhaseManager< - W, WorkflowEngine.Result, Pair, PO> { + implements PhaseManager, ProvisionData, PO> { private final W activeWorkflow; private final WorkflowDefinition definition; @@ -464,45 +426,39 @@ public Phase3Run(Target target, WorkflowDefinition definition, int size, W activ } @Override - public WorkMonitor, JsonNode> createMonitor(PO operation) { - return new BaseOperationMonitor>(operation) { + public TerminalHandler> createTerminal(PO operation) { + return new TerminalHandler<>() { @Override - protected void failed() { + public void failed() { final var realInputs = activeWorkflow.realInputs(); startTransaction( tx -> { final var index = activeWorkflow.realInputTryNext(tx); if (index < realInputs.size()) { - final var monitor = - new DelayWorkMonitor, JsonNode>(); - final var initialState = - target - .engine() - .run( - definition.language(), - definition.contents(), - definition.accessoryFiles(), - activeWorkflow.id(), - realInputs.get(index), - activeWorkflow.engineArguments(), - monitor); + final var relaunch = + TaskStarter.launch( + definition, activeWorkflow, target.engine(), realInputs.get(index)); final var nextPhaseManager = new Phase3Run(target, definition, 1, activeWorkflow); - monitor.set(nextPhaseManager.createMonitor(operation)); final var operations = workflow() .phase( - nextPhaseManager.phase(), List.of(new Pair<>("", initialState)), tx); + nextPhaseManager.phase(), + List.of(TaskStarter.toPair(relaunch, mapper())), + tx); if (operations.size() != 1) { // The backing store has decided to abandon this workflow run. return; } - monitor.set(nextPhaseManager.createMonitor(operations.get(0))); + relaunch.start( + BaseProcessor.this, + operations.get(0), + nextPhaseManager.createTerminal(operations.get(0))); } }); } @Override - protected JsonNode serialize(Result result) { + public JsonNode serialize(Result result) { final var output = mapper().createObjectNode(); output.set("output", result.output()); output.set("cleanupState", result.cleanupState().orElse(NullNode.getInstance())); @@ -511,17 +467,21 @@ protected JsonNode serialize(Result result) { } @Override - protected void succeeded(Result result) { + public void succeeded(Result result) { if (result.output() == null) { - permanentFailure("No output from workflow"); + startTransaction( + transaction -> { + operation.status(OperationStatus.FAILED, transaction); + operation.recoveryState( + JsonNodeFactory.instance.textNode("No output from workflow"), transaction); + }); return; } startTransaction( transaction -> { result.cleanupState().ifPresent(c -> workflow().cleanup(c, transaction)); workflow().runUrl(result.workflowRunUrl(), transaction); - final var tasks = - new ArrayList>>(); + final var tasks = new ArrayList>(); final var allIds = workflow().inputIds(); final var remainingIds = new HashSet<>(allIds); remainingIds.removeAll(workflow().requestedExternalIds()); @@ -530,13 +490,7 @@ protected void succeeded(Result result) { .forEach( p -> tasks.add( - WrappedMonitor.start( - new ProvisionData(allIds), - PrepareOutputProvisioning.ProvisioningOutWorkMonitor::new, - (language, workflowId, monitor) -> - new Pair<>( - "$" + p.name(), - p.provision(result.workflowRunUrl(), monitor))))); + TaskStarter.launch(p, allIds, Map.of(), result.workflowRunUrl()))); if (definition .outputs() .allMatch( @@ -553,7 +507,8 @@ protected void succeeded(Result result) { activeWorkflow.metadata().get(output.name()), allIds, remainingIds, - () -> isOk.set(false))) + () -> isOk.set(false), + activeWorkflow.id())) .forEach(tasks::add); return isOk.get(); } else { @@ -580,8 +535,7 @@ public Phase phase() { } @Override - public PhaseManager, ?, PO> startNext( - int size) { + public PhaseManager startNext(int size) { return new Phase4ProvisionOut(target, definition, size, activeWorkflow); } @@ -591,8 +545,7 @@ public W workflow() { } } - private class Phase4ProvisionOut - implements PhaseManager, Void, PO> { + private class Phase4ProvisionOut implements PhaseManager { private final W activeWorkflow; private final WorkflowDefinition definition; @@ -608,21 +561,22 @@ public Phase4ProvisionOut( } @Override - public WorkMonitor, JsonNode> createMonitor( - PO operation) { - return new BaseOperationMonitor>(operation) { + public TerminalHandler createTerminal(PO operation) { + return new TerminalHandler<>() { @Override - protected void failed() { + public void failed() { // Do nothing. } @Override - protected JsonNode serialize(Pair result) { + public JsonNode serialize(ProvisionData result) { final var node = mapper().createObjectNode(); - node.putPOJO("info", result.first()); + final var info = node.putObject("info"); + info.putPOJO("ids", result.ids()); + info.putPOJO("labels", result.labels()); result - .second() + .result() .visit( new ResultVisitor() { @Override @@ -645,11 +599,11 @@ public void url(String url, Map labels) { } @Override - protected void succeeded(Pair result) { + public void succeeded(ProvisionData result) { startTransaction( transaction -> { result - .second() + .result() .visit( new ResultVisitor() { @Override @@ -657,23 +611,19 @@ public void file( String storagePath, String md5, long size, String metatype) { workflow() .provisionFile( - result.first().getIds(), + result.ids(), storagePath, md5, metatype, size, - result.first().getLabels(), + result.labels(), transaction); } @Override public void url(String url, Map labels) { workflow() - .provisionUrl( - result.first().getIds(), - url, - result.first().getLabels(), - transaction); + .provisionUrl(result.ids(), url, result.labels(), transaction); } }); if (size.decrementAndGet() == 0) { @@ -683,9 +633,7 @@ public void url(String url, Map labels) { } else { startNextPhase( Phase4ProvisionOut.this, - Collections.singletonList( - (lang, workflowId, monitor) -> - new Pair<>("", target.engine().cleanup(cleanup, monitor))), + List.of(TaskStarter.launchCleanup(target.engine(), cleanup)), transaction); } } @@ -726,18 +674,18 @@ private Phase5Cleanup(WorkflowDefinition definition, W activeWorkflow) { } @Override - public WorkMonitor createMonitor(PO operation) { - return new BaseOperationMonitor(operation) { + public TerminalHandler createTerminal(PO operation) { + return new TerminalHandler<>() { @Override - protected void failed() {} + public void failed() {} @Override - protected JsonNode serialize(Void result) { + public JsonNode serialize(Void result) { return JsonNodeFactory.instance.nullNode(); } @Override - protected void succeeded(Void result) { + public void succeeded(Void result) { startTransaction(activeWorkflow::succeeded); } }; @@ -882,32 +830,37 @@ protected void recover( new Phase1Preflight( target, activeOperations.size(), workflow, definition, workflow.isPreflightOkay()); for (final var operation : activeOperations) { - target - .provisionerFor(WorkflowOutputDataType.valueOf(operation.type()).format()) - .preflightRecover(operation.recoveryState(), p1.createMonitor(operation)); + TaskStarter.of( + operation.type(), + target + .provisionerFor(WorkflowOutputDataType.valueOf(operation.type()).format()) + .runPreflight() + .recover(operation.recoveryState())) + .start(this, operation, p1.createTerminal(operation)); } break; case PROVISION_IN: final var p2 = new Phase2ProvisionIn(target, activeOperations.size(), definition, workflow); for (final var operation : activeOperations) { - WrappedMonitor.recover( - operation.recoveryState(), - v -> { - try { - return List.of(mapper().treeToValue(v, JsonPath[].class)); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }, - PrepareInputProvisioning.ProvisionInMonitor::new, - target.provisionerFor(InputProvisionFormat.valueOf(operation.type()))::recover, - p2.createMonitor(operation)); + PrepareInputProvisioning.recover( + definition.language(), + operation, + target.provisionerFor( + InputProvisionFormat.valueOf(operation.type().substring(1)))) + .start(this, operation, p2.createTerminal(operation)); } break; case RUNNING: final var p3 = new Phase3Run(target, definition, activeOperations.size(), workflow); for (final var operation : activeOperations) { - target.engine().recover(operation.recoveryState(), p3.createMonitor(operation)); + TaskStarter.of( + "", + target + .engine() + .run() + .map(WorkflowEngine.Result::serialize) + .recover(operation.recoveryState())) + .start(this, operation, p3.createTerminal(operation)); } break; case PROVISION_OUT: @@ -915,44 +868,33 @@ protected void recover( new Phase4ProvisionOut(target, definition, activeOperations.size(), workflow); for (final var operation : activeOperations) { if (operation.type().startsWith("$")) { - WrappedMonitor.recover( - operation.recoveryState(), - v -> { - try { - return mapper().treeToValue(v, ProvisionData.class); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }, - ProvisioningOutWorkMonitor::new, - recoverType.runtime( - target - .runtimeProvisioners() - .filter(p -> p.name().equals(operation.type().substring(1))) - .findAny() - .orElseThrow()), - p4.createMonitor(operation)); + TaskStarter.of( + operation.type(), + recoverType.prepare( + TaskStarter.wrapRuntimeProvisioner( + target + .runtimeProvisioners() + .filter(p -> p.name().equals(operation.type().substring(1))) + .findAny() + .orElseThrow()), + operation.recoveryState())) + .start(this, operation, p4.createTerminal(operation)); } else { - WrappedMonitor.recover( - operation.recoveryState(), - v -> { - try { - return mapper().treeToValue(v, ProvisionData.class); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }, - ProvisioningOutWorkMonitor::new, - recoverType.provisionOut( - target.provisionerFor(OutputProvisionFormat.valueOf(operation.type()))), - p4.createMonitor(operation)); + TaskStarter.of( + operation.type(), + recoverType.prepare( + TaskStarter.wrapOutputProvisioner( + target.provisionerFor(OutputProvisionFormat.valueOf(operation.type()))), + operation.recoveryState())) + .start(this, operation, p4.createTerminal(operation)); } } break; case CLEANUP: final var p5 = new Phase5Cleanup(definition, workflow); for (final var operation : activeOperations) { - target.engine().recoverCleanup(operation.recoveryState(), p5.createMonitor(operation)); + TaskStarter.of("", target.engine().cleanup().recover(operation.recoveryState())) + .start(this, operation, p5.createTerminal(operation)); } break; case FAILED: @@ -998,17 +940,8 @@ protected void start(Target target, WorkflowDefinition definition, W workflow, T currentPhase.workflow().phase(Phase.FAILED, Collections.emptyList(), transaction); return; } - final var monitors = new ArrayList>(); final var initialStates = - nextPhaseSteps.stream() - .map( - t -> { - final var monitor = new DelayWorkMonitor(); - monitors.add(monitor); - return t.start( - currentPhase.definition().language(), currentPhase.workflow().id(), monitor); - }) - .collect(Collectors.toList()); + nextPhaseSteps.stream().map((starter) -> TaskStarter.toPair(starter, mapper())).toList(); final var nextPhaseManager = currentPhase.startNext(initialStates.size()); final var operations = currentPhase.workflow().phase(nextPhaseManager.phase(), initialStates, transaction); @@ -1018,7 +951,7 @@ protected void start(Target target, WorkflowDefinition definition, W workflow, T } for (var index = 0; index < nextPhaseSteps.size(); index++) { final var operation = operations.get(index); - monitors.get(index).set(nextPhaseManager.createMonitor(operation)); + nextPhaseSteps.get(index).start(this, operation, nextPhaseManager.createTerminal(operation)); } } diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/DelayWorkMonitor.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/DelayWorkMonitor.java deleted file mode 100644 index 056b852e..00000000 --- a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/DelayWorkMonitor.java +++ /dev/null @@ -1,87 +0,0 @@ -package ca.on.oicr.gsi.vidarr.core; - -import ca.on.oicr.gsi.vidarr.WorkMonitor; -import ca.on.oicr.gsi.vidarr.WorkMonitor.Status; -import com.fasterxml.jackson.databind.JsonNode; -import java.util.Deque; -import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.concurrent.TimeUnit; - -/** - * A workflow monitor that buffers request until later - * - * @param the output type - * @param the recovery state type - */ -final class DelayWorkMonitor implements WorkMonitor { - private WorkMonitor delegate; - private final Deque waiting = new ConcurrentLinkedDeque<>(); - - public void complete(T result) { - executeSafely(() -> delegate.complete(result)); - } - - private void executeSafely(Runnable task) { - boolean run; - synchronized (this) { - if (delegate == null) { - waiting.add(task); - run = false; - } else { - run = true; - } - } - if (run) { - task.run(); - } - } - - @Override - public void log(System.Logger.Level level, String message) { - executeSafely(() -> delegate.log(level, message)); - } - - public void permanentFailure(String reason) { - executeSafely(() -> delegate.permanentFailure(reason)); - } - - public void scheduleTask(long delay, TimeUnit units, Runnable task) { - executeSafely(() -> delegate.scheduleTask(delay, units, task)); - } - - @Override - public void storeDebugInfo(JsonNode information) { - executeSafely(() -> delegate.storeDebugInfo(information)); - } - - public void scheduleTask(Runnable task) { - executeSafely(() -> delegate.scheduleTask(task)); - } - - /** - * Add a real work monitor that will handle the work - * - * @param monitor the monitor to use - */ - public void set(WorkMonitor monitor) { - synchronized (this) { - if (delegate == null) { - delegate = monitor; - } else { - throw new IllegalStateException("Cannot replace work monitor once activated."); - } - } - Runnable task; - while ((task = waiting.pollFirst()) != null) { - task.run(); - } - } - - public void storeRecoveryInformation(S state) { - executeSafely(() -> delegate.storeRecoveryInformation(state)); - } - - public void updateState(Status status) { - executeSafely(() -> delegate.updateState(status)); - } -} diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/InputProvisioningStateExternal.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/InputProvisioningStateExternal.java new file mode 100644 index 00000000..8364713f --- /dev/null +++ b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/InputProvisioningStateExternal.java @@ -0,0 +1,6 @@ +package ca.on.oicr.gsi.vidarr.core; + +import com.fasterxml.jackson.databind.JsonNode; +import java.util.List; + +public record InputProvisioningStateExternal(List mutation, JsonNode metadata) {} diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/InputProvisioningStateInternal.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/InputProvisioningStateInternal.java new file mode 100644 index 00000000..9a39f477 --- /dev/null +++ b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/InputProvisioningStateInternal.java @@ -0,0 +1,5 @@ +package ca.on.oicr.gsi.vidarr.core; + +import java.util.List; + +public record InputProvisioningStateInternal(List mutation, String id, String path) {} diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/JsonMutation.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/JsonMutation.java index 909ffed5..882ce00a 100644 --- a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/JsonMutation.java +++ b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/JsonMutation.java @@ -2,20 +2,9 @@ import com.fasterxml.jackson.databind.JsonNode; import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; /** Describe a change to a JSON document */ -public final class JsonMutation { - private List path; - private JsonNode result; - - public JsonMutation() {} - - public JsonMutation(JsonNode result, Stream path) { - this.result = result; - this.path = path.collect(Collectors.toList()); - } +public record JsonMutation(List path, JsonNode result) { public List getPath() { return path; @@ -24,12 +13,4 @@ public List getPath() { public JsonNode getResult() { return result; } - - public void setPath(List path) { - this.path = path; - } - - public void setResult(JsonNode result) { - this.result = result; - } } diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/MonitorWithType.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/MonitorWithType.java deleted file mode 100644 index 0adb1fa5..00000000 --- a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/MonitorWithType.java +++ /dev/null @@ -1,64 +0,0 @@ -package ca.on.oicr.gsi.vidarr.core; - -import ca.on.oicr.gsi.vidarr.WorkMonitor; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import java.util.concurrent.TimeUnit; - -/** - * A work monitor with an additional type attribute that gets serialised with the data - * - * @param the inner state type - */ -final class MonitorWithType implements WorkMonitor { - private final WorkMonitor monitor; - private final String type; - - public MonitorWithType(WorkMonitor monitor, String type) { - this.monitor = monitor; - this.type = type; - } - - @Override - public void complete(T result) { - monitor.complete(result); - } - - @Override - public void log(System.Logger.Level level, String message) { - monitor.log(level, message); - } - - @Override - public void permanentFailure(String reason) { - monitor.permanentFailure(reason); - } - - @Override - public void scheduleTask(long delay, TimeUnit units, Runnable task) { - monitor.scheduleTask(delay, units, task); - } - - @Override - public void storeDebugInfo(JsonNode information) { - monitor.storeDebugInfo(information); - } - - @Override - public void scheduleTask(Runnable task) { - monitor.scheduleTask(task); - } - - @Override - public void storeRecoveryInformation(JsonNode state) { - final var array = JsonNodeFactory.instance.arrayNode(); - array.add(type); - array.add(state); - monitor.storeRecoveryInformation(array); - } - - @Override - public void updateState(Status status) { - monitor.updateState(status); - } -} diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/OneOfInputProvisioner.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/OneOfInputProvisioner.java index 6beefa30..e7801129 100644 --- a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/OneOfInputProvisioner.java +++ b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/OneOfInputProvisioner.java @@ -6,24 +6,27 @@ import ca.on.oicr.gsi.vidarr.InputProvisionFormat; import ca.on.oicr.gsi.vidarr.InputProvisioner; import ca.on.oicr.gsi.vidarr.InputProvisionerProvider; -import ca.on.oicr.gsi.vidarr.WorkMonitor; +import ca.on.oicr.gsi.vidarr.OperationAction; +import ca.on.oicr.gsi.vidarr.OperationAction.BranchState; import ca.on.oicr.gsi.vidarr.WorkflowLanguage; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.xml.stream.XMLStreamException; /** An input provisioner that selects between multiple input provisioners using a tagged union */ -public final class OneOfInputProvisioner implements InputProvisioner { +public final class OneOfInputProvisioner implements InputProvisioner { public static InputProvisionerProvider provider() { return () -> Stream.of(new Pair<>("oneOf", OneOfInputProvisioner.class)); } private String internal; - private Map provisioners; + private Map> provisioners; - public OneOfInputProvisioner(Map provisioners, String internal) { + public OneOfInputProvisioner( + Map> provisioners, String internal) { this.provisioners = provisioners; this.internal = internal; } @@ -52,47 +55,49 @@ public String getInternal() { return internal; } - public Map getProvisioners() { + public Map> getProvisioners() { return provisioners; } @Override - public JsonNode provision( - WorkflowLanguage language, String id, String path, WorkMonitor monitor) { - final var output = JsonNodeFactory.instance.arrayNode(); - output.add(internal); - output.add( - provisioners - .get(internal) - .provision(language, id, path, new MonitorWithType<>(monitor, internal))); - return output; + public BranchState provision(WorkflowLanguage language, String id, String path) { + return provision(provisioners.get(internal), language, id, path); + } + + private BranchState provision( + InputProvisioner provisioner, + WorkflowLanguage language, + String id, + String path) { + return provisioner.run().intoBranch(internal, provisioner.provision(language, id, path)); } @Override - public JsonNode provisionExternal( - WorkflowLanguage language, JsonNode metadata, WorkMonitor monitor) { + public BranchState provisionExternal(WorkflowLanguage language, JsonNode metadata) { final var type = metadata.get("type").asText(); - final var output = JsonNodeFactory.instance.arrayNode(); - output.add(type); - output.add( - provisioners - .get(type) - .provisionExternal( - language, metadata.get("contents"), new MonitorWithType<>(monitor, type))); - return output; + return provisionExternal(type, provisioners.get(type), language, metadata.get("contents")); + } + + private BranchState provisionExternal( + String name, + InputProvisioner provisioner, + WorkflowLanguage language, + JsonNode metadata) { + return provisioner.run().intoBranch(name, provisioner.provisionExternal(language, metadata)); } @Override - public void recover(JsonNode state, WorkMonitor monitor) { - final var type = state.get(0).asText(); - provisioners.get(type).recover(state.get(1), new MonitorWithType<>(monitor, type)); + public OperationAction run() { + return OperationAction.branch( + provisioners.entrySet().stream() + .collect(Collectors.toMap(Entry::getKey, e -> e.getValue().run()))); } public void setInternal(String internal) { this.internal = internal; } - public void setProvisioners(Map provisioners) { + public void setProvisioners(Map> provisioners) { this.provisioners = provisioners; } diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/OneOfOutputProvisioner.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/OneOfOutputProvisioner.java index 20cf25c1..4b261131 100644 --- a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/OneOfOutputProvisioner.java +++ b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/OneOfOutputProvisioner.java @@ -3,25 +3,28 @@ import ca.on.oicr.gsi.Pair; import ca.on.oicr.gsi.status.SectionRenderer; import ca.on.oicr.gsi.vidarr.BasicType; +import ca.on.oicr.gsi.vidarr.OperationAction; +import ca.on.oicr.gsi.vidarr.OperationAction.BranchState; import ca.on.oicr.gsi.vidarr.OutputProvisionFormat; import ca.on.oicr.gsi.vidarr.OutputProvisioner; import ca.on.oicr.gsi.vidarr.OutputProvisionerProvider; -import ca.on.oicr.gsi.vidarr.WorkMonitor; import com.fasterxml.jackson.databind.JsonNode; import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.xml.stream.XMLStreamException; /** An output provisioner that combines multiple other plugins using a tagged union */ -public final class OneOfOutputProvisioner implements OutputProvisioner { +public final class OneOfOutputProvisioner implements OutputProvisioner { public static OutputProvisionerProvider provider() { return () -> Stream.of(new Pair<>("oneOf", OneOfOutputProvisioner.class)); } - private final Map provisioners; + private final Map> provisioners; - public OneOfOutputProvisioner(Map provisioners) { + public OneOfOutputProvisioner(Map> provisioners) { this.provisioners = provisioners; } @@ -39,39 +42,43 @@ public void configuration(SectionRenderer sectionRenderer) throws XMLStreamExcep } @Override - public JsonNode preflightCheck(JsonNode metadata, WorkMonitor monitor) { + public BranchState preflightCheck(JsonNode metadata) { final var type = metadata.get("type").asText(); - return provisioners - .get(type) - .preflightCheck(metadata.get("contents"), new MonitorWithType<>(monitor, type)); + return preflightCheck(type, provisioners.get(type), metadata.get("contents")); } - @Override - public void preflightRecover(JsonNode state, WorkMonitor monitor) { - final var type = state.get(0).asText(); - provisioners.get(type).preflightRecover(state.get(1), new MonitorWithType<>(monitor, type)); + private BranchState preflightCheck( + String type, OutputProvisioner provisioner, JsonNode metadata) { + return provisioner.runPreflight().intoBranch(type, provisioner.preflightCheck(metadata)); } @Override - public JsonNode provision( - String workflowRunId, String data, JsonNode metadata, WorkMonitor monitor) { + public BranchState provision(String workflowRunId, String data, JsonNode metadata) { final var type = metadata.get("type").asText(); - return provisioners - .get(type) - .provision( - workflowRunId, data, metadata.get("contents"), new MonitorWithType<>(monitor, type)); + return provision(type, provisioners.get(type), workflowRunId, data, metadata.get("contents")); + } + + private BranchState provision( + String type, + OutputProvisioner provisioner, + String workflowRunId, + String data, + JsonNode metadata) { + return provisioner.run().intoBranch(type, provisioner.provision(workflowRunId, data, metadata)); } @Override - public void recover(JsonNode state, WorkMonitor monitor) { - final var type = state.get(0).asText(); - provisioners.get(type).recover(state.get(1), new MonitorWithType<>(monitor, type)); + public OperationAction run() { + return OperationAction.branch( + provisioners.entrySet().stream() + .collect(Collectors.toMap(Entry::getKey, e -> e.getValue().run()))); } @Override - public void retry(JsonNode state, WorkMonitor monitor) { - final var type = state.get(0).asText(); - provisioners.get(type).retry(state.get(1), new MonitorWithType<>(monitor, type)); + public OperationAction runPreflight() { + return OperationAction.branch( + provisioners.entrySet().stream() + .collect(Collectors.toMap(Entry::getKey, e -> e.getValue().runPreflight()))); } @Override diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/OutputProvisionState.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/OutputProvisionState.java new file mode 100644 index 00000000..a65b62e5 --- /dev/null +++ b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/OutputProvisionState.java @@ -0,0 +1,14 @@ +package ca.on.oicr.gsi.vidarr.core; + +import ca.on.oicr.gsi.vidarr.api.ExternalId; +import com.fasterxml.jackson.databind.JsonNode; +import java.util.Map; +import java.util.Set; + +/** The accessory data that must be stored during output provisioning */ +public record OutputProvisionState( + Set ids, + Map labels, + String workflowRunId, + String data, + JsonNode metadata) {} diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/PrepareInputProvisioning.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/PrepareInputProvisioning.java index 810ce17b..9a3e8d66 100644 --- a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/PrepareInputProvisioning.java +++ b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/PrepareInputProvisioning.java @@ -1,10 +1,15 @@ package ca.on.oicr.gsi.vidarr.core; import ca.on.oicr.gsi.Pair; +import ca.on.oicr.gsi.vidarr.ActiveOperation; import ca.on.oicr.gsi.vidarr.BasicType; import ca.on.oicr.gsi.vidarr.InputProvisionFormat; +import ca.on.oicr.gsi.vidarr.InputProvisioner; import ca.on.oicr.gsi.vidarr.InputType; -import ca.on.oicr.gsi.vidarr.WorkMonitor; +import ca.on.oicr.gsi.vidarr.OperationAction; +import ca.on.oicr.gsi.vidarr.OperationStatefulStep; +import ca.on.oicr.gsi.vidarr.OperationStatefulStep.Child; +import ca.on.oicr.gsi.vidarr.WorkflowLanguage; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.NullNode; @@ -18,23 +23,89 @@ /** Start the input provisioning tasks */ final class PrepareInputProvisioning implements InputType.Visitor { - public static class ProvisionInMonitor - extends WrappedMonitor, JsonNode, JsonMutation> { + private static TaskStarter launch( + InputProvisionFormat format, + WorkflowLanguage language, + InputProvisioner provisioner, + List mutation, + String id, + String filePath) { + return TaskStarter.of( + "i" + format.name(), + wrapProvisionActionInternal(language, provisioner, provisioner.run()) + .launch(new InputProvisioningStateInternal(mutation, id, filePath))); + } - public ProvisionInMonitor( - List jsonPath, WorkMonitor monitor) { - super(jsonPath, monitor); - } + private static TaskStarter launchExternal( + InputProvisionFormat format, + WorkflowLanguage language, + InputProvisioner provisioner, + List mutation, + JsonNode metadata) { + return TaskStarter.of( + "e" + format.name(), + wrapProvisionActionExternal(language, provisioner, provisioner.run()) + .launch(new InputProvisioningStateExternal(mutation, metadata))); + } - @Override - protected JsonMutation mix(List accessory, JsonNode result) { - return new JsonMutation(result, accessory.stream()); + public static TaskStarter recover( + WorkflowLanguage language, + ActiveOperation operation, + InputProvisioner provisioner) { + switch (operation.type().charAt(0)) { + case 'i': + return TaskStarter.of( + operation.type(), + wrapProvisionActionInternal(language, provisioner, provisioner.run()) + .recover(operation.recoveryState())); + case 'e': + return TaskStarter.of( + operation.type(), + wrapProvisionActionExternal(language, provisioner, provisioner.run()) + .recover(operation.recoveryState())); + default: + throw new IllegalArgumentException("Illegal type for operation: " + operation.type()); } } + static + OperationAction< + Child, + InputProvisioningStateExternal, + JsonMutation> + wrapProvisionActionExternal( + WorkflowLanguage language, + InputProvisioner provisioner, + OperationAction action) { + return OperationAction.load( + InputProvisioningStateExternal.class, InputProvisioningStateExternal::metadata) + .then( + OperationStatefulStep.subStep( + (state, metadata) -> provisioner.provisionExternal(language, metadata), action)) + .map((state, result) -> new JsonMutation(state.state().mutation(), result)); + } + + static + OperationAction< + Child, + InputProvisioningStateInternal, + JsonMutation> + wrapProvisionActionInternal( + WorkflowLanguage language, + InputProvisioner provisioner, + OperationAction action) { + return OperationAction.load( + InputProvisioningStateInternal.class, InputProvisioningStateInternal::id) + .then( + OperationStatefulStep.subStep( + (state, id) -> provisioner.provision(language, id, state.path()), action)) + .map((state, result) -> new JsonMutation(state.state().mutation(), result)); + } + private final Consumer> consumer; private final JsonNode input; private final List jsonPath; + private final WorkflowLanguage language; private final FileResolver resolver; private final Map>> retryModifications; private final Target target; @@ -43,12 +114,14 @@ public PrepareInputProvisioning( Target target, JsonNode input, Stream jsonPath, + WorkflowLanguage language, FileResolver resolver, Consumer> consumer, Map>> retryModifications) { this.target = target; this.input = input; this.jsonPath = jsonPath.collect(Collectors.toList()); + this.language = language; this.resolver = resolver; this.consumer = consumer; this.retryModifications = retryModifications; @@ -86,6 +159,7 @@ public JsonNode dictionary(InputType key, InputType value) { target, entry.getValue(), Stream.concat(jsonPath.stream(), Stream.of(JsonPath.object(entry.getKey()))), + language, resolver, consumer, retryModifications))); @@ -106,6 +180,7 @@ public JsonNode dictionary(InputType key, InputType value) { inputEntry.get(0), Stream.concat( jsonPath.stream(), Stream.of(JsonPath.array(i), JsonPath.array(0))), + language, resolver, consumer, retryModifications))); @@ -116,6 +191,7 @@ public JsonNode dictionary(InputType key, InputType value) { inputEntry.get(1), Stream.concat( jsonPath.stream(), Stream.of(JsonPath.array(i), JsonPath.array(1))), + language, resolver, consumer, retryModifications))); @@ -155,14 +231,8 @@ private JsonNode handle(InputProvisionFormat format) { switch (input.get("type").asText()) { case "EXTERNAL": consumer.accept( - WrappedMonitor.start( - jsonPath, - ProvisionInMonitor::new, - (language, workflowId, monitor) -> - new Pair<>( - format.name(), - handler.provisionExternal( - language, input.get("contents").get("configuration"), monitor)))); + launchExternal( + format, language, handler, jsonPath, input.get("contents").get("configuration"))); break; case "INTERNAL": if (input.has("contents") @@ -171,13 +241,7 @@ private JsonNode handle(InputProvisionFormat format) { && input.get("contents").get(0).isTextual()) { final var id = input.get("contents").get(0).asText(); final var filePath = resolver.pathForId(id).map(FileMetadata::path).orElseThrow(); - consumer.accept( - WrappedMonitor.start( - jsonPath, - ProvisionInMonitor::new, - (language, workflowId, monitor) -> - new Pair<>( - format.name(), handler.provision(language, id, filePath, monitor)))); + consumer.accept(launch(format, language, handler, jsonPath, id, filePath)); } else { throw new IllegalArgumentException("Invalid input file for BY_ID"); } @@ -213,6 +277,7 @@ public JsonNode list(InputType inner) { target, input.get(i), Stream.concat(jsonPath.stream(), Stream.of(JsonPath.array(i))), + language, resolver, consumer, retryModifications))); @@ -238,6 +303,7 @@ public JsonNode object(Stream> contents) { input.get(p.first()), Stream.concat( jsonPath.stream(), Stream.of(JsonPath.object(p.first()))), + language, resolver, consumer, retryModifications)))); @@ -269,6 +335,7 @@ public JsonNode pair(InputType left, InputType right) { target, input.get(0), Stream.concat(jsonPath.stream(), Stream.of(JsonPath.object("left"))), + language, resolver, consumer, retryModifications))); @@ -279,6 +346,7 @@ public JsonNode pair(InputType left, InputType right) { target, input.get(1), Stream.concat(jsonPath.stream(), Stream.of(JsonPath.object("right"))), + language, resolver, consumer, retryModifications))); @@ -296,6 +364,7 @@ public JsonNode pair(InputType left, InputType right) { target, input.get("left"), Stream.concat(jsonPath.stream(), Stream.of(JsonPath.object("left"))), + language, resolver, consumer, retryModifications))); @@ -306,6 +375,7 @@ public JsonNode pair(InputType left, InputType right) { target, input.get("right"), Stream.concat(jsonPath.stream(), Stream.of(JsonPath.object("right"))), + language, resolver, consumer, retryModifications))); @@ -356,6 +426,7 @@ public JsonNode taggedUnion(Stream> elements) { target, input.get("contents"), jsonPath.stream(), + language, resolver, consumer, retryModifications))) @@ -381,6 +452,7 @@ public void accept(InputType t) { target, input.get(index), Stream.concat(jsonPath.stream(), Stream.of(JsonPath.array(index))), + language, resolver, consumer, retryModifications))); diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/PrepareOutputProvisioning.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/PrepareOutputProvisioning.java index 2ab70f48..a242ba69 100644 --- a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/PrepareOutputProvisioning.java +++ b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/PrepareOutputProvisioning.java @@ -1,14 +1,15 @@ package ca.on.oicr.gsi.vidarr.core; import ca.on.oicr.gsi.Pair; -import ca.on.oicr.gsi.vidarr.OutputProvisioner; -import ca.on.oicr.gsi.vidarr.OutputProvisioner.Result; import ca.on.oicr.gsi.vidarr.OutputType; -import ca.on.oicr.gsi.vidarr.WorkMonitor; import ca.on.oicr.gsi.vidarr.api.ExternalId; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.*; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.Spliterators; +import java.util.TreeMap; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -20,22 +21,7 @@ */ final class PrepareOutputProvisioning extends BaseOutputExtractor< - Stream>>, - Stream>>> { - static final class ProvisioningOutWorkMonitor - extends WrappedMonitor< - ProvisionData, OutputProvisioner.Result, Pair> { - - public ProvisioningOutWorkMonitor( - ProvisionData accessory, WorkMonitor, JsonNode> monitor) { - super(accessory, monitor); - } - - @Override - protected Pair mix(ProvisionData accessory, Result result) { - return new Pair<>(accessory, result); - } - } + Stream>, Stream>> { private static Map extractLabels(JsonNode node) { final var labels = new TreeMap(); @@ -53,6 +39,7 @@ private static Map extractLabels(JsonNode node) { private final Set remainingInputIds; private final Runnable outputIsBad; private final Target target; + private final String workflowRunId; public PrepareOutputProvisioning( ObjectMapper mapper, @@ -61,17 +48,19 @@ public PrepareOutputProvisioning( JsonNode metadata, Set allInputIds, Set remainingInputIds, - Runnable outputIsBad) { + Runnable outputIsBad, + String workflowRunId) { super(output, metadata); this.mapper = mapper; this.target = target; this.allInputIds = allInputIds; this.remainingInputIds = remainingInputIds; this.outputIsBad = outputIsBad; + this.workflowRunId = workflowRunId; } @Override - protected Stream>> handle( + protected Stream> handle( WorkflowOutputDataType format, boolean optional, JsonNode metadata, @@ -100,9 +89,9 @@ yield stream(output.get("left"), optional) }) .map( p -> { - final var provisionData = new ProvisionData(); - provisionData.setLabels(p.second()); - provisionData.setIds( + final var data = p.first(); + final var labels = p.second(); + final Set ids = outputData.visit( new OutputDataVisitor<>() { @Override @@ -119,14 +108,9 @@ public Set external(Stream ids) { public Set remaining() { return remainingInputIds; } - })); - return WrappedMonitor.start( - provisionData, - ProvisioningOutWorkMonitor::new, - (workflowLanguage, workflowId, o) -> - new Pair<>( - format.format().name(), - handler.provision(workflowId, p.first(), metadata, o))); + }); + return TaskStarter.launch( + format.format(), handler, ids, labels, workflowRunId, data, metadata); }); } @@ -136,17 +120,24 @@ protected ObjectMapper mapper() { } @Override - protected Stream>> mergeChildren( - Stream>>> stream) { + protected Stream> mergeChildren( + Stream>> stream) { return stream.flatMap(Function.identity()); } @Override - protected Stream>> processChild( + protected Stream> processChild( Map key, String name, OutputType type, JsonNode metadata, JsonNode output) { return type.apply( new PrepareOutputProvisioning( - mapper, target, output, metadata, allInputIds, remainingInputIds, outputIsBad)); + mapper, + target, + output, + metadata, + allInputIds, + remainingInputIds, + outputIsBad, + workflowRunId)); } private Stream stream(JsonNode node, boolean optional) { @@ -157,7 +148,7 @@ private Stream stream(JsonNode node, boolean optional) { } @Override - public Stream>> unknown() { + public Stream> unknown() { throw new IllegalArgumentException(); } } diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/PreparePreflightChecks.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/PreparePreflightChecks.java index b25190bd..812bf411 100644 --- a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/PreparePreflightChecks.java +++ b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/PreparePreflightChecks.java @@ -1,6 +1,5 @@ package ca.on.oicr.gsi.vidarr.core; -import ca.on.oicr.gsi.Pair; import ca.on.oicr.gsi.vidarr.OutputType; import ca.on.oicr.gsi.vidarr.api.ExternalId; import com.fasterxml.jackson.databind.JsonNode; @@ -64,9 +63,7 @@ public Void remaining() { return null; } }); - preflightTask.accept( - (workflowLanguage, workflowId, o) -> - new Pair<>(format.name(), handler.preflightCheck(metadata, o))); + preflightTask.accept(TaskStarter.launch(format, handler, metadata)); return true; } } diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/ProvisionData.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/ProvisionData.java index 5193c0e0..ad0d88ef 100644 --- a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/ProvisionData.java +++ b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/ProvisionData.java @@ -1,36 +1,9 @@ package ca.on.oicr.gsi.vidarr.core; +import ca.on.oicr.gsi.vidarr.OutputProvisioner.Result; import ca.on.oicr.gsi.vidarr.api.ExternalId; import java.util.Map; import java.util.Set; -/** The accessory data that must be stored during output provisioning */ -public final class ProvisionData { - private Set ids; - private Map labels; - - public ProvisionData() { - // Do nothing. - } - - public ProvisionData(Set ids) { - this.ids = ids; - labels = Map.of(); - } - - public Set getIds() { - return ids; - } - - public Map getLabels() { - return labels; - } - - public void setIds(Set ids) { - this.ids = ids; - } - - public void setLabels(Map labels) { - this.labels = labels; - } -} +public record ProvisionData( + Set ids, Map labels, Result result) {} diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/RawInputProvisioner.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/RawInputProvisioner.java index e8c7caad..0ff0c049 100644 --- a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/RawInputProvisioner.java +++ b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/RawInputProvisioner.java @@ -11,7 +11,7 @@ import javax.xml.stream.XMLStreamException; /** Provision input using the stored path as the input path with no alteration */ -public final class RawInputProvisioner extends BaseJsonInputProvisioner { +public final class RawInputProvisioner implements InputProvisioner { private static final ObjectMapper MAPPER = new ObjectMapper(); public static InputProvisionerProvider provider() { @@ -20,9 +20,7 @@ public static InputProvisionerProvider provider() { private Set formats; - public RawInputProvisioner() { - super(MAPPER, String.class, String.class); - } + public RawInputProvisioner() {} @Override public boolean canProvision(InputProvisionFormat format) { @@ -43,32 +41,29 @@ public BasicType externalTypeFor(InputProvisionFormat format) { } } - public Set getFormats() { - return formats; + @Override + public RawInputState provision(WorkflowLanguage language, String id, String path) { + return new RawInputState(JsonNodeFactory.instance.textNode(path)); } @Override - public void startup() { - // Always ok. + public RawInputState provisionExternal(WorkflowLanguage language, JsonNode metadata) { + return new RawInputState(metadata); } @Override - protected String provisionExternal( - WorkflowLanguage language, String metadata, WorkMonitor monitor) { - monitor.scheduleTask(() -> monitor.complete(JsonNodeFactory.instance.textNode(metadata))); - return metadata; + public OperationAction run() { + return OperationAction.load(RawInputState.class, RawInputState::path) + .then(OperationStep.require(JsonNode::isTextual, "Input is not text")); } - @Override - public String provisionRegistered( - WorkflowLanguage language, String id, String path, WorkMonitor monitor) { - monitor.scheduleTask(() -> monitor.complete(JsonNodeFactory.instance.textNode(path))); - return path; + public Set getFormats() { + return formats; } @Override - protected void recover(String state, WorkMonitor monitor) { - monitor.scheduleTask(() -> monitor.complete(JsonNodeFactory.instance.textNode(state))); + public void startup() { + // Always ok. } public void setFormats(Set formats) { diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/RawInputState.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/RawInputState.java new file mode 100644 index 00000000..563818a5 --- /dev/null +++ b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/RawInputState.java @@ -0,0 +1,5 @@ +package ca.on.oicr.gsi.vidarr.core; + +import com.fasterxml.jackson.databind.JsonNode; + +public record RawInputState(JsonNode path) {} diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/RecoveryType.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/RecoveryType.java index 0bf871f2..f1ff8afa 100644 --- a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/RecoveryType.java +++ b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/RecoveryType.java @@ -1,35 +1,28 @@ package ca.on.oicr.gsi.vidarr.core; -import ca.on.oicr.gsi.vidarr.OutputProvisioner; -import ca.on.oicr.gsi.vidarr.OutputProvisioner.Result; -import ca.on.oicr.gsi.vidarr.RuntimeProvisioner; -import ca.on.oicr.gsi.vidarr.core.WrappedMonitor.RecoveryStarter; +import ca.on.oicr.gsi.vidarr.OperationAction; +import ca.on.oicr.gsi.vidarr.OperationAction.Launcher; +import com.fasterxml.jackson.databind.JsonNode; public enum RecoveryType { RECOVER { @Override - RecoveryStarter provisionOut(OutputProvisioner provisioner) { - return provisioner::recover; - } - - @Override - RecoveryStarter runtime(RuntimeProvisioner provisioner) { - return provisioner::recover; + public + Launcher prepare( + OperationAction action, JsonNode state) { + return action.recover(state); } }, RETRY { @Override - RecoveryStarter provisionOut(OutputProvisioner provisioner) { - return provisioner::retry; - } - - @Override - RecoveryStarter runtime(RuntimeProvisioner provisioner) { - return provisioner::retry; + public + Launcher prepare( + OperationAction action, JsonNode state) { + return action.retry(state); } }; - abstract RecoveryStarter provisionOut(OutputProvisioner outputProvisioner); - - abstract RecoveryStarter runtime(RuntimeProvisioner provisioner); + public abstract + Launcher prepare( + OperationAction action, JsonNode state); } diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/RuntimeProvisionState.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/RuntimeProvisionState.java new file mode 100644 index 00000000..ae4951cf --- /dev/null +++ b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/RuntimeProvisionState.java @@ -0,0 +1,9 @@ +package ca.on.oicr.gsi.vidarr.core; + +import ca.on.oicr.gsi.vidarr.api.ExternalId; +import java.util.Map; +import java.util.Set; + +/** The accessory data that must be stored during output provisioning */ +public record RuntimeProvisionState( + Set ids, Map labels, String workflowRunUrl) {} diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/Target.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/Target.java index 299146c7..a98a8b65 100644 --- a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/Target.java +++ b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/Target.java @@ -16,22 +16,22 @@ public interface Target { /** All consumable resources */ Stream> consumableResources(); /** The workflow engine plugin to use */ - WorkflowEngine engine(); + WorkflowEngine engine(); /** * The input provisioner plugins to use * * @param type the format being provisioned in */ - InputProvisioner provisionerFor(InputProvisionFormat type); + InputProvisioner provisionerFor(InputProvisionFormat type); /** * The output provision plugins to use * * @param type the format being provisioned out */ - OutputProvisioner provisionerFor(OutputProvisionFormat type); + OutputProvisioner provisionerFor(OutputProvisionFormat type); /** Any runtime provisioners to use on every workflow run */ - Stream runtimeProvisioners(); + Stream> runtimeProvisioners(); } diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/TaskStarter.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/TaskStarter.java index 60eea444..c24846ea 100644 --- a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/TaskStarter.java +++ b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/TaskStarter.java @@ -1,24 +1,183 @@ package ca.on.oicr.gsi.vidarr.core; import ca.on.oicr.gsi.Pair; -import ca.on.oicr.gsi.vidarr.WorkMonitor; -import ca.on.oicr.gsi.vidarr.WorkflowLanguage; +import ca.on.oicr.gsi.vidarr.ActiveOperation; +import ca.on.oicr.gsi.vidarr.OperationAction; +import ca.on.oicr.gsi.vidarr.OperationAction.Launcher; +import ca.on.oicr.gsi.vidarr.OperationStatefulStep; +import ca.on.oicr.gsi.vidarr.OperationStatefulStep.Child; +import ca.on.oicr.gsi.vidarr.OperationStatus; +import ca.on.oicr.gsi.vidarr.OutputProvisionFormat; +import ca.on.oicr.gsi.vidarr.OutputProvisioner; +import ca.on.oicr.gsi.vidarr.RuntimeProvisioner; +import ca.on.oicr.gsi.vidarr.WorkflowDefinition; +import ca.on.oicr.gsi.vidarr.WorkflowEngine; +import ca.on.oicr.gsi.vidarr.api.ExternalId; +import ca.on.oicr.gsi.vidarr.core.BaseProcessor.TerminalHandler; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.util.Map; +import java.util.Set; /** * Start a task for a workflow run once the matching operation is available * - * @param the return type of the operation + * @param the return type of the operation */ -public interface TaskStarter { - /** - * Start the operation - * - * @param workflowLanguage the workflow language - * @param workflowRunId the workflow run ID - * @param operation the operation that the task will use to complete its result - * @return the initial recovery state to be serialised to the database - */ - Pair start( - WorkflowLanguage workflowLanguage, String workflowRunId, WorkMonitor operation); +interface TaskStarter { + static TaskStarter launch( + RuntimeProvisioner provisioner, + Set ids, + Map labels, + String workflowRunUrl) { + return of( + "$" + provisioner.name(), + wrapRuntimeAction(provisioner, provisioner.run()) + .launch(new RuntimeProvisionState(ids, labels, workflowRunUrl))); + } + + static TaskStarter launch( + OutputProvisionFormat format, + OutputProvisioner provisioner, + Set ids, + Map labels, + String workflowRunId, + String data, + JsonNode metadata) { + + return of( + format.name(), + wrapOutputProvisioner(provisioner, provisioner.run()) + .launch(new OutputProvisionState(ids, labels, workflowRunId, data, metadata))); + } + + static + TaskStarter> launch( + WorkflowDefinition definition, + ActiveWorkflow activeWorkflow, + WorkflowEngine engine, + ObjectNode input) { + return of( + "", + engine + .run() + .map(WorkflowEngine.Result::serialize) + .launch( + engine.start( + definition.language(), + definition.contents(), + definition.accessoryFiles(), + activeWorkflow.id(), + input, + activeWorkflow.engineArguments()))); + } + + static TaskStarter launch( + WorkflowOutputDataType format, + OutputProvisioner provisioner, + JsonNode metadata) { + return of( + format.name(), provisioner.runPreflight().launch(provisioner.preflightCheck(metadata))); + } + + static TaskStarter launchCleanup( + WorkflowEngine engine, JsonNode savedState) { + try { + return of("", engine.cleanup().launch(engine.cleanup().deserializeOriginal(savedState))); + } catch (JsonProcessingException e) { + return new TaskStarter<>() { + @Override + public String name() { + return ""; + } + + @Override + public > void start( + BaseProcessor processor, PO operation, TerminalHandler handler) { + processor.startTransaction( + tx -> { + operation.status(OperationStatus.FAILED, tx); + operation.recoveryState(processor.mapper().valueToTree(e.getMessage()), tx); + }); + handler.failed(); + } + + @Override + public JsonNode state(ObjectMapper mapper) { + return mapper.nullNode(); + } + }; + } + } + + static TaskStarter of( + String name, Launcher launcher) { + return new TaskStarter<>() { + @Override + public String name() { + return name; + } + + @Override + public > void start( + BaseProcessor processor, PO operation, TerminalHandler handler) { + launcher.launch(operation, processor, processor.createNext(operation, handler)); + } + + @Override + public JsonNode state(ObjectMapper mapper) { + return launcher.state(); + } + }; + } + + static Pair toPair(TaskStarter starter, ObjectMapper mapper) { + return new Pair<>(starter.name(), starter.state(mapper)); + } + + static OperationAction wrapOutputProvisioner( + OutputProvisioner provisioner) { + return wrapOutputProvisioner(provisioner, provisioner.run()); + } + + private static + OperationAction, OutputProvisionState, ProvisionData> + wrapOutputProvisioner( + OutputProvisioner provisioner, + OperationAction action) { + return OperationAction.load(OutputProvisionState.class, OutputProvisionState::workflowRunId) + .then( + OperationStatefulStep.subStep( + (state, id) -> provisioner.provision(id, state.data(), state.metadata()), action)) + .map( + (state, result) -> + new ProvisionData(state.state().ids(), state.state().labels(), result)); + } + + private static + OperationAction, RuntimeProvisionState, ProvisionData> + wrapRuntimeAction( + RuntimeProvisioner provisioner, + OperationAction action) { + return OperationAction.load(RuntimeProvisionState.class, RuntimeProvisionState::workflowRunUrl) + .then(OperationStatefulStep.subStep((state, url) -> provisioner.provision(url), action)) + .map( + (state, result) -> + new ProvisionData(state.state().ids(), state.state().labels(), result)); + } + + static + OperationAction wrapRuntimeProvisioner( + RuntimeProvisioner provisioner) { + return wrapRuntimeAction(provisioner, provisioner.run()); + } + + String name(); + + > void start( + BaseProcessor processor, PO operation, TerminalHandler handler); + + JsonNode state(ObjectMapper mapper); } diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/WrappedMonitor.java b/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/WrappedMonitor.java deleted file mode 100644 index 2dbfe0c0..00000000 --- a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/WrappedMonitor.java +++ /dev/null @@ -1,129 +0,0 @@ -package ca.on.oicr.gsi.vidarr.core; - -import ca.on.oicr.gsi.Pair; -import ca.on.oicr.gsi.vidarr.WorkMonitor; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; - -/** - * A worked monitor that adds extra information during state serialisation - * - * @param the type of the added information - * @param the return type of the inner operation - * @param the return type of the whole operation - */ -abstract class WrappedMonitor implements WorkMonitor { - public interface MonitorConstructor { - WrappedMonitor create(A accessory, WorkMonitor monitor); - } - - public interface RecoveryStarter { - void recover(JsonNode state, WorkMonitor monitor); - } - - /** - * Restart a task from stored database state - * - * @param state the state provided from the database - * @param accessoryLoader a function to deserialise the database state - * @param constructor a constructor for the monitor - * @param task a method to restart a task - * @param operation the outer operation - * @param the type of the added information - * @param the return type of the inner operation - * @param the return type of the whole operation - */ - public static void recover( - JsonNode state, - Function accessoryLoader, - MonitorConstructor constructor, - RecoveryStarter task, - WorkMonitor operation) { - task.recover(state.get(1), constructor.create(accessoryLoader.apply(state.get(0)), operation)); - } - - /** - * Modify a task to include extra information - * - * @param accessory the extra information to store - * @param constructor a constructor for the monitor - * @param task the task to wrap - * @param the type of the added information - * @param the return type of the inner operation - * @param the return type of the whole operation - * @return a modified task that captures the accessory data - */ - public static TaskStarter start( - A accessory, MonitorConstructor constructor, TaskStarter task) { - return (workflowLanguage, workflowId, operation) -> { - final var result = - task.start(workflowLanguage, workflowId, constructor.create(accessory, operation)); - final var array = JsonNodeFactory.instance.arrayNode(2); - array.insertPOJO(0, accessory); - array.insertPOJO(1, result.second()); - return new Pair<>(result.first(), array); - }; - } - - private final A accessory; - private final WorkMonitor monitor; - - public WrappedMonitor(A accessory, WorkMonitor monitor) { - this.accessory = accessory; - this.monitor = monitor; - } - - @Override - public final void complete(R result) { - monitor.complete(mix(accessory, result)); - } - - @Override - public void log(System.Logger.Level level, String message) { - monitor.log(level, message); - } - - /** - * Combine the accessory information and the result from the operation to produce a new output - * - * @param accessory the accessory information carried along - * @param result the result provided by the inner operation - * @return the result to be provided to the outer operation - */ - protected abstract S mix(A accessory, R result); - - @Override - public final void permanentFailure(String reason) { - monitor.permanentFailure(reason); - } - - @Override - public final void scheduleTask(long delay, TimeUnit units, Runnable task) { - monitor.scheduleTask(delay, units, task); - } - - @Override - public final void scheduleTask(Runnable task) { - monitor.scheduleTask(task); - } - - @Override - public void storeDebugInfo(JsonNode information) { - monitor.storeDebugInfo(information); - } - - @Override - public final void storeRecoveryInformation(JsonNode state) { - final var array = JsonNodeFactory.instance.arrayNode(2); - array.insertPOJO(0, accessory); - array.insertPOJO(1, state); - monitor.storeRecoveryInformation(array); - } - - @Override - public final void updateState(Status status) { - monitor.updateState(status); - } -} diff --git a/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/CleanupState.java b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/CleanupState.java new file mode 100644 index 00000000..5b1dd4ce --- /dev/null +++ b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/CleanupState.java @@ -0,0 +1,5 @@ +package ca.on.oicr.gsi.vidarr.cromwell; + +public record CleanupState() { + +} diff --git a/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/CromwellMetadataURL.java b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/CromwellMetadataURL.java index 4b98868f..22b495dd 100644 --- a/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/CromwellMetadataURL.java +++ b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/CromwellMetadataURL.java @@ -10,7 +10,7 @@ * WorkflowEngine and OutputProvisioner use configuration to control this behaviour. */ final class CromwellMetadataURL { - protected static URI formatMetadataURL(String rootUrl, String cromwellId, Boolean includeCalls) { + protected static URI formatMetadataURL(String rootUrl, String cromwellId, boolean includeCalls) { String metadataAPIformat = "%s/api/workflows/v1/%s/metadata"; // Note that the URL specifies to *exclude* keys, hence the inverted bool test diff --git a/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/CromwellOutputProvisioner.java b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/CromwellOutputProvisioner.java index e4a8b18f..6ed9c71e 100644 --- a/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/CromwellOutputProvisioner.java +++ b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/CromwellOutputProvisioner.java @@ -1,18 +1,39 @@ package ca.on.oicr.gsi.vidarr.cromwell; -import static ca.on.oicr.gsi.vidarr.cromwell.CromwellWorkflowEngine.*; +import static ca.on.oicr.gsi.vidarr.OperationAction.load; +import static ca.on.oicr.gsi.vidarr.OperationStatefulStep.log; +import static ca.on.oicr.gsi.vidarr.OperationStatefulStep.onInnerState; +import static ca.on.oicr.gsi.vidarr.OperationStatefulStep.poll; +import static ca.on.oicr.gsi.vidarr.OperationStatefulStep.repeatUntilSuccess; +import static ca.on.oicr.gsi.vidarr.OperationStep.debugInfo; +import static ca.on.oicr.gsi.vidarr.OperationStep.http; +import static ca.on.oicr.gsi.vidarr.OperationStep.log; +import static ca.on.oicr.gsi.vidarr.OperationStep.monitorWhen; +import static ca.on.oicr.gsi.vidarr.OperationStep.requireJsonSuccess; +import static ca.on.oicr.gsi.vidarr.OperationStep.requirePresent; +import static ca.on.oicr.gsi.vidarr.OperationStep.status; +import static ca.on.oicr.gsi.vidarr.cromwell.CromwellWorkflowEngine.CROMWELL_FAILURES; +import static ca.on.oicr.gsi.vidarr.cromwell.CromwellWorkflowEngine.MAPPER; +import static ca.on.oicr.gsi.vidarr.cromwell.CromwellWorkflowEngine.statusFromCromwell; import ca.on.oicr.gsi.Pair; import ca.on.oicr.gsi.status.SectionRenderer; -import ca.on.oicr.gsi.vidarr.*; +import ca.on.oicr.gsi.vidarr.BasicType; +import ca.on.oicr.gsi.vidarr.JsonBodyHandler; +import ca.on.oicr.gsi.vidarr.OperationAction; +import ca.on.oicr.gsi.vidarr.OperationStatefulStep; +import ca.on.oicr.gsi.vidarr.OperationStatefulStep.Child; +import ca.on.oicr.gsi.vidarr.OperationStatefulStep.RepeatCounter; +import ca.on.oicr.gsi.vidarr.OutputProvisionFormat; +import ca.on.oicr.gsi.vidarr.OutputProvisioner; +import ca.on.oicr.gsi.vidarr.OutputProvisionerProvider; +import ca.on.oicr.gsi.vidarr.WorkingStatus; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import java.net.URI; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.nio.file.Path; +import java.lang.System.Logger.Level; import java.time.Duration; -import java.util.*; -import java.util.concurrent.TimeUnit; +import java.util.List; +import java.util.Optional; import java.util.stream.Stream; /** @@ -20,9 +41,8 @@ * workflow */ public class CromwellOutputProvisioner - extends BaseJsonOutputProvisioner { - private static final int CHECK_DELAY = 1; - private static final List> EXTENSION_TO_META_TYPE = + implements OutputProvisioner { + static final List> EXTENSION_TO_META_TYPE = List.of( new Pair<>(".bam", "application/bam"), new Pair<>(".bai", "application/bam-index"), @@ -62,6 +82,7 @@ public static OutputProvisionerProvider provider() { private int[] chunks; private String cromwellUrl; + private boolean debugCalls; private String fileField; private String fileSizeField; private String md5Field; @@ -71,142 +92,14 @@ public static OutputProvisionerProvider provider() { private ObjectNode workflowOptions = MAPPER.createObjectNode(); private String workflowSource; private String workflowUrl; - private Boolean debugCalls; - public CromwellOutputProvisioner() { - super(MAPPER, ProvisionState.class, Void.class, OutputMetadata.class); - } + public CromwellOutputProvisioner() {} @Override public boolean canProvision(OutputProvisionFormat format) { return format == OutputProvisionFormat.FILES; } - private void check( - ProvisionState state, WorkMonitor monitor) { - try { - monitor.log( - System.Logger.Level.INFO, - String.format("Checking Cromwell job %s on %s", state.getCromwellId(), cromwellUrl)); - CROMWELL_REQUESTS.labels(cromwellUrl).inc(); - CLIENT - .sendAsync( - HttpRequest.newBuilder() - .uri( - CromwellMetadataURL.formatMetadataURL( - cromwellUrl, state.getCromwellId(), debugCalls)) - .timeout(Duration.ofMinutes(1)) - .GET() - .build(), - new JsonBodyHandler<>(MAPPER, WorkflowMetadataResponse.class)) - .thenApply(HttpResponse::body) - .thenAccept( - s -> { - final var result = s.get(); - monitor.log( - System.Logger.Level.INFO, - String.format( - "Cromwell job %s on %s is in state %s", - state.getCromwellId(), cromwellUrl, result.getStatus())); - monitor.storeDebugInfo(result.debugInfo()); - switch (result.getStatus()) { - // In the case of failures ("Aborted" or "Failed"), request the full metadata - // from Cromwell if we don't already have it - // so we can have call info for debugging. - case "Aborted": - case "Failed": - if (debugCalls) { - monitor.log( - System.Logger.Level.INFO, - String.format( - "Cromwell job %s is failed. Cromwell OutputProvisioner " - + "is configured to have already fetched calls info. Skipping " - + "second request.", - state.getCromwellId())); - monitor.permanentFailure("Cromwell failure: " + result.getStatus()); - break; - } - monitor.log( - System.Logger.Level.INFO, - String.format( - "Cromwell job %s is failed, fetching call info on %s", - state.getCromwellId(), cromwellUrl)); - CROMWELL_REQUESTS.labels(cromwellUrl).inc(); - - CLIENT - .sendAsync( - HttpRequest.newBuilder() - .uri( - CromwellMetadataURL.formatMetadataURL( - cromwellUrl, state.getCromwellId(), true)) - .timeout(Duration.ofMinutes(1)) - .GET() - .build(), - new JsonBodyHandler<>(MAPPER, WorkflowMetadataResponse.class)) - .thenApply(HttpResponse::body) - .thenAccept( - s2 -> { - final var fullResult = s2.get(); - monitor.log( - System.Logger.Level.INFO, - String.format( - "Successfully fetched full metadata for Cromwell job %s on %s", - state.getCromwellId(), cromwellUrl)); - monitor.storeDebugInfo(fullResult.debugInfo()); - monitor.permanentFailure("Cromwell failure: " + result.getStatus()); - }) - .exceptionally( - t2 -> { - t2.printStackTrace(); - monitor.log( - System.Logger.Level.WARNING, - String.format( - "Failed to get Cromwell job %s on %s due to %s", - state.getCromwellId(), cromwellUrl, t2.getMessage())); - CROMWELL_FAILURES.labels(cromwellUrl).inc(); - - // TODO: this may schedule 2 requests to cromwell /metadata now. - // Consider - // a failure-unique check - monitor.scheduleTask( - CHECK_DELAY, TimeUnit.MINUTES, () -> check(state, monitor)); - return null; - }); - - break; - case "Succeeded": - finish(state, monitor); - break; - default: - monitor.updateState(statusFromCromwell(result.getStatus())); - monitor.scheduleTask( - CHECK_DELAY, TimeUnit.MINUTES, () -> check(state, monitor)); - } - }) - .exceptionally( - t -> { - t.printStackTrace(); - monitor.log( - System.Logger.Level.WARNING, - String.format( - "Failed to get Cromwell job %s on %s due to %s", - state.getCromwellId(), cromwellUrl, t.getMessage())); - CROMWELL_FAILURES.labels(cromwellUrl).inc(); - monitor.scheduleTask(CHECK_DELAY, TimeUnit.MINUTES, () -> check(state, monitor)); - return null; - }); - } catch (Exception e) { - e.printStackTrace(); - monitor.log( - System.Logger.Level.WARNING, - String.format( - "Failed to get Cromwell job %s on %s due to %s", - state.getCromwellId(), cromwellUrl, e.getMessage())); - CROMWELL_FAILURES.labels(cromwellUrl).inc(); - monitor.scheduleTask(CHECK_DELAY, TimeUnit.MINUTES, () -> check(state, monitor)); - } - } - @Override public void configuration(SectionRenderer sectionRenderer) { sectionRenderer.line("Input Parameter for File Path", fileField); @@ -222,51 +115,17 @@ public void configuration(SectionRenderer sectionRenderer) { } } - private void finish( - ProvisionState state, WorkMonitor monitor) { - monitor.log( - System.Logger.Level.INFO, - String.format( - "Reaping results of Cromwell job %s on %s", state.getCromwellId(), cromwellUrl)); - CROMWELL_REQUESTS.labels(cromwellUrl).inc(); - CLIENT - .sendAsync( - HttpRequest.newBuilder() - .uri( - URI.create( - String.format( - "%s/api/workflows/v1/%s/outputs", cromwellUrl, state.getCromwellId()))) - .timeout(Duration.ofMinutes(1)) - .GET() - .build(), - new JsonBodyHandler<>(MAPPER, WorkflowOutputResponse.class)) - .thenApply(HttpResponse::body) - .thenAccept( - s -> { - final var result = s.get(); - monitor.log( - System.Logger.Level.INFO, - String.format( - "Got results of Cromwell job %s on %s", state.getCromwellId(), cromwellUrl)); - monitor.complete( - Result.file( - result.getOutputs().get(storagePathField).asText(), - result.getOutputs().get(md5Field).asText(), - Long.parseLong(result.getOutputs().get(fileSizeField).asText()), - state.getMetaType())); - }) - .exceptionally( - t -> { - monitor.log( - System.Logger.Level.WARNING, - String.format( - "Failed to get results of Cromwell job %s on %s", - state.getCromwellId(), cromwellUrl)); - t.printStackTrace(); - CROMWELL_FAILURES.labels(cromwellUrl).inc(); - monitor.scheduleTask(CHECK_DELAY, TimeUnit.MINUTES, () -> check(state, monitor)); - return null; - }); + private OutputProvisioner.Result extractOutput( + Child, ?> state, WorkflowOutputResponse result) { + return Result.file( + result.getOutputs().get(storagePathField).asText(), + result.getOutputs().get(md5Field).asText(), + Long.parseLong(result.getOutputs().get(fileSizeField).asText()), + EXTENSION_TO_META_TYPE.stream() + .filter(p -> state.state().state().fileName().endsWith(p.first())) + .findFirst() + .map(Pair::second) + .orElseThrow()); } public int[] getChunks() { @@ -314,149 +173,62 @@ public String getWorkflowUrl() { } @Override - protected Void preflightCheck( - OutputMetadata metadata, WorkMonitor monitor) { - monitor.scheduleTask(() -> monitor.complete(true)); - return null; - } - - @Override - protected void preflightRecover(Void state, WorkMonitor monitor) { - monitor.scheduleTask(() -> monitor.complete(true)); + public PreflightState preflightCheck(JsonNode metadata) { + return new PreflightState(); } @Override - protected ProvisionState provision( - String workflowId, - String data, - OutputMetadata metadata, - WorkMonitor monitor) { - final var state = new ProvisionState(); - state.setFileName(data); - state.setVidarrId(workflowId); - var path = Path.of(metadata.getOutputDirectory()); - int startIndex = 0; - for (final var length : chunks) { - if (length < 1) { - break; - } - final var endIndex = Math.min(workflowId.length(), startIndex + length); - if (endIndex == startIndex) { - break; - } - path = path.resolve(workflowId.substring(startIndex, endIndex)); - startIndex = endIndex; - } - - state.setOutputPrefix(path.resolve(workflowId).toString()); - - state.setMetaType( - EXTENSION_TO_META_TYPE.stream() - .filter(p -> data.endsWith(p.first())) - .findFirst() - .map(Pair::second) - .orElseThrow()); - recover(state, monitor); - return state; + public ProvisionState provision(String workflowRunId, String data, JsonNode metadata) { + return new ProvisionState(cromwellUrl, data, metadata, workflowRunId); } @Override - protected void recover(ProvisionState state, WorkMonitor monitor) { - if (state.getCromwellId() == null) { - monitor.scheduleTask( - () -> { - try { - monitor.log( - System.Logger.Level.INFO, - String.format( - "Launching provisioning out job on Cromwell %s for %s", - cromwellUrl, state.getFileName())); - final var body = - new MultiPartBodyPublisher() - .addPart( - workflowUrl == null ? "workflowSource" : "workflowUrl", - workflowUrl == null ? workflowSource : workflowUrl) - .addPart( - "labels", - MAPPER.writeValueAsString( - Collections.singletonMap( - "vidarr-id", - state - .getVidarrId() - .substring(Math.max(0, state.getVidarrId().length() - 255))))) - .addPart( - "workflowInputs", - MAPPER.writeValueAsString( - Map.of( - fileField, - state.getFileName(), - outputPrefixField, - state.getOutputPrefix()))) - .addPart("workflowOptions", MAPPER.writeValueAsString(workflowOptions)) - .addPart("workflowType", "WDL") - .addPart("workflowTypeVersion", wdlVersion); - CROMWELL_REQUESTS.labels(cromwellUrl).inc(); - CLIENT - .sendAsync( - HttpRequest.newBuilder() - .uri(URI.create(String.format("%s/api/workflows/v1", cromwellUrl))) - .timeout(Duration.ofMinutes(1)) - .header("Content-Type", body.getContentType()) - .POST(body.build()) - .build(), - new JsonBodyHandler<>(MAPPER, WorkflowStatusResponse.class)) - .thenApply(HttpResponse::body) - .thenAccept( - s -> { - final var result = s.get(); - if (result.getId() == null) { - monitor.permanentFailure("Cromwell to launch workflow."); - return; - } - state.setCromwellId(result.getId()); - monitor.storeRecoveryInformation(state); - monitor.updateState(statusFromCromwell(result.getStatus())); - monitor.scheduleTask( - CHECK_DELAY, TimeUnit.MINUTES, () -> check(state, monitor)); - monitor.log( - System.Logger.Level.INFO, - String.format( - "Provisioning for %s on Cromwell %s is %s", - state.getFileName(), cromwellUrl, result.getId())); - }) - .exceptionally( - t -> { - monitor.log( - System.Logger.Level.WARNING, - String.format( - "Failed to launch provisioning out job on Cromwell %s for %s", - cromwellUrl, state.getFileName())); - t.printStackTrace(); - CROMWELL_FAILURES.labels(cromwellUrl).inc(); - - // Call recover() rather than check(): recover() starts with a check for - // a null cromwell ID. There's a chance Exception t is 'header parser - // received no bytes', or another case where we don't have a cromwell id. - // Prevents looping 'Checking Cromwell job null'. recover() calls check() - // if a cromwell id is present. - monitor.scheduleTask( - CHECK_DELAY, TimeUnit.MINUTES, () -> recover(state, monitor)); - return null; - }); - } catch (Exception e) { - CROMWELL_FAILURES.labels(cromwellUrl).inc(); - monitor.permanentFailure(e.toString()); - } - }); - } else { - check(state, monitor); - } + public OperationAction run() { + return load(ProvisionState.class, (state) -> state.buildLaunchRequest(this)) + .then(http(new JsonBodyHandler<>(MAPPER, WorkflowStatusResponse.class))) + .then( + log( + Level.INFO, + (response) -> + String.format("Got response %d on %s", response.statusCode(), cromwellUrl))) + .then(monitorWhen(CROMWELL_FAILURES, result -> result.statusCode() / 100 != 2, cromwellUrl)) + .then(requireJsonSuccess()) + .map(result -> Optional.ofNullable(result.getId()).filter(id -> !id.equals("null"))) + .then(requirePresent()) + .then(status(WorkingStatus.QUEUED)) + .then( + log( + Level.INFO, + id -> String.format("Started Cromwell provision-out %s on %s", id, cromwellUrl))) + .then(repeatUntilSuccess(Duration.ofMinutes(10), 5)) + .then( + OperationStatefulStep.subStep( + onInnerState(ProvisionState.class, ProvisionState::checkTask), + load(StateStarted.class, (state) -> state.buildCheckRequest(debugCalls)) + .then(http(new JsonBodyHandler<>(MAPPER, WorkflowMetadataResponse.class))) + .then(requireJsonSuccess()) + .then(debugInfo(WorkflowMetadataResponse::debugInfo)) + .then( + log( + Level.INFO, + (state, response) -> + String.format( + "Status of Cromwell provision-out %s on %s: %s", + state.cromwellId(), + state.cromwellServer(), + response.getStatus()))) + .then(status(response -> statusFromCromwell(response.getStatus()))) + .map(WorkflowMetadataResponse::pollStatus) + .then(poll(Duration.ofMinutes(5))) + .reload(StateStarted::buildOutputsRequest) + .then(http(new JsonBodyHandler<>(MAPPER, WorkflowOutputResponse.class))) + .then(requireJsonSuccess()))) + .map(this::extractOutput); } @Override - protected void retry(ProvisionState state, WorkMonitor monitor) { - state.setCromwellId(null); - recover(state, monitor); + public OperationAction runPreflight() { + return OperationAction.value(PreflightState.class, true); } public void setChunks(int[] chunks) { @@ -467,6 +239,10 @@ public void setCromwellUrl(String cromwellUrl) { this.cromwellUrl = cromwellUrl; } + public void setDebugCalls(boolean debugCalls) { + this.debugCalls = debugCalls; + } + public void setFileField(String fileField) { this.fileField = fileField; } @@ -525,8 +301,4 @@ public BasicType typeFor(OutputProvisionFormat format) { throw new IllegalArgumentException("Cannot provision non-file output"); } } - - public void setDebugCalls(Boolean debugCalls) { - this.debugCalls = debugCalls; - } } diff --git a/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/CromwellWorkflowEngine.java b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/CromwellWorkflowEngine.java index 2892709b..f9e174e5 100644 --- a/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/CromwellWorkflowEngine.java +++ b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/CromwellWorkflowEngine.java @@ -1,38 +1,44 @@ package ca.on.oicr.gsi.vidarr.cromwell; +import static ca.on.oicr.gsi.vidarr.OperationAction.load; +import static ca.on.oicr.gsi.vidarr.OperationAction.value; +import static ca.on.oicr.gsi.vidarr.OperationStatefulStep.log; +import static ca.on.oicr.gsi.vidarr.OperationStatefulStep.onInnerState; +import static ca.on.oicr.gsi.vidarr.OperationStatefulStep.poll; +import static ca.on.oicr.gsi.vidarr.OperationStatefulStep.repeatUntilSuccess; +import static ca.on.oicr.gsi.vidarr.OperationStatefulStep.subStep; +import static ca.on.oicr.gsi.vidarr.OperationStep.debugInfo; +import static ca.on.oicr.gsi.vidarr.OperationStep.http; +import static ca.on.oicr.gsi.vidarr.OperationStep.monitorWhen; +import static ca.on.oicr.gsi.vidarr.OperationStep.requireJsonSuccess; +import static ca.on.oicr.gsi.vidarr.OperationStep.requirePresent; +import static ca.on.oicr.gsi.vidarr.OperationStep.status; + import ca.on.oicr.gsi.Pair; import ca.on.oicr.gsi.status.SectionRenderer; -import ca.on.oicr.gsi.vidarr.*; -import com.fasterxml.jackson.core.JsonProcessingException; +import ca.on.oicr.gsi.vidarr.BasicType; +import ca.on.oicr.gsi.vidarr.JsonBodyHandler; +import ca.on.oicr.gsi.vidarr.OperationAction; +import ca.on.oicr.gsi.vidarr.OperationStep; +import ca.on.oicr.gsi.vidarr.WorkflowEngine; +import ca.on.oicr.gsi.vidarr.WorkflowEngineProvider; +import ca.on.oicr.gsi.vidarr.WorkflowLanguage; +import ca.on.oicr.gsi.vidarr.WorkingStatus; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import io.prometheus.client.Counter; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.net.URI; +import java.lang.System.Logger.Level; import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.net.http.HttpResponse.BodyHandlers; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; import java.util.Map; import java.util.Optional; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; import javax.xml.stream.XMLStreamException; /** Run workflows using Cromwell */ -public final class CromwellWorkflowEngine - extends BaseJsonWorkflowEngine { +public final class CromwellWorkflowEngine implements WorkflowEngine { private static final int CHECK_DELAY = 1; static final HttpClient CLIENT = HttpClient.newBuilder() @@ -54,161 +60,28 @@ public final class CromwellWorkflowEngine .register(); static final ObjectMapper MAPPER = new ObjectMapper(); - private static Stream findAllParents(String file) { - final var parents = new ArrayList(); - for (var path = Path.of(file).getParent(); path != null; path = path.getParent()) { - parents.add(path); - } - return parents.stream(); - } - public static WorkflowEngineProvider provider() { return () -> Stream.of(new Pair<>("cromwell", CromwellWorkflowEngine.class)); } - static WorkMonitor.Status statusFromCromwell(String status) { + static WorkingStatus statusFromCromwell(String status) { return switch (status) { - case "On Hold" -> WorkMonitor.Status.WAITING; - case "Submitted" -> WorkMonitor.Status.QUEUED; - case "Running" -> WorkMonitor.Status.RUNNING; - default -> WorkMonitor.Status.UNKNOWN; + case "On Hold" -> WorkingStatus.WAITING; + case "Submitted" -> WorkingStatus.QUEUED; + case "Running" -> WorkingStatus.RUNNING; + default -> WorkingStatus.UNKNOWN; }; } + private boolean debugInflightRuns; private Map engineParameters; private String url; - // TODO Optimally, this would be Optional - private Boolean debugInflightRuns; - - public CromwellWorkflowEngine() { - super(MAPPER, EngineState.class, String.class, Void.class); - } - - private void check(EngineState state, WorkMonitor, EngineState> monitor) { - try { - monitor.log( - System.Logger.Level.INFO, - String.format("Checking Cromwell workflow %s on %s", state.getCromwellId(), url)); - CROMWELL_REQUESTS.labels(url).inc(); - CLIENT - .sendAsync( - HttpRequest.newBuilder() - .uri( - CromwellMetadataURL.formatMetadataURL( - url, state.getCromwellId(), debugInflightRuns)) - .timeout(Duration.ofMinutes(1)) - .GET() - .build(), - new JsonBodyHandler<>(MAPPER, WorkflowMetadataResponse.class)) - .thenApply(HttpResponse::body) - .thenAccept( - s -> { - final var result = s.get(); - monitor.log( - System.Logger.Level.INFO, - String.format( - "Status for Cromwell workflow %s on %s is %s", - state.getCromwellId(), url, result.getStatus())); - monitor.storeDebugInfo(result.debugInfo()); - switch (result.getStatus()) { - // In the case of failures ("Aborted" or "Failed"), request the full metadata - // from Cromwell if we don't already have it - // so we can have call info for debugging. - case "Aborted": - case "Failed": - if (debugInflightRuns) { - monitor.log( - System.Logger.Level.INFO, - String.format("Cromwell job %s is failed. Cromwell WorkflowEngine is " - + "configured to have already fetched calls info. Skipping second " - + "request.", state.getCromwellId()) - ); - monitor.permanentFailure("Cromwell failure: " + result.getStatus()); - break; - } - monitor.log( - System.Logger.Level.INFO, - String.format( - "Cromwell job %s is failed, fetching call info on %s", - state.getCromwellId(), url)); - CROMWELL_REQUESTS.labels(url).inc(); - - CLIENT - .sendAsync( - HttpRequest.newBuilder() - .uri( - CromwellMetadataURL.formatMetadataURL( - url, state.getCromwellId(), true)) - .timeout(Duration.ofMinutes(1)) - .GET() - .build(), - new JsonBodyHandler<>(MAPPER, WorkflowMetadataResponse.class)) - .thenApply(HttpResponse::body) - .thenAccept( - s2 -> { - final var fullResult = s2.get(); - monitor.log( - System.Logger.Level.INFO, - String.format( - "Successfully fetched full metadata for Cromwell job %s on %s", - state.getCromwellId(), url)); - monitor.storeDebugInfo(fullResult.debugInfo()); - monitor.permanentFailure("Cromwell failure: " + result.getStatus()); - }) - .exceptionally( - t2 -> { - t2.printStackTrace(); - monitor.log( - System.Logger.Level.WARNING, - String.format( - "Failed to get Cromwell job %s on %s due to %s", - state.getCromwellId(), url, t2.getMessage())); - CROMWELL_FAILURES.labels(url).inc(); - - // TODO: this schedules 2 requests to cromwell /metadata now. Consider - // a failure-unique check - monitor.scheduleTask( - CHECK_DELAY, TimeUnit.MINUTES, () -> check(state, monitor)); - return null; - }); - break; - case "Succeeded": - finish(state, monitor); - break; - default: - monitor.updateState(statusFromCromwell(result.getStatus())); - monitor.scheduleTask(5, TimeUnit.MINUTES, () -> check(state, monitor)); - } - }) - .exceptionally( - t -> { - t.printStackTrace(); - monitor.log( - System.Logger.Level.WARNING, - String.format( - "Failed to get status for Cromwell workflow %s on %s due to %s", - state.getCromwellId(), url, t.getMessage())); - CROMWELL_FAILURES.labels(url).inc(); - monitor.scheduleTask(5, TimeUnit.MINUTES, () -> check(state, monitor)); - return null; - }); - } catch (Exception e) { - e.printStackTrace(); - monitor.log( - System.Logger.Level.WARNING, - String.format( - "Failed to get status for Cromwell workflow %s on %s due to %s", - state.getCromwellId(), url, e.getMessage())); - CROMWELL_FAILURES.labels(url).inc(); - monitor.scheduleTask(5, TimeUnit.MINUTES, () -> check(state, monitor)); - } - } + public CromwellWorkflowEngine() {} @Override - protected Void cleanup(String cleanupState, WorkMonitor monitor) { - monitor.scheduleTask(() -> monitor.complete(null)); - return null; + public OperationAction cleanup() { + return value(CleanupState.class, null); } @Override @@ -225,86 +98,87 @@ public Optional engineParameters() { fields.entrySet().stream().map(e -> new Pair<>(e.getKey(), e.getValue())))); } - private void finish(EngineState state, WorkMonitor, EngineState> monitor) { - CROMWELL_REQUESTS.labels(url).inc(); - monitor.log( - System.Logger.Level.INFO, - String.format("Reaping output of Cromwell workflow %s on %s", state.getCromwellId(), url)); - CLIENT - .sendAsync( - HttpRequest.newBuilder() - .uri( - URI.create( - String.format( - "%s/api/workflows/v1/%s/outputs", url, state.getCromwellId()))) - .timeout(Duration.ofMinutes(1)) - .GET() - .build(), - new JsonBodyHandler<>(MAPPER, WorkflowOutputResponse.class)) - .thenApply(HttpResponse::body) - .thenAccept( - s -> { - final var result = s.get(); - monitor.log( - System.Logger.Level.INFO, - String.format( - "Got output of Cromwell workflow %s on %s", state.getCromwellId(), url)); - monitor.complete( - new Result<>( - result.getOutputs(), - // Note: This instance of the cromwell URL is for use by OutputProvisioners - // Don't include 'excludeKeys', includeCalls needs to be true - CromwellMetadataURL.formatMetadataURL(url, state.getCromwellId(), true) - .toString(), - Optional.empty())); - }) - .exceptionally( - t -> { - t.printStackTrace(); - monitor.log( - System.Logger.Level.INFO, - String.format( - "Failed to get output of Cromwell workflow %s on %s", - state.getCromwellId(), url)); - CROMWELL_FAILURES.labels(url).inc(); - monitor.scheduleTask(CHECK_DELAY, TimeUnit.MINUTES, () -> check(state, monitor)); - return null; - }); - } - public String getUrl() { return url; } @Override - protected void recover(EngineState state, WorkMonitor, EngineState> monitor) { - if (state.getCromwellId() == null) { - monitor.scheduleTask(() -> startTask(state, monitor)); - } else { - check(state, monitor); - } + public OperationAction> run() { + return load(StateUnstarted.class, StateUnstarted::buildLaunchRequest) + .then(http(new JsonBodyHandler<>(MAPPER, WorkflowStatusResponse.class))) + .then( + log( + Level.INFO, + (state, response) -> + String.format( + "Got response %d on %s", response.statusCode(), state.cromwellServer()))) + .then(monitorWhen(CROMWELL_FAILURES, OperationStep::isHttpOk, url)) + .then(requireJsonSuccess()) + .map(result -> Optional.ofNullable(result.getId()).filter(id -> !id.equals("null"))) + .then(requirePresent()) + .then(status(WorkingStatus.QUEUED)) + .then( + log( + Level.INFO, + (state, id) -> + String.format( + "Started Cromwell workflow %s on %s", id, state.cromwellServer()))) + .then(repeatUntilSuccess(Duration.ofMinutes(10), 5)) + .then( + subStep( + onInnerState(StateUnstarted.class, StateUnstarted::checkTask), + load(StateStarted.class, (state) -> state.buildCheckRequest(debugInflightRuns)) + .then(http(new JsonBodyHandler<>(MAPPER, WorkflowMetadataResponse.class))) + .then(requireJsonSuccess()) + .then(debugInfo(WorkflowMetadataResponse::debugInfo)) + .then( + log( + Level.INFO, + (state, response) -> + String.format( + "Status of Cromwell workflow %s on %s: %s", + state.cromwellId(), + state.cromwellServer(), + response.getStatus()))) + .then(status(response -> statusFromCromwell(response.getStatus()))) + .map(WorkflowMetadataResponse::pollStatus) + .then(poll(Duration.ofMinutes(5))) + .reload(StateStarted::buildOutputsRequest) + .then(http(new JsonBodyHandler<>(MAPPER, WorkflowOutputResponse.class))) + .then(requireJsonSuccess()) + .map( + (state, output) -> + new Result<>( + output.getOutputs(), + state.runtimeProvisionerUrl(), + Optional.empty())))); + } + + public void setDebugInflightRuns(boolean debugInflightRuns) { + this.debugInflightRuns = debugInflightRuns; } - @Override - protected void recoverCleanup(Void state, WorkMonitor monitor) { - monitor.complete(null); + public void setEngineParameters(Map engineParameters) { + this.engineParameters = engineParameters; + } + + public void setUrl(String url) { + this.url = url; } @Override - protected EngineState runWorkflow( + public StateUnstarted start( WorkflowLanguage workflowLanguage, String workflow, Stream> accessoryFiles, String vidarrId, ObjectNode workflowParameters, - JsonNode engineParameters, - WorkMonitor, EngineState> monitor) { - final var state = new EngineState(); + JsonNode engineParameters) { /* Cromwell and Shesmu/Vidarr handle optional parameters differently. * Shesmu/Vidarr encoding missing values as "null", but Cromwell encodes * them as absent. If a value is "null", Cromwell will complain that it * can't be parsed rather than drop the null and use its default. - * Therefore, we delete all the nulls so we have something Cromwell will + * Therefore, we delete all the nulls so that we have something Cromwell will * accept. There's no situation where Cromwell would _need_ to see a null. */ final var filteredParameters = MAPPER.createObjectNode(); @@ -315,134 +189,15 @@ protected EngineState runWorkflow( filteredParameters.set(field.getKey(), field.getValue()); } } - state.setWorkflowInputFiles( - accessoryFiles.collect(Collectors.toMap(Pair::first, Pair::second))); - state.setEngineParameters(engineParameters); - state.setParameters(filteredParameters); - state.setVidarrId(vidarrId); - state.setWorkflowLanguage(workflowLanguage); - state.setWorkflowSource(workflow); - monitor.scheduleTask(() -> startTask(state, monitor)); - return state; - } - public void setEngineParameters(Map engineParameters) { - this.engineParameters = engineParameters; - } - - public void setUrl(String url) { - this.url = url; - } - - private void startTask(EngineState state, WorkMonitor, EngineState> monitor) { - try { - monitor.log(System.Logger.Level.INFO, String.format("Starting Cromwell workflow on %s", url)); - final var body = - new MultiPartBodyPublisher() - .addPart("workflowSource", state.getWorkflowSource()) - .addPart("workflowInputs", MAPPER.writeValueAsString(state.getParameters())) - .addPart("workflowType", "WDL") - .addPart( - "workflowTypeVersion", - switch (state.getWorkflowLanguage()) { - case WDL_1_0 -> "1.0"; - case WDL_1_1 -> "1.1"; - default -> "draft1"; - }) - .addPart( - "labels", - MAPPER.writeValueAsString( - Collections.singletonMap( - "vidarr-id", - state - .getVidarrId() - .substring(Math.max(0, state.getVidarrId().length() - 255))))) - .addPart("workflowOptions", MAPPER.writeValueAsString(state.getEngineParameters())); - if (!state.getWorkflowInputFiles().isEmpty()) { - // Cromwell doesn't deduplicate these and stores them all in its database, so it doesn't - // matter if we make the effort to ensure these ZIP files are byte-for-byte identical. - final var zipOutput = new ByteArrayOutputStream(); - try (final var zipFile = new ZipOutputStream(zipOutput)) { - - // We have to create all the parent directories or the better-files compressor that - // Cromwell uses will fail to decompress. A directory entry is one that ends with a / and - // has no data. - final var parentDirectories = - state.getWorkflowInputFiles().keySet().stream() - .flatMap(CromwellWorkflowEngine::findAllParents) - .distinct() - .sorted(Comparator.comparing(Path::getNameCount)) - .collect(Collectors.toList()); - for (final var parentDirectory : parentDirectories) { - zipFile.putNextEntry(new ZipEntry(parentDirectory.toString() + "/")); - zipFile.closeEntry(); - } - - for (final var accessory : state.getWorkflowInputFiles().entrySet()) { - zipFile.putNextEntry(new ZipEntry(accessory.getKey())); - zipFile.write(accessory.getValue().getBytes(StandardCharsets.UTF_8)); - zipFile.closeEntry(); - } - } - final var zipContents = zipOutput.toByteArray(); - body.addPart( - "workflowDependencies", () -> new ByteArrayInputStream(zipContents), null, null); - } - - CROMWELL_REQUESTS.labels(url).inc(); - CLIENT - .sendAsync( - HttpRequest.newBuilder() - .uri(URI.create(String.format("%s/api/workflows/v1", url))) - .timeout(Duration.ofMinutes(1)) - .header("Content-Type", body.getContentType()) - .POST(body.build()) - .build(), - BodyHandlers.ofString(StandardCharsets.UTF_8)) - .thenAccept( - r -> { - if (r.statusCode() / 100 != 2) { - monitor.permanentFailure( - String.format( - "Cromwell returned HTTP status %d on submission: %s", - r.statusCode(), r.body())); - return; - } - try { - final var result = MAPPER.readValue(r.body(), WorkflowStatusResponse.class); - if (result.getId() == null || result.getId().equals("null")) { - monitor.permanentFailure("Cromwell failed to launch workflow."); - return; - } - state.setCromwellId(result.getId()); - monitor.storeRecoveryInformation(state); - monitor.updateState(statusFromCromwell(result.getStatus())); - monitor.scheduleTask(CHECK_DELAY, TimeUnit.MINUTES, () -> check(state, monitor)); - monitor.log( - System.Logger.Level.INFO, - String.format( - "Started Cromwell workflow %s on %s", state.getCromwellId(), url)); - } catch (JsonProcessingException e) { - e.printStackTrace(); - monitor.permanentFailure(e.getMessage()); - } - }) - .exceptionally( - t -> { - monitor.log( - System.Logger.Level.INFO, - String.format("Failed to launch Cromwell workflow on %s", url)); - t.printStackTrace(); - CROMWELL_FAILURES.labels(url).inc(); - monitor.scheduleTask(CHECK_DELAY, TimeUnit.MINUTES, () -> check(state, monitor)); - return null; - }); - } catch (Exception e) { - monitor.log( - System.Logger.Level.INFO, String.format("Failed to launch Cromwell workflow on %s", url)); - CROMWELL_FAILURES.labels(url).inc(); - monitor.permanentFailure(e.toString()); - } + return new StateUnstarted( + url, + vidarrId, + engineParameters, + filteredParameters, + accessoryFiles.collect(Collectors.toMap(Pair::first, Pair::second)), + workflowLanguage, + workflow); } @Override @@ -454,8 +209,4 @@ public void startup() { public boolean supports(WorkflowLanguage language) { return language == WorkflowLanguage.WDL_1_0 || language == WorkflowLanguage.WDL_1_1; } - - public void setDebugInflightRuns(Boolean debugInflightRuns) { - this.debugInflightRuns = debugInflightRuns; - } } diff --git a/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/EngineState.java b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/EngineState.java deleted file mode 100644 index 678e6294..00000000 --- a/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/EngineState.java +++ /dev/null @@ -1,73 +0,0 @@ -package ca.on.oicr.gsi.vidarr.cromwell; - -import ca.on.oicr.gsi.vidarr.WorkflowLanguage; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import java.util.Map; - -/** The current state of a running workflow to be recorded in the database */ -public final class EngineState { - private String cromwellId; - private JsonNode engineParameters; - private ObjectNode parameters; - private String vidarrId; - private Map workflowInputFiles; - private WorkflowLanguage workflowLanguage; - private String workflowSource; - - public String getCromwellId() { - return cromwellId; - } - - public JsonNode getEngineParameters() { - return engineParameters; - } - - public ObjectNode getParameters() { - return parameters; - } - - public String getVidarrId() { - return vidarrId; - } - - public Map getWorkflowInputFiles() { - return workflowInputFiles; - } - - public WorkflowLanguage getWorkflowLanguage() { - return workflowLanguage; - } - - public String getWorkflowSource() { - return workflowSource; - } - - public void setCromwellId(String cromwellId) { - this.cromwellId = cromwellId; - } - - public void setEngineParameters(JsonNode engineParameters) { - this.engineParameters = engineParameters; - } - - public void setParameters(ObjectNode parameters) { - this.parameters = parameters; - } - - public void setVidarrId(String vidarrId) { - this.vidarrId = vidarrId; - } - - public void setWorkflowInputFiles(Map workflowInputFiles) { - this.workflowInputFiles = workflowInputFiles; - } - - public void setWorkflowLanguage(WorkflowLanguage workflowLanguage) { - this.workflowLanguage = workflowLanguage; - } - - public void setWorkflowSource(String workflowSource) { - this.workflowSource = workflowSource; - } -} diff --git a/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/PreflightState.java b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/PreflightState.java new file mode 100644 index 00000000..856124f2 --- /dev/null +++ b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/PreflightState.java @@ -0,0 +1,3 @@ +package ca.on.oicr.gsi.vidarr.cromwell; + +public record PreflightState() {} diff --git a/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/ProvisionState.java b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/ProvisionState.java index dc0a8160..f52217c1 100644 --- a/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/ProvisionState.java +++ b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/ProvisionState.java @@ -1,50 +1,72 @@ package ca.on.oicr.gsi.vidarr.cromwell; -/** The current provisioning state, to be recorded in the database */ -public final class ProvisionState { - private String cromwellId; - private String fileName; - private String metaType; - private String outputPrefix; - private String vidarrId; - - public String getCromwellId() { - return cromwellId; - } - - public String getFileName() { - return fileName; - } - - public String getMetaType() { - return metaType; - } +import static ca.on.oicr.gsi.vidarr.cromwell.CromwellWorkflowEngine.MAPPER; - public String getOutputPrefix() { - return outputPrefix; - } - - public String getVidarrId() { - return vidarrId; - } +import ca.on.oicr.gsi.vidarr.MultiPartBodyPublisher; +import com.fasterxml.jackson.databind.JsonNode; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpRequest; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Collections; +import java.util.Map; - public void setCromwellId(String cromwellId) { - this.cromwellId = cromwellId; - } +/** The current provisioning state, to be recorded in the database */ +public record ProvisionState( + String cromwellUrl, String fileName, JsonNode metadata, String vidarrId) { - public void setFileName(String fileName) { - this.fileName = fileName; - } + public HttpRequest buildLaunchRequest(CromwellOutputProvisioner provisioner) throws IOException { + final var outputMetadata = MAPPER.convertValue(metadata, OutputMetadata.class); + var path = Path.of(outputMetadata.getOutputDirectory()); + int startIndex = 0; + for (final var length : provisioner.getChunks()) { + if (length < 1) { + break; + } + final var endIndex = Math.min(vidarrId.length(), startIndex + length); + if (endIndex == startIndex) { + break; + } + path = path.resolve(vidarrId.substring(startIndex, endIndex)); + startIndex = endIndex; + } - public void setMetaType(String metaType) { - this.metaType = metaType; - } + final var outputPrefix = path.resolve(vidarrId).toString(); - public void setOutputPrefix(String outputPrefix) { - this.outputPrefix = outputPrefix; + final var body = + new MultiPartBodyPublisher() + .addPart( + provisioner.getWorkflowUrl() == null ? "workflowSource" : "workflowUrl", + provisioner.getWorkflowUrl() == null + ? provisioner.getWorkflowSource() + : provisioner.getWorkflowUrl()) + .addPart("workflowType", "WDL") + .addPart("workflowTypeVersion", provisioner.getWdlVersion()) + .addPart( + "labels", + MAPPER.writeValueAsString( + Collections.singletonMap( + "vidarr-id", vidarrId.substring(Math.max(0, vidarrId.length() - 255))))) + .addPart( + "workflowInputs", + MAPPER.writeValueAsString( + Map.of( + provisioner.getFileField(), + fileName, + provisioner.getOutputPrefixField(), + outputPrefix))) + .addPart( + "workflowOptions", MAPPER.writeValueAsString(provisioner.getWorkflowOptions())); + return HttpRequest.newBuilder() + .uri(URI.create(String.format("%s/api/workflows/v1", cromwellUrl))) + .timeout(Duration.ofMinutes(1)) + .header("Content-Type", body.getContentType()) + .POST(body.build()) + .build(); } - public void setVidarrId(String vidarrId) { - this.vidarrId = vidarrId; + public StateStarted checkTask(String cromwellId) { + return new StateStarted(cromwellId, cromwellUrl); } } diff --git a/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/StateStarted.java b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/StateStarted.java new file mode 100644 index 00000000..466bcea2 --- /dev/null +++ b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/StateStarted.java @@ -0,0 +1,32 @@ +package ca.on.oicr.gsi.vidarr.cromwell; + +import java.net.URI; +import java.net.http.HttpRequest; +import java.time.Duration; + +public record StateStarted(String cromwellId, String cromwellServer) { + public HttpRequest buildCheckRequest(boolean debugInflightRuns) { + return HttpRequest.newBuilder() + .uri(CromwellMetadataURL.formatMetadataURL(cromwellServer, cromwellId, debugInflightRuns)) + .timeout(Duration.ofMinutes(1)) + .GET() + .build(); + } + + public HttpRequest buildOutputsRequest() { + return HttpRequest.newBuilder() + .uri( + URI.create(String.format("%s/api/workflows/v1/%s/outputs", cromwellServer, cromwellId))) + .timeout(Duration.ofMinutes(1)) + .GET() + .build(); + } + + public String runtimeProvisionerUrl() { + // Note: This instance of the cromwell URL is + // for use by OutputProvisioners + // Don't include 'excludeKeys', includeCalls + // needs to be true + return CromwellMetadataURL.formatMetadataURL(cromwellServer(), cromwellId(), true).toString(); + } +} diff --git a/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/StateUnstarted.java b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/StateUnstarted.java new file mode 100644 index 00000000..0a2478e5 --- /dev/null +++ b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/StateUnstarted.java @@ -0,0 +1,106 @@ +package ca.on.oicr.gsi.vidarr.cromwell; + +import static ca.on.oicr.gsi.vidarr.cromwell.CromwellWorkflowEngine.MAPPER; + +import ca.on.oicr.gsi.vidarr.MultiPartBodyPublisher; +import ca.on.oicr.gsi.vidarr.WorkflowLanguage; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpRequest; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Map; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** The current state of a running workflow to be recorded in the database */ +public record StateUnstarted( + String cromwellServer, + String vidarrId, + JsonNode engineParameters, + ObjectNode parameters, + Map workflowInputFiles, + WorkflowLanguage workflowLanguage, + String workflowSource) { + public StateStarted checkTask(String cromwellId) { + return new StateStarted(cromwellId, cromwellServer); + } + + public HttpRequest buildLaunchRequest() throws IOException { + final var body = + new MultiPartBodyPublisher() + .addPart("workflowSource", this.workflowSource()) + .addPart("workflowInputs", MAPPER.writeValueAsString(this.parameters())) + .addPart("workflowType", "WDL") + .addPart( + "workflowTypeVersion", + switch (this.workflowLanguage()) { + case WDL_1_0 -> "1.0"; + case WDL_1_1 -> "1.1"; + default -> "draft1"; + }) + .addPart( + "labels", + MAPPER.writeValueAsString( + Collections.singletonMap( + "vidarr-id", + this.vidarrId().substring(Math.max(0, this.vidarrId().length() - 255))))) + .addPart("workflowOptions", MAPPER.writeValueAsString(this.engineParameters())); + if (!this.workflowInputFiles().isEmpty()) { + // Cromwell doesn't deduplicate these and stores them all in its database, + // so it doesn't + // matter if we make the effort to ensure these ZIP files are + // byte-for-byte identical. + final var zipOutput = new ByteArrayOutputStream(); + try (final var zipFile = new ZipOutputStream(zipOutput)) { + + // We have to create all the parent directories or the better-files + // compressor that + // Cromwell uses will fail to decompress. A directory entry is one that + // ends with a / and + // has no data. + final var parentDirectories = + this.workflowInputFiles().keySet().stream() + .flatMap(StateUnstarted::findAllParents) + .distinct() + .sorted(Comparator.comparing(Path::getNameCount)) + .toList(); + for (final var parentDirectory : parentDirectories) { + zipFile.putNextEntry(new ZipEntry(parentDirectory.toString() + "/")); + zipFile.closeEntry(); + } + + for (final var accessory : this.workflowInputFiles().entrySet()) { + zipFile.putNextEntry(new ZipEntry(accessory.getKey())); + zipFile.write(accessory.getValue().getBytes(StandardCharsets.UTF_8)); + zipFile.closeEntry(); + } + } + final var zipContents = zipOutput.toByteArray(); + body.addPart("workflowDependencies", () -> new ByteArrayInputStream(zipContents), null, null); + } + return HttpRequest.newBuilder() + .uri(URI.create(String.format("%s/api/workflows/v1", this.cromwellServer()))) + .timeout(Duration.ofMinutes(1)) + .header("Content-Type", body.getContentType()) + .POST(body.build()) + .build(); + } + + private static Stream findAllParents(String file) { + final var parents = new ArrayList(); + for (var path = Path.of(file).getParent(); path != null; path = path.getParent()) { + parents.add(path); + } + return parents.stream(); + } +} diff --git a/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/WorkflowMetadataResponse.java b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/WorkflowMetadataResponse.java index 0477be43..9ba863c9 100644 --- a/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/WorkflowMetadataResponse.java +++ b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/WorkflowMetadataResponse.java @@ -1,5 +1,8 @@ package ca.on.oicr.gsi.vidarr.cromwell; +import static ca.on.oicr.gsi.vidarr.cromwell.CromwellWorkflowEngine.statusFromCromwell; + +import ca.on.oicr.gsi.vidarr.PollResult; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.JsonNode; import java.util.List; @@ -9,10 +12,10 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class WorkflowMetadataResponse { private Map> calls; + private List failures = List.of(); private String id; private String status; private String workflowRoot; - private List failures = List.of(); public JsonNode debugInfo() { final var debugInfo = CromwellWorkflowEngine.MAPPER.createObjectNode(); @@ -23,7 +26,7 @@ public JsonNode debugInfo() { debugInfo.putPOJO("cromwellFailures", failures); // `calls` might be empty if workflow run is not failed - if (null != calls && calls.size() != 0) { + if (null != calls && !calls.isEmpty()) { final var cromwellCalls = debugInfo.putArray("cromwellCalls"); calls.forEach( (task, calls) -> @@ -65,6 +68,14 @@ public String getWorkflowRoot() { return workflowRoot; } + public PollResult pollStatus() { + return switch (status) { + case "Aborted", "Failed" -> PollResult.failed("Cromwell state is: " + status); + case "Succeeded" -> PollResult.finished(); + default -> PollResult.active(statusFromCromwell(status)); + }; + } + public void setCalls(Map> calls) { this.calls = calls; } diff --git a/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/WorkflowStatusResponse.java b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/WorkflowStatusResponse.java index 8b72edd1..fa2c5dcc 100644 --- a/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/WorkflowStatusResponse.java +++ b/vidarr-cromwell/src/main/java/ca/on/oicr/gsi/vidarr/cromwell/WorkflowStatusResponse.java @@ -23,4 +23,8 @@ public void setId(String id) { public void setStatus(String status) { this.status = status; } + + public boolean hasSucceeded() { + return status.equals("Succeeded"); + } } diff --git a/vidarr-pluginapi/pom.xml b/vidarr-pluginapi/pom.xml index 3ba5fec1..14ffdc12 100644 --- a/vidarr-pluginapi/pom.xml +++ b/vidarr-pluginapi/pom.xml @@ -30,6 +30,10 @@ io.undertow undertow-core + + io.prometheus + simpleclient + junit junit diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/ActiveOperation.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ActiveOperation.java similarity index 58% rename from vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/ActiveOperation.java rename to vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ActiveOperation.java index 6a0d0d5d..1cabc3c1 100644 --- a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/ActiveOperation.java +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ActiveOperation.java @@ -1,6 +1,8 @@ -package ca.on.oicr.gsi.vidarr.core; +package ca.on.oicr.gsi.vidarr; import com.fasterxml.jackson.databind.JsonNode; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; /** * A running operation that can be updated by the scheduler @@ -8,6 +10,42 @@ * @param the transaction type associated with this implementation */ public interface ActiveOperation { + + /** + * The database where transaction operations can be serialized + * + * @param the transaction type + */ + interface TransactionManager { + + /** + * Performs an operation in a transaction. + * + * @param transaction a callback to perform in a transaction; not that the transaction object + * should not outlive this method call + */ + void inTransaction(Consumer transaction); + + /** + * Request that Vidarr schedule a callback at the next available opportunity + * + * @param task the task to execute + */ + void scheduleTask(Runnable task); + + /** + * Request that Vidarr schedule a callback at a specified time in the future + * + *

This scheduling is best-effort; Vidarr may execute a task earlier or later than requested + * based on load and priority. + * + * @param delay the amount of time to wait; if delay is < 1, this is equivalent to {@link + * #scheduleTask(Runnable)} + * @param units the time units to wait + * @param task the task to execute + */ + void scheduleTask(long delay, TimeUnit units, Runnable task); + } /** * Change the current client-visible debugging information * @@ -63,7 +101,7 @@ public interface ActiveOperation { String type(); /** - * Change the plugin type assocated with this operation + * Change the plugin type associated with this operation * * @param type the new plugin type * @param transaction the transaction to perform the update in diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/BaseJsonInputProvisioner.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/BaseJsonInputProvisioner.java deleted file mode 100644 index 227dd489..00000000 --- a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/BaseJsonInputProvisioner.java +++ /dev/null @@ -1,133 +0,0 @@ -package ca.on.oicr.gsi.vidarr; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.NullNode; -import java.util.concurrent.TimeUnit; - -/** - * A provisioner where input and state is serialised and deserialised using Jackson - * - * @param the type of metadata (information from the submitter) - * @param the type of stored state - */ -public abstract class BaseJsonInputProvisioner implements InputProvisioner { - private class JsonWorkMonitor implements WorkMonitor { - private final WorkMonitor original; - - private JsonWorkMonitor(WorkMonitor original) { - this.original = original; - } - - @Override - public void complete(T result) { - original.complete(result); - } - - @Override - public void log(System.Logger.Level level, String message) { - original.log(level, message); - } - - @Override - public void permanentFailure(String reason) { - original.permanentFailure(reason); - } - - @Override - public void scheduleTask(long delay, TimeUnit units, Runnable task) { - original.scheduleTask(delay, units, task); - } - - @Override - public void scheduleTask(Runnable task) { - original.scheduleTask(task); - } - - @Override - public void storeDebugInfo(JsonNode information) { - original.storeDebugInfo(information); - } - - public void storeRecoveryInformation(S state) { - original.storeRecoveryInformation(mapper.valueToTree(state)); - } - - @Override - public void updateState(Status status) { - original.updateState(status); - } - } - - private final ObjectMapper mapper; - private final Class metadataClass; - private final Class stateClass; - - /** - * Construct a new input provisioner with data mapping - * - * @param mapper the Jackson object mapper configured to serialise the state, data, and metadata - * @param stateClass the class object for the state - * @param metadataClass the class object for the metadata (from the submitter) - */ - protected BaseJsonInputProvisioner( - ObjectMapper mapper, Class stateClass, Class metadataClass) { - this.stateClass = stateClass; - this.mapper = mapper; - this.metadataClass = metadataClass; - } - - @Override - public final JsonNode provision( - WorkflowLanguage language, String id, String path, WorkMonitor monitor) { - try { - return mapper.valueToTree( - provisionRegistered(language, id, path, new JsonWorkMonitor<>(monitor))); - } catch (Exception e) { - monitor.permanentFailure(e.toString()); - return NullNode.getInstance(); - } - } - - /** - * Begin provisioning out a new output - * - * @see #provisionExternal(WorkflowLanguage, JsonNode, WorkMonitor) - */ - protected abstract S provisionExternal( - WorkflowLanguage language, M metadata, WorkMonitor monitor); - - @Override - public final JsonNode provisionExternal( - WorkflowLanguage language, JsonNode metadata, WorkMonitor monitor) { - try { - return mapper.valueToTree( - provisionExternal( - language, - mapper.treeToValue(metadata, metadataClass), - new JsonWorkMonitor<>(monitor))); - } catch (Exception e) { - monitor.permanentFailure(e.toString()); - return NullNode.getInstance(); - } - } - - public abstract S provisionRegistered( - WorkflowLanguage language, String id, String path, WorkMonitor monitor); - - /** - * Restart a provisioning process from state saved in the database - * - * @see #recover(JsonNode, WorkMonitor) - */ - protected abstract void recover(S state, WorkMonitor monitor); - - @Override - public final void recover(JsonNode state, WorkMonitor monitor) { - try { - recover(mapper.treeToValue(state, stateClass), new JsonWorkMonitor<>(monitor)); - } catch (Exception e) { - monitor.permanentFailure(e.toString()); - } - } -} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/BaseJsonOutputProvisioner.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/BaseJsonOutputProvisioner.java deleted file mode 100644 index 96893e31..00000000 --- a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/BaseJsonOutputProvisioner.java +++ /dev/null @@ -1,171 +0,0 @@ -package ca.on.oicr.gsi.vidarr; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.NullNode; -import java.util.concurrent.TimeUnit; - -/** - * A provisioner where input and state is serialised and deserialised using Jackson - * - * @param the type of metadata (information from the submitter) - * @param the type of stored state - * @param the type of stored state during preflight - */ -public abstract class BaseJsonOutputProvisioner implements OutputProvisioner { - private class JsonWorkMonitor implements WorkMonitor { - private final WorkMonitor original; - - private JsonWorkMonitor(WorkMonitor original) { - this.original = original; - } - - @Override - public void complete(T result) { - original.complete(result); - } - - @Override - public void log(System.Logger.Level level, String message) { - original.log(level, message); - } - - @Override - public void permanentFailure(String reason) { - original.permanentFailure(reason); - } - - @Override - public void scheduleTask(long delay, TimeUnit units, Runnable task) { - original.scheduleTask(delay, units, task); - } - - @Override - public void scheduleTask(Runnable task) { - original.scheduleTask(task); - } - - @Override - public void storeDebugInfo(JsonNode information) { - original.storeDebugInfo(information); - } - - public void storeRecoveryInformation(U state) { - original.storeRecoveryInformation(mapper.valueToTree(state)); - } - - @Override - public void updateState(Status status) { - original.updateState(status); - } - } - - private final ObjectMapper mapper; - private final Class metadataClass; - private final Class preflightStateClass; - private final Class stateClass; - - /** - * Construct a new output provisioner with data mapping - * - * @param mapper the Jackson object mapper configured to serialise the state, data, and metadata - * @param stateClass the class object for the state - * @param preflightStateClass the class object for the preflight state - * @param metadataClass the class object for the metadata (from the submitter) - */ - protected BaseJsonOutputProvisioner( - ObjectMapper mapper, - Class stateClass, - Class preflightStateClass, - Class metadataClass) { - this.stateClass = stateClass; - this.mapper = mapper; - this.preflightStateClass = preflightStateClass; - this.metadataClass = metadataClass; - } - /** - * Check that the metadata provided by the submitter is valid. - * - * @see #preflightCheck(JsonNode, WorkMonitor) - */ - protected abstract F preflightCheck(M metadata, WorkMonitor monitor); - - @Override - public final JsonNode preflightCheck(JsonNode metadata, WorkMonitor monitor) { - try { - return mapper.valueToTree( - preflightCheck( - mapper.treeToValue(metadata, metadataClass), new JsonWorkMonitor<>(monitor))); - } catch (Exception e) { - monitor.permanentFailure(e.toString()); - return NullNode.getInstance(); - } - } - - protected abstract void preflightRecover(F state, WorkMonitor monitor); - - @Override - public final void preflightRecover(JsonNode state, WorkMonitor monitor) { - try { - preflightRecover( - mapper.treeToValue(state, preflightStateClass), new JsonWorkMonitor<>(monitor)); - } catch (Exception e) { - monitor.permanentFailure(e.toString()); - } - } - - /** - * Begin provisioning out a new output - * - * @see OutputProvisioner#provision(String, String, JsonNode, WorkMonitor) - */ - protected abstract S provision( - String workflowId, String data, M metadata, WorkMonitor monitor); - - @Override - public final JsonNode provision( - String workflowRunId, String data, JsonNode metadata, WorkMonitor monitor) { - try { - return mapper.valueToTree( - provision( - workflowRunId, - data, - mapper.treeToValue(metadata, metadataClass), - new JsonWorkMonitor<>(monitor))); - } catch (Exception e) { - monitor.permanentFailure(e.toString()); - return NullNode.getInstance(); - } - } - - /** - * Restart a provisioning process from state saved in the database - * - * @see #recover(JsonNode, WorkMonitor) - */ - protected abstract void recover(S state, WorkMonitor monitor); - - @Override - public final void recover(JsonNode state, WorkMonitor monitor) { - try { - recover(mapper.treeToValue(state, stateClass), new JsonWorkMonitor<>(monitor)); - } catch (Exception e) { - monitor.permanentFailure(e.toString()); - } - } - /** - * Restart a failed provisioning process from state saved in the database - * - * @see #retry(JsonNode, WorkMonitor) - */ - protected abstract void retry(S state, WorkMonitor monitor); - - @Override - public final void retry(JsonNode state, WorkMonitor monitor) { - try { - retry(mapper.treeToValue(state, stateClass), new JsonWorkMonitor<>(monitor)); - } catch (Exception e) { - monitor.permanentFailure(e.toString()); - } - } -} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/BaseJsonRuntimeProvisioner.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/BaseJsonRuntimeProvisioner.java deleted file mode 100644 index c897e0f2..00000000 --- a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/BaseJsonRuntimeProvisioner.java +++ /dev/null @@ -1,125 +0,0 @@ -package ca.on.oicr.gsi.vidarr; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.NullNode; -import java.util.concurrent.TimeUnit; - -/** - * A runtime provisioner where input and state is serialised and deserialised using Jackson - * - * @param the type of stored state - */ -public abstract class BaseJsonRuntimeProvisioner implements RuntimeProvisioner { - private class JsonWorkMonitor implements WorkMonitor { - private final WorkMonitor original; - - private JsonWorkMonitor(WorkMonitor original) { - this.original = original; - } - - @Override - public void complete(T result) { - original.complete(result); - } - - @Override - public void log(System.Logger.Level level, String message) { - original.log(level, message); - } - - @Override - public void permanentFailure(String reason) { - original.permanentFailure(reason); - } - - @Override - public void scheduleTask(long delay, TimeUnit units, Runnable task) { - original.scheduleTask(delay, units, task); - } - - @Override - public void scheduleTask(Runnable task) { - original.scheduleTask(task); - } - - @Override - public void storeDebugInfo(JsonNode information) { - original.storeDebugInfo(information); - } - - public void storeRecoveryInformation(U state) { - original.storeRecoveryInformation(mapper.valueToTree(state)); - } - - @Override - public void updateState(Status status) { - original.updateState(status); - } - } - - private final ObjectMapper mapper; - private final Class stateClass; - - /** - * Construct a new output provisioner with data mapping - * - * @param mapper the Jackson object mapper configured to serialise the state, data, and metadata - * @param stateClass the class object for the state - */ - protected BaseJsonRuntimeProvisioner(ObjectMapper mapper, Class stateClass) { - this.stateClass = stateClass; - this.mapper = mapper; - } - /** - * Begin provisioning out a new output - * - * @see OutputProvisioner#provision(String, String, JsonNode, WorkMonitor) - */ - protected abstract S provision( - String workflowId, String data, WorkMonitor monitor); - - @Override - public final JsonNode provision( - String workflowRunId, WorkMonitor monitor) { - try { - return mapper.valueToTree(provision(workflowRunId, new JsonWorkMonitor<>(monitor))); - } catch (Exception e) { - monitor.permanentFailure(e.toString()); - return NullNode.getInstance(); - } - } - - /** - * Restart a provisioning process from state saved in the database - * - * @see #recover(JsonNode, WorkMonitor) - */ - protected abstract void recover(S state, WorkMonitor monitor); - - @Override - public final void recover( - JsonNode state, WorkMonitor monitor) { - try { - recover(mapper.treeToValue(state, stateClass), new JsonWorkMonitor<>(monitor)); - } catch (Exception e) { - monitor.permanentFailure(e.toString()); - } - } - - /** - * Restart a failed provisioning process from state saved in the database - * - * @see #recover(JsonNode, WorkMonitor) - */ - protected abstract void retry(S state, WorkMonitor monitor); - - @Override - public final void retry(JsonNode state, WorkMonitor monitor) { - try { - retry(mapper.treeToValue(state, stateClass), new JsonWorkMonitor<>(monitor)); - } catch (Exception e) { - monitor.permanentFailure(e.toString()); - } - } -} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/BaseJsonWorkflowEngine.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/BaseJsonWorkflowEngine.java deleted file mode 100644 index 9149aeca..00000000 --- a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/BaseJsonWorkflowEngine.java +++ /dev/null @@ -1,191 +0,0 @@ -package ca.on.oicr.gsi.vidarr; - -import ca.on.oicr.gsi.Pair; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.fasterxml.jackson.databind.node.ObjectNode; -import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; - -/** - * A workflow engine where the state is serialised using Jackson - * - * @param the Java type for the state - * @param the Java type for the cleanup - * @param the Java type for the cleanup state - */ -public abstract class BaseJsonWorkflowEngine implements WorkflowEngine { - private abstract class BaseJsonWorkMonitor implements WorkMonitor { - protected final WorkMonitor original; - - BaseJsonWorkMonitor(WorkMonitor original) { - this.original = original; - } - - @Override - public void log(System.Logger.Level level, String message) { - original.log(level, message); - } - - @Override - public void permanentFailure(String reason) { - original.permanentFailure(reason); - } - - @Override - public void scheduleTask(long delay, TimeUnit units, Runnable task) { - original.scheduleTask(delay, units, task); - } - - @Override - public void scheduleTask(Runnable task) { - original.scheduleTask(task); - } - - @Override - public void storeDebugInfo(JsonNode information) { - original.storeDebugInfo(information); - } - - public void storeRecoveryInformation(U state) { - original.storeRecoveryInformation(mapper.valueToTree(state)); - } - - @Override - public void updateState(Status status) { - original.updateState(status); - } - } - - private class JsonCleanupWorkMonitor extends BaseJsonWorkMonitor { - - public JsonCleanupWorkMonitor(WorkMonitor monitor) { - super(monitor); - } - - @Override - public void complete(Void result) { - original.complete(result); - } - } - - private class JsonWorkMonitor extends BaseJsonWorkMonitor, Result, S> { - - JsonWorkMonitor(WorkMonitor, JsonNode> original) { - super(original); - } - - @Override - public void complete(Result result) { - original.complete( - new Result<>( - result.output(), - result.workflowRunUrl(), - result.cleanupState().map(mapper::valueToTree))); - } - } - - private final Class cleanupClass; - private final Class cleanupStateClass; - private final ObjectMapper mapper; - private final Class stateClass; - - /** - * Construct a workflow engine with a state mapper - * - * @param mapper the Jackson object mapper configured to serialise the state - * @param stateClass the class object for the state - * @param cleanupClass the class object for the cleanup - * @param cleanupStateClass the class object for the cleanup state - */ - protected BaseJsonWorkflowEngine( - ObjectMapper mapper, Class stateClass, Class cleanupClass, Class cleanupStateClass) { - this.mapper = mapper; - this.stateClass = stateClass; - this.cleanupClass = cleanupClass; - this.cleanupStateClass = cleanupStateClass; - } - - /** - * Clean up the output of a workflow (i.e., delete its on-disk output) after provisioning has been - * completed. - * - * @see #cleanup(JsonNode, WorkMonitor) - */ - protected abstract D cleanup(C cleanupState, WorkMonitor monitor); - - @Override - public final JsonNode cleanup(JsonNode cleanupState, WorkMonitor monitor) { - try { - return mapper.valueToTree( - cleanup( - mapper.treeToValue(cleanupState, cleanupClass), new JsonCleanupWorkMonitor(monitor))); - } catch (Exception e) { - monitor.permanentFailure(e.toString()); - return JsonNodeFactory.instance.nullNode(); - } - } - - /** - * Restart a running process from state saved in the database - * - * @see #recover(JsonNode, WorkMonitor) - */ - protected abstract void recover(S state, WorkMonitor, S> monitor); - - @Override - public final void recover(JsonNode state, WorkMonitor, JsonNode> monitor) { - try { - recover(mapper.treeToValue(state, stateClass), new JsonWorkMonitor(monitor)); - } catch (Exception e) { - monitor.permanentFailure(e.toString()); - } - } - - protected abstract void recoverCleanup(D state, WorkMonitor monitor); - - @Override - public void recoverCleanup(JsonNode state, WorkMonitor monitor) { - try { - recoverCleanup( - mapper.treeToValue(state, cleanupStateClass), new JsonCleanupWorkMonitor(monitor)); - } catch (Exception e) { - monitor.permanentFailure(e.toString()); - } - } - - @Override - public final JsonNode run( - WorkflowLanguage workflowLanguage, - String workflow, - Stream> accessoryFiles, - String vidarrId, - ObjectNode workflowParameters, - JsonNode engineParameters, - WorkMonitor, JsonNode> monitor) { - return mapper.valueToTree( - runWorkflow( - workflowLanguage, - workflow, - accessoryFiles, - vidarrId, - workflowParameters, - engineParameters, - new JsonWorkMonitor(monitor))); - } - /** - * Start a new workflow - * - * @see WorkflowEngine#run(WorkflowLanguage, String, Stream, String, ObjectNode, JsonNode, - * WorkMonitor) - */ - protected abstract S runWorkflow( - WorkflowLanguage workflowLanguage, - String workflow, - Stream> accessoryFiles, - String vidarrId, - ObjectNode workflowParameters, - JsonNode engineParameters, - WorkMonitor, S> monitor); -} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/InputProvisioner.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/InputProvisioner.java index e8f003b5..d8dc5911 100644 --- a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/InputProvisioner.java +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/InputProvisioner.java @@ -18,12 +18,16 @@ import java.util.stream.Collectors; import javax.xml.stream.XMLStreamException; -/** A mechanism to collect output data from a workflow and push it into an appropriate data store */ +/** + * A mechanism to collect output data from a workflow and push it into an appropriate data store + * + * @param the state information used for provisioning in data + */ @JsonTypeIdResolver(InputProvisioner.InputProvisionerIdResolver.class) @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, include = As.PROPERTY, property = "type") -public interface InputProvisioner { +public interface InputProvisioner { final class InputProvisionerIdResolver extends TypeIdResolverBase { - private final Map> knownIds = + private final Map>> knownIds = ServiceLoader.load(InputProvisionerProvider.class).stream() .map(Provider::get) .flatMap(InputProvisionerProvider::types) @@ -72,44 +76,33 @@ public JavaType typeFromId(DatabindContext context, String id) throws IOExceptio * Begin provisioning out a new input that was registered in Vidarr * *

This method should not do any externally-visible work. Anything it needs should be done in a - * {@link WorkMonitor#scheduleTask(Runnable)} callback so that Vidarr can execute it once the - * database is in a healthy state. + * {@link #run()} so that Vidarr can execute it once the database is in a healthy state. * * @param language the workflow language the output will be consumed by * @param id the Vidarr ID for the file * @param path the output path registered in Vidarr for the file - * @param monitor the monitor structure for writing the output of the provisioning process * @return the initial state of the provision out process */ - JsonNode provision( - WorkflowLanguage language, String id, String path, WorkMonitor monitor); + State provision(WorkflowLanguage language, String id, String path); /** * Begin provisioning out a new input that was not registered in Vidarr * *

This method should not do any externally-visible work. Anything it needs should be done in a - * {@link WorkMonitor#scheduleTask(Runnable)} callback so that Vidarr can execute it once the - * database is in a healthy state. + * {@link #run()} so that Vidarr can execute it once the database is in a healthy state. * * @param language the workflow language the output will be consumed by * @param metadata the information coming from the submitter to direct provisioning - * @param monitor the monitor structure for writing the output of the provisioning process * @return the initial state of the provision out process */ - JsonNode provisionExternal( - WorkflowLanguage language, JsonNode metadata, WorkMonitor monitor); + State provisionExternal(WorkflowLanguage language, JsonNode metadata); /** - * Restart a provisioning process from state saved in the database + * Create a declarative structure to execute the provisioning * - *

This method should not do any externally-visible work. Anything it needs should be done in a - * {@link WorkMonitor#scheduleTask(Runnable)} callback so that Vidarr can execute it once the - * database is in a healthy state. - * - * @param state the frozen database state - * @param monitor the monitor structure for writing the output of the provisioning process + * @return the sequence of operations that should be performed */ - void recover(JsonNode state, WorkMonitor monitor); + OperationAction run(); /** * Called to initialise this input provisioner. * diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/InputProvisionerProvider.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/InputProvisionerProvider.java index 19f6a1f2..462808da 100644 --- a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/InputProvisionerProvider.java +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/InputProvisionerProvider.java @@ -6,5 +6,5 @@ /** Reads JSON configuration and instantiates input provisioners appropriately */ public interface InputProvisionerProvider { /** Provides the type names and classes this plugin provides */ - Stream>> types(); + Stream>>> types(); } diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationAction.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationAction.java new file mode 100644 index 00000000..27e6507f --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationAction.java @@ -0,0 +1,362 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import ca.on.oicr.gsi.vidarr.OperationStatefulStep.StatefulTransformer; +import ca.on.oicr.gsi.vidarr.OperationStep.Transformer; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Map; + +/** + * An operation action defines a process that takes some initial state and executes a sequence of + * commands to generate an output value. All actions along the way can be logged to a database to + * allow crash recovery. + * + *

This class enforces the use of Java's records for state encapsulation in order to encourage + * all state to be immutable. It cannot guarantee that, but it is the operation assumption of this + * design. Mutating the state directly is undefined behaviour. + * + * @param the state that will be logged to the database; this type is usually unimportant, + * but if it changes, the database recovery will not work + * @param the initial state object generated by the workflow or provisioner + * @param the return value for this action + */ +public abstract sealed class OperationAction< + State extends Record, OriginalState extends Record, Value> + permits OperationActionBranch, + OperationActionConstant, + OperationActionDoStep, + OperationActionDoStatefulStep, + OperationActionLoad, + OperationActionReload { + + /** + * A delayed start and serialization object + * + * @param the state being captured + * @param the output type of this operation + */ + public interface Launcher { + + /** + * Start execution + * + * @param operation the active operation associated with this action + * @param transactionManager the transaction manager to allow writing state and running delayed + * operations + * @param next the flow controller to call when the operation is complete (both for success and + * failure) + * @param the type of the transactions used by the operation + */ + void launch( + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next); + + /** + * Serialize the current state as JSON + * + * @return the serialized state + */ + JsonNode state(); + } + + /** + * Create the initial value for an operation action from the state + * + * @param the state type for the operation + * @param the value type produced + */ + public interface Loader { + + /** + * Create the initial value from the provided state + * + * @param state the state to use + * @return the value for the next step + * @throws Exception any exceptions will be caught and redirected through Vidarr's logging and + * operation framework + */ + Result prepare(State state) throws Exception; + } + + /** + * The state associated with a branching operation + * + * @param name the name of the branch + * @param inner the state of that branch + */ + public record BranchState(String name, JsonNode inner) {} + + static final ObjectMapper MAPPER = new ObjectMapper(); + + /** + * Create a new action that can follow many paths + * + *

This operation has limits to the type safety it can provide; it requires that all paths + * present during creation of the state are available during subsequent runs. + * + * @param branches the possible name operations to use; they must all produce the same output + * @return a new action that picks the correct path and executes it + * @param the output value + */ + public static OperationAction branch( + Map> branches) { + return new OperationActionBranch<>(branches); + } + + /** + * Create a new action that returns a value based on the state + * + * @param stateClass the class object for the state that is used + * @param loader the function to extract the value from the state + * @return the new action to return this value + * @param the type of the state + * @param the type to be returned + */ + public static OperationAction load( + Class stateClass, Loader loader) { + return new OperationActionLoad<>(stateClass, loader); + } + + /** + * Create a new action that returns a value based on the state + * + * @param typeReference a type reference for the state that is used + * @param loader the function to extract the value from the state + * @return the new action to return this value + * @param the type of the state + * @param the type to be returned + */ + public static OperationAction load( + TypeReference typeReference, Loader loader) { + return new OperationActionLoad<>(typeReference, loader); + } + + /** + * Create a new action that returns a fixed value + * + * @param stateClass the class object for the state that is used + * @param value the value to return + * @return the new action to return this value + * @param the type of the state + * @param the type to be returned + */ + public static OperationAction value( + Class stateClass, Value value) { + return new OperationActionConstant<>(stateClass, value); + } + + /** + * Create a new action that returns a fixed value + * + * @param typeReference a type reference for the state that is used + * @param value the value to return + * @return the new action to return this value + * @param the type of the state + * @param the type to be returned + */ + public static OperationAction value( + TypeReference typeReference, Value value) { + return new OperationActionConstant<>(typeReference, value); + } + + abstract State buildState(OriginalState originalState); + + /** + * Deserialize an initial state for this operation from JSON + * + * @param originalState the original state as JSON + * @return the deserialized original state + * @throws JsonProcessingException thrown if the JSON does not match the correct structure for the + * original state + */ + public abstract OriginalState deserializeOriginal(JsonNode originalState) + throws JsonProcessingException; + + /** + * Create a new branch state by serializing a state object + * + * @param name the name of the branch + * @param originalState the initial state of that branch + * @return the enclosing branch state + */ + public BranchState intoBranch(String name, OriginalState originalState) { + return new BranchState(name, MAPPER.valueToTree(buildState(originalState))); + } + + abstract JavaType jacksonType(); + + /** + * Begin execution from a new state + * + * @param originalState the new state to start with + * @return a launcher to being execution + */ + public final Launcher launch(OriginalState originalState) { + final var state = buildState(originalState); + final var jsonState = MAPPER.valueToTree(state); + return new Launcher<>() { + @Override + public void launch( + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next) { + run(state, operation, transactionManager, next); + } + + @Override + public JsonNode state() { + return jsonState; + } + }; + } + + /** + * Transform the current value into another with state information + * + *

This is syntactic sugar for {@link OperationStatefulStep#mapping(StatefulTransformer)} + * + * @param mapper the transformation step to apply + * @return a new operation that will apply this transformation + * @param the result type of the transformation + */ + public final OperationAction map( + StatefulTransformer mapper) { + return then(OperationStatefulStep.mapping(mapper)); + } + + /** + * Transform the current value into another + * + *

This is syntactic sugar for {@link OperationStep#mapping(Transformer)} + * + * @param mapper the transformation step to apply + * @return a new operation that will apply this transformation + * @param the result type of the transformation + */ + public final OperationAction map( + Transformer mapper) { + return then(OperationStep.mapping(mapper)); + } + + /** + * Restart an operation from the database + * + * @param state the JSON state previously serialised + * @return a launcher to being execution + */ + public final Launcher recover(JsonNode state) { + final State s = MAPPER.convertValue(state, jacksonType()); + return new Launcher<>() { + @Override + public void launch( + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next) { + transactionManager.scheduleTask(() -> run(s, operation, transactionManager, next)); + } + + @Override + public JsonNode state() { + return MAPPER.valueToTree(s); + } + }; + } + + /** + * Recover and operation from the database and discard and error conditions and allow it to start + * again. + * + * @param state the JSON state previously serialised + * @return a launcher to being execution + */ + public final Launcher retry(JsonNode state) { + try { + final var s = rewind(MAPPER.convertValue(state, jacksonType())); + + return new Launcher<>() { + @Override + public void launch( + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next) { + transactionManager.scheduleTask(() -> run(s, operation, transactionManager, next)); + } + + @Override + public JsonNode state() { + return MAPPER.valueToTree(s); + } + }; + } catch (JsonProcessingException e) { + return new Launcher<>() { + final String message = "Failed to rewind state: " + e.getMessage(); + + @Override + public void launch( + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next) { + next.error(message); + } + + @Override + public JsonNode state() { + return MAPPER.nullNode(); + } + }; + } + } + + abstract State rewind(State state) throws JsonProcessingException; + + /** + * Discard the current value and load a new one from the state + * + * @param loader a function to load the value from the state + * @return an action that will load this value + * @param the type of the returned value from this operation + */ + public final OperationAction reload( + Loader loader) { + return new OperationActionReload<>(this, loader); + } + + abstract void run( + State state, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow flow); + + /** + * Transform the current operation in a way that uses state + * + * @param flow the flow control to apply; this flow control might modify the state + * @return an action that applies this modification + * @param the type of the state with this flow control + * @param the value type after the operation + */ + public final + OperationAction then( + OperationStatefulStep flow) { + return new OperationActionDoStatefulStep<>(this, flow); + } + + /** + * Perform an operation on the current value and return a new value + * + *

Note that these steps are independent of the state and only operate the input value + * + * @param step the step to perform; it must take the current value as input + * @return a new action that will perform the step + * @param the type of the returned value from this operation + */ + public final OperationAction then( + OperationStep step) { + return new OperationActionDoStep<>(this, step); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionBranch.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionBranch.java new file mode 100644 index 00000000..960cf059 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionBranch.java @@ -0,0 +1,98 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import ca.on.oicr.gsi.vidarr.OperationAction.BranchState; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.JsonNode; +import java.util.Map; + +final class OperationActionBranch + extends OperationAction { + + private final Map> branches; + + OperationActionBranch(Map> branches) { + this.branches = branches; + } + + @Override + BranchState buildState(BranchState branchState) { + return branchState; + } + + @Override + public BranchState deserializeOriginal(JsonNode originalState) throws JsonProcessingException { + return MAPPER.treeToValue(originalState, BranchState.class); + } + + @Override + JavaType jacksonType() { + return MAPPER.getTypeFactory().constructType(BranchState.class); + } + + @Override + BranchState rewind(BranchState state) throws JsonProcessingException { + return new BranchState(state.name(), rewind(branches.get(state.name()), state.inner())); + } + + private JsonNode rewind( + OperationAction action, JsonNode inner) throws JsonProcessingException { + return MAPPER.valueToTree(action.rewind(MAPPER.treeToValue(inner, action.jacksonType()))); + } + + @Override + void run( + BranchState branchState, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow flow) { + final var branch = branches.get(branchState.name()); + if (branch == null) { + flow.error( + String.format("Branch is looking for path %s, but this is unknown", branchState.name())); + } else { + startInner( + branch, + branchState.name(), + MAPPER.convertValue(branchState.inner(), branch.jacksonType()), + transactionManager, + operation, + flow); + } + } + + void startInner( + OperationAction action, + String name, + InnerState innerState, + TransactionManager transactionManager, + ActiveOperation operation, + OperationControlFlow next) { + action.run( + innerState, + operation, + transactionManager, + new OperationControlFlow<>() { + @Override + public void cancel() { + next.cancel(); + } + + @Override + public void error(String error) { + next.error(error); + } + + @Override + public void next(Output output) { + next.next(output); + } + + @Override + public JsonNode serializeNestedState(InnerState innerState) { + return next.serializeNestedState(new BranchState(name, MAPPER.valueToTree(innerState))); + } + }); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionConstant.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionConstant.java new file mode 100644 index 00000000..8790aca0 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionConstant.java @@ -0,0 +1,53 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.JsonNode; + +final class OperationActionConstant + extends OperationAction { + + private final JavaType type; + private final Output value; + + OperationActionConstant(Class type, Output value) { + this.value = value; + this.type = MAPPER.constructType(type); + } + + OperationActionConstant(TypeReference type, Output value) { + this.value = value; + this.type = MAPPER.constructType(type); + } + + @Override + State buildState(State state) { + return state; + } + + @Override + public State deserializeOriginal(JsonNode originalState) throws JsonProcessingException { + return MAPPER.treeToValue(originalState, type); + } + + @Override + JavaType jacksonType() { + return type; + } + + @Override + State rewind(State state) throws JsonProcessingException { + return state; + } + + @Override + void run( + State state, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow flow) { + flow.next(value); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionDoStatefulStep.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionDoStatefulStep.java new file mode 100644 index 00000000..807398e9 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionDoStatefulStep.java @@ -0,0 +1,51 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.JsonNode; + +final class OperationActionDoStatefulStep< + State extends Record, NextState extends Record, OriginalState extends Record, Input, Output> + extends OperationAction { + + private final OperationStatefulStep step; + private final OperationAction input; + + public OperationActionDoStatefulStep( + OperationAction input, + OperationStatefulStep step) { + super(); + this.input = input; + this.step = step; + } + + @Override + NextState buildState(OriginalState originalState) { + return step.buildState(input.buildState(originalState)); + } + + @Override + public OriginalState deserializeOriginal(JsonNode originalState) throws JsonProcessingException { + return input.deserializeOriginal(originalState); + } + + @Override + JavaType jacksonType() { + return step.jacksonType(input.jacksonType()); + } + + @Override + NextState rewind(NextState state) throws JsonProcessingException { + return step.rewind(state, input); + } + + @Override + void run( + NextState nextState, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow flow) { + this.step.run(input, nextState, operation, transactionManager, flow); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionDoStep.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionDoStep.java new file mode 100644 index 00000000..0c156998 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionDoStep.java @@ -0,0 +1,78 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.JsonNode; + +final class OperationActionDoStep + extends OperationAction { + + private final OperationAction input; + private final OperationStep step; + + public OperationActionDoStep( + OperationAction input, OperationStep step) { + super(); + this.input = input; + this.step = step; + } + + @Override + State buildState(OriginalState originalState) { + return input.buildState(originalState); + } + + @Override + public OriginalState deserializeOriginal(JsonNode originalState) throws JsonProcessingException { + return input.deserializeOriginal(originalState); + } + + @Override + JavaType jacksonType() { + return input.jacksonType(); + } + + @Override + State rewind(State state) throws JsonProcessingException { + return input.rewind(state); + } + + @Override + void run( + State state, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow flow) { + input.run( + state, + operation, + transactionManager, + new OperationControlFlow<>() { + @Override + public void cancel() { + flow.cancel(); + } + + @Override + public void error(String error) { + flow.error(error); + } + + @Override + public void next(Input input) { + if (!operation.isLive()) { + flow.cancel(); + return; + } + + step.run(input, operation, transactionManager, flow); + } + + @Override + public JsonNode serializeNestedState(State state) { + return flow.serializeNestedState(state); + } + }); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionLoad.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionLoad.java new file mode 100644 index 00000000..6fc940a8 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionLoad.java @@ -0,0 +1,64 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.JsonNode; + +final class OperationActionLoad + extends OperationAction { + + private final Loader loader; + private final JavaType type; + + OperationActionLoad(Class type, Loader loader) { + this.loader = loader; + this.type = MAPPER.constructType(type); + } + + OperationActionLoad(TypeReference type, Loader loader) { + this.loader = loader; + this.type = MAPPER.constructType(type); + } + + @Override + State buildState(State state) { + return state; + } + + @Override + public State deserializeOriginal(JsonNode originalState) throws JsonProcessingException { + return MAPPER.treeToValue(originalState, type); + } + + @Override + JavaType jacksonType() { + return type; + } + + @Override + State rewind(State state) throws JsonProcessingException { + return state; + } + + @Override + void run( + State state, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow flow) { + if (!operation.isLive()) { + flow.cancel(); + return; + } + Result value; + try { + value = loader.prepare(state); + } catch (Exception e) { + flow.error(e.getMessage()); + return; + } + flow.next(value); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionReload.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionReload.java new file mode 100644 index 00000000..46108e2f --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationActionReload.java @@ -0,0 +1,83 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.JsonNode; + +final class OperationActionReload + extends OperationAction { + + private final OperationAction input; + private final Loader loader; + + OperationActionReload( + OperationAction input, Loader loader) { + this.input = input; + this.loader = loader; + } + + @Override + State buildState(OriginalState originalState) { + return input.buildState(originalState); + } + + @Override + public OriginalState deserializeOriginal(JsonNode originalState) throws JsonProcessingException { + return input.deserializeOriginal(originalState); + } + + @Override + JavaType jacksonType() { + return input.jacksonType(); + } + + @Override + State rewind(State state) throws JsonProcessingException { + return input.rewind(state); + } + + @Override + void run( + State state, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow flow) { + input.run( + state, + operation, + transactionManager, + new OperationControlFlow<>() { + @Override + public void cancel() { + flow.cancel(); + } + + @Override + public void error(String error) { + flow.error(error); + } + + @Override + public void next(Input value) { + if (!operation.isLive()) { + flow.cancel(); + return; + } + Result output; + try { + output = loader.prepare(state); + } catch (Exception e) { + flow.error(e.getMessage()); + return; + } + flow.next(output); + } + + @Override + public JsonNode serializeNestedState(State state) { + return flow.serializeNestedState(state); + } + }); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationControlFlow.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationControlFlow.java new file mode 100644 index 00000000..bc1c870c --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationControlFlow.java @@ -0,0 +1,43 @@ +package ca.on.oicr.gsi.vidarr; + +import com.fasterxml.jackson.databind.JsonNode; + +/** + * Control flow for executing {@link OperationAction} + * + * @param the type of the operation's state + * @param the type the operation should yield + */ +public interface OperationControlFlow { + + /** + * Perform cleanup for an operation has been externally terminated + * + *

This should bubble up the call stack + */ + void cancel(); + + /** + * Abort a running operation due to an error + * + *

Perform cleanup as necessary + * + * @param error the error message that should be reported + */ + void error(String error); + + /** + * Return a successful value from an operation + * + * @param result the output value + */ + void next(Result result); + + /** + * Convert the state to JSON, performing and wrapping required + * + * @param state the state object to serialize + * @return the JSON-encoded form of the state object and any necessary enclosing state + */ + JsonNode serializeNestedState(State state); +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStep.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStep.java new file mode 100644 index 00000000..3bdc8fff --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStep.java @@ -0,0 +1,314 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.JsonNode; +import java.lang.System.Logger.Level; +import java.time.Duration; +import java.util.Optional; +import java.util.function.BiPredicate; + +/** + * Stateful operation steps modify the values being carried by an {@link OperationAction} and can + * access and modify the state + * + *

It's reasonable to consider every step as a function that takes an input value and provides an + * output value. Unlike a normal function, steps can be asynchronous and write information to + * Vidarr's database. Stateful steps may need to track additional information and so wrap the + * original state as necessary to keep track of the correct program flow + * + * @param the state the operation "gets" from the previous step + * @param the state that initiated the chain of steps + * @param the state the operation "provides" to the next step + * @param the parameter type + * @param the return type + */ +public abstract sealed class OperationStatefulStep< + InputState extends Record, + OutputState extends Record, + OriginalState extends Record, + Input, + Output> + permits OperationStatefulStepDebugInfo, + OperationStatefulStepLog, + OperationStatefulStepMapping, + OperationStatefulStepPoll, + OperationStatefulStepRepeatUntilSuccess, + OperationStatefulStepRequire, + OperationStatefulStepStatus, + OperationStatefulStepSubStep { + + /** + * A state which can be unwrapped into an inner state + * + *

The states generated by some of these steps implement this interface to make it easy to + * crack into the "inner" (first) state. It is not typical for states in workflow engines and + * provisioners to implement this interface. + */ + public interface InnerState { + + /** + * Try to find the inner state + * + * @param clazz the class corresponding to the inner + * @return the inner state of the expected type + * @param the type of the inner state + */ + T loadInner(Class clazz); + } + /** + * A simple mapping function + * + *

This is analogous to {@link java.util.function.BiFunction}, but it can throw an exception + * which will be caught and logged to Vidarr's database. + * + * @param the state type + * @param the parameter type + * @param the return type + */ + public interface StatefulTransformer { + + /** + * Call the function with an argument + * + * @param state the current state + * @param input the argument to use + * @return the transformed value + * @throws Exception any exceptions will be caught and redirected through Vidarr's logging and + * operation framework + */ + Output transform(State state, Input input) throws Exception; + } + + /** + * The state for a {@link #subStep(StatefulTransformer, OperationAction)} + * + * @param child the state for the sub-step + * @param state the state for the main path + * @param the type of the sub-step state + * @param the type of the main path state + */ + public record Child(Optional child, State state) + implements InnerState { + + @Override + public T loadInner(Class clazz) { + return state instanceof InnerState + ? ((InnerState) state).loadInner(clazz) + : clazz.cast(state); + } + } + + public record RepeatCounter(int attempts, State state) implements InnerState { + + @Override + public T loadInner(Class clazz) { + return state instanceof InnerState + ? ((InnerState) state).loadInner(clazz) + : clazz.cast(state); + } + } + + /** + * Write new debugging information for this value and state + * + * @param fetch the transformation to produce debugging information to write to the database. + * Vidarr imposes no schema on this data; it is up to the client to interpret it + * @return a step to perform write this debugging information + * @param the current state type + * @param the original state type + * @param the type of the input value + */ + public static + OperationStatefulStep debugInfo( + StatefulTransformer fetch) { + return new OperationStatefulStepDebugInfo<>(fetch); + } + + /** + * Log information about the current value and state + * + * @param level the logging level + * @param message a transformer to generate the log message + * @return a step to write this log message + * @param the current state type + * @param the original state type + * @param the type of the input value + */ + public static + OperationStatefulStep log( + Level level, StatefulTransformer message) { + return new OperationStatefulStepLog<>(level, message); + } + + /** + * Change the value using a state-aware function + * + * @param transformer the function to apply to the input value and state + * @return a step to call this function + * @param the current state type + * @param the original state type + * @param the type of the input + * @param the type of the output + */ + public static + OperationStatefulStep mapping( + StatefulTransformer transformer) { + return new OperationStatefulStepMapping<>(transformer); + } + + /** + * Unwrap a complex state object to access the original inner state + * + * @param clazz the type of the inner state + * @param transformer the function to apply to the inner state + * @return a new function that performs the unwrapping and then calls the provided function + * @param the outer (wrapped) state + * @param the inner (unwrapped) state + * @param the type of the input + * @param the type of the output + */ + public static + StatefulTransformer onInnerState( + Class clazz, StatefulTransformer transformer) { + return (state, input) -> + transformer.transform( + state instanceof OperationStatefulStep.InnerState inner + ? inner.loadInner(clazz) + : clazz.cast(state), + input); + } + + /** + * Run an operation repeatedly until it completes + * + *

This is meant for repeated accessing an external service. Note that if any step in the chain + * fails, the poll will not be reattempted. + * + * @param delay the best-effort time to wait between reattempts + * @return a step that reattempts the previous steps + * @param the type of the previous state + * @param the original state + */ + public static + OperationStatefulStep poll(Duration delay) { + return new OperationStatefulStepPoll<>(delay); + } + + /** + * Run an operation repeated until it succeeds + * + *

Runs an operation multiple time and retries if it fails on any error-producing step. + * + * @param delay the best-effort time to wait between reattempts + * @param maximumAttempts the maximum number of retries before declaring failure + * @return a step that reattempts the previous steps until success + * @param the type of the previous state + * @param the original state + * @param the type of the input and (unchanged) output + */ + public static + OperationStatefulStep, OriginalState, Value, Value> + repeatUntilSuccess(Duration delay, int maximumAttempts) { + return new OperationStatefulStepRepeatUntilSuccess<>(delay, maximumAttempts); + } + + /** + * Abort the operation if a condition is not met + * + * @param success the test to determine if the sequence should continue (true) or go into an error + * state (false) + * @param failureMessage the message to display when a failure occurs + * @return a step to check this condition + * @param the type of the previous state + * @param the original state + * @param the type of the input and (unchanged) output + */ + public static + OperationStatefulStep require( + BiPredicate success, String failureMessage) { + return new OperationStatefulStepRequire<>(success, failureMessage); + } + /** + * Change the status of this operation based on the current value and state + * + * @param fetch a function that examines the current value and state to produce a corresponding + * status + * @return a step that changes the status + * @param the type of the previous state + * @param the original state + * @param the type of the input and (unchanged) output + */ + public static + OperationStatefulStep status( + StatefulTransformer fetch) { + return new OperationStatefulStepStatus<>(fetch); + } + + /** + * Performs a single branching path between steps. + * + *

This step will first allow the previous steps to run. Once successful, it will check-point + * this information then run a second sequence of steps. If restarted, it will restart at the + * checkpoint rather than the beginning. + * + * @param spawn a function which examines the current state and input to produce a new initial + * state for the subtask + * @param subtask the action to perform in the subtask + * @return a steps which executes the subtask + * @param the type of the state of the main steps + * @param the type of the state of the child steps + * @param the type of the original state of the main steps + * @param the type of the original state of the child steps + * @param the type of the input information used for creating the child state + * @param the type of the output produced by the child task + */ + public static < + State extends Record, + SubState extends Record, + OriginalState extends Record, + OriginalSubState extends Record, + Input, + Output> + OperationStatefulStep, OriginalState, Input, Output> subStep( + StatefulTransformer spawn, + OperationAction subtask) { + return new OperationStatefulStepSubStep<>(spawn, subtask); + } + + /** + * Constructs enclosing state from new input state + * + * @param inputState the input state to wrap + * @return the enclosed output state. + */ + abstract OutputState buildState(InputState inputState); + + /** + * Get the Jackson type for this step's state + * + * @param incoming the Jackson type of the inner type + * @return the enclosing state type + */ + abstract JavaType jacksonType(JavaType incoming); + + /** + * Reset the state to allow restarting a failed operation during manual intervention + * + * @param state the original state + * @param input the upstream operation + * @return the reset state + * @throws JsonProcessingException allow JSON decoding errors to occur + */ + abstract OutputState rewind( + OutputState state, OperationAction input) + throws JsonProcessingException; + + abstract void run( + OperationAction input, + OutputState state, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next); +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepDebugInfo.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepDebugInfo.java new file mode 100644 index 00000000..88d5b17f --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepDebugInfo.java @@ -0,0 +1,82 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.JsonNode; + +final class OperationStatefulStepDebugInfo< + State extends Record, OriginalState extends Record, Value> + extends OperationStatefulStep { + + private final StatefulTransformer fetch; + + public OperationStatefulStepDebugInfo(StatefulTransformer fetch) { + super(); + this.fetch = fetch; + } + + @Override + State buildState(State state) { + return state; + } + + @Override + JavaType jacksonType(JavaType incoming) { + return incoming; + } + + @Override + State rewind(State state, OperationAction input) + throws JsonProcessingException { + return input.rewind(state); + } + + @Override + public void run( + OperationAction input, + State nextState, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next) { + input.run( + nextState, + operation, + transactionManager, + new OperationControlFlow<>() { + @Override + public void cancel() { + next.cancel(); + } + + @Override + public void error(String error) { + next.error(error); + } + + @Override + public void next(Value value) { + if (!operation.isLive()) { + next.cancel(); + return; + } + + JsonNode newDebugInfo; + try { + newDebugInfo = fetch.transform(nextState, value); + } catch (Exception e) { + next.error(e.getMessage()); + return; + } + transactionManager.inTransaction( + transaction -> operation.debugInfo(newDebugInfo, transaction)); + next.next(value); + } + + @Override + public JsonNode serializeNestedState(State state) { + return next.serializeNestedState(state); + } + }); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepLog.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepLog.java new file mode 100644 index 00000000..248a110f --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepLog.java @@ -0,0 +1,83 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.JsonNode; +import java.lang.System.Logger.Level; + +final class OperationStatefulStepLog + extends OperationStatefulStep { + + private final StatefulTransformer fetch; + private final Level level; + + public OperationStatefulStepLog(Level level, StatefulTransformer fetch) { + super(); + this.level = level; + this.fetch = fetch; + } + + @Override + State buildState(State state) { + return state; + } + + @Override + JavaType jacksonType(JavaType incoming) { + return incoming; + } + + @Override + State rewind(State state, OperationAction input) + throws JsonProcessingException { + return input.rewind(state); + } + + @Override + public void run( + OperationAction input, + State nextState, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next) { + input.run( + nextState, + operation, + transactionManager, + new OperationControlFlow<>() { + @Override + public void cancel() { + next.cancel(); + } + + @Override + public void error(String error) { + next.error(error); + } + + @Override + public void next(Value value) { + if (!operation.isLive()) { + next.cancel(); + return; + } + + String message; + try { + message = fetch.transform(nextState, value); + } catch (Exception e) { + next.error(e.getMessage()); + return; + } + operation.log(level, message); + next.next(value); + } + + @Override + public JsonNode serializeNestedState(State state) { + return next.serializeNestedState(state); + } + }); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepMapping.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepMapping.java new file mode 100644 index 00000000..6701e137 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepMapping.java @@ -0,0 +1,79 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.JsonNode; + +final class OperationStatefulStepMapping< + State extends Record, OriginalState extends Record, Input, Output> + extends OperationStatefulStep { + + private final StatefulTransformer transformer; + + OperationStatefulStepMapping(StatefulTransformer transformer) { + this.transformer = transformer; + } + + @Override + State buildState(State state) { + return state; + } + + @Override + JavaType jacksonType(JavaType incoming) { + return incoming; + } + + @Override + State rewind(State state, OperationAction input) + throws JsonProcessingException { + return input.rewind(state); + } + + @Override + public void run( + OperationAction input, + State nextState, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next) { + input.run( + nextState, + operation, + transactionManager, + new OperationControlFlow<>() { + @Override + public void cancel() { + next.cancel(); + } + + @Override + public void error(String error) { + next.error(error); + } + + @Override + public void next(Input input) { + if (!operation.isLive()) { + next.cancel(); + return; + } + + Output output; + try { + output = transformer.transform(nextState, input); + } catch (Exception e) { + next.error(e.getMessage()); + return; + } + next.next(output); + } + + @Override + public JsonNode serializeNestedState(State state) { + return next.serializeNestedState(state); + } + }); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepPoll.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepPoll.java new file mode 100644 index 00000000..98e5522d --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepPoll.java @@ -0,0 +1,97 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import ca.on.oicr.gsi.vidarr.PollResult.Visitor; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.JsonNode; +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +final class OperationStatefulStepPoll + extends OperationStatefulStep { + + private final Duration delay; + + public OperationStatefulStepPoll(Duration delay) { + super(); + this.delay = delay; + } + + @Override + State buildState(State state) { + return state; + } + + @Override + JavaType jacksonType(JavaType incoming) { + return incoming; + } + + @Override + State rewind(State state, OperationAction input) + throws JsonProcessingException { + return input.rewind(state); + } + + @Override + public void run( + OperationAction input, + State nextState, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next) { + input.run( + nextState, + operation, + transactionManager, + new OperationControlFlow() { + @Override + public void cancel() { + next.cancel(); + } + + @Override + public void error(String error) { + next.error(error); + } + + @Override + public void next(PollResult pollResult) { + if (!operation.isLive()) { + next.cancel(); + return; + } + pollResult.visit( + new Visitor() { + @Override + public void active(WorkingStatus status) { + transactionManager.inTransaction( + transaction -> operation.status(OperationStatus.of(status), transaction)); + transactionManager.scheduleTask( + delay.get(TimeUnit.SECONDS.toChronoUnit()), + TimeUnit.SECONDS, + () -> + OperationStatefulStepPoll.this.run( + input, nextState, operation, transactionManager, next)); + } + + @Override + public void failed(String error) { + next.error(error); + } + + @Override + public void finished() { + next.next(null); + } + }); + } + + @Override + public JsonNode serializeNestedState(State state) { + return next.serializeNestedState(state); + } + }); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepRepeatUntilSuccess.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepRepeatUntilSuccess.java new file mode 100644 index 00000000..8e09ae7b --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepRepeatUntilSuccess.java @@ -0,0 +1,91 @@ +package ca.on.oicr.gsi.vidarr; + +import static ca.on.oicr.gsi.vidarr.OperationAction.MAPPER; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import ca.on.oicr.gsi.vidarr.OperationStatefulStep.RepeatCounter; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.JsonNode; +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +final class OperationStatefulStepRepeatUntilSuccess< + State extends Record, OriginalState extends Record, Value> + extends OperationStatefulStep, OriginalState, Value, Value> { + + private final Duration delay; + private final int maximumAttempts; + + public OperationStatefulStepRepeatUntilSuccess(Duration delay, int maximumAttempts) { + super(); + this.delay = delay; + this.maximumAttempts = maximumAttempts; + } + + @Override + RepeatCounter buildState(State state) { + return new RepeatCounter<>(0, state); + } + + @Override + JavaType jacksonType(JavaType incoming) { + return MAPPER.getTypeFactory().constructParametricType(RepeatCounter.class, incoming); + } + + @Override + RepeatCounter rewind( + RepeatCounter state, OperationAction input) + throws JsonProcessingException { + return new RepeatCounter<>(0, input.rewind(state.state())); + } + + @Override + public void run( + OperationAction input, + RepeatCounter state, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow, Value> next) { + input.run( + state.state(), + operation, + transactionManager, + new OperationControlFlow<>() { + @Override + public void cancel() { + next.cancel(); + } + + @Override + public void error(String error) { + if (state.attempts() < maximumAttempts) { + final var nextState = new RepeatCounter<>(state.attempts() + 1, state.state()); + final var recoveryState = next.serializeNestedState(nextState); + transactionManager.inTransaction(tx -> operation.recoveryState(recoveryState, tx)); + transactionManager.scheduleTask( + delay.get(TimeUnit.SECONDS.toChronoUnit()), + TimeUnit.SECONDS, + () -> run(input, nextState, operation, transactionManager, next)); + } else { + next.error(error); + } + } + + @Override + public void next(Value value) { + if (operation.isLive()) { + next.next(value); + + } else { + next.cancel(); + } + } + + @Override + public JsonNode serializeNestedState(State innerState) { + return next.serializeNestedState(new RepeatCounter<>(state.attempts(), innerState)); + } + }); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepRequire.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepRequire.java new file mode 100644 index 00000000..e73d0b59 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepRequire.java @@ -0,0 +1,87 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.JsonNode; +import java.util.function.BiPredicate; + +final class OperationStatefulStepRequire + extends OperationStatefulStep { + + private final String failureMessage; + private final BiPredicate predicate; + + OperationStatefulStepRequire(BiPredicate predicate, String failureMessage) { + this.predicate = predicate; + this.failureMessage = failureMessage; + } + + @Override + State buildState(State state) { + return state; + } + + @Override + JavaType jacksonType(JavaType incoming) { + return incoming; + } + + @Override + State rewind(State state, OperationAction input) + throws JsonProcessingException { + return input.rewind(state); + } + + @Override + public void run( + OperationAction input, + State nextState, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next) { + input.run( + nextState, + operation, + transactionManager, + new OperationControlFlow<>() { + + @Override + public void cancel() { + next.cancel(); + } + + @Override + public void error(String error) { + next.error(error); + } + + @Override + public void next(Value value) { + if (!operation.isLive()) { + next.cancel(); + return; + } + + boolean check; + try { + check = predicate.test(nextState, value); + } catch (Exception e) { + next.error(e.getMessage()); + return; + } + if (check) { + next.next(value); + + } else { + next.error(failureMessage); + } + } + + @Override + public JsonNode serializeNestedState(State state) { + return next.serializeNestedState(state); + } + }); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepStatus.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepStatus.java new file mode 100644 index 00000000..8ed99e24 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepStatus.java @@ -0,0 +1,81 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.JsonNode; + +final class OperationStatefulStepStatus + extends OperationStatefulStep { + + private final StatefulTransformer fetch; + + public OperationStatefulStepStatus(StatefulTransformer fetch) { + super(); + this.fetch = fetch; + } + + @Override + State buildState(State state) { + return state; + } + + @Override + JavaType jacksonType(JavaType incoming) { + return incoming; + } + + @Override + State rewind(State state, OperationAction input) + throws JsonProcessingException { + return input.rewind(state); + } + + @Override + public void run( + OperationAction input, + State nextState, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next) { + input.run( + nextState, + operation, + transactionManager, + new OperationControlFlow<>() { + @Override + public void cancel() { + next.cancel(); + } + + @Override + public void error(String error) { + next.error(error); + } + + @Override + public void next(Value value) { + if (!operation.isLive()) { + next.cancel(); + return; + } + + WorkingStatus status; + try { + status = fetch.transform(nextState, value); + } catch (Exception e) { + next.error(e.getMessage()); + return; + } + transactionManager.inTransaction( + transaction -> operation.status(OperationStatus.of(status), transaction)); + next.next(value); + } + + @Override + public JsonNode serializeNestedState(State state) { + return next.serializeNestedState(state); + } + }); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepSubStep.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepSubStep.java new file mode 100644 index 00000000..528c07ef --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatefulStepSubStep.java @@ -0,0 +1,146 @@ +package ca.on.oicr.gsi.vidarr; + +import static ca.on.oicr.gsi.vidarr.OperationAction.MAPPER; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import ca.on.oicr.gsi.vidarr.OperationStatefulStep.Child; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.JsonNode; +import java.util.Optional; + +final class OperationStatefulStepSubStep< + State extends Record, + SubState extends Record, + OriginalState extends Record, + OriginalSubState extends Record, + Input, + Output> + extends OperationStatefulStep, OriginalState, Input, Output> { + + private final StatefulTransformer spawn; + private final OperationAction subtask; + + public OperationStatefulStepSubStep( + StatefulTransformer spawn, + OperationAction subtask) { + super(); + this.spawn = spawn; + this.subtask = subtask; + } + + @Override + Child buildState(State state) { + return new Child<>(Optional.empty(), state); + } + + @Override + JavaType jacksonType(JavaType incoming) { + return MAPPER + .getTypeFactory() + .constructParametricType(Child.class, incoming, subtask.jacksonType()); + } + + @Override + Child rewind( + Child state, OperationAction input) + throws JsonProcessingException { + return new Child<>(Optional.empty(), input.rewind(state.state())); + } + + @Override + public void run( + OperationAction input, + Child state, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow, Output> next) { + state + .child() + .ifPresentOrElse( + childState -> + subtask.run( + childState, + operation, + transactionManager, + new OperationControlFlow() { + @Override + public void cancel() { + next.cancel(); + } + + @Override + public void error(String error) { + next.error(error); + } + + @Override + public void next(Output output) { + if (!operation.isLive()) { + next.cancel(); + return; + } + next.next(output); + } + + @Override + public JsonNode serializeNestedState(SubState subState) { + return next.serializeNestedState( + new Child<>(Optional.of(subState), state.state())); + } + }), + () -> + input.run( + state.state(), + operation, + transactionManager, + new OperationControlFlow<>() { + @Override + public void cancel() { + next.cancel(); + } + + @Override + public void error(String error) { + next.error(error); + } + + @Override + public void next(Input value) { + if (!operation.isLive()) { + next.cancel(); + return; + } + + OriginalSubState originalSubState; + try { + originalSubState = spawn.transform(state.state(), value); + } catch (Exception e) { + next.error(e.getMessage()); + return; + } + transactionManager.scheduleTask( + () -> { + if (!operation.isLive()) { + next.cancel(); + return; + } + final var nextState = + new Child<>( + Optional.of(subtask.buildState(originalSubState)), + state.state()); + final var recoveryState = next.serializeNestedState(nextState); + transactionManager.inTransaction( + tx -> operation.recoveryState(recoveryState, tx)); + transactionManager.scheduleTask( + () -> run(input, nextState, operation, transactionManager, next)); + }); + } + + @Override + public JsonNode serializeNestedState(State state) { + return next.serializeNestedState(new Child<>(Optional.empty(), state)); + } + })); + } +} diff --git a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/OperationStatus.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatus.java similarity index 75% rename from vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/OperationStatus.java rename to vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatus.java index 2f58c333..838db03d 100644 --- a/vidarr-core/src/main/java/ca/on/oicr/gsi/vidarr/core/OperationStatus.java +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStatus.java @@ -1,6 +1,4 @@ -package ca.on.oicr.gsi.vidarr.core; - -import ca.on.oicr.gsi.vidarr.WorkMonitor.Status; +package ca.on.oicr.gsi.vidarr; /** The status for an operation */ public enum OperationStatus { @@ -13,7 +11,7 @@ public enum OperationStatus { FAILED, SUCCEEDED; - public static OperationStatus of(Status status) { + public static OperationStatus of(WorkingStatus status) { return switch (status) { case WAITING -> PLUGIN_WAITING; case QUEUED -> PLUGIN_QUEUED; diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStep.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStep.java new file mode 100644 index 00000000..cdda640f --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStep.java @@ -0,0 +1,295 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import com.fasterxml.jackson.databind.JsonNode; +import io.prometheus.client.Counter; +import java.lang.ProcessBuilder.Redirect; +import java.lang.System.Logger.Level; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandler; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.function.Supplier; + +/** + * Operation steps modify the values being carried by an {@link OperationAction} without changing + * the state + * + *

It's reasonable to consider every step as a function that takes an input value and provides an + * output value. Unlike a normal function, steps can be asynchronous and write information to + * Vidarr's database. + * + * @param the parameter type + * @param the return type + */ +public abstract sealed class OperationStep + permits OperationStepCompletableFuture, + OperationStepDebugInfo, + OperationStepLog, + OperationStepMapping, + OperationStepMonitor, + OperationStepRequire, + OperationStepStatus, + OperationStepThen { + + /** + * A simple mapping function + * + *

This is analogous to {@link java.util.function.Function}, but it can throw an exception + * which will be caught and logged to Vidarr's database. + * + * @param the parameter type + * @param the return type + */ + public interface Transformer { + + /** + * Call the function with an argument + * + * @param input the argument to use + * @return the transformed value + * @throws Exception any exceptions will be caught and redirected through Vidarr's logging and + * operation framework + */ + Output transform(Input input) throws Exception; + } + + private static final HttpClient HTTP_CLIENT = HttpClient.newHttpClient(); + + /** + * Write new debugging information for this value + * + * @param fetch the transformation to produce debugging information to write to the database. + * Vidarr imposes no schema on this data; it is up to the client to interpret it + * @return a step to perform write this debugging information + * @param the type of the input value + */ + public static OperationStep debugInfo(Transformer fetch) { + return new OperationStepDebugInfo<>(fetch); + } + + /** + * Perform an asynchronous step using a {@link CompletableFuture} from Java's thread pool + * infrastructure + * + * @return a step to create and wait for a future + * @param the type produced by the completable future + */ + public static OperationStep, Value> future() { + return new OperationStepCompletableFuture<>(); + } + + /** + * Perform an HTTP request and collect the output + * + * @param body the handler to extract the body of the HTTP request + * @return a step to perform this HTTP request + * @param the type of the response body + */ + public static OperationStep> http(BodyHandler body) { + return OperationStep.>>mapping( + httpRequest -> HTTP_CLIENT.sendAsync(httpRequest, body)) + .then(future()); + } + + /** + * Check if an HTTP response was successful + * + * @param response the HTTP response to check + * @return true if the code is 2xx; false otherwise + */ + public static boolean isHttpOk(HttpResponse response) { + return response.statusCode() / 100 == 2; + } + + /** + * Log information about the current value + * + * @param level the logging level + * @param message a transformer to generate the log message + * @return a step to write this log message + * @param the type of the input value + */ + public static OperationStep log( + Level level, Transformer message) { + return new OperationStepLog<>(level, message); + } + + /** + * Change the input value using a function + * + * @param transformer the function to apply to the input value + * @return a step to call this function + * @param the type of the input + * @param the type of the output + */ + public static OperationStep mapping( + Transformer transformer) { + return new OperationStepMapping<>(transformer); + } + + /** + * Increment a Prometheus counter + * + * @param counter the counter to increment + * @param labels the label values for the counter + * @return a step to increment this counter + * @param the type of the input and (unchanged) output + */ + public static OperationStep monitor(Counter counter, String... labels) { + return monitorWhen(counter, x -> true, labels); + } + + /** + * Increment a Prometheus counter if a condition + * + * @param counter the counter to increment + * @param success a test to determine if the counter should be incremented + * @param labels the label values for the counter + * @return a step to increment this counter + * @param the type of the input and (unchanged) output + */ + public static OperationStep monitorWhen( + Counter counter, Predicate success, String... labels) { + return new OperationStepMonitor<>(counter, success, labels); + } + + /** + * Abort the operation if a condition is not met + * + * @param success the test to determine if the sequence should continue (true) or go into an error + * state (false) + * @param failureMessage the message to display when a failure occurs + * @return a step to check this condition + * @param the type of the input and (unchanged) output + */ + public static OperationStep require( + Predicate success, String failureMessage) { + return new OperationStepRequire<>(success, failureMessage); + } + + /** + * Check that an HTTP response returned a 2xx code and extract the body + * + * @return a step that performs this check and extraction + * @param the type of the HTTP response body + */ + public static OperationStep, Body> requireHttpSuccess() { + return mapping( + response -> { + if (isHttpOk(response)) { + return response.body(); + } else { + throw new RuntimeException( + String.format( + "HTTP request to %s failed with status: %d", + response.uri(), response.statusCode())); + } + }); + } + + /** + * Check that an HTTP response with a JSON body returned a 2xx code and decode the body + * + * @return a step that performs this check and extraction + * @param the type of the HTTP response JSON + */ + public static OperationStep>, Body> requireJsonSuccess() { + return OperationStep.>requireHttpSuccess().then(mapping(Supplier::get)); + } + + /** + * Require an optional value is not empty + * + * @return a step that performs this check + * @param the value inside the optional + */ + public static OperationStep, Value> requirePresent() { + return mapping(Optional::orElseThrow); + } + + /** + * Unconditionally update the status of the operation + * + * @param status the status to change to + * @return a step that changes the status + * @param the type of the input and (unchanged) output + */ + public static OperationStep status(WorkingStatus status) { + return status(v -> status); + } + + /** + * Change the status of this operation based on the current value + * + * @param fetch a function that examines the current value and produces a corresponding status + * @return a step that changes the status + * @param the type of the input and (unchanged) output + */ + public static OperationStep status( + Transformer fetch) { + return new OperationStepStatus<>(fetch); + } + + /** + * Launch a program on the system running the Vidarr server + * + * @param output the handling of standard output that is desired + * @return a step that runs this process + * @param the data collected from standard output + */ + public static OperationStep> subprocess( + ProcessOutputHandler output) { + return mapping( + input -> { + final var build = new ProcessBuilder().command(input.command()); + if (input.standardInput().isPresent()) { + build.redirectInput(Redirect.PIPE); + } + final var outputGenerator = output.prepare(build); + final var process = build.start(); + if (input.standardInput().isPresent()) { + try (final var stdin = process.getOutputStream()) { + stdin.write(input.standardInput().get()); + } + } + if (input.maximumWait().isPresent()) { + final var duration = input.maximumWait().get(); + if (!process.waitFor(duration.get(TimeUnit.SECONDS.toChronoUnit()), TimeUnit.SECONDS)) { + process.destroy(); + throw new RuntimeException( + String.format("Killed process %d after timeout", process.pid())); + } + } else { + process.waitFor(); + } + return new ProcessOutput<>( + process.exitValue(), outputGenerator.get(process.exitValue() == 0)); + }); + } + + abstract void run( + Input input, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next); + + /** + * Perform two operations in sequence + * + *

This provides the same functionality as {@link OperationAction#then(OperationStep)}. + * Normally, the {@link OperationAction#then(OperationStep)} makes for more linear, readable code, + * but this method can be useful for pre-composing utility steps. + * + * @param step the following step + * @return a combined step + * @param the final output after the second step is applied + */ + public final OperationStep then(OperationStep step) { + return new OperationStepThen<>(this, step); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepCompletableFuture.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepCompletableFuture.java new file mode 100644 index 00000000..71f1638f --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepCompletableFuture.java @@ -0,0 +1,28 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import java.util.concurrent.CompletableFuture; + +final class OperationStepCompletableFuture + extends OperationStep, Value> { + + public OperationStepCompletableFuture() { + super(); + } + + @Override + public void run( + CompletableFuture input, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next) { + input.whenComplete( + (result, throwable) -> { + if (throwable == null) { + next.next(result); + } else { + next.error(throwable.getMessage()); + } + }); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepDebugInfo.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepDebugInfo.java new file mode 100644 index 00000000..50591384 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepDebugInfo.java @@ -0,0 +1,30 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import com.fasterxml.jackson.databind.JsonNode; + +final class OperationStepDebugInfo extends OperationStep { + + private final Transformer debugInfo; + + public OperationStepDebugInfo(Transformer debugInfo) { + this.debugInfo = debugInfo; + } + + @Override + public void run( + Value input, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next) { + final JsonNode newDebugInfo; + try { + newDebugInfo = debugInfo.transform(input); + } catch (Exception e) { + next.error(e.getMessage()); + return; + } + transactionManager.inTransaction(tx -> operation.debugInfo(newDebugInfo, tx)); + next.next(input); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepLog.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepLog.java new file mode 100644 index 00000000..ff8da448 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepLog.java @@ -0,0 +1,31 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import java.lang.System.Logger.Level; + +final class OperationStepLog extends OperationStep { + + private final Level level; + private final Transformer message; + + public OperationStepLog(Level level, Transformer message) { + super(); + this.level = level; + this.message = message; + } + + @Override + public void run( + Value input, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next) { + try { + operation.log(level, message.transform(input)); + } catch (Exception e) { + next.error(e.getMessage()); + return; + } + next.next(input); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepMapping.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepMapping.java new file mode 100644 index 00000000..9c23f407 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepMapping.java @@ -0,0 +1,28 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; + +final class OperationStepMapping extends OperationStep { + + private final Transformer transformer; + + OperationStepMapping(Transformer transformer) { + this.transformer = transformer; + } + + @Override + public void run( + Input input, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next) { + final Output result; + try { + result = transformer.transform(input); + } catch (Exception e) { + next.error(e.getMessage()); + return; + } + next.next(result); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepMonitor.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepMonitor.java new file mode 100644 index 00000000..d4dcc4e6 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepMonitor.java @@ -0,0 +1,35 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import io.prometheus.client.Counter; +import java.util.function.Predicate; + +final class OperationStepMonitor extends OperationStep { + + private final Counter counter; + private final String[] labels; + private final Predicate success; + + OperationStepMonitor(Counter counter, Predicate success, String[] labels) { + this.counter = counter; + this.success = success; + this.labels = labels; + } + + @Override + public void run( + Value input, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next) { + try { + if (success.test(input)) { + counter.labels(labels).inc(); + } + } catch (Exception e) { + next.error(e.getMessage()); + return; + } + next.next(input); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepRequire.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepRequire.java new file mode 100644 index 00000000..70b12c5b --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepRequire.java @@ -0,0 +1,36 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import java.util.function.Predicate; + +final class OperationStepRequire extends OperationStep { + + private final String failureMessage; + private final Predicate success; + + public OperationStepRequire(Predicate success, String failureMessage) { + super(); + this.success = success; + this.failureMessage = failureMessage; + } + + @Override + public void run( + Result result, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next) { + boolean wasSuccess; + try { + wasSuccess = success.test(result); + } catch (Exception e) { + next.error(e.getMessage()); + return; + } + if (wasSuccess) { + next.next(result); + } else { + next.error(failureMessage); + } + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepStatus.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepStatus.java new file mode 100644 index 00000000..f86b7d61 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepStatus.java @@ -0,0 +1,29 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; + +final class OperationStepStatus extends OperationStep { + + private final Transformer status; + + public OperationStepStatus(Transformer status) { + this.status = status; + } + + @Override + public void run( + Value input, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next) { + final WorkingStatus newStatus; + try { + newStatus = status.transform(input); + } catch (Exception e) { + next.error(e.getMessage()); + return; + } + transactionManager.inTransaction(tx -> operation.status(OperationStatus.of(newStatus), tx)); + next.next(input); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepThen.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepThen.java new file mode 100644 index 00000000..0fda40f2 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OperationStepThen.java @@ -0,0 +1,49 @@ +package ca.on.oicr.gsi.vidarr; + +import ca.on.oicr.gsi.vidarr.ActiveOperation.TransactionManager; +import com.fasterxml.jackson.databind.JsonNode; + +final class OperationStepThen extends OperationStep { + + private final OperationStep first; + private final OperationStep second; + + public OperationStepThen( + OperationStep first, OperationStep second) { + this.first = first; + this.second = second; + } + + @Override + public void run( + Input input, + ActiveOperation operation, + TransactionManager transactionManager, + OperationControlFlow next) { + first.run( + input, + operation, + transactionManager, + new OperationControlFlow() { + @Override + public void cancel() { + next.cancel(); + } + + @Override + public void error(String error) { + next.error(error); + } + + @Override + public void next(Intermediate output) { + second.run(output, operation, transactionManager, next); + } + + @Override + public JsonNode serializeNestedState(State state) { + return next.serializeNestedState(state); + } + }); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OutputProvisioner.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OutputProvisioner.java index 247872ce..507bc201 100644 --- a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OutputProvisioner.java +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OutputProvisioner.java @@ -29,12 +29,15 @@ * *

OutputProvisioner uses jackson-databind to map information from the server's '.vidarrconfig' * file to member non-static fields. The @JsonIgnore annotation prevents this. + * + * @param the state information used to run the preflight check + * @param the state information used for provisioning out data */ @JsonTypeIdResolver(OutputProvisioner.OutputProvisionerIdResolver.class) @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, include = As.PROPERTY, property = "type") -public interface OutputProvisioner { +public interface OutputProvisioner { final class OutputProvisionerIdResolver extends TypeIdResolverBase { - private final Map> knownIds = + private final Map>> knownIds = ServiceLoader.load(OutputProvisionerProvider.class).stream() .map(Provider::get) .flatMap(OutputProvisionerProvider::types) @@ -141,63 +144,38 @@ private Result() {} void configuration(SectionRenderer sectionRenderer) throws XMLStreamException; /** - * Check that the metadata provided by the submitter is valid. + * Prepare state to check that the metadata provided by the submitter is valid. * * @param metadata the metadata provided by the submitter - * @param monitor the monitor structure for writing the output of the checking process - */ - JsonNode preflightCheck(JsonNode metadata, WorkMonitor monitor); - - /** - * Restart a preflight check process from state saved in the database - * - * @param state the frozen database state - * @param monitor the monitor structure for writing the output of the provisioning process */ - void preflightRecover(JsonNode state, WorkMonitor monitor); + PreflightState preflightCheck(JsonNode metadata); /** - * Begin provisioning out a new output + * Prepare state to provision out data * *

This method should not do any externally-visible work. Anything it needs should be done in a - * {@link WorkMonitor#scheduleTask(Runnable)} callback so that Vidarr can execute it once the - * database is in a healthy state. + * {@link #run()} so that Vidarr can execute it once the database is in a healthy state. * * @param workflowRunId the workflow run ID assigned by Vidarr * @param data the output coming from the workflow * @param metadata the information coming from the submitter to direct provisioning - * @param monitor the monitor structure for writing the output of the provisioning process * @return the initial state of the provision out process */ - JsonNode provision( - String workflowRunId, String data, JsonNode metadata, WorkMonitor monitor); + State provision(String workflowRunId, String data, JsonNode metadata); /** - * Restart a provisioning process from state saved in the database + * Create a declarative structure to execute the provisioning * - *

This method should not do any externally-visible work. Anything it needs should be done in a - * {@link WorkMonitor#scheduleTask(Runnable)} callback so that Vidarr can execute it once the - * database is in a healthy state. - * - * @param state the frozen database state - * @param monitor the monitor structure for writing the output of the provisioning process + * @return the sequence of operations that should be performed */ - void recover(JsonNode state, WorkMonitor monitor); + OperationAction run(); /** - * Restart a provisioning process from state saved in the database from a failed task. - * - *

This method should not do any externally-visible work. Anything it needs should be done in a - * {@link WorkMonitor#scheduleTask(Runnable)} callback so that Vidarr can execute it once the - * database is in a healthy state. - * - *

This is meant to allow retrying the provision out process after a failure such as out of - * disk that doesn't require reprocessing the data. + * Create a declarative structure to execute the pre-flight check * - * @param state the frozen database state - * @param monitor the monitor structure for writing the output of the provisioning process + * @return the sequence of operations that should be performed */ - void retry(JsonNode state, WorkMonitor monitor); + OperationAction runPreflight(); /** * Called to initialise this output provisioner. diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OutputProvisionerProvider.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OutputProvisionerProvider.java index 08462eac..727d6efc 100644 --- a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OutputProvisionerProvider.java +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/OutputProvisionerProvider.java @@ -6,5 +6,5 @@ /** Reads JSON configuration and instantiates provisioners appropriately */ public interface OutputProvisionerProvider { /** Provides the type names and classes this plugin provides */ - Stream>> types(); + Stream>>> types(); } diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/PollResult.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/PollResult.java new file mode 100644 index 00000000..fc28d592 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/PollResult.java @@ -0,0 +1,65 @@ +package ca.on.oicr.gsi.vidarr; + +import java.time.Duration; + +/** Indicate how a {@link OperationStatefulStep#poll(Duration)} step should proceed */ +public abstract sealed class PollResult + permits PollResultActive, PollResultFailed, PollResultFinished { + + /** Consumes a poll result */ + public interface Visitor { + + /** + * Indicates the task is ongoing. + * + * @param status the current status associated with the task + */ + void active(WorkingStatus status); + + /** + * Indicates the task has failed. + * + * @param error an error associated with the task + */ + void failed(String error); + + /** Indicates the task has finished successfully. */ + void finished(); + } + + /** + * Indicate that the task is still ongoing and will need to be polled again + * + * @param status the current status of the task, for setting the Vidarr status + * @return the indicator to the poll task + */ + public static PollResult active(WorkingStatus status) { + return new PollResultActive(status); + } + + /** + * Indicate that the task has failed and the steps need to enter an error state + * + * @param error the error message to provide + * @return the indicator to the poll task + */ + public static PollResult failed(String error) { + return new PollResultFailed(error); + } + + /** + * Indicate that the task has finished successfully and the next step should be run + * + * @return the indicator to the poll task + */ + public static PollResult finished() { + return new PollResultFinished(); + } + + /** + * Check which state the result is in + * + * @param visitor a consumer of the result + */ + public abstract void visit(Visitor visitor); +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/PollResultActive.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/PollResultActive.java new file mode 100644 index 00000000..6621c77e --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/PollResultActive.java @@ -0,0 +1,16 @@ +package ca.on.oicr.gsi.vidarr; + +final class PollResultActive extends PollResult { + + private final WorkingStatus status; + + public PollResultActive(WorkingStatus status) { + super(); + this.status = status; + } + + @Override + public void visit(Visitor visitor) { + visitor.active(status); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/PollResultFailed.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/PollResultFailed.java new file mode 100644 index 00000000..34d5c32f --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/PollResultFailed.java @@ -0,0 +1,16 @@ +package ca.on.oicr.gsi.vidarr; + +final class PollResultFailed extends PollResult { + + private final String message; + + public PollResultFailed(String message) { + super(); + this.message = message; + } + + @Override + public void visit(Visitor visitor) { + visitor.failed(message); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/PollResultFinished.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/PollResultFinished.java new file mode 100644 index 00000000..50eec150 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/PollResultFinished.java @@ -0,0 +1,9 @@ +package ca.on.oicr.gsi.vidarr; + +final class PollResultFinished extends PollResult { + + @Override + public void visit(Visitor visitor) { + visitor.finished(); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessInput.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessInput.java new file mode 100644 index 00000000..c277fe6c --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessInput.java @@ -0,0 +1,14 @@ +package ca.on.oicr.gsi.vidarr; + +import java.time.Duration; +import java.util.Optional; + +/** + * The information required to run a local process + * + * @param standardInput data to be provided to standard input + * @param maximumWait the maximum allowed runtime for a process + * @param command the command to run and its arguments + */ +public record ProcessInput( + Optional standardInput, Optional maximumWait, String... command) {} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutput.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutput.java new file mode 100644 index 00000000..c98bcd75 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutput.java @@ -0,0 +1,20 @@ +package ca.on.oicr.gsi.vidarr; + +/** + * The output status information from a locally-run process/program + * + * @param exitCode the process's final exit code + * @param standardOutput the data gathered from standard output + * @param the type of the standard output data + */ +public record ProcessOutput(int exitCode, Body standardOutput) { + + /** + * Checks if the process exited successfully + * + * @return true if the exit code is zero + */ + public boolean success() { + return exitCode == 0; + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutputAsJson.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutputAsJson.java new file mode 100644 index 00000000..ede0db08 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutputAsJson.java @@ -0,0 +1,34 @@ +package ca.on.oicr.gsi.vidarr; + +import static ca.on.oicr.gsi.vidarr.OperationAction.MAPPER; + +import com.fasterxml.jackson.databind.JavaType; +import java.io.File; +import java.io.IOException; + +final class ProcessOutputAsJson extends ProcessOutputHandler { + + private final boolean successOnly; + private final JavaType type; + + public ProcessOutputAsJson(JavaType type, boolean successOnly) { + this.type = type; + this.successOnly = successOnly; + } + + @Override + public OutputGenerator prepare(ProcessBuilder build) throws IOException { + final var output = File.createTempFile("vidarr", ".out"); + output.deleteOnExit(); + return (boolean success) -> { + if (!success && successOnly) { + return null; + } + try { + return MAPPER.readValue(output, type); + } finally { + output.delete(); + } + }; + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutputHandler.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutputHandler.java new file mode 100644 index 00000000..7188943f --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutputHandler.java @@ -0,0 +1,69 @@ +package ca.on.oicr.gsi.vidarr; + +import static ca.on.oicr.gsi.vidarr.OperationAction.MAPPER; + +import com.fasterxml.jackson.core.type.TypeReference; +import java.io.IOException; +import java.nio.file.Path; + +/** + * Determines how the standard output of a process is handled + * + * @param the type of data that will be returned to the operation + */ +public abstract sealed class ProcessOutputHandler + permits ProcessOutputToStandardOutput, ProcessOutputToFile, ProcessOutputAsJson { + public interface OutputGenerator { + Output get(boolean success) throws Exception; + } + + /** + * Parse standard output as a JSON object + * + * @param clazz a Java class to indicate the kind of JSON data + * @param successOnly if true and the process exists with non-zero exit status, the output will + * not attempt to be parsed and a null value will be returned instead. + * @return a handler that will parse the JSON data + * @param the type of JSON data being parsed + */ + public static ProcessOutputHandler readOutput( + Class clazz, boolean successOnly) { + return new ProcessOutputAsJson<>(MAPPER.getTypeFactory().constructType(clazz), successOnly); + } + + /** + * Parse standard output as a JSON object + * + * @param typeReference a type reference to indicate the kind of JSON data + * @param successOnly if true and the process exists with non-zero exit status, the output will + * not attempt to be parsed and a null value will be returned instead. + * @return a handler that will parse the JSON data + * @param the type of JSON data being parsed + */ + public static ProcessOutputHandler readOutput( + TypeReference typeReference, boolean successOnly) { + return new ProcessOutputAsJson<>( + MAPPER.getTypeFactory().constructType(typeReference), successOnly); + } + + /** + * Write the data to a temporary file and provide the path to that file + * + * @return handler that will cause the child process to write to a file; if the file is not + * deleted, it will be deleted on JVM exit + */ + public static ProcessOutputHandler toFile() { + return new ProcessOutputToFile(); + } + + /** + * Write the data to Vidarr's standard output and provide nothing + * + * @return handler that will cause the child process to inherit the server's standard output + */ + public static ProcessOutputHandler toStandardOutput() { + return new ProcessOutputToStandardOutput(); + } + + abstract OutputGenerator prepare(ProcessBuilder build) throws IOException; +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutputToFile.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutputToFile.java new file mode 100644 index 00000000..ba7faef2 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutputToFile.java @@ -0,0 +1,15 @@ +package ca.on.oicr.gsi.vidarr; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; + +final class ProcessOutputToFile extends ProcessOutputHandler { + + @Override + public OutputGenerator prepare(ProcessBuilder build) throws IOException { + final var output = File.createTempFile("vidarr", ".out"); + output.deleteOnExit(); + return (success) -> output.toPath(); + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutputToStandardOutput.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutputToStandardOutput.java new file mode 100644 index 00000000..cc854282 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/ProcessOutputToStandardOutput.java @@ -0,0 +1,12 @@ +package ca.on.oicr.gsi.vidarr; + +import java.lang.ProcessBuilder.Redirect; + +final class ProcessOutputToStandardOutput extends ProcessOutputHandler { + + @Override + public OutputGenerator prepare(ProcessBuilder build) { + build.redirectOutput(Redirect.INHERIT); + return (success) -> null; + } +} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/RuntimeProvisioner.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/RuntimeProvisioner.java index 1aa925a1..6c018ed7 100644 --- a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/RuntimeProvisioner.java +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/RuntimeProvisioner.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; import com.fasterxml.jackson.databind.DatabindContext; import com.fasterxml.jackson.databind.JavaType; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.annotation.JsonTypeIdResolver; import com.fasterxml.jackson.databind.jsontype.impl.TypeIdResolverBase; import java.io.IOException; @@ -30,14 +29,16 @@ * *

RuntimeProvisioner uses jackson-databind to map information from the server's '.vidarrconfig' * file to member non-static fields. The @JsonIgnore annotation prevents this. + * + * @param the state information used for provisioning out data */ @JsonTypeIdResolver(RuntimeProvisioner.RuntimeProvisionerIdResolver.class) @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, include = As.PROPERTY, property = "type") -public interface RuntimeProvisioner { +public interface RuntimeProvisioner { final class RuntimeProvisionerIdResolver extends TypeIdResolverBase { - private final Map> knownIds = + private final Map>> knownIds = ServiceLoader.load(RuntimeProvisionerProvider.class).stream() .map(Provider::get) .flatMap(RuntimeProvisionerProvider::types) @@ -78,39 +79,21 @@ public JavaType typeFromId(DatabindContext context, String id) throws IOExceptio String name(); /** - * Begin provisioning out a new output - * - * @param workflowRunUrl the URL provided by the {@link WorkflowEngine.Result#workflowRunUrl()} - * @param monitor WorkMonitor for writing the output of the checking process and scheduling - * asynchronous tasks. OutputProvisioner.Result is the expected output type. JsonNode is the - * format of the state records. - * @return JsonNode used by WrappedMonitor in BaseProcessor.Phase3Run to serialize to the database - */ - JsonNode provision( - String workflowRunUrl, WorkMonitor monitor); - - /** - * Restart a provisioning process from state saved in the database + * Create state for provisioning the output * - *

Rebuild state from `state` object then schedule appropriate next step with - * `monitor.scheduleTask()` + *

This method should do not work. It should only create state for use by the {@link #run()} * - * @param state the frozen database state - * @param monitor the monitor structure for writing the output of the provisioning process + * @param workflowRunUrl the URL provided by the {@link WorkflowEngine.Result#workflowRunUrl()} + * @return the state that will be used by {@link #run()} */ - void recover(JsonNode state, WorkMonitor monitor); + State provision(String workflowRunUrl); /** - * Restart a provisioning process from state saved in the database that previously failed - * - *

Rebuild state from `state` object then schedule appropriate next step with - * `monitor.scheduleTask()`. This is meant to allow retrying the provision out process after a - * failure such as out of disk that doesn't require reprocessing the data. + * Create a declarative structure to execute the provisioning * - * @param state the frozen database state - * @param monitor the monitor structure for writing the output of the provisioning process + * @return the sequence of operations that should be performed */ - void retry(JsonNode state, WorkMonitor monitor); + OperationAction run(); /** * Called to initialise this runtime provisioner. diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/RuntimeProvisionerProvider.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/RuntimeProvisionerProvider.java index ae402bd8..6d2593e8 100644 --- a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/RuntimeProvisionerProvider.java +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/RuntimeProvisionerProvider.java @@ -6,5 +6,5 @@ /** Reads JSON configuration and instantiates provisioners appropriately */ public interface RuntimeProvisionerProvider { /** Provides the type names and classes this plugin provides */ - Stream>> types(); + Stream>>> types(); } diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/WorkMonitor.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/WorkMonitor.java deleted file mode 100644 index 9609a87e..00000000 --- a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/WorkMonitor.java +++ /dev/null @@ -1,137 +0,0 @@ -package ca.on.oicr.gsi.vidarr; - -import com.fasterxml.jackson.databind.JsonNode; -import java.util.concurrent.TimeUnit; - -/** - * A mechanism to inform the Vidarr server about the state of a task - * - *

When Vidarr asks a plugin to launch a job for running or provisioning, it provides an object - * for the plugin to use to manage the state of that record. Vidarr does not impose a particular - * scheduling or process behaviour on plugins. Plugins may manage their tasks however they please - * and asynchronously report their status back to Vidarr using a monitor. - * - *

complete() and permanentFailure() are not to be called directly, but rather inside the - * Runnable parameter of a call to scheduleTask(). This is to prevent database corruption - Vidarr - * needs to wait for database transactions to complete before doing any scheduled tasks, including - * marking tasks as completed or failed. Calling these methods directly would risk trying to write - * during or after that transaction rolls back in an error state. - * - *

See vidarr-core BaseProcessor.java's startNextPhase(), BaseProcessor's BaseOperationMonitor, - * and vidarr-server DatabaseWorkflow's phase() - * - * @param the output type expected from the task - * @param the format state information is stored in - */ -public interface WorkMonitor { - - /** The status, as reported to the user, about a task */ - enum Status { - /** - * The task's state cannot be accurately determined - * - *

This may indicate a remote system is not responding. - */ - UNKNOWN, - /** The task is not executing because it is waiting for resources to become available */ - WAITING, - /** The task has been scheduled for execution */ - QUEUED, - /** The task is currently executing */ - RUNNING, - /** The task has been requested to stop executing for load reasons */ - THROTTLED - } - - /** - * The task is complete - * - *

Do not call directly from plugin. Call inside the Runnable parameter of {@link - * #scheduleTask(Runnable)} This is to prevent database corruption. - * - *

This can only be called once and cannot be called if {@link #permanentFailure(String)} has - * been called. - * - * @param result the output data from the task - */ - void complete(T result); - - /** - * Write something interesting - * - * @param level how important this message is - * @param message the message to display - */ - void log(System.Logger.Level level, String message); - - /** - * Indicate that the task is unrecoverably broken - * - *

Do not call directly from plugin. Call inside the Runnable parameter of {@link - * #scheduleTask(Runnable)} This is to prevent database corruption. - * - *

Once called, the workflow and related provisioning steps will be considered a failure. - * - *

This cannot be called after {@link #complete(Object)} - * - * @param reason the reason for the failure - */ - void permanentFailure(String reason); - - /** - * Request that Vidarr schedule a callback at the next available opportunity - * - *

This cannot be called after {@link #complete(Object)} or {@link #permanentFailure(String)} - * - * @param task the task to execute - */ - void scheduleTask(Runnable task); - - /** - * Request that Vidarr schedule a callback at a specified time in the future - * - *

This scheduling is best-effort; Vidarr may execute a task earlier or later than requested - * based on load and priority. - * - *

This cannot be called after {@link #complete(Object)} or {@link #permanentFailure(String)} - * - * @param delay the amount of time to wait; if delay is < 1, this is equivalent to {@link - * #scheduleTask(Runnable)} - * @param units the time units to wait - * @param task the task to execute - */ - void scheduleTask(long delay, TimeUnit units, Runnable task); - - /** - * Write debugging status information that will be made available to clients - * - * @param information the current information to be presented - */ - void storeDebugInfo(JsonNode information); - - /** - * Write the recovery information to stable store - * - *

If Vidarr is shutdown or crashes during execution, it will attempt to recover all running - * tasks using information stored in its database. This updates this data for this task in the - * case of crash recovery. The plugin is responsible for interpreting this data and being able to - * recover state from other versions of the plugin in the case of upgrade. - * - *

This cannot be called after {@link #complete(Object)} or {@link #permanentFailure(String)} - * - * @param state the data to store - */ - void storeRecoveryInformation(S state); - - /** - * Update the state of the task - * - *

Vidarr uses this state information to provide more detailed information to the user about - * the health of a task. It does not affect decision-making. - * - *

This cannot be called after {@link #complete(Object)} or {@link #permanentFailure(String)} - * - * @param status the new status of the task - */ - void updateState(Status status); -} diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/WorkflowEngine.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/WorkflowEngine.java index 4ac5a3ba..771bb467 100644 --- a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/WorkflowEngine.java +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/WorkflowEngine.java @@ -1,5 +1,7 @@ package ca.on.oicr.gsi.vidarr; +import static ca.on.oicr.gsi.vidarr.OperationAction.MAPPER; + import ca.on.oicr.gsi.Pair; import ca.on.oicr.gsi.status.SectionRenderer; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -21,12 +23,39 @@ import java.util.stream.Stream; import javax.xml.stream.XMLStreamException; -/** Defines an engine that knows how to execute workflows and track the results */ +/** + * Defines an engine that knows how to execute workflows and track the results + * + * @param the initial state information used to launch the workflow + * @param the state used to perform any cleanup after provision out + */ @JsonTypeIdResolver(WorkflowEngine.WorkflowEngineIdResolver.class) @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, include = As.PROPERTY, property = "type") -public interface WorkflowEngine { +public interface WorkflowEngine { + + /** + * The output data from a workflow + * + * @param output the JSON data emitted by the workflow + * @param workflowRunUrl the URL of the completed workflow run for provisioning metrics and logs + * @param cleanupState a state that can be used later to trigger cleanup of the workflow's output + * once provisioning out has been completed + */ + record Result(JsonNode output, String workflowRunUrl, Optional cleanupState) { + + /** + * Convert the cleanup state to a JSON object + * + * @return a replacement output with the cleanup state serialized, if present + */ + public Result serialize() { + return new WorkflowEngine.Result<>( + output(), workflowRunUrl(), cleanupState().map(MAPPER::valueToTree)); + } + } + final class WorkflowEngineIdResolver extends TypeIdResolverBase { - private final Map> knownIds = + private final Map>> knownIds = ServiceLoader.load(WorkflowEngineProvider.class).stream() .map(Provider::get) .flatMap(WorkflowEngineProvider::types) @@ -58,58 +87,13 @@ public JavaType typeFromId(DatabindContext context, String id) throws IOExceptio } } - /** The output data from a workflow */ - class Result { - private final Optional cleanupState; - private final JsonNode output; - private final String workflowRunUrl; - - /** - * Create new workflow output data - * - * @param output the JSON data emitted by the workflow - * @param workflowRunUrl the URL of the completed workflow run for provisioning metrics and logs - * @param cleanupState a state that can be used later to trigger cleanup of the workflow's - * output once provisioning out has been completed - */ - public Result(JsonNode output, String workflowRunUrl, Optional cleanupState) { - this.output = output; - this.workflowRunUrl = workflowRunUrl; - this.cleanupState = cleanupState; - } - - /** The information the plugin requires to clean up the output, if required. */ - public Optional cleanupState() { - return cleanupState; - } - - /** The workflow output's output */ - public JsonNode output() { - return output; - } - - /** - * The URL identifying the workflow run so the metrics and logs workflows can collect - * information about the completed job. - */ - public String workflowRunUrl() { - return workflowRunUrl; - } - } - /** - * Clean up the output of a workflow (i.e., delete its on-disk output) after provisioning has been - * completed. - * - *

This method should not do any externally-visible work. Anything it needs should be done in a - * {@link WorkMonitor#scheduleTask(Runnable)} callback so that Vidarr can execute it once the - * database is in a healthy state. + * Create a declarative structure to clean up the output of a workflow * - * @param cleanupState the clean up state provided with the workflow's output - * @param monitor the monitor structure for clean up process; since no output is required, supply - * null as the output value + *

This method should not do any externally-visible work. It creates a set of operations so + * that Vidarr can execute it once the database is in a healthy state. */ - JsonNode cleanup(JsonNode cleanupState, WorkMonitor monitor); + OperationAction cleanup(); /** Display configuration status */ void configuration(SectionRenderer sectionRenderer) throws XMLStreamException; @@ -123,35 +107,17 @@ public String workflowRunUrl() { Optional engineParameters(); /** - * Restart a running process from state saved in the database - * - *

This method should not do any externally-visible work. Anything it needs should be done in a - * {@link WorkMonitor#scheduleTask(Runnable)} callback so that Vidarr can execute it once the - * database is in a healthy state. - * - * @param state the frozen database state - * @param monitor the monitor structure for writing the output of the workflow process - */ - void recover(JsonNode state, WorkMonitor, JsonNode> monitor); - - /** - * Restart a running clean up process from state saved in the database - * - *

This method should not do any externally-visible work. Anything it needs should be done in a - * {@link WorkMonitor#scheduleTask(Runnable)} callback so that Vidarr can execute it once the - * database is in a healthy state. + * Create a declarative structure to execute a workflow * - * @param state the frozen database state - * @param monitor the monitor structure for writing the output of the cleanup process + * @return the sequence of operations that should be performed */ - void recoverCleanup(JsonNode state, WorkMonitor monitor); + OperationAction> run(); /** * Start a new workflow * - *

This method should not do any externally-visible work. Anything it needs should be done in a - * {@link WorkMonitor#scheduleTask(Runnable)} callback so that Vidarr can execute it once the - * database is in a healthy state. + *

This method should not do any externally-visible work. It should simply populate the state + * structure that will be used by {@link #run()}. * * @param workflowLanguage the language the workflow was written in * @param workflow the contents of the workflow @@ -159,17 +125,15 @@ public String workflowRunUrl() { * @param vidarrId the ID of the workflow being executed * @param workflowParameters the input parameters to the workflow * @param engineParameters the input configuration parameters to the workflow engine - * @param monitor the monitor structure for writing the output of the workflow process * @return the initial state of the provision out process */ - JsonNode run( + State start( WorkflowLanguage workflowLanguage, String workflow, Stream> accessoryFiles, String vidarrId, ObjectNode workflowParameters, - JsonNode engineParameters, - WorkMonitor, JsonNode> monitor); + JsonNode engineParameters); /** * Called to initialise this workflow engine. diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/WorkflowEngineProvider.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/WorkflowEngineProvider.java index ee763882..59063093 100644 --- a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/WorkflowEngineProvider.java +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/WorkflowEngineProvider.java @@ -7,5 +7,5 @@ public interface WorkflowEngineProvider { /** Provides the type names and classes this plugin provides */ - Stream>> types(); + Stream>>> types(); } diff --git a/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/WorkingStatus.java b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/WorkingStatus.java new file mode 100644 index 00000000..77fe99a9 --- /dev/null +++ b/vidarr-pluginapi/src/main/java/ca/on/oicr/gsi/vidarr/WorkingStatus.java @@ -0,0 +1,19 @@ +package ca.on.oicr.gsi.vidarr; + +/** The status, as reported to the user, about a task */ +public enum WorkingStatus { + /** + * The task's state cannot be accurately determined + * + *

This may indicate a remote system is not responding. + */ + UNKNOWN, + /** The task is not executing because it is waiting for resources to become available */ + WAITING, + /** The task has been scheduled for execution */ + QUEUED, + /** The task is currently executing */ + RUNNING, + /** The task has been requested to stop executing for load reasons */ + THROTTLED +} diff --git a/vidarr-pluginapi/src/main/java/module-info.java b/vidarr-pluginapi/src/main/java/module-info.java index 34709020..9ff0d1e6 100644 --- a/vidarr-pluginapi/src/main/java/module-info.java +++ b/vidarr-pluginapi/src/main/java/module-info.java @@ -32,6 +32,7 @@ requires transitive com.fasterxml.jackson.core; requires transitive com.fasterxml.jackson.databind; requires transitive com.fasterxml.jackson.datatype.jsr310; + requires transitive simpleclient; requires java.xml; requires java.net.http; requires transitive undertow.core; diff --git a/vidarr-server/pom.xml b/vidarr-server/pom.xml index fef8a499..bb283da3 100644 --- a/vidarr-server/pom.xml +++ b/vidarr-server/pom.xml @@ -219,7 +219,7 @@ .* - ca.on.oicr.gsi.vidarr.core.OperationStatus + ca.on.oicr.gsi.vidarr.OperationStatus OperationStatus::valueOf OperationStatus::name diff --git a/vidarr-server/src/main/java/ca/on/oicr/gsi/vidarr/server/DatabaseBackedProcessor.java b/vidarr-server/src/main/java/ca/on/oicr/gsi/vidarr/server/DatabaseBackedProcessor.java index bd97f0cd..375a0a24 100644 --- a/vidarr-server/src/main/java/ca/on/oicr/gsi/vidarr/server/DatabaseBackedProcessor.java +++ b/vidarr-server/src/main/java/ca/on/oicr/gsi/vidarr/server/DatabaseBackedProcessor.java @@ -16,6 +16,7 @@ import ca.on.oicr.gsi.Pair; import ca.on.oicr.gsi.vidarr.BasicType; import ca.on.oicr.gsi.vidarr.InputType; +import ca.on.oicr.gsi.vidarr.OperationStatus; import ca.on.oicr.gsi.vidarr.OutputType; import ca.on.oicr.gsi.vidarr.WorkflowDefinition; import ca.on.oicr.gsi.vidarr.api.BulkVersionRequest; @@ -29,7 +30,6 @@ import ca.on.oicr.gsi.vidarr.core.ExtractOutputKeys; import ca.on.oicr.gsi.vidarr.core.ExtractRetryValues; import ca.on.oicr.gsi.vidarr.core.FileMetadata; -import ca.on.oicr.gsi.vidarr.core.OperationStatus; import ca.on.oicr.gsi.vidarr.core.OutputCompatibility; import ca.on.oicr.gsi.vidarr.core.Phase; import ca.on.oicr.gsi.vidarr.core.RecoveryType; @@ -242,7 +242,7 @@ private static WorkflowDefinition buildDefinitionFromRecord( private static Stream checkConsumableResource( Map consumableResources, Pair resource) { if (consumableResources == null) { - return Stream.of(String.format("Missing consumableResources attribute")); + return Stream.of("Missing consumableResources attribute"); } return consumableResources.containsKey(resource.first()) ? resource diff --git a/vidarr-server/src/main/java/ca/on/oicr/gsi/vidarr/server/DatabaseOperation.java b/vidarr-server/src/main/java/ca/on/oicr/gsi/vidarr/server/DatabaseOperation.java index ea1c3f76..eed2c9ae 100644 --- a/vidarr-server/src/main/java/ca/on/oicr/gsi/vidarr/server/DatabaseOperation.java +++ b/vidarr-server/src/main/java/ca/on/oicr/gsi/vidarr/server/DatabaseOperation.java @@ -2,8 +2,8 @@ import static ca.on.oicr.gsi.vidarr.server.jooq.Tables.*; -import ca.on.oicr.gsi.vidarr.core.ActiveOperation; -import ca.on.oicr.gsi.vidarr.core.OperationStatus; +import ca.on.oicr.gsi.vidarr.ActiveOperation; +import ca.on.oicr.gsi.vidarr.OperationStatus; import ca.on.oicr.gsi.vidarr.core.Phase; import com.fasterxml.jackson.databind.JsonNode; import java.util.List; diff --git a/vidarr-server/src/main/java/ca/on/oicr/gsi/vidarr/server/Main.java b/vidarr-server/src/main/java/ca/on/oicr/gsi/vidarr/server/Main.java index b2cbe7b2..8970e413 100644 --- a/vidarr-server/src/main/java/ca/on/oicr/gsi/vidarr/server/Main.java +++ b/vidarr-server/src/main/java/ca/on/oicr/gsi/vidarr/server/Main.java @@ -29,6 +29,7 @@ import ca.on.oicr.gsi.vidarr.InputType; import ca.on.oicr.gsi.vidarr.JsonBodyHandler; import ca.on.oicr.gsi.vidarr.JsonPost; +import ca.on.oicr.gsi.vidarr.OperationStatus; import ca.on.oicr.gsi.vidarr.OutputProvisionFormat; import ca.on.oicr.gsi.vidarr.OutputProvisioner; import ca.on.oicr.gsi.vidarr.OutputType; @@ -40,7 +41,6 @@ import ca.on.oicr.gsi.vidarr.core.BaseProcessor; import ca.on.oicr.gsi.vidarr.core.ExtractInputVidarrIds; import ca.on.oicr.gsi.vidarr.core.FileMetadata; -import ca.on.oicr.gsi.vidarr.core.OperationStatus; import ca.on.oicr.gsi.vidarr.core.Phase; import ca.on.oicr.gsi.vidarr.core.Target; import ca.on.oicr.gsi.vidarr.server.DatabaseBackedProcessor.DeleteResultHandler; @@ -443,20 +443,20 @@ private static HttpHandler monitor(HttpHandler handler) { private final ReentrantReadWriteLock epochLock = new ReentrantReadWriteLock(); private final ScheduledExecutorService executor = Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors()); - private final Map inputProvisioners; + private final Map> inputProvisioners; private final Semaphore loadCounter = new Semaphore(3); private final MaxInFlightByWorkflow maxInFlightPerWorkflow = new MaxInFlightByWorkflow(); private final PriorityByWorkflow priorityPerWorkflow = new PriorityByWorkflow(); private final Map otherServers; - private final Map outputProvisioners; + private final Map> outputProvisioners; private final int port; private final DatabaseBackedProcessor processor; - private final Map runtimeProvisioners; + private final Map> runtimeProvisioners; private final String selfName; private final String selfUrl; private final Map targets; private final Path unloadDirectory; - private final Map workflowEngines; + private final Map> workflowEngines; private final StatusPage status = new StatusPage(this) { @Override @@ -562,10 +562,10 @@ public void emit(SectionRenderer sectionRenderer) "", maxInFlightPerWorkflow), new Pair( "priority", priorityPerWorkflow))) - .collect(Collectors.toList()); - private final WorkflowEngine engine = + .toList(); + private final WorkflowEngine engine = workflowEngines.get(e.getValue().getWorkflowEngine()); - private final Map + private final Map> inputProvisioners = e.getValue().getInputProvisioners().stream() .map(Main.this.inputProvisioners::get) @@ -573,13 +573,17 @@ public void emit(SectionRenderer sectionRenderer) p -> Stream.of(InputProvisionFormat.values()) .filter(p::canProvision) - .map(f -> new Pair<>(f, p))) + .map( + f -> + new Pair< + InputProvisionFormat, + InputProvisioner>(f, p))) .collect( Collectors - ., - InputProvisionFormat, InputProvisioner> + .>, + InputProvisionFormat, InputProvisioner> toMap(Pair::first, Pair::second)); - private final Map + private final Map> outputProvisioners = e.getValue().getOutputProvisioners().stream() .map(Main.this.outputProvisioners::get) @@ -587,16 +591,22 @@ public void emit(SectionRenderer sectionRenderer) p -> Stream.of(OutputProvisionFormat.values()) .filter(p::canProvision) - .map(f -> new Pair<>(f, p))) + .map( + f -> + new Pair< + OutputProvisionFormat, + OutputProvisioner>(f, p))) .collect( Collectors - ., - OutputProvisionFormat, OutputProvisioner> + .>, + OutputProvisionFormat, OutputProvisioner> toMap(Pair::first, Pair::second)); - private final List runtimeProvisioners = + private final List> runtimeProvisioners = e.getValue().getRuntimeProvisioners().stream() - .map(Main.this.runtimeProvisioners::get) - .collect(Collectors.toList()); + .>map(Main.this.runtimeProvisioners::get) + .toList(); @Override public Stream> consumableResources() { @@ -604,22 +614,23 @@ public Stream> consumableResources() { } @Override - public WorkflowEngine engine() { + public WorkflowEngine engine() { return engine; } @Override - public InputProvisioner provisionerFor(InputProvisionFormat type) { + public InputProvisioner provisionerFor(InputProvisionFormat type) { return inputProvisioners.get(type); } @Override - public OutputProvisioner provisionerFor(OutputProvisionFormat type) { + public OutputProvisioner provisionerFor( + OutputProvisionFormat type) { return outputProvisioners.get(type); } @Override - public Stream runtimeProvisioners() { + public Stream> runtimeProvisioners() { return runtimeProvisioners.stream(); } })); diff --git a/vidarr-server/src/main/java/ca/on/oicr/gsi/vidarr/server/dto/ServerConfiguration.java b/vidarr-server/src/main/java/ca/on/oicr/gsi/vidarr/server/dto/ServerConfiguration.java index a69561d6..d7624864 100644 --- a/vidarr-server/src/main/java/ca/on/oicr/gsi/vidarr/server/dto/ServerConfiguration.java +++ b/vidarr-server/src/main/java/ca/on/oicr/gsi/vidarr/server/dto/ServerConfiguration.java @@ -18,16 +18,16 @@ public final class ServerConfiguration { private String dbPass; private int dbPort; private String dbUser; - private Map inputProvisioners; + private Map> inputProvisioners; private String name; private Map otherServers; - private Map outputProvisioners; + private Map> outputProvisioners; private int port = 8080; - private Map runtimeProvisioners; + private Map> runtimeProvisioners; private Map targets; private String unloadDirectory = "."; private String url; - private Map workflowEngines; + private Map> workflowEngines; public Map getConsumableResources() { return consumableResources; @@ -53,7 +53,7 @@ public String getDbUser() { return dbUser; } - public Map getInputProvisioners() { + public Map> getInputProvisioners() { return inputProvisioners; } @@ -65,7 +65,7 @@ public Map getOtherServers() { return otherServers; } - public Map getOutputProvisioners() { + public Map> getOutputProvisioners() { return outputProvisioners; } @@ -73,7 +73,7 @@ public int getPort() { return port; } - public Map getRuntimeProvisioners() { + public Map> getRuntimeProvisioners() { return runtimeProvisioners; } @@ -89,7 +89,7 @@ public String getUrl() { return url; } - public Map getWorkflowEngines() { + public Map> getWorkflowEngines() { return workflowEngines; } @@ -117,7 +117,7 @@ public void setDbUser(String dbUser) { this.dbUser = dbUser; } - public void setInputProvisioners(Map inputProvisioners) { + public void setInputProvisioners(Map> inputProvisioners) { this.inputProvisioners = inputProvisioners; } @@ -129,7 +129,7 @@ public void setOtherServers(Map otherServers) { this.otherServers = otherServers; } - public void setOutputProvisioners(Map outputProvisioners) { + public void setOutputProvisioners(Map> outputProvisioners) { this.outputProvisioners = outputProvisioners; } @@ -137,7 +137,7 @@ public void setPort(int port) { this.port = port; } - public void setRuntimeProvisioners(Map runtimeProvisioners) { + public void setRuntimeProvisioners(Map> runtimeProvisioners) { this.runtimeProvisioners = runtimeProvisioners; } @@ -153,7 +153,7 @@ public void setUrl(String url) { this.url = url; } - public void setWorkflowEngines(Map workflowEngines) { + public void setWorkflowEngines(Map> workflowEngines) { this.workflowEngines = workflowEngines; } } diff --git a/vidarr-server/src/main/resources/db/migration/V0019__delete_recovery.sql b/vidarr-server/src/main/resources/db/migration/V0019__delete_recovery.sql new file mode 100644 index 00000000..50b6aa57 --- /dev/null +++ b/vidarr-server/src/main/resources/db/migration/V0019__delete_recovery.sql @@ -0,0 +1,6 @@ +DELETE FROM active_operation WHERE workflow_run_id IN (SELECT active_workflow_run.id FROM active_workflow_run); +DELETE FROM analysis_external_id WHERE analysis_external_id.external_id_id IN (SELECT external_id.id FROM external_id WHERE external_id.workflow_run_id IN (SELECT active_workflow_run.id FROM active_workflow_run)); +DELETE FROM external_id WHERE external_id.workflow_run_id IN (SELECT active_workflow_run.id FROM active_workflow_run); +DELETE FROM analysis WHERE analysis.workflow_run_id IN (SELECT active_workflow_run.id FROM active_workflow_run); +DELETE FROM active_workflow_run; +DELETE FROM workflow_run WHERE completed IS NULL; diff --git a/vidarr-sh/src/main/java/ca/on/oicr/gsi/vidarr/sh/CleanupState.java b/vidarr-sh/src/main/java/ca/on/oicr/gsi/vidarr/sh/CleanupState.java new file mode 100644 index 00000000..7b757123 --- /dev/null +++ b/vidarr-sh/src/main/java/ca/on/oicr/gsi/vidarr/sh/CleanupState.java @@ -0,0 +1,3 @@ +package ca.on.oicr.gsi.vidarr.sh; + +public record CleanupState() {} diff --git a/vidarr-sh/src/main/java/ca/on/oicr/gsi/vidarr/sh/ShellState.java b/vidarr-sh/src/main/java/ca/on/oicr/gsi/vidarr/sh/ShellState.java deleted file mode 100644 index f2a323a2..00000000 --- a/vidarr-sh/src/main/java/ca/on/oicr/gsi/vidarr/sh/ShellState.java +++ /dev/null @@ -1,23 +0,0 @@ -package ca.on.oicr.gsi.vidarr.sh; - -/** The shell workflow engine's state to be stored in the database */ -public final class ShellState { - private String outputPath; - private long pid; - - public String getOutputPath() { - return outputPath; - } - - public long getPid() { - return pid; - } - - public void setOutputPath(String outputPath) { - this.outputPath = outputPath; - } - - public void setPid(long pid) { - this.pid = pid; - } -} diff --git a/vidarr-sh/src/main/java/ca/on/oicr/gsi/vidarr/sh/StateInitial.java b/vidarr-sh/src/main/java/ca/on/oicr/gsi/vidarr/sh/StateInitial.java new file mode 100644 index 00000000..8d766044 --- /dev/null +++ b/vidarr-sh/src/main/java/ca/on/oicr/gsi/vidarr/sh/StateInitial.java @@ -0,0 +1,6 @@ +package ca.on.oicr.gsi.vidarr.sh; + +import com.fasterxml.jackson.databind.JsonNode; + +/** The shell workflow engine's state to be stored in the database */ +public record StateInitial(String workflow, JsonNode input, boolean hasAccessoryFiles) {} diff --git a/vidarr-sh/src/main/java/ca/on/oicr/gsi/vidarr/sh/UnixShellWorkflowEngine.java b/vidarr-sh/src/main/java/ca/on/oicr/gsi/vidarr/sh/UnixShellWorkflowEngine.java index b0baee18..ae160840 100644 --- a/vidarr-sh/src/main/java/ca/on/oicr/gsi/vidarr/sh/UnixShellWorkflowEngine.java +++ b/vidarr-sh/src/main/java/ca/on/oicr/gsi/vidarr/sh/UnixShellWorkflowEngine.java @@ -1,38 +1,38 @@ package ca.on.oicr.gsi.vidarr.sh; +import static ca.on.oicr.gsi.vidarr.OperationStatefulStep.require; +import static ca.on.oicr.gsi.vidarr.OperationStep.require; +import static ca.on.oicr.gsi.vidarr.OperationStep.subprocess; +import static ca.on.oicr.gsi.vidarr.ProcessOutputHandler.readOutput; + import ca.on.oicr.gsi.Pair; import ca.on.oicr.gsi.status.SectionRenderer; -import ca.on.oicr.gsi.vidarr.*; -import ca.on.oicr.gsi.vidarr.WorkMonitor.Status; +import ca.on.oicr.gsi.vidarr.BasicType; +import ca.on.oicr.gsi.vidarr.OperationAction; +import ca.on.oicr.gsi.vidarr.ProcessInput; +import ca.on.oicr.gsi.vidarr.ProcessOutput; +import ca.on.oicr.gsi.vidarr.WorkflowEngine; +import ca.on.oicr.gsi.vidarr.WorkflowEngineProvider; +import ca.on.oicr.gsi.vidarr.WorkflowLanguage; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import java.io.File; -import java.io.IOException; -import java.lang.ProcessBuilder.Redirect; import java.util.Optional; -import java.util.OptionalInt; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; import java.util.stream.Stream; /** Run commands using UNIX shell locally */ -public final class UnixShellWorkflowEngine - extends BaseJsonWorkflowEngine { +public final class UnixShellWorkflowEngine implements WorkflowEngine { private static final ObjectMapper MAPPER = new ObjectMapper(); public static WorkflowEngineProvider provider() { return () -> Stream.of(new Pair<>("sh", UnixShellWorkflowEngine.class)); } - public UnixShellWorkflowEngine() { - super(MAPPER, ShellState.class, String.class, String.class); - } + public UnixShellWorkflowEngine() {} @Override - public String cleanup(String cleanupState, WorkMonitor monitor) { - recoverCleanup(cleanupState, monitor); - return cleanupState; + public OperationAction cleanup() { + return OperationAction.value(CleanupState.class, null); } @Override @@ -46,129 +46,43 @@ public Optional engineParameters() { } @Override - public void startup() { - // Always ok. - } - - @Override - public boolean supports(WorkflowLanguage language) { - return language == WorkflowLanguage.UNIX_SHELL; - } - - @Override - public void recover(ShellState state, WorkMonitor, ShellState> monitor) { - // When we recover, we have no way to recover a process's exit status, so we'll just assume it - // exited successfully. - ProcessHandle.of(state.getPid()) - .ifPresentOrElse( - handle -> - waitForCompletion( - monitor, - new File(state.getOutputPath()), - () -> handle.isAlive() ? OptionalInt.empty() : OptionalInt.of(0)), - () -> monitor.permanentFailure("Cannot recover UNIX process between restarts.")); - } - - @Override - protected void recoverCleanup(String path, WorkMonitor monitor) { - monitor.scheduleTask( - () -> { - final var output = new File(path); - if (output.exists()) { - if (output.delete()) { - System.err.printf("Failed to delete file: %s\n", output); - } - } - monitor.complete(null); - }); + public OperationAction> run() { + return OperationAction.load( + StateInitial.class, + state -> + new ProcessInput( + Optional.of(MAPPER.writeValueAsBytes(state.input())), + Optional.empty(), + "sh", + "-c", + state.workflow())) + .then( + require( + (state, process) -> !state.hasAccessoryFiles(), + "Cannot run shell with accessory files.")) + .then(subprocess(readOutput(JsonNode.class, true))) + .then(require(ProcessOutput::success, "Process failed")) + .map(output -> new Result<>(output.standardOutput(), "/", Optional.empty())); } @Override - public ShellState runWorkflow( + public StateInitial start( WorkflowLanguage workflowLanguage, String workflow, Stream> accessoryFiles, String vidarrId, ObjectNode workflowParameters, - JsonNode engineParameters, - WorkMonitor, ShellState> monitor) { - final var state = new ShellState(); - final var fail = accessoryFiles.count() > 0; - monitor.scheduleTask( - () -> { - if (fail) { - monitor.permanentFailure("Cannot handle accessory files."); - return; - } - try { - final File outputFile = File.createTempFile("vidarr-sh", ".out"); - state.setOutputPath(outputFile.getAbsolutePath()); - monitor.storeRecoveryInformation(state); - monitor.updateState(Status.WAITING); - monitor.scheduleTask( - () -> { - try { - final var process = - new ProcessBuilder() - .command("sh", "-c", workflow) - .redirectInput(Redirect.PIPE) - .redirectOutput(Redirect.to(outputFile)) - .start(); - try (final var stdin = process.getOutputStream()) { - MAPPER.writeValue(stdin, workflowParameters); - } - monitor.updateState(Status.RUNNING); - state.setPid(process.pid()); - monitor.storeRecoveryInformation(state); - waitForCompletion( - monitor, - outputFile, - () -> - process.isAlive() - ? OptionalInt.empty() - : OptionalInt.of(process.exitValue())); - } catch (IOException e) { - monitor.permanentFailure(e.getMessage()); - } - }); - } catch (IOException e) { - monitor.permanentFailure(e.getMessage()); - } - }); - return state; + JsonNode engineParameters) { + return new StateInitial(workflow, workflowParameters, accessoryFiles.findAny().isPresent()); } - private void waitForCompletion( - WorkMonitor, ShellState> monitor, - File outputFile, - Supplier checkExit) { - monitor.scheduleTask( - 1, - TimeUnit.MINUTES, - new Runnable() { - @Override - public void run() { - checkExit - .get() - .ifPresentOrElse( - exit -> { - if (exit == 0) { - try { - monitor.complete( - new Result<>( - MAPPER.readTree(outputFile), - outputFile.toURI().toASCIIString(), - Optional.of(outputFile.getAbsolutePath()))); - } catch (IOException e) { - monitor.permanentFailure(e.getMessage()); - } + @Override + public void startup() { + // Always ok. + } - } else { - monitor.permanentFailure("Process exited with an error."); - } - }, - () -> monitor.scheduleTask(1, TimeUnit.MINUTES, this)); - } - }); + @Override + public boolean supports(WorkflowLanguage language) { + return language == WorkflowLanguage.UNIX_SHELL; } }