[cas] Add ValidateCAS action to ensure data coherence (swiftlang#460)

Use llvm-cas's new validate-if-needed action to ensure the correctness
of CAS data in the case of power failure or similar situations. This
action is added to prepareForBuilding to ensure that it is run before
any other other CAS accesses. Note that validate-if-needed internally
avoids performing unnecessary work - in particular, it only validates
data once for every machine boot.

rdar://150295950

Author: benlangmuir

Sources/SWBBuildSystem/BuildOperation.swift

@@ -419,7 +419,7 @@ package final class BuildOperation: BuildSystemOperation {
         }
 
         // Perform any needed steps before we kick off the build.
-        if let (warnings, errors) = prepareForBuilding() {
+        if let (warnings, errors) = await prepareForBuilding() {
             // Emit any warnings and errors.  If there were any errors, then bail out.
             for message in warnings { buildOutputDelegate.warning(message) }
             for message in errors { buildOutputDelegate.error(message) }
@@ -809,7 +809,7 @@ package final class BuildOperation: BuildSystemOperation {
         return delegate.buildComplete(self, status: effectiveStatus, delegate: buildOutputDelegate, metrics: .init(counters: aggregatedCounters))
     }
 
-    func prepareForBuilding() -> ([String], [String])? {
+    func prepareForBuilding() async -> ([String], [String])? {
         let warnings = [String]()       // Not presently used
         var errors = [String]()
 
@@ -829,9 +829,61 @@ package final class BuildOperation: BuildSystemOperation {
             }
         }
 
+        if UserDefaults.enableCASValidation {
+            for info in buildDescription.casValidationInfos {
+                do {
+                    try await validateCAS(info)
+                } catch {
+                    errors.append("cas validation failed for \(info.options.casPath.str)")
+                }
+            }
+        }
+
         return (warnings.count > 0 || errors.count > 0) ? (warnings, errors) : nil
     }
 
+    func validateCAS(_ info: BuildDescription.CASValidationInfo) async throws {
+        assert(UserDefaults.enableCASValidation)
+
+        let casPath = info.options.casPath
+        let ruleInfo = "ValidateCAS \(casPath.str) \(info.llvmCasExec.str)"
+
+        let signatureCtx = InsecureHashContext()
+        signatureCtx.add(string: "ValidateCAS")
+        signatureCtx.add(string: casPath.str)
+        signatureCtx.add(string: info.llvmCasExec.str)
+        let signature = signatureCtx.signature
+
+        let activityId = delegate.beginActivity(self, ruleInfo: ruleInfo, executionDescription: "Validate CAS contents at \(casPath.str)", signature: signature, target: nil, parentActivity: nil)
+        var status: BuildOperationTaskEnded.Status = .failed
+        defer {
+            delegate.endActivity(self, id: activityId, signature: signature, status: status)
+        }
+
+        var commandLine = [
+            info.llvmCasExec.str,
+            "-cas", casPath.str,
+            "-validate-if-needed",
+            "-check-hash",
+            "-allow-recovery",
+        ]
+        if let pluginPath = info.options.pluginPath {
+            commandLine.append(contentsOf: [
+                "-fcas-plugin-path", pluginPath.str
+            ])
+        }
+        let result: Processes.ExecutionResult = try await clientDelegate.executeExternalTool(commandLine: commandLine)
+        // In a task we might use a discovered tool info to detect if the tool supports validation, but without that scaffolding, just check the specific error.
+        if result.exitStatus == .exit(1) && result.stderr.contains(ByteString("Unknown command line argument '-validate-if-needed'")) {
+            delegate.emit(data: ByteString("validation not supported").bytes, for: activityId, signature: signature)
+            status = .succeeded
+        } else {
+            delegate.emit(data: ByteString(result.stderr).bytes, for: activityId, signature: signature)
+            delegate.emit(data: ByteString(result.stdout).bytes, for: activityId, signature: signature)
+            status = result.exitStatus.isSuccess ? .succeeded : result.exitStatus.wasCanceled ? .cancelled : .failed
+        }
+    }
+
     /// Cancel the executing build operation.
     package func cancel() {
         queue.blocking_sync() {

Sources/SWBCore/Settings/BuiltinMacros.swift

@@ -1125,6 +1125,7 @@ public final class BuiltinMacros {
     public static let TAPI_HEADER_SEARCH_PATHS = BuiltinMacros.declarePathListMacro("TAPI_HEADER_SEARCH_PATHS")
     public static let USE_HEADER_SYMLINKS = BuiltinMacros.declareBooleanMacro("USE_HEADER_SYMLINKS")
     public static let USE_HIERARCHICAL_LAYOUT_FOR_COPIED_ASIDE_PRODUCTS = BuiltinMacros.declareBooleanMacro("USE_HIERARCHICAL_LAYOUT_FOR_COPIED_ASIDE_PRODUCTS")
+    public static let VALIDATE_CAS_EXEC = BuiltinMacros.declareStringMacro("VALIDATE_CAS_EXEC")
     public static let VALIDATE_PLIST_FILES_WHILE_COPYING = BuiltinMacros.declareBooleanMacro("VALIDATE_PLIST_FILES_WHILE_COPYING")
     public static let VALIDATE_PRODUCT = BuiltinMacros.declareBooleanMacro("VALIDATE_PRODUCT")
     public static let VALIDATE_DEPENDENCIES = BuiltinMacros.declareEnumMacro("VALIDATE_DEPENDENCIES") as EnumMacroDeclaration<BooleanWarningLevel>
@@ -2325,6 +2326,7 @@ public final class BuiltinMacros {
         USE_HEADER_SYMLINKS,
         USE_HIERARCHICAL_LAYOUT_FOR_COPIED_ASIDE_PRODUCTS,
         VALIDATE_PLIST_FILES_WHILE_COPYING,
+        VALIDATE_CAS_EXEC,
         VALIDATE_PRODUCT,
         VALIDATE_DEPENDENCIES,
         VALIDATE_DEVELOPMENT_ASSET_PATHS,

Sources/SWBCore/TaskGeneration.swift

@@ -768,7 +768,7 @@ extension CoreClientTargetDiagnosticProducingDelegate {
 private let externalToolExecutionQueue = AsyncOperationQueue(concurrentTasks: ProcessInfo.processInfo.activeProcessorCount)
 
 extension CoreClientDelegate {
-    func executeExternalTool(commandLine: [String], workingDirectory: String? = nil, environment: [String: String] = [:]) async throws -> Processes.ExecutionResult {
+    package func executeExternalTool(commandLine: [String], workingDirectory: String? = nil, environment: [String: String] = [:]) async throws -> Processes.ExecutionResult {
         switch try await executeExternalTool(commandLine: commandLine, workingDirectory: workingDirectory, environment: environment) {
         case .deferred:
             guard let url = commandLine.first.map(URL.init(fileURLWithPath:)) else {