Import pipeline (#48)

* import pipeline wip

* lazy hash

* blockRef

* update import pipeline

* validator activity stats

* fix format

.swiftformat

@@ -1,2 +1,3 @@
 --extensionacl on-declarations
 --maxwidth 150
+--asynccapturing debugCheck

.swiftlint.yml

@@ -8,13 +8,11 @@ disabled_rules:
     - cyclomatic_complexity
     - blanket_disable_command
     - type_body_length
+    - identifier_name
 
 excluded:
     - "**/.build"
 
-identifier_name:
-    min_length: 1
-
 type_name:
     min_length: 1
 

Blockchain/Package.resolved

@@ -1,5 +1,5 @@
 {
-  "originHash" : "6b228bab7212903c926ae4a95f87303c43f7fea238e14956c10f5bb056e2bdf8",
+  "originHash" : "b411c6a0bd87c7eb9917abe3f20455e142ffe77b06622bdfbf0cf5a8c002daf8",
   "pins" : [
     {
       "identity" : "blake2.swift",
@@ -19,6 +19,15 @@
         "revision" : "dac3e7161de34c60c82794d031de0231b5a5746e"
       }
     },
+    {
+      "identity" : "swift-atomics",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-atomics.git",
+      "state" : {
+        "revision" : "cd142fd2f64be2100422d658e7411e39489da985",
+        "version" : "1.2.0"
+      }
+    },
     {
       "identity" : "swift-crypto",
       "kind" : "remoteSourceControl",

Blockchain/Sources/Blockchain/Blockchain.swift

@@ -5,46 +5,24 @@ import Utils
 /// Includes the canonical chain as well as pending forks.
 /// Assume all blocks and states are valid and have been validated.
 public class Blockchain {
-    public private(set) var heads: [StateRef]
-    public private(set) var finalizedHead: StateRef
+    public let config: ProtocolConfigRef
 
-    private var stateByBlockHash: [Data32: StateRef] = [:]
-    private var stateByTimeslot: [TimeslotIndex: [StateRef]] = [:]
+    private let dataProvider: BlockchainDataProvider
 
-    public init(heads: [StateRef], finalizedHead: StateRef) {
-        assert(heads.contains(where: { $0 === finalizedHead }))
-
-        self.heads = heads
-        self.finalizedHead = finalizedHead
-
-        for head in heads {
-            addState(head)
-        }
+    public init(config: ProtocolConfigRef, dataProvider: BlockchainDataProvider) async {
+        self.config = config
+        self.dataProvider = dataProvider
     }
 
-    public var bestHead: StateRef {
-        // heads with the highest timestamp / latest block
-        heads.max(by: { $0.value.lastBlock.header.timeslotIndex < $1.value.lastBlock.header.timeslotIndex })!
-    }
-
-    public func newHead(_ head: StateRef) {
-        // TODO: check if the parent is known
-        // TODO: Update heads, by either extending an existing one or adding a new fork
-        addState(head)
-    }
-
-    private func addState(_ state: StateRef) {
-        stateByBlockHash[state.value.lastBlock.header.hash()] = state
-        stateByTimeslot[state.value.lastBlock.header.timeslotIndex, default: []].append(state)
-    }
-}
-
-extension Blockchain {
-    public subscript(hash: Data32) -> StateRef? {
-        stateByBlockHash[hash]
+    public func importBlock(_ block: BlockRef) async throws {
+        let runtime = Runtime(config: config)
+        let parent = try await dataProvider.getState(hash: block.header.parentHash)
+        let state = try runtime.apply(block: block, state: parent)
+        try await dataProvider.add(state: state)
     }
 
-    public subscript(index: TimeslotIndex) -> [StateRef]? {
-        stateByTimeslot[index]
+    public func finalize(hash: Data32) async throws {
+        // TODO: purge forks
+        try await dataProvider.setFinalizedHead(hash: hash)
     }
 }

Blockchain/Sources/Blockchain/BlockchainDataProvider.swift

@@ -0,0 +1,38 @@
+import Utils
+
+public enum BlockchainDataProviderError: Error {
+    case unknownHash
+}
+
+public protocol BlockchainDataProvider {
+    func hasHeader(hash: Data32) async throws -> Bool
+    func isHead(hash: Data32) async throws -> Bool
+
+    func getHeader(hash: Data32) async throws -> HeaderRef
+    func getBlock(hash: Data32) async throws -> BlockRef
+    func getState(hash: Data32) async throws -> StateRef
+    func getFinalizedHead() async throws -> Data32
+    func getHeads() async throws -> [Data32]
+    func getBlockHash(index: TimeslotIndex) async throws -> [Data32]
+
+    func add(state: StateRef, isHead: Bool) async throws
+    func setFinalizedHead(hash: Data32) async throws
+    // remove header, block and state
+    func remove(hash: Data32) async throws
+
+    // protected method
+    func _updateHeadNoCheck(hash: Data32, parent: Data32) async throws
+}
+
+extension BlockchainDataProvider {
+    public func updateHead(hash: Data32, parent: Data32) async throws {
+        try await debugCheck(await hasHeader(hash: hash))
+        try await debugCheck(await hasHeader(hash: parent))
+
+        try await _updateHeadNoCheck(hash: hash, parent: parent)
+    }
+
+    public func add(state: StateRef) async throws {
+        try await add(state: state, isHead: false)
+    }
+}

Blockchain/Sources/Blockchain/InMemoryDataProvider.swift

@@ -0,0 +1,95 @@
+import Utils
+
+// TODO: add tests
+public actor InMemoryDataProvider: Sendable {
+    public private(set) var heads: [StateRef]
+    public private(set) var finalizedHead: StateRef
+
+    private var stateByBlockHash: [Data32: StateRef] = [:]
+    private var hashByTimeslot: [TimeslotIndex: [Data32]] = [:]
+
+    public init(genesis: StateRef) async {
+        heads = [genesis]
+        finalizedHead = genesis
+
+        addState(genesis)
+    }
+
+    private func addState(_ state: StateRef) {
+        stateByBlockHash[state.value.lastBlock.hash] = state
+        hashByTimeslot[state.value.lastBlock.header.timeslotIndex, default: []].append(state.value.lastBlock.hash)
+    }
+}
+
+extension InMemoryDataProvider: BlockchainDataProvider {
+    public func hasHeader(hash: Data32) async throws -> Bool {
+        stateByBlockHash[hash] != nil
+    }
+
+    public func isHead(hash: Data32) async throws -> Bool {
+        heads.contains(where: { $0.value.lastBlock.hash == hash })
+    }
+
+    public func getHeader(hash: Data32) async throws -> HeaderRef {
+        guard let header = stateByBlockHash[hash]?.value.lastBlock.header.asRef() else {
+            throw BlockchainDataProviderError.unknownHash
+        }
+        return header
+    }
+
+    public func getBlock(hash: Data32) async throws -> BlockRef {
+        guard let block = stateByBlockHash[hash]?.value.lastBlock else {
+            throw BlockchainDataProviderError.unknownHash
+        }
+        return block
+    }
+
+    public func getState(hash: Data32) async throws -> StateRef {
+        guard let state = stateByBlockHash[hash] else {
+            throw BlockchainDataProviderError.unknownHash
+        }
+        return state
+    }
+
+    public func getFinalizedHead() async throws -> Data32 {
+        finalizedHead.value.lastBlock.hash
+    }
+
+    public func getHeads() async throws -> [Data32] {
+        heads.map(\.value.lastBlock.hash)
+    }
+
+    public func getBlockHash(index: TimeslotIndex) async throws -> [Data32] {
+        hashByTimeslot[index] ?? []
+    }
+
+    public func add(state: StateRef, isHead: Bool) async throws {
+        addState(state)
+        if isHead {
+            try await updateHead(hash: state.value.lastBlock.hash, parent: state.value.lastBlock.header.parentHash)
+        }
+    }
+
+    public func setFinalizedHead(hash: Data32) async throws {
+        guard let state = stateByBlockHash[hash] else {
+            throw BlockchainDataProviderError.unknownHash
+        }
+        finalizedHead = state
+    }
+
+    public func _updateHeadNoCheck(hash: Data32, parent: Data32) async throws {
+        for i in 0 ..< heads.count where heads[i].value.lastBlock.hash == parent {
+            assert(stateByBlockHash[hash] != nil)
+            heads[i] = stateByBlockHash[hash]!
+            return
+        }
+    }
+
+    public func remove(hash: Data32) async throws {
+        guard let state = stateByBlockHash[hash] else {
+            return
+        }
+        stateByBlockHash.removeValue(forKey: hash)
+        hashByTimeslot.removeValue(forKey: state.value.lastBlock.header.timeslotIndex)
+    }
+}

Blockchain/Sources/Blockchain/Runtime.swift

@@ -0,0 +1,28 @@
+// the STF
+public final class Runtime {
+    public enum Error: Swift.Error {
+        case safroleError(SafroleError)
+    }
+
+    public let config: ProtocolConfigRef
+
+    public init(config: ProtocolConfigRef) {
+        self.config = config
+    }
+
+    public func apply(block: BlockRef, state prevState: StateRef) throws(Error) -> StateRef {
+        var newState = prevState.value
+        newState.lastBlock = block
+        let res = newState.updateSafrole(
+            slot: block.header.timeslotIndex, entropy: newState.entropyPool.0, extrinsics: block.extrinsic.tickets
+        )
+        switch res {
+        case let .success((state: postState, epochMark: _, ticketsMark: _)):
+            newState.mergeWith(postState: postState)
+        case let .failure(err):
+            throw .safroleError(err)
+        }
+
+        return StateRef(newState)
+    }
+}

Blockchain/Sources/Blockchain/Safrole.swift

@@ -148,7 +148,7 @@ public protocol Safrole {
             SafroleError
         >
 
-    func mergeWith(postState: SafrolePostState) -> Self
+    mutating func mergeWith(postState: SafrolePostState)
 }
 
 func outsideInReorder<T>(_ array: [T]) -> [T] {

Blockchain/Sources/Blockchain/Types/Block.swift

@@ -11,7 +11,30 @@ public struct Block: Sendable, Equatable {
     }
 }
 
-public typealias BlockRef = Ref<Block>
+extension Block {
+    public func asRef() -> BlockRef {
+        BlockRef(self)
+    }
+}
+
+public final class BlockRef: Ref<Block>, @unchecked Sendable {
+    public required init(_ value: Block) {
+        lazy = Lazy {
+            Ref(value.header.hash())
+        }
+
+        super.init(value)
+    }
+
+    private let lazy: Lazy<Ref<Data32>>
+
+    public var hash: Data32 {
+        lazy.value.value
+    }
+
+    public var header: Header { value.header }
+    public var extrinsic: Extrinsic { value.extrinsic }
+}
 
 extension Block: Dummy {
     public typealias Config = ProtocolConfigRef