Merge branch 'master' into dev-issues-278

* master:
  rename (#289)
  data store (#286)
  improve peer test (#288)

.swiftformat

@@ -2,3 +2,4 @@
 --maxwidth 140
 --asynccapturing debugCheck
 --closurevoid preserve
+--disable typeSugar

Blockchain/Sources/Blockchain/AuxDataStore/DataStore.swift

@@ -0,0 +1,47 @@
+import Utils
+
+enum DataStoreError: Error {
+    case invalidPackageHash(Data32)
+    case invalidSegmentRoot(Data32)
+}
+
+public final class DataStore: Sendable {
+    private let impl: DataStoreProtocol
+    private let network: DataStoreNetworkProtocol
+
+    public init(_ impl: DataStoreProtocol, _ network: DataStoreNetworkProtocol) {
+        self.impl = impl
+        self.network = network
+    }
+
+    public func fetchSegment(segments: [WorkItem.ImportedDataSegment]) async throws -> [Data4104] {
+        var result: [Data4104] = []
+
+        for segment in segments {
+            let segmentRoot = switch segment.root {
+            case let .segmentRoot(root):
+                root
+            case let .workPackageHash(hash):
+                try await impl.getSegmentRoot(forWorkPackageHash: hash).unwrap(orError: DataStoreError.invalidPackageHash(hash))
+            }
+            let erasureRoot = try await impl.getEasureRoot(forSegmentRoot: segmentRoot)
+                .unwrap(orError: DataStoreError.invalidSegmentRoot(segmentRoot))
+
+            if let localData = try await impl.get(erasureRoot: erasureRoot, index: segment.index) {
+                result.append(localData)
+            } else {
+                // TODO: use network for fetch shards and reconstruct the segment
+                fatalError("not implemented")
+            }
+        }
+
+        return result
+    }
+
+    public func set(data: Data4104, erasureRoot: Data32, index: UInt16) async throws {
+        try await impl.set(data: data, erasureRoot: erasureRoot, index: index)
+
+        // TODO; erasure code the data and store each chunk
+        // so assurer can query them later with CE137
+    }
+}

Blockchain/Sources/Blockchain/AuxDataStore/DataStoreProtocol.swift

@@ -0,0 +1,22 @@
+import Utils
+
+public protocol DataStoreProtocol: Sendable {
+    // segment root => erasure root
+    func getEasureRoot(forSegmentRoot: Data32) async throws -> Data32?
+    func set(erasureRoot: Data32, forSegmentRoot: Data32) async throws
+    func delete(erasureRoot: Data32) async throws
+
+    // work package hash => segment root
+    func getSegmentRoot(forWorkPackageHash: Data32) async throws -> Data32?
+    func set(segmentRoot: Data32, forWorkPackageHash: Data32) async throws
+    func delete(segmentRoot: Data32) async throws
+
+    // erasure root + index => segment data
+    func get(erasureRoot: Data32, index: UInt16) async throws -> Data4104?
+    func set(data: Data4104, erasureRoot: Data32, index: UInt16) async throws
+}
+
+public protocol DataStoreNetworkProtocol: Sendable {
+    // Use CE139/CE140 to fetch remote chunk
+    func fetchRemoteChunk(erasureRoot: Data32, shardIndex: UInt16, segmentIndices: [UInt16]) async throws -> Data12?
+}

Blockchain/Sources/Blockchain/AuxDataStore/InMemoryDataStoreBackend.swift

@@ -0,0 +1,54 @@
+import Utils
+
+public actor InMemoryDataStoreBackend {
+    // segment root => erasure root
+    private var erasureRootBySegmentRoot: [Data32: Data32] = [:]
+
+    // work package hash => segment root
+    private var segmentRootByWorkPackageHash: [Data32: Data32] = [:]
+
+    // erasure root + index => segment data
+    private var chunks: [Data32: [UInt16: Data4104]] = [:]
+
+    public init() {}
+}
+
+extension InMemoryDataStoreBackend: DataStoreProtocol {
+    public func getEasureRoot(forSegmentRoot: Data32) async throws -> Data32? {
+        erasureRootBySegmentRoot[forSegmentRoot]
+    }
+
+    public func set(erasureRoot: Data32, forSegmentRoot: Data32) async throws {
+        erasureRootBySegmentRoot[forSegmentRoot] = erasureRoot
+    }
+
+    public func delete(erasureRoot: Data32) async throws {
+        erasureRootBySegmentRoot.removeValue(forKey: erasureRoot)
+    }
+
+    public func getSegmentRoot(forWorkPackageHash: Data32) async throws -> Data32? {
+        segmentRootByWorkPackageHash[forWorkPackageHash]
+    }
+
+    public func set(segmentRoot: Data32, forWorkPackageHash: Data32) async throws {
+        segmentRootByWorkPackageHash[forWorkPackageHash] = segmentRoot
+    }
+
+    public func delete(segmentRoot: Data32) async throws {
+        segmentRootByWorkPackageHash.removeValue(forKey: segmentRoot)
+    }
+
+    public func get(erasureRoot: Data32, index: UInt16) async throws -> Data4104? {
+        chunks[erasureRoot]?[index]
+    }
+
+    public func set(data: Data4104, erasureRoot: Data32, index: UInt16) async throws {
+        chunks[erasureRoot, default: [:]][index] = data
+    }
+}
+
+extension InMemoryDataStoreBackend: DataStoreNetworkProtocol {
+    public func fetchRemoteChunk(erasureRoot _: Data32, shardIndex _: UInt16, segmentIndices _: [UInt16]) async throws -> Data12? {
+        nil
+    }
+}

Blockchain/Sources/Blockchain/BlockchainServices.swift

@@ -32,7 +32,8 @@ public class BlockchainServices {
         self.genesisBlock = genesisBlock
         self.genesisState = genesisState
         dataProvider = try! await BlockchainDataProvider(InMemoryDataProvider(genesisState: genesisState, genesisBlock: genesisBlock))
-        dataStore = DataStore(InMemoryDataStore(), basePath: URL(fileURLWithPath: "/tmp/boka-test-data"))
+        let dataStoreBackend = InMemoryDataStoreBackend()
+        dataStore = DataStore(dataStoreBackend, dataStoreBackend)
 
         storeMiddleware = StoreMiddleware()
         eventBus = EventBus(eventMiddleware: .serial(Middleware(storeMiddleware), .noError), handlerMiddleware: .noError)
@@ -82,7 +83,7 @@ public class BlockchainServices {
                 eventBus: eventBus,
                 keystore: keystore,
                 scheduler: scheduler,
-                extrinsicPool: ExtrinsicPoolService(config: config, dataProvider: dataProvider, eventBus: eventBus)
+                safroleTicketPool: SafroleTicketPoolService(config: config, dataProvider: dataProvider, eventBus: eventBus)
             )
             _blockAuthorRef = _blockAuthor
             return _blockAuthor!

Blockchain/Sources/Blockchain/Config/ProtocolConfig.swift

@@ -85,8 +85,7 @@ public struct ProtocolConfig: Sendable, Codable, Equatable {
     /// WM = 2^11: The maximum number of entries in a work-package manifest.
     public var maxWorkPackageManifestEntries: Int
 
-    /// WB = 12 * 2^20: The maximum size of an encoded work-package together with its extrinsic data and import impli-
-    /// cations, in octets.
+    /// WB = 12 * 2^20: The maximum size of an encoded work-package together with its extrinsic data and import implications, in octets.
     public var maxEncodedWorkPackageSize: Int
 
     /// WG = WP*WE = 4104: The size of a segment in octets.

Blockchain/Sources/Blockchain/Types/WorkPackageBundle.swift

@@ -5,6 +5,6 @@ import Utils
 public struct WorkPackageBundle: Sendable, Equatable, Codable {
     public var workPackage: WorkPackage
     public var extrinsic: [Data]
-    public var importSegments: [[Data]]
-    public var justifications: [[Data]]
+    public var importSegments: [Data4104]
+    public var justifications: [Data]
 }

Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCalls.swift

@@ -743,9 +743,9 @@ public class Import: HostCall {
     public static var identifier: UInt8 { 18 }
 
     public let context: RefineContext.ContextType
-    public let importSegments: [Data]
+    public let importSegments: [Data4104]
 
-    public init(context: RefineContext.ContextType, importSegments: [Data]) {
+    public init(context: RefineContext.ContextType, importSegments: [Data4104]) {
         self.context = context
         self.importSegments = importSegments
     }
@@ -762,7 +762,7 @@ public class Import: HostCall {
         let isWritable = state.isMemoryWritable(address: startAddr, length: Int(length))
 
         if let segment, isWritable {
-            try state.writeMemory(address: startAddr, values: segment)
+            try state.writeMemory(address: startAddr, values: segment.data)
         }
 
         if !isWritable {
@@ -803,13 +803,15 @@ public class Export: HostCall {
             segment = data
         }
 
-        if segment == nil {
-            state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.OOB.rawValue)
-        } else if exportSegmentOffset + UInt64(segment!.count) >= UInt64(config.value.maxWorkPackageManifestEntries) {
-            state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.FULL.rawValue)
+        if let segment {
+            if exportSegmentOffset + UInt64(segment.count) >= UInt64(config.value.maxWorkPackageManifestEntries) {
+                state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.FULL.rawValue)
+            } else {
+                state.writeRegister(Registers.Index(raw: 7), exportSegmentOffset + UInt64(segment.count))
+                context.exports.append(Data4104(segment)!)
+            }
         } else {
-            state.writeRegister(Registers.Index(raw: 7), exportSegmentOffset + UInt64(segment!.count))
-            context.exports.append(segment!)
+            state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.OOB.rawValue)
         }
     }
 }

Blockchain/Sources/Blockchain/VMInvocations/InvocationContexts/RefineContext.swift

@@ -1,6 +1,7 @@
 import Foundation
 import PolkaVM
 import TracingUtils
+import Utils
 
 private let logger = Logger(label: "RefineContext")
 
@@ -13,12 +14,12 @@ public struct InnerPvm {
 public class RefineContext: InvocationContext {
     public typealias ContextType = (
         pvms: [UInt64: InnerPvm],
-        exports: [Data]
+        exports: [Data4104]
     )
 
     public let config: ProtocolConfigRef
     public var context: ContextType
-    public let importSegments: [Data]
+    public let importSegments: [Data4104]
     public let exportSegmentOffset: UInt64
     public let service: ServiceIndex
     public let serviceAccounts: ServiceAccounts
@@ -27,7 +28,7 @@ public class RefineContext: InvocationContext {
     public init(
         config: ProtocolConfigRef,
         context: ContextType,
-        importSegments: [Data],
+        importSegments: [Data4104],
         exportSegmentOffset: UInt64,
         service: ServiceIndex,
         serviceAccounts: some ServiceAccounts,

Blockchain/Sources/Blockchain/VMInvocations/Invocations/RefineInvocation.swift

@@ -15,10 +15,10 @@ public protocol RefineInvocation {
         refinementCtx: RefinementContext, // c
         authorizerHash: Data32,
         authorizationOutput: Data,
-        importSegments: [Data],
+        importSegments: [Data4104],
         extrinsicDataBlobs: [Data],
         exportSegmentOffset: UInt64
-    ) async throws -> (result: Result<Data, WorkResultError>, exports: [Data])
+    ) async throws -> (result: Result<Data, WorkResultError>, exports: [Data4104])
 }
 
 extension RefineInvocation {
@@ -33,10 +33,10 @@ extension RefineInvocation {
         refinementCtx: RefinementContext, // c
         authorizerHash: Data32,
         authorizationOutput: Data,
-        importSegments: [Data],
+        importSegments: [Data4104],
         extrinsicDataBlobs: [Data],
         exportSegmentOffset: UInt64
-    ) async throws -> (result: Result<Data, WorkResultError>, exports: [Data]) {
+    ) async throws -> (result: Result<Data, WorkResultError>, exports: [Data4104]) {
         let codeBlob = try await serviceAccounts.historicalLookup(
             serviceAccount: service,
             timeslot: refinementCtx.lookupAnchor.timeslot,

Blockchain/Sources/Blockchain/Validator/BlockAuthor.swift

@@ -7,7 +7,7 @@ import Utils
 public final class BlockAuthor: ServiceBase2, @unchecked Sendable {
     private let dataProvider: BlockchainDataProvider
     private let keystore: KeyStore
-    private let extrinsicPool: ExtrinsicPoolService
+    private let safroleTicketPool: SafroleTicketPoolService
 
     private let tickets: ThreadSafeContainer<[RuntimeEvents.SafroleTicketsGenerated]> = .init([])
 
@@ -17,11 +17,11 @@ public final class BlockAuthor: ServiceBase2, @unchecked Sendable {
         eventBus: EventBus,
         keystore: KeyStore,
         scheduler: Scheduler,
-        extrinsicPool: ExtrinsicPoolService
+        safroleTicketPool: SafroleTicketPoolService
     ) async {
         self.dataProvider = dataProvider
         self.keystore = keystore
-        self.extrinsicPool = extrinsicPool
+        self.safroleTicketPool = safroleTicketPool
 
         super.init(id: "BlockAuthor", config: config, eventBus: eventBus, scheduler: scheduler)
 
@@ -57,7 +57,7 @@ public final class BlockAuthor: ServiceBase2, @unchecked Sendable {
         let stateRoot = await state.value.stateRoot
         let epoch = timeslot.timeslotToEpochIndex(config: config)
 
-        let pendingTickets = await extrinsicPool.getPendingTickets(epoch: epoch)
+        let pendingTickets = await safroleTicketPool.getPendingTickets(epoch: epoch)
         let existingTickets = SortedArray(sortedUnchecked: state.value.safroleState.ticketsAccumulator.array.map(\.id))
         let tickets = pendingTickets.array
             .lazy