From 15794f5cad319a58a8ed768af3947e2228ced0ce Mon Sep 17 00:00:00 2001 From: Lucy Satheesan Date: Tue, 18 Jul 2023 21:04:39 -0700 Subject: [PATCH 01/48] ioring pitch: first steps --- Package.resolved | 16 ++ Package.swift | 9 +- Sources/CSystem/include/CSystemLinux.h | 1 + Sources/CSystem/include/io_uring.h | 50 ++++ Sources/System/IOCompletion.swift | 50 ++++ Sources/System/IORequest.swift | 140 ++++++++++ Sources/System/IORing.swift | 350 +++++++++++++++++++++++++ Sources/System/IORingBuffer.swift | 0 Sources/System/IORingFileSlot.swift | 8 + Sources/System/Lock.swift | 37 +++ Sources/System/ManagedIORing.swift | 33 +++ 11 files changed, 692 insertions(+), 2 deletions(-) create mode 100644 Package.resolved create mode 100644 Sources/CSystem/include/io_uring.h create mode 100644 Sources/System/IOCompletion.swift create mode 100644 Sources/System/IORequest.swift create mode 100644 Sources/System/IORing.swift create mode 100644 Sources/System/IORingBuffer.swift create mode 100644 Sources/System/IORingFileSlot.swift create mode 100644 Sources/System/Lock.swift create mode 100644 Sources/System/ManagedIORing.swift diff --git a/Package.resolved b/Package.resolved new file mode 100644 index 00000000..b10a9832 --- /dev/null +++ b/Package.resolved @@ -0,0 +1,16 @@ +{ + "object": { + "pins": [ + { + "package": "swift-atomics", + "repositoryURL": "https://github.com/apple/swift-atomics", + "state": { + "branch": null, + "revision": "6c89474e62719ddcc1e9614989fff2f68208fe10", + "version": "1.1.0" + } + } + ] + }, + "version": 1 +} diff --git a/Package.swift b/Package.swift index b081a872..60f1a562 100644 --- a/Package.swift +++ b/Package.swift @@ -17,14 +17,19 @@ let package = Package( products: [ .library(name: "SystemPackage", targets: ["SystemPackage"]), ], - dependencies: [], + dependencies: [ + .package(url: "https://github.com/apple/swift-atomics", from: "1.1.0") + ], targets: [ .target( name: "CSystem", dependencies: []), .target( name: "SystemPackage", - dependencies: ["CSystem"], + dependencies: [ + "CSystem", + .product(name: "Atomics", package: "swift-atomics") + ], path: "Sources/System", cSettings: [ .define("_CRT_SECURE_NO_WARNINGS") diff --git a/Sources/CSystem/include/CSystemLinux.h b/Sources/CSystem/include/CSystemLinux.h index b172d658..6489c4f3 100644 --- a/Sources/CSystem/include/CSystemLinux.h +++ b/Sources/CSystem/include/CSystemLinux.h @@ -21,5 +21,6 @@ #include #include #include +#include "io_uring.h" #endif diff --git a/Sources/CSystem/include/io_uring.h b/Sources/CSystem/include/io_uring.h new file mode 100644 index 00000000..9e3d9fb3 --- /dev/null +++ b/Sources/CSystem/include/io_uring.h @@ -0,0 +1,50 @@ +#include +#include +#include + +#include +#include + +#ifdef __alpha__ +/* + * alpha is the only exception, all other architectures + * have common numbers for new system calls. + */ +# ifndef __NR_io_uring_setup +# define __NR_io_uring_setup 535 +# endif +# ifndef __NR_io_uring_enter +# define __NR_io_uring_enter 536 +# endif +# ifndef __NR_io_uring_register +# define __NR_io_uring_register 537 +# endif +#else /* !__alpha__ */ +# ifndef __NR_io_uring_setup +# define __NR_io_uring_setup 425 +# endif +# ifndef __NR_io_uring_enter +# define __NR_io_uring_enter 426 +# endif +# ifndef __NR_io_uring_register +# define __NR_io_uring_register 427 +# endif +#endif + +int io_uring_register(int fd, unsigned int opcode, void *arg, + unsigned int nr_args) +{ + return syscall(__NR_io_uring_register, fd, opcode, arg, nr_args); +} + +int io_uring_setup(unsigned int entries, struct io_uring_params *p) +{ + return syscall(__NR_io_uring_setup, entries, p); +} + +int io_uring_enter(int fd, unsigned int to_submit, unsigned int min_complete, + unsigned int flags, sigset_t *sig) +{ + return syscall(__NR_io_uring_enter, fd, to_submit, min_complete, + flags, sig, _NSIG / 8); +} diff --git a/Sources/System/IOCompletion.swift b/Sources/System/IOCompletion.swift new file mode 100644 index 00000000..5e226322 --- /dev/null +++ b/Sources/System/IOCompletion.swift @@ -0,0 +1,50 @@ +@_implementationOnly import CSystem + +public struct IOCompletion { + let rawValue: io_uring_cqe +} + +extension IOCompletion { + public struct Flags: OptionSet, Hashable, Codable { + public let rawValue: UInt32 + + public init(rawValue: UInt32) { + self.rawValue = rawValue + } + + public static let allocatedBuffer = Flags(rawValue: 1 << 0) + public static let moreCompletions = Flags(rawValue: 1 << 1) + public static let socketNotEmpty = Flags(rawValue: 1 << 2) + public static let isNotificationEvent = Flags(rawValue: 1 << 3) + } +} + +extension IOCompletion { + public var userData: UInt64 { + get { + return rawValue.user_data + } + } + + public var result: Int32 { + get { + return rawValue.res + } + } + + public var flags: IOCompletion.Flags { + get { + return Flags(rawValue: rawValue.flags & 0x0000FFFF) + } + } + + public var bufferIndex: UInt16? { + get { + if self.flags.contains(.allocatedBuffer) { + return UInt16(rawValue.flags >> 16) + } else { + return nil + } + } + } +} diff --git a/Sources/System/IORequest.swift b/Sources/System/IORequest.swift new file mode 100644 index 00000000..efa03f3b --- /dev/null +++ b/Sources/System/IORequest.swift @@ -0,0 +1,140 @@ +@_implementationOnly import CSystem + +public struct IORequest { + internal var rawValue: io_uring_sqe + + public init() { + self.rawValue = io_uring_sqe() + } +} + +extension IORequest { + public enum Operation: UInt8 { + case nop = 0 + case readv = 1 + case writev = 2 + case fsync = 3 + case readFixed = 4 + case writeFixed = 5 + case pollAdd = 6 + case pollRemove = 7 + case syncFileRange = 8 + case sendMessage = 9 + case receiveMessage = 10 + // ... + case openAt = 18 + case read = 22 + case write = 23 + case openAt2 = 28 + + } + + public struct Flags: OptionSet, Hashable, Codable { + public let rawValue: UInt8 + + public init(rawValue: UInt8) { + self.rawValue = rawValue + } + + public static let fixedFile = Flags(rawValue: 1 << 0) + public static let drainQueue = Flags(rawValue: 1 << 1) + public static let linkRequest = Flags(rawValue: 1 << 2) + public static let hardlinkRequest = Flags(rawValue: 1 << 3) + public static let asynchronous = Flags(rawValue: 1 << 4) + public static let selectBuffer = Flags(rawValue: 1 << 5) + public static let skipSuccess = Flags(rawValue: 1 << 6) + } + + public var operation: Operation { + get { Operation(rawValue: rawValue.opcode)! } + set { rawValue.opcode = newValue.rawValue } + } + + public var flags: Flags { + get { Flags(rawValue: rawValue.flags) } + set { rawValue.flags = newValue.rawValue } + } + + public var fileDescriptor: FileDescriptor { + get { FileDescriptor(rawValue: rawValue.fd) } + set { rawValue.fd = newValue.rawValue } + } + + public var offset: UInt64? { + get { + if (rawValue.off == UInt64.max) { + return nil + } else { + return rawValue.off + } + } + set { + if let val = newValue { + rawValue.off = val + } else { + rawValue.off = UInt64.max + } + } + } + + public var buffer: UnsafeMutableRawBufferPointer { + get { + let ptr = UnsafeMutableRawPointer(bitPattern: UInt(exactly: rawValue.addr)!) + return UnsafeMutableRawBufferPointer(start: ptr, count: Int(rawValue.len)) + } + + set { + // TODO: cleanup? + rawValue.addr = UInt64(Int(bitPattern: newValue.baseAddress!)) + rawValue.len = UInt32(exactly: newValue.count)! + } + } +} + +extension IORequest { + static func nop() -> IORequest { + var req = IORequest() + req.operation = .nop + return req + } + + static func read( + from fileDescriptor: FileDescriptor, + into buffer: UnsafeMutableRawBufferPointer, + at offset: UInt64? = nil + ) -> IORequest { + var req = IORequest.readWrite( + op: Operation.read, + fd: fileDescriptor, + buffer: buffer, + offset: offset + ) + fatalError() + } + + static func read( + fixedFile: Int // TODO: AsyncFileDescriptor + ) -> IORequest { + fatalError() + } + + static func write( + + ) -> IORequest { + fatalError() + } + + internal static func readWrite( + op: Operation, + fd: FileDescriptor, + buffer: UnsafeMutableRawBufferPointer, + offset: UInt64? = nil + ) -> IORequest { + var req = IORequest() + req.operation = op + req.fileDescriptor = fd + req.offset = offset + req.buffer = buffer + return req + } +} \ No newline at end of file diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift new file mode 100644 index 00000000..6945d8c0 --- /dev/null +++ b/Sources/System/IORing.swift @@ -0,0 +1,350 @@ +@_implementationOnly import CSystem +import Glibc +import Atomics + +// XXX: this *really* shouldn't be here. oh well. +extension UnsafeMutableRawPointer { + func advanced(by offset: UInt32) -> UnsafeMutableRawPointer { + return advanced(by: Int(offset)) + } +} + +// all pointers in this struct reference kernel-visible memory +struct SQRing { + let kernelHead: UnsafeAtomic + let kernelTail: UnsafeAtomic + var userTail: UInt32 + + // from liburing: the kernel should never change these + // might change in the future with resizable rings? + let ringMask: UInt32 + // let ringEntries: UInt32 - absorbed into array.count + + // ring flags bitfield + // currently used by the kernel only in SQPOLL mode to indicate + // when the polling thread needs to be woken up + let flags: UnsafeAtomic + + // ring array + // maps indexes between the actual ring and the submissionQueueEntries list, + // allowing the latter to be used as a kind of freelist with enough work? + // currently, just 1:1 mapping (0.. +} + +struct CQRing { + let kernelHead: UnsafeAtomic + let kernelTail: UnsafeAtomic + + // TODO: determine if this is actually used + var userHead: UInt32 + + let ringMask: UInt32 + + let cqes: UnsafeBufferPointer +} + +// XXX: This should be a non-copyable type (?) +// demo only runs on Swift 5.8.1 +public final class IORing: Sendable { + let ringFlags: UInt32 + let ringDescriptor: Int32 + + var submissionRing: SQRing + var submissionMutex: Mutex + // FEAT: set this eventually + let submissionPolling: Bool = false + + var completionRing: CQRing + var completionMutex: Mutex + + let submissionQueueEntries: UnsafeMutableBufferPointer + + var registeredFiles: UnsafeMutableBufferPointer? + + // kept around for unmap / cleanup + let ringSize: Int + let ringPtr: UnsafeMutableRawPointer + + public init(queueDepth: UInt32) throws { + var params = io_uring_params() + + ringDescriptor = withUnsafeMutablePointer(to: ¶ms) { + return io_uring_setup(queueDepth, $0); + } + + if (params.features & IORING_FEAT_SINGLE_MMAP == 0 + || params.features & IORING_FEAT_NODROP == 0) { + close(ringDescriptor) + // TODO: error handling + fatalError("kernel not new enough") + } + + if (ringDescriptor < 0) { + // TODO: error handling + } + + let submitRingSize = params.sq_off.array + + params.sq_entries * UInt32(MemoryLayout.size) + + let completionRingSize = params.cq_off.cqes + + params.cq_entries * UInt32(MemoryLayout.size) + + ringSize = Int(max(submitRingSize, completionRingSize)) + + ringPtr = mmap( + /* addr: */ nil, + /* len: */ ringSize, + /* prot: */ PROT_READ | PROT_WRITE, + /* flags: */ MAP_SHARED | MAP_POPULATE, + /* fd: */ ringDescriptor, + /* offset: */ __off_t(IORING_OFF_SQ_RING) + ); + + if (ringPtr == MAP_FAILED) { + perror("mmap"); + // TODO: error handling + fatalError() + } + + let kernelHead = UnsafeAtomic(at: + ringPtr.advanced(by: params.sq_off.head) + .assumingMemoryBound(to: UInt32.AtomicRepresentation.self) + ) + + submissionRing = SQRing( + kernelHead: UnsafeAtomic( + at: ringPtr.advanced(by: params.sq_off.head) + .assumingMemoryBound(to: UInt32.AtomicRepresentation.self) + ), + kernelTail: UnsafeAtomic( + at: ringPtr.advanced(by: params.sq_off.tail) + .assumingMemoryBound(to: UInt32.AtomicRepresentation.self) + ), + userTail: 0, // no requests yet + ringMask: ringPtr.advanced(by: params.sq_off.ring_mask) + .assumingMemoryBound(to: UInt32.self).pointee, + flags: UnsafeAtomic( + at: ringPtr.advanced(by: params.sq_off.flags) + .assumingMemoryBound(to: UInt32.AtomicRepresentation.self) + ), + array: UnsafeMutableBufferPointer( + start: ringPtr.advanced(by: params.sq_off.array) + .assumingMemoryBound(to: UInt32.self), + count: Int(ringPtr.advanced(by: params.sq_off.ring_entries) + .assumingMemoryBound(to: UInt32.self).pointee) + ) + ) + + // fill submission ring array with 1:1 map to underlying SQEs + for i in 0...size, + /* prot: */ PROT_READ | PROT_WRITE, + /* flags: */ MAP_SHARED | MAP_POPULATE, + /* fd: */ ringDescriptor, + /* offset: */ __off_t(IORING_OFF_SQES) + ); + + if (sqes == MAP_FAILED) { + perror("mmap"); + // TODO: error handling + fatalError() + } + + submissionQueueEntries = UnsafeMutableBufferPointer( + start: sqes!.assumingMemoryBound(to: io_uring_sqe.self), + count: Int(params.sq_entries) + ) + + completionRing = CQRing( + kernelHead: UnsafeAtomic( + at: ringPtr.advanced(by: params.cq_off.head) + .assumingMemoryBound(to: UInt32.AtomicRepresentation.self) + ), + kernelTail: UnsafeAtomic( + at: ringPtr.advanced(by: params.cq_off.tail) + .assumingMemoryBound(to: UInt32.AtomicRepresentation.self) + ), + userHead: 0, // no completions yet + ringMask: ringPtr.advanced(by: params.cq_off.ring_mask) + .assumingMemoryBound(to: UInt32.self).pointee, + cqes: UnsafeBufferPointer( + start: ringPtr.advanced(by: params.cq_off.cqes) + .assumingMemoryBound(to: io_uring_cqe.self), + count: Int(ringPtr.advanced(by: params.cq_off.ring_entries) + .assumingMemoryBound(to: UInt32.self).pointee) + ) + ) + + self.submissionMutex = Mutex() + self.completionMutex = Mutex() + + self.ringFlags = params.flags + } + + func blockingConsumeCompletion() -> IOCompletion { + self.completionMutex.lock() + defer { self.completionMutex.unlock() } + + if let completion = _tryConsumeCompletion() { + return completion + } else { + _waitForCompletion() + return _tryConsumeCompletion().unsafelyUnwrapped + } + } + + func _waitForCompletion() { + // TODO: error handling + io_uring_enter(ringDescriptor, 0, 1, IORING_ENTER_GETEVENTS, nil) + } + + func tryConsumeCompletion() -> IOCompletion? { + self.completionMutex.lock() + defer { self.completionMutex.unlock() } + return _tryConsumeCompletion() + } + + func _tryConsumeCompletion() -> IOCompletion? { + let tail = completionRing.kernelTail.load(ordering: .acquiring) + var head = completionRing.kernelHead.load(ordering: .relaxed) + + if tail != head { + // 32 byte copy - oh well + let res = completionRing.cqes[Int(head & completionRing.ringMask)] + completionRing.kernelHead.store(head + 1, ordering: .relaxed) + return IOCompletion(rawValue: res) + } + + return nil + } + + + func registerFiles(count: UInt32) { + // TODO: implement + guard self.registeredFiles == nil else { fatalError() } + let fileBuf = UnsafeMutableBufferPointer.allocate(capacity: Int(count)) + fileBuf.initialize(repeating: UInt32.max) + io_uring_register( + self.ringDescriptor, + IORING_REGISTER_FILES, + fileBuf.baseAddress!, + count + ) + // TODO: error handling + self.registeredFiles = fileBuf + } + + func unregisterFiles() { + if self.registeredFiles != nil { + io_uring_register( + self.ringDescriptor, + IORING_UNREGISTER_FILES, + self.registeredFiles!.baseAddress!, + UInt32(self.registeredFiles!.count) + ) + // TODO: error handling + self.registeredFiles!.deallocate() + self.registeredFiles = nil + } + } + + // register a group of buffers + func registerBuffers(bufSize: UInt32, count: UInt32) { + // + + } + + func getBuffer() -> (index: Int, buf: UnsafeRawBufferPointer) { + fatalError() + } + + // TODO: types + func submitRequests() { + self.submissionMutex.lock() + defer { self.submissionMutex.unlock() } + self._submitRequests() + } + + func _submitRequests() { + let flushedEvents = _flushQueue() + + // Ring always needs enter right now; + // TODO: support SQPOLL here + + let ret = io_uring_enter(ringDescriptor, flushedEvents, 0, 0, nil) + // TODO: handle errors + } + + internal func _flushQueue() -> UInt32 { + self.submissionRing.kernelTail.store( + self.submissionRing.userTail, ordering: .relaxed + ) + return self.submissionRing.userTail - + self.submissionRing.kernelHead.load(ordering: .relaxed) + } + + + func writeRequest(_ request: __owned IORequest) -> Bool { + self.submissionMutex.lock() + defer { self.submissionMutex.unlock() } + return _writeRequest(request) + } + + internal func _writeRequest(_ request: __owned IORequest) -> Bool { + if let entry = _getSubmissionEntry() { + entry.pointee = request.rawValue + return true + } + return false + } + + internal func _blockingGetSubmissionEntry() -> UnsafeMutablePointer { + while true { + if let entry = _getSubmissionEntry() { + return entry + } + // TODO: actually block here instead of spinning + } + + } + + internal func _getSubmissionEntry() -> UnsafeMutablePointer? { + let next = self.submissionRing.userTail + 1 + + // FEAT: smp load when SQPOLL in use (not in MVP) + let kernelHead = self.submissionRing.kernelHead.load(ordering: .relaxed) + + // FEAT: 128-bit event support (not in MVP) + if (next - kernelHead <= self.submissionRing.array.count) { + // let sqe = &sq->sqes[(sq->sqe_tail & sq->ring_mask) << shift]; + let sqeIndex = Int( + self.submissionRing.userTail & self.submissionRing.ringMask + ) + + let sqe = self.submissionQueueEntries + .baseAddress.unsafelyUnwrapped + .advanced(by: sqeIndex) + + self.submissionRing.userTail = next; + return sqe + } + return nil + } + + deinit { + munmap(ringPtr, ringSize); + munmap( + UnsafeMutableRawPointer(submissionQueueEntries.baseAddress!), + submissionQueueEntries.count * MemoryLayout.size + ) + close(ringDescriptor) + } +}; + diff --git a/Sources/System/IORingBuffer.swift b/Sources/System/IORingBuffer.swift new file mode 100644 index 00000000..e69de29b diff --git a/Sources/System/IORingFileSlot.swift b/Sources/System/IORingFileSlot.swift new file mode 100644 index 00000000..d0a7c666 --- /dev/null +++ b/Sources/System/IORingFileSlot.swift @@ -0,0 +1,8 @@ +class IORingFileSlot { + + + deinit { + // return file slot + + } +} \ No newline at end of file diff --git a/Sources/System/Lock.swift b/Sources/System/Lock.swift new file mode 100644 index 00000000..c5204ba0 --- /dev/null +++ b/Sources/System/Lock.swift @@ -0,0 +1,37 @@ +// TODO: write against kernel APIs directly? +import Glibc + +public final class Mutex { + @usableFromInline let mutex: UnsafeMutablePointer + + @inlinable init() { + self.mutex = UnsafeMutablePointer.allocate(capacity: 1) + self.mutex.initialize(to: pthread_mutex_t()) + pthread_mutex_init(self.mutex, nil) + } + + @inlinable deinit { + defer { mutex.deallocate() } + guard pthread_mutex_destroy(mutex) == 0 else { + preconditionFailure("unable to destroy mutex") + } + } + + // XXX: this is because we need to lock the mutex in the context of a submit() function + // and unlock *before* the UnsafeContinuation returns. + // Code looks like: { + // // prepare request + // io_uring_get_sqe() + // io_uring_prep_foo(...) + // return await withUnsafeContinuation { + // sqe->user_data = ...; io_uring_submit(); unlock(); + // } + // } + @inlinable @inline(__always) public func lock() { + pthread_mutex_lock(mutex) + } + + @inlinable @inline(__always) public func unlock() { + pthread_mutex_unlock(mutex) + } +} diff --git a/Sources/System/ManagedIORing.swift b/Sources/System/ManagedIORing.swift new file mode 100644 index 00000000..32c84faf --- /dev/null +++ b/Sources/System/ManagedIORing.swift @@ -0,0 +1,33 @@ +final public class ManagedIORing: @unchecked Sendable { + var internalRing: IORing + + init(queueDepth: UInt32) throws { + self.internalRing = try IORing(queueDepth: queueDepth) + self.startWaiter() + } + + private func startWaiter() { + Task.detached { + while (!Task.isCancelled) { + let cqe = self.internalRing.blockingConsumeCompletion() + + let cont = unsafeBitCast(cqe.userData, to: UnsafeContinuation.self) + cont.resume(returning: cqe) + } + } + } + + @_unsafeInheritExecutor + public func submitAndWait(_ request: __owned IORequest) async -> IOCompletion { + self.internalRing.submissionMutex.lock() + return await withUnsafeContinuation { cont in + let entry = internalRing._blockingGetSubmissionEntry() + entry.pointee = request.rawValue + entry.pointee.user_data = unsafeBitCast(cont, to: UInt64.self) + self.internalRing._submitRequests() + self.internalRing.submissionMutex.unlock() + } + } + + +} \ No newline at end of file From 6338029b74b84b79795a0cc24bcc189883ec083d Mon Sep 17 00:00:00 2001 From: Lucy Satheesan Date: Mon, 31 Jul 2023 22:28:43 -0700 Subject: [PATCH 02/48] stage 2: IORequest enum --- Sources/System/IORequest.swift | 206 ++++++++++---------------- Sources/System/IORing.swift | 218 +++++++++++++++++++++------- Sources/System/IORingBuffer.swift | 0 Sources/System/IORingError.swift | 3 + Sources/System/IORingFileSlot.swift | 8 - Sources/System/Lock.swift | 2 +- Sources/System/ManagedIORing.swift | 11 +- Sources/System/RawIORequest.swift | 139 ++++++++++++++++++ 8 files changed, 396 insertions(+), 191 deletions(-) delete mode 100644 Sources/System/IORingBuffer.swift create mode 100644 Sources/System/IORingError.swift delete mode 100644 Sources/System/IORingFileSlot.swift create mode 100644 Sources/System/RawIORequest.swift diff --git a/Sources/System/IORequest.swift b/Sources/System/IORequest.swift index efa03f3b..6b26a4d1 100644 --- a/Sources/System/IORequest.swift +++ b/Sources/System/IORequest.swift @@ -1,140 +1,82 @@ -@_implementationOnly import CSystem - -public struct IORequest { - internal var rawValue: io_uring_sqe +import struct CSystem.io_uring_sqe + +public enum IORequest { + case nop // nothing here + case openat( + atDirectory: FileDescriptor, + path: UnsafePointer, + FileDescriptor.AccessMode, + options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), + permissions: FilePermissions? = nil, + intoSlot: IORingFileSlot? = nil + ) + case read( + file: File, + buffer: Buffer, + offset: UInt64 = 0 + ) + case write( + file: File, + buffer: Buffer, + offset: UInt64 = 0 + ) + + public enum Buffer { + case registered(IORingBuffer) + case unregistered(UnsafeMutableRawBufferPointer) + } - public init() { - self.rawValue = io_uring_sqe() + public enum File { + case registered(IORingFileSlot) + case unregistered(FileDescriptor) } } extension IORequest { - public enum Operation: UInt8 { - case nop = 0 - case readv = 1 - case writev = 2 - case fsync = 3 - case readFixed = 4 - case writeFixed = 5 - case pollAdd = 6 - case pollRemove = 7 - case syncFileRange = 8 - case sendMessage = 9 - case receiveMessage = 10 - // ... - case openAt = 18 - case read = 22 - case write = 23 - case openAt2 = 28 - - } - - public struct Flags: OptionSet, Hashable, Codable { - public let rawValue: UInt8 - - public init(rawValue: UInt8) { - self.rawValue = rawValue - } - - public static let fixedFile = Flags(rawValue: 1 << 0) - public static let drainQueue = Flags(rawValue: 1 << 1) - public static let linkRequest = Flags(rawValue: 1 << 2) - public static let hardlinkRequest = Flags(rawValue: 1 << 3) - public static let asynchronous = Flags(rawValue: 1 << 4) - public static let selectBuffer = Flags(rawValue: 1 << 5) - public static let skipSuccess = Flags(rawValue: 1 << 6) - } - - public var operation: Operation { - get { Operation(rawValue: rawValue.opcode)! } - set { rawValue.opcode = newValue.rawValue } - } - - public var flags: Flags { - get { Flags(rawValue: rawValue.flags) } - set { rawValue.flags = newValue.rawValue } - } - - public var fileDescriptor: FileDescriptor { - get { FileDescriptor(rawValue: rawValue.fd) } - set { rawValue.fd = newValue.rawValue } - } - - public var offset: UInt64? { - get { - if (rawValue.off == UInt64.max) { - return nil - } else { - return rawValue.off - } - } - set { - if let val = newValue { - rawValue.off = val - } else { - rawValue.off = UInt64.max - } - } - } - - public var buffer: UnsafeMutableRawBufferPointer { - get { - let ptr = UnsafeMutableRawPointer(bitPattern: UInt(exactly: rawValue.addr)!) - return UnsafeMutableRawBufferPointer(start: ptr, count: Int(rawValue.len)) - } - - set { - // TODO: cleanup? - rawValue.addr = UInt64(Int(bitPattern: newValue.baseAddress!)) - rawValue.len = UInt32(exactly: newValue.count)! + @inlinable @inline(__always) + public func makeRawRequest() -> RawIORequest { + var request = RawIORequest() + switch self { + case .nop: + request.operation = .nop + case .openat(let atDirectory, let path, let mode, let options, let permissions, let slot): + // TODO: use rawValue less + request.operation = .openAt + request.fileDescriptor = atDirectory + request.rawValue.addr = unsafeBitCast(path, to: UInt64.self) + request.rawValue.open_flags = UInt32(bitPattern: options.rawValue | mode.rawValue) + request.rawValue.len = permissions?.rawValue ?? 0 + request.rawValue.file_index = UInt32(slot?.index ?? 0) + case .read(let file, let buffer, let offset), .write(let file, let buffer, let offset): + if case .read = self { + if case .registered = buffer { + request.operation = .readFixed + } else { + request.operation = .read + } + } else { + if case .registered = buffer { + request.operation = .writeFixed + } else { + request.operation = .write + } + } + switch file { + case .registered(let regFile): + request.rawValue.fd = Int32(exactly: regFile.index)! + request.flags = .fixedFile + case .unregistered(let fd): + request.fileDescriptor = fd + } + switch buffer { + case .registered(let regBuf): + request.buffer = regBuf.unsafeBuffer + request.rawValue.buf_index = UInt16(exactly: regBuf.index)! + case .unregistered(let buf): + request.buffer = buf + } + request.offset = offset } + return request } } - -extension IORequest { - static func nop() -> IORequest { - var req = IORequest() - req.operation = .nop - return req - } - - static func read( - from fileDescriptor: FileDescriptor, - into buffer: UnsafeMutableRawBufferPointer, - at offset: UInt64? = nil - ) -> IORequest { - var req = IORequest.readWrite( - op: Operation.read, - fd: fileDescriptor, - buffer: buffer, - offset: offset - ) - fatalError() - } - - static func read( - fixedFile: Int // TODO: AsyncFileDescriptor - ) -> IORequest { - fatalError() - } - - static func write( - - ) -> IORequest { - fatalError() - } - - internal static func readWrite( - op: Operation, - fd: FileDescriptor, - buffer: UnsafeMutableRawBufferPointer, - offset: UInt64? = nil - ) -> IORequest { - var req = IORequest() - req.operation = op - req.fileDescriptor = fd - req.offset = offset - req.buffer = buffer - return req - } -} \ No newline at end of file diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 6945d8c0..0c2f1384 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -1,6 +1,8 @@ @_implementationOnly import CSystem -import Glibc -import Atomics +import struct CSystem.io_uring_sqe + +@_implementationOnly import Atomics +import Glibc // needed for mmap // XXX: this *really* shouldn't be here. oh well. extension UnsafeMutableRawPointer { @@ -10,7 +12,7 @@ extension UnsafeMutableRawPointer { } // all pointers in this struct reference kernel-visible memory -struct SQRing { +@usableFromInline struct SQRing { let kernelHead: UnsafeAtomic let kernelTail: UnsafeAtomic var userTail: UInt32 @@ -44,14 +46,90 @@ struct CQRing { let cqes: UnsafeBufferPointer } +internal class ResourceManager: @unchecked Sendable { + typealias Resource = T + let resourceList: UnsafeMutableBufferPointer + var freeList: [Int] + let mutex: Mutex + + init(_ res: UnsafeMutableBufferPointer) { + self.resourceList = res + self.freeList = [Int](resourceList.indices) + self.mutex = Mutex() + } + + func getResource() -> IOResource? { + self.mutex.lock() + defer { self.mutex.unlock() } + if let index = freeList.popLast() { + return IOResource( + rescource: resourceList[index], + index: index, + manager: self + ) + } else { + return nil + } + } + + func releaseResource(index: Int) { + self.mutex.lock() + defer { self.mutex.unlock() } + self.freeList.append(index) + } +} + +public class IOResource { + typealias Resource = T + @usableFromInline let resource: T + @usableFromInline let index: Int + let manager: ResourceManager + + internal init( + rescource: T, + index: Int, + manager: ResourceManager + ) { + self.resource = rescource + self.index = index + self.manager = manager + } + + func withResource() { + + } + + deinit { + self.manager.releaseResource(index: self.index) + } +} + +public typealias IORingFileSlot = IOResource +public typealias IORingBuffer = IOResource + +extension IORingFileSlot { + public var unsafeFileSlot: Int { + return index + } +} +extension IORingBuffer { + public var unsafeBuffer: UnsafeMutableRawBufferPointer { + get { + return .init(start: resource.iov_base, count: resource.iov_len) + } + } +} + + + // XXX: This should be a non-copyable type (?) // demo only runs on Swift 5.8.1 -public final class IORing: Sendable { +public final class IORing: @unchecked Sendable { let ringFlags: UInt32 let ringDescriptor: Int32 - var submissionRing: SQRing - var submissionMutex: Mutex + @usableFromInline var submissionRing: SQRing + @usableFromInline var submissionMutex: Mutex // FEAT: set this eventually let submissionPolling: Bool = false @@ -59,13 +137,14 @@ public final class IORing: Sendable { var completionMutex: Mutex let submissionQueueEntries: UnsafeMutableBufferPointer - - var registeredFiles: UnsafeMutableBufferPointer? // kept around for unmap / cleanup let ringSize: Int let ringPtr: UnsafeMutableRawPointer + var registeredFiles: ResourceManager? + var registeredBuffers: ResourceManager? + public init(queueDepth: UInt32) throws { var params = io_uring_params() @@ -77,7 +156,7 @@ public final class IORing: Sendable { || params.features & IORING_FEAT_NODROP == 0) { close(ringDescriptor) // TODO: error handling - fatalError("kernel not new enough") + throw IORingError.missingRequiredFeatures } if (ringDescriptor < 0) { @@ -104,14 +183,9 @@ public final class IORing: Sendable { if (ringPtr == MAP_FAILED) { perror("mmap"); // TODO: error handling - fatalError() + fatalError("mmap failed in ring setup") } - let kernelHead = UnsafeAtomic(at: - ringPtr.advanced(by: params.sq_off.head) - .assumingMemoryBound(to: UInt32.AtomicRepresentation.self) - ) - submissionRing = SQRing( kernelHead: UnsafeAtomic( at: ringPtr.advanced(by: params.sq_off.head) @@ -154,7 +228,7 @@ public final class IORing: Sendable { if (sqes == MAP_FAILED) { perror("mmap"); // TODO: error handling - fatalError() + fatalError("sqe mmap failed in ring setup") } submissionQueueEntries = UnsafeMutableBufferPointer( @@ -195,16 +269,29 @@ public final class IORing: Sendable { if let completion = _tryConsumeCompletion() { return completion } else { - _waitForCompletion() + while true { + let res = io_uring_enter(ringDescriptor, 0, 1, IORING_ENTER_GETEVENTS, nil) + // error handling: + // EAGAIN / EINTR (try again), + // EBADF / EBADFD / EOPNOTSUPP / ENXIO + // (failure in ring lifetime management, fatal), + // EINVAL (bad constant flag?, fatal), + // EFAULT (bad address for argument from library, fatal) + // EBUSY (not enough space for events; implies events filled + // by kernel between kernelTail load and now) + if res >= 0 || res == -EBUSY { + break + } else if res == -EAGAIN || res == -EINTR { + continue + } + fatalError("fatal error in receiving requests: " + + Errno(rawValue: -res).debugDescription + ) + } return _tryConsumeCompletion().unsafelyUnwrapped } } - func _waitForCompletion() { - // TODO: error handling - io_uring_enter(ringDescriptor, 0, 1, IORING_ENTER_GETEVENTS, nil) - } - func tryConsumeCompletion() -> IOCompletion? { self.completionMutex.lock() defer { self.completionMutex.unlock() } @@ -213,7 +300,7 @@ public final class IORing: Sendable { func _tryConsumeCompletion() -> IOCompletion? { let tail = completionRing.kernelTail.load(ordering: .acquiring) - var head = completionRing.kernelHead.load(ordering: .relaxed) + let head = completionRing.kernelHead.load(ordering: .relaxed) if tail != head { // 32 byte copy - oh well @@ -227,7 +314,6 @@ public final class IORing: Sendable { func registerFiles(count: UInt32) { - // TODO: implement guard self.registeredFiles == nil else { fatalError() } let fileBuf = UnsafeMutableBufferPointer.allocate(capacity: Int(count)) fileBuf.initialize(repeating: UInt32.max) @@ -238,31 +324,43 @@ public final class IORing: Sendable { count ) // TODO: error handling - self.registeredFiles = fileBuf + self.registeredFiles = ResourceManager(fileBuf) } func unregisterFiles() { - if self.registeredFiles != nil { - io_uring_register( - self.ringDescriptor, - IORING_UNREGISTER_FILES, - self.registeredFiles!.baseAddress!, - UInt32(self.registeredFiles!.count) - ) - // TODO: error handling - self.registeredFiles!.deallocate() - self.registeredFiles = nil - } + fatalError("failed to unregister files") + } + + func getFile() -> IORingFileSlot? { + return self.registeredFiles?.getResource() } // register a group of buffers func registerBuffers(bufSize: UInt32, count: UInt32) { - // + let iovecs = UnsafeMutableBufferPointer.allocate(capacity: Int(count)) + let intBufSize = Int(bufSize) + for i in 0.. IORingBuffer? { + return self.registeredBuffers?.getResource() } - func getBuffer() -> (index: Int, buf: UnsafeRawBufferPointer) { - fatalError() + func unregisterBuffers() { + fatalError("failed to unregister buffers: TODO") } // TODO: types @@ -277,9 +375,22 @@ public final class IORing: Sendable { // Ring always needs enter right now; // TODO: support SQPOLL here - - let ret = io_uring_enter(ringDescriptor, flushedEvents, 0, 0, nil) - // TODO: handle errors + while true { + let ret = io_uring_enter(ringDescriptor, flushedEvents, 0, 0, nil) + // error handling: + // EAGAIN / EINTR (try again), + // EBADF / EBADFD / EOPNOTSUPP / ENXIO + // (failure in ring lifetime management, fatal), + // EINVAL (bad constant flag?, fatal), + // EFAULT (bad address for argument from library, fatal) + if ret == -EAGAIN || ret == -EINTR { + continue + } else if ret < 0 { + fatalError("fatal error in submitting requests: " + + Errno(rawValue: -ret).debugDescription + ) + } + } } internal func _flushQueue() -> UInt32 { @@ -291,20 +402,28 @@ public final class IORing: Sendable { } + @inlinable @inline(__always) func writeRequest(_ request: __owned IORequest) -> Bool { self.submissionMutex.lock() defer { self.submissionMutex.unlock() } - return _writeRequest(request) + return _writeRequest(request.makeRawRequest()) } - internal func _writeRequest(_ request: __owned IORequest) -> Bool { - if let entry = _getSubmissionEntry() { - entry.pointee = request.rawValue - return true - } - return false + @inlinable @inline(__always) + func writeAndSubmit(_ request: __owned IORequest) -> Bool { + self.submissionMutex.lock() + defer { self.submissionMutex.unlock() } + return _writeRequest(request.makeRawRequest()) + } + + @inlinable @inline(__always) + internal func _writeRequest(_ request: __owned RawIORequest) -> Bool { + let entry = _blockingGetSubmissionEntry() + entry.pointee = request.rawValue + return true } + @inlinable @inline(__always) internal func _blockingGetSubmissionEntry() -> UnsafeMutablePointer { while true { if let entry = _getSubmissionEntry() { @@ -315,6 +434,7 @@ public final class IORing: Sendable { } + @usableFromInline @inline(__always) internal func _getSubmissionEntry() -> UnsafeMutablePointer? { let next = self.submissionRing.userTail + 1 diff --git a/Sources/System/IORingBuffer.swift b/Sources/System/IORingBuffer.swift deleted file mode 100644 index e69de29b..00000000 diff --git a/Sources/System/IORingError.swift b/Sources/System/IORingError.swift new file mode 100644 index 00000000..d87b2938 --- /dev/null +++ b/Sources/System/IORingError.swift @@ -0,0 +1,3 @@ +enum IORingError: Error { + case missingRequiredFeatures +} diff --git a/Sources/System/IORingFileSlot.swift b/Sources/System/IORingFileSlot.swift deleted file mode 100644 index d0a7c666..00000000 --- a/Sources/System/IORingFileSlot.swift +++ /dev/null @@ -1,8 +0,0 @@ -class IORingFileSlot { - - - deinit { - // return file slot - - } -} \ No newline at end of file diff --git a/Sources/System/Lock.swift b/Sources/System/Lock.swift index c5204ba0..fd20c641 100644 --- a/Sources/System/Lock.swift +++ b/Sources/System/Lock.swift @@ -1,7 +1,7 @@ // TODO: write against kernel APIs directly? import Glibc -public final class Mutex { +@usableFromInline final class Mutex { @usableFromInline let mutex: UnsafeMutablePointer @inlinable init() { diff --git a/Sources/System/ManagedIORing.swift b/Sources/System/ManagedIORing.swift index 32c84faf..580eacc1 100644 --- a/Sources/System/ManagedIORing.swift +++ b/Sources/System/ManagedIORing.swift @@ -3,6 +3,8 @@ final public class ManagedIORing: @unchecked Sendable { init(queueDepth: UInt32) throws { self.internalRing = try IORing(queueDepth: queueDepth) + self.internalRing.registerBuffers(bufSize: 655336, count: 4) + self.internalRing.registerFiles(count: 32) self.startWaiter() } @@ -22,12 +24,19 @@ final public class ManagedIORing: @unchecked Sendable { self.internalRing.submissionMutex.lock() return await withUnsafeContinuation { cont in let entry = internalRing._blockingGetSubmissionEntry() - entry.pointee = request.rawValue + entry.pointee = request.makeRawRequest().rawValue entry.pointee.user_data = unsafeBitCast(cont, to: UInt64.self) self.internalRing._submitRequests() self.internalRing.submissionMutex.unlock() } } + internal func getFileSlot() -> IORingFileSlot? { + self.internalRing.getFile() + } + + internal func getBuffer() -> IORingBuffer? { + self.internalRing.getBuffer() + } } \ No newline at end of file diff --git a/Sources/System/RawIORequest.swift b/Sources/System/RawIORequest.swift new file mode 100644 index 00000000..fa50b439 --- /dev/null +++ b/Sources/System/RawIORequest.swift @@ -0,0 +1,139 @@ +// TODO: investigate @usableFromInline / @_implementationOnly dichotomy +@_implementationOnly import CSystem +import struct CSystem.io_uring_sqe + +public struct RawIORequest { + @usableFromInline var rawValue: io_uring_sqe + + public init() { + self.rawValue = io_uring_sqe() + } +} + +extension RawIORequest { + public enum Operation: UInt8 { + case nop = 0 + case readv = 1 + case writev = 2 + case fsync = 3 + case readFixed = 4 + case writeFixed = 5 + case pollAdd = 6 + case pollRemove = 7 + case syncFileRange = 8 + case sendMessage = 9 + case receiveMessage = 10 + // ... + case openAt = 18 + case read = 22 + case write = 23 + case openAt2 = 28 + + } + + public struct Flags: OptionSet, Hashable, Codable { + public let rawValue: UInt8 + + public init(rawValue: UInt8) { + self.rawValue = rawValue + } + + public static let fixedFile = Flags(rawValue: 1 << 0) + public static let drainQueue = Flags(rawValue: 1 << 1) + public static let linkRequest = Flags(rawValue: 1 << 2) + public static let hardlinkRequest = Flags(rawValue: 1 << 3) + public static let asynchronous = Flags(rawValue: 1 << 4) + public static let selectBuffer = Flags(rawValue: 1 << 5) + public static let skipSuccess = Flags(rawValue: 1 << 6) + } + + public var operation: Operation { + get { Operation(rawValue: rawValue.opcode)! } + set { rawValue.opcode = newValue.rawValue } + } + + public var flags: Flags { + get { Flags(rawValue: rawValue.flags) } + set { rawValue.flags = newValue.rawValue } + } + + public var fileDescriptor: FileDescriptor { + get { FileDescriptor(rawValue: rawValue.fd) } + set { rawValue.fd = newValue.rawValue } + } + + public var offset: UInt64? { + get { + if (rawValue.off == UInt64.max) { + return nil + } else { + return rawValue.off + } + } + set { + if let val = newValue { + rawValue.off = val + } else { + rawValue.off = UInt64.max + } + } + } + + public var buffer: UnsafeMutableRawBufferPointer { + get { + let ptr = UnsafeMutableRawPointer(bitPattern: UInt(exactly: rawValue.addr)!) + return UnsafeMutableRawBufferPointer(start: ptr, count: Int(rawValue.len)) + } + + set { + // TODO: cleanup? + rawValue.addr = UInt64(Int(bitPattern: newValue.baseAddress!)) + rawValue.len = UInt32(exactly: newValue.count)! + } + } + + public enum RequestFlags { + case readWriteFlags(ReadWriteFlags) + // case fsyncFlags(FsyncFlags?) + // poll_events + // poll32_events + // sync_range_flags + // msg_flags + // timeout_flags + // accept_flags + // cancel_flags + case openFlags(FileDescriptor.OpenOptions) + // statx_flags + // fadvise_advice + // splice_flags + } + + public struct ReadWriteFlags: OptionSet, Hashable, Codable { + public var rawValue: UInt32 + public init(rawValue: UInt32) { + self.rawValue = rawValue + } + + public static let highPriority = ReadWriteFlags(rawValue: 1 << 0) + + // sync with only data integrity + public static let dataSync = ReadWriteFlags(rawValue: 1 << 1) + + // sync with full data + file integrity + public static let fileSync = ReadWriteFlags(rawValue: 1 << 2) + + // return -EAGAIN if operation blocks + public static let noWait = ReadWriteFlags(rawValue: 1 << 3) + + // append to end of the file + public static let append = ReadWriteFlags(rawValue: 1 << 4) + } +} + +extension RawIORequest { + static func nop() -> RawIORequest { + var req = RawIORequest() + req.operation = .nop + return req + } +} \ No newline at end of file From 996e940942f976f7a2de8751375c21413812ac41 Mon Sep 17 00:00:00 2001 From: Lucy Satheesan Date: Mon, 31 Jul 2023 22:28:54 -0700 Subject: [PATCH 03/48] initial AsyncFileDescriptor work --- Sources/System/AsyncFileDescriptor.swift | 71 ++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 Sources/System/AsyncFileDescriptor.swift diff --git a/Sources/System/AsyncFileDescriptor.swift b/Sources/System/AsyncFileDescriptor.swift new file mode 100644 index 00000000..97427812 --- /dev/null +++ b/Sources/System/AsyncFileDescriptor.swift @@ -0,0 +1,71 @@ +@_implementationOnly import CSystem + +public class AsyncFileDescriptor { + var open: Bool = true + @usableFromInline let fileSlot: IORingFileSlot + @usableFromInline let ring: ManagedIORing + + static func openat( + atDirectory: FileDescriptor = FileDescriptor(rawValue: AT_FDCWD), + path: FilePath, + _ mode: FileDescriptor.AccessMode, + options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), + permissions: FilePermissions? = nil, + onRing ring: ManagedIORing + ) async throws -> AsyncFileDescriptor { + // todo; real error type + guard let fileSlot = ring.getFileSlot() else { + throw IORingError.missingRequiredFeatures + } + let cstr = path.withCString { + return $0 // bad + } + let res = await ring.submitAndWait(.openat( + atDirectory: atDirectory, + path: cstr, + mode, + options: options, + permissions: permissions, intoSlot: fileSlot + )) + if res.result < 0 { + throw Errno(rawValue: -res.result) + } + + return AsyncFileDescriptor( + fileSlot, ring: ring + ) + } + + internal init(_ fileSlot: IORingFileSlot, ring: ManagedIORing) { + self.fileSlot = fileSlot + self.ring = ring + } + + func close() async throws { + self.open = false + fatalError() + } + + @inlinable @inline(__always) @_unsafeInheritExecutor + func read( + into buffer: IORequest.Buffer, + atAbsoluteOffset offset: UInt64 = UInt64.max + ) async throws -> UInt32 { + let res = await ring.submitAndWait(.read( + file: .registered(self.fileSlot), + buffer: buffer, + offset: offset + )) + if res.result < 0 { + throw Errno(rawValue: -res.result) + } else { + return UInt32(bitPattern: res.result) + } + } + + deinit { + if (self.open) { + // TODO: close + } + } +} From 1f821a8539af62aab858f356ab9aa21592742ccb Mon Sep 17 00:00:00 2001 From: Lucy Satheesan Date: Tue, 8 Aug 2023 09:44:32 -0700 Subject: [PATCH 04/48] AsyncSequence draft implementation --- Sources/System/AsyncFileDescriptor.swift | 52 ++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/Sources/System/AsyncFileDescriptor.swift b/Sources/System/AsyncFileDescriptor.swift index 97427812..6e39e055 100644 --- a/Sources/System/AsyncFileDescriptor.swift +++ b/Sources/System/AsyncFileDescriptor.swift @@ -69,3 +69,55 @@ public class AsyncFileDescriptor { } } } + +extension AsyncFileDescriptor: AsyncSequence { + public func makeAsyncIterator() -> FileIterator { + return .init(self) + } + + public typealias AsyncIterator = FileIterator + public typealias Element = UInt8 +} + +public struct FileIterator: AsyncIteratorProtocol { + @usableFromInline let file: AsyncFileDescriptor + @usableFromInline var buffer: IORingBuffer + @usableFromInline var done: Bool + + @usableFromInline internal var currentByte: UnsafeRawPointer? + @usableFromInline internal var lastByte: UnsafeRawPointer? + + init(_ file: AsyncFileDescriptor) { + self.file = file + self.buffer = file.ring.getBuffer()! + self.done = false + } + + @inlinable @inline(__always) + public mutating func nextBuffer() async throws { + let buffer = self.buffer + + let bytesRead = try await file.read(into: .registered(buffer)) + if _fastPath(bytesRead != 0) { + let bufPointer = buffer.unsafeBuffer.baseAddress.unsafelyUnwrapped + self.currentByte = UnsafeRawPointer(bufPointer) + self.lastByte = UnsafeRawPointer(bufPointer.advanced(by: Int(bytesRead))) + } else { + self.done = true + } + } + + @inlinable @inline(__always) @_unsafeInheritExecutor + public mutating func next() async throws -> UInt8? { + if _fastPath(currentByte != lastByte) { + // SAFETY: both pointers should be non-nil if they're not equal + let byte = currentByte.unsafelyUnwrapped.load(as: UInt8.self) + currentByte = currentByte.unsafelyUnwrapped + 1 + return byte + } else if done { + return nil + } + try await nextBuffer() + return try await next() + } +} From 0762f57d806df3f5ad9e268d18eddbe4f3cb2594 Mon Sep 17 00:00:00 2001 From: Lucy Satheesan Date: Thu, 10 Aug 2023 16:03:21 -0700 Subject: [PATCH 05/48] migrate CSystem to systemLibrary for some reason, the linker on my linux machine fails to link the tests otherwise. investigate / fix before merging. --- Package.swift | 5 ++--- Sources/CSystem/{include => }/CSystemLinux.h | 0 Sources/CSystem/{include => }/CSystemWindows.h | 0 Sources/CSystem/{include => }/io_uring.h | 5 +++++ Sources/CSystem/{include => }/module.modulemap | 0 Sources/CSystem/shims.c | 18 ------------------ 6 files changed, 7 insertions(+), 21 deletions(-) rename Sources/CSystem/{include => }/CSystemLinux.h (100%) rename Sources/CSystem/{include => }/CSystemWindows.h (100%) rename Sources/CSystem/{include => }/io_uring.h (94%) rename Sources/CSystem/{include => }/module.modulemap (100%) delete mode 100644 Sources/CSystem/shims.c diff --git a/Package.swift b/Package.swift index 60f1a562..1c058acf 100644 --- a/Package.swift +++ b/Package.swift @@ -21,9 +21,8 @@ let package = Package( .package(url: "https://github.com/apple/swift-atomics", from: "1.1.0") ], targets: [ - .target( - name: "CSystem", - dependencies: []), + .systemLibrary( + name: "CSystem"), .target( name: "SystemPackage", dependencies: [ diff --git a/Sources/CSystem/include/CSystemLinux.h b/Sources/CSystem/CSystemLinux.h similarity index 100% rename from Sources/CSystem/include/CSystemLinux.h rename to Sources/CSystem/CSystemLinux.h diff --git a/Sources/CSystem/include/CSystemWindows.h b/Sources/CSystem/CSystemWindows.h similarity index 100% rename from Sources/CSystem/include/CSystemWindows.h rename to Sources/CSystem/CSystemWindows.h diff --git a/Sources/CSystem/include/io_uring.h b/Sources/CSystem/io_uring.h similarity index 94% rename from Sources/CSystem/include/io_uring.h rename to Sources/CSystem/io_uring.h index 9e3d9fb3..5c05ed8b 100644 --- a/Sources/CSystem/include/io_uring.h +++ b/Sources/CSystem/io_uring.h @@ -5,6 +5,9 @@ #include #include +#ifndef SWIFT_IORING_C_WRAPPER +#define SWIFT_IORING_C_WRAPPER + #ifdef __alpha__ /* * alpha is the only exception, all other architectures @@ -48,3 +51,5 @@ int io_uring_enter(int fd, unsigned int to_submit, unsigned int min_complete, return syscall(__NR_io_uring_enter, fd, to_submit, min_complete, flags, sig, _NSIG / 8); } + +#endif diff --git a/Sources/CSystem/include/module.modulemap b/Sources/CSystem/module.modulemap similarity index 100% rename from Sources/CSystem/include/module.modulemap rename to Sources/CSystem/module.modulemap diff --git a/Sources/CSystem/shims.c b/Sources/CSystem/shims.c deleted file mode 100644 index f492a2ae..00000000 --- a/Sources/CSystem/shims.c +++ /dev/null @@ -1,18 +0,0 @@ -/* - This source file is part of the Swift System open source project - - Copyright (c) 2020 Apple Inc. and the Swift System project authors - Licensed under Apache License v2.0 with Runtime Library Exception - - See https://swift.org/LICENSE.txt for license information -*/ - -#ifdef __linux__ - -#include - -#endif - -#if defined(_WIN32) -#include -#endif From d099546787499e97973db394abb3de388e7e5305 Mon Sep 17 00:00:00 2001 From: Lucy Satheesan Date: Fri, 11 Aug 2023 09:06:17 -0700 Subject: [PATCH 06/48] fix access control --- Sources/System/AsyncFileDescriptor.swift | 14 +++++----- Sources/System/IORing.swift | 34 +++++++++--------------- Sources/System/ManagedIORing.swift | 2 +- 3 files changed, 22 insertions(+), 28 deletions(-) diff --git a/Sources/System/AsyncFileDescriptor.swift b/Sources/System/AsyncFileDescriptor.swift index 6e39e055..2143847f 100644 --- a/Sources/System/AsyncFileDescriptor.swift +++ b/Sources/System/AsyncFileDescriptor.swift @@ -1,12 +1,13 @@ @_implementationOnly import CSystem -public class AsyncFileDescriptor { - var open: Bool = true + +public final class AsyncFileDescriptor { + @usableFromInline var open: Bool = true @usableFromInline let fileSlot: IORingFileSlot @usableFromInline let ring: ManagedIORing - static func openat( - atDirectory: FileDescriptor = FileDescriptor(rawValue: AT_FDCWD), + public static func openat( + atDirectory: FileDescriptor = FileDescriptor(rawValue: -100), path: FilePath, _ mode: FileDescriptor.AccessMode, options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), @@ -25,7 +26,8 @@ public class AsyncFileDescriptor { path: cstr, mode, options: options, - permissions: permissions, intoSlot: fileSlot + permissions: permissions, + intoSlot: fileSlot )) if res.result < 0 { throw Errno(rawValue: -res.result) @@ -47,7 +49,7 @@ public class AsyncFileDescriptor { } @inlinable @inline(__always) @_unsafeInheritExecutor - func read( + public func read( into buffer: IORequest.Buffer, atAbsoluteOffset offset: UInt64 = UInt64.max ) async throws -> UInt32 { diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 0c2f1384..f91dfc04 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -262,7 +262,7 @@ public final class IORing: @unchecked Sendable { self.ringFlags = params.flags } - func blockingConsumeCompletion() -> IOCompletion { + public func blockingConsumeCompletion() -> IOCompletion { self.completionMutex.lock() defer { self.completionMutex.unlock() } @@ -292,7 +292,7 @@ public final class IORing: @unchecked Sendable { } } - func tryConsumeCompletion() -> IOCompletion? { + public func tryConsumeCompletion() -> IOCompletion? { self.completionMutex.lock() defer { self.completionMutex.unlock() } return _tryConsumeCompletion() @@ -312,8 +312,7 @@ public final class IORing: @unchecked Sendable { return nil } - - func registerFiles(count: UInt32) { + public func registerFiles(count: UInt32) { guard self.registeredFiles == nil else { fatalError() } let fileBuf = UnsafeMutableBufferPointer.allocate(capacity: Int(count)) fileBuf.initialize(repeating: UInt32.max) @@ -327,16 +326,15 @@ public final class IORing: @unchecked Sendable { self.registeredFiles = ResourceManager(fileBuf) } - func unregisterFiles() { + public func unregisterFiles() { fatalError("failed to unregister files") } - func getFile() -> IORingFileSlot? { + public func getFile() -> IORingFileSlot? { return self.registeredFiles?.getResource() } - // register a group of buffers - func registerBuffers(bufSize: UInt32, count: UInt32) { + public func registerBuffers(bufSize: UInt32, count: UInt32) { let iovecs = UnsafeMutableBufferPointer.allocate(capacity: Int(count)) let intBufSize = Int(bufSize) for i in 0.. IORingBuffer? { + public func getBuffer() -> IORingBuffer? { return self.registeredBuffers?.getResource() } - func unregisterBuffers() { + public func unregisterBuffers() { fatalError("failed to unregister buffers: TODO") } - // TODO: types - func submitRequests() { + public func submitRequests() { self.submissionMutex.lock() defer { self.submissionMutex.unlock() } self._submitRequests() } - func _submitRequests() { + internal func _submitRequests() { let flushedEvents = _flushQueue() // Ring always needs enter right now; @@ -389,6 +386,8 @@ public final class IORing: @unchecked Sendable { fatalError("fatal error in submitting requests: " + Errno(rawValue: -ret).debugDescription ) + } else { + break } } } @@ -403,14 +402,7 @@ public final class IORing: @unchecked Sendable { @inlinable @inline(__always) - func writeRequest(_ request: __owned IORequest) -> Bool { - self.submissionMutex.lock() - defer { self.submissionMutex.unlock() } - return _writeRequest(request.makeRawRequest()) - } - - @inlinable @inline(__always) - func writeAndSubmit(_ request: __owned IORequest) -> Bool { + public func writeRequest(_ request: __owned IORequest) -> Bool { self.submissionMutex.lock() defer { self.submissionMutex.unlock() } return _writeRequest(request.makeRawRequest()) diff --git a/Sources/System/ManagedIORing.swift b/Sources/System/ManagedIORing.swift index 580eacc1..7f5bec22 100644 --- a/Sources/System/ManagedIORing.swift +++ b/Sources/System/ManagedIORing.swift @@ -1,7 +1,7 @@ final public class ManagedIORing: @unchecked Sendable { var internalRing: IORing - init(queueDepth: UInt32) throws { + public init(queueDepth: UInt32) throws { self.internalRing = try IORing(queueDepth: queueDepth) self.internalRing.registerBuffers(bufSize: 655336, count: 4) self.internalRing.registerFiles(count: 32) From b58450413e04141a7fa4e32b7cb68b3fa90809e1 Mon Sep 17 00:00:00 2001 From: Lucy Satheesan Date: Fri, 11 Aug 2023 09:07:08 -0700 Subject: [PATCH 07/48] fix off-by-one in IORequest.openat --- Sources/System/IORequest.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Sources/System/IORequest.swift b/Sources/System/IORequest.swift index 6b26a4d1..39d7cdb7 100644 --- a/Sources/System/IORequest.swift +++ b/Sources/System/IORequest.swift @@ -46,7 +46,9 @@ extension IORequest { request.rawValue.addr = unsafeBitCast(path, to: UInt64.self) request.rawValue.open_flags = UInt32(bitPattern: options.rawValue | mode.rawValue) request.rawValue.len = permissions?.rawValue ?? 0 - request.rawValue.file_index = UInt32(slot?.index ?? 0) + if let fileSlot = slot { + request.rawValue.file_index = UInt32(fileSlot.index + 1) + } case .read(let file, let buffer, let offset), .write(let file, let buffer, let offset): if case .read = self { if case .registered = buffer { From b596783bd9a5e9d34f99e06b3624c0bf9e243c7d Mon Sep 17 00:00:00 2001 From: Lucy Satheesan Date: Fri, 11 Aug 2023 09:07:47 -0700 Subject: [PATCH 08/48] implement closing --- Sources/System/AsyncFileDescriptor.swift | 12 +++++++++--- Sources/System/IORequest.swift | 9 +++++++++ Sources/System/RawIORequest.swift | 6 +++++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/Sources/System/AsyncFileDescriptor.swift b/Sources/System/AsyncFileDescriptor.swift index 2143847f..504aec9f 100644 --- a/Sources/System/AsyncFileDescriptor.swift +++ b/Sources/System/AsyncFileDescriptor.swift @@ -43,9 +43,15 @@ public final class AsyncFileDescriptor { self.ring = ring } - func close() async throws { + @inlinable @inline(__always) @_unsafeInheritExecutor + public func close() async throws { + let res = await ring.submitAndWait(.close( + .registered(self.fileSlot) + )) + if res.result < 0 { + throw Errno(rawValue: -res.result) + } self.open = false - fatalError() } @inlinable @inline(__always) @_unsafeInheritExecutor @@ -67,7 +73,7 @@ public final class AsyncFileDescriptor { deinit { if (self.open) { - // TODO: close + // TODO: close or error? TBD } } } diff --git a/Sources/System/IORequest.swift b/Sources/System/IORequest.swift index 39d7cdb7..9548b3fb 100644 --- a/Sources/System/IORequest.swift +++ b/Sources/System/IORequest.swift @@ -20,6 +20,7 @@ public enum IORequest { buffer: Buffer, offset: UInt64 = 0 ) + case close(File) public enum Buffer { case registered(IORingBuffer) @@ -78,6 +79,14 @@ extension IORequest { request.buffer = buf } request.offset = offset + case .close(let file): + request.operation = .close + switch file { + case .registered(let regFile): + request.rawValue.file_index = UInt32(regFile.index + 1) + case .unregistered(let normalFile): + request.fileDescriptor = normalFile + } } return request } diff --git a/Sources/System/RawIORequest.swift b/Sources/System/RawIORequest.swift index fa50b439..520cb85c 100644 --- a/Sources/System/RawIORequest.swift +++ b/Sources/System/RawIORequest.swift @@ -25,10 +25,14 @@ extension RawIORequest { case receiveMessage = 10 // ... case openAt = 18 + case close = 19 + case filesUpdate = 20 + case statx = 21 case read = 22 case write = 23 + // ... case openAt2 = 28 - + // ... } public struct Flags: OptionSet, Hashable, Codable { From 6b4084c69f8a691976d7fe4deeaa80e099649356 Mon Sep 17 00:00:00 2001 From: Lucy Satheesan Date: Fri, 11 Aug 2023 09:08:43 -0700 Subject: [PATCH 09/48] introduce IORing unit tests --- Tests/LinuxMain.swift | 8 -- .../AsyncFileDescriptorTests.swift | 40 ++++++ Tests/SystemTests/IORequestTests.swift | 68 +++++++++ Tests/SystemTests/IORingTests.swift | 21 +++ Tests/SystemTests/ManagedIORingTests.swift | 19 +++ Tests/SystemTests/XCTestManifests.swift | 132 ------------------ 6 files changed, 148 insertions(+), 140 deletions(-) delete mode 100644 Tests/LinuxMain.swift create mode 100644 Tests/SystemTests/AsyncFileDescriptorTests.swift create mode 100644 Tests/SystemTests/IORequestTests.swift create mode 100644 Tests/SystemTests/IORingTests.swift create mode 100644 Tests/SystemTests/ManagedIORingTests.swift delete mode 100644 Tests/SystemTests/XCTestManifests.swift diff --git a/Tests/LinuxMain.swift b/Tests/LinuxMain.swift deleted file mode 100644 index 695f4e5b..00000000 --- a/Tests/LinuxMain.swift +++ /dev/null @@ -1,8 +0,0 @@ -import XCTest - -import SystemTests - -var tests = [XCTestCaseEntry]() -tests += SystemTests.__allTests() - -XCTMain(tests) diff --git a/Tests/SystemTests/AsyncFileDescriptorTests.swift b/Tests/SystemTests/AsyncFileDescriptorTests.swift new file mode 100644 index 00000000..0f1c103c --- /dev/null +++ b/Tests/SystemTests/AsyncFileDescriptorTests.swift @@ -0,0 +1,40 @@ +import XCTest + +#if SYSTEM_PACKAGE +import SystemPackage +#else +import System +#endif + +final class AsyncFileDescriptorTests: XCTestCase { + func testOpen() async throws { + let ring = try ManagedIORing(queueDepth: 32) + let file = try await AsyncFileDescriptor.openat( + path: "/dev/zero", + .readOnly, + onRing: ring + ) + } + + func testOpenClose() async throws { + let ring = try ManagedIORing(queueDepth: 32) + let file = try await AsyncFileDescriptor.openat( + path: "/dev/zero", + .readOnly, + onRing: ring + ) + await try file.close() + } + + func testDevNullEmpty() async throws { + let ring = try ManagedIORing(queueDepth: 32) + let file = try await AsyncFileDescriptor.openat( + path: "/dev/null", + .readOnly, + onRing: ring + ) + for try await _ in file { + XCTFail("/dev/null should be empty") + } + } +} diff --git a/Tests/SystemTests/IORequestTests.swift b/Tests/SystemTests/IORequestTests.swift new file mode 100644 index 00000000..a44a607e --- /dev/null +++ b/Tests/SystemTests/IORequestTests.swift @@ -0,0 +1,68 @@ +import XCTest + +#if SYSTEM_PACKAGE +@testable import SystemPackage +#else +import System +#endif + +func requestBytes(_ request: RawIORequest) -> [UInt8] { + return withUnsafePointer(to: request) { + let requestBuf = UnsafeBufferPointer(start: $0, count: 1) + let rawBytes = UnsafeRawBufferPointer(requestBuf) + return .init(rawBytes) + } +} + +// This test suite compares various IORequests bit-for-bit to IORequests +// that were generated with liburing or manually written out, +// which are known to work correctly. +final class IORequestTests: XCTestCase { + func testNop() { + let req = IORequest.nop.makeRawRequest() + let sourceBytes = requestBytes(req) + // convenient property of nop: it's all zeros! + // for some unknown reason, liburing sets the fd field to -1. + // we're not trying to be bug-compatible with it, so 0 *should* work. + XCTAssertEqual(sourceBytes, .init(repeating: 0, count: 64)) + } + + func testOpenatFixedFile() throws { + // TODO: come up with a better way of getting a FileSlot. + let buf = UnsafeMutableBufferPointer.allocate(capacity: 2) + let resmgr = ResourceManager.init(buf) + + let pathPtr = UnsafePointer(bitPattern: 0x414141410badf00d)! + let fileSlot = resmgr.getResource()! + let req = IORequest.openat( + atDirectory: FileDescriptor(rawValue: -100), + path: pathPtr, + .readOnly, + options: [], + permissions: nil, + intoSlot: fileSlot + ) + + let expectedRequest: [UInt8] = { + var bin = [UInt8].init(repeating: 0, count: 64) + bin[0] = 0x12 // opcode for the request + // bin[1] = 0 - no request flags + // bin[2...3] = 0 - padding + bin[4...7] = [0x9c, 0xff, 0xff, 0xff] // -100 in UInt32 - dirfd + // bin[8...15] = 0 - zeroes + withUnsafeBytes(of: pathPtr) { + // path pointer + bin[16...23] = ArraySlice($0) + } + // bin[24...43] = 0 - zeroes + withUnsafeBytes(of: UInt32(fileSlot.index + 1)) { + // file index + 1 - yes, unfortunately + bin[44...47] = ArraySlice($0) + } + return bin + }() + + let actualRequest = requestBytes(req.makeRawRequest()) + XCTAssertEqual(expectedRequest, actualRequest) + } +} diff --git a/Tests/SystemTests/IORingTests.swift b/Tests/SystemTests/IORingTests.swift new file mode 100644 index 00000000..78baa984 --- /dev/null +++ b/Tests/SystemTests/IORingTests.swift @@ -0,0 +1,21 @@ +import XCTest + +#if SYSTEM_PACKAGE +import SystemPackage +#else +import System +#endif + +final class IORingTests: XCTestCase { + func testInit() throws { + let ring = try IORing(queueDepth: 32) + } + + func testNop() throws { + let ring = try IORing(queueDepth: 32) + ring.writeRequest(.nop) + ring.submitRequests() + let completion = ring.blockingConsumeCompletion() + XCTAssertEqual(completion.result, 0) + } +} diff --git a/Tests/SystemTests/ManagedIORingTests.swift b/Tests/SystemTests/ManagedIORingTests.swift new file mode 100644 index 00000000..e7ad3f59 --- /dev/null +++ b/Tests/SystemTests/ManagedIORingTests.swift @@ -0,0 +1,19 @@ +import XCTest + +#if SYSTEM_PACKAGE +import SystemPackage +#else +import System +#endif + +final class ManagedIORingTests: XCTestCase { + func testInit() throws { + let ring = try ManagedIORing(queueDepth: 32) + } + + func testNop() async throws { + let ring = try ManagedIORing(queueDepth: 32) + let completion = await ring.submitAndWait(.nop) + XCTAssertEqual(completion.result, 0) + } +} diff --git a/Tests/SystemTests/XCTestManifests.swift b/Tests/SystemTests/XCTestManifests.swift deleted file mode 100644 index de99bd81..00000000 --- a/Tests/SystemTests/XCTestManifests.swift +++ /dev/null @@ -1,132 +0,0 @@ -#if !canImport(ObjectiveC) && swift(<5.5) -import XCTest - -extension ErrnoTest { - // DO NOT MODIFY: This is autogenerated, use: - // `swift test --generate-linuxmain` - // to regenerate. - static let __allTests__ErrnoTest = [ - ("testConstants", testConstants), - ("testPatternMatching", testPatternMatching), - ] -} - -extension FileDescriptorTest { - // DO NOT MODIFY: This is autogenerated, use: - // `swift test --generate-linuxmain` - // to regenerate. - static let __allTests__FileDescriptorTest = [ - ("testConstants", testConstants), - ("testStandardDescriptors", testStandardDescriptors), - ] -} - -extension FileOperationsTest { - // DO NOT MODIFY: This is autogenerated, use: - // `swift test --generate-linuxmain` - // to regenerate. - static let __allTests__FileOperationsTest = [ - ("testAdHocOpen", testAdHocOpen), - ("testAdHocPipe", testAdHocPipe), - ("testGithubIssues", testGithubIssues), - ("testHelpers", testHelpers), - ("testSyscalls", testSyscalls), - ] -} - -extension FilePathComponentsTest { - // DO NOT MODIFY: This is autogenerated, use: - // `swift test --generate-linuxmain` - // to regenerate. - static let __allTests__FilePathComponentsTest = [ - ("testAdHocRRC", testAdHocRRC), - ("testCases", testCases), - ("testSeparatorNormalization", testSeparatorNormalization), - ] -} - -extension FilePathParsingTest { - // DO NOT MODIFY: This is autogenerated, use: - // `swift test --generate-linuxmain` - // to regenerate. - static let __allTests__FilePathParsingTest = [ - ("testNormalization", testNormalization), - ] -} - -extension FilePathSyntaxTest { - // DO NOT MODIFY: This is autogenerated, use: - // `swift test --generate-linuxmain` - // to regenerate. - static let __allTests__FilePathSyntaxTest = [ - ("testAdHocMutations", testAdHocMutations), - ("testFailableStringInitializers", testFailableStringInitializers), - ("testLexicallyRelative", testLexicallyRelative), - ("testPartialWindowsRoots", testPartialWindowsRoots), - ("testPathSyntax", testPathSyntax), - ("testPrefixSuffix", testPrefixSuffix), - ] -} - -extension FilePathTest { - // DO NOT MODIFY: This is autogenerated, use: - // `swift test --generate-linuxmain` - // to regenerate. - static let __allTests__FilePathTest = [ - ("testFilePath", testFilePath), - ] -} - -extension FilePermissionsTest { - // DO NOT MODIFY: This is autogenerated, use: - // `swift test --generate-linuxmain` - // to regenerate. - static let __allTests__FilePermissionsTest = [ - ("testPermissions", testPermissions), - ] -} - -extension MockingTest { - // DO NOT MODIFY: This is autogenerated, use: - // `swift test --generate-linuxmain` - // to regenerate. - static let __allTests__MockingTest = [ - ("testMocking", testMocking), - ] -} - -extension SystemCharTest { - // DO NOT MODIFY: This is autogenerated, use: - // `swift test --generate-linuxmain` - // to regenerate. - static let __allTests__SystemCharTest = [ - ("testIsLetter", testIsLetter), - ] -} - -extension SystemStringTest { - // DO NOT MODIFY: This is autogenerated, use: - // `swift test --generate-linuxmain` - // to regenerate. - static let __allTests__SystemStringTest = [ - ("testAdHoc", testAdHoc), - ("testPlatformString", testPlatformString), - ] -} - -public func __allTests() -> [XCTestCaseEntry] { - return [ - testCase(ErrnoTest.__allTests__ErrnoTest), - testCase(FileDescriptorTest.__allTests__FileDescriptorTest), - testCase(FileOperationsTest.__allTests__FileOperationsTest), - testCase(FilePathComponentsTest.__allTests__FilePathComponentsTest), - testCase(FilePathParsingTest.__allTests__FilePathParsingTest), - testCase(FilePathSyntaxTest.__allTests__FilePathSyntaxTest), - testCase(FilePathTest.__allTests__FilePathTest), - testCase(FilePermissionsTest.__allTests__FilePermissionsTest), - testCase(MockingTest.__allTests__MockingTest), - testCase(SystemCharTest.__allTests__SystemCharTest), - testCase(SystemStringTest.__allTests__SystemStringTest), - ] -} -#endif From baab9b26b77bca2622e7f3741ceddc62a51e00fe Mon Sep 17 00:00:00 2001 From: David Smith Date: Thu, 24 Oct 2024 21:21:07 +0000 Subject: [PATCH 10/48] Starting to move to noncopyable structs, and away from swift-atomics --- Sources/System/IORing.swift | 72 ++++++++++++++--------------- Tests/SystemTests/IORingTests.swift | 2 +- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index f91dfc04..0ac783af 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -1,7 +1,7 @@ @_implementationOnly import CSystem import struct CSystem.io_uring_sqe -@_implementationOnly import Atomics +@_implementationOnly import Synchronization import Glibc // needed for mmap // XXX: this *really* shouldn't be here. oh well. @@ -12,9 +12,9 @@ extension UnsafeMutableRawPointer { } // all pointers in this struct reference kernel-visible memory -@usableFromInline struct SQRing { - let kernelHead: UnsafeAtomic - let kernelTail: UnsafeAtomic +@usableFromInline struct SQRing: ~Copyable { + let kernelHead: UnsafePointer> + let kernelTail: UnsafePointer> var userTail: UInt32 // from liburing: the kernel should never change these @@ -25,7 +25,7 @@ extension UnsafeMutableRawPointer { // ring flags bitfield // currently used by the kernel only in SQPOLL mode to indicate // when the polling thread needs to be woken up - let flags: UnsafeAtomic + let flags: UnsafePointer> // ring array // maps indexes between the actual ring and the submissionQueueEntries list, @@ -34,9 +34,9 @@ extension UnsafeMutableRawPointer { let array: UnsafeMutableBufferPointer } -struct CQRing { - let kernelHead: UnsafeAtomic - let kernelTail: UnsafeAtomic +struct CQRing: ~Copyable { + let kernelHead: UnsafePointer> + let kernelTail: UnsafePointer> // TODO: determine if this is actually used var userHead: UInt32 @@ -124,7 +124,7 @@ extension IORingBuffer { // XXX: This should be a non-copyable type (?) // demo only runs on Swift 5.8.1 -public final class IORing: @unchecked Sendable { +public struct IORing: @unchecked Sendable, ~Copyable { let ringFlags: UInt32 let ringDescriptor: Int32 @@ -187,20 +187,20 @@ public final class IORing: @unchecked Sendable { } submissionRing = SQRing( - kernelHead: UnsafeAtomic( - at: ringPtr.advanced(by: params.sq_off.head) - .assumingMemoryBound(to: UInt32.AtomicRepresentation.self) + kernelHead: UnsafePointer>( + ringPtr.advanced(by: params.sq_off.head) + .assumingMemoryBound(to: Atomic.self) ), - kernelTail: UnsafeAtomic( - at: ringPtr.advanced(by: params.sq_off.tail) - .assumingMemoryBound(to: UInt32.AtomicRepresentation.self) + kernelTail: UnsafePointer>( + ringPtr.advanced(by: params.sq_off.tail) + .assumingMemoryBound(to: Atomic.self) ), userTail: 0, // no requests yet ringMask: ringPtr.advanced(by: params.sq_off.ring_mask) .assumingMemoryBound(to: UInt32.self).pointee, - flags: UnsafeAtomic( - at: ringPtr.advanced(by: params.sq_off.flags) - .assumingMemoryBound(to: UInt32.AtomicRepresentation.self) + flags: UnsafePointer>( + ringPtr.advanced(by: params.sq_off.flags) + .assumingMemoryBound(to: Atomic.self) ), array: UnsafeMutableBufferPointer( start: ringPtr.advanced(by: params.sq_off.array) @@ -237,13 +237,13 @@ public final class IORing: @unchecked Sendable { ) completionRing = CQRing( - kernelHead: UnsafeAtomic( - at: ringPtr.advanced(by: params.cq_off.head) - .assumingMemoryBound(to: UInt32.AtomicRepresentation.self) + kernelHead: UnsafePointer>( + ringPtr.advanced(by: params.cq_off.head) + .assumingMemoryBound(to: Atomic.self) ), - kernelTail: UnsafeAtomic( - at: ringPtr.advanced(by: params.cq_off.tail) - .assumingMemoryBound(to: UInt32.AtomicRepresentation.self) + kernelTail: UnsafePointer>( + ringPtr.advanced(by: params.cq_off.tail) + .assumingMemoryBound(to: Atomic.self) ), userHead: 0, // no completions yet ringMask: ringPtr.advanced(by: params.cq_off.ring_mask) @@ -299,20 +299,20 @@ public final class IORing: @unchecked Sendable { } func _tryConsumeCompletion() -> IOCompletion? { - let tail = completionRing.kernelTail.load(ordering: .acquiring) - let head = completionRing.kernelHead.load(ordering: .relaxed) + let tail = completionRing.kernelTail.pointee.load(ordering: .acquiring) + let head = completionRing.kernelHead.pointee.load(ordering: .relaxed) if tail != head { // 32 byte copy - oh well let res = completionRing.cqes[Int(head & completionRing.ringMask)] - completionRing.kernelHead.store(head + 1, ordering: .relaxed) + completionRing.kernelHead.pointee.store(head + 1, ordering: .relaxed) return IOCompletion(rawValue: res) } return nil } - public func registerFiles(count: UInt32) { + public mutating func registerFiles(count: UInt32) { guard self.registeredFiles == nil else { fatalError() } let fileBuf = UnsafeMutableBufferPointer.allocate(capacity: Int(count)) fileBuf.initialize(repeating: UInt32.max) @@ -334,7 +334,7 @@ public final class IORing: @unchecked Sendable { return self.registeredFiles?.getResource() } - public func registerBuffers(bufSize: UInt32, count: UInt32) { + public mutating func registerBuffers(bufSize: UInt32, count: UInt32) { let iovecs = UnsafeMutableBufferPointer.allocate(capacity: Int(count)) let intBufSize = Int(bufSize) for i in 0.. UInt32 { - self.submissionRing.kernelTail.store( + self.submissionRing.kernelTail.pointee.store( self.submissionRing.userTail, ordering: .relaxed ) return self.submissionRing.userTail - - self.submissionRing.kernelHead.load(ordering: .relaxed) + self.submissionRing.kernelHead.pointee.load(ordering: .relaxed) } @inlinable @inline(__always) - public func writeRequest(_ request: __owned IORequest) -> Bool { + public mutating func writeRequest(_ request: __owned IORequest) -> Bool { self.submissionMutex.lock() defer { self.submissionMutex.unlock() } return _writeRequest(request.makeRawRequest()) } @inlinable @inline(__always) - internal func _writeRequest(_ request: __owned RawIORequest) -> Bool { + internal mutating func _writeRequest(_ request: __owned RawIORequest) -> Bool { let entry = _blockingGetSubmissionEntry() entry.pointee = request.rawValue return true } @inlinable @inline(__always) - internal func _blockingGetSubmissionEntry() -> UnsafeMutablePointer { + internal mutating func _blockingGetSubmissionEntry() -> UnsafeMutablePointer { while true { if let entry = _getSubmissionEntry() { return entry @@ -427,11 +427,11 @@ public final class IORing: @unchecked Sendable { } @usableFromInline @inline(__always) - internal func _getSubmissionEntry() -> UnsafeMutablePointer? { + internal mutating func _getSubmissionEntry() -> UnsafeMutablePointer? { let next = self.submissionRing.userTail + 1 // FEAT: smp load when SQPOLL in use (not in MVP) - let kernelHead = self.submissionRing.kernelHead.load(ordering: .relaxed) + let kernelHead = self.submissionRing.kernelHead.pointee.load(ordering: .relaxed) // FEAT: 128-bit event support (not in MVP) if (next - kernelHead <= self.submissionRing.array.count) { diff --git a/Tests/SystemTests/IORingTests.swift b/Tests/SystemTests/IORingTests.swift index 78baa984..4604f500 100644 --- a/Tests/SystemTests/IORingTests.swift +++ b/Tests/SystemTests/IORingTests.swift @@ -12,7 +12,7 @@ final class IORingTests: XCTestCase { } func testNop() throws { - let ring = try IORing(queueDepth: 32) + var ring = try IORing(queueDepth: 32) ring.writeRequest(.nop) ring.submitRequests() let completion = ring.blockingConsumeCompletion() From 60299362ed85e53915891a562b60d621afc608df Mon Sep 17 00:00:00 2001 From: David Smith Date: Thu, 24 Oct 2024 21:27:22 +0000 Subject: [PATCH 11/48] One more noncopyable struct --- Sources/System/IORing.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 0ac783af..656e1683 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -79,7 +79,7 @@ internal class ResourceManager: @unchecked Sendable { } } -public class IOResource { +public struct IOResource: ~Copyable { typealias Resource = T @usableFromInline let resource: T @usableFromInline let index: Int From 10c070abd648bb66da947c77b5a5c645ef1d53fd Mon Sep 17 00:00:00 2001 From: David Smith Date: Mon, 28 Oct 2024 20:00:07 +0000 Subject: [PATCH 12/48] WIP, more ~Copyable adoption --- Sources/System/IORequest.swift | 145 ++++--- Sources/System/IORing.swift | 368 +++++++++--------- Sources/System/Lock.swift | 37 -- Sources/System/ManagedIORing.swift | 25 +- .../AsyncFileDescriptorTests.swift | 2 +- 5 files changed, 295 insertions(+), 282 deletions(-) delete mode 100644 Sources/System/Lock.swift diff --git a/Sources/System/IORequest.swift b/Sources/System/IORequest.swift index 9548b3fb..2ecd3f46 100644 --- a/Sources/System/IORequest.swift +++ b/Sources/System/IORequest.swift @@ -1,9 +1,9 @@ import struct CSystem.io_uring_sqe -public enum IORequest { +public enum IORequest: ~Copyable { case nop // nothing here case openat( - atDirectory: FileDescriptor, + atDirectory: FileDescriptor, path: UnsafePointer, FileDescriptor.AccessMode, options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), @@ -22,71 +22,106 @@ public enum IORequest { ) case close(File) - public enum Buffer { + public enum Buffer: ~Copyable { case registered(IORingBuffer) case unregistered(UnsafeMutableRawBufferPointer) } - public enum File { + public enum File: ~Copyable { case registered(IORingFileSlot) case unregistered(FileDescriptor) } } +@inlinable @inline(__always) +internal func makeRawRequest_readWrite_registered( + file: consuming IORequest.File, + buffer: consuming IORingBuffer, + offset: UInt64, + request: consuming RawIORequest +) -> RawIORequest { + switch file { + case .registered(let regFile): + request.rawValue.fd = Int32(exactly: regFile.index)! + request.flags = .fixedFile + case .unregistered(let fd): + request.fileDescriptor = fd + } + request.buffer = buffer.unsafeBuffer + request.rawValue.buf_index = UInt16(exactly: buffer.index)! + request.offset = offset + return request +} + +@inlinable @inline(__always) +internal func makeRawRequest_readWrite_unregistered( + file: consuming IORequest.File, + buffer: UnsafeMutableRawBufferPointer, + offset: UInt64, + request: consuming RawIORequest +) -> RawIORequest { + switch file { + case .registered(let regFile): + request.rawValue.fd = Int32(exactly: regFile.index)! + request.flags = .fixedFile + case .unregistered(let fd): + request.fileDescriptor = fd + } + request.buffer = buffer + request.offset = offset + return request +} + extension IORequest { @inlinable @inline(__always) - public func makeRawRequest() -> RawIORequest { + public consuming func makeRawRequest() -> RawIORequest { var request = RawIORequest() - switch self { - case .nop: - request.operation = .nop - case .openat(let atDirectory, let path, let mode, let options, let permissions, let slot): - // TODO: use rawValue less - request.operation = .openAt - request.fileDescriptor = atDirectory - request.rawValue.addr = unsafeBitCast(path, to: UInt64.self) - request.rawValue.open_flags = UInt32(bitPattern: options.rawValue | mode.rawValue) - request.rawValue.len = permissions?.rawValue ?? 0 - if let fileSlot = slot { - request.rawValue.file_index = UInt32(fileSlot.index + 1) - } - case .read(let file, let buffer, let offset), .write(let file, let buffer, let offset): - if case .read = self { - if case .registered = buffer { - request.operation = .readFixed - } else { - request.operation = .read - } - } else { - if case .registered = buffer { - request.operation = .writeFixed - } else { - request.operation = .write - } - } - switch file { - case .registered(let regFile): - request.rawValue.fd = Int32(exactly: regFile.index)! - request.flags = .fixedFile - case .unregistered(let fd): - request.fileDescriptor = fd - } - switch buffer { - case .registered(let regBuf): - request.buffer = regBuf.unsafeBuffer - request.rawValue.buf_index = UInt16(exactly: regBuf.index)! - case .unregistered(let buf): - request.buffer = buf - } - request.offset = offset - case .close(let file): - request.operation = .close - switch file { - case .registered(let regFile): - request.rawValue.file_index = UInt32(regFile.index + 1) - case .unregistered(let normalFile): - request.fileDescriptor = normalFile - } + switch consume self { + case .nop: + request.operation = .nop + case .openat(let atDirectory, let path, let mode, let options, let permissions, let slot): + // TODO: use rawValue less + request.operation = .openAt + request.fileDescriptor = atDirectory + request.rawValue.addr = unsafeBitCast(path, to: UInt64.self) + request.rawValue.open_flags = UInt32(bitPattern: options.rawValue | mode.rawValue) + request.rawValue.len = permissions?.rawValue ?? 0 + if let fileSlot = slot { + request.rawValue.file_index = UInt32(fileSlot.index + 1) + } + case .write(let file, let buffer, let offset): + switch consume buffer { + case .registered(let buffer): + request.operation = .writeFixed + return makeRawRequest_readWrite_registered( + file: file, buffer: buffer, offset: offset, request: request) + + case .unregistered(let buffer): + request.operation = .write + return makeRawRequest_readWrite_unregistered( + file: file, buffer: buffer, offset: offset, request: request) + } + case .read(let file, let buffer, let offset): + + switch consume buffer { + case .registered(let buffer): + request.operation = .readFixed + return makeRawRequest_readWrite_registered( + file: file, buffer: buffer, offset: offset, request: request) + + case .unregistered(let buffer): + request.operation = .read + return makeRawRequest_readWrite_unregistered( + file: file, buffer: buffer, offset: offset, request: request) + } + case .close(let file): + request.operation = .close + switch file { + case .registered(let regFile): + request.rawValue.file_index = UInt32(regFile.index + 1) + case .unregistered(let normalFile): + request.fileDescriptor = normalFile + } } return request } diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 656e1683..3e24408f 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -1,8 +1,8 @@ @_implementationOnly import CSystem -import struct CSystem.io_uring_sqe +import Glibc // needed for mmap +import Synchronization -@_implementationOnly import Synchronization -import Glibc // needed for mmap +import struct CSystem.io_uring_sqe // XXX: this *really* shouldn't be here. oh well. extension UnsafeMutableRawPointer { @@ -26,7 +26,7 @@ extension UnsafeMutableRawPointer { // currently used by the kernel only in SQPOLL mode to indicate // when the polling thread needs to be woken up let flags: UnsafePointer> - + // ring array // maps indexes between the actual ring and the submissionQueueEntries list, // allowing the latter to be used as a kind of freelist with enough work? @@ -48,34 +48,40 @@ struct CQRing: ~Copyable { internal class ResourceManager: @unchecked Sendable { typealias Resource = T - let resourceList: UnsafeMutableBufferPointer - var freeList: [Int] - let mutex: Mutex + + struct Resources { + let resourceList: UnsafeMutableBufferPointer + var freeList: [Int] + } + + let mutex: Mutex init(_ res: UnsafeMutableBufferPointer) { - self.resourceList = res - self.freeList = [Int](resourceList.indices) - self.mutex = Mutex() + mutex = Mutex( + Resources( + resourceList: res, + freeList: [Int](res.indices) + )) } func getResource() -> IOResource? { - self.mutex.lock() - defer { self.mutex.unlock() } - if let index = freeList.popLast() { - return IOResource( - rescource: resourceList[index], - index: index, - manager: self - ) - } else { - return nil + mutex.withLock { resources in + if let index = resources.freeList.popLast() { + return IOResource( + resource: resources.resourceList[index], + index: index, + manager: self + ) + } else { + return nil + } } } func releaseResource(index: Int) { - self.mutex.lock() - defer { self.mutex.unlock() } - self.freeList.append(index) + mutex.withLock { resources in + resources.freeList.append(index) + } } } @@ -86,11 +92,11 @@ public struct IOResource: ~Copyable { let manager: ResourceManager internal init( - rescource: T, + resource: T, index: Int, manager: ResourceManager ) { - self.resource = rescource + self.resource = resource self.index = index self.manager = manager } @@ -114,13 +120,89 @@ extension IORingFileSlot { } extension IORingBuffer { public var unsafeBuffer: UnsafeMutableRawBufferPointer { - get { - return .init(start: resource.iov_base, count: resource.iov_len) + return .init(start: resource.iov_base, count: resource.iov_len) + } +} + +@inlinable @inline(__always) +internal func _writeRequest(_ request: __owned RawIORequest, ring: inout SQRing, submissionQueueEntries: UnsafeMutableBufferPointer) + -> Bool +{ + let entry = _blockingGetSubmissionEntry(ring: &ring, submissionQueueEntries: submissionQueueEntries) + entry.pointee = request.rawValue + return true +} + +@inlinable @inline(__always) +internal func _blockingGetSubmissionEntry(ring: inout SQRing, submissionQueueEntries: UnsafeMutableBufferPointer) -> UnsafeMutablePointer< + io_uring_sqe +> { + while true { + if let entry = _getSubmissionEntry(ring: &ring, submissionQueueEntries: submissionQueueEntries) { + return entry } + // TODO: actually block here instead of spinning } + +} + +internal func _submitRequests(ring: inout SQRing, ringDescriptor: Int32) { + let flushedEvents = _flushQueue(ring: &ring) + + // Ring always needs enter right now; + // TODO: support SQPOLL here + while true { + let ret = io_uring_enter(ringDescriptor, flushedEvents, 0, 0, nil) + // error handling: + // EAGAIN / EINTR (try again), + // EBADF / EBADFD / EOPNOTSUPP / ENXIO + // (failure in ring lifetime management, fatal), + // EINVAL (bad constant flag?, fatal), + // EFAULT (bad address for argument from library, fatal) + if ret == -EAGAIN || ret == -EINTR { + continue + } else if ret < 0 { + fatalError( + "fatal error in submitting requests: " + Errno(rawValue: -ret).debugDescription + ) + } else { + break + } + } +} + +internal func _flushQueue(ring: inout SQRing) -> UInt32 { + ring.kernelTail.pointee.store( + ring.userTail, ordering: .relaxed + ) + return ring.userTail - ring.kernelHead.pointee.load(ordering: .relaxed) } +@usableFromInline @inline(__always) +internal func _getSubmissionEntry(ring: inout SQRing, submissionQueueEntries: UnsafeMutableBufferPointer) -> UnsafeMutablePointer< + io_uring_sqe +>? { + let next = ring.userTail + 1 + // FEAT: smp load when SQPOLL in use (not in MVP) + let kernelHead = ring.kernelHead.pointee.load(ordering: .relaxed) + + // FEAT: 128-bit event support (not in MVP) + if next - kernelHead <= ring.array.count { + // let sqe = &sq->sqes[(sq->sqe_tail & sq->ring_mask) << shift]; + let sqeIndex = Int( + ring.userTail & ring.ringMask + ) + + let sqe = submissionQueueEntries + .baseAddress.unsafelyUnwrapped + .advanced(by: sqeIndex) + + ring.userTail = next + return sqe + } + return nil +} // XXX: This should be a non-copyable type (?) // demo only runs on Swift 5.8.1 @@ -128,15 +210,13 @@ public struct IORing: @unchecked Sendable, ~Copyable { let ringFlags: UInt32 let ringDescriptor: Int32 - @usableFromInline var submissionRing: SQRing - @usableFromInline var submissionMutex: Mutex + @usableFromInline let submissionMutex: Mutex // FEAT: set this eventually let submissionPolling: Bool = false - var completionRing: CQRing - var completionMutex: Mutex + let completionMutex: Mutex - let submissionQueueEntries: UnsafeMutableBufferPointer + @usableFromInline let submissionQueueEntries: UnsafeMutableBufferPointer // kept around for unmap / cleanup let ringSize: Int @@ -149,28 +229,31 @@ public struct IORing: @unchecked Sendable, ~Copyable { var params = io_uring_params() ringDescriptor = withUnsafeMutablePointer(to: ¶ms) { - return io_uring_setup(queueDepth, $0); + return io_uring_setup(queueDepth, $0) } - if (params.features & IORING_FEAT_SINGLE_MMAP == 0 - || params.features & IORING_FEAT_NODROP == 0) { + if params.features & IORING_FEAT_SINGLE_MMAP == 0 + || params.features & IORING_FEAT_NODROP == 0 + { close(ringDescriptor) // TODO: error handling throw IORingError.missingRequiredFeatures } - - if (ringDescriptor < 0) { + + if ringDescriptor < 0 { // TODO: error handling } - let submitRingSize = params.sq_off.array + let submitRingSize = + params.sq_off.array + params.sq_entries * UInt32(MemoryLayout.size) - - let completionRingSize = params.cq_off.cqes + + let completionRingSize = + params.cq_off.cqes + params.cq_entries * UInt32(MemoryLayout.size) ringSize = Int(max(submitRingSize, completionRingSize)) - + ringPtr = mmap( /* addr: */ nil, /* len: */ ringSize, @@ -178,35 +261,36 @@ public struct IORing: @unchecked Sendable, ~Copyable { /* flags: */ MAP_SHARED | MAP_POPULATE, /* fd: */ ringDescriptor, /* offset: */ __off_t(IORING_OFF_SQ_RING) - ); + ) - if (ringPtr == MAP_FAILED) { - perror("mmap"); + if ringPtr == MAP_FAILED { + perror("mmap") // TODO: error handling fatalError("mmap failed in ring setup") } - submissionRing = SQRing( + let submissionRing = SQRing( kernelHead: UnsafePointer>( ringPtr.advanced(by: params.sq_off.head) - .assumingMemoryBound(to: Atomic.self) + .assumingMemoryBound(to: Atomic.self) ), kernelTail: UnsafePointer>( ringPtr.advanced(by: params.sq_off.tail) - .assumingMemoryBound(to: Atomic.self) + .assumingMemoryBound(to: Atomic.self) ), - userTail: 0, // no requests yet + userTail: 0, // no requests yet ringMask: ringPtr.advanced(by: params.sq_off.ring_mask) .assumingMemoryBound(to: UInt32.self).pointee, flags: UnsafePointer>( ringPtr.advanced(by: params.sq_off.flags) - .assumingMemoryBound(to: Atomic.self) + .assumingMemoryBound(to: Atomic.self) ), array: UnsafeMutableBufferPointer( start: ringPtr.advanced(by: params.sq_off.array) .assumingMemoryBound(to: UInt32.self), - count: Int(ringPtr.advanced(by: params.sq_off.ring_entries) - .assumingMemoryBound(to: UInt32.self).pointee) + count: Int( + ringPtr.advanced(by: params.sq_off.ring_entries) + .assumingMemoryBound(to: UInt32.self).pointee) ) ) @@ -223,10 +307,10 @@ public struct IORing: @unchecked Sendable, ~Copyable { /* flags: */ MAP_SHARED | MAP_POPULATE, /* fd: */ ringDescriptor, /* offset: */ __off_t(IORING_OFF_SQES) - ); + ) - if (sqes == MAP_FAILED) { - perror("mmap"); + if sqes == MAP_FAILED { + perror("mmap") // TODO: error handling fatalError("sqe mmap failed in ring setup") } @@ -236,76 +320,77 @@ public struct IORing: @unchecked Sendable, ~Copyable { count: Int(params.sq_entries) ) - completionRing = CQRing( + let completionRing = CQRing( kernelHead: UnsafePointer>( ringPtr.advanced(by: params.cq_off.head) - .assumingMemoryBound(to: Atomic.self) + .assumingMemoryBound(to: Atomic.self) ), kernelTail: UnsafePointer>( ringPtr.advanced(by: params.cq_off.tail) - .assumingMemoryBound(to: Atomic.self) + .assumingMemoryBound(to: Atomic.self) ), - userHead: 0, // no completions yet + userHead: 0, // no completions yet ringMask: ringPtr.advanced(by: params.cq_off.ring_mask) .assumingMemoryBound(to: UInt32.self).pointee, cqes: UnsafeBufferPointer( start: ringPtr.advanced(by: params.cq_off.cqes) .assumingMemoryBound(to: io_uring_cqe.self), - count: Int(ringPtr.advanced(by: params.cq_off.ring_entries) - .assumingMemoryBound(to: UInt32.self).pointee) + count: Int( + ringPtr.advanced(by: params.cq_off.ring_entries) + .assumingMemoryBound(to: UInt32.self).pointee) ) ) - self.submissionMutex = Mutex() - self.completionMutex = Mutex() + self.submissionMutex = Mutex(submissionRing) + self.completionMutex = Mutex(completionRing) self.ringFlags = params.flags } public func blockingConsumeCompletion() -> IOCompletion { - self.completionMutex.lock() - defer { self.completionMutex.unlock() } - - if let completion = _tryConsumeCompletion() { - return completion - } else { - while true { - let res = io_uring_enter(ringDescriptor, 0, 1, IORING_ENTER_GETEVENTS, nil) - // error handling: - // EAGAIN / EINTR (try again), - // EBADF / EBADFD / EOPNOTSUPP / ENXIO - // (failure in ring lifetime management, fatal), - // EINVAL (bad constant flag?, fatal), - // EFAULT (bad address for argument from library, fatal) - // EBUSY (not enough space for events; implies events filled - // by kernel between kernelTail load and now) - if res >= 0 || res == -EBUSY { - break - } else if res == -EAGAIN || res == -EINTR { - continue + completionMutex.withLock { ring in + if let completion = _tryConsumeCompletion(ring: &ring) { + return completion + } else { + while true { + let res = io_uring_enter(ringDescriptor, 0, 1, IORING_ENTER_GETEVENTS, nil) + // error handling: + // EAGAIN / EINTR (try again), + // EBADF / EBADFD / EOPNOTSUPP / ENXIO + // (failure in ring lifetime management, fatal), + // EINVAL (bad constant flag?, fatal), + // EFAULT (bad address for argument from library, fatal) + // EBUSY (not enough space for events; implies events filled + // by kernel between kernelTail load and now) + if res >= 0 || res == -EBUSY { + break + } else if res == -EAGAIN || res == -EINTR { + continue + } + fatalError( + "fatal error in receiving requests: " + + Errno(rawValue: -res).debugDescription + ) } - fatalError("fatal error in receiving requests: " + - Errno(rawValue: -res).debugDescription - ) + return _tryConsumeCompletion(ring: &ring).unsafelyUnwrapped } - return _tryConsumeCompletion().unsafelyUnwrapped } } public func tryConsumeCompletion() -> IOCompletion? { - self.completionMutex.lock() - defer { self.completionMutex.unlock() } - return _tryConsumeCompletion() + completionMutex.withLock { ring in + return _tryConsumeCompletion(ring: &ring) + } } - func _tryConsumeCompletion() -> IOCompletion? { - let tail = completionRing.kernelTail.pointee.load(ordering: .acquiring) - let head = completionRing.kernelHead.pointee.load(ordering: .relaxed) - + func _tryConsumeCompletion(ring: inout CQRing) -> IOCompletion? { + let tail = ring.kernelTail.pointee.load(ordering: .acquiring) + let head = ring.kernelHead.pointee.load(ordering: .relaxed) + if tail != head { // 32 byte copy - oh well - let res = completionRing.cqes[Int(head & completionRing.ringMask)] - completionRing.kernelHead.pointee.store(head + 1, ordering: .relaxed) + let res = ring.cqes[Int(head & ring.ringMask)] + ring.kernelHead.pointee.store(head + 1, ordering: .relaxed) return IOCompletion(rawValue: res) } @@ -340,7 +425,8 @@ public struct IORing: @unchecked Sendable, ~Copyable { for i in 0.. UInt32 { - self.submissionRing.kernelTail.pointee.store( - self.submissionRing.userTail, ordering: .relaxed - ) - return self.submissionRing.userTail - - self.submissionRing.kernelHead.pointee.load(ordering: .relaxed) - } - - @inlinable @inline(__always) public mutating func writeRequest(_ request: __owned IORequest) -> Bool { - self.submissionMutex.lock() - defer { self.submissionMutex.unlock() } - return _writeRequest(request.makeRawRequest()) - } - - @inlinable @inline(__always) - internal mutating func _writeRequest(_ request: __owned RawIORequest) -> Bool { - let entry = _blockingGetSubmissionEntry() - entry.pointee = request.rawValue - return true - } - - @inlinable @inline(__always) - internal mutating func _blockingGetSubmissionEntry() -> UnsafeMutablePointer { - while true { - if let entry = _getSubmissionEntry() { - return entry - } - // TODO: actually block here instead of spinning + let raw = request.makeRawRequest() + return submissionMutex.withLock { ring in + return _writeRequest(raw, ring: &ring, submissionQueueEntries: submissionQueueEntries) } - - } - - @usableFromInline @inline(__always) - internal mutating func _getSubmissionEntry() -> UnsafeMutablePointer? { - let next = self.submissionRing.userTail + 1 - - // FEAT: smp load when SQPOLL in use (not in MVP) - let kernelHead = self.submissionRing.kernelHead.pointee.load(ordering: .relaxed) - - // FEAT: 128-bit event support (not in MVP) - if (next - kernelHead <= self.submissionRing.array.count) { - // let sqe = &sq->sqes[(sq->sqe_tail & sq->ring_mask) << shift]; - let sqeIndex = Int( - self.submissionRing.userTail & self.submissionRing.ringMask - ) - - let sqe = self.submissionQueueEntries - .baseAddress.unsafelyUnwrapped - .advanced(by: sqeIndex) - - self.submissionRing.userTail = next; - return sqe - } - return nil } deinit { - munmap(ringPtr, ringSize); + munmap(ringPtr, ringSize) munmap( UnsafeMutableRawPointer(submissionQueueEntries.baseAddress!), submissionQueueEntries.count * MemoryLayout.size ) close(ringDescriptor) } -}; - +} diff --git a/Sources/System/Lock.swift b/Sources/System/Lock.swift deleted file mode 100644 index fd20c641..00000000 --- a/Sources/System/Lock.swift +++ /dev/null @@ -1,37 +0,0 @@ -// TODO: write against kernel APIs directly? -import Glibc - -@usableFromInline final class Mutex { - @usableFromInline let mutex: UnsafeMutablePointer - - @inlinable init() { - self.mutex = UnsafeMutablePointer.allocate(capacity: 1) - self.mutex.initialize(to: pthread_mutex_t()) - pthread_mutex_init(self.mutex, nil) - } - - @inlinable deinit { - defer { mutex.deallocate() } - guard pthread_mutex_destroy(mutex) == 0 else { - preconditionFailure("unable to destroy mutex") - } - } - - // XXX: this is because we need to lock the mutex in the context of a submit() function - // and unlock *before* the UnsafeContinuation returns. - // Code looks like: { - // // prepare request - // io_uring_get_sqe() - // io_uring_prep_foo(...) - // return await withUnsafeContinuation { - // sqe->user_data = ...; io_uring_submit(); unlock(); - // } - // } - @inlinable @inline(__always) public func lock() { - pthread_mutex_lock(mutex) - } - - @inlinable @inline(__always) public func unlock() { - pthread_mutex_unlock(mutex) - } -} diff --git a/Sources/System/ManagedIORing.swift b/Sources/System/ManagedIORing.swift index 7f5bec22..88fb72e7 100644 --- a/Sources/System/ManagedIORing.swift +++ b/Sources/System/ManagedIORing.swift @@ -5,15 +5,16 @@ final public class ManagedIORing: @unchecked Sendable { self.internalRing = try IORing(queueDepth: queueDepth) self.internalRing.registerBuffers(bufSize: 655336, count: 4) self.internalRing.registerFiles(count: 32) - self.startWaiter() + self.startWaiter() } private func startWaiter() { Task.detached { - while (!Task.isCancelled) { + while !Task.isCancelled { let cqe = self.internalRing.blockingConsumeCompletion() - let cont = unsafeBitCast(cqe.userData, to: UnsafeContinuation.self) + let cont = unsafeBitCast( + cqe.userData, to: UnsafeContinuation.self) cont.resume(returning: cqe) } } @@ -21,14 +22,18 @@ final public class ManagedIORing: @unchecked Sendable { @_unsafeInheritExecutor public func submitAndWait(_ request: __owned IORequest) async -> IOCompletion { - self.internalRing.submissionMutex.lock() + var consumeOnceWorkaround: IORequest? = request return await withUnsafeContinuation { cont in - let entry = internalRing._blockingGetSubmissionEntry() - entry.pointee = request.makeRawRequest().rawValue - entry.pointee.user_data = unsafeBitCast(cont, to: UInt64.self) - self.internalRing._submitRequests() - self.internalRing.submissionMutex.unlock() + return internalRing.submissionMutex.withLock { ring in + let request = consumeOnceWorkaround.take()! + let entry = _blockingGetSubmissionEntry( + ring: &ring, submissionQueueEntries: internalRing.submissionQueueEntries) + entry.pointee = request.makeRawRequest().rawValue + entry.pointee.user_data = unsafeBitCast(cont, to: UInt64.self) + _submitRequests(ring: &ring, ringDescriptor: internalRing.ringDescriptor) + } } + } internal func getFileSlot() -> IORingFileSlot? { @@ -39,4 +44,4 @@ final public class ManagedIORing: @unchecked Sendable { self.internalRing.getBuffer() } -} \ No newline at end of file +} diff --git a/Tests/SystemTests/AsyncFileDescriptorTests.swift b/Tests/SystemTests/AsyncFileDescriptorTests.swift index 0f1c103c..1baba962 100644 --- a/Tests/SystemTests/AsyncFileDescriptorTests.swift +++ b/Tests/SystemTests/AsyncFileDescriptorTests.swift @@ -23,7 +23,7 @@ final class AsyncFileDescriptorTests: XCTestCase { .readOnly, onRing: ring ) - await try file.close() + try await file.close() } func testDevNullEmpty() async throws { From 32966f975d8d4285059c29eaccd1cee46c56b84f Mon Sep 17 00:00:00 2001 From: David Smith Date: Mon, 28 Oct 2024 23:43:59 +0000 Subject: [PATCH 13/48] It builds again! With some horrible hacks --- Sources/System/AsyncFileDescriptor.swift | 117 +++++++++++------- Sources/System/IOCompletion.swift | 1 + Sources/System/IORequest.swift | 58 ++++----- Sources/System/IORing.swift | 22 ++-- Sources/System/ManagedIORing.swift | 4 +- Sources/System/RawIORequest.swift | 2 +- .../AsyncFileDescriptorTests.swift | 2 +- Tests/SystemTests/IORequestTests.swift | 4 +- 8 files changed, 124 insertions(+), 86 deletions(-) diff --git a/Sources/System/AsyncFileDescriptor.swift b/Sources/System/AsyncFileDescriptor.swift index 504aec9f..a00f41de 100644 --- a/Sources/System/AsyncFileDescriptor.swift +++ b/Sources/System/AsyncFileDescriptor.swift @@ -1,13 +1,12 @@ @_implementationOnly import CSystem - -public final class AsyncFileDescriptor { +public struct AsyncFileDescriptor: ~Copyable { @usableFromInline var open: Bool = true @usableFromInline let fileSlot: IORingFileSlot @usableFromInline let ring: ManagedIORing - + public static func openat( - atDirectory: FileDescriptor = FileDescriptor(rawValue: -100), + atDirectory: FileDescriptor = FileDescriptor(rawValue: -100), path: FilePath, _ mode: FileDescriptor.AccessMode, options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), @@ -19,35 +18,37 @@ public final class AsyncFileDescriptor { throw IORingError.missingRequiredFeatures } let cstr = path.withCString { - return $0 // bad + return $0 // bad } - let res = await ring.submitAndWait(.openat( - atDirectory: atDirectory, - path: cstr, - mode, - options: options, - permissions: permissions, - intoSlot: fileSlot - )) + let res = await ring.submitAndWait( + .openat( + atDirectory: atDirectory, + path: cstr, + mode, + options: options, + permissions: permissions, + intoSlot: fileSlot.borrow() + )) if res.result < 0 { throw Errno(rawValue: -res.result) } - + return AsyncFileDescriptor( fileSlot, ring: ring ) } - internal init(_ fileSlot: IORingFileSlot, ring: ManagedIORing) { - self.fileSlot = fileSlot + internal init(_ fileSlot: consuming IORingFileSlot, ring: ManagedIORing) { + self.fileSlot = consume fileSlot self.ring = ring } @inlinable @inline(__always) @_unsafeInheritExecutor - public func close() async throws { - let res = await ring.submitAndWait(.close( - .registered(self.fileSlot) - )) + public consuming func close() async throws { + let res = await ring.submitAndWait( + .close( + .registered(self.fileSlot) + )) if res.result < 0 { throw Errno(rawValue: -res.result) } @@ -56,14 +57,16 @@ public final class AsyncFileDescriptor { @inlinable @inline(__always) @_unsafeInheritExecutor public func read( - into buffer: IORequest.Buffer, + into buffer: inout UnsafeMutableRawBufferPointer, atAbsoluteOffset offset: UInt64 = UInt64.max ) async throws -> UInt32 { - let res = await ring.submitAndWait(.read( - file: .registered(self.fileSlot), - buffer: buffer, - offset: offset - )) + let file = fileSlot.borrow() + let res = await ring.submitAndWait( + .readUnregistered( + file: .registered(file), + buffer: buffer, + offset: offset + )) if res.result < 0 { throw Errno(rawValue: -res.result) } else { @@ -71,23 +74,54 @@ public final class AsyncFileDescriptor { } } - deinit { - if (self.open) { - // TODO: close or error? TBD + @inlinable @inline(__always) @_unsafeInheritExecutor + public func read( + into buffer: borrowing IORingBuffer, //TODO: should be inout? + atAbsoluteOffset offset: UInt64 = UInt64.max + ) async throws -> UInt32 { + let res = await ring.submitAndWait( + .read( + file: .registered(self.fileSlot.borrow()), + buffer: buffer.borrow(), + offset: offset + )) + if res.result < 0 { + throw Errno(rawValue: -res.result) + } else { + return UInt32(bitPattern: res.result) } } + + //TODO: temporary workaround until AsyncSequence supports ~Copyable + public consuming func toBytes() -> AsyncFileDescriptorSequence { + AsyncFileDescriptorSequence(self) + } + + //TODO: can we do the linear types thing and error if they don't consume it manually? + // deinit { + // if self.open { + // TODO: close or error? TBD + // } + // } } -extension AsyncFileDescriptor: AsyncSequence { +public class AsyncFileDescriptorSequence: AsyncSequence { + var descriptor: AsyncFileDescriptor? + public func makeAsyncIterator() -> FileIterator { - return .init(self) + return .init(descriptor.take()!) + } + + internal init(_ descriptor: consuming AsyncFileDescriptor) { + self.descriptor = consume descriptor } public typealias AsyncIterator = FileIterator public typealias Element = UInt8 } -public struct FileIterator: AsyncIteratorProtocol { +//TODO: only a class due to ~Copyable limitations +public class FileIterator: AsyncIteratorProtocol { @usableFromInline let file: AsyncFileDescriptor @usableFromInline var buffer: IORingBuffer @usableFromInline var done: Bool @@ -95,28 +129,27 @@ public struct FileIterator: AsyncIteratorProtocol { @usableFromInline internal var currentByte: UnsafeRawPointer? @usableFromInline internal var lastByte: UnsafeRawPointer? - init(_ file: AsyncFileDescriptor) { - self.file = file + init(_ file: consuming AsyncFileDescriptor) { self.buffer = file.ring.getBuffer()! + self.file = file self.done = false } @inlinable @inline(__always) - public mutating func nextBuffer() async throws { - let buffer = self.buffer - - let bytesRead = try await file.read(into: .registered(buffer)) + public func nextBuffer() async throws { + let bytesRead = Int(try await file.read(into: buffer)) if _fastPath(bytesRead != 0) { - let bufPointer = buffer.unsafeBuffer.baseAddress.unsafelyUnwrapped + let unsafeBuffer = buffer.unsafeBuffer + let bufPointer = unsafeBuffer.baseAddress.unsafelyUnwrapped self.currentByte = UnsafeRawPointer(bufPointer) - self.lastByte = UnsafeRawPointer(bufPointer.advanced(by: Int(bytesRead))) + self.lastByte = UnsafeRawPointer(bufPointer.advanced(by: bytesRead)) } else { - self.done = true + done = true } } @inlinable @inline(__always) @_unsafeInheritExecutor - public mutating func next() async throws -> UInt8? { + public func next() async throws -> UInt8? { if _fastPath(currentByte != lastByte) { // SAFETY: both pointers should be non-nil if they're not equal let byte = currentByte.unsafelyUnwrapped.load(as: UInt8.self) diff --git a/Sources/System/IOCompletion.swift b/Sources/System/IOCompletion.swift index 5e226322..8bf173c9 100644 --- a/Sources/System/IOCompletion.swift +++ b/Sources/System/IOCompletion.swift @@ -1,5 +1,6 @@ @_implementationOnly import CSystem +//TODO: should be ~Copyable, but requires UnsafeContinuation add ~Copyable support public struct IOCompletion { let rawValue: io_uring_cqe } diff --git a/Sources/System/IORequest.swift b/Sources/System/IORequest.swift index 2ecd3f46..2a47c62e 100644 --- a/Sources/System/IORequest.swift +++ b/Sources/System/IORequest.swift @@ -1,7 +1,7 @@ import struct CSystem.io_uring_sqe public enum IORequest: ~Copyable { - case nop // nothing here + case nop // nothing here case openat( atDirectory: FileDescriptor, path: UnsafePointer, @@ -12,21 +12,26 @@ public enum IORequest: ~Copyable { ) case read( file: File, - buffer: Buffer, + buffer: IORingBuffer, + offset: UInt64 = 0 + ) + case readUnregistered( + file: File, + buffer: UnsafeMutableRawBufferPointer, offset: UInt64 = 0 ) case write( file: File, - buffer: Buffer, + buffer: IORingBuffer, + offset: UInt64 = 0 + ) + case writeUnregistered( + file: File, + buffer: UnsafeMutableRawBufferPointer, offset: UInt64 = 0 ) case close(File) - public enum Buffer: ~Copyable { - case registered(IORingBuffer) - case unregistered(UnsafeMutableRawBufferPointer) - } - public enum File: ~Copyable { case registered(IORingFileSlot) case unregistered(FileDescriptor) @@ -90,30 +95,21 @@ extension IORequest { request.rawValue.file_index = UInt32(fileSlot.index + 1) } case .write(let file, let buffer, let offset): - switch consume buffer { - case .registered(let buffer): - request.operation = .writeFixed - return makeRawRequest_readWrite_registered( - file: file, buffer: buffer, offset: offset, request: request) - - case .unregistered(let buffer): - request.operation = .write - return makeRawRequest_readWrite_unregistered( - file: file, buffer: buffer, offset: offset, request: request) - } + request.operation = .writeFixed + return makeRawRequest_readWrite_registered( + file: file, buffer: buffer, offset: offset, request: request) + case .writeUnregistered(let file, let buffer, let offset): + request.operation = .write + return makeRawRequest_readWrite_unregistered( + file: file, buffer: buffer, offset: offset, request: request) case .read(let file, let buffer, let offset): - - switch consume buffer { - case .registered(let buffer): - request.operation = .readFixed - return makeRawRequest_readWrite_registered( - file: file, buffer: buffer, offset: offset, request: request) - - case .unregistered(let buffer): - request.operation = .read - return makeRawRequest_readWrite_unregistered( - file: file, buffer: buffer, offset: offset, request: request) - } + request.operation = .readFixed + return makeRawRequest_readWrite_registered( + file: file, buffer: buffer, offset: offset, request: request) + case .readUnregistered(let file, let buffer, let offset): + request.operation = .read + return makeRawRequest_readWrite_unregistered( + file: file, buffer: buffer, offset: offset, request: request) case .close(let file): request.operation = .close switch file { diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 3e24408f..2f18cb59 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -51,7 +51,7 @@ internal class ResourceManager: @unchecked Sendable { struct Resources { let resourceList: UnsafeMutableBufferPointer - var freeList: [Int] + var freeList: [Int] //TODO: bitvector? } let mutex: Mutex @@ -90,23 +90,33 @@ public struct IOResource: ~Copyable { @usableFromInline let resource: T @usableFromInline let index: Int let manager: ResourceManager + let isBorrow: Bool //TODO: this is a workaround for lifetime issues and should be removed internal init( resource: T, index: Int, - manager: ResourceManager + manager: ResourceManager, + isBorrow: Bool = false ) { self.resource = resource self.index = index self.manager = manager + self.isBorrow = isBorrow } func withResource() { } + //TODO: this is a workaround for lifetime issues and should be removed + @usableFromInline func borrow() -> IOResource { + IOResource(resource: resource, index: index, manager: manager, isBorrow: true) + } + deinit { - self.manager.releaseResource(index: self.index) + if !isBorrow { + manager.releaseResource(index: self.index) + } } } @@ -204,8 +214,6 @@ internal func _getSubmissionEntry(ring: inout SQRing, submissionQueueEntries: Un return nil } -// XXX: This should be a non-copyable type (?) -// demo only runs on Swift 5.8.1 public struct IORing: @unchecked Sendable, ~Copyable { let ringFlags: UInt32 let ringDescriptor: Int32 @@ -455,9 +463,9 @@ public struct IORing: @unchecked Sendable, ~Copyable { @inlinable @inline(__always) public mutating func writeRequest(_ request: __owned IORequest) -> Bool { - let raw = request.makeRawRequest() + var raw: RawIORequest? = request.makeRawRequest() return submissionMutex.withLock { ring in - return _writeRequest(raw, ring: &ring, submissionQueueEntries: submissionQueueEntries) + return _writeRequest(raw.take()!, ring: &ring, submissionQueueEntries: submissionQueueEntries) } } diff --git a/Sources/System/ManagedIORing.swift b/Sources/System/ManagedIORing.swift index 88fb72e7..73ea2314 100644 --- a/Sources/System/ManagedIORing.swift +++ b/Sources/System/ManagedIORing.swift @@ -37,11 +37,11 @@ final public class ManagedIORing: @unchecked Sendable { } internal func getFileSlot() -> IORingFileSlot? { - self.internalRing.getFile() + internalRing.getFile() } internal func getBuffer() -> IORingBuffer? { - self.internalRing.getBuffer() + internalRing.getBuffer() } } diff --git a/Sources/System/RawIORequest.swift b/Sources/System/RawIORequest.swift index 520cb85c..52407b03 100644 --- a/Sources/System/RawIORequest.swift +++ b/Sources/System/RawIORequest.swift @@ -2,7 +2,7 @@ @_implementationOnly import CSystem import struct CSystem.io_uring_sqe -public struct RawIORequest { +public struct RawIORequest: ~Copyable { @usableFromInline var rawValue: io_uring_sqe public init() { diff --git a/Tests/SystemTests/AsyncFileDescriptorTests.swift b/Tests/SystemTests/AsyncFileDescriptorTests.swift index 1baba962..7de22724 100644 --- a/Tests/SystemTests/AsyncFileDescriptorTests.swift +++ b/Tests/SystemTests/AsyncFileDescriptorTests.swift @@ -33,7 +33,7 @@ final class AsyncFileDescriptorTests: XCTestCase { .readOnly, onRing: ring ) - for try await _ in file { + for try await _ in file.toBytes() { XCTFail("/dev/null should be empty") } } diff --git a/Tests/SystemTests/IORequestTests.swift b/Tests/SystemTests/IORequestTests.swift index a44a607e..a1e14533 100644 --- a/Tests/SystemTests/IORequestTests.swift +++ b/Tests/SystemTests/IORequestTests.swift @@ -6,7 +6,7 @@ import XCTest import System #endif -func requestBytes(_ request: RawIORequest) -> [UInt8] { +func requestBytes(_ request: consuming RawIORequest) -> [UInt8] { return withUnsafePointer(to: request) { let requestBuf = UnsafeBufferPointer(start: $0, count: 1) let rawBytes = UnsafeRawBufferPointer(requestBuf) @@ -40,7 +40,7 @@ final class IORequestTests: XCTestCase { .readOnly, options: [], permissions: nil, - intoSlot: fileSlot + intoSlot: fileSlot.borrow() ) let expectedRequest: [UInt8] = { From 822e4814d18de785c06fe4edbe31f750c3898f1b Mon Sep 17 00:00:00 2001 From: David Smith Date: Mon, 28 Oct 2024 23:49:37 +0000 Subject: [PATCH 14/48] Adopt isolation parameters --- Sources/System/AsyncFileDescriptor.swift | 16 +++++++++------- Sources/System/ManagedIORing.swift | 3 +-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Sources/System/AsyncFileDescriptor.swift b/Sources/System/AsyncFileDescriptor.swift index a00f41de..68d9e33c 100644 --- a/Sources/System/AsyncFileDescriptor.swift +++ b/Sources/System/AsyncFileDescriptor.swift @@ -43,8 +43,8 @@ public struct AsyncFileDescriptor: ~Copyable { self.ring = ring } - @inlinable @inline(__always) @_unsafeInheritExecutor - public consuming func close() async throws { + @inlinable @inline(__always) + public consuming func close(isolation actor: isolated (any Actor)? = #isolation) async throws { let res = await ring.submitAndWait( .close( .registered(self.fileSlot) @@ -55,10 +55,11 @@ public struct AsyncFileDescriptor: ~Copyable { self.open = false } - @inlinable @inline(__always) @_unsafeInheritExecutor + @inlinable @inline(__always) public func read( into buffer: inout UnsafeMutableRawBufferPointer, - atAbsoluteOffset offset: UInt64 = UInt64.max + atAbsoluteOffset offset: UInt64 = UInt64.max, + isolation actor: isolated (any Actor)? = #isolation ) async throws -> UInt32 { let file = fileSlot.borrow() let res = await ring.submitAndWait( @@ -74,10 +75,11 @@ public struct AsyncFileDescriptor: ~Copyable { } } - @inlinable @inline(__always) @_unsafeInheritExecutor + @inlinable @inline(__always) public func read( into buffer: borrowing IORingBuffer, //TODO: should be inout? - atAbsoluteOffset offset: UInt64 = UInt64.max + atAbsoluteOffset offset: UInt64 = UInt64.max, + isolation actor: isolated (any Actor)? = #isolation ) async throws -> UInt32 { let res = await ring.submitAndWait( .read( @@ -148,7 +150,7 @@ public class FileIterator: AsyncIteratorProtocol { } } - @inlinable @inline(__always) @_unsafeInheritExecutor + @inlinable @inline(__always) public func next() async throws -> UInt8? { if _fastPath(currentByte != lastByte) { // SAFETY: both pointers should be non-nil if they're not equal diff --git a/Sources/System/ManagedIORing.swift b/Sources/System/ManagedIORing.swift index 73ea2314..30ea5ed6 100644 --- a/Sources/System/ManagedIORing.swift +++ b/Sources/System/ManagedIORing.swift @@ -20,8 +20,7 @@ final public class ManagedIORing: @unchecked Sendable { } } - @_unsafeInheritExecutor - public func submitAndWait(_ request: __owned IORequest) async -> IOCompletion { + public func submitAndWait(_ request: __owned IORequest, isolation actor: isolated (any Actor)? = #isolation) async -> IOCompletion { var consumeOnceWorkaround: IORequest? = request return await withUnsafeContinuation { cont in return internalRing.submissionMutex.withLock { ring in From a9f92a6e316826c6f80d3963f3207bae4d573818 Mon Sep 17 00:00:00 2001 From: David Smith Date: Mon, 28 Oct 2024 17:23:25 -0700 Subject: [PATCH 15/48] Delete stray Package.resolved changes --- Package.resolved | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 Package.resolved diff --git a/Package.resolved b/Package.resolved deleted file mode 100644 index b10a9832..00000000 --- a/Package.resolved +++ /dev/null @@ -1,16 +0,0 @@ -{ - "object": { - "pins": [ - { - "package": "swift-atomics", - "repositoryURL": "https://github.com/apple/swift-atomics", - "state": { - "branch": null, - "revision": "6c89474e62719ddcc1e9614989fff2f68208fe10", - "version": "1.1.0" - } - } - ] - }, - "version": 1 -} From 1a3e37d7919aba47d376200d7c6f998e31aa84ba Mon Sep 17 00:00:00 2001 From: David Smith Date: Mon, 28 Oct 2024 17:24:46 -0700 Subject: [PATCH 16/48] Fix mismerge --- Sources/CSystem/{ => include}/CSystemLinux.h | 0 Sources/CSystem/{ => include}/CSystemWASI.h | 0 Sources/CSystem/{ => include}/CSystemWindows.h | 0 Sources/CSystem/{ => include}/io_uring.h | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename Sources/CSystem/{ => include}/CSystemLinux.h (100%) rename Sources/CSystem/{ => include}/CSystemWASI.h (100%) rename Sources/CSystem/{ => include}/CSystemWindows.h (100%) rename Sources/CSystem/{ => include}/io_uring.h (100%) diff --git a/Sources/CSystem/CSystemLinux.h b/Sources/CSystem/include/CSystemLinux.h similarity index 100% rename from Sources/CSystem/CSystemLinux.h rename to Sources/CSystem/include/CSystemLinux.h diff --git a/Sources/CSystem/CSystemWASI.h b/Sources/CSystem/include/CSystemWASI.h similarity index 100% rename from Sources/CSystem/CSystemWASI.h rename to Sources/CSystem/include/CSystemWASI.h diff --git a/Sources/CSystem/CSystemWindows.h b/Sources/CSystem/include/CSystemWindows.h similarity index 100% rename from Sources/CSystem/CSystemWindows.h rename to Sources/CSystem/include/CSystemWindows.h diff --git a/Sources/CSystem/io_uring.h b/Sources/CSystem/include/io_uring.h similarity index 100% rename from Sources/CSystem/io_uring.h rename to Sources/CSystem/include/io_uring.h From f369347b73ccbd40740cb905a12b82d09083757e Mon Sep 17 00:00:00 2001 From: David Smith Date: Mon, 28 Oct 2024 17:25:31 -0700 Subject: [PATCH 17/48] Fix mismerge --- Sources/CSystem/{ => include}/module.modulemap | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Sources/CSystem/{ => include}/module.modulemap (100%) diff --git a/Sources/CSystem/module.modulemap b/Sources/CSystem/include/module.modulemap similarity index 100% rename from Sources/CSystem/module.modulemap rename to Sources/CSystem/include/module.modulemap From ef94a3719363a97cc8b7e6fa30a096a7d74e9f3c Mon Sep 17 00:00:00 2001 From: David Smith Date: Thu, 5 Dec 2024 03:11:11 +0000 Subject: [PATCH 18/48] Refactoring, and give up on resources being noncopyable structs --- Sources/System/AsyncFileDescriptor.swift | 60 ++-- Sources/System/IOCompletion.swift | 2 +- Sources/System/IORequest.swift | 273 +++++++++++++++--- Sources/System/IORing.swift | 18 +- .../Internals/WindowsSyscallAdapters.swift | 2 +- .../AsyncFileDescriptorTests.swift | 36 ++- Tests/SystemTests/IORequestTests.swift | 14 +- Tests/SystemTests/IORingTests.swift | 4 +- Tests/SystemTests/ManagedIORingTests.swift | 4 +- 9 files changed, 305 insertions(+), 108 deletions(-) diff --git a/Sources/System/AsyncFileDescriptor.swift b/Sources/System/AsyncFileDescriptor.swift index 68d9e33c..f8075d6c 100644 --- a/Sources/System/AsyncFileDescriptor.swift +++ b/Sources/System/AsyncFileDescriptor.swift @@ -5,29 +5,30 @@ public struct AsyncFileDescriptor: ~Copyable { @usableFromInline let fileSlot: IORingFileSlot @usableFromInline let ring: ManagedIORing - public static func openat( - atDirectory: FileDescriptor = FileDescriptor(rawValue: -100), + public static func open( path: FilePath, - _ mode: FileDescriptor.AccessMode, + in directory: FileDescriptor = FileDescriptor(rawValue: -100), + on ring: ManagedIORing, + mode: FileDescriptor.AccessMode, options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), - permissions: FilePermissions? = nil, - onRing ring: ManagedIORing + permissions: FilePermissions? = nil ) async throws -> AsyncFileDescriptor { // todo; real error type guard let fileSlot = ring.getFileSlot() else { throw IORingError.missingRequiredFeatures } + //TODO: need an async-friendly withCString let cstr = path.withCString { return $0 // bad } let res = await ring.submitAndWait( - .openat( - atDirectory: atDirectory, - path: cstr, - mode, - options: options, - permissions: permissions, - intoSlot: fileSlot.borrow() + IORequest( + opening: cstr, + in: directory, + into: fileSlot, + mode: mode, + options: options, + permissions: permissions )) if res.result < 0 { throw Errno(rawValue: -res.result) @@ -45,10 +46,7 @@ public struct AsyncFileDescriptor: ~Copyable { @inlinable @inline(__always) public consuming func close(isolation actor: isolated (any Actor)? = #isolation) async throws { - let res = await ring.submitAndWait( - .close( - .registered(self.fileSlot) - )) + let res = await ring.submitAndWait(IORequest(closing: fileSlot)) if res.result < 0 { throw Errno(rawValue: -res.result) } @@ -61,12 +59,11 @@ public struct AsyncFileDescriptor: ~Copyable { atAbsoluteOffset offset: UInt64 = UInt64.max, isolation actor: isolated (any Actor)? = #isolation ) async throws -> UInt32 { - let file = fileSlot.borrow() let res = await ring.submitAndWait( - .readUnregistered( - file: .registered(file), - buffer: buffer, - offset: offset + IORequest( + reading: fileSlot, + into: buffer, + at: offset )) if res.result < 0 { throw Errno(rawValue: -res.result) @@ -77,15 +74,15 @@ public struct AsyncFileDescriptor: ~Copyable { @inlinable @inline(__always) public func read( - into buffer: borrowing IORingBuffer, //TODO: should be inout? + into buffer: IORingBuffer, //TODO: should be inout? atAbsoluteOffset offset: UInt64 = UInt64.max, isolation actor: isolated (any Actor)? = #isolation ) async throws -> UInt32 { let res = await ring.submitAndWait( - .read( - file: .registered(self.fileSlot.borrow()), - buffer: buffer.borrow(), - offset: offset + IORequest( + reading: fileSlot, + into: buffer, + at: offset )) if res.result < 0 { throw Errno(rawValue: -res.result) @@ -100,11 +97,12 @@ public struct AsyncFileDescriptor: ~Copyable { } //TODO: can we do the linear types thing and error if they don't consume it manually? - // deinit { - // if self.open { - // TODO: close or error? TBD - // } - // } + // deinit { + // if self.open { + // close() + // // TODO: close or error? TBD + // } + // } } public class AsyncFileDescriptorSequence: AsyncSequence { diff --git a/Sources/System/IOCompletion.swift b/Sources/System/IOCompletion.swift index 8bf173c9..1702f9e8 100644 --- a/Sources/System/IOCompletion.swift +++ b/Sources/System/IOCompletion.swift @@ -21,7 +21,7 @@ extension IOCompletion { } extension IOCompletion { - public var userData: UInt64 { + public var userData: UInt64 { //TODO: naming? get { return rawValue.user_data } diff --git a/Sources/System/IORequest.swift b/Sources/System/IORequest.swift index 2a47c62e..0af87176 100644 --- a/Sources/System/IORequest.swift +++ b/Sources/System/IORequest.swift @@ -1,57 +1,90 @@ import struct CSystem.io_uring_sqe -public enum IORequest: ~Copyable { +@usableFromInline +internal enum IORequestCore: ~Copyable { case nop // nothing here case openat( + atDirectory: FileDescriptor, + path: UnsafePointer, + FileDescriptor.AccessMode, + options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), + permissions: FilePermissions? = nil + ) + case openatSlot( atDirectory: FileDescriptor, path: UnsafePointer, FileDescriptor.AccessMode, options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), permissions: FilePermissions? = nil, - intoSlot: IORingFileSlot? = nil + intoSlot: IORingFileSlot ) case read( - file: File, + file: FileDescriptor, buffer: IORingBuffer, offset: UInt64 = 0 ) case readUnregistered( - file: File, + file: FileDescriptor, + buffer: UnsafeMutableRawBufferPointer, + offset: UInt64 = 0 + ) + case readSlot( + file: IORingFileSlot, + buffer: IORingBuffer, + offset: UInt64 = 0 + ) + case readUnregisteredSlot( + file: IORingFileSlot, buffer: UnsafeMutableRawBufferPointer, offset: UInt64 = 0 ) case write( - file: File, + file: FileDescriptor, buffer: IORingBuffer, offset: UInt64 = 0 ) case writeUnregistered( - file: File, + file: FileDescriptor, buffer: UnsafeMutableRawBufferPointer, offset: UInt64 = 0 ) - case close(File) - - public enum File: ~Copyable { - case registered(IORingFileSlot) - case unregistered(FileDescriptor) - } + case writeSlot( + file: IORingFileSlot, + buffer: IORingBuffer, + offset: UInt64 = 0 + ) + case writeUnregisteredSlot( + file: IORingFileSlot, + buffer: UnsafeMutableRawBufferPointer, + offset: UInt64 = 0 + ) + case close(FileDescriptor) + case closeSlot(IORingFileSlot) } @inlinable @inline(__always) internal func makeRawRequest_readWrite_registered( - file: consuming IORequest.File, - buffer: consuming IORingBuffer, + file: FileDescriptor, + buffer: IORingBuffer, offset: UInt64, request: consuming RawIORequest ) -> RawIORequest { - switch file { - case .registered(let regFile): - request.rawValue.fd = Int32(exactly: regFile.index)! - request.flags = .fixedFile - case .unregistered(let fd): - request.fileDescriptor = fd - } + request.fileDescriptor = file + request.buffer = buffer.unsafeBuffer + request.rawValue.buf_index = UInt16(exactly: buffer.index)! + request.offset = offset + return request +} + +@inlinable @inline(__always) +internal func makeRawRequest_readWrite_registered_slot( + file: IORingFileSlot, + buffer: IORingBuffer, + offset: UInt64, + request: consuming RawIORequest +) -> RawIORequest { + request.rawValue.fd = Int32(exactly: file.index)! + request.flags = .fixedFile request.buffer = buffer.unsafeBuffer request.rawValue.buf_index = UInt16(exactly: buffer.index)! request.offset = offset @@ -60,64 +93,222 @@ internal func makeRawRequest_readWrite_registered( @inlinable @inline(__always) internal func makeRawRequest_readWrite_unregistered( - file: consuming IORequest.File, + file: FileDescriptor, buffer: UnsafeMutableRawBufferPointer, offset: UInt64, request: consuming RawIORequest ) -> RawIORequest { - switch file { - case .registered(let regFile): - request.rawValue.fd = Int32(exactly: regFile.index)! - request.flags = .fixedFile - case .unregistered(let fd): - request.fileDescriptor = fd - } + request.fileDescriptor = file request.buffer = buffer request.offset = offset return request } +@inlinable @inline(__always) +internal func makeRawRequest_readWrite_unregistered_slot( + file: IORingFileSlot, + buffer: UnsafeMutableRawBufferPointer, + offset: UInt64, + request: consuming RawIORequest +) -> RawIORequest { + request.rawValue.fd = Int32(exactly: file.index)! + request.flags = .fixedFile + request.buffer = buffer + request.offset = offset + return request +} + +public struct IORequest : ~Copyable { + @usableFromInline var core: IORequestCore + + @inlinable internal consuming func extractCore() -> IORequestCore { + return core + } +} + extension IORequest { + public init() { //TODO: why do we have nop? + core = .nop + } + + public init( + reading file: IORingFileSlot, + into buffer: IORingBuffer, + at offset: UInt64 = 0 + ) { + core = .readSlot(file: file, buffer: buffer, offset: offset) + } + + public init( + reading file: FileDescriptor, + into buffer: IORingBuffer, + at offset: UInt64 = 0 + ) { + core = .read(file: file, buffer: buffer, offset: offset) + } + + public init( + reading file: IORingFileSlot, + into buffer: UnsafeMutableRawBufferPointer, + at offset: UInt64 = 0 + ) { + core = .readUnregisteredSlot(file: file, buffer: buffer, offset: offset) + } + + public init( + reading file: FileDescriptor, + into buffer: UnsafeMutableRawBufferPointer, + at offset: UInt64 = 0 + ) { + core = .readUnregistered(file: file, buffer: buffer, offset: offset) + } + + public init( + writing buffer: IORingBuffer, + into file: IORingFileSlot, + at offset: UInt64 = 0 + ) { + core = .writeSlot(file: file, buffer: buffer, offset: offset) + } + + public init( + writing buffer: IORingBuffer, + into file: FileDescriptor, + at offset: UInt64 = 0 + ) { + core = .write(file: file, buffer: buffer, offset: offset) + } + + public init( + writing buffer: UnsafeMutableRawBufferPointer, + into file: IORingFileSlot, + at offset: UInt64 = 0 + ) { + core = .writeUnregisteredSlot(file: file, buffer: buffer, offset: offset) + } + + public init( + writing buffer: UnsafeMutableRawBufferPointer, + into file: FileDescriptor, + at offset: UInt64 = 0 + ) { + core = .writeUnregistered(file: file, buffer: buffer, offset: offset) + } + + public init( + closing file: FileDescriptor + ) { + core = .close(file) + } + + public init( + closing file: IORingFileSlot + ) { + core = .closeSlot(file) + } + + + public init( + opening path: UnsafePointer, + in directory: FileDescriptor, + into slot: IORingFileSlot, + mode: FileDescriptor.AccessMode, + options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), + permissions: FilePermissions? = nil + ) { + core = .openatSlot(atDirectory: directory, path: path, mode, options: options, permissions: permissions, intoSlot: slot) + } + + public init( + opening path: UnsafePointer, + in directory: FileDescriptor, + mode: FileDescriptor.AccessMode, + options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), + permissions: FilePermissions? = nil + ) { + core = .openat(atDirectory: directory, path: path, mode, options: options, permissions: permissions) + } + + + public init( + opening path: FilePath, + in directory: FileDescriptor, + into slot: IORingFileSlot, + mode: FileDescriptor.AccessMode, + options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), + permissions: FilePermissions? = nil + ) { + fatalError("Implement me") + } + + public init( + opening path: FilePath, + in directory: FileDescriptor, + mode: FileDescriptor.AccessMode, + options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), + permissions: FilePermissions? = nil + ) { + fatalError("Implement me") + } + @inlinable @inline(__always) public consuming func makeRawRequest() -> RawIORequest { var request = RawIORequest() - switch consume self { + switch extractCore() { case .nop: request.operation = .nop - case .openat(let atDirectory, let path, let mode, let options, let permissions, let slot): + case .openatSlot(let atDirectory, let path, let mode, let options, let permissions, let fileSlot): // TODO: use rawValue less request.operation = .openAt request.fileDescriptor = atDirectory - request.rawValue.addr = unsafeBitCast(path, to: UInt64.self) + request.rawValue.addr = UInt64(UInt(bitPattern: path)) + request.rawValue.open_flags = UInt32(bitPattern: options.rawValue | mode.rawValue) + request.rawValue.len = permissions?.rawValue ?? 0 + request.rawValue.file_index = UInt32(fileSlot.index + 1) + case .openat(let atDirectory, let path, let mode, let options, let permissions): + request.operation = .openAt + request.fileDescriptor = atDirectory + request.rawValue.addr = UInt64(UInt(bitPattern: path)) request.rawValue.open_flags = UInt32(bitPattern: options.rawValue | mode.rawValue) request.rawValue.len = permissions?.rawValue ?? 0 - if let fileSlot = slot { - request.rawValue.file_index = UInt32(fileSlot.index + 1) - } case .write(let file, let buffer, let offset): request.operation = .writeFixed return makeRawRequest_readWrite_registered( file: file, buffer: buffer, offset: offset, request: request) + case .writeSlot(let file, let buffer, let offset): + request.operation = .writeFixed + return makeRawRequest_readWrite_registered_slot( + file: file, buffer: buffer, offset: offset, request: request) case .writeUnregistered(let file, let buffer, let offset): request.operation = .write return makeRawRequest_readWrite_unregistered( file: file, buffer: buffer, offset: offset, request: request) + case .writeUnregisteredSlot(let file, let buffer, let offset): + request.operation = .write + return makeRawRequest_readWrite_unregistered_slot( + file: file, buffer: buffer, offset: offset, request: request) case .read(let file, let buffer, let offset): request.operation = .readFixed return makeRawRequest_readWrite_registered( file: file, buffer: buffer, offset: offset, request: request) + case .readSlot(let file, let buffer, let offset): + request.operation = .readFixed + return makeRawRequest_readWrite_registered_slot( + file: file, buffer: buffer, offset: offset, request: request) case .readUnregistered(let file, let buffer, let offset): request.operation = .read return makeRawRequest_readWrite_unregistered( file: file, buffer: buffer, offset: offset, request: request) + case .readUnregisteredSlot(let file, let buffer, let offset): + request.operation = .read + return makeRawRequest_readWrite_unregistered_slot( + file: file, buffer: buffer, offset: offset, request: request) case .close(let file): request.operation = .close - switch file { - case .registered(let regFile): - request.rawValue.file_index = UInt32(regFile.index + 1) - case .unregistered(let normalFile): - request.fileDescriptor = normalFile - } + request.fileDescriptor = file + case .closeSlot(let file): + request.operation = .close + request.rawValue.file_index = UInt32(file.index + 1) } return request } diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 2f18cb59..57c21611 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -46,7 +46,7 @@ struct CQRing: ~Copyable { let cqes: UnsafeBufferPointer } -internal class ResourceManager: @unchecked Sendable { +internal final class ResourceManager: @unchecked Sendable { typealias Resource = T struct Resources { @@ -85,38 +85,28 @@ internal class ResourceManager: @unchecked Sendable { } } -public struct IOResource: ~Copyable { +public class IOResource { typealias Resource = T @usableFromInline let resource: T @usableFromInline let index: Int let manager: ResourceManager - let isBorrow: Bool //TODO: this is a workaround for lifetime issues and should be removed internal init( resource: T, index: Int, - manager: ResourceManager, - isBorrow: Bool = false + manager: ResourceManager ) { self.resource = resource self.index = index self.manager = manager - self.isBorrow = isBorrow } func withResource() { } - //TODO: this is a workaround for lifetime issues and should be removed - @usableFromInline func borrow() -> IOResource { - IOResource(resource: resource, index: index, manager: manager, isBorrow: true) - } - deinit { - if !isBorrow { - manager.releaseResource(index: self.index) - } + manager.releaseResource(index: self.index) } } diff --git a/Sources/System/Internals/WindowsSyscallAdapters.swift b/Sources/System/Internals/WindowsSyscallAdapters.swift index d56d33e4..706881ec 100644 --- a/Sources/System/Internals/WindowsSyscallAdapters.swift +++ b/Sources/System/Internals/WindowsSyscallAdapters.swift @@ -187,7 +187,7 @@ internal func pwrite( internal func pipe( _ fds: UnsafeMutablePointer, bytesReserved: UInt32 = 4096 ) -> CInt { -  return _pipe(fds, bytesReserved, _O_BINARY | _O_NOINHERIT); + return _pipe(fds, bytesReserved, _O_BINARY | _O_NOINHERIT); } @inline(__always) diff --git a/Tests/SystemTests/AsyncFileDescriptorTests.swift b/Tests/SystemTests/AsyncFileDescriptorTests.swift index 7de22724..ebd308fc 100644 --- a/Tests/SystemTests/AsyncFileDescriptorTests.swift +++ b/Tests/SystemTests/AsyncFileDescriptorTests.swift @@ -9,32 +9,50 @@ import System final class AsyncFileDescriptorTests: XCTestCase { func testOpen() async throws { let ring = try ManagedIORing(queueDepth: 32) - let file = try await AsyncFileDescriptor.openat( + _ = try await AsyncFileDescriptor.open( path: "/dev/zero", - .readOnly, - onRing: ring + on: ring, + mode: .readOnly ) } func testOpenClose() async throws { let ring = try ManagedIORing(queueDepth: 32) - let file = try await AsyncFileDescriptor.openat( + let file = try await AsyncFileDescriptor.open( path: "/dev/zero", - .readOnly, - onRing: ring + on: ring, + mode: .readOnly ) try await file.close() } func testDevNullEmpty() async throws { let ring = try ManagedIORing(queueDepth: 32) - let file = try await AsyncFileDescriptor.openat( + let file = try await AsyncFileDescriptor.open( path: "/dev/null", - .readOnly, - onRing: ring + on: ring, + mode: .readOnly ) for try await _ in file.toBytes() { XCTFail("/dev/null should be empty") } } + + func testRead() async throws { + let ring = try ManagedIORing(queueDepth: 32) + let file = try await AsyncFileDescriptor.open( + path: "/dev/zero", + on: ring, + mode: .readOnly + ) + let bytes = file.toBytes() + var counter = 0 + for try await byte in bytes { + XCTAssert(byte == 0) + counter &+= 1 + if counter > 16384 { + break + } + } + } } diff --git a/Tests/SystemTests/IORequestTests.swift b/Tests/SystemTests/IORequestTests.swift index a1e14533..78be4cf7 100644 --- a/Tests/SystemTests/IORequestTests.swift +++ b/Tests/SystemTests/IORequestTests.swift @@ -19,7 +19,7 @@ func requestBytes(_ request: consuming RawIORequest) -> [UInt8] { // which are known to work correctly. final class IORequestTests: XCTestCase { func testNop() { - let req = IORequest.nop.makeRawRequest() + let req = IORequest().makeRawRequest() let sourceBytes = requestBytes(req) // convenient property of nop: it's all zeros! // for some unknown reason, liburing sets the fd field to -1. @@ -34,13 +34,13 @@ final class IORequestTests: XCTestCase { let pathPtr = UnsafePointer(bitPattern: 0x414141410badf00d)! let fileSlot = resmgr.getResource()! - let req = IORequest.openat( - atDirectory: FileDescriptor(rawValue: -100), - path: pathPtr, - .readOnly, + let req = IORequest( + opening: pathPtr, + in: FileDescriptor(rawValue: -100), + into: fileSlot, + mode: .readOnly, options: [], - permissions: nil, - intoSlot: fileSlot.borrow() + permissions: nil ) let expectedRequest: [UInt8] = { diff --git a/Tests/SystemTests/IORingTests.swift b/Tests/SystemTests/IORingTests.swift index 4604f500..85846d3e 100644 --- a/Tests/SystemTests/IORingTests.swift +++ b/Tests/SystemTests/IORingTests.swift @@ -8,12 +8,12 @@ import System final class IORingTests: XCTestCase { func testInit() throws { - let ring = try IORing(queueDepth: 32) + _ = try IORing(queueDepth: 32) } func testNop() throws { var ring = try IORing(queueDepth: 32) - ring.writeRequest(.nop) + ring.writeRequest(IORequest()) ring.submitRequests() let completion = ring.blockingConsumeCompletion() XCTAssertEqual(completion.result, 0) diff --git a/Tests/SystemTests/ManagedIORingTests.swift b/Tests/SystemTests/ManagedIORingTests.swift index e7ad3f59..4b8ea28b 100644 --- a/Tests/SystemTests/ManagedIORingTests.swift +++ b/Tests/SystemTests/ManagedIORingTests.swift @@ -8,12 +8,12 @@ import System final class ManagedIORingTests: XCTestCase { func testInit() throws { - let ring = try ManagedIORing(queueDepth: 32) + _ = try ManagedIORing(queueDepth: 32) } func testNop() async throws { let ring = try ManagedIORing(queueDepth: 32) - let completion = await ring.submitAndWait(.nop) + let completion = await ring.submitAndWait(IORequest()) XCTAssertEqual(completion.result, 0) } } From 7ea32aefce1f716cb0147f43b64eacf184f1914f Mon Sep 17 00:00:00 2001 From: David Smith Date: Wed, 11 Dec 2024 23:00:50 +0000 Subject: [PATCH 19/48] More refactoring, and working timeout support on ManagedIORing --- Sources/System/AsyncFileDescriptor.swift | 11 ++- Sources/System/IORing.swift | 82 +++++++++++++++++----- Sources/System/IORingError.swift | 5 +- Sources/System/ManagedIORing.swift | 74 ++++++++++++++++--- Sources/System/RawIORequest.swift | 42 ++++++++++- Tests/SystemTests/IORingTests.swift | 2 +- Tests/SystemTests/ManagedIORingTests.swift | 31 +++++++- 7 files changed, 208 insertions(+), 39 deletions(-) diff --git a/Sources/System/AsyncFileDescriptor.swift b/Sources/System/AsyncFileDescriptor.swift index f8075d6c..a76d90e5 100644 --- a/Sources/System/AsyncFileDescriptor.swift +++ b/Sources/System/AsyncFileDescriptor.swift @@ -21,8 +21,7 @@ public struct AsyncFileDescriptor: ~Copyable { let cstr = path.withCString { return $0 // bad } - let res = await ring.submitAndWait( - IORequest( + let res = try await ring.submit(request: IORequest( opening: cstr, in: directory, into: fileSlot, @@ -46,7 +45,7 @@ public struct AsyncFileDescriptor: ~Copyable { @inlinable @inline(__always) public consuming func close(isolation actor: isolated (any Actor)? = #isolation) async throws { - let res = await ring.submitAndWait(IORequest(closing: fileSlot)) + let res = try await ring.submit(request: IORequest(closing: fileSlot)) if res.result < 0 { throw Errno(rawValue: -res.result) } @@ -59,8 +58,7 @@ public struct AsyncFileDescriptor: ~Copyable { atAbsoluteOffset offset: UInt64 = UInt64.max, isolation actor: isolated (any Actor)? = #isolation ) async throws -> UInt32 { - let res = await ring.submitAndWait( - IORequest( + let res = try await ring.submit(request: IORequest( reading: fileSlot, into: buffer, at: offset @@ -78,8 +76,7 @@ public struct AsyncFileDescriptor: ~Copyable { atAbsoluteOffset offset: UInt64 = UInt64.max, isolation actor: isolated (any Actor)? = #isolation ) async throws -> UInt32 { - let res = await ring.submitAndWait( - IORequest( + let res = try await ring.submit(request: IORequest( reading: fileSlot, into: buffer, at: offset diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 57c21611..00461d14 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -138,7 +138,10 @@ internal func _blockingGetSubmissionEntry(ring: inout SQRing, submissionQueueEnt io_uring_sqe > { while true { - if let entry = _getSubmissionEntry(ring: &ring, submissionQueueEntries: submissionQueueEntries) { + if let entry = _getSubmissionEntry( + ring: &ring, + submissionQueueEntries: submissionQueueEntries + ) { return entry } // TODO: actually block here instead of spinning @@ -146,13 +149,18 @@ internal func _blockingGetSubmissionEntry(ring: inout SQRing, submissionQueueEnt } -internal func _submitRequests(ring: inout SQRing, ringDescriptor: Int32) { - let flushedEvents = _flushQueue(ring: &ring) - - // Ring always needs enter right now; +//TODO: omitting signal mask for now +//Tell the kernel that we've submitted requests and/or are waiting for completions +internal func _enter( + ringDescriptor: Int32, + numEvents: UInt32, + minCompletions: UInt32, + flags: UInt32 +) throws -> Int32 { + // Ring always needs enter right now; // TODO: support SQPOLL here while true { - let ret = io_uring_enter(ringDescriptor, flushedEvents, 0, 0, nil) + let ret = io_uring_enter(ringDescriptor, numEvents, minCompletions, flags, nil) // error handling: // EAGAIN / EINTR (try again), // EBADF / EBADFD / EOPNOTSUPP / ENXIO @@ -160,32 +168,47 @@ internal func _submitRequests(ring: inout SQRing, ringDescriptor: Int32) { // EINVAL (bad constant flag?, fatal), // EFAULT (bad address for argument from library, fatal) if ret == -EAGAIN || ret == -EINTR { + //TODO: should we wait a bit on AGAIN? continue } else if ret < 0 { fatalError( "fatal error in submitting requests: " + Errno(rawValue: -ret).debugDescription ) } else { - break + return ret } } } +internal func _submitRequests(ring: inout SQRing, ringDescriptor: Int32) throws { + let flushedEvents = _flushQueue(ring: &ring) + _ = try _enter(ringDescriptor: ringDescriptor, numEvents: flushedEvents, minCompletions: 0, flags: 0) +} + +internal func _getUnconsumedSubmissionCount(ring: inout SQRing) -> UInt32 { + return ring.userTail - ring.kernelHead.pointee.load(ordering: .acquiring) +} + +internal func _getUnconsumedCompletionCount(ring: inout CQRing) -> UInt32 { + return ring.kernelTail.pointee.load(ordering: .acquiring) - ring.kernelHead.pointee.load(ordering: .acquiring) +} + +//TODO: pretty sure this is supposed to do more than it does internal func _flushQueue(ring: inout SQRing) -> UInt32 { ring.kernelTail.pointee.store( - ring.userTail, ordering: .relaxed + ring.userTail, ordering: .releasing ) - return ring.userTail - ring.kernelHead.pointee.load(ordering: .relaxed) + return _getUnconsumedSubmissionCount(ring: &ring) } @usableFromInline @inline(__always) internal func _getSubmissionEntry(ring: inout SQRing, submissionQueueEntries: UnsafeMutableBufferPointer) -> UnsafeMutablePointer< io_uring_sqe >? { - let next = ring.userTail + 1 + let next = ring.userTail &+ 1 //this is expected to wrap // FEAT: smp load when SQPOLL in use (not in MVP) - let kernelHead = ring.kernelHead.pointee.load(ordering: .relaxed) + let kernelHead = ring.kernelHead.pointee.load(ordering: .acquiring) // FEAT: 128-bit event support (not in MVP) if next - kernelHead <= ring.array.count { @@ -383,18 +406,45 @@ public struct IORing: @unchecked Sendable, ~Copyable { func _tryConsumeCompletion(ring: inout CQRing) -> IOCompletion? { let tail = ring.kernelTail.pointee.load(ordering: .acquiring) - let head = ring.kernelHead.pointee.load(ordering: .relaxed) + let head = ring.kernelHead.pointee.load(ordering: .acquiring) if tail != head { // 32 byte copy - oh well let res = ring.cqes[Int(head & ring.ringMask)] - ring.kernelHead.pointee.store(head + 1, ordering: .relaxed) + ring.kernelHead.pointee.store(head &+ 1, ordering: .releasing) return IOCompletion(rawValue: res) } return nil } + internal func handleRegistrationResult(_ result: Int32) throws { + //TODO: error handling + } + + public mutating func registerEventFD(ring: inout IORing, _ descriptor: FileDescriptor) throws { + var rawfd = descriptor.rawValue + let result = withUnsafePointer(to: &rawfd) { fdptr in + return io_uring_register( + ring.ringDescriptor, + IORING_REGISTER_EVENTFD, + UnsafeMutableRawPointer(mutating: fdptr), + 1 + ) + } + try handleRegistrationResult(result) + } + + public mutating func unregisterEventFD(ring: inout IORing) throws { + let result = io_uring_register( + ring.ringDescriptor, + IORING_UNREGISTER_EVENTFD, + nil, + 0 + ) + try handleRegistrationResult(result) + } + public mutating func registerFiles(count: UInt32) { guard self.registeredFiles == nil else { fatalError() } let fileBuf = UnsafeMutableBufferPointer.allocate(capacity: Int(count)) @@ -445,9 +495,9 @@ public struct IORing: @unchecked Sendable, ~Copyable { fatalError("failed to unregister buffers: TODO") } - public func submitRequests() { - submissionMutex.withLock { ring in - _submitRequests(ring: &ring, ringDescriptor: ringDescriptor) + public func submitRequests() throws { + try submissionMutex.withLock { ring in + try _submitRequests(ring: &ring, ringDescriptor: ringDescriptor) } } diff --git a/Sources/System/IORingError.swift b/Sources/System/IORingError.swift index d87b2938..fda58bcb 100644 --- a/Sources/System/IORingError.swift +++ b/Sources/System/IORingError.swift @@ -1,3 +1,6 @@ -enum IORingError: Error { +//TODO: make this not an enum +public enum IORingError: Error, Equatable { case missingRequiredFeatures + case operationCanceled + case unknown } diff --git a/Sources/System/ManagedIORing.swift b/Sources/System/ManagedIORing.swift index 30ea5ed6..99c112d6 100644 --- a/Sources/System/ManagedIORing.swift +++ b/Sources/System/ManagedIORing.swift @@ -1,3 +1,16 @@ +fileprivate func handleCompletionError( + _ result: Int32, + for continuation: UnsafeContinuation) { + var error: IORingError = .unknown + switch result { + case -(_ECANCELED): + error = .operationCanceled + default: + error = .unknown + } + continuation.resume(throwing: error) +} + final public class ManagedIORing: @unchecked Sendable { var internalRing: IORing @@ -11,25 +24,64 @@ final public class ManagedIORing: @unchecked Sendable { private func startWaiter() { Task.detached { while !Task.isCancelled { + //TODO: should timeout handling be sunk into IORing? let cqe = self.internalRing.blockingConsumeCompletion() + if cqe.userData == 0 { + continue + } let cont = unsafeBitCast( - cqe.userData, to: UnsafeContinuation.self) - cont.resume(returning: cqe) + cqe.userData, to: UnsafeContinuation.self) + + if cqe.result < 0 { + var err = system_strerror(cqe.result * -1) + let len = system_strlen(err!) + err!.withMemoryRebound(to: UInt8.self, capacity: len) { + let errStr = String(decoding: UnsafeBufferPointer(start: $0, count: len), as: UTF8.self) + print("\(errStr)") + } + handleCompletionError(cqe.result, for: cont) + } else { + cont.resume(returning: cqe) + } } } } - public func submitAndWait(_ request: __owned IORequest, isolation actor: isolated (any Actor)? = #isolation) async -> IOCompletion { + public func submit( + request: __owned IORequest, + timeout: Duration? = nil, + isolation actor: isolated (any Actor)? = #isolation + ) async throws -> IOCompletion { var consumeOnceWorkaround: IORequest? = request - return await withUnsafeContinuation { cont in - return internalRing.submissionMutex.withLock { ring in - let request = consumeOnceWorkaround.take()! - let entry = _blockingGetSubmissionEntry( - ring: &ring, submissionQueueEntries: internalRing.submissionQueueEntries) - entry.pointee = request.makeRawRequest().rawValue - entry.pointee.user_data = unsafeBitCast(cont, to: UInt64.self) - _submitRequests(ring: &ring, ringDescriptor: internalRing.ringDescriptor) + return try await withUnsafeThrowingContinuation { cont in + do { + try internalRing.submissionMutex.withLock { ring in + let request = consumeOnceWorkaround.take()! + let entry = _blockingGetSubmissionEntry( + ring: &ring, submissionQueueEntries: internalRing.submissionQueueEntries) + entry.pointee = request.makeRawRequest().rawValue + entry.pointee.user_data = unsafeBitCast(cont, to: UInt64.self) + if let timeout { + //TODO: if IORING_FEAT_MIN_TIMEOUT is supported we can do this more efficiently + let timeoutEntry = _blockingGetSubmissionEntry( + ring: &ring, + submissionQueueEntries: internalRing.submissionQueueEntries + ) + try RawIORequest.withTimeoutRequest( + linkedTo: entry, + in: timeoutEntry, + duration: timeout, + flags: .relativeTime + ) { + try _submitRequests(ring: &ring, ringDescriptor: internalRing.ringDescriptor) + } + } else { + try _submitRequests(ring: &ring, ringDescriptor: internalRing.ringDescriptor) + } + } + } catch (let e) { + cont.resume(throwing: e) } } diff --git a/Sources/System/RawIORequest.swift b/Sources/System/RawIORequest.swift index 52407b03..5a1fd03d 100644 --- a/Sources/System/RawIORequest.swift +++ b/Sources/System/RawIORequest.swift @@ -24,6 +24,8 @@ extension RawIORequest { case sendMessage = 9 case receiveMessage = 10 // ... + case link_timeout = 15 + // ... case openAt = 18 case close = 19 case filesUpdate = 20 @@ -103,7 +105,7 @@ extension RawIORequest { // poll32_events // sync_range_flags // msg_flags - // timeout_flags + case timeoutFlags(TimeOutFlags) // accept_flags // cancel_flags case openFlags(FileDescriptor.OpenOptions) @@ -132,12 +134,48 @@ extension RawIORequest { // append to end of the file public static let append = ReadWriteFlags(rawValue: 1 << 4) } + + public struct TimeOutFlags: OptionSet, Hashable, Codable { + public var rawValue: UInt32 + + public init(rawValue: UInt32) { + self.rawValue = rawValue + } + + public static let relativeTime: RawIORequest.TimeOutFlags = TimeOutFlags(rawValue: 0) + public static let absoluteTime: RawIORequest.TimeOutFlags = TimeOutFlags(rawValue: 1 << 0) + } } extension RawIORequest { static func nop() -> RawIORequest { - var req = RawIORequest() + var req: RawIORequest = RawIORequest() req.operation = .nop return req } + + //TODO: typed errors + static func withTimeoutRequest( + linkedTo opEntry: UnsafeMutablePointer, + in timeoutEntry: UnsafeMutablePointer, + duration: Duration, + flags: TimeOutFlags, + work: () throws -> R) rethrows -> R { + + opEntry.pointee.flags |= Flags.linkRequest.rawValue + opEntry.pointee.off = 1 + var ts = __kernel_timespec( + tv_sec: duration.components.seconds, + tv_nsec: duration.components.attoseconds / 1_000_000_000 + ) + return try withUnsafePointer(to: &ts) { tsPtr in + var req: RawIORequest = RawIORequest() + req.operation = .link_timeout + req.rawValue.timeout_flags = flags.rawValue + req.rawValue.len = 1 + req.rawValue.addr = UInt64(UInt(bitPattern: tsPtr)) + timeoutEntry.pointee = req.rawValue + return try work() + } + } } \ No newline at end of file diff --git a/Tests/SystemTests/IORingTests.swift b/Tests/SystemTests/IORingTests.swift index 85846d3e..d0102a0b 100644 --- a/Tests/SystemTests/IORingTests.swift +++ b/Tests/SystemTests/IORingTests.swift @@ -14,7 +14,7 @@ final class IORingTests: XCTestCase { func testNop() throws { var ring = try IORing(queueDepth: 32) ring.writeRequest(IORequest()) - ring.submitRequests() + try ring.submitRequests() let completion = ring.blockingConsumeCompletion() XCTAssertEqual(completion.result, 0) } diff --git a/Tests/SystemTests/ManagedIORingTests.swift b/Tests/SystemTests/ManagedIORingTests.swift index 4b8ea28b..24324037 100644 --- a/Tests/SystemTests/ManagedIORingTests.swift +++ b/Tests/SystemTests/ManagedIORingTests.swift @@ -13,7 +13,36 @@ final class ManagedIORingTests: XCTestCase { func testNop() async throws { let ring = try ManagedIORing(queueDepth: 32) - let completion = await ring.submitAndWait(IORequest()) + let completion = try await ring.submit(request: IORequest()) XCTAssertEqual(completion.result, 0) } + + func testTimeout() async throws { + let ring = try ManagedIORing(queueDepth: 32) + var pipes: (Int32, Int32) = (0, 0) + withUnsafeMutableBytes(of: &pipes) { ptr in + ptr.withMemoryRebound(to: UInt32.self) { tptr in + let res = pipe(tptr.baseAddress!) + XCTAssertEqual(res, 0) + } + } + let buffer = UnsafeMutableRawBufferPointer.allocate(byteCount: 128, alignment: 16) + do { + let completion = try await ring.submit( + request: IORequest(reading: FileDescriptor(rawValue: pipes.0), into: buffer), + timeout: .seconds(0.1) + ) + print("\(completion)") + XCTFail("An error should be thrown") + } catch (let e) { + if let err = e as? IORingError { + XCTAssertEqual(err, .operationCanceled) + } else { + XCTFail() + } + buffer.deallocate() + close(pipes.0) + close(pipes.1) + } + } } From 55fd6e7150cb6f48fa0203c1df29746be35b1a00 Mon Sep 17 00:00:00 2001 From: David Smith Date: Thu, 12 Dec 2024 01:20:24 +0000 Subject: [PATCH 20/48] Add support for timeout-on-wait to IORing, don't have tests yet --- Sources/System/IORing.swift | 37 ++++++++++++++++++++-- Sources/System/ManagedIORing.swift | 33 ++++++++++--------- Tests/SystemTests/IORingTests.swift | 2 +- Tests/SystemTests/ManagedIORingTests.swift | 2 +- 4 files changed, 53 insertions(+), 21 deletions(-) diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 00461d14..872d9960 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -368,13 +368,26 @@ public struct IORing: @unchecked Sendable, ~Copyable { self.ringFlags = params.flags } - public func blockingConsumeCompletion() -> IOCompletion { + private func _blockingConsumeCompletionGuts( + extraArgs: UnsafeMutablePointer? = nil + ) throws(IORingError) -> IOCompletion { completionMutex.withLock { ring in if let completion = _tryConsumeCompletion(ring: &ring) { return completion } else { while true { - let res = io_uring_enter(ringDescriptor, 0, 1, IORING_ENTER_GETEVENTS, nil) + var sz = 0 + if extraArgs != nil { + sz = MemoryLayout.size + } + let res = io_uring_enter2( + ringDescriptor, + 0, + 1, + IORING_ENTER_GETEVENTS, + extraArgs, + sz + ) // error handling: // EAGAIN / EINTR (try again), // EBADF / EBADFD / EOPNOTSUPP / ENXIO @@ -398,6 +411,26 @@ public struct IORing: @unchecked Sendable, ~Copyable { } } + public func blockingConsumeCompletion(timeout: Duration? = nil) throws -> IOCompletion { + if let timeout { + var ts = __kernel_timespec( + tv_sec: timeout.components.seconds, + tv_nsec: timeout.components.attoseconds / 1_000_000_000 + ) + return try withUnsafePointer(to: &ts) { tsPtr in + var args = io_uring_getevents_arg( + sigmask: 0, + sigmask_sz: 0, + pad: 0, + ts: UInt64(UInt(bitPattern: tsPtr)) + ) + return try _blockingConsumeCompletionGuts(extraArgs: &args) + } + } else { + return try _blockingConsumeCompletionGuts() + } + } + public func tryConsumeCompletion() -> IOCompletion? { completionMutex.withLock { ring in return _tryConsumeCompletion(ring: &ring) diff --git a/Sources/System/ManagedIORing.swift b/Sources/System/ManagedIORing.swift index 99c112d6..b6df140d 100644 --- a/Sources/System/ManagedIORing.swift +++ b/Sources/System/ManagedIORing.swift @@ -25,7 +25,8 @@ final public class ManagedIORing: @unchecked Sendable { Task.detached { while !Task.isCancelled { //TODO: should timeout handling be sunk into IORing? - let cqe = self.internalRing.blockingConsumeCompletion() + //TODO: sort out the error handling here + let cqe = try! self.internalRing.blockingConsumeCompletion() if cqe.userData == 0 { continue @@ -34,12 +35,6 @@ final public class ManagedIORing: @unchecked Sendable { cqe.userData, to: UnsafeContinuation.self) if cqe.result < 0 { - var err = system_strerror(cqe.result * -1) - let len = system_strlen(err!) - err!.withMemoryRebound(to: UInt8.self, capacity: len) { - let errStr = String(decoding: UnsafeBufferPointer(start: $0, count: len), as: UTF8.self) - print("\(errStr)") - } handleCompletionError(cqe.result, for: cont) } else { cont.resume(returning: cqe) @@ -64,17 +59,21 @@ final public class ManagedIORing: @unchecked Sendable { entry.pointee.user_data = unsafeBitCast(cont, to: UInt64.self) if let timeout { //TODO: if IORING_FEAT_MIN_TIMEOUT is supported we can do this more efficiently - let timeoutEntry = _blockingGetSubmissionEntry( - ring: &ring, - submissionQueueEntries: internalRing.submissionQueueEntries - ) - try RawIORequest.withTimeoutRequest( - linkedTo: entry, - in: timeoutEntry, - duration: timeout, - flags: .relativeTime - ) { + if true { //replace with IORING_FEAT_MIN_TIMEOUT feature check try _submitRequests(ring: &ring, ringDescriptor: internalRing.ringDescriptor) + } else { + let timeoutEntry = _blockingGetSubmissionEntry( + ring: &ring, + submissionQueueEntries: internalRing.submissionQueueEntries + ) + try RawIORequest.withTimeoutRequest( + linkedTo: entry, + in: timeoutEntry, + duration: timeout, + flags: .relativeTime + ) { + try _submitRequests(ring: &ring, ringDescriptor: internalRing.ringDescriptor) + } } } else { try _submitRequests(ring: &ring, ringDescriptor: internalRing.ringDescriptor) diff --git a/Tests/SystemTests/IORingTests.swift b/Tests/SystemTests/IORingTests.swift index d0102a0b..2c25e2b3 100644 --- a/Tests/SystemTests/IORingTests.swift +++ b/Tests/SystemTests/IORingTests.swift @@ -15,7 +15,7 @@ final class IORingTests: XCTestCase { var ring = try IORing(queueDepth: 32) ring.writeRequest(IORequest()) try ring.submitRequests() - let completion = ring.blockingConsumeCompletion() + let completion = try ring.blockingConsumeCompletion() XCTAssertEqual(completion.result, 0) } } diff --git a/Tests/SystemTests/ManagedIORingTests.swift b/Tests/SystemTests/ManagedIORingTests.swift index 24324037..a499f6c6 100644 --- a/Tests/SystemTests/ManagedIORingTests.swift +++ b/Tests/SystemTests/ManagedIORingTests.swift @@ -17,7 +17,7 @@ final class ManagedIORingTests: XCTestCase { XCTAssertEqual(completion.result, 0) } - func testTimeout() async throws { + func testSubmitTimeout() async throws { let ring = try ManagedIORing(queueDepth: 32) var pipes: (Int32, Int32) = (0, 0) withUnsafeMutableBytes(of: &pipes) { ptr in From e5fdf9ea96ff918aac8d8f1667909c74c1bc3b7c Mon Sep 17 00:00:00 2001 From: David Smith Date: Thu, 6 Feb 2025 00:41:30 +0000 Subject: [PATCH 21/48] Remove managed abstractions for now --- Sources/System/AsyncFileDescriptor.swift | 161 ------------------ Sources/System/ManagedIORing.swift | 97 ----------- .../AsyncFileDescriptorTests.swift | 58 ------- Tests/SystemTests/ManagedIORingTests.swift | 48 ------ 4 files changed, 364 deletions(-) delete mode 100644 Sources/System/AsyncFileDescriptor.swift delete mode 100644 Sources/System/ManagedIORing.swift delete mode 100644 Tests/SystemTests/AsyncFileDescriptorTests.swift delete mode 100644 Tests/SystemTests/ManagedIORingTests.swift diff --git a/Sources/System/AsyncFileDescriptor.swift b/Sources/System/AsyncFileDescriptor.swift deleted file mode 100644 index a76d90e5..00000000 --- a/Sources/System/AsyncFileDescriptor.swift +++ /dev/null @@ -1,161 +0,0 @@ -@_implementationOnly import CSystem - -public struct AsyncFileDescriptor: ~Copyable { - @usableFromInline var open: Bool = true - @usableFromInline let fileSlot: IORingFileSlot - @usableFromInline let ring: ManagedIORing - - public static func open( - path: FilePath, - in directory: FileDescriptor = FileDescriptor(rawValue: -100), - on ring: ManagedIORing, - mode: FileDescriptor.AccessMode, - options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), - permissions: FilePermissions? = nil - ) async throws -> AsyncFileDescriptor { - // todo; real error type - guard let fileSlot = ring.getFileSlot() else { - throw IORingError.missingRequiredFeatures - } - //TODO: need an async-friendly withCString - let cstr = path.withCString { - return $0 // bad - } - let res = try await ring.submit(request: IORequest( - opening: cstr, - in: directory, - into: fileSlot, - mode: mode, - options: options, - permissions: permissions - )) - if res.result < 0 { - throw Errno(rawValue: -res.result) - } - - return AsyncFileDescriptor( - fileSlot, ring: ring - ) - } - - internal init(_ fileSlot: consuming IORingFileSlot, ring: ManagedIORing) { - self.fileSlot = consume fileSlot - self.ring = ring - } - - @inlinable @inline(__always) - public consuming func close(isolation actor: isolated (any Actor)? = #isolation) async throws { - let res = try await ring.submit(request: IORequest(closing: fileSlot)) - if res.result < 0 { - throw Errno(rawValue: -res.result) - } - self.open = false - } - - @inlinable @inline(__always) - public func read( - into buffer: inout UnsafeMutableRawBufferPointer, - atAbsoluteOffset offset: UInt64 = UInt64.max, - isolation actor: isolated (any Actor)? = #isolation - ) async throws -> UInt32 { - let res = try await ring.submit(request: IORequest( - reading: fileSlot, - into: buffer, - at: offset - )) - if res.result < 0 { - throw Errno(rawValue: -res.result) - } else { - return UInt32(bitPattern: res.result) - } - } - - @inlinable @inline(__always) - public func read( - into buffer: IORingBuffer, //TODO: should be inout? - atAbsoluteOffset offset: UInt64 = UInt64.max, - isolation actor: isolated (any Actor)? = #isolation - ) async throws -> UInt32 { - let res = try await ring.submit(request: IORequest( - reading: fileSlot, - into: buffer, - at: offset - )) - if res.result < 0 { - throw Errno(rawValue: -res.result) - } else { - return UInt32(bitPattern: res.result) - } - } - - //TODO: temporary workaround until AsyncSequence supports ~Copyable - public consuming func toBytes() -> AsyncFileDescriptorSequence { - AsyncFileDescriptorSequence(self) - } - - //TODO: can we do the linear types thing and error if they don't consume it manually? - // deinit { - // if self.open { - // close() - // // TODO: close or error? TBD - // } - // } -} - -public class AsyncFileDescriptorSequence: AsyncSequence { - var descriptor: AsyncFileDescriptor? - - public func makeAsyncIterator() -> FileIterator { - return .init(descriptor.take()!) - } - - internal init(_ descriptor: consuming AsyncFileDescriptor) { - self.descriptor = consume descriptor - } - - public typealias AsyncIterator = FileIterator - public typealias Element = UInt8 -} - -//TODO: only a class due to ~Copyable limitations -public class FileIterator: AsyncIteratorProtocol { - @usableFromInline let file: AsyncFileDescriptor - @usableFromInline var buffer: IORingBuffer - @usableFromInline var done: Bool - - @usableFromInline internal var currentByte: UnsafeRawPointer? - @usableFromInline internal var lastByte: UnsafeRawPointer? - - init(_ file: consuming AsyncFileDescriptor) { - self.buffer = file.ring.getBuffer()! - self.file = file - self.done = false - } - - @inlinable @inline(__always) - public func nextBuffer() async throws { - let bytesRead = Int(try await file.read(into: buffer)) - if _fastPath(bytesRead != 0) { - let unsafeBuffer = buffer.unsafeBuffer - let bufPointer = unsafeBuffer.baseAddress.unsafelyUnwrapped - self.currentByte = UnsafeRawPointer(bufPointer) - self.lastByte = UnsafeRawPointer(bufPointer.advanced(by: bytesRead)) - } else { - done = true - } - } - - @inlinable @inline(__always) - public func next() async throws -> UInt8? { - if _fastPath(currentByte != lastByte) { - // SAFETY: both pointers should be non-nil if they're not equal - let byte = currentByte.unsafelyUnwrapped.load(as: UInt8.self) - currentByte = currentByte.unsafelyUnwrapped + 1 - return byte - } else if done { - return nil - } - try await nextBuffer() - return try await next() - } -} diff --git a/Sources/System/ManagedIORing.swift b/Sources/System/ManagedIORing.swift deleted file mode 100644 index b6df140d..00000000 --- a/Sources/System/ManagedIORing.swift +++ /dev/null @@ -1,97 +0,0 @@ -fileprivate func handleCompletionError( - _ result: Int32, - for continuation: UnsafeContinuation) { - var error: IORingError = .unknown - switch result { - case -(_ECANCELED): - error = .operationCanceled - default: - error = .unknown - } - continuation.resume(throwing: error) -} - -final public class ManagedIORing: @unchecked Sendable { - var internalRing: IORing - - public init(queueDepth: UInt32) throws { - self.internalRing = try IORing(queueDepth: queueDepth) - self.internalRing.registerBuffers(bufSize: 655336, count: 4) - self.internalRing.registerFiles(count: 32) - self.startWaiter() - } - - private func startWaiter() { - Task.detached { - while !Task.isCancelled { - //TODO: should timeout handling be sunk into IORing? - //TODO: sort out the error handling here - let cqe = try! self.internalRing.blockingConsumeCompletion() - - if cqe.userData == 0 { - continue - } - let cont = unsafeBitCast( - cqe.userData, to: UnsafeContinuation.self) - - if cqe.result < 0 { - handleCompletionError(cqe.result, for: cont) - } else { - cont.resume(returning: cqe) - } - } - } - } - - public func submit( - request: __owned IORequest, - timeout: Duration? = nil, - isolation actor: isolated (any Actor)? = #isolation - ) async throws -> IOCompletion { - var consumeOnceWorkaround: IORequest? = request - return try await withUnsafeThrowingContinuation { cont in - do { - try internalRing.submissionMutex.withLock { ring in - let request = consumeOnceWorkaround.take()! - let entry = _blockingGetSubmissionEntry( - ring: &ring, submissionQueueEntries: internalRing.submissionQueueEntries) - entry.pointee = request.makeRawRequest().rawValue - entry.pointee.user_data = unsafeBitCast(cont, to: UInt64.self) - if let timeout { - //TODO: if IORING_FEAT_MIN_TIMEOUT is supported we can do this more efficiently - if true { //replace with IORING_FEAT_MIN_TIMEOUT feature check - try _submitRequests(ring: &ring, ringDescriptor: internalRing.ringDescriptor) - } else { - let timeoutEntry = _blockingGetSubmissionEntry( - ring: &ring, - submissionQueueEntries: internalRing.submissionQueueEntries - ) - try RawIORequest.withTimeoutRequest( - linkedTo: entry, - in: timeoutEntry, - duration: timeout, - flags: .relativeTime - ) { - try _submitRequests(ring: &ring, ringDescriptor: internalRing.ringDescriptor) - } - } - } else { - try _submitRequests(ring: &ring, ringDescriptor: internalRing.ringDescriptor) - } - } - } catch (let e) { - cont.resume(throwing: e) - } - } - - } - - internal func getFileSlot() -> IORingFileSlot? { - internalRing.getFile() - } - - internal func getBuffer() -> IORingBuffer? { - internalRing.getBuffer() - } - -} diff --git a/Tests/SystemTests/AsyncFileDescriptorTests.swift b/Tests/SystemTests/AsyncFileDescriptorTests.swift deleted file mode 100644 index ebd308fc..00000000 --- a/Tests/SystemTests/AsyncFileDescriptorTests.swift +++ /dev/null @@ -1,58 +0,0 @@ -import XCTest - -#if SYSTEM_PACKAGE -import SystemPackage -#else -import System -#endif - -final class AsyncFileDescriptorTests: XCTestCase { - func testOpen() async throws { - let ring = try ManagedIORing(queueDepth: 32) - _ = try await AsyncFileDescriptor.open( - path: "/dev/zero", - on: ring, - mode: .readOnly - ) - } - - func testOpenClose() async throws { - let ring = try ManagedIORing(queueDepth: 32) - let file = try await AsyncFileDescriptor.open( - path: "/dev/zero", - on: ring, - mode: .readOnly - ) - try await file.close() - } - - func testDevNullEmpty() async throws { - let ring = try ManagedIORing(queueDepth: 32) - let file = try await AsyncFileDescriptor.open( - path: "/dev/null", - on: ring, - mode: .readOnly - ) - for try await _ in file.toBytes() { - XCTFail("/dev/null should be empty") - } - } - - func testRead() async throws { - let ring = try ManagedIORing(queueDepth: 32) - let file = try await AsyncFileDescriptor.open( - path: "/dev/zero", - on: ring, - mode: .readOnly - ) - let bytes = file.toBytes() - var counter = 0 - for try await byte in bytes { - XCTAssert(byte == 0) - counter &+= 1 - if counter > 16384 { - break - } - } - } -} diff --git a/Tests/SystemTests/ManagedIORingTests.swift b/Tests/SystemTests/ManagedIORingTests.swift deleted file mode 100644 index a499f6c6..00000000 --- a/Tests/SystemTests/ManagedIORingTests.swift +++ /dev/null @@ -1,48 +0,0 @@ -import XCTest - -#if SYSTEM_PACKAGE -import SystemPackage -#else -import System -#endif - -final class ManagedIORingTests: XCTestCase { - func testInit() throws { - _ = try ManagedIORing(queueDepth: 32) - } - - func testNop() async throws { - let ring = try ManagedIORing(queueDepth: 32) - let completion = try await ring.submit(request: IORequest()) - XCTAssertEqual(completion.result, 0) - } - - func testSubmitTimeout() async throws { - let ring = try ManagedIORing(queueDepth: 32) - var pipes: (Int32, Int32) = (0, 0) - withUnsafeMutableBytes(of: &pipes) { ptr in - ptr.withMemoryRebound(to: UInt32.self) { tptr in - let res = pipe(tptr.baseAddress!) - XCTAssertEqual(res, 0) - } - } - let buffer = UnsafeMutableRawBufferPointer.allocate(byteCount: 128, alignment: 16) - do { - let completion = try await ring.submit( - request: IORequest(reading: FileDescriptor(rawValue: pipes.0), into: buffer), - timeout: .seconds(0.1) - ) - print("\(completion)") - XCTFail("An error should be thrown") - } catch (let e) { - if let err = e as? IORingError { - XCTAssertEqual(err, .operationCanceled) - } else { - XCTFail() - } - buffer.deallocate() - close(pipes.0) - close(pipes.1) - } - } -} From fdbcecab0bf531d5f739f5414969599b9be663aa Mon Sep 17 00:00:00 2001 From: David Smith Date: Thu, 6 Feb 2025 00:42:21 +0000 Subject: [PATCH 22/48] Eliminate internal locking and add multiple consume support --- Sources/CSystem/include/io_uring.h | 10 +- Sources/System/IORing.swift | 223 ++++++++++++++++++----------- 2 files changed, 150 insertions(+), 83 deletions(-) diff --git a/Sources/CSystem/include/io_uring.h b/Sources/CSystem/include/io_uring.h index 5c05ed8b..5cab757c 100644 --- a/Sources/CSystem/include/io_uring.h +++ b/Sources/CSystem/include/io_uring.h @@ -45,11 +45,17 @@ int io_uring_setup(unsigned int entries, struct io_uring_params *p) return syscall(__NR_io_uring_setup, entries, p); } +int io_uring_enter2(int fd, unsigned int to_submit, unsigned int min_complete, + unsigned int flags, void *args, size_t sz) +{ + return syscall(__NR_io_uring_enter, fd, to_submit, min_complete, + flags, args, _NSIG / 8); +} + int io_uring_enter(int fd, unsigned int to_submit, unsigned int min_complete, unsigned int flags, sigset_t *sig) { - return syscall(__NR_io_uring_enter, fd, to_submit, min_complete, - flags, sig, _NSIG / 8); + return io_uring_enter2(fd, to_submit, min_complete, flags, sig, _NSIG / 8); } #endif diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 872d9960..0328538f 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -51,7 +51,7 @@ internal final class ResourceManager: @unchecked Sendable { struct Resources { let resourceList: UnsafeMutableBufferPointer - var freeList: [Int] //TODO: bitvector? + var freeList: [Int] //TODO: bitvector? } let mutex: Mutex @@ -125,21 +125,27 @@ extension IORingBuffer { } @inlinable @inline(__always) -internal func _writeRequest(_ request: __owned RawIORequest, ring: inout SQRing, submissionQueueEntries: UnsafeMutableBufferPointer) +internal func _writeRequest( + _ request: __owned RawIORequest, ring: inout SQRing, + submissionQueueEntries: UnsafeMutableBufferPointer +) -> Bool { - let entry = _blockingGetSubmissionEntry(ring: &ring, submissionQueueEntries: submissionQueueEntries) + let entry = _blockingGetSubmissionEntry( + ring: &ring, submissionQueueEntries: submissionQueueEntries) entry.pointee = request.rawValue return true } @inlinable @inline(__always) -internal func _blockingGetSubmissionEntry(ring: inout SQRing, submissionQueueEntries: UnsafeMutableBufferPointer) -> UnsafeMutablePointer< +internal func _blockingGetSubmissionEntry( + ring: inout SQRing, submissionQueueEntries: UnsafeMutableBufferPointer +) -> UnsafeMutablePointer< io_uring_sqe > { while true { if let entry = _getSubmissionEntry( - ring: &ring, + ring: &ring, submissionQueueEntries: submissionQueueEntries ) { return entry @@ -152,12 +158,12 @@ internal func _blockingGetSubmissionEntry(ring: inout SQRing, submissionQueueEnt //TODO: omitting signal mask for now //Tell the kernel that we've submitted requests and/or are waiting for completions internal func _enter( - ringDescriptor: Int32, + ringDescriptor: Int32, numEvents: UInt32, minCompletions: UInt32, flags: UInt32 ) throws -> Int32 { - // Ring always needs enter right now; + // Ring always needs enter right now; // TODO: support SQPOLL here while true { let ret = io_uring_enter(ringDescriptor, numEvents, minCompletions, flags, nil) @@ -180,32 +186,36 @@ internal func _enter( } } -internal func _submitRequests(ring: inout SQRing, ringDescriptor: Int32) throws { - let flushedEvents = _flushQueue(ring: &ring) - _ = try _enter(ringDescriptor: ringDescriptor, numEvents: flushedEvents, minCompletions: 0, flags: 0) +internal func _submitRequests(ring: borrowing SQRing, ringDescriptor: Int32) throws { + let flushedEvents = _flushQueue(ring: ring) + _ = try _enter( + ringDescriptor: ringDescriptor, numEvents: flushedEvents, minCompletions: 0, flags: 0) } -internal func _getUnconsumedSubmissionCount(ring: inout SQRing) -> UInt32 { +internal func _getUnconsumedSubmissionCount(ring: borrowing SQRing) -> UInt32 { return ring.userTail - ring.kernelHead.pointee.load(ordering: .acquiring) } -internal func _getUnconsumedCompletionCount(ring: inout CQRing) -> UInt32 { - return ring.kernelTail.pointee.load(ordering: .acquiring) - ring.kernelHead.pointee.load(ordering: .acquiring) +internal func _getUnconsumedCompletionCount(ring: borrowing CQRing) -> UInt32 { + return ring.kernelTail.pointee.load(ordering: .acquiring) + - ring.kernelHead.pointee.load(ordering: .acquiring) } //TODO: pretty sure this is supposed to do more than it does -internal func _flushQueue(ring: inout SQRing) -> UInt32 { +internal func _flushQueue(ring: borrowing SQRing) -> UInt32 { ring.kernelTail.pointee.store( ring.userTail, ordering: .releasing ) - return _getUnconsumedSubmissionCount(ring: &ring) + return _getUnconsumedSubmissionCount(ring: ring) } @usableFromInline @inline(__always) -internal func _getSubmissionEntry(ring: inout SQRing, submissionQueueEntries: UnsafeMutableBufferPointer) -> UnsafeMutablePointer< +internal func _getSubmissionEntry( + ring: inout SQRing, submissionQueueEntries: UnsafeMutableBufferPointer +) -> UnsafeMutablePointer< io_uring_sqe >? { - let next = ring.userTail &+ 1 //this is expected to wrap + let next = ring.userTail &+ 1 //this is expected to wrap // FEAT: smp load when SQPOLL in use (not in MVP) let kernelHead = ring.kernelHead.pointee.load(ordering: .acquiring) @@ -227,15 +237,15 @@ internal func _getSubmissionEntry(ring: inout SQRing, submissionQueueEntries: Un return nil } -public struct IORing: @unchecked Sendable, ~Copyable { +public struct IORing: ~Copyable { let ringFlags: UInt32 let ringDescriptor: Int32 - @usableFromInline let submissionMutex: Mutex + @usableFromInline var submissionRing: SQRing // FEAT: set this eventually let submissionPolling: Bool = false - let completionMutex: Mutex + let completionRing: CQRing @usableFromInline let submissionQueueEntries: UnsafeMutableBufferPointer @@ -362,82 +372,136 @@ public struct IORing: @unchecked Sendable, ~Copyable { ) ) - self.submissionMutex = Mutex(submissionRing) - self.completionMutex = Mutex(completionRing) + self.submissionRing = submissionRing + self.completionRing = completionRing self.ringFlags = params.flags } private func _blockingConsumeCompletionGuts( - extraArgs: UnsafeMutablePointer? = nil - ) throws(IORingError) -> IOCompletion { - completionMutex.withLock { ring in - if let completion = _tryConsumeCompletion(ring: &ring) { - return completion - } else { - while true { - var sz = 0 - if extraArgs != nil { - sz = MemoryLayout.size - } - let res = io_uring_enter2( - ringDescriptor, - 0, - 1, - IORING_ENTER_GETEVENTS, - extraArgs, - sz - ) - // error handling: - // EAGAIN / EINTR (try again), - // EBADF / EBADFD / EOPNOTSUPP / ENXIO - // (failure in ring lifetime management, fatal), - // EINVAL (bad constant flag?, fatal), - // EFAULT (bad address for argument from library, fatal) - // EBUSY (not enough space for events; implies events filled - // by kernel between kernelTail load and now) - if res >= 0 || res == -EBUSY { - break - } else if res == -EAGAIN || res == -EINTR { - continue - } - fatalError( - "fatal error in receiving requests: " - + Errno(rawValue: -res).debugDescription - ) + minimumCount: UInt32, + extraArgs: UnsafeMutablePointer? = nil, + consumer: (IOCompletion?, IORingError?, Bool) throws -> Void + ) rethrows { + var count = 0 + while let completion = _tryConsumeCompletion(ring: completionRing) { + count += 1 + try consumer(completion, nil, false) + } + + if count < minimumCount { + while count < minimumCount { + var sz = 0 + if extraArgs != nil { + sz = MemoryLayout.size + } + let res = io_uring_enter2( + ringDescriptor, + 0, + minimumCount, + IORING_ENTER_GETEVENTS, + extraArgs, + sz + ) + // error handling: + // EAGAIN / EINTR (try again), + // EBADF / EBADFD / EOPNOTSUPP / ENXIO + // (failure in ring lifetime management, fatal), + // EINVAL (bad constant flag?, fatal), + // EFAULT (bad address for argument from library, fatal) + // EBUSY (not enough space for events; implies events filled + // by kernel between kernelTail load and now) + if res >= 0 || res == -EBUSY { + break + } else if res == -EAGAIN || res == -EINTR { + continue + } + fatalError( + "fatal error in receiving requests: " + + Errno(rawValue: -res).debugDescription + ) + while let completion = _tryConsumeCompletion(ring: completionRing) { + try consumer(completion, nil, false) } - return _tryConsumeCompletion(ring: &ring).unsafelyUnwrapped } + try consumer(nil, nil, true) } } - public func blockingConsumeCompletion(timeout: Duration? = nil) throws -> IOCompletion { + internal func _blockingConsumeOneCompletion( + extraArgs: UnsafeMutablePointer? = nil + ) throws -> IOCompletion { + var result: IOCompletion? = nil + try _blockingConsumeCompletionGuts(minimumCount: 1, extraArgs: extraArgs) { + (completion, error, done) in + if let error { + throw error + } + if let completion { + result = completion + } + } + return result.unsafelyUnwrapped + } + + public func blockingConsumeCompletion( + timeout: Duration? = nil + ) throws -> IOCompletion { if let timeout { var ts = __kernel_timespec( - tv_sec: timeout.components.seconds, - tv_nsec: timeout.components.attoseconds / 1_000_000_000 + tv_sec: timeout.components.seconds, + tv_nsec: timeout.components.attoseconds / 1_000_000_000 ) return try withUnsafePointer(to: &ts) { tsPtr in var args = io_uring_getevents_arg( - sigmask: 0, - sigmask_sz: 0, - pad: 0, + sigmask: 0, + sigmask_sz: 0, + pad: 0, ts: UInt64(UInt(bitPattern: tsPtr)) ) - return try _blockingConsumeCompletionGuts(extraArgs: &args) + return try _blockingConsumeOneCompletion(extraArgs: &args) } } else { - return try _blockingConsumeCompletionGuts() + return try _blockingConsumeOneCompletion() } } - public func tryConsumeCompletion() -> IOCompletion? { - completionMutex.withLock { ring in - return _tryConsumeCompletion(ring: &ring) + public func blockingConsumeCompletions( + minimumCount: UInt32 = 1, + timeout: Duration? = nil, + consumer: (IOCompletion?, IORingError?, Bool) throws -> Void + ) throws { + var x = Glibc.stat() + let y = x.st_size + if let timeout { + var ts = __kernel_timespec( + tv_sec: timeout.components.seconds, + tv_nsec: timeout.components.attoseconds / 1_000_000_000 + ) + return try withUnsafePointer(to: &ts) { tsPtr in + var args = io_uring_getevents_arg( + sigmask: 0, + sigmask_sz: 0, + pad: 0, + ts: UInt64(UInt(bitPattern: tsPtr)) + ) + try _blockingConsumeCompletionGuts( + minimumCount: minimumCount, extraArgs: &args, consumer: consumer) + } + } else { + try _blockingConsumeCompletionGuts(minimumCount: minimumCount, consumer: consumer) } } - func _tryConsumeCompletion(ring: inout CQRing) -> IOCompletion? { + // public func peekNextCompletion() -> IOCompletion { + + // } + + public func tryConsumeCompletion() -> IOCompletion? { + return _tryConsumeCompletion(ring: completionRing) + } + + func _tryConsumeCompletion(ring: borrowing CQRing) -> IOCompletion? { let tail = ring.kernelTail.pointee.load(ordering: .acquiring) let head = ring.kernelHead.pointee.load(ordering: .acquiring) @@ -459,9 +523,9 @@ public struct IORing: @unchecked Sendable, ~Copyable { var rawfd = descriptor.rawValue let result = withUnsafePointer(to: &rawfd) { fdptr in return io_uring_register( - ring.ringDescriptor, + ring.ringDescriptor, IORING_REGISTER_EVENTFD, - UnsafeMutableRawPointer(mutating: fdptr), + UnsafeMutableRawPointer(mutating: fdptr), 1 ) } @@ -470,9 +534,9 @@ public struct IORing: @unchecked Sendable, ~Copyable { public mutating func unregisterEventFD(ring: inout IORing) throws { let result = io_uring_register( - ring.ringDescriptor, + ring.ringDescriptor, IORING_UNREGISTER_EVENTFD, - nil, + nil, 0 ) try handleRegistrationResult(result) @@ -529,17 +593,14 @@ public struct IORing: @unchecked Sendable, ~Copyable { } public func submitRequests() throws { - try submissionMutex.withLock { ring in - try _submitRequests(ring: &ring, ringDescriptor: ringDescriptor) - } + try _submitRequests(ring: submissionRing, ringDescriptor: ringDescriptor) } @inlinable @inline(__always) public mutating func writeRequest(_ request: __owned IORequest) -> Bool { var raw: RawIORequest? = request.makeRawRequest() - return submissionMutex.withLock { ring in - return _writeRequest(raw.take()!, ring: &ring, submissionQueueEntries: submissionQueueEntries) - } + return _writeRequest( + raw.take()!, ring: &submissionRing, submissionQueueEntries: submissionQueueEntries) } deinit { From 7107a570b4ad1a37a0ff86d510bc8159fb18dbb8 Mon Sep 17 00:00:00 2001 From: David Smith Date: Tue, 11 Feb 2025 21:01:49 +0000 Subject: [PATCH 23/48] Fix import visibility --- Sources/System/IORequest.swift | 10 +++++----- Sources/System/IORing.swift | 2 -- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Sources/System/IORequest.swift b/Sources/System/IORequest.swift index 0af87176..0e1a3b06 100644 --- a/Sources/System/IORequest.swift +++ b/Sources/System/IORequest.swift @@ -1,4 +1,4 @@ -import struct CSystem.io_uring_sqe +@_implementationOnly import struct CSystem.io_uring_sqe @usableFromInline internal enum IORequestCore: ~Copyable { @@ -62,7 +62,7 @@ internal enum IORequestCore: ~Copyable { case closeSlot(IORingFileSlot) } -@inlinable @inline(__always) +@inline(__always) internal func makeRawRequest_readWrite_registered( file: FileDescriptor, buffer: IORingBuffer, @@ -76,7 +76,7 @@ internal func makeRawRequest_readWrite_registered( return request } -@inlinable @inline(__always) +@inline(__always) internal func makeRawRequest_readWrite_registered_slot( file: IORingFileSlot, buffer: IORingBuffer, @@ -104,7 +104,7 @@ internal func makeRawRequest_readWrite_unregistered( return request } -@inlinable @inline(__always) +@inline(__always) internal func makeRawRequest_readWrite_unregistered_slot( file: IORingFileSlot, buffer: UnsafeMutableRawBufferPointer, @@ -251,7 +251,7 @@ extension IORequest { fatalError("Implement me") } - @inlinable @inline(__always) + @inline(__always) public consuming func makeRawRequest() -> RawIORequest { var request = RawIORequest() switch extractCore() { diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 0328538f..472a69b2 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -471,8 +471,6 @@ public struct IORing: ~Copyable { timeout: Duration? = nil, consumer: (IOCompletion?, IORingError?, Bool) throws -> Void ) throws { - var x = Glibc.stat() - let y = x.st_size if let timeout { var ts = __kernel_timespec( tv_sec: timeout.components.seconds, From 02481e0daabc0fb342c175681347efe8f2d44337 Mon Sep 17 00:00:00 2001 From: David Smith Date: Tue, 11 Feb 2025 21:06:16 +0000 Subject: [PATCH 24/48] More import fixes --- Sources/System/IORing.swift | 12 ++++++------ Sources/System/RawIORequest.swift | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 472a69b2..6dff66f1 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -2,7 +2,7 @@ import Glibc // needed for mmap import Synchronization -import struct CSystem.io_uring_sqe +@_implementationOnly import struct CSystem.io_uring_sqe // XXX: this *really* shouldn't be here. oh well. extension UnsafeMutableRawPointer { @@ -124,7 +124,7 @@ extension IORingBuffer { } } -@inlinable @inline(__always) +@inline(__always) internal func _writeRequest( _ request: __owned RawIORequest, ring: inout SQRing, submissionQueueEntries: UnsafeMutableBufferPointer @@ -137,7 +137,7 @@ internal func _writeRequest( return true } -@inlinable @inline(__always) +@inline(__always) internal func _blockingGetSubmissionEntry( ring: inout SQRing, submissionQueueEntries: UnsafeMutableBufferPointer ) -> UnsafeMutablePointer< @@ -209,7 +209,7 @@ internal func _flushQueue(ring: borrowing SQRing) -> UInt32 { return _getUnconsumedSubmissionCount(ring: ring) } -@usableFromInline @inline(__always) +@inline(__always) internal func _getSubmissionEntry( ring: inout SQRing, submissionQueueEntries: UnsafeMutableBufferPointer ) -> UnsafeMutablePointer< @@ -247,7 +247,7 @@ public struct IORing: ~Copyable { let completionRing: CQRing - @usableFromInline let submissionQueueEntries: UnsafeMutableBufferPointer + let submissionQueueEntries: UnsafeMutableBufferPointer // kept around for unmap / cleanup let ringSize: Int @@ -594,7 +594,7 @@ public struct IORing: ~Copyable { try _submitRequests(ring: submissionRing, ringDescriptor: ringDescriptor) } - @inlinable @inline(__always) + @inline(__always) public mutating func writeRequest(_ request: __owned IORequest) -> Bool { var raw: RawIORequest? = request.makeRawRequest() return _writeRequest( diff --git a/Sources/System/RawIORequest.swift b/Sources/System/RawIORequest.swift index 5a1fd03d..5b884993 100644 --- a/Sources/System/RawIORequest.swift +++ b/Sources/System/RawIORequest.swift @@ -1,9 +1,9 @@ // TODO: investigate @usableFromInline / @_implementationOnly dichotomy @_implementationOnly import CSystem -import struct CSystem.io_uring_sqe +@_implementationOnly import struct CSystem.io_uring_sqe public struct RawIORequest: ~Copyable { - @usableFromInline var rawValue: io_uring_sqe + var rawValue: io_uring_sqe public init() { self.rawValue = io_uring_sqe() From bb03f0fba3c480c3a625c9894b2f5dc32e9885c2 Mon Sep 17 00:00:00 2001 From: David Smith Date: Tue, 11 Feb 2025 22:33:10 +0000 Subject: [PATCH 25/48] Redesign registered resources API --- Sources/System/IORing.swift | 148 ++++++++++--------------- Tests/SystemTests/IORequestTests.swift | 6 +- 2 files changed, 60 insertions(+), 94 deletions(-) diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 6dff66f1..3305a9bd 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -11,6 +11,12 @@ extension UnsafeMutableRawPointer { } } +extension UnsafeMutableRawBufferPointer { + func to_iovec() -> iovec { + iovec(iov_base: baseAddress, iov_len: count) + } +} + // all pointers in this struct reference kernel-visible memory @usableFromInline struct SQRing: ~Copyable { let kernelHead: UnsafePointer> @@ -46,67 +52,17 @@ struct CQRing: ~Copyable { let cqes: UnsafeBufferPointer } -internal final class ResourceManager: @unchecked Sendable { - typealias Resource = T - - struct Resources { - let resourceList: UnsafeMutableBufferPointer - var freeList: [Int] //TODO: bitvector? - } - - let mutex: Mutex - - init(_ res: UnsafeMutableBufferPointer) { - mutex = Mutex( - Resources( - resourceList: res, - freeList: [Int](res.indices) - )) - } - - func getResource() -> IOResource? { - mutex.withLock { resources in - if let index = resources.freeList.popLast() { - return IOResource( - resource: resources.resourceList[index], - index: index, - manager: self - ) - } else { - return nil - } - } - } - - func releaseResource(index: Int) { - mutex.withLock { resources in - resources.freeList.append(index) - } - } -} - -public class IOResource { +public struct IOResource { typealias Resource = T @usableFromInline let resource: T @usableFromInline let index: Int - let manager: ResourceManager internal init( resource: T, - index: Int, - manager: ResourceManager + index: Int ) { self.resource = resource self.index = index - self.manager = manager - } - - func withResource() { - - } - - deinit { - manager.releaseResource(index: self.index) } } @@ -253,8 +209,8 @@ public struct IORing: ~Copyable { let ringSize: Int let ringPtr: UnsafeMutableRawPointer - var registeredFiles: ResourceManager? - var registeredBuffers: ResourceManager? + var _registeredFiles: [UInt32]? + var _registeredBuffers: [iovec]? public init(queueDepth: UInt32) throws { var params = io_uring_params() @@ -517,11 +473,11 @@ public struct IORing: ~Copyable { //TODO: error handling } - public mutating func registerEventFD(ring: inout IORing, _ descriptor: FileDescriptor) throws { + public mutating func registerEventFD(_ descriptor: FileDescriptor) throws { var rawfd = descriptor.rawValue let result = withUnsafePointer(to: &rawfd) { fdptr in return io_uring_register( - ring.ringDescriptor, + ringDescriptor, IORING_REGISTER_EVENTFD, UnsafeMutableRawPointer(mutating: fdptr), 1 @@ -530,9 +486,9 @@ public struct IORing: ~Copyable { try handleRegistrationResult(result) } - public mutating func unregisterEventFD(ring: inout IORing) throws { + public mutating func unregisterEventFD() throws { let result = io_uring_register( - ring.ringDescriptor, + ringDescriptor, IORING_UNREGISTER_EVENTFD, nil, 0 @@ -540,50 +496,64 @@ public struct IORing: ~Copyable { try handleRegistrationResult(result) } - public mutating func registerFiles(count: UInt32) { - guard self.registeredFiles == nil else { fatalError() } - let fileBuf = UnsafeMutableBufferPointer.allocate(capacity: Int(count)) - fileBuf.initialize(repeating: UInt32.max) - io_uring_register( - self.ringDescriptor, - IORING_REGISTER_FILES, - fileBuf.baseAddress!, - count - ) + public mutating func registerFileSlots(count: Int) { + precondition(_registeredFiles == nil) + precondition(count < UInt32.max) + let files = [UInt32](repeating: UInt32.max, count: count) + + let regResult = files.withUnsafeBufferPointer { bPtr in + io_uring_register( + self.ringDescriptor, + IORING_REGISTER_FILES, + UnsafeMutableRawPointer(mutating:bPtr.baseAddress!), + UInt32(truncatingIfNeeded: count) + ) + } + // TODO: error handling - self.registeredFiles = ResourceManager(fileBuf) + _registeredFiles = files } public func unregisterFiles() { fatalError("failed to unregister files") } - public func getFile() -> IORingFileSlot? { - return self.registeredFiles?.getResource() + public var registeredFileSlots: some RandomAccessCollection { + RegisteredResources(resources: _registeredFiles ?? []) } - public mutating func registerBuffers(bufSize: UInt32, count: UInt32) { - let iovecs = UnsafeMutableBufferPointer.allocate(capacity: Int(count)) - let intBufSize = Int(bufSize) - for i in 0..: RandomAccessCollection { + let resources: [T] + + var startIndex: Int { 0 } + var endIndex: Int { resources.endIndex } + init(resources: [T]) { + self.resources = resources + } + subscript(position: Int) -> IOResource { + IOResource(resource: resources[position], index: position) + } } - public func getBuffer() -> IORingBuffer? { - return self.registeredBuffers?.getResource() + public var registeredBuffers: some RandomAccessCollection { + RegisteredResources(resources: _registeredBuffers ?? []) } public func unregisterBuffers() { diff --git a/Tests/SystemTests/IORequestTests.swift b/Tests/SystemTests/IORequestTests.swift index 78be4cf7..e553a546 100644 --- a/Tests/SystemTests/IORequestTests.swift +++ b/Tests/SystemTests/IORequestTests.swift @@ -28,12 +28,8 @@ final class IORequestTests: XCTestCase { } func testOpenatFixedFile() throws { - // TODO: come up with a better way of getting a FileSlot. - let buf = UnsafeMutableBufferPointer.allocate(capacity: 2) - let resmgr = ResourceManager.init(buf) - let pathPtr = UnsafePointer(bitPattern: 0x414141410badf00d)! - let fileSlot = resmgr.getResource()! + let fileSlot: IORingFileSlot = IORingFileSlot(resource: UInt32.max, index: 0) let req = IORequest( opening: pathPtr, in: FileDescriptor(rawValue: -100), From a396967e3a77b780a78b733564137be7a8b4b7dc Mon Sep 17 00:00:00 2001 From: David Smith Date: Tue, 11 Feb 2025 22:38:01 +0000 Subject: [PATCH 26/48] More registration tweaks --- Sources/System/IORing.swift | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 3305a9bd..292303b7 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -496,7 +496,7 @@ public struct IORing: ~Copyable { try handleRegistrationResult(result) } - public mutating func registerFileSlots(count: Int) { + public mutating func registerFileSlots(count: Int) -> some RandomAccessCollection { precondition(_registeredFiles == nil) precondition(count < UInt32.max) let files = [UInt32](repeating: UInt32.max, count: count) @@ -512,6 +512,7 @@ public struct IORing: ~Copyable { // TODO: error handling _registeredFiles = files + return registeredFileSlots } public func unregisterFiles() { @@ -522,9 +523,10 @@ public struct IORing: ~Copyable { RegisteredResources(resources: _registeredFiles ?? []) } - public mutating func registerBuffers(_ buffers: UnsafeMutableRawBufferPointer...) { + public mutating func registerBuffers(_ buffers: UnsafeMutableRawBufferPointer...) -> some RandomAccessCollection { precondition(buffers.count < UInt32.max) precondition(_registeredBuffers == nil) + //TODO: check if io_uring has preconditions it needs for the buffers (e.g. alignment) let iovecs = buffers.map { $0.to_iovec() } let regResult = iovecs.withUnsafeBufferPointer { bPtr in io_uring_register( @@ -537,6 +539,7 @@ public struct IORing: ~Copyable { // TODO: error handling _registeredBuffers = iovecs + return registeredBuffers } struct RegisteredResources: RandomAccessCollection { From d4ca4129a033c04efb863827e9178101434c44df Mon Sep 17 00:00:00 2001 From: David Smith Date: Tue, 11 Feb 2025 23:01:33 +0000 Subject: [PATCH 27/48] Some renaming, and implement linked requests --- Sources/System/IORequest.swift | 4 ++-- Sources/System/IORing.swift | 19 +++++++++++++++++-- Sources/System/RawIORequest.swift | 9 +++++++-- Tests/SystemTests/IORingTests.swift | 2 +- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/Sources/System/IORequest.swift b/Sources/System/IORequest.swift index 0e1a3b06..f406f5a0 100644 --- a/Sources/System/IORequest.swift +++ b/Sources/System/IORequest.swift @@ -1,7 +1,7 @@ @_implementationOnly import struct CSystem.io_uring_sqe @usableFromInline -internal enum IORequestCore: ~Copyable { +internal enum IORequestCore { case nop // nothing here case openat( atDirectory: FileDescriptor, @@ -118,7 +118,7 @@ internal func makeRawRequest_readWrite_unregistered_slot( return request } -public struct IORequest : ~Copyable { +public struct IORequest { @usableFromInline var core: IORequestCore @inlinable internal consuming func extractCore() -> IORequestCore { diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 292303b7..fab2f8e4 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -567,13 +567,28 @@ public struct IORing: ~Copyable { try _submitRequests(ring: submissionRing, ringDescriptor: ringDescriptor) } - @inline(__always) - public mutating func writeRequest(_ request: __owned IORequest) -> Bool { + public mutating func prepare(request: __owned IORequest) -> Bool { var raw: RawIORequest? = request.makeRawRequest() return _writeRequest( raw.take()!, ring: &submissionRing, submissionQueueEntries: submissionQueueEntries) } + //@inlinable //TODO: make sure the array allocation gets optimized out... + public mutating func prepare(linkedRequests: IORequest...) { + guard linkedRequests.count > 0 else { + return + } + let last = linkedRequests.last! + for req in linkedRequests.dropLast() { + var raw = req.makeRawRequest() + raw.linkToNextRequest() + _writeRequest( + raw, ring: &submissionRing, submissionQueueEntries: submissionQueueEntries) + } + _writeRequest( + last.makeRawRequest(), ring: &submissionRing, submissionQueueEntries: submissionQueueEntries) + } + deinit { munmap(ringPtr, ringSize) munmap( diff --git a/Sources/System/RawIORequest.swift b/Sources/System/RawIORequest.swift index 5b884993..40b72366 100644 --- a/Sources/System/RawIORequest.swift +++ b/Sources/System/RawIORequest.swift @@ -2,6 +2,7 @@ @_implementationOnly import CSystem @_implementationOnly import struct CSystem.io_uring_sqe +//TODO: make this internal public struct RawIORequest: ~Copyable { var rawValue: io_uring_sqe @@ -11,7 +12,7 @@ public struct RawIORequest: ~Copyable { } extension RawIORequest { - public enum Operation: UInt8 { + enum Operation: UInt8 { case nop = 0 case readv = 1 case writev = 2 @@ -53,7 +54,7 @@ extension RawIORequest { public static let skipSuccess = Flags(rawValue: 1 << 6) } - public var operation: Operation { + var operation: Operation { get { Operation(rawValue: rawValue.opcode)! } set { rawValue.opcode = newValue.rawValue } } @@ -63,6 +64,10 @@ extension RawIORequest { set { rawValue.flags = newValue.rawValue } } + public mutating func linkToNextRequest() { + flags = Flags(rawValue: flags.rawValue | Flags.linkRequest.rawValue) + } + public var fileDescriptor: FileDescriptor { get { FileDescriptor(rawValue: rawValue.fd) } set { rawValue.fd = newValue.rawValue } diff --git a/Tests/SystemTests/IORingTests.swift b/Tests/SystemTests/IORingTests.swift index 2c25e2b3..4ae12fcf 100644 --- a/Tests/SystemTests/IORingTests.swift +++ b/Tests/SystemTests/IORingTests.swift @@ -13,7 +13,7 @@ final class IORingTests: XCTestCase { func testNop() throws { var ring = try IORing(queueDepth: 32) - ring.writeRequest(IORequest()) + ring.prepare(request: IORequest()) try ring.submitRequests() let completion = try ring.blockingConsumeCompletion() XCTAssertEqual(completion.result, 0) From 5bfed03c971a5440f8c5e8489466352be8c2f298 Mon Sep 17 00:00:00 2001 From: David Smith Date: Tue, 11 Feb 2025 23:27:03 +0000 Subject: [PATCH 28/48] Switch to static methods for constructing requests --- Sources/System/IORequest.swift | 96 +++++++++++--------------- Tests/SystemTests/IORequestTests.swift | 5 +- Tests/SystemTests/IORingTests.swift | 2 +- 3 files changed, 43 insertions(+), 60 deletions(-) diff --git a/Sources/System/IORequest.swift b/Sources/System/IORequest.swift index f406f5a0..4cdda768 100644 --- a/Sources/System/IORequest.swift +++ b/Sources/System/IORequest.swift @@ -127,127 +127,111 @@ public struct IORequest { } extension IORequest { - public init() { //TODO: why do we have nop? - core = .nop + public static func nop() -> IORequest { + IORequest(core: .nop) } - public init( - reading file: IORingFileSlot, + public static func reading(_ file: IORingFileSlot, into buffer: IORingBuffer, at offset: UInt64 = 0 - ) { - core = .readSlot(file: file, buffer: buffer, offset: offset) + ) -> IORequest { + IORequest(core: .readSlot(file: file, buffer: buffer, offset: offset)) } - public init( - reading file: FileDescriptor, + public static func reading(_ file: FileDescriptor, into buffer: IORingBuffer, at offset: UInt64 = 0 - ) { - core = .read(file: file, buffer: buffer, offset: offset) + ) -> IORequest { + IORequest(core: .read(file: file, buffer: buffer, offset: offset)) } - public init( - reading file: IORingFileSlot, + public static func reading(_ file: IORingFileSlot, into buffer: UnsafeMutableRawBufferPointer, at offset: UInt64 = 0 - ) { - core = .readUnregisteredSlot(file: file, buffer: buffer, offset: offset) + ) -> IORequest { + IORequest(core: .readUnregisteredSlot(file: file, buffer: buffer, offset: offset)) } - public init( - reading file: FileDescriptor, + public static func reading(_ file: FileDescriptor, into buffer: UnsafeMutableRawBufferPointer, at offset: UInt64 = 0 - ) { - core = .readUnregistered(file: file, buffer: buffer, offset: offset) + ) -> IORequest { + IORequest(core: .readUnregistered(file: file, buffer: buffer, offset: offset)) } - public init( - writing buffer: IORingBuffer, + public static func writing(_ buffer: IORingBuffer, into file: IORingFileSlot, at offset: UInt64 = 0 - ) { - core = .writeSlot(file: file, buffer: buffer, offset: offset) + ) -> IORequest { + IORequest(core: .writeSlot(file: file, buffer: buffer, offset: offset)) } - public init( - writing buffer: IORingBuffer, + public static func writing(_ buffer: IORingBuffer, into file: FileDescriptor, at offset: UInt64 = 0 - ) { - core = .write(file: file, buffer: buffer, offset: offset) + ) -> IORequest { + IORequest(core: .write(file: file, buffer: buffer, offset: offset)) } - public init( - writing buffer: UnsafeMutableRawBufferPointer, + public static func writing(_ buffer: UnsafeMutableRawBufferPointer, into file: IORingFileSlot, at offset: UInt64 = 0 - ) { - core = .writeUnregisteredSlot(file: file, buffer: buffer, offset: offset) + ) -> IORequest { + IORequest(core: .writeUnregisteredSlot(file: file, buffer: buffer, offset: offset)) } - public init( - writing buffer: UnsafeMutableRawBufferPointer, + public static func writing(_ buffer: UnsafeMutableRawBufferPointer, into file: FileDescriptor, at offset: UInt64 = 0 - ) { - core = .writeUnregistered(file: file, buffer: buffer, offset: offset) + ) -> IORequest { + IORequest(core: .writeUnregistered(file: file, buffer: buffer, offset: offset)) } - public init( - closing file: FileDescriptor - ) { - core = .close(file) + public static func closing(_ file: FileDescriptor) -> IORequest { + IORequest(core: .close(file)) } - public init( - closing file: IORingFileSlot - ) { - core = .closeSlot(file) + public static func closing(_ file: IORingFileSlot) -> IORequest { + IORequest(core: .closeSlot(file)) } - public init( - opening path: UnsafePointer, + public static func opening(_ path: UnsafePointer, in directory: FileDescriptor, into slot: IORingFileSlot, mode: FileDescriptor.AccessMode, options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), permissions: FilePermissions? = nil - ) { - core = .openatSlot(atDirectory: directory, path: path, mode, options: options, permissions: permissions, intoSlot: slot) + ) -> IORequest { + IORequest(core :.openatSlot(atDirectory: directory, path: path, mode, options: options, permissions: permissions, intoSlot: slot)) } - public init( - opening path: UnsafePointer, + public static func opening(_ path: UnsafePointer, in directory: FileDescriptor, mode: FileDescriptor.AccessMode, options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), permissions: FilePermissions? = nil - ) { - core = .openat(atDirectory: directory, path: path, mode, options: options, permissions: permissions) + ) -> IORequest { + IORequest(core: .openat(atDirectory: directory, path: path, mode, options: options, permissions: permissions)) } - public init( - opening path: FilePath, + public static func opening(_ path: FilePath, in directory: FileDescriptor, into slot: IORingFileSlot, mode: FileDescriptor.AccessMode, options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), permissions: FilePermissions? = nil - ) { + ) -> IORequest { fatalError("Implement me") } - public init( - opening path: FilePath, + public static func opening(_ path: FilePath, in directory: FileDescriptor, mode: FileDescriptor.AccessMode, options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), permissions: FilePermissions? = nil - ) { + ) -> IORequest { fatalError("Implement me") } diff --git a/Tests/SystemTests/IORequestTests.swift b/Tests/SystemTests/IORequestTests.swift index e553a546..f9f95803 100644 --- a/Tests/SystemTests/IORequestTests.swift +++ b/Tests/SystemTests/IORequestTests.swift @@ -19,7 +19,7 @@ func requestBytes(_ request: consuming RawIORequest) -> [UInt8] { // which are known to work correctly. final class IORequestTests: XCTestCase { func testNop() { - let req = IORequest().makeRawRequest() + let req = IORequest.nop().makeRawRequest() let sourceBytes = requestBytes(req) // convenient property of nop: it's all zeros! // for some unknown reason, liburing sets the fd field to -1. @@ -30,8 +30,7 @@ final class IORequestTests: XCTestCase { func testOpenatFixedFile() throws { let pathPtr = UnsafePointer(bitPattern: 0x414141410badf00d)! let fileSlot: IORingFileSlot = IORingFileSlot(resource: UInt32.max, index: 0) - let req = IORequest( - opening: pathPtr, + let req = IORequest.opening(pathPtr, in: FileDescriptor(rawValue: -100), into: fileSlot, mode: .readOnly, diff --git a/Tests/SystemTests/IORingTests.swift b/Tests/SystemTests/IORingTests.swift index 4ae12fcf..6421e0ca 100644 --- a/Tests/SystemTests/IORingTests.swift +++ b/Tests/SystemTests/IORingTests.swift @@ -13,7 +13,7 @@ final class IORingTests: XCTestCase { func testNop() throws { var ring = try IORing(queueDepth: 32) - ring.prepare(request: IORequest()) + ring.prepare(request: IORequest.nop()) try ring.submitRequests() let completion = try ring.blockingConsumeCompletion() XCTAssertEqual(completion.result, 0) From 74366c33418fafe8268d225f56046590ef01d083 Mon Sep 17 00:00:00 2001 From: David Smith Date: Wed, 12 Feb 2025 20:18:51 +0000 Subject: [PATCH 29/48] Improve submit API --- Sources/System/IORing.swift | 15 ++++++++++++--- Tests/SystemTests/IORingTests.swift | 3 +-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index fab2f8e4..e238faba 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -563,7 +563,7 @@ public struct IORing: ~Copyable { fatalError("failed to unregister buffers: TODO") } - public func submitRequests() throws { + public func submitPreparedRequests() throws { try _submitRequests(ring: submissionRing, ringDescriptor: ringDescriptor) } @@ -573,8 +573,7 @@ public struct IORing: ~Copyable { raw.take()!, ring: &submissionRing, submissionQueueEntries: submissionQueueEntries) } - //@inlinable //TODO: make sure the array allocation gets optimized out... - public mutating func prepare(linkedRequests: IORequest...) { + mutating func prepare(linkedRequests: some BidirectionalCollection) { guard linkedRequests.count > 0 else { return } @@ -589,6 +588,16 @@ public struct IORing: ~Copyable { last.makeRawRequest(), ring: &submissionRing, submissionQueueEntries: submissionQueueEntries) } + //@inlinable //TODO: make sure the array allocation gets optimized out... + public mutating func prepare(linkedRequests: IORequest...) { + prepare(linkedRequests: linkedRequests) + } + + public mutating func submit(linkedRequests: IORequest...) throws { + prepare(linkedRequests: linkedRequests) + try submitPreparedRequests() + } + deinit { munmap(ringPtr, ringSize) munmap( diff --git a/Tests/SystemTests/IORingTests.swift b/Tests/SystemTests/IORingTests.swift index 6421e0ca..306516be 100644 --- a/Tests/SystemTests/IORingTests.swift +++ b/Tests/SystemTests/IORingTests.swift @@ -13,8 +13,7 @@ final class IORingTests: XCTestCase { func testNop() throws { var ring = try IORing(queueDepth: 32) - ring.prepare(request: IORequest.nop()) - try ring.submitRequests() + try ring.submit(linkedRequests: IORequest.nop()) let completion = try ring.blockingConsumeCompletion() XCTAssertEqual(completion.result, 0) } From 7f6e673750128d8b9a47934b83e0b0c523921b56 Mon Sep 17 00:00:00 2001 From: David Smith Date: Wed, 12 Feb 2025 23:43:29 +0000 Subject: [PATCH 30/48] Adjust registered resources API --- Sources/System/IORing.swift | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index e238faba..0500be09 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -53,7 +53,7 @@ struct CQRing: ~Copyable { } public struct IOResource { - typealias Resource = T + public typealias Resource = T @usableFromInline let resource: T @usableFromInline let index: Int @@ -496,7 +496,7 @@ public struct IORing: ~Copyable { try handleRegistrationResult(result) } - public mutating func registerFileSlots(count: Int) -> some RandomAccessCollection { + public mutating func registerFileSlots(count: Int) -> RegisteredResources { precondition(_registeredFiles == nil) precondition(count < UInt32.max) let files = [UInt32](repeating: UInt32.max, count: count) @@ -519,11 +519,11 @@ public struct IORing: ~Copyable { fatalError("failed to unregister files") } - public var registeredFileSlots: some RandomAccessCollection { + public var registeredFileSlots: RegisteredResources { RegisteredResources(resources: _registeredFiles ?? []) } - public mutating func registerBuffers(_ buffers: UnsafeMutableRawBufferPointer...) -> some RandomAccessCollection { + public mutating func registerBuffers(_ buffers: UnsafeMutableRawBufferPointer...) -> RegisteredResources { precondition(buffers.count < UInt32.max) precondition(_registeredBuffers == nil) //TODO: check if io_uring has preconditions it needs for the buffers (e.g. alignment) @@ -542,20 +542,23 @@ public struct IORing: ~Copyable { return registeredBuffers } - struct RegisteredResources: RandomAccessCollection { + public struct RegisteredResources: RandomAccessCollection { let resources: [T] - var startIndex: Int { 0 } - var endIndex: Int { resources.endIndex } + public var startIndex: Int { 0 } + public var endIndex: Int { resources.endIndex } init(resources: [T]) { self.resources = resources } - subscript(position: Int) -> IOResource { + public subscript(position: Int) -> IOResource { IOResource(resource: resources[position], index: position) } + public subscript(position: Int16) -> IOResource { + IOResource(resource: resources[Int(position)], index: Int(position)) + } } - public var registeredBuffers: some RandomAccessCollection { + public var registeredBuffers: RegisteredResources { RegisteredResources(resources: _registeredBuffers ?? []) } From 0c6ef16ca91cd82ab78a52a5cac83b113ba8e67d Mon Sep 17 00:00:00 2001 From: David Smith Date: Wed, 12 Feb 2025 23:45:18 +0000 Subject: [PATCH 31/48] Fix type --- Sources/System/IORing.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 0500be09..fb0c20aa 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -553,7 +553,7 @@ public struct IORing: ~Copyable { public subscript(position: Int) -> IOResource { IOResource(resource: resources[position], index: position) } - public subscript(position: Int16) -> IOResource { + public subscript(position: UInt16) -> IOResource { IOResource(resource: resources[Int(position)], index: Int(position)) } } From a22e5f685866614a027ff4a394e2af71d4b412c8 Mon Sep 17 00:00:00 2001 From: David Smith Date: Thu, 13 Feb 2025 22:09:42 +0000 Subject: [PATCH 32/48] Add a version of registerBuffers that isn't varargs --- Sources/System/IORing.swift | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index fb0c20aa..b7189917 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -496,16 +496,18 @@ public struct IORing: ~Copyable { try handleRegistrationResult(result) } - public mutating func registerFileSlots(count: Int) -> RegisteredResources { + public mutating func registerFileSlots(count: Int) -> RegisteredResources< + IORingFileSlot.Resource + > { precondition(_registeredFiles == nil) precondition(count < UInt32.max) - let files = [UInt32](repeating: UInt32.max, count: count) + let files = [UInt32](repeating: UInt32.max, count: count) let regResult = files.withUnsafeBufferPointer { bPtr in io_uring_register( self.ringDescriptor, IORING_REGISTER_FILES, - UnsafeMutableRawPointer(mutating:bPtr.baseAddress!), + UnsafeMutableRawPointer(mutating: bPtr.baseAddress!), UInt32(truncatingIfNeeded: count) ) } @@ -523,7 +525,9 @@ public struct IORing: ~Copyable { RegisteredResources(resources: _registeredFiles ?? []) } - public mutating func registerBuffers(_ buffers: UnsafeMutableRawBufferPointer...) -> RegisteredResources { + public mutating func registerBuffers(_ buffers: some Collection) + -> RegisteredResources + { precondition(buffers.count < UInt32.max) precondition(_registeredBuffers == nil) //TODO: check if io_uring has preconditions it needs for the buffers (e.g. alignment) @@ -542,6 +546,12 @@ public struct IORing: ~Copyable { return registeredBuffers } + public mutating func registerBuffers(_ buffers: UnsafeMutableRawBufferPointer...) + -> RegisteredResources + { + registerBuffers(buffers) + } + public struct RegisteredResources: RandomAccessCollection { let resources: [T] @@ -588,7 +598,8 @@ public struct IORing: ~Copyable { raw, ring: &submissionRing, submissionQueueEntries: submissionQueueEntries) } _writeRequest( - last.makeRawRequest(), ring: &submissionRing, submissionQueueEntries: submissionQueueEntries) + last.makeRawRequest(), ring: &submissionRing, + submissionQueueEntries: submissionQueueEntries) } //@inlinable //TODO: make sure the array allocation gets optimized out... From 6983196ad6f980fde158811e13a828d70b7a93f0 Mon Sep 17 00:00:00 2001 From: David Smith Date: Thu, 13 Feb 2025 23:31:06 +0000 Subject: [PATCH 33/48] Add unlinkAt support --- Sources/System/IORequest.swift | 14 ++++++++++++++ Sources/System/RawIORequest.swift | 1 + 2 files changed, 15 insertions(+) diff --git a/Sources/System/IORequest.swift b/Sources/System/IORequest.swift index 4cdda768..8705f0d6 100644 --- a/Sources/System/IORequest.swift +++ b/Sources/System/IORequest.swift @@ -60,6 +60,10 @@ internal enum IORequestCore { ) case close(FileDescriptor) case closeSlot(IORingFileSlot) + case unlinkAt( + atDirectory: FileDescriptor, + path: UnsafePointer + ) } @inline(__always) @@ -235,6 +239,12 @@ extension IORequest { fatalError("Implement me") } + public static func unlinking(_ path: UnsafePointer, + in directory: FileDescriptor + ) -> IORequest { + IORequest(core: .unlinkAt(atDirectory: directory, path: path)) + } + @inline(__always) public consuming func makeRawRequest() -> RawIORequest { var request = RawIORequest() @@ -293,6 +303,10 @@ extension IORequest { case .closeSlot(let file): request.operation = .close request.rawValue.file_index = UInt32(file.index + 1) + case .unlinkAt(let atDirectory, let path): + request.operation = .unlinkAt + request.fileDescriptor = atDirectory + request.rawValue.addr = UInt64(UInt(bitPattern: path)) } return request } diff --git a/Sources/System/RawIORequest.swift b/Sources/System/RawIORequest.swift index 40b72366..8f1a2b1b 100644 --- a/Sources/System/RawIORequest.swift +++ b/Sources/System/RawIORequest.swift @@ -36,6 +36,7 @@ extension RawIORequest { // ... case openAt2 = 28 // ... + case unlinkAt = 36 } public struct Flags: OptionSet, Hashable, Codable { From 5ba137703c63791b5088ac44ef1ebd2662566d0a Mon Sep 17 00:00:00 2001 From: David Smith Date: Thu, 13 Feb 2025 23:43:19 +0000 Subject: [PATCH 34/48] Dubious approach to this, but I want to try it out a bit --- Sources/System/RawIORequest.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Sources/System/RawIORequest.swift b/Sources/System/RawIORequest.swift index 8f1a2b1b..78f4f6de 100644 --- a/Sources/System/RawIORequest.swift +++ b/Sources/System/RawIORequest.swift @@ -101,6 +101,7 @@ extension RawIORequest { // TODO: cleanup? rawValue.addr = UInt64(Int(bitPattern: newValue.baseAddress!)) rawValue.len = UInt32(exactly: newValue.count)! + rawValue.user_data = rawValue.addr //TODO: this is kind of a hack, but I need to decide how best to get the buffer out on the other side } } From 006464963162f26a722f600b975a5ad5145e7150 Mon Sep 17 00:00:00 2001 From: David Smith Date: Fri, 14 Feb 2025 18:24:09 +0000 Subject: [PATCH 35/48] Turn on single issuer as an experiment --- Sources/System/IORing.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index b7189917..5dd53d37 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -214,6 +214,7 @@ public struct IORing: ~Copyable { public init(queueDepth: UInt32) throws { var params = io_uring_params() + params.flags = IORING_SETUP_SINGLE_ISSUER //TODO make this configurable ringDescriptor = withUnsafeMutablePointer(to: ¶ms) { return io_uring_setup(queueDepth, $0) From 901f4c80c48dd983ae87b115736f0589d6963b73 Mon Sep 17 00:00:00 2001 From: David Smith Date: Fri, 14 Feb 2025 18:25:46 +0000 Subject: [PATCH 36/48] Revert "Turn on single issuer as an experiment" This reverts commit 006464963162f26a722f600b975a5ad5145e7150. --- Sources/System/IORing.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 5dd53d37..b7189917 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -214,7 +214,6 @@ public struct IORing: ~Copyable { public init(queueDepth: UInt32) throws { var params = io_uring_params() - params.flags = IORING_SETUP_SINGLE_ISSUER //TODO make this configurable ringDescriptor = withUnsafeMutablePointer(to: ¶ms) { return io_uring_setup(queueDepth, $0) From 283e8d6b7b747f082ed2b7c9bd763229729d9d76 Mon Sep 17 00:00:00 2001 From: David Smith Date: Fri, 14 Feb 2025 18:26:31 +0000 Subject: [PATCH 37/48] Reapply "Turn on single issuer as an experiment" This reverts commit 901f4c80c48dd983ae87b115736f0589d6963b73. --- Sources/System/IORing.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index b7189917..5dd53d37 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -214,6 +214,7 @@ public struct IORing: ~Copyable { public init(queueDepth: UInt32) throws { var params = io_uring_params() + params.flags = IORING_SETUP_SINGLE_ISSUER //TODO make this configurable ringDescriptor = withUnsafeMutablePointer(to: ¶ms) { return io_uring_setup(queueDepth, $0) From d895f2a828f6c6c884057f43f0b14aa7fac10aff Mon Sep 17 00:00:00 2001 From: David Smith Date: Fri, 14 Feb 2025 18:27:17 +0000 Subject: [PATCH 38/48] Revert "Reapply "Turn on single issuer as an experiment"" This reverts commit 283e8d6b7b747f082ed2b7c9bd763229729d9d76. --- Sources/System/IORing.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 5dd53d37..b7189917 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -214,7 +214,6 @@ public struct IORing: ~Copyable { public init(queueDepth: UInt32) throws { var params = io_uring_params() - params.flags = IORING_SETUP_SINGLE_ISSUER //TODO make this configurable ringDescriptor = withUnsafeMutablePointer(to: ¶ms) { return io_uring_setup(queueDepth, $0) From f3b8cc473dfb84705aaaa374d8385794ba4a5efe Mon Sep 17 00:00:00 2001 From: David Smith Date: Fri, 28 Feb 2025 00:45:31 +0000 Subject: [PATCH 39/48] Actually consume events we waited for --- Sources/System/IORing.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index b7189917..505f3e02 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -376,9 +376,9 @@ public struct IORing: ~Copyable { "fatal error in receiving requests: " + Errno(rawValue: -res).debugDescription ) - while let completion = _tryConsumeCompletion(ring: completionRing) { - try consumer(completion, nil, false) - } + } + while let completion = _tryConsumeCompletion(ring: completionRing) { + try consumer(completion, nil, false) } try consumer(nil, nil, true) } From d338de147879b044c636992b6fe892a4b90ac8fc Mon Sep 17 00:00:00 2001 From: David Smith Date: Fri, 28 Feb 2025 00:49:31 +0000 Subject: [PATCH 40/48] Only get one completion if we asked for one completion --- Sources/System/IORing.swift | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 505f3e02..d459422a 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -336,6 +336,7 @@ public struct IORing: ~Copyable { private func _blockingConsumeCompletionGuts( minimumCount: UInt32, + maximumCount: UInt32, extraArgs: UnsafeMutablePointer? = nil, consumer: (IOCompletion?, IORingError?, Bool) throws -> Void ) rethrows { @@ -343,6 +344,10 @@ public struct IORing: ~Copyable { while let completion = _tryConsumeCompletion(ring: completionRing) { count += 1 try consumer(completion, nil, false) + if count == maximumCount { + try consumer(nil, nil, true) + return + } } if count < minimumCount { @@ -377,8 +382,13 @@ public struct IORing: ~Copyable { + Errno(rawValue: -res).debugDescription ) } + var count = 0 while let completion = _tryConsumeCompletion(ring: completionRing) { + count += 1 try consumer(completion, nil, false) + if count == maximumCount { + break + } } try consumer(nil, nil, true) } @@ -388,7 +398,7 @@ public struct IORing: ~Copyable { extraArgs: UnsafeMutablePointer? = nil ) throws -> IOCompletion { var result: IOCompletion? = nil - try _blockingConsumeCompletionGuts(minimumCount: 1, extraArgs: extraArgs) { + try _blockingConsumeCompletionGuts(minimumCount: 1, maximumCount: 1, extraArgs: extraArgs) { (completion, error, done) in if let error { throw error @@ -440,10 +450,10 @@ public struct IORing: ~Copyable { ts: UInt64(UInt(bitPattern: tsPtr)) ) try _blockingConsumeCompletionGuts( - minimumCount: minimumCount, extraArgs: &args, consumer: consumer) + minimumCount: minimumCount, maximumCount: UInt32.max, extraArgs: &args, consumer: consumer) } } else { - try _blockingConsumeCompletionGuts(minimumCount: minimumCount, consumer: consumer) + try _blockingConsumeCompletionGuts(minimumCount: minimumCount, maximumCount: UInt32.max, consumer: consumer) } } From 5e2467325960b37a53b145fab6efc896ce9b1d6f Mon Sep 17 00:00:00 2001 From: David Smith Date: Mon, 3 Mar 2025 21:53:26 +0000 Subject: [PATCH 41/48] Switch from unsafe pointers to FilePaths, using hacks --- Sources/System/IORequest.swift | 97 +++++++++++++++----------- Sources/System/RawIORequest.swift | 3 +- Tests/SystemTests/IORequestTests.swift | 2 +- 3 files changed, 58 insertions(+), 44 deletions(-) diff --git a/Sources/System/IORequest.swift b/Sources/System/IORequest.swift index 8705f0d6..e09bd895 100644 --- a/Sources/System/IORequest.swift +++ b/Sources/System/IORequest.swift @@ -5,14 +5,14 @@ internal enum IORequestCore { case nop // nothing here case openat( atDirectory: FileDescriptor, - path: UnsafePointer, + path: FilePath, FileDescriptor.AccessMode, options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), permissions: FilePermissions? = nil ) case openatSlot( atDirectory: FileDescriptor, - path: UnsafePointer, + path: FilePath, FileDescriptor.AccessMode, options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), permissions: FilePermissions? = nil, @@ -60,9 +60,9 @@ internal enum IORequestCore { ) case close(FileDescriptor) case closeSlot(IORingFileSlot) - case unlinkAt( + case unlinkAt( atDirectory: FileDescriptor, - path: UnsafePointer + path: FilePath ) } @@ -135,56 +135,64 @@ extension IORequest { IORequest(core: .nop) } - public static func reading(_ file: IORingFileSlot, + public static func reading( + _ file: IORingFileSlot, into buffer: IORingBuffer, at offset: UInt64 = 0 ) -> IORequest { IORequest(core: .readSlot(file: file, buffer: buffer, offset: offset)) } - public static func reading(_ file: FileDescriptor, + public static func reading( + _ file: FileDescriptor, into buffer: IORingBuffer, at offset: UInt64 = 0 ) -> IORequest { IORequest(core: .read(file: file, buffer: buffer, offset: offset)) } - public static func reading(_ file: IORingFileSlot, + public static func reading( + _ file: IORingFileSlot, into buffer: UnsafeMutableRawBufferPointer, at offset: UInt64 = 0 ) -> IORequest { IORequest(core: .readUnregisteredSlot(file: file, buffer: buffer, offset: offset)) } - public static func reading(_ file: FileDescriptor, + public static func reading( + _ file: FileDescriptor, into buffer: UnsafeMutableRawBufferPointer, at offset: UInt64 = 0 ) -> IORequest { IORequest(core: .readUnregistered(file: file, buffer: buffer, offset: offset)) } - public static func writing(_ buffer: IORingBuffer, + public static func writing( + _ buffer: IORingBuffer, into file: IORingFileSlot, at offset: UInt64 = 0 ) -> IORequest { IORequest(core: .writeSlot(file: file, buffer: buffer, offset: offset)) } - public static func writing(_ buffer: IORingBuffer, + public static func writing( + _ buffer: IORingBuffer, into file: FileDescriptor, at offset: UInt64 = 0 ) -> IORequest { IORequest(core: .write(file: file, buffer: buffer, offset: offset)) } - public static func writing(_ buffer: UnsafeMutableRawBufferPointer, + public static func writing( + _ buffer: UnsafeMutableRawBufferPointer, into file: IORingFileSlot, at offset: UInt64 = 0 ) -> IORequest { IORequest(core: .writeUnregisteredSlot(file: file, buffer: buffer, offset: offset)) } - public static func writing(_ buffer: UnsafeMutableRawBufferPointer, + public static func writing( + _ buffer: UnsafeMutableRawBufferPointer, into file: FileDescriptor, at offset: UInt64 = 0 ) -> IORequest { @@ -199,47 +207,35 @@ extension IORequest { IORequest(core: .closeSlot(file)) } - - public static func opening(_ path: UnsafePointer, - in directory: FileDescriptor, - into slot: IORingFileSlot, - mode: FileDescriptor.AccessMode, - options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), - permissions: FilePermissions? = nil - ) -> IORequest { - IORequest(core :.openatSlot(atDirectory: directory, path: path, mode, options: options, permissions: permissions, intoSlot: slot)) - } - - public static func opening(_ path: UnsafePointer, - in directory: FileDescriptor, - mode: FileDescriptor.AccessMode, - options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), - permissions: FilePermissions? = nil - ) -> IORequest { - IORequest(core: .openat(atDirectory: directory, path: path, mode, options: options, permissions: permissions)) - } - - - public static func opening(_ path: FilePath, + public static func opening( + _ path: FilePath, in directory: FileDescriptor, into slot: IORingFileSlot, mode: FileDescriptor.AccessMode, options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), permissions: FilePermissions? = nil ) -> IORequest { - fatalError("Implement me") + IORequest( + core: .openatSlot( + atDirectory: directory, path: path, mode, options: options, + permissions: permissions, intoSlot: slot)) } - public static func opening(_ path: FilePath, + public static func opening( + _ path: FilePath, in directory: FileDescriptor, mode: FileDescriptor.AccessMode, options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), permissions: FilePermissions? = nil ) -> IORequest { - fatalError("Implement me") + IORequest( + core: .openat( + atDirectory: directory, path: path, mode, options: options, permissions: permissions + )) } - public static func unlinking(_ path: UnsafePointer, + public static func unlinking( + _ path: FilePath, in directory: FileDescriptor ) -> IORequest { IORequest(core: .unlinkAt(atDirectory: directory, path: path)) @@ -251,20 +247,31 @@ extension IORequest { switch extractCore() { case .nop: request.operation = .nop - case .openatSlot(let atDirectory, let path, let mode, let options, let permissions, let fileSlot): + case .openatSlot( + let atDirectory, let path, let mode, let options, let permissions, let fileSlot): // TODO: use rawValue less request.operation = .openAt request.fileDescriptor = atDirectory - request.rawValue.addr = UInt64(UInt(bitPattern: path)) + request.rawValue.addr = UInt64( + UInt( + bitPattern: path.withPlatformString { ptr in + ptr //this is unsavory, but we keep it alive by storing path alongside it in the request + })) request.rawValue.open_flags = UInt32(bitPattern: options.rawValue | mode.rawValue) request.rawValue.len = permissions?.rawValue ?? 0 request.rawValue.file_index = UInt32(fileSlot.index + 1) + request.path = path case .openat(let atDirectory, let path, let mode, let options, let permissions): request.operation = .openAt request.fileDescriptor = atDirectory - request.rawValue.addr = UInt64(UInt(bitPattern: path)) + request.rawValue.addr = UInt64( + UInt( + bitPattern: path.withPlatformString { ptr in + ptr //this is unsavory, but we keep it alive by storing path alongside it in the request + })) request.rawValue.open_flags = UInt32(bitPattern: options.rawValue | mode.rawValue) request.rawValue.len = permissions?.rawValue ?? 0 + request.path = path case .write(let file, let buffer, let offset): request.operation = .writeFixed return makeRawRequest_readWrite_registered( @@ -306,7 +313,13 @@ extension IORequest { case .unlinkAt(let atDirectory, let path): request.operation = .unlinkAt request.fileDescriptor = atDirectory - request.rawValue.addr = UInt64(UInt(bitPattern: path)) + request.rawValue.addr = UInt64( + UInt( + bitPattern: path.withPlatformString { ptr in + ptr //this is unsavory, but we keep it alive by storing path alongside it in the request + }) + ) + request.path = path } return request } diff --git a/Sources/System/RawIORequest.swift b/Sources/System/RawIORequest.swift index 78f4f6de..980771cc 100644 --- a/Sources/System/RawIORequest.swift +++ b/Sources/System/RawIORequest.swift @@ -4,7 +4,8 @@ //TODO: make this internal public struct RawIORequest: ~Copyable { - var rawValue: io_uring_sqe + var rawValue: io_uring_sqe + var path: FilePath? //buffer owner for the path pointer that the sqe may have public init() { self.rawValue = io_uring_sqe() diff --git a/Tests/SystemTests/IORequestTests.swift b/Tests/SystemTests/IORequestTests.swift index f9f95803..4aaf7543 100644 --- a/Tests/SystemTests/IORequestTests.swift +++ b/Tests/SystemTests/IORequestTests.swift @@ -30,7 +30,7 @@ final class IORequestTests: XCTestCase { func testOpenatFixedFile() throws { let pathPtr = UnsafePointer(bitPattern: 0x414141410badf00d)! let fileSlot: IORingFileSlot = IORingFileSlot(resource: UInt32.max, index: 0) - let req = IORequest.opening(pathPtr, + let req = IORequest.opening(FilePath(platformString: pathPtr), in: FileDescriptor(rawValue: -100), into: fileSlot, mode: .readOnly, From 49dd7977ca34dc0ea40adcbd031586f4269819bc Mon Sep 17 00:00:00 2001 From: David Smith Date: Mon, 3 Mar 2025 22:07:28 +0000 Subject: [PATCH 42/48] Add a combined "submit and consume" operation --- Sources/System/IORing.swift | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index d459422a..1f223fa8 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -450,10 +450,12 @@ public struct IORing: ~Copyable { ts: UInt64(UInt(bitPattern: tsPtr)) ) try _blockingConsumeCompletionGuts( - minimumCount: minimumCount, maximumCount: UInt32.max, extraArgs: &args, consumer: consumer) + minimumCount: minimumCount, maximumCount: UInt32.max, extraArgs: &args, + consumer: consumer) } } else { - try _blockingConsumeCompletionGuts(minimumCount: minimumCount, maximumCount: UInt32.max, consumer: consumer) + try _blockingConsumeCompletionGuts( + minimumCount: minimumCount, maximumCount: UInt32.max, consumer: consumer) } } @@ -590,6 +592,20 @@ public struct IORing: ~Copyable { try _submitRequests(ring: submissionRing, ringDescriptor: ringDescriptor) } + public func submitPreparedRequestsAndConsumeCompletions( + minimumCount: UInt32 = 1, + timeout: Duration? = nil, + consumer: (IOCompletion?, IORingError?, Bool) throws -> Void + ) throws { + //TODO: optimize this to one uring_enter + try submitPreparedRequests() + try blockingConsumeCompletions( + minimumCount: minimumCount, + timeout: timeout, + consumer: consumer + ) + } + public mutating func prepare(request: __owned IORequest) -> Bool { var raw: RawIORequest? = request.makeRawRequest() return _writeRequest( From 91155fd52772b5cb1cf6980461f4429f3301f0e9 Mon Sep 17 00:00:00 2001 From: David Smith Date: Mon, 3 Mar 2025 22:15:32 +0000 Subject: [PATCH 43/48] Plumb error handling through completion consumers --- Sources/System/IORing.swift | 12 ++++++++++-- Sources/System/IORingError.swift | 6 +++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 1f223fa8..4525281a 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -343,7 +343,11 @@ public struct IORing: ~Copyable { var count = 0 while let completion = _tryConsumeCompletion(ring: completionRing) { count += 1 - try consumer(completion, nil, false) + if completion.result < 0 { + try consumer(nil, IORingError(completionResult: completion.result), false) + } else { + try consumer(completion, nil, false) + } if count == maximumCount { try consumer(nil, nil, true) return @@ -385,8 +389,12 @@ public struct IORing: ~Copyable { var count = 0 while let completion = _tryConsumeCompletion(ring: completionRing) { count += 1 + if completion.result < 0 { + try consumer(nil, IORingError(completionResult: completion.result), false) + } else { try consumer(completion, nil, false) - if count == maximumCount { + } + if count == maximumCount { break } } diff --git a/Sources/System/IORingError.swift b/Sources/System/IORingError.swift index fda58bcb..fbd70bce 100644 --- a/Sources/System/IORingError.swift +++ b/Sources/System/IORingError.swift @@ -2,5 +2,9 @@ public enum IORingError: Error, Equatable { case missingRequiredFeatures case operationCanceled - case unknown + case unknown(errorCode: Int) + + internal init(completionResult: Int32) { + self = .unknown(errorCode: Int(completionResult)) //TODO, flesh this out + } } From 9ad16c0aae9f005202eab849f1e6278179453238 Mon Sep 17 00:00:00 2001 From: David Smith Date: Mon, 3 Mar 2025 22:37:51 +0000 Subject: [PATCH 44/48] Plumb through userData --- Sources/System/IORequest.swift | 175 +++++++++++++++++++----------- Sources/System/RawIORequest.swift | 1 - 2 files changed, 114 insertions(+), 62 deletions(-) diff --git a/Sources/System/IORequest.swift b/Sources/System/IORequest.swift index e09bd895..7b602060 100644 --- a/Sources/System/IORequest.swift +++ b/Sources/System/IORequest.swift @@ -8,7 +8,8 @@ internal enum IORequestCore { path: FilePath, FileDescriptor.AccessMode, options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), - permissions: FilePermissions? = nil + permissions: FilePermissions? = nil, + userData: UInt64 = 0 ) case openatSlot( atDirectory: FileDescriptor, @@ -16,53 +17,69 @@ internal enum IORequestCore { FileDescriptor.AccessMode, options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), permissions: FilePermissions? = nil, - intoSlot: IORingFileSlot + intoSlot: IORingFileSlot, + userData: UInt64 = 0 ) case read( file: FileDescriptor, buffer: IORingBuffer, - offset: UInt64 = 0 + offset: UInt64 = 0, + userData: UInt64 = 0 ) case readUnregistered( file: FileDescriptor, buffer: UnsafeMutableRawBufferPointer, - offset: UInt64 = 0 + offset: UInt64 = 0, + userData: UInt64 = 0 ) case readSlot( file: IORingFileSlot, buffer: IORingBuffer, - offset: UInt64 = 0 + offset: UInt64 = 0, + userData: UInt64 = 0 ) case readUnregisteredSlot( file: IORingFileSlot, buffer: UnsafeMutableRawBufferPointer, - offset: UInt64 = 0 + offset: UInt64 = 0, + userData: UInt64 = 0 ) case write( file: FileDescriptor, buffer: IORingBuffer, - offset: UInt64 = 0 + offset: UInt64 = 0, + userData: UInt64 = 0 ) case writeUnregistered( file: FileDescriptor, buffer: UnsafeMutableRawBufferPointer, - offset: UInt64 = 0 + offset: UInt64 = 0, + userData: UInt64 = 0 ) case writeSlot( file: IORingFileSlot, buffer: IORingBuffer, - offset: UInt64 = 0 + offset: UInt64 = 0, + userData: UInt64 = 0 ) case writeUnregisteredSlot( file: IORingFileSlot, buffer: UnsafeMutableRawBufferPointer, - offset: UInt64 = 0 + offset: UInt64 = 0, + userData: UInt64 = 0 + ) + case close( + FileDescriptor, + userData: UInt64 = 0 + ) + case closeSlot( + IORingFileSlot, + userData: UInt64 = 0 ) - case close(FileDescriptor) - case closeSlot(IORingFileSlot) case unlinkAt( atDirectory: FileDescriptor, - path: FilePath + path: FilePath, + userData: UInt64 = 0 ) } @@ -71,6 +88,7 @@ internal func makeRawRequest_readWrite_registered( file: FileDescriptor, buffer: IORingBuffer, offset: UInt64, + userData: UInt64 = 0, request: consuming RawIORequest ) -> RawIORequest { request.fileDescriptor = file @@ -85,6 +103,7 @@ internal func makeRawRequest_readWrite_registered_slot( file: IORingFileSlot, buffer: IORingBuffer, offset: UInt64, + userData: UInt64 = 0, request: consuming RawIORequest ) -> RawIORequest { request.rawValue.fd = Int32(exactly: file.index)! @@ -100,6 +119,7 @@ internal func makeRawRequest_readWrite_unregistered( file: FileDescriptor, buffer: UnsafeMutableRawBufferPointer, offset: UInt64, + userData: UInt64 = 0, request: consuming RawIORequest ) -> RawIORequest { request.fileDescriptor = file @@ -113,6 +133,7 @@ internal func makeRawRequest_readWrite_unregistered_slot( file: IORingFileSlot, buffer: UnsafeMutableRawBufferPointer, offset: UInt64, + userData: UInt64 = 0, request: consuming RawIORequest ) -> RawIORequest { request.rawValue.fd = Int32(exactly: file.index)! @@ -131,80 +152,101 @@ public struct IORequest { } extension IORequest { - public static func nop() -> IORequest { + public static func nop(userData: UInt64 = 0) -> IORequest { IORequest(core: .nop) } public static func reading( _ file: IORingFileSlot, into buffer: IORingBuffer, - at offset: UInt64 = 0 + at offset: UInt64 = 0, + userData: UInt64 = 0 ) -> IORequest { - IORequest(core: .readSlot(file: file, buffer: buffer, offset: offset)) + IORequest(core: .readSlot(file: file, buffer: buffer, offset: offset, userData: userData)) } public static func reading( _ file: FileDescriptor, into buffer: IORingBuffer, - at offset: UInt64 = 0 + at offset: UInt64 = 0, + userData: UInt64 = 0 ) -> IORequest { - IORequest(core: .read(file: file, buffer: buffer, offset: offset)) + IORequest(core: .read(file: file, buffer: buffer, offset: offset, userData: userData)) } public static func reading( _ file: IORingFileSlot, into buffer: UnsafeMutableRawBufferPointer, - at offset: UInt64 = 0 + at offset: UInt64 = 0, + userData: UInt64 = 0 ) -> IORequest { - IORequest(core: .readUnregisteredSlot(file: file, buffer: buffer, offset: offset)) + IORequest( + core: .readUnregisteredSlot( + file: file, buffer: buffer, offset: offset, userData: userData)) } public static func reading( _ file: FileDescriptor, into buffer: UnsafeMutableRawBufferPointer, - at offset: UInt64 = 0 + at offset: UInt64 = 0, + userData: UInt64 = 0 ) -> IORequest { - IORequest(core: .readUnregistered(file: file, buffer: buffer, offset: offset)) + IORequest( + core: .readUnregistered(file: file, buffer: buffer, offset: offset, userData: userData)) } public static func writing( _ buffer: IORingBuffer, into file: IORingFileSlot, - at offset: UInt64 = 0 + at offset: UInt64 = 0, + userData: UInt64 = 0 ) -> IORequest { - IORequest(core: .writeSlot(file: file, buffer: buffer, offset: offset)) + IORequest(core: .writeSlot(file: file, buffer: buffer, offset: offset, userData: userData)) } public static func writing( _ buffer: IORingBuffer, into file: FileDescriptor, - at offset: UInt64 = 0 + at offset: UInt64 = 0, + userData: UInt64 = 0 ) -> IORequest { - IORequest(core: .write(file: file, buffer: buffer, offset: offset)) + IORequest(core: .write(file: file, buffer: buffer, offset: offset, userData: userData)) } public static func writing( _ buffer: UnsafeMutableRawBufferPointer, into file: IORingFileSlot, - at offset: UInt64 = 0 + at offset: UInt64 = 0, + userData: UInt64 = 0 ) -> IORequest { - IORequest(core: .writeUnregisteredSlot(file: file, buffer: buffer, offset: offset)) + IORequest( + core: .writeUnregisteredSlot( + file: file, buffer: buffer, offset: offset, userData: userData)) } public static func writing( _ buffer: UnsafeMutableRawBufferPointer, into file: FileDescriptor, - at offset: UInt64 = 0 + at offset: UInt64 = 0, + userData: UInt64 = 0 ) -> IORequest { - IORequest(core: .writeUnregistered(file: file, buffer: buffer, offset: offset)) + IORequest( + core: .writeUnregistered(file: file, buffer: buffer, offset: offset, userData: userData) + ) } - public static func closing(_ file: FileDescriptor) -> IORequest { - IORequest(core: .close(file)) + public static func closing( + _ file: FileDescriptor, + userData: UInt64 = 0 + ) -> IORequest { + IORequest(core: .close(file, userData: userData)) } - public static func closing(_ file: IORingFileSlot) -> IORequest { - IORequest(core: .closeSlot(file)) + public static func closing( + _ file: IORingFileSlot, + userData: UInt64 = 0 + ) -> IORequest { + IORequest(core: .closeSlot(file, userData: userData)) } public static func opening( @@ -213,12 +255,13 @@ extension IORequest { into slot: IORingFileSlot, mode: FileDescriptor.AccessMode, options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), - permissions: FilePermissions? = nil + permissions: FilePermissions? = nil, + userData: UInt64 = 0 ) -> IORequest { IORequest( core: .openatSlot( atDirectory: directory, path: path, mode, options: options, - permissions: permissions, intoSlot: slot)) + permissions: permissions, intoSlot: slot, userData: userData)) } public static func opening( @@ -226,19 +269,22 @@ extension IORequest { in directory: FileDescriptor, mode: FileDescriptor.AccessMode, options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), - permissions: FilePermissions? = nil + permissions: FilePermissions? = nil, + userData: UInt64 = 0 ) -> IORequest { IORequest( core: .openat( - atDirectory: directory, path: path, mode, options: options, permissions: permissions + atDirectory: directory, path: path, mode, options: options, + permissions: permissions, userData: userData )) } public static func unlinking( _ path: FilePath, - in directory: FileDescriptor + in directory: FileDescriptor, + userData: UInt64 = 0 ) -> IORequest { - IORequest(core: .unlinkAt(atDirectory: directory, path: path)) + IORequest(core: .unlinkAt(atDirectory: directory, path: path, userData: userData)) } @inline(__always) @@ -248,7 +294,8 @@ extension IORequest { case .nop: request.operation = .nop case .openatSlot( - let atDirectory, let path, let mode, let options, let permissions, let fileSlot): + let atDirectory, let path, let mode, let options, let permissions, let fileSlot, + let userData): // TODO: use rawValue less request.operation = .openAt request.fileDescriptor = atDirectory @@ -261,7 +308,9 @@ extension IORequest { request.rawValue.len = permissions?.rawValue ?? 0 request.rawValue.file_index = UInt32(fileSlot.index + 1) request.path = path - case .openat(let atDirectory, let path, let mode, let options, let permissions): + request.rawValue.user_data = userData + case .openat( + let atDirectory, let path, let mode, let options, let permissions, let userData): request.operation = .openAt request.fileDescriptor = atDirectory request.rawValue.addr = UInt64( @@ -272,45 +321,48 @@ extension IORequest { request.rawValue.open_flags = UInt32(bitPattern: options.rawValue | mode.rawValue) request.rawValue.len = permissions?.rawValue ?? 0 request.path = path - case .write(let file, let buffer, let offset): + request.rawValue.user_data = userData + case .write(let file, let buffer, let offset, let userData): request.operation = .writeFixed return makeRawRequest_readWrite_registered( - file: file, buffer: buffer, offset: offset, request: request) - case .writeSlot(let file, let buffer, let offset): + file: file, buffer: buffer, offset: offset, userData: userData, request: request) + case .writeSlot(let file, let buffer, let offset, let userData): request.operation = .writeFixed return makeRawRequest_readWrite_registered_slot( - file: file, buffer: buffer, offset: offset, request: request) - case .writeUnregistered(let file, let buffer, let offset): + file: file, buffer: buffer, offset: offset, userData: userData, request: request) + case .writeUnregistered(let file, let buffer, let offset, let userData): request.operation = .write return makeRawRequest_readWrite_unregistered( - file: file, buffer: buffer, offset: offset, request: request) - case .writeUnregisteredSlot(let file, let buffer, let offset): + file: file, buffer: buffer, offset: offset, userData: userData, request: request) + case .writeUnregisteredSlot(let file, let buffer, let offset, let userData): request.operation = .write return makeRawRequest_readWrite_unregistered_slot( - file: file, buffer: buffer, offset: offset, request: request) - case .read(let file, let buffer, let offset): + file: file, buffer: buffer, offset: offset, userData: userData, request: request) + case .read(let file, let buffer, let offset, let userData): request.operation = .readFixed return makeRawRequest_readWrite_registered( - file: file, buffer: buffer, offset: offset, request: request) - case .readSlot(let file, let buffer, let offset): + file: file, buffer: buffer, offset: offset, userData: userData, request: request) + case .readSlot(let file, let buffer, let offset, let userData): request.operation = .readFixed return makeRawRequest_readWrite_registered_slot( - file: file, buffer: buffer, offset: offset, request: request) - case .readUnregistered(let file, let buffer, let offset): + file: file, buffer: buffer, offset: offset, userData: userData, request: request) + case .readUnregistered(let file, let buffer, let offset, let userData): request.operation = .read return makeRawRequest_readWrite_unregistered( - file: file, buffer: buffer, offset: offset, request: request) - case .readUnregisteredSlot(let file, let buffer, let offset): + file: file, buffer: buffer, offset: offset, userData: userData, request: request) + case .readUnregisteredSlot(let file, let buffer, let offset, let userData): request.operation = .read return makeRawRequest_readWrite_unregistered_slot( - file: file, buffer: buffer, offset: offset, request: request) - case .close(let file): + file: file, buffer: buffer, offset: offset, userData: userData, request: request) + case .close(let file, let userData): request.operation = .close request.fileDescriptor = file - case .closeSlot(let file): + request.rawValue.user_data = userData + case .closeSlot(let file, let userData): request.operation = .close request.rawValue.file_index = UInt32(file.index + 1) - case .unlinkAt(let atDirectory, let path): + request.rawValue.user_data = userData + case .unlinkAt(let atDirectory, let path, let userData): request.operation = .unlinkAt request.fileDescriptor = atDirectory request.rawValue.addr = UInt64( @@ -320,6 +372,7 @@ extension IORequest { }) ) request.path = path + request.rawValue.user_data = userData } return request } diff --git a/Sources/System/RawIORequest.swift b/Sources/System/RawIORequest.swift index 980771cc..50c97c61 100644 --- a/Sources/System/RawIORequest.swift +++ b/Sources/System/RawIORequest.swift @@ -102,7 +102,6 @@ extension RawIORequest { // TODO: cleanup? rawValue.addr = UInt64(Int(bitPattern: newValue.baseAddress!)) rawValue.len = UInt32(exactly: newValue.count)! - rawValue.user_data = rawValue.addr //TODO: this is kind of a hack, but I need to decide how best to get the buffer out on the other side } } From 880ec9086e0517cd8a90e1f58588cfcebdcec487 Mon Sep 17 00:00:00 2001 From: David Smith Date: Mon, 3 Mar 2025 22:51:35 +0000 Subject: [PATCH 45/48] Add a pointer convenience for getting the user data --- Sources/System/IOCompletion.swift | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Sources/System/IOCompletion.swift b/Sources/System/IOCompletion.swift index 1702f9e8..9b41214e 100644 --- a/Sources/System/IOCompletion.swift +++ b/Sources/System/IOCompletion.swift @@ -23,19 +23,25 @@ extension IOCompletion { extension IOCompletion { public var userData: UInt64 { //TODO: naming? get { - return rawValue.user_data + rawValue.user_data + } + } + + public var userPointer: UnsafeRawPointer? { + get { + UnsafeRawPointer(bitPattern: UInt(rawValue.user_data)) } } public var result: Int32 { get { - return rawValue.res + rawValue.res } } public var flags: IOCompletion.Flags { get { - return Flags(rawValue: rawValue.flags & 0x0000FFFF) + Flags(rawValue: rawValue.flags & 0x0000FFFF) } } From 48455a913557acf7e7e5f0b7a209e7b6fa9a2c6a Mon Sep 17 00:00:00 2001 From: David Smith Date: Mon, 3 Mar 2025 23:04:52 +0000 Subject: [PATCH 46/48] Fix plumbing --- Sources/System/IORequest.swift | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Sources/System/IORequest.swift b/Sources/System/IORequest.swift index 7b602060..1eed6edf 100644 --- a/Sources/System/IORequest.swift +++ b/Sources/System/IORequest.swift @@ -95,6 +95,7 @@ internal func makeRawRequest_readWrite_registered( request.buffer = buffer.unsafeBuffer request.rawValue.buf_index = UInt16(exactly: buffer.index)! request.offset = offset + request.rawValue.user_data = userData return request } @@ -111,10 +112,11 @@ internal func makeRawRequest_readWrite_registered_slot( request.buffer = buffer.unsafeBuffer request.rawValue.buf_index = UInt16(exactly: buffer.index)! request.offset = offset + request.rawValue.user_data = userData return request } -@inlinable @inline(__always) +@inline(__always) internal func makeRawRequest_readWrite_unregistered( file: FileDescriptor, buffer: UnsafeMutableRawBufferPointer, @@ -125,6 +127,7 @@ internal func makeRawRequest_readWrite_unregistered( request.fileDescriptor = file request.buffer = buffer request.offset = offset + request.rawValue.user_data = userData return request } @@ -140,6 +143,7 @@ internal func makeRawRequest_readWrite_unregistered_slot( request.flags = .fixedFile request.buffer = buffer request.offset = offset + request.rawValue.user_data = userData return request } From 72c316ba86750e2cd3e223afd8ee698795e4efe0 Mon Sep 17 00:00:00 2001 From: David Smith Date: Mon, 3 Mar 2025 23:59:36 +0000 Subject: [PATCH 47/48] Make completions noncopyable again --- Sources/System/IOCompletion.swift | 3 +-- Sources/System/IORing.swift | 14 +++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Sources/System/IOCompletion.swift b/Sources/System/IOCompletion.swift index 9b41214e..ee81797e 100644 --- a/Sources/System/IOCompletion.swift +++ b/Sources/System/IOCompletion.swift @@ -1,7 +1,6 @@ @_implementationOnly import CSystem -//TODO: should be ~Copyable, but requires UnsafeContinuation add ~Copyable support -public struct IOCompletion { +public struct IOCompletion: ~Copyable { let rawValue: io_uring_cqe } diff --git a/Sources/System/IORing.swift b/Sources/System/IORing.swift index 4525281a..79840c1c 100644 --- a/Sources/System/IORing.swift +++ b/Sources/System/IORing.swift @@ -338,7 +338,7 @@ public struct IORing: ~Copyable { minimumCount: UInt32, maximumCount: UInt32, extraArgs: UnsafeMutablePointer? = nil, - consumer: (IOCompletion?, IORingError?, Bool) throws -> Void + consumer: (consuming IOCompletion?, IORingError?, Bool) throws -> Void ) rethrows { var count = 0 while let completion = _tryConsumeCompletion(ring: completionRing) { @@ -407,15 +407,15 @@ public struct IORing: ~Copyable { ) throws -> IOCompletion { var result: IOCompletion? = nil try _blockingConsumeCompletionGuts(minimumCount: 1, maximumCount: 1, extraArgs: extraArgs) { - (completion, error, done) in + (completion: consuming IOCompletion?, error, done) in if let error { throw error } - if let completion { - result = completion + if let completion { + result = consume completion } } - return result.unsafelyUnwrapped + return result.take()! } public func blockingConsumeCompletion( @@ -443,7 +443,7 @@ public struct IORing: ~Copyable { public func blockingConsumeCompletions( minimumCount: UInt32 = 1, timeout: Duration? = nil, - consumer: (IOCompletion?, IORingError?, Bool) throws -> Void + consumer: (consuming IOCompletion?, IORingError?, Bool) throws -> Void ) throws { if let timeout { var ts = __kernel_timespec( @@ -603,7 +603,7 @@ public struct IORing: ~Copyable { public func submitPreparedRequestsAndConsumeCompletions( minimumCount: UInt32 = 1, timeout: Duration? = nil, - consumer: (IOCompletion?, IORingError?, Bool) throws -> Void + consumer: (consuming IOCompletion?, IORingError?, Bool) throws -> Void ) throws { //TODO: optimize this to one uring_enter try submitPreparedRequests() From 7fff872441b6acb03bb8922300c09ef5f2afae2b Mon Sep 17 00:00:00 2001 From: David Smith Date: Thu, 6 Mar 2025 12:09:57 -0800 Subject: [PATCH 48/48] Add the draft proposal so I can link to it --- NNNN-swift-system-io-uring.md | 434 ++++++++++++++++++++++++++++++++++ 1 file changed, 434 insertions(+) create mode 100644 NNNN-swift-system-io-uring.md diff --git a/NNNN-swift-system-io-uring.md b/NNNN-swift-system-io-uring.md new file mode 100644 index 00000000..9b0eb1dc --- /dev/null +++ b/NNNN-swift-system-io-uring.md @@ -0,0 +1,434 @@ +# IORing, a Swift System API for io_uring + +* Proposal: [SE-NNNN](NNNN-filename.md) +* Authors: [Lucy Satheesan](https://github.com/oxy), [David Smith](https://github.com/Catfish-Man/) +* Review Manager: TBD +* Status: **Awaiting implementation** +* Implementation: [apple/swift-system#208](https://github.com/apple/swift-system/pull/208) + +## Introduction + +`io_uring` is Linux's solution to asynchronous and batched syscalls, with a particular focus on IO. We propose a low-level Swift API for it in Swift System that could either be used directly by projects with unusual needs, or via intermediaries like Swift NIO, to address scalability and thread pool starvation issues. + +## Motivation + +Up until recently, the overwhelmingly dominant file IO syscalls on major Unix platforms have been synchronous, e.g. `read(2)`. This design is very simple and proved sufficient for many uses for decades, but is less than ideal for Swift's needs in a few major ways: + +1. Requiring an entire OS thread for each concurrent operation imposes significant memory overhead +2. Requiring a separate syscall for each operation imposes significant CPU/time overhead to switch into and out of kernel mode repeatedly. This has been exacerbated in recent years by mitigations for the Spectre family of security exploits increasing the cost of syscalls. +3. Swift's N:M coroutine-on-thread-pool concurrency model assumes that threads will not be blocked. Each thread waiting for a syscall means a CPU core being left idle. In practice systems like NIO that deal in highly concurrent IO have had to work around this by providing their own thread pools. + +Non-file IO (network, pipes, etc…) has been in a somewhat better place with `epoll` and `kqueue` for asynchronously waiting for readability, but syscall overhead remains a significant issue for highly scalable systems. + +With the introduction of `io_uring` in 2019, Linux now has the kernel level tools to address these three problems directly. However, `io_uring` is quite complex and maps poorly into Swift. We expect that by providing a Swift interface to it, we can enable Swift on Linux servers to scale better and be more efficient than it has been in the past. + +## Proposed solution + +`struct IORing: ~Copyable` provides facilities for + +* Registering and unregistering resources (files and buffers), an `io_uring` specific variation on Unix file descriptors that improves their efficiency +* Registering and unregistering eventfds, which allow asynchronous waiting for completions +* Enqueueing IO requests +* Dequeueing IO completions + +`class IOResource` represents, via its two typealiases `IORingFileSlot` and `IORingBuffer`, registered file descriptors and buffers. Ideally we'd express the lifetimes of these as being dependent on the lifetime of the ring, but so far that's proven intractable, so we use a reference type. We expect that the up-front overhead of this should be negligible for larger operations, and smaller or one-shot operations can use non-registered buffers and file descriptors. + +`struct IORequest: ~Copyable` represents an IO operation that can be enqueued for the kernel to execute. It supports a wide variety of operations matching traditional unix file and socket operations. + +IORequest operations are expressed as overloaded static methods on `IORequest`, e.g. `openat` is spelled + +```swift + public static func opening( + _ path: FilePath, + in directory: FileDescriptor, + into slot: IORingFileSlot, + mode: FileDescriptor.AccessMode, + options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), + permissions: FilePermissions? = nil, + context: UInt64 = 0 + ) -> IORequest + + public static func opening( + _ path: FilePath, + in directory: FileDescriptor, + mode: FileDescriptor.AccessMode, + options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), + permissions: FilePermissions? = nil, + context: UInt64 = 0 + ) -> IORequest +``` + +which allows clients to decide whether they want to open the file into a slot on the ring, or have it return a file descriptor via a completion. Similarly, read operations have overloads for "use a buffer from the ring" or "read into this `UnsafeMutableBufferPointer`" + +Multiple `IORequests` can be enqueued on a single `IORing` using the `prepare(…)` family of methods, and then submitted together using `submitPreparedRequests`, allowing for things like "open this file, read its contents, and then close it" to be a single syscall. Conveniences are provided for preparing and submitting requests in one call. + +Since IO operations can execute in parallel or out of order by default, linked chains of operations can be established with `prepare(linkedRequests:…)` and related methods. Separate chains can still execute in parallel, and if an operation early in the chain fails, all subsequent operations will deliver cancellation errors as their completion. + +Already-completed results can be retrieved from the ring using `tryConsumeCompletion`, which never waits but may return nil, or `blockingConsumeCompletion(timeout:)`, which synchronously waits (up to an optional timeout) until an operation completes. There's also a bulk version of `blockingConsumeCompletion`, which may reduce the number of syscalls issued. It takes a closure which will be called repeatedly as completions are available (see Future Directions for potential improvements to this API). + +Since neither polling nor synchronously waiting is optimal in many cases, `IORing` also exposes the ability to register an eventfd (see `man eventfd(2)`), which will become readable when completions are available on the ring. This can then be monitored asynchronously with `epoll`, `kqueue`, or for clients who are linking libdispatch, `DispatchSource`. + +`struct IOCompletion: ~Copyable` represents the result of an IO operation and provides + +* Flags indicating various operation-specific metadata about the now-completed syscall +* The context associated with the operation when it was enqueued, as an `UnsafeRawPointer` or a `UInt64` +* The result of the operation, as an `Int32` with operation-specific meaning +* The error, if one occurred + +Unfortunately the underlying kernel API makes it relatively difficult to determine which `IORequest` led to a given `IOCompletion`, so it's expected that users will need to create this association themselves via the context parameter. + +`IORingError` represents failure of an operation. + +`IORing.Features` describes the supported features of the underlying kernel `IORing` implementation, which can be used to provide graceful reduction in functionality when running on older systems. + +## Detailed design + +```swift +public class IOResource { } +public typealias IORingFileSlot = IOResource +public typealias IORingBuffer = IOResource + +extension IORingBuffer { + public var unsafeBuffer: UnsafeMutableRawBufferPointer +} + +// IORing is intentionally not Sendable, to avoid internal locking overhead +public struct IORing: ~Copyable { + + public init(queueDepth: UInt32) throws(IORingError) + + public mutating func registerEventFD(_ descriptor: FileDescriptor) throws(IORingError) + public mutating func unregisterEventFD(_ descriptor: FileDescriptor) throws(IORingError) + + // An IORing.RegisteredResources is a view into the buffers or files registered with the ring, if any + public struct RegisteredResources: RandomAccessCollection { + public subscript(position: Int) -> IOResource + public subscript(position: UInt16) -> IOResource // This is useful because io_uring likes to use UInt16s as indexes + } + + public mutating func registerFileSlots(count: Int) throws(IORingError) -> RegisteredResources + + public func unregisterFiles() + + public var registeredFileSlots: RegisteredResources + + public mutating func registerBuffers( + _ buffers: some Collection + ) throws(IORingError) -> RegisteredResources + + public mutating func registerBuffers( + _ buffers: UnsafeMutableRawBufferPointer... + ) throws(IORingError) -> RegisteredResources + + public func unregisterBuffers() + + public var registeredBuffers: RegisteredResources + + public func prepare(requests: IORequest...) + public func prepare(linkedRequests: IORequest...) + + public func submitPreparedRequests(timeout: Duration? = nil) throws(IORingError) + public func submit(requests: IORequest..., timeout: Duration? = nil) throws(IORingError) + public func submit(linkedRequests: IORequest..., timeout: Duration? = nil) throws(IORingError) + + public func submitPreparedRequests() throws(IORingError) + public func submitPreparedRequestsAndWait(timeout: Duration? = nil) throws(IORingError) + + public func submitPreparedRequestsAndConsumeCompletions( + minimumCount: UInt32 = 1, + timeout: Duration? = nil, + consumer: (consuming IOCompletion?, IORingError?, Bool) throws(E) -> Void + ) throws(E) + + public func blockingConsumeCompletion( + timeout: Duration? = nil + ) throws(IORingError) -> IOCompletion + + public func blockingConsumeCompletions( + minimumCount: UInt32 = 1, + timeout: Duration? = nil, + consumer: (consuming IOCompletion?, IORingError?, Bool) throws(E) -> Void + ) throws(E) + + public func tryConsumeCompletion() -> IOCompletion? + + public struct Features { + //IORING_FEAT_SINGLE_MMAP is handled internally + public var nonDroppingCompletions: Bool //IORING_FEAT_NODROP + public var stableSubmissions: Bool //IORING_FEAT_SUBMIT_STABLE + public var currentFilePosition: Bool //IORING_FEAT_RW_CUR_POS + public var assumingTaskCredentials: Bool //IORING_FEAT_CUR_PERSONALITY + public var fastPolling: Bool //IORING_FEAT_FAST_POLL + public var epoll32BitFlags: Bool //IORING_FEAT_POLL_32BITS + public var pollNonFixedFiles: Bool //IORING_FEAT_SQPOLL_NONFIXED + public var extendedArguments: Bool //IORING_FEAT_EXT_ARG + public var nativeWorkers: Bool //IORING_FEAT_NATIVE_WORKERS + public var resourceTags: Bool //IORING_FEAT_RSRC_TAGS + public var allowsSkippingSuccessfulCompletions: Bool //IORING_FEAT_CQE_SKIP + public var improvedLinkedFiles: Bool //IORING_FEAT_LINKED_FILE + public var registerRegisteredRings: Bool //IORING_FEAT_REG_REG_RING + public var minimumTimeout: Bool //IORING_FEAT_MIN_TIMEOUT + public var bundledSendReceive: Bool //IORING_FEAT_RECVSEND_BUNDLE + } + public static var supportedFeatures: Features +} + +public struct IORequest: ~Copyable { + public static func nop(context: UInt64 = 0) -> IORequest + + // overloads for each combination of registered vs unregistered buffer/descriptor + // Read + public static func reading( + _ file: IORingFileSlot, + into buffer: IORingBuffer, + at offset: UInt64 = 0, + context: UInt64 = 0 + ) -> IORequest + + public static func reading( + _ file: FileDescriptor, + into buffer: IORingBuffer, + at offset: UInt64 = 0, + context: UInt64 = 0 + ) -> IORequest + + public static func reading( + _ file: IORingFileSlot, + into buffer: UnsafeMutableRawBufferPointer, + at offset: UInt64 = 0, + context: UInt64 = 0 + ) -> IORequest + + public static func reading( + _ file: FileDescriptor, + into buffer: UnsafeMutableRawBufferPointer, + at offset: UInt64 = 0, + context: UInt64 = 0 + ) -> IORequest + + // Write + public static func writing( + _ buffer: IORingBuffer, + into file: IORingFileSlot, + at offset: UInt64 = 0, + context: UInt64 = 0 + ) -> IORequest + + public static func writing( + _ buffer: IORingBuffer, + into file: FileDescriptor, + at offset: UInt64 = 0, + context: UInt64 = 0 + ) -> IORequest + + public static func writing( + _ buffer: UnsafeMutableRawBufferPointer, + into file: IORingFileSlot, + at offset: UInt64 = 0, + context: UInt64 = 0 + ) -> IORequest + + public static func writing( + _ buffer: UnsafeMutableRawBufferPointer, + into file: FileDescriptor, + at offset: UInt64 = 0, + context: UInt64 = 0 + ) -> IORequest + + // Close + public static func closing( + _ file: FileDescriptor, + context: UInt64 = 0 + ) -> IORequest + + public static func closing( + _ file: IORingFileSlot, + context: UInt64 = 0 + ) -> IORequest + + // Open At + public static func opening( + _ path: FilePath, + in directory: FileDescriptor, + into slot: IORingFileSlot, + mode: FileDescriptor.AccessMode, + options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), + permissions: FilePermissions? = nil, + context: UInt64 = 0 + ) -> IORequest + + public static func opening( + _ path: FilePath, + in directory: FileDescriptor, + mode: FileDescriptor.AccessMode, + options: FileDescriptor.OpenOptions = FileDescriptor.OpenOptions(), + permissions: FilePermissions? = nil, + context: UInt64 = 0 + ) -> IORequest + + public static func unlinking( + _ path: FilePath, + in directory: FileDescriptor, + context: UInt64 = 0 + ) -> IORequest + + // Other operations follow in the same pattern +} + +public struct IOCompletion { + + public struct Flags: OptionSet, Hashable, Codable { + public let rawValue: UInt32 + + public init(rawValue: UInt32) + + public static let moreCompletions: Flags + public static let socketNotEmpty: Flags + public static let isNotificationEvent: Flags + } + + //These are both the same value, but having both eliminates some ugly casts in client code + public var context: UInt64 + public var contextPointer: UnsafeRawPointer + + public var result: Int32 + + public var error: IORingError? // Convenience wrapper over `result` + + public var flags: Flags +} + +public struct IORingError: Error, Equatable { + static var missingRequiredFeatures: IORingError + static var operationCanceled: IORingError + static var timedOut: IORingError + static var resourceRegistrationFailed: IORingError + // Other error values to be filled out as the set of supported operations expands in the future + static var unknown: IORingError(errorCode: Int) +} + +``` + +## Usage Examples + +### Blocking + +```swift +let ring = try IORing(queueDepth: 2) + +//Make space on the ring for our file (this is optional, but improves performance with repeated use) +let file = ring.registerFiles(count: 1)[0] + +var statInfo = Glibc.stat() // System doesn't have an abstraction for stat() right now +// Build our requests to open the file and find out how big it is +ring.prepare(linkedRequests: + .opening(path, + in: parentDirectory, + into: file, + mode: mode, + options: openOptions, + permissions: nil + ), + .readingMetadataOf(file, + into: &statInfo + ) +) +//batch submit 2 syscalls in 1! +try ring.submitPreparedRequestsAndConsumeCompletions(minimumCount: 2) { (completion: consuming IOCompletion?, error, done) in + if let error { + throw error //or other error handling as desired + } +} + +// We could register our buffer with the ring too, but we're only using it once +let buffer = UnsafeMutableRawBufferPointer.allocate(Int(statInfo.st_size)) + +// Build our requests to read the file and close it +ring.prepare(linkedRequests: + .reading(file, + into: buffer + ), + .closing(file) +) + +//batch submit 2 syscalls in 1! +try ring.submitPreparedRequestsAndConsumeCompletions(minimumCount: 2) { (completion: consuming IOCompletion?, error, done) in + if let error { + throw error //or other error handling as desired + } +} + +processBuffer(buffer) +``` + +### Using libdispatch to wait for the read asynchronously + +```swift +//Initial setup as above up through creating buffer, omitted for brevity + +//Make the read request with a context so we can get the buffer out of it in the completion handler +… +.reading(file, into: buffer, context: UInt64(buffer.baseAddress!)) +… + +// Make an eventfd and register it with the ring +let eventfd = eventfd(0, 0) +ring.registerEventFD(eventfd) + +// Make a read source to monitor the eventfd for readability +let readabilityMonitor = DispatchSource.makeReadSource(fileDescriptor: eventfd) +readabilityMonitor.setEventHandler { + let completion = ring.blockingConsumeCompletion() + if let error = completion.error { + //handle failure to read the file + } + processBuffer(completion.contextPointer) +} +readabilityMonitor.activate() + +ring.submitPreparedRequests //note, not "AndConsumeCompletions" this time +``` + +## Source compatibility + +This is an all-new API in Swift System, so has no backwards compatibility implications. Of note, though, this API is only available on Linux. + +## ABI compatibility + +Swift on Linux does not have a stable ABI, and we will likely take advantage of this to evolve IORing as compiler support improves, as described in Future Directions. + +## Implications on adoption + +This feature is intrinsically linked to Linux kernel support, so constrains the deployment target of anything that adopts it to newer kernels. Exactly which features of the evolving io_uring syscall surface area we need is under consideration. + +## Future directions + +* While most Swift users on Darwin are not limited by IO scalability issues, the thread pool considerations still make introducing something similar to this appealing if and when the relevant OS support is available. We should attempt to the best of our ability to not design this in a way that's gratuitously incompatible with non-Linux OSs, although Swift System does not attempt to have an API that's identical on all platforms. +* The set of syscalls covered by `io_uring` has grown significantly and is still growing. We should leave room for supporting additional operations in the future. +* Once same-element requirements and pack counts as integer generic arguments are supported by the compiler, we should consider adding something along the lines of the following to allow preparing, submitting, and waiting for an entire set of operations at once: + +``` +func submitLinkedRequestsAndWait( + _ requests: repeat each Request +) where Request == IORequest + -> InlineArray<(repeat each Request).count, IOCompletion> +``` +* Once mutable borrows are supported, we should consider replacing the closure-taking bulk completion APIs (e.g. `blockingConsumeCompletions(…)`) with ones that return a sequence of completions instead +* We should consider making more types noncopyable as compiler support improves +* liburing has a "peek next completion" operation that doesn't consume it, and then a "mark consumed" operation. We may want to add something similar +* liburing has support for operations allocating their own buffers and returning them via the completion, we may want to support this +* We may want to provide API for asynchronously waiting, rather than just exposing the eventfd to let people roll their own async waits. Doing this really well has *considerable* implications for the concurrency runtime though. +* We should almost certainly expose API for more of the configuration options in `io_uring_setup` +* The API for feature probing is functional but not especially nice. Finding a better way to present that concept would be desirable. + +## Alternatives considered + +* We could use a NIO-style separate thread pool, but we believe `io_uring` is likely a better option for scalability. We may still want to provide a thread-pool backed version as an option, because many Linux systems currently disable `io_uring` due to security concerns. +* We could multiplex all IO onto a single actor as `AsyncBytes` currently does, but this has a number of downsides that make it entirely unsuitable to server usage. Most notably, it eliminates IO parallelism entirely. +* Using POSIX AIO instead of or as well as io_uring would greatly increase our ability to support older kernels and other Unix systems, but it has well-documented performance and usability issues that have prevented its adoption elsewhere, and apply just as much to Swift. +* Earlier versions of this proposal had higher level "managed" abstractions over IORing. These have been removed due to lack of interest from clients, but could be added back later if needed. +* I considered making any or all of `IORingError`, `IOCompletion`, and `IORequest` nested struct declarations inside `IORing`. The main reason I haven't done so is I was a little concerned about the ambiguity of having a type called `Error`. I'd be particularly interested in feedback on this choice. + +## Acknowledgments + +The NIO team, in particular Cory Benfield and Franz Busch, have provided invaluable feedback and direction on this project.