diff --git a/oro-kernel/src/iface/kernel/mem_token_v0.rs b/oro-kernel/src/iface/kernel/mem_token_v0.rs index 9d7dcea..1d6de22 100644 --- a/oro-kernel/src/iface/kernel/mem_token_v0.rs +++ b/oro-kernel/src/iface/kernel/mem_token_v0.rs @@ -68,19 +68,15 @@ impl KernelInterface for MemTokenV0 { _ => InterfaceResponse::immediate(SysError::BadKey, 0), } } - Token::SlotMap(token, side) => { - // SAFETY(qix-): Ensure that the `usize` fits within a `u64`, - // SAFETY(qix-): otherwise the below `as` casts will truncate. - ::oro_macro::assert::fits_within::(); - + Token::PortEndpoint(token) => { match key { key!("type") => InterfaceResponse::ok(t.type_id()), - key!("subtype") => InterfaceResponse::ok(*side as u64), + key!("subtype") => InterfaceResponse::ok(token.side() as u64), key!("forget") => InterfaceResponse::immediate(SysError::WriteOnly, 0), - key!("pagesize") => InterfaceResponse::ok(token.page_size() as u64), - key!("pages") => InterfaceResponse::ok(token.page_count() as u64), - key!("size") => InterfaceResponse::ok(token.size() as u64), - key!("commit") => InterfaceResponse::ok(token.commit() as u64), + key!("pagesize") => InterfaceResponse::ok(4096), + key!("pages") => InterfaceResponse::ok(1), + key!("size") => InterfaceResponse::ok(4096), + key!("commit") => InterfaceResponse::ok(1), key!("base") => InterfaceResponse::immediate(SysError::WriteOnly, 0), _ => InterfaceResponse::immediate(SysError::BadKey, 0), } diff --git a/oro-kernel/src/iface/root_ring/test_ports.rs b/oro-kernel/src/iface/root_ring/test_ports.rs index ea74194..edca625 100644 --- a/oro-kernel/src/iface/root_ring/test_ports.rs +++ b/oro-kernel/src/iface/root_ring/test_ports.rs @@ -9,15 +9,28 @@ use core::marker::PhantomData; use oro::{key, syscall::Error as SysError}; use crate::{ - arch::Arch, interface::Interface, port::Port, syscall::InterfaceResponse, tab::Tab, + arch::Arch, + interface::Interface, + port::{PortEnd, PortState}, + syscall::InterfaceResponse, + tab::Tab, thread::Thread, }; +/// Interface specific errors +#[repr(u64)] +pub enum Error { + /// The endpoint is already claimed. + Claimed = key!("claimed"), + /// The system is out of memory. + OutOfMemory = key!("oom"), +} + /// Temporary interface for testing ports. /// /// # Do not use! /// Do not use this interface. It's just here to test ports. -pub struct RootTestPorts(Tab, PhantomData); +pub struct RootTestPorts(Tab, PhantomData); impl RootTestPorts { /// Creates a new `RootTestPorts` instance. @@ -26,12 +39,7 @@ impl RootTestPorts { /// Panics if the system is out of memory. #[must_use] pub fn new() -> Self { - Self( - Port::new() - .and_then(|p| crate::tab::get().add(p)) - .expect("out of memory"), - PhantomData, - ) + Self(PortState::new().expect("out of memory"), PhantomData) } } @@ -47,17 +55,36 @@ impl Interface for RootTestPorts { match key { key!("health") => InterfaceResponse::ok(1337), - key!("prodtkn") => { - // Mark it owned by the current thread's instead. - let tkn = self.0.with(|p| p.producer()); - thread.with_mut(|t| t.insert_token(tkn.clone())); - InterfaceResponse::ok(tkn.id()) - } - key!("cnsmtkn") => { - // Mark it owned by the current thread's instead. - let tkn = self.0.with(|p| p.consumer()); - thread.with_mut(|t| t.insert_token(tkn.clone())); - InterfaceResponse::ok(tkn.id()) + key!("prodtkn") | key!("cnsmtkn") => { + match PortState::endpoint( + &self.0, + if key == key!("prodtkn") { + PortEnd::Producer + } else { + PortEnd::Consumer + }, + ) + .map(|r| { + r.map_err(|_| { + InterfaceResponse::immediate( + SysError::InterfaceError, + Error::Claimed as u64, + ) + }) + }) + .ok_or(InterfaceResponse::immediate( + SysError::InterfaceError, + Error::OutOfMemory as u64, + )) + .flatten() + .map(|tkn| { + let id = tkn.id(); + thread.with_mut(|t| t.insert_token(tkn)); + InterfaceResponse::ok(id) + }) { + Ok(r) => r, + Err(e) => e, + } } _ => InterfaceResponse::immediate(SysError::BadKey, 0), } diff --git a/oro-kernel/src/lib.rs b/oro-kernel/src/lib.rs index 1a53557..b015a95 100644 --- a/oro-kernel/src/lib.rs +++ b/oro-kernel/src/lib.rs @@ -32,6 +32,9 @@ // SAFETY(qix-): has equally good codegen. Either, way this is zero risk. // SAFETY(qix-): https://github.com/rust-lang/rust/issues/90850 #![feature(downcast_unchecked)] +// SAFETY(qix-): This is almost stabilized. +// SAFETY(qix-): https://github.com/rust-lang/rust/issues/70142 +#![feature(result_flattening)] pub mod arch; pub mod iface; diff --git a/oro-kernel/src/port.rs b/oro-kernel/src/port.rs index 73211f6..01fa2fd 100644 --- a/oro-kernel/src/port.rs +++ b/oro-kernel/src/port.rs @@ -1,93 +1,212 @@ //! Implements Oro ports. -use oro_mem::phys::PhysAddr; +use core::sync::atomic::{AtomicU64, Ordering::SeqCst}; -use crate::{ - tab::Tab, - token::{NormalToken, SlotMapEndpoint, Token}, +use oro::key; +use oro_mem::{ + global_alloc::GlobalPfa, + pfa::Alloc, + phys::{Phys, PhysAddr}, }; -/// A singular port connection. -pub struct Port { - /// The producer side memory token. - producer_token: Tab, - /// The consumer side memory token. - consumer_token: Tab, - /// The base address of the producer page. - /// - /// This is a *volatile* page, exactly 4096 bytes in size (512 `u64`s). - /// This pointer is guaranteed to be valid for the lifetime - /// of this `Port`. +use crate::{tab::Tab, token::Token}; + +/// "Internal" state of a port. +pub struct PortState { + /// The physical page belonging to the producer. /// /// **This may be the same as `consumer_page`!** - producer_page: *mut u64, - /// The base address of the consumer page. - /// - /// This is a *volatile* page, exactly 4096 bytes in size (512 `u64`s). - /// This pointer is guaranteed to be valid for the lifetime - /// of this `Port`. + producer_phys: Phys, + /// The physical page belonging to the consumer. /// /// **This may be the same as `producer_page`!** - consumer_page: *mut u64, + consumer_phys: Phys, + /// The producer's current tab index, or `0` if the producer is not active. + producer_index: AtomicU64, + /// The consumer's current tab index, or `0` if the consumer is not active. + consumer_index: AtomicU64, } -impl Port { +impl PortState { /// Creates a new port. /// /// Returns `None` if the system is out of memory. #[must_use] - pub fn new() -> Option { - let (producer_phys, producer_tab) = { - let mut t = NormalToken::new_4kib(1); - let phys = t.get_or_allocate(0)?; - let t = Token::SlotMap(t, SlotMapEndpoint::Producer); - let tab = crate::tab::get().add(t)?; - (phys, tab) - }; - - let (consumer_phys, consumer_tab) = { - let mut t = NormalToken::new_4kib(1); - let phys = t.get_or_allocate(0)?; - let t = Token::SlotMap(t, SlotMapEndpoint::Consumer); - let tab = crate::tab::get().add(t)?; - (phys, tab) - }; - - let this = Self { - producer_token: producer_tab, - consumer_token: consumer_tab, - // SAFETY: We just allocated these pages, and they're guaranteed aligned to a u64, so they are valid. - producer_page: unsafe { producer_phys.as_mut_ptr_unchecked::() }, - // SAFETY: We just allocated these pages, and they're guaranteed aligned to a u64, so they are valid. - consumer_page: unsafe { consumer_phys.as_mut_ptr_unchecked::() }, + pub fn new() -> Option> { + // SAFETY: We're allocating the page right as we're constructing the `Phys`. + let producer_phys = unsafe { Phys::from_address_unchecked(GlobalPfa.allocate()?) }; + // SAFETY: We're allocating the page right as we're constructing the `Phys`. + let consumer_phys = unsafe { + Phys::from_address_unchecked(GlobalPfa.allocate().or_else(|| { + GlobalPfa.free(producer_phys.address_u64()); + None + })?) }; // Zero out the pages. // SAFETY: We just allocated these pages, so they're guaranteed to exist. + // SAFETY: Further, it's always going to be aligned to a u8. + // SAFETY: Lastly, these writes have exclusive access to the memory. unsafe { - this.producer_page.cast::().write_bytes(0, 4096); - this.consumer_page.cast::().write_bytes(0, 4096); + producer_phys + .as_mut_ptr_unchecked::() + .write_bytes(0, 4096); + consumer_phys + .as_mut_ptr_unchecked::() + .write_bytes(0, 4096); } - Some(this) + // Make sure all cores see the zero. + ::core::sync::atomic::fence(SeqCst); + + crate::tab::get() + .add(Self { + producer_phys, + consumer_phys, + producer_index: AtomicU64::new(0), + consumer_index: AtomicU64::new(0), + }) + .or_else(|| { + // Free the pages; the Tab allocation failed. + // SAFETY: We had just allocated it; we can free it safely. + unsafe { + GlobalPfa.free(producer_phys.address_u64()); + GlobalPfa.free(consumer_phys.address_u64()); + } + + None + }) + } + + /// Tries to create a port endpoint from the given `PortState`. + /// + /// Returns `Some(Ok(tab))` if the endpoint was successfully created and is thus + /// unused elsewhere, or `Some(Err(tab))` if the endpoint already exists and is + /// still live. + /// + /// Returns `None` if the system is out of memory. + /// + /// **This is a relatively slow operation; do not call in tight loops!** + #[must_use] + pub fn endpoint(state: &Tab, end: PortEnd) -> Option, Tab>> { + state.with(|this| { + let index_ref = match end { + PortEnd::Producer => &this.producer_index, + PortEnd::Consumer => &this.consumer_index, + }; + + let mut current_index = index_ref.load(SeqCst); + + loop { + if current_index != 0 { + // Is it still live? + if let Some(existing_tab) = crate::tab::get().lookup(current_index) { + return Some(Err(existing_tab)); + } + } + + let tab = crate::tab::get().add(Token::PortEndpoint(PortEndpointToken { + state: state.clone(), + end, + }))?; + + let id = tab.id(); + + if let Err(existing_index) = + index_ref.compare_exchange(current_index, id, SeqCst, SeqCst) + { + // Another thread got to it first; check the liveness again. + current_index = existing_index; + } else { + // We got it! + return Some(Ok(tab)); + } + } + }) + } +} + +impl Drop for PortState { + fn drop(&mut self) { + // Make sure that, somehow, there are no active endpoints. + // Given the design of tabs, this should never be the case. + // However, it's still a good idea to check. + debug_assert!( + { + let v = self.producer_index.load(SeqCst); + v == 0 || crate::tab::get().lookup_any(v).is_none() + }, + "producer endpoint still active" + ); + debug_assert!( + { + let v = self.consumer_index.load(SeqCst); + v == 0 || crate::tab::get().lookup_any(v).is_none() + }, + "consumer endpoint still active" + ); + + // SAFETY: We're freeing the pages right as we're dropping the `Phys`. + unsafe { + GlobalPfa.free(self.producer_phys.address_u64()); + GlobalPfa.free(self.consumer_phys.address_u64()); + } } +} - /// Gets the producer side memory token for this port. +/// A slot map endpoint - either producer or consumer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u64)] +pub enum PortEnd { + /// The producer side of the slot map. + Producer = key!("producer"), + /// The consumer side of the slot map. + Consumer = key!("consumer"), +} + +/// A port endpoint token. +pub struct PortEndpointToken { + /// The internal port state tab. + state: Tab, + /// Which end the endpoint is. + end: PortEnd, +} + +impl PortEndpointToken { + /// Returns the [`PortEnd`] of the endpoint. + #[inline] #[must_use] - pub fn producer(&self) -> Tab { - self.producer_token.clone() + pub fn side(&self) -> PortEnd { + self.end } - /// Gets the consumer side memory token for this port. + /// Returns the [`Phys`] address of the endpoint's page. + #[inline] #[must_use] - pub fn consumer(&self) -> Tab { - self.consumer_token.clone() + pub fn phys(&self) -> Phys { + match self.end { + PortEnd::Producer => self.state.with(|s| s.producer_phys), + PortEnd::Consumer => self.state.with(|s| s.consumer_phys), + } } - /// Advances the consumer and producer pages. + /// Advances the port's internal copy state. + /// + /// For direct-mapped ports, this is a no-op. + /// + /// This is also a no-op for producers. pub fn advance(&self) { - let _ = self.producer_page; - let _ = self.consumer_page; - todo!("advance"); + if self.end == PortEnd::Producer { + return; + } + + self.state.with(|st| { + if st.consumer_phys == st.producer_phys { + // Direct-mapped port; no need to advance. + return; + } + + todo!("advance!"); + }); } } diff --git a/oro-kernel/src/tab.rs b/oro-kernel/src/tab.rs index 6313075..f4acba2 100644 --- a/oro-kernel/src/tab.rs +++ b/oro-kernel/src/tab.rs @@ -1190,8 +1190,8 @@ pub enum TabType { Module = 5, /// A [`crate::token::Token`]. Token = 6, - /// A [`crate::port::Port`]. - Port = 7, + /// A [`crate::port::PortState`]. + PortState = 7, } impl Tabbed for crate::thread::Thread { @@ -1218,6 +1218,6 @@ impl Tabbed for crate::token::Token { const TY: TabType = TabType::Token; } -impl Tabbed for crate::port::Port { - const TY: TabType = TabType::Port; +impl Tabbed for crate::port::PortState { + const TY: TabType = TabType::PortState; } diff --git a/oro-kernel/src/thread.rs b/oro-kernel/src/thread.rs index 78c46c4..3d74097 100644 --- a/oro-kernel/src/thread.rs +++ b/oro-kernel/src/thread.rs @@ -10,8 +10,10 @@ // TODO(qix-): them better. use oro::{key, syscall::Error as SysError}; +use oro_debug::dbg_warn; use oro_macro::AsU64; use oro_mem::{ + alloc::vec::Vec, global_alloc::GlobalPfa, mapper::{AddressSegment, AddressSpace as _, MapError, UnmapError}, pfa::Alloc, @@ -22,6 +24,7 @@ use crate::{ AddressSpace, UserHandle, arch::{Arch, ThreadHandle}, instance::Instance, + port::PortEnd, ring::Ring, scheduler::PageFaultType, syscall::{InFlightState, InFlightSystemCall, InFlightSystemCallHandle, SystemCallResponse}, @@ -96,6 +99,10 @@ pub struct Thread { // TODO(qix-): replaced with a more efficient data structure // TODO(qix-): in the future. tls_token_vmap: Table<(Tab, usize)>, + /// A list of active port endpoint page virtual bases for the current time slice. + /// + /// Unmapped when the thread is paused. + active_ports: Vec, } impl Thread { @@ -201,6 +208,7 @@ impl Thread { run_state_transition: None, data: TypeTable::new(), tls_token_vmap: Table::new(), + active_ports: Vec::new(), }; let tab = crate::tab::get().add(this).ok_or(MapError::OutOfMemory)?; @@ -317,6 +325,22 @@ impl Thread { match &self.state { State::Running(core) => { if *core == core_id { + // For each active port endpoint, unmap it. + for virt in &self.active_ports { + let segment = AddressSpace::::user_thread_local_data(); + if let Err(err) = segment.unmap(self.handle.mapper(), *virt) { + dbg_warn!( + "failed to unmap port endpoint at {virt:#016X} for thread \ + {:#016X}: {err:?} (port may misbehave)", + self.id + ); + } + } + + // Clear the active ports list + self.active_ports.clear(); + + // We're paused now. self.state = State::Paused(core_id); Ok(()) } else { @@ -517,27 +541,24 @@ impl Thread { .map_err(PageFaultError::MapError)?; Ok(()) } - Token::SlotMap(t, _side) => { + Token::PortEndpoint(ep) => { debug_assert!( page_idx == 0, "slot map tokens must be exactly one page; attempt was made to commit \ page index >0" ); - debug_assert!( - t.page_size() == 4096, - "page size != 4096 is not implemented" - ); - debug_assert!( - t.page_count() == 1, - "slot map tokens must be exactly one page" - ); let segment = AddressSpace::::user_thread_local_data(); - let Some(phys) = t.get(0) else { - unreachable!("slot map token must be allocated before use"); - }; + segment - .map(self.handle.mapper(), virt, phys.address_u64()) + .map(self.handle.mapper(), virt, ep.phys().address_u64()) .map_err(PageFaultError::MapError)?; + + ep.advance(); + + if ep.side() == PortEnd::Consumer { + self.active_ports.push(virt); + } + Ok(()) } } @@ -608,21 +629,8 @@ impl Thread { Ok(()) }) } - Token::SlotMap(t, _side) => { - debug_assert!( - t.page_size() == 4096, - "page size != 4096 is not implemented" - ); - debug_assert!( - t.page_size().is_power_of_two(), - "page size is not a power of 2" - ); - debug_assert!( - t.page_count() == 1, - "slot map tokens must be exactly one page" - ); - - if (virt & (t.page_size() - 1)) != 0 { + Token::PortEndpoint(_) => { + if (virt & 4095) != 0 { return Err(TokenMapError::VirtNotAligned); } @@ -634,7 +642,7 @@ impl Thread { // Make sure that no other token exists in the vmap. if !virt_resides_within::(&segment, virt) - || !virt_resides_within::(&segment, virt + t.page_size() - 1) + || !virt_resides_within::(&segment, virt + 4095) { return Err(TokenMapError::VirtOutOfRange); } diff --git a/oro-kernel/src/token.rs b/oro-kernel/src/token.rs index f455e89..9f7849e 100644 --- a/oro-kernel/src/token.rs +++ b/oro-kernel/src/token.rs @@ -24,6 +24,8 @@ use oro_mem::{ phys::{Phys, PhysAddr}, }; +use crate::port::PortEndpointToken; + /// A singular memory token. See module level documentation for more information. // SAFETY(qix-): Do not change discriminant values. Only add new ones. // SAFETY(qix-): Further, this enum MUST be `repr(u64)` to ensure that the @@ -34,9 +36,9 @@ pub enum Token { // NOTE(qix-): as a sentinel value. /// A [`NormalToken`] memory token. Represents one or more physical pages. Normal(NormalToken) = key!("normal"), - /// A slot map token, which is a [`NormalToken`] that is enforced to be mapped - /// to a TLS segment and only take up a single page. - SlotMap(NormalToken, SlotMapEndpoint) = key!("portslot"), + /// A [`PortEndpointToken`] memory token. Represents a port endpoint created + /// by [`crate::port::PortState::endpoint()`]. + PortEndpoint(PortEndpointToken) = key!("port"), } impl Token { @@ -164,13 +166,3 @@ impl NormalToken { } } } - -/// A slot map endpoint - either producer or consumer. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[repr(u64)] -pub enum SlotMapEndpoint { - /// The producer side of the slot map. - Producer = key!("producer"), - /// The consumer side of the slot map. - Consumer = key!("consumer"), -}