From 8e6f4333634cbd96c0beee2ce7b7683ce32b4695 Mon Sep 17 00:00:00 2001 From: DriedYelloPeach Date: Tue, 10 Jun 2025 18:07:06 -0700 Subject: [PATCH] feat: new protocol for proc-macro-api --- Cargo.lock | 94 +++++++++++ crates/proc-macro-api/Cargo.toml | 3 +- crates/proc-macro-api/src/lib.rs | 10 ++ crates/proc-macro-api/src/new_protocol/msg.rs | 158 ++++++++++++++++++ .../proc-macro-api/src/new_protocol/proto.rs | 42 +++++ crates/proc-macro-api/src/task.rs | 41 +++++ 6 files changed, 347 insertions(+), 1 deletion(-) create mode 100644 crates/proc-macro-api/src/new_protocol/msg.rs create mode 100644 crates/proc-macro-api/src/new_protocol/proto.rs create mode 100644 crates/proc-macro-api/src/task.rs diff --git a/Cargo.lock b/Cargo.lock index 01de430925dc..98dba58ccaf4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -44,6 +44,15 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -119,6 +128,12 @@ version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66bb12751a83493ef4b8da1120451a262554e216a247f14b48cb5e8fe7ed8bdf" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "camino" version = "1.1.9" @@ -239,6 +254,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "cobs" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" + [[package]] name = "countme" version = "3.0.1" @@ -260,6 +281,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -416,6 +443,18 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "ena" version = "0.14.3" @@ -516,6 +555,15 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -542,6 +590,20 @@ dependencies = [ "hashbrown 0.15.2", ] +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32", + "rustc_version", + "serde", + "spin", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.5.0" @@ -1510,6 +1572,19 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" +[[package]] +name = "postcard" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170a2601f67cc9dba8edd8c4870b15f71a6a2dc196daec8c83f72b59dff628a8" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless", + "serde", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -1523,6 +1598,7 @@ dependencies = [ "indexmap", "intern", "paths", + "postcard", "rustc-hash 2.1.1", "serde", "serde_derive", @@ -1919,6 +1995,15 @@ dependencies = [ "smallvec", ] +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "ryu" version = "1.0.20" @@ -2104,6 +2189,15 @@ dependencies = [ "vfs", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" diff --git a/crates/proc-macro-api/Cargo.toml b/crates/proc-macro-api/Cargo.toml index f5ba40a994b1..52af72363c18 100644 --- a/crates/proc-macro-api/Cargo.toml +++ b/crates/proc-macro-api/Cargo.toml @@ -14,6 +14,7 @@ rust-version.workspace = true [dependencies] serde.workspace = true serde_derive.workspace = true +postcard = { version = "1.1.1", features = ["alloc"] } serde_json = { workspace = true, features = ["unbounded_depth"] } tracing.workspace = true rustc-hash.workspace = true @@ -24,7 +25,7 @@ paths = { workspace = true, features = ["serde1"] } tt.workspace = true stdx.workspace = true # span = {workspace = true, default-features = false} does not work -span = { path = "../span", version = "0.0.0", default-features = false} +span = { path = "../span", version = "0.0.0", default-features = false } intern.workspace = true diff --git a/crates/proc-macro-api/src/lib.rs b/crates/proc-macro-api/src/lib.rs index 25c30b6db4a4..bf78d310d8ea 100644 --- a/crates/proc-macro-api/src/lib.rs +++ b/crates/proc-macro-api/src/lib.rs @@ -9,6 +9,16 @@ pub mod legacy_protocol { pub mod json; pub mod msg; } + +#[allow(dead_code)] +pub mod new_protocol { + pub mod msg; + pub mod proto; +} + +#[allow(dead_code)] +pub mod task; + mod process; use paths::{AbsPath, AbsPathBuf}; diff --git a/crates/proc-macro-api/src/new_protocol/msg.rs b/crates/proc-macro-api/src/new_protocol/msg.rs new file mode 100644 index 000000000000..1449566fe11f --- /dev/null +++ b/crates/proc-macro-api/src/new_protocol/msg.rs @@ -0,0 +1,158 @@ +//! New Protocol Message Definitions covers the functionalities of `legacy_protocol` and with several changes: +//! +//! - Using [postcard](https://github.com/jamesmunns/postcard) as serde library, which provides binary, lightweight serialization. +//! - As we change to postcard, no need to use FlatTree, use `tt::TopSubtree` directly +//! +//! One possible communication may look like this: +//! client ------------------------- server +//! >-------Request---------> +//! <-------Query-----------< +//! >-------Reply-----------> +//! <-------Response-------< + +use paths::Utf8PathBuf; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +use crate::ProcMacroKind; +use crate::legacy_protocol::msg::FlatTree; + +/// Represents messages send from client to proc-macr-srv +#[derive(Debug, Serialize, Deserialize)] +pub enum C2SMsg { + Request(Request), + // NOTE: Reply is left empty as a placeholder + Reply, +} + +/// Represents messages send from client to proc-macr-srv +#[derive(Debug, Serialize, Deserialize)] +pub enum S2CMsg { + Response(Response), + // NOTE: Query is left empty as a place holder + Query, +} + +// NOTE: Directly copied from legacy_protocol +#[derive(Debug, Serialize, Deserialize)] +pub enum Request { + ListMacros { dylib_path: Utf8PathBuf }, + ExpandMacro(Box), + ApiVersionCheck {}, + SetConfig(ServerConfig), +} + +// NOTE: Directly copied from legacy_protocol +#[derive(Debug, Serialize, Deserialize)] +pub enum Response { + ListMacros(Result, String>), + ExpandMacro(Result), + ApiVersionCheck(u32), + SetConfig(ServerConfig), + ExpandMacroExtended(Result), +} + +// NOTE: Directly copied from legacy_protocol +#[derive(Debug, Serialize, Deserialize, Default)] +#[serde(default)] +pub struct ServerConfig { + pub span_mode: SpanMode, +} + +// NOTE: Directly copied from legacy_protocol, +// except the tree field +#[derive(Debug, Serialize, Deserialize)] +pub struct ExpandMacroExtended { + /// The expanded syntax tree. + pub tree: TreeWrapper, + /// Additional span data mappings. + pub span_data_table: Vec, +} + +// NOTE: Directly copied from legacy_protocol +#[derive(Debug, Serialize, Deserialize)] +pub struct PanicMessage(pub String); + +// NOTE: Directly copied from legacy_protocol +#[derive(Copy, Clone, Default, Debug, Serialize, Deserialize)] +pub enum SpanMode { + #[default] + Id, + RustAnalyzer, +} + +// NOTE: Directly copied from legacy_protocol +#[derive(Debug, Serialize, Deserialize)] +pub struct ExpandMacro { + pub lib: Utf8PathBuf, + + pub env: Vec<(String, String)>, + + pub current_dir: Option, + + #[serde(flatten)] + pub data: ExpandMacroData, +} + +// TODO: Maybe ideal we want to ser/de TokenTree directly +// +// Something like this should be ideal +// pub struct TreeWrapper(tt::TokenTree); +// currently, just use this wrapper to build the backbones +#[derive(Debug)] +pub struct TreeWrapper(FlatTree); + +// TODO: How to serialize/deserialize TokenTree? +// Seems impossible to derive all the underlying wrapped types to serialize/deserialize, which will change tons of code +// +impl Serialize for TreeWrapper { + fn serialize(&self, _serializer: Ser) -> Result + where + Ser: Serializer, + { + todo!("Implement Serialize for TreeWrapper if needed") + } +} + +impl<'de> Deserialize<'de> for TreeWrapper { + fn deserialize(_deserializer: D) -> Result + where + D: Deserializer<'de>, + { + todo!("Implement Deserialize for TreeWrapper if needed") + } +} + +// NOTE: Directly copied from legacy_protocol, +// except the tree field +#[derive(Debug, Serialize, Deserialize)] +pub struct ExpandMacroData { + pub macro_body: TreeWrapper, + + pub macro_name: String, + + pub attributes: Option, + #[serde(skip_serializing_if = "ExpnGlobals::skip_serializing_if")] + #[serde(default)] + pub has_global_spans: ExpnGlobals, + + #[serde(skip_serializing_if = "Vec::is_empty")] + #[serde(default)] + pub span_data_table: Vec, +} + +// NOTE: Directly copied from legacy_protocol, +#[derive(Copy, Clone, Default, Debug, Serialize, Deserialize)] +pub struct ExpnGlobals { + #[serde(skip_serializing)] + #[serde(default)] + pub serialize: bool, + pub def_site: usize, + pub call_site: usize, + pub mixed_site: usize, +} + +impl ExpnGlobals { + fn skip_serializing_if(&self) -> bool { + !self.serialize + } +} diff --git a/crates/proc-macro-api/src/new_protocol/proto.rs b/crates/proc-macro-api/src/new_protocol/proto.rs new file mode 100644 index 000000000000..59848d90d09a --- /dev/null +++ b/crates/proc-macro-api/src/new_protocol/proto.rs @@ -0,0 +1,42 @@ +//! Implement how to send and receive new protocol message on stdio stream +//! +//! Current implementation encoded message to | message length | message content | + +use serde::de::DeserializeOwned; +use std::io::{self, BufRead, Read, Write}; + +use super::msg::{C2SMsg, S2CMsg}; + +fn read_u64_be(reader: &mut R) -> io::Result { + let mut buf = [0u8; 8]; + reader.read_exact(&mut buf)?; + Ok(usize::from_be_bytes(buf)) +} + +fn write_u64_be(writer: &mut W, value: usize) -> io::Result<()> { + writer.write_all(&value.to_be_bytes()) // Convert and write as Big-Endian +} + +trait ProtoPostcard: serde::Serialize + DeserializeOwned { + fn receive_proto(reader: &mut R) -> io::Result { + let msg_len = read_u64_be(reader)? as usize; + let mut buf = vec![0u8; msg_len]; + + reader.read_exact(&mut buf)?; + postcard::from_bytes::(&buf) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) + } + + fn send_proto(&self, writer: &mut W) -> io::Result<()> { + let bytes = postcard::to_allocvec(self) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + + write_u64_be(writer, bytes.len())?; + writer.write_all(&bytes)?; + writer.flush() + } +} + +// NOTE: With default implementation, C2SMsg and S2CMsg can both be send through stdio +impl ProtoPostcard for C2SMsg {} +impl ProtoPostcard for S2CMsg {} diff --git a/crates/proc-macro-api/src/task.rs b/crates/proc-macro-api/src/task.rs new file mode 100644 index 000000000000..f3bf877beab8 --- /dev/null +++ b/crates/proc-macro-api/src/task.rs @@ -0,0 +1,41 @@ +//! task.rs is used to abstract ProcMacroServerProcess::send_task in process.rs +//! The ultimate goal is to encapsulate legacy_protocol and don't directly use legacy_protocol in process.rs +//! +use crate::legacy_protocol::msg::{Request, Response}; +use crate::new_protocol::msg::{C2SMsg, S2CMsg}; + +pub trait TaskClient { + type Task; + type TaskResult; + + fn send_task(&self, task: Self::Task) -> Self::TaskResult; +} + +pub struct JsonTaskClient; + +impl TaskClient for JsonTaskClient { + type Task = Request; + type TaskResult = Response; + + // Implement send_task for Json Legacy Protocol + // Basically what we have done in process.rs + fn send_task(&self, _task: Self::Task) -> Self::TaskResult { + todo!("implement JsonTaskClient::send_task"); + } +} + +pub struct PostcardTaskClient; + +impl TaskClient for PostcardTaskClient { + type Task = C2SMsg; + type TaskResult = S2CMsg; + + // Implement send_task for Postcard Protocol + // As this new Protol will allow back and forth communication + // send_task will abstract these back and forth details + // Task will be C2SMsg::Request, which is basically legacy_protocol Request + // TaskResult will be S2CMsg::Response, which is basically legacy_protocol Response + fn send_task(&self, _task: Self::Task) -> Self::TaskResult { + todo!("implement PostcardTaskClient::send_task, back and forth"); + } +}