API prototype (#1628)

* Prototype a new api strategy
- increase code sharing between grpc and http
- set up our codebase for simplifying traits in xmtp_mls
- make it easier to add/remove/modify api
- make api more rust idiomatic

* fix header panic

* temporarily exclude d14n

* fmt toml

Author: insipx

.github/workflows/test-http-api.yml

@@ -47,4 +47,4 @@ jobs:
       - name: build tests
         run: cargo nextest run --config-file ".cargo/nextest.toml" --no-run --tests --workspace --exclude xmtp_api_grpc --exclude xmtpv3 --exclude bindings_node --exclude bindings_wasm --features http-api
       - name: cargo test
-        run: cargo nextest run --config-file ".cargo/nextest.toml" --workspace --exclude xmtp_api_grpc --exclude xmtpv3 --exclude bindings_node --exclude bindings_wasm --features http-api --test-threads 2
+        run: cargo nextest run --config-file ".cargo/nextest.toml" --workspace --exclude xmtp_api_grpc --exclude xmtpv3 --exclude bindings_node --exclude bindings_wasm --exclude xmtp_api_d14n --features http-api --test-threads 2

Cargo.lock

@@ -18,6 +18,7 @@ members = [
   "xmtp_debug",
   "xmtp_content_types",
   "common",
+  "xmtp_api_d14n",
 ]
 
 # Make the feature resolver explicit.
@@ -71,8 +72,8 @@ openssl-sys = { version = "0.9", features = ["vendored"] }
 parking_lot = "0.12.3"
 pbjson = "0.7.0"
 pbjson-types = "0.7.0"
-prost = "^0.13"
-prost-types = "^0.13"
+prost = { version = "^0.13", default-features = false }
+prost-types = { version = "^0.13", default-features = false }
 rand = "0.8.5"
 regex = "1.10.4"
 rustc-hex = "2.1.0"

Cargo.toml

@@ -24,6 +24,8 @@ use std::sync::{
     Arc,
 };
 
+pub type BoxedRetry = Retry<Box<dyn Strategy>, Box<dyn Strategy>>;
+
 pub struct NotSpecialized;
 
 /// Specifies which errors are retryable.

common/src/retry.rs

@@ -13,12 +13,13 @@ pub enum StreamHandleError {
     ChannelClosed,
     #[error("The stream was closed")]
     StreamClosed,
-    #[error(transparent)]
-    JoinHandleError(#[from] tokio::task::JoinError),
     #[error("Stream Cancelled")]
     Cancelled,
     #[error("Stream Panicked With {0}")]
     Panicked(String),
+    #[cfg(not(target_arch = "wasm32"))]
+    #[error(transparent)]
+    JoinHandleError(#[from] tokio::task::JoinError),
 }
 /// A handle to a spawned Stream
 /// the spawned stream can be 'joined` by awaiting its Future implementation.

common/src/stream_handles.rs

@@ -0,0 +1,21 @@
+[package]
+name = "xmtp_api_d14n"
+edition = "2021"
+license.workspace = true
+version.workspace = true
+
+[dependencies]
+derive_builder = "0.20"
+once_cell.workspace = true
+parking_lot.workspace = true
+prost.workspace = true
+prost-types.workspace = true
+xmtp_proto.workspace = true
+
+[dev-dependencies]
+xmtp_api_grpc.workspace = true
+tokio.workspace = true
+
+[features]
+http-api = []
+grpc-api = []

xmtp_api_d14n/Cargo.toml

@@ -0,0 +1,4 @@
+# Decentralization API for XMTP
+
+Backwards-compatible with non-d14n endpoints (endpoints/v3/*), includes new d14n
+endpoints (endpoints/d14n).

xmtp_api_d14n/README.md

@@ -0,0 +1,39 @@
+use derive_builder::Builder;
+use prost::Message;
+use std::borrow::Cow;
+use xmtp_proto::traits::{BodyError, Endpoint};
+use xmtp_proto::xmtp::mls::api::v1::FetchKeyPackagesRequest;
+use xmtp_proto::xmtp::xmtpv4::message_api::FILE_DECRIPTOR_SET;
+use xmtp_proto::xmtp::xmtpv4::message_api::{GetInboxIdsRequest, GetInboxIdsResponse};
+
+#[derive(Debug, Builder, Default)]
+#[builder(setter(strip_option))]
+pub struct GetInboxIds {
+    #[builder(setter(into))]
+    envelopes: Vec<ClientEnvelope>,
+}
+
+impl GetInboxIds {
+    pub fn builder() -> GetInboxIdsBuilder {
+        Default::default()
+    }
+}
+
+impl Endpoint for GetInboxIds {
+    type Output = GetInboxIdsResponse;
+
+    fn http_endpoint(&self) -> Cow<'static, str> {
+        todo!()
+    }
+
+    fn grpc_endpoint(&self) -> Cow<'static, str> {
+        crate::path_and_query::<PublishClientEnvelopesRequest>(FILE_DESCRIPTOR_SET)
+    }
+
+    fn body(&self) -> Result<Vec<u8>, BodyError> {
+        Ok(PublishClientEnvelopesRequest {
+            envelopes: self.envelopes.clone(),
+        }
+        .encode_to_vec())
+    }
+}

xmtp_api_d14n/src/endpoints/d14n/get_inbox_ids.rs

@@ -0,0 +1,5 @@
+mod publish_client_envelopes;
+pub use publish_client_envelopes::*;
+
+mod query_envelopes;
+pub use query_envelopes::*;

xmtp_api_d14n/src/endpoints/d14n/mod.rs

@@ -0,0 +1,38 @@
+use derive_builder::Builder;
+use prost::Message;
+use std::borrow::Cow;
+use xmtp_proto::traits::{BodyError, Endpoint};
+use xmtp_proto::xmtp::xmtpv4::envelopes::ClientEnvelope;
+use xmtp_proto::xmtp::xmtpv4::payer_api::PublishClientEnvelopesRequest;
+use xmtp_proto::xmtp::xmtpv4::payer_api::FILE_DESCRIPTOR_SET;
+
+#[derive(Debug, Builder, Default)]
+#[builder(setter(strip_option))]
+pub struct PublishClientEnvelopes {
+    #[builder(setter(into))]
+    envelopes: Vec<ClientEnvelope>,
+}
+
+impl PublishClientEnvelopes {
+    pub fn builder() -> PublishClientEnvelopesBuilder {
+        Default::default()
+    }
+}
+
+impl Endpoint for PublishClientEnvelopes {
+    type Output = ();
+    fn http_endpoint(&self) -> Cow<'static, str> {
+        todo!()
+    }
+
+    fn grpc_endpoint(&self) -> Cow<'static, str> {
+        crate::path_and_query::<PublishClientEnvelopesRequest>(FILE_DESCRIPTOR_SET)
+    }
+
+    fn body(&self) -> Result<Vec<u8>, BodyError> {
+        Ok(PublishClientEnvelopesRequest {
+            envelopes: self.envelopes.clone(),
+        }
+        .encode_to_vec())
+    }
+}

xmtp_api_d14n/src/endpoints/d14n/publish_client_envelopes.rs

@@ -0,0 +1,82 @@
+use derive_builder::Builder;
+use prost::Message;
+use std::borrow::Cow;
+use xmtp_proto::traits::{BodyError, Endpoint};
+use xmtp_proto::xmtp::xmtpv4::message_api::EnvelopesQuery;
+use xmtp_proto::xmtp::xmtpv4::message_api::FILE_DESCRIPTOR_SET;
+use xmtp_proto::xmtp::xmtpv4::message_api::{QueryEnvelopesRequest, QueryEnvelopesResponse};
+
+/// Query a single thing
+#[derive(Debug, Builder, Default, Clone)]
+pub struct QueryEnvelope {
+    #[builder(setter(into))]
+    topics: Vec<Vec<u8>>,
+    #[builder(setter(into))]
+    originator_node_ids: Vec<u32>,
+}
+
+impl QueryEnvelope {
+    pub fn builder() -> QueryEnvelopeBuilder {
+        Default::default()
+    }
+}
+
+impl Endpoint for QueryEnvelope {
+    type Output = QueryEnvelopesResponse;
+
+    fn http_endpoint(&self) -> Cow<'static, str> {
+        todo!()
+    }
+
+    fn grpc_endpoint(&self) -> Cow<'static, str> {
+        crate::path_and_query::<QueryEnvelopesRequest>(FILE_DESCRIPTOR_SET)
+    }
+
+    fn body(&self) -> Result<Vec<u8>, BodyError> {
+        Ok(QueryEnvelopesRequest {
+            query: Some(EnvelopesQuery {
+                topics: self.topics.clone(),
+                originator_node_ids: self.originator_node_ids.clone(),
+                last_seen: None,
+            }),
+            limit: 1,
+        }
+        .encode_to_vec())
+    }
+}
+
+/// Batch Query
+#[derive(Debug, Builder, Default)]
+#[builder(setter(strip_option))]
+pub struct QueryEnvelopes {
+    #[builder(setter(into))]
+    envelopes: EnvelopesQuery,
+    #[builder(setter(into))]
+    limit: u32,
+}
+
+impl QueryEnvelopes {
+    pub fn builder() -> QueryEnvelopesBuilder {
+        Default::default()
+    }
+}
+
+impl Endpoint for QueryEnvelopes {
+    type Output = QueryEnvelopesResponse;
+
+    fn http_endpoint(&self) -> Cow<'static, str> {
+        todo!()
+    }
+
+    fn grpc_endpoint(&self) -> Cow<'static, str> {
+        crate::path_and_query::<QueryEnvelopesRequest>(FILE_DESCRIPTOR_SET)
+    }
+
+    fn body(&self) -> Result<Vec<u8>, BodyError> {
+        Ok(QueryEnvelopesRequest {
+            query: Some(self.envelopes.clone()),
+            limit: self.limit,
+        }
+        .encode_to_vec())
+    }
+}

xmtp_api_d14n/src/endpoints/d14n/query_envelopes.rs

@@ -0,0 +1,5 @@
+pub mod d14n;
+pub mod v3;
+
+pub use d14n::*;
+pub use v3::*;

xmtp_api_d14n/src/endpoints/mod.rs

@@ -0,0 +1 @@
+

xmtp_api_d14n/src/endpoints/v3/identity/mod.rs

@@ -0,0 +1,68 @@
+use derive_builder::Builder;
+use prost::Message;
+use std::borrow::Cow;
+use xmtp_proto::traits::{BodyError, Endpoint};
+use xmtp_proto::xmtp::mls::api::v1::FILE_DESCRIPTOR_SET;
+use xmtp_proto::xmtp::mls::api::v1::{FetchKeyPackagesRequest, FetchKeyPackagesResponse};
+
+#[derive(Debug, Builder, Default)]
+#[builder(setter(strip_option))]
+pub struct FetchKeyPackages {
+    #[builder(setter(into))]
+    installation_keys: Vec<Vec<u8>>,
+}
+
+impl FetchKeyPackages {
+    pub fn builder() -> FetchKeyPackagesBuilder {
+        Default::default()
+    }
+}
+
+impl Endpoint for FetchKeyPackages {
+    type Output = FetchKeyPackagesResponse;
+    fn http_endpoint(&self) -> Cow<'static, str> {
+        todo!()
+    }
+
+    fn grpc_endpoint(&self) -> Cow<'static, str> {
+        crate::path_and_query::<FetchKeyPackagesRequest>(FILE_DESCRIPTOR_SET)
+    }
+
+    fn body(&self) -> Result<Vec<u8>, BodyError> {
+        Ok(FetchKeyPackagesRequest {
+            installation_keys: self.installation_keys.clone(),
+        }
+        .encode_to_vec())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use xmtp_api_grpc::{grpc_client::GrpcClient, LOCALHOST_ADDRESS};
+    use xmtp_proto::api_client::ApiBuilder;
+    use xmtp_proto::traits::Query;
+
+    #[test]
+    fn test_file_descriptor() {
+        let pnq = crate::path_and_query::<FetchKeyPackagesRequest>(FILE_DESCRIPTOR_SET);
+        println!("{}", pnq);
+    }
+
+    #[tokio::test]
+    async fn test_fetch_key_packages() {
+        let mut client = GrpcClient::builder();
+        client.set_app_version("0.0.0".into()).unwrap();
+        client.set_tls(false);
+        client.set_host(LOCALHOST_ADDRESS.to_string());
+        let client = client.build().await.unwrap();
+
+        let endpoint = FetchKeyPackages::builder()
+            .installation_keys(vec![vec![1, 2, 3]])
+            .build()
+            .unwrap();
+
+        let result: FetchKeyPackagesResponse = endpoint.query(&client).await.unwrap();
+        assert_eq!(result.key_packages, vec![]);
+    }
+}