Move tagged_blob macro here from jf-utils

It conceptually belongs here anyways. Also, this fixes the
dependency tree so that `commit` can depend on `tagged_blob` without
depending on `jf-utils` (which would create a cycle). This will
enable tagged base 64 serialization support directly on the
`Commitment` type, obviating the need for many newtype structs.
That will be done in a subsequent change.

During the move, I streamlined the `tagged_blob` macro a bit. I
removed the `TaggedBlob` wrapper type which was only used to add
custom serde impls and to convert to and from for serializing other
types. Instead, serialization now goes directly through `TaggedBase64`,
and I have implemented serialization and deserialization for the
`TaggedBase64` type itself.

Since `TaggedBlob` is gone, I renamed the macro `tagged` for simplicity.

Author: jbearer

Cargo.toml

@@ -20,9 +20,12 @@ wasm-debug = ["dep:console_error_panic_hook"]
 build-cli = ["dep:clap"]
 
 [dependencies]
-crc-any = { version = "2.4.1", default-features = false }
-
+ark-serialize = { version = "0.3.0", default-features = false, features = ["derive"] }
 base64 = "0.13.0"
+crc-any = { version = "2.4.1", default-features = false }
+serde = { version = "1.0", features = ["derive"] }
+snafu = { version = "0.7", features = ["backtraces"] }
+tagged-base64-macros = { path = "macros" }
 
 # Command line argument processing
 clap = { version = "4.0", optional = true, features = ["derive"] }
@@ -40,9 +43,12 @@ web-sys = { version = "0.3.49", optional = true, features = ["console", "Headers
 console_error_panic_hook = { version = "0.1.7", optional = true }
 
 [dev-dependencies]
+ark-std = { version = "0.3.0", default-features = false }
+bincode = "1.3"
 getrandom = { version = "0.2", features = ["js"] }
 quickcheck = "1.0"
 quickcheck_macros = "1.0"
+serde_json = "1.0"
 wasm-bindgen-test = { version = "0.3.28" }
 
 [profile.release]

macros/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "tagged-base64-macros"
+description = "Procedural macros associated with tagged-base64"
+version = "0.2.0"
+authors = ["Espresso Systems <hello@espressosys.com>"]
+edition = "2018"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+ark-std = { version = "0.3.0", default-features = false }
+syn = { version = "1.0", features = ["extra-traits"] }
+quote = "1.0"
+
+[dev-dependencies]
+ark-serialize = { version = "0.3.0", default-features = false, features = ["derive"] }
+ark-bls12-381 = { version = "0.3.0", default-features = false, features = ["curve"] }
+bincode = { version = "1.3.3", default-features = false }
+rand_chacha = { version = "0.3.1" }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0.61"

macros/src/lib.rs

@@ -0,0 +1,156 @@
+// Copyright (c) 2022 Espresso Systems (espressosys.com)
+#![no_std]
+
+extern crate proc_macro;
+
+use proc_macro::TokenStream;
+use quote::quote;
+use syn::{parse_macro_input, AttributeArgs, Item, Meta, NestedMeta};
+
+/// Derive serdes for a type which serializes as a binary blob.
+///
+/// This macro can be used to easily derive friendly serde implementations for a binary type which
+/// implements [CanonicalSerialize](ark_serialize::CanonicalSerialize) and
+/// [CanonicalDeserialize](ark_serialize::CanonicalDeserialize). This is useful for cryptographic
+/// primitives and other types which do not have a human-readable serialization, but which may be
+/// embedded in structs with a human-readable serialization. The serde implementations derived by
+/// this macro will serialize the type as bytes for binary encodings and as base 64 for human
+/// readable encodings.
+///
+/// Specifically, this macro does 4 things when applied to a type definition:
+/// * It adds `#[derive(Serialize, Deserialize)]` to the type definition, along with serde
+///   attributes to serialize using [TaggedBase64].
+/// * It creates an implementation of [Tagged] for the type using the specified tag. This tag will
+///   be used to identify base 64 strings which represent this type in human-readable encodings.
+/// * It creates an implementation of `TryFrom<TaggedBase64>` for the type `T`, which is needed to
+///   make the `serde(try_from)` attribute work.
+/// * It creates implementations of [Display](ark_std::fmt::Display) and
+///   [FromStr](ark_std::str::FromStr) using tagged base 64 as a display format. This allows tagged
+///   blob types to be conveniently displayed and read to and from user interfaces in a manner
+///   consistent with how they are serialized.
+///
+/// Usage example:
+///
+/// ```
+/// #[macro_use] extern crate tagged_base64_macros;
+/// use ark_serialize::*;
+///
+/// #[tagged("PRIM")]
+/// #[derive(Clone, CanonicalSerialize, CanonicalDeserialize, /* any other derives */)]
+/// pub struct CryptoPrim(
+///     // This type can only be serialied as an opaque, binary blob using ark_serialize.
+///     pub(crate) ark_bls12_381::Fr,
+/// );
+/// ```
+///
+/// The type `CryptoPrim` can now be serialized as binary:
+/// ```
+/// # use ark_serialize::*;
+/// # use ark_std::UniformRand;
+/// # use tagged_base64_macros::tagged;
+/// # use rand_chacha::{ChaChaRng, rand_core::SeedableRng};
+/// # #[tagged("PRIM")]
+/// # #[derive(Clone, CanonicalSerialize, CanonicalDeserialize, /* any other derives */)]
+/// # struct CryptoPrim(ark_bls12_381::Fr);
+/// # let crypto_prim = CryptoPrim(ark_bls12_381::Fr::rand(&mut ChaChaRng::from_seed([42; 32])));
+/// bincode::serialize(&crypto_prim).unwrap();
+/// ```
+/// or as base64:
+/// ```
+/// # use ark_serialize::*;
+/// # use ark_std::UniformRand;
+/// # use tagged_base64_macros::tagged;
+/// # use rand_chacha::{ChaChaRng, rand_core::SeedableRng};
+/// # #[tagged("PRIM")]
+/// # #[derive(Clone, CanonicalSerialize, CanonicalDeserialize, /* any other derives */)]
+/// # struct CryptoPrim(ark_bls12_381::Fr);
+/// # let crypto_prim = CryptoPrim(ark_bls12_381::Fr::rand(&mut ChaChaRng::from_seed([42; 32])));
+/// serde_json::to_string(&crypto_prim).unwrap();
+/// ```
+/// which will produce a tagged base64 string like
+/// "PRIM~8oaujwbov8h4eEq7HFpqW6mIXhVbtJGxLUgiKrGpMCoJ".
+#[proc_macro_attribute]
+pub fn tagged(args: TokenStream, input: TokenStream) -> TokenStream {
+    let args = parse_macro_input!(args as AttributeArgs);
+    let input = parse_macro_input!(input as Item);
+    let (name, generics) = match &input {
+        Item::Struct(item) => (&item.ident, &item.generics),
+        Item::Enum(item) => (&item.ident, &item.generics),
+        _ => panic!("expected struct or enum"),
+    };
+    let (impl_generics, ty_generics, where_clause) = generics.split_for_impl();
+    let tag: &dyn quote::ToTokens = match args.as_slice() {
+        [NestedMeta::Lit(tag)] => tag,
+        [NestedMeta::Meta(Meta::Path(path))] => path,
+        x => panic!(
+            "`tagged` takes one argument, the tag, as a string literal or expression, found {:?}",
+            x
+        ),
+    };
+    let output = quote! {
+        #[derive(serde::Serialize, serde::Deserialize)]
+        #[serde(try_from = "tagged_base64::TaggedBase64", into = "tagged_base64::TaggedBase64")]
+        // Override the inferred bound for Serialize/Deserialize impls. If we're converting to and
+        // from CanonicalBytes as an intermediate, the impls should work for any generic parameters.
+        #[serde(bound = "")]
+        #input
+
+        impl #impl_generics tagged_base64::Tagged for #name #ty_generics #where_clause {
+            fn tag() -> ark_std::string::String {
+                ark_std::string::String::from(#tag)
+            }
+        }
+
+        impl #impl_generics core::convert::TryFrom<tagged_base64::TaggedBase64>
+            for #name #ty_generics
+        #where_clause
+        {
+            type Error = tagged_base64::Tb64Error;
+            fn try_from(t: tagged_base64::TaggedBase64) -> Result<Self, Self::Error> {
+                if t.tag() == <#name #ty_generics>::tag() {
+                    <Self as CanonicalDeserialize>::deserialize(t.as_ref())
+                        .map_err(|_| tagged_base64::Tb64Error::InvalidData)
+                } else {
+                    Err(tagged_base64::Tb64Error::InvalidTag)
+                }
+            }
+        }
+
+        impl #impl_generics core::convert::From<#name #ty_generics> for tagged_base64::TaggedBase64
+            #where_clause
+        {
+            fn from(x: #name #ty_generics) -> Self {
+                (&x).into()
+            }
+        }
+
+        impl #impl_generics core::convert::From<&#name #ty_generics> for tagged_base64::TaggedBase64
+            #where_clause
+        {
+            fn from(x: &#name #ty_generics) -> Self {
+                let mut bytes = ark_std::vec![];
+                x.serialize(&mut bytes).unwrap();
+                Self::new(&<#name #ty_generics>::tag(), &bytes).unwrap()
+            }
+        }
+
+        impl #impl_generics ark_std::fmt::Display for #name #ty_generics #where_clause {
+            fn fmt(&self, f: &mut ark_std::fmt::Formatter<'_>) -> ark_std::fmt::Result {
+                ark_std::write!(
+                    f, "{}",
+                    tagged_base64::TaggedBase64::from(self)
+                )
+            }
+        }
+
+        impl #impl_generics ark_std::str::FromStr for #name #ty_generics #where_clause {
+            type Err = tagged_base64::Tb64Error;
+            fn from_str(s: &str) -> Result<Self, Self::Err> {
+                use core::convert::TryFrom;
+                Self::try_from(tagged_base64::TaggedBase64::from_str(s)?)
+                    .map_err(|_| tagged_base64::Tb64Error::InvalidData)
+            }
+        }
+    };
+    output.into()
+}

src/lib.rs

@@ -41,15 +41,23 @@
 //! well as display and input in a user interface.
 
 #![allow(clippy::unused_unit)]
+use ark_serialize::*;
 use core::fmt;
 #[cfg(target_arch = "wasm32")]
 use core::fmt::Display;
 use core::str::FromStr;
 use crc_any::CRC;
+use serde::{
+    de::{Deserialize, Deserializer, Error as DeError},
+    ser::{Error as SerError, Serialize, Serializer},
+};
+use snafu::Snafu;
 
 #[cfg(target_arch = "wasm32")]
 use wasm_bindgen::prelude::*;
 
+pub use tagged_base64_macros::tagged;
+
 /// Separator that does not appear in URL-safe base64 encoding and can
 /// appear in URLs without percent-encoding.
 pub const TB64_DELIM: char = '~';
@@ -60,13 +68,49 @@ pub const TB64_CONFIG: base64::Config = base64::URL_SAFE_NO_PAD;
 /// A structure holding a string tag, vector of bytes, and a checksum
 /// covering the tag and the bytes.
 #[cfg_attr(target_arch = "wasm32", wasm_bindgen)]
-#[derive(Clone, Debug, Eq, PartialEq)]
+#[derive(Clone, Debug, Eq, PartialEq, CanonicalSerialize, CanonicalDeserialize)]
 pub struct TaggedBase64 {
     tag: String,
     value: Vec<u8>,
     checksum: u8,
 }
 
+impl Serialize for TaggedBase64 {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        if serializer.is_human_readable() {
+            // If we are serializing to a human-readable format, be nice and just display the
+            // tagged base 64 as a string.
+            Serialize::serialize(&self.to_string(), serializer)
+        } else {
+            // For binary formats, convert to bytes (using CanonicalSerialize) and write the bytes.
+            let mut bytes = vec![];
+            CanonicalSerialize::serialize(self, &mut bytes).map_err(S::Error::custom)?;
+            Serialize::serialize(&bytes, serializer)
+        }
+    }
+}
+
+impl<'a> Deserialize<'a> for TaggedBase64 {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'a>,
+    {
+        if deserializer.is_human_readable() {
+            // If we are deserializing a human-readable format, the serializer would have written
+            // the tagged base 64 as a string, so deserialize a string and then parse it.
+            Self::from_str(Deserialize::deserialize(deserializer)?).map_err(D::Error::custom)
+        } else {
+            // Otherwise, this is a binary format; deserialize bytes and then convert the bytes to
+            // TaggedBase64 using CanonicalDeserialize.
+            let bytes = <Vec<u8> as Deserialize>::deserialize(deserializer)?;
+            CanonicalDeserialize::deserialize(bytes.as_slice()).map_err(D::Error::custom)
+        }
+    }
+}
+
 /// JavaScript-compatible wrapper for TaggedBase64
 ///
 /// The primary difference is that JsTaggedBase64 returns errors
@@ -77,7 +121,7 @@ pub struct JsTaggedBase64 {
     tb64: TaggedBase64,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Snafu)]
 pub enum Tb64Error {
     /// An invalid character was found in the tag.
     InvalidTag,
@@ -87,38 +131,25 @@ pub enum Tb64Error {
     MissingChecksum,
     /// An invalid byte was found while decoding the base64-encoded value.
     /// The offset and offending byte are provided.
-    InvalidByte(usize, u8),
+    #[snafu(display(
+        "An invalid byte ({:#x}) was found at offset {} while decoding the base64-encoded value.",
+        byte,
+        offset
+    ))]
+    InvalidByte { offset: usize, byte: u8 },
     /// The last non-padding input symbol's encoded 6 bits have
     /// nonzero bits that will be discarded. This is indicative of
     /// corrupted or truncated Base64. Unlike InvalidByte, which
     /// reports symbols that aren't in the alphabet, this error is for
     /// symbols that are in the alphabet but represent nonsensical
     /// encodings.
-    InvalidLastSymbol(usize, u8),
+    InvalidLastSymbol { offset: usize, byte: u8 },
     /// The length of the base64-encoded value is invalid.
     InvalidLength,
     /// The checksum was truncated or did not match.
     InvalidChecksum,
-}
-
-impl fmt::Display for Tb64Error {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match *self {
-            Tb64Error::InvalidTag =>
-                write!(f, "An invalid character was found in the tag."),
-            Tb64Error::MissingDelimiter =>
-                write!(f, "Missing delimiter ({}).", TB64_DELIM),
-            Tb64Error::MissingChecksum =>
-                write!(f, "Missing checksum in value."),
-            Tb64Error::InvalidByte(offset, byte) =>
-                write!(f, "An invalid byte ({:#0x}) was found at offset {} while decoding the base64-encoded value. The offset and offending byte are provided.", byte, offset),
-            Tb64Error::InvalidLastSymbol(offset, byte) => write!(f, "The last non-padding input symbol's encoded 6 bits have nonzero bits that will be discarded. This is indicative of corrupted or truncated Base64. Unlike InvalidByte, which reports symbols that aren't in the alphabet, this error is for symbols that are in the alphabet but represent nonsensical encodings. Invalid byte ({:#0x}) at offset {}.", byte, offset),
-            Tb64Error::InvalidLength =>
-                write!(f, "The length of the base64-encoded value is invalid."),
-            Tb64Error::InvalidChecksum =>
-                write!(f, "The checksum was truncated or did not match."),
-        }
-    }
+    /// The data did not encode the expected type.
+    InvalidData,
 }
 
 /// Converts a TaggedBase64 value to a String.
@@ -296,15 +327,23 @@ impl TaggedBase64 {
     /// Wraps the underlying base64 decoder.
     pub fn decode_raw(value: &str) -> Result<Vec<u8>, Tb64Error> {
         base64::decode_config(value, TB64_CONFIG).map_err(|err| match err {
-            base64::DecodeError::InvalidByte(offset, byte) => Tb64Error::InvalidByte(offset, byte),
+            base64::DecodeError::InvalidByte(offset, byte) => {
+                Tb64Error::InvalidByte { offset, byte }
+            }
             base64::DecodeError::InvalidLength => Tb64Error::InvalidLength,
             base64::DecodeError::InvalidLastSymbol(offset, byte) => {
-                Tb64Error::InvalidLastSymbol(offset, byte)
+                Tb64Error::InvalidLastSymbol { offset, byte }
             }
         })
     }
 }
 
+impl AsRef<[u8]> for TaggedBase64 {
+    fn as_ref(&self) -> &[u8] {
+        &self.value
+    }
+}
+
 /// Converts any object that supports the Display trait to a JsValue for
 /// passing to Javascript.
 ///
@@ -375,3 +414,15 @@ impl JsTaggedBase64 {
         self.tb64.to_string()
     }
 }
+
+/// Trait for types whose serialization is not human-readable.
+///
+/// Such types have a human-readable tag which is used to identify tagged base
+/// 64 blobs representing a serialization of that type.
+///
+/// Rather than implement this trait manually, it is recommended to use the
+/// [macro@tagged] macro to specify a tag for your type. That macro also
+/// derives appropriate serde implementations for serializing as an opaque blob.
+pub trait Tagged {
+    fn tag() -> String;
+}