Skip to content

Commit

Permalink
bincode v1 を除去する (#443)
Browse files Browse the repository at this point in the history
依存 crate に bincode v1 と v2 が混在している。bincode v1 を除去する。

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

- **リファクタ**
	- ジオメトリのスライシングとタイルの書き込み処理において、シリアライゼーションとデシリアライゼーションの方法を改善しました。
	- シリアライゼーションとデシリアライゼーションのエラータイプを更新し、より具体的なエラータイプを使用するように変更しました。

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
  • Loading branch information
ciscorn authored Mar 8, 2024
1 parent b2e121a commit 8aec2e4
Show file tree
Hide file tree
Showing 17 changed files with 78 additions and 58 deletions.
4 changes: 2 additions & 2 deletions app/src-tauri/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,13 @@ tauri-build = { version = "1.5.1", features = [] }
[dependencies]
serde_json = "1.0"
serde = { version = "1.0", features = ["derive"] }
tauri = { version = "1.5.4", features = [ "shell-open", "fs-read-dir", "dialog-all"] }
tauri = { version = "1.6.1", features = [ "shell-open", "fs-read-dir", "dialog-all"] }
nusamai = { path = "../../nusamai" }
nusamai-geometry = { path = "../../nusamai-geometry" }
nusamai-geojson = { path = "../../nusamai-geojson" }
nusamai-plateau = { path = "../../nusamai-plateau" }
nusamai-citygml = {path = "../../nusamai-citygml" }
log = "0.4.20"
log = "0.4.21"
tauri-plugin-log = { git = "https://github.com/tauri-apps/plugins-workspace", branch = "v1" }
thiserror = "1.0.57"

Expand Down
4 changes: 2 additions & 2 deletions nusamai-3dtiles/nusamai-3dtiles-json/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
serde = { version = "1.0.196", features = ["derive"] }
serde_json = { version = "1.0.113", features = ["indexmap", "float_roundtrip"] }
serde = { version = "1.0.197", features = ["derive"] }
serde_json = { version = "1.0.114", features = ["indexmap", "float_roundtrip"] }
serde_repr = "0.1.18"
nusamai-gltf-json = { path = "../../nusamai-gltf/nusamai-gltf-json" }

Expand Down
10 changes: 5 additions & 5 deletions nusamai-citygml/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@ default = ["serde"]
serde = ["dep:serde", "serde_json", "nusamai-geometry/serde"]

[dependencies]
ahash = "0.8.8"
chrono = { version = "0.4.34", features = ["serde"], default-features = false }
indexmap = { version = "2.2", features = ["serde"] }
log = "0.4.20"
ahash = "0.8.11"
chrono = { version = "0.4.35", features = ["serde"], default-features = false }
indexmap = { version = "2.2.5", features = ["serde"] }
log = "0.4.21"
macros = { path = "./macros" }
nusamai-geometry = { path = "../nusamai-geometry", features = ["serde"]}
nusamai-projection = { path = "../nusamai-projection"}
quick-xml = "0.31"
serde = { version = "1.0", features = ["derive"], optional = true }
serde_json = { version = "1.0.113", features = ["indexmap"], optional = true }
serde_json = { version = "1.0.114", features = ["indexmap"], optional = true }
thiserror = "1.0"
url = { version = "2.5.0", features = ["serde"] }
6 changes: 3 additions & 3 deletions nusamai-czml/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ version = "0.1.0"
edition = "2021"

[dependencies]
chrono = { version = "0.4.34", features = ["serde"] }
serde = { version = "1.0.196", features = ["derive"] }
serde_json = { version = "1.0.113", features = ["float_roundtrip"] }
chrono = { version = "0.4.35", features = ["serde"] }
serde = { version = "1.0.197", features = ["derive"] }
serde_json = { version = "1.0.114", features = ["float_roundtrip"] }
nusamai-geometry = { path = "../nusamai-geometry" }

[dev-dependencies]
Expand Down
2 changes: 1 addition & 1 deletion nusamai-geojson/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@ edition = "2021"
[dependencies]
geojson = "0.24.1"
nusamai-geometry = { path = "../nusamai-geometry" }
serde_json = { version = "1.0.113", features = ["indexmap"] }
serde_json = { version = "1.0.114", features = ["indexmap"] }
12 changes: 6 additions & 6 deletions nusamai-geometry/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,16 @@ edition = "2021"

[dependencies]
num-traits = "0.2.18"
serde = { version = "1.0.196", features = ["derive"], optional = true }
serde = { version = "1.0.197", features = ["derive"], optional = true }

[dev-dependencies]
byteorder = "1.5.0"
geo-types = "0.7.12"
geo-types = "0.7.13"
geojson = "0.24.1"
indexmap = "2.2.3"
indexmap = "2.2.5"
quick-xml = "0.31.0"
thiserror = "1.0.57"
earcut-rs = { git = "https://github.com/MIERUNE/earcut-rs.git" }
clap = { version = "4.5.0", features = ["derive"] }
serde = { version = "1.0.196", features = ["derive"] }
serde_json = "1.0.113"
clap = { version = "4.5.2", features = ["derive"] }
serde = { version = "1.0.197", features = ["derive"] }
serde_json = "1.0.114"
6 changes: 3 additions & 3 deletions nusamai-gltf/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@ version = "0.1.0"

[dependencies]
nusamai-gltf-json = { "path" = "nusamai-gltf-json" }
clap = { version = "4.5.0", features = ["derive"] }
clap = { version = "4.5.2", features = ["derive"] }
nusamai-geometry = { path = "../nusamai-geometry" }
quick-xml = "0.31.0"
thiserror = "1.0.57"
earcut-rs = { git = "https://github.com/MIERUNE/earcut-rs.git" }
indexmap = "2.2.3"
indexmap = "2.2.5"
byteorder = "1.5.0"
serde_json = "1.0.113"
serde_json = "1.0.114"

[dev-dependencies]
glob = "0.3.1"
4 changes: 2 additions & 2 deletions nusamai-gltf/nusamai-gltf-json/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
serde = { version = "1.0.196", features = ["derive"] }
serde_json = { version = "1.0.113", features = ["indexmap", "float_roundtrip"] }
serde = { version = "1.0.197", features = ["derive"] }
serde_json = { version = "1.0.114", features = ["indexmap", "float_roundtrip"] }
serde_repr = "0.1.18"

[dev-dependencies]
Expand Down
2 changes: 1 addition & 1 deletion nusamai-gpkg/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ sqlx = { version = "0.7.3", features = ["sqlite", "runtime-tokio"] }
nusamai-geometry = { path = "../nusamai-geometry" }
thiserror = "1.0.57"
url = "2.5.0"
indexmap = "2.2.3"
indexmap = "2.2.5"

[dev-dependencies]
tokio = { version = "1.36", features = ["full"] }
4 changes: 2 additions & 2 deletions nusamai-mvt/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ version = "0.1.0"
edition = "2021"

[dependencies]
ahash = "0.8.8"
indexmap = "2.2.3"
ahash = "0.8.11"
indexmap = "2.2.5"
prost = "0.12.3"

[build-dependencies]
Expand Down
12 changes: 6 additions & 6 deletions nusamai-plateau/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,20 @@ serde = ["dep:serde"]

[dependencies]
quick-xml = "0.31.0"
serde = { version = "1.0.196", features = ["derive", "rc"], optional = true }
serde = { version = "1.0.197", features = ["derive", "rc"], optional = true }
nusamai-citygml = { path = "../nusamai-citygml", features = ["serde"]}
nusamai-geometry = { path = "../nusamai-geometry" }
chrono = { version = "0.4.34", features = ["serde"], default-features = false }
chrono = { version = "0.4.35", features = ["serde"], default-features = false }
url = "2.5.0"
stretto = "0.8.3"
hashbrown = { version = "0.14.3", features = ["serde"] }
indexmap = "2.2.3"
log = "0.4.20"
indexmap = "2.2.5"
log = "0.4.21"

[dev-dependencies]
zstd = { version = "0.13.0", features = ["zdict_builder"] }
bincode = { version = "2.0.0-rc.3", default-features = false, features = ["std", "serde"] }
clap = { version = "4.5", features = ["derive"] }
serde = { version = "1.0.196", features = ["derive"] }
serde = { version = "1.0.197", features = ["derive"] }
lz4_flex = "0.11.2"
serde_json = "1.0.113"
serde_json = "1.0.114"
20 changes: 10 additions & 10 deletions nusamai/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,16 @@ version.workspace = true
edition = "2021"

[dependencies]
indexmap = { version = "2.2.3", features = ["serde"] }
rayon = "1.8.1"
serde = { version = "1.0.196", features = ["derive"] }
indexmap = { version = "2.2.5", features = ["serde"] }
rayon = "1.9.0"
serde = { version = "1.0.197", features = ["derive"] }
nusamai-plateau = { path = "../nusamai-plateau" }
nusamai-citygml = { path = "../nusamai-citygml" }
quick-xml = "0.31.0"
clap = { version = "4.5.0", features = ["derive", "string"] }
clap = { version = "4.5.2", features = ["derive", "string"] }
thiserror = "1.0.57"
ctrlc = "3.4.2"
bincode = "1.3.3"
bincode = { version = "2.0.0-rc.3", default-features = false, features = ["std", "serde"] }
lz4_flex = "0.11.2"
nusamai-geojson = { path = "../nusamai-geojson" }
nusamai-gltf = { path = "../nusamai-gltf" }
Expand All @@ -24,7 +24,7 @@ nusamai-czml = { path = "../nusamai-czml" }
nusamai-projection = { path = "../nusamai-projection" }
nusamai-mvt = { path = "../nusamai-mvt" }
geojson = "0.24.1"
serde_json = { version = "1.0.113", features = ["indexmap"] }
serde_json = { version = "1.0.114", features = ["indexmap"] }
url = "2.5.0"
nusamai-gpkg = { path = "../nusamai-gpkg" }
tokio = { version = "1.36", features = ["full"] }
Expand All @@ -33,20 +33,20 @@ hashbrown = { version = "0.14.3", features = ["serde"] }
ext-sort = { version = "0.1.4", features = ["memory-limit"] }
deepsize = "0.2.0"
serde_bytes = "0.11.14"
log = { version = "0.4.20" }
log = { version = "0.4.21" }
pretty_env_logger = "0.5.0"
itertools = "0.12.1"
prost = "0.12.3"
bytesize = "1.3.0"
ahash = "0.8.8"
ahash = "0.8.11"
nusamai-shapefile = { path = "../nusamai-shapefile" }
shapefile = "0.5.0"
earcut-rs = { git = "https://github.com/MIERUNE/earcut-rs.git" }
glob = "0.3.1"
shellexpand = "3.1.0"
kml = "0.8.5"
nusamai-kml = { path = "../nusamai-kml" }
image = { version = "0.24.8", default-features = false, features = ["tiff", "jpeg", "jpeg_rayon", "webp-encoder"] }
image = { version = "0.24.9", default-features = false, features = ["tiff", "jpeg", "jpeg_rayon", "webp-encoder"] }
flate2 = "1.0.28"
chrono = "0.4.35"

Expand All @@ -56,4 +56,4 @@ tokio = { version = "1.36", features = ["full"] }
nusamai-geometry = { path = "../nusamai-geometry" }
byteorder = "1.5.0"
glob = "0.3.1"
assert_cmd = "2.0.13"
assert_cmd = "2.0.14"
7 changes: 5 additions & 2 deletions nusamai/src/sink/cesiumtiles/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,8 @@ fn geometry_slicing_stage(
min_zoom: u8,
max_zoom: u8,
) -> Result<()> {
let bincode_config = bincode::config::standard();

// Convert CityObjects to sliced features
upstream.into_iter().par_bridge().try_for_each(|parcel| {
feedback.ensure_not_canceled()?;
Expand All @@ -182,7 +184,7 @@ fn geometry_slicing_stage(
feedback.ensure_not_canceled()?;

if let Value::Object(obj) = &parcel.entity.root {
let bytes = bincode::serialize(&feature).unwrap();
let bytes = bincode::serde::encode_to_vec(&feature, bincode_config).unwrap();
let serialized_feature = SerializedSlicedFeature {
tile_id: tile_id_conv.zxy_to_id(z, x, y),
typename: obj.typename.to_string(),
Expand Down Expand Up @@ -253,6 +255,7 @@ fn tile_writing_stage(
) -> Result<()> {
let ellipsoid = nusamai_projection::ellipsoid::wgs84();
let contents: Arc<Mutex<Vec<TileContent>>> = Default::default();
let bincode_config = bincode::config::standard();

// Make a glTF (.glb) file for each tile
receiver_sorted
Expand Down Expand Up @@ -314,7 +317,7 @@ fn tile_writing_stage(
feedback.ensure_not_canceled()?;

let feature = {
let mut feature: SlicedFeature = bincode::deserialize(&serialized_feat.body)
let (mut feature, _): (SlicedFeature, _) = bincode::serde::decode_from_slice(&serialized_feat.body, bincode_config)
.map_err(|err| {
PipelineError::Other(format!(
"Failed to deserialize a sliced feature: {:?}",
Expand Down
10 changes: 6 additions & 4 deletions nusamai/src/sink/cesiumtiles/sort.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ impl<T> ExternalChunk<T> for BincodeExternalChunk<T>
where
T: serde::ser::Serialize + serde::de::DeserializeOwned,
{
type SerializationError = bincode::Error;
type DeserializationError = bincode::Error;
type SerializationError = bincode::error::EncodeError;
type DeserializationError = bincode::error::DecodeError;

fn new(reader: io::Take<io::BufReader<fs::File>>) -> Self {
Self {
Expand All @@ -27,8 +27,9 @@ where
mut chunk_writer: &mut io::BufWriter<fs::File>,
items: impl IntoIterator<Item = T>,
) -> Result<(), Self::SerializationError> {
let bincode_config = bincode::config::standard();
for item in items.into_iter() {
bincode::serialize_into(&mut chunk_writer, &item)?;
bincode::serde::encode_into_std_write(&item, &mut chunk_writer, bincode_config)?;
}
Ok(())
}
Expand All @@ -41,10 +42,11 @@ where
type Item = Result<T, <Self as ExternalChunk<T>>::DeserializationError>;

fn next(&mut self) -> Option<Self::Item> {
let bincode_config = bincode::config::standard();
if self.reader.limit() == 0 {
None
} else {
match bincode::deserialize_from(&mut self.reader) {
match bincode::serde::decode_from_std_read(&mut self.reader, bincode_config) {
Ok(result) => Some(Ok(result)),
Err(err) => Some(Err(err)),
}
Expand Down
19 changes: 15 additions & 4 deletions nusamai/src/sink/mvt/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,8 @@ fn geometry_slicing_stage(
sender_sliced: mpsc::SyncSender<SerializedSlicedFeature>,
mvt_options: &MvtParams,
) -> Result<()> {
let bincode_config = bincode::config::standard();

// Convert CityObjects to sliced features
upstream.into_iter().par_bridge().try_for_each(|parcel| {
feedback.ensure_not_canceled()?;
Expand All @@ -211,7 +213,7 @@ fn geometry_slicing_stage(
geometry: mpoly,
properties: parcel.entity.root.clone(),
};
let bytes = bincode::serialize(&feature).unwrap();
let bytes = bincode::serde::encode_to_vec(&feature, bincode_config).unwrap();
let sfeat = SerializedSlicedFeature {
tile_id: tile_id_conv.zxy_to_id(z, x, y),
body: bytes,
Expand Down Expand Up @@ -333,9 +335,18 @@ fn make_tile(default_detail: i32, serialized_feats: &[SerializedSlicedFeature])
let mut int_ring_buf = Vec::new();
let mut int_ring_buf2 = Vec::new();
let extent = 1 << default_detail;
let bincode_config = bincode::config::standard();

for serialized_feat in serialized_feats {
let feat: SlicedFeature = bincode::deserialize(&serialized_feat.body).unwrap();
let mpoly = feat.geometry;
let (feature, _): (SlicedFeature, _) = bincode::serde::decode_from_slice(
&serialized_feat.body,
bincode_config,
)
.map_err(|err| {
PipelineError::Other(format!("Failed to deserialize a sliced feature: {:?}", err))
})?;

let mpoly = feature.geometry;
let mut int_mpoly = MultiPolygon2::<i16>::new();

for poly in &mpoly {
Expand Down Expand Up @@ -403,7 +414,7 @@ fn make_tile(default_detail: i32, serialized_feats: &[SerializedSlicedFeature])
let mut id = None;
let mut tags: Vec<u32> = Vec::new();

let layer = if let object::Value::Object(obj) = &feat.properties {
let layer = if let object::Value::Object(obj) = &feature.properties {
let layer = layers.entry_ref(obj.typename.as_ref()).or_default();

// Encode attributes as MVT tags
Expand Down
10 changes: 6 additions & 4 deletions nusamai/src/sink/mvt/sort.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ impl<T> ExternalChunk<T> for BincodeExternalChunk<T>
where
T: serde::ser::Serialize + serde::de::DeserializeOwned,
{
type SerializationError = bincode::Error;
type DeserializationError = bincode::Error;
type SerializationError = bincode::error::EncodeError;
type DeserializationError = bincode::error::DecodeError;

fn new(reader: io::Take<io::BufReader<fs::File>>) -> Self {
Self {
Expand All @@ -27,8 +27,9 @@ where
mut chunk_writer: &mut io::BufWriter<fs::File>,
items: impl IntoIterator<Item = T>,
) -> Result<(), Self::SerializationError> {
let bincode_config = bincode::config::standard();
for item in items.into_iter() {
bincode::serialize_into(&mut chunk_writer, &item)?;
bincode::serde::encode_into_std_write(&item, &mut chunk_writer, bincode_config)?;
}
Ok(())
}
Expand All @@ -41,10 +42,11 @@ where
type Item = Result<T, <Self as ExternalChunk<T>>::DeserializationError>;

fn next(&mut self) -> Option<Self::Item> {
let bincode_config = bincode::config::standard();
if self.reader.limit() == 0 {
None
} else {
match bincode::deserialize_from(&mut self.reader) {
match bincode::serde::decode_from_std_read(&mut self.reader, bincode_config) {
Ok(result) => Some(Ok(result)),
Err(err) => Some(Err(err)),
}
Expand Down
4 changes: 3 additions & 1 deletion nusamai/src/sink/serde/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ impl DataSink for SerdeSink {

fn run(&mut self, upstream: Receiver, feedback: &Feedback, _schema: &Schema) -> Result<()> {
let (sender, receiver) = std::sync::mpsc::sync_channel(1000);
let bincode_config = bincode::config::standard();

let (ra, rb) = rayon::join(
|| {
Expand All @@ -76,7 +77,8 @@ impl DataSink for SerdeSink {
feedback.ensure_not_canceled()?;

buf.clear();
bincode::serialize_into(buf as &mut Vec<u8>, &parcel.entity).unwrap();
bincode::serde::encode_into_std_write(parcel.entity, buf, bincode_config)
.unwrap();
if sender.send(lz4_flex::compress_prepend_size(buf)).is_err() {
return Err(PipelineError::Canceled);
};
Expand Down

0 comments on commit 8aec2e4

Please sign in to comment.