From aa73c404bf01641a37ede12c98e4aff66f053f88 Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Sun, 16 Jun 2024 21:34:30 -0400 Subject: [PATCH 1/8] Replace users lib with whoami for better compatibility --- Cargo.lock | 39 ++++++++++++++++++++++++++++----------- rust/Cargo.toml | 2 +- rust/src/security/user.rs | 10 ++-------- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8d28bd3..4f8a14c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -742,9 +742,9 @@ dependencies = [ "thiserror", "tokio", "url", - "users", "uuid", "which", + "whoami", ] [[package]] @@ -1108,7 +1108,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.1", "smallvec", "windows-targets 0.52.5", ] @@ -1396,6 +1396,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.1" @@ -1748,15 +1757,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "users" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24cc0f6d6f267b73e5a2cadf007ba8f9bc39c6a6f9666f8cf25ea809a153b032" -dependencies = [ - "libc", -] - [[package]] name = "uuid" version = "1.8.0" @@ -1788,6 +1788,12 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.92" @@ -1864,6 +1870,17 @@ dependencies = [ "rustix", ] +[[package]] +name = "whoami" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" +dependencies = [ + "redox_syscall 0.4.1", + "wasite", + "web-sys", +] + [[package]] name = "winapi-util" version = "0.1.8" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index dc9bb10..433f435 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -41,9 +41,9 @@ socket2 = "0.5" thiserror = { workspace = true } tokio = { workspace = true, features = ["rt", "rt-multi-thread", "net", "io-util", "macros", "sync", "time"] } url = "2" -users = { version = "0.11", default-features = false } uuid = { version = "1", features = ["v4"] } which = { version = "4", optional = true } +whoami = "1" [build-dependencies] prost-build = { version = "0.12", optional = true } diff --git a/rust/src/security/user.rs b/rust/src/security/user.rs index 623b955..0a0225a 100644 --- a/rust/src/security/user.rs +++ b/rust/src/security/user.rs @@ -6,7 +6,7 @@ use std::fs; use std::io; use std::path::PathBuf; -use users::get_current_username; +use whoami::username; use crate::proto::common::CredentialsProto; use crate::proto::common::TokenProto; @@ -334,13 +334,7 @@ impl User { } pub(crate) fn get_simpler_user() -> UserInfo { - let effective_user = env::var(HADOOP_USER_NAME).ok().unwrap_or_else(|| { - get_current_username() - .unwrap() - .to_str() - .unwrap() - .to_string() - }); + let effective_user = env::var(HADOOP_USER_NAME).ok().unwrap_or_else(username); UserInfo { real_user: None, effective_user: Some(effective_user), From 99560e1e1387b91b2b748f69f1ecf362cdd3eb0d Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Sun, 16 Jun 2024 21:34:54 -0400 Subject: [PATCH 2/8] Fix clippy lint --- rust/src/security/user.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/src/security/user.rs b/rust/src/security/user.rs index 0a0225a..414ca40 100644 --- a/rust/src/security/user.rs +++ b/rust/src/security/user.rs @@ -125,6 +125,7 @@ impl BlockTokenIdentifier { } #[derive(Debug)] +#[allow(dead_code)] pub struct Token { pub alias: String, pub identifier: Vec, From d46837c723f00ae0ce8a7ffbbfbdbfd9957a8176 Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Sun, 16 Jun 2024 22:00:10 -0400 Subject: [PATCH 3/8] Run CI builds on mac and windows --- .github/workflows/rust-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.yml index 5811d24..c9be1d9 100644 --- a/.github/workflows/rust-test.yml +++ b/.github/workflows/rust-test.yml @@ -27,8 +27,8 @@ jobs: matrix: os: - ubuntu-latest - # - macos-11 - # - windows-latest + - macos-11 + - windows-latest runs-on: ${{ matrix.os }} steps: From f5ab2e5723846d01b6551410e639206fb3b9c5ab Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Mon, 17 Jun 2024 06:35:39 -0400 Subject: [PATCH 4/8] Don't check with benchmark --- .github/workflows/rust-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.yml index 8022d15..d68299e 100644 --- a/.github/workflows/rust-test.yml +++ b/.github/workflows/rust-test.yml @@ -41,7 +41,7 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: build and lint with clippy - run: cargo clippy --all-targets --features integration-test,benchmark -- -D warnings + run: cargo clippy --all-targets --features integration-test -- -D warnings - name: Check docs run: cargo doc @@ -50,7 +50,7 @@ jobs: run: cargo check --tests - name: Check all features - run: cargo check --all-targets --features integration-test,benchmark + run: cargo check --all-targets --features integration-test test: strategy: From c39e0efa18f639113bb35da78b9b165d85225193 Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Mon, 17 Jun 2024 06:59:44 -0400 Subject: [PATCH 5/8] Better feature flagging for bechmarks --- .github/workflows/rust-test.yml | 4 ++-- rust/Cargo.toml | 6 ++++-- rust/benches/io.rs | 14 +++++++++----- rust/benches/rpc.rs | 5 ++--- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.yml index d68299e..8022d15 100644 --- a/.github/workflows/rust-test.yml +++ b/.github/workflows/rust-test.yml @@ -41,7 +41,7 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: build and lint with clippy - run: cargo clippy --all-targets --features integration-test -- -D warnings + run: cargo clippy --all-targets --features integration-test,benchmark -- -D warnings - name: Check docs run: cargo doc @@ -50,7 +50,7 @@ jobs: run: cargo check --tests - name: Check all features - run: cargo check --all-targets --features integration-test + run: cargo check --all-targets --features integration-test,benchmark test: strategy: diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 971c781..bc5093c 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -21,8 +21,6 @@ cipher = "0.4" crc = "3.2" ctr = "0.9" des = "0.8" -# Just used for benchmarks -fs-hdfs3 = { version = "0.1.12", optional = true } futures = "0.3" g2p = "1" hex = "0.4" @@ -46,6 +44,10 @@ uuid = { version = "1", features = ["v4"] } which = { version = "4", optional = true } whoami = "1" +[target.'cfg(unix)'.dependencies] +# Just used for benchmarks +fs-hdfs3 = { version = "0.1.12", optional = true } + [build-dependencies] prost-build = { version = "0.12", optional = true } protobuf-src = { version = "1.1", optional = true } diff --git a/rust/benches/io.rs b/rust/benches/io.rs index 761ac55..a77c965 100644 --- a/rust/benches/io.rs +++ b/rust/benches/io.rs @@ -1,8 +1,8 @@ +#![cfg(unix)] use std::collections::HashSet; use bytes::{Buf, BufMut, BytesMut}; use criterion::*; -use hdfs::hdfs::get_hdfs; use hdfs_native::{ minidfs::{DfsFeatures, MiniDfs}, Client, WriteOptions, @@ -49,8 +49,9 @@ fn bench(c: &mut Criterion) { }) }); group.sample_size(50); + #[cfg(unix)] group.bench_function("read-libhdfs", |b| { - let fs = get_hdfs().unwrap(); + let fs = hdfs::hdfs::get_hdfs().unwrap(); b.iter(|| { let mut buf = BytesMut::zeroed(ints_to_write * 4); let mut bytes_read = 0; @@ -79,8 +80,9 @@ fn bench(c: &mut Criterion) { .iter(|| async { reader.read_range(0, reader.file_length()).await.unwrap() }) }); group.sample_size(50); + #[cfg(unix)] group.bench_function("read-libhdfs", |b| { - let fs = get_hdfs().unwrap(); + let fs = hdfs::hdfs::get_hdfs().unwrap(); b.iter(|| { let mut buf = BytesMut::zeroed(ints_to_write * 4); let mut bytes_read = 0; @@ -120,8 +122,9 @@ fn bench(c: &mut Criterion) { }) }); group.sample_size(10); + #[cfg(unix)] group.bench_function("write-libhdfs", |b| { - let fs = get_hdfs().unwrap(); + let fs = hdfs::hdfs::get_hdfs().unwrap(); b.iter(|| { let mut buf = buf.clone(); let writer = fs.create_with_overwrite("/bench-write", true).unwrap(); @@ -154,8 +157,9 @@ fn bench(c: &mut Criterion) { }) }); group.sample_size(10); + #[cfg(unix)] group.bench_function("write-libhdfs", |b| { - let fs = get_hdfs().unwrap(); + let fs = hdfs::hdfs::get_hdfs().unwrap(); b.iter(|| { let mut buf = buf.clone(); let writer = fs diff --git a/rust/benches/rpc.rs b/rust/benches/rpc.rs index 0b7cd29..8dabc85 100644 --- a/rust/benches/rpc.rs +++ b/rust/benches/rpc.rs @@ -1,7 +1,6 @@ use std::collections::HashSet; use criterion::*; -use hdfs::hdfs::get_hdfs; use hdfs_native::{minidfs::MiniDfs, Client, WriteOptions}; fn bench(c: &mut Criterion) { @@ -25,14 +24,14 @@ fn bench(c: &mut Criterion) { .unwrap(); }); - let fs = get_hdfs().unwrap(); - let mut group = c.benchmark_group("rpc"); group.bench_function("getFileInfo-native", |b| { b.to_async(&rt) .iter(|| async { client.get_file_info("/bench").await.unwrap() }) }); + #[cfg(unix)] group.bench_function("getFileInfo-libhdfs", |b| { + let fs = hdfs::hdfs::get_hdfs().unwrap(); b.iter(|| fs.get_file_status("/bench").unwrap()) }); } From 7b7d484d960c0a8a841d978fb0b31f52b4612ec4 Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Mon, 17 Jun 2024 07:56:56 -0400 Subject: [PATCH 6/8] Update rust/benches/io.rs --- rust/benches/io.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/benches/io.rs b/rust/benches/io.rs index a77c965..718935c 100644 --- a/rust/benches/io.rs +++ b/rust/benches/io.rs @@ -1,4 +1,3 @@ -#![cfg(unix)] use std::collections::HashSet; use bytes::{Buf, BufMut, BytesMut}; From 64421ce1b73245ba02bba6d34e272ec014a79f04 Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Mon, 17 Jun 2024 14:13:00 +0000 Subject: [PATCH 7/8] Try to fix lint --- rust/benches/io.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/benches/io.rs b/rust/benches/io.rs index 718935c..e53b218 100644 --- a/rust/benches/io.rs +++ b/rust/benches/io.rs @@ -1,5 +1,6 @@ use std::collections::HashSet; +#[allow(unused_imports)] use bytes::{Buf, BufMut, BytesMut}; use criterion::*; use hdfs_native::{ From 938e6dab51dcf8d051aefb9a2aa368f45d742f86 Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Mon, 17 Jun 2024 14:41:59 +0000 Subject: [PATCH 8/8] Use auto anylinux and use latest macos --- .github/workflows/python-release.yml | 1 + .github/workflows/python-test.yml | 1 + .github/workflows/rust-test.yml | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index 9df5ffb..68e0f95 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -34,6 +34,7 @@ jobs: target: ${{ matrix.target }} args: --release --out dist --find-interpreter --manifest-path python/Cargo.toml sccache: 'true' + manylinux: auto - name: Upload wheels uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 9241232..d83500d 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -73,6 +73,7 @@ jobs: with: args: --release --out dist --find-interpreter --manifest-path python/Cargo.toml sccache: 'true' + manylinux: auto - name: Upload wheels if: github.ref == 'refs/heads/master' diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.yml index 8022d15..626568f 100644 --- a/.github/workflows/rust-test.yml +++ b/.github/workflows/rust-test.yml @@ -27,7 +27,7 @@ jobs: matrix: os: - ubuntu-latest - - macos-11 + - macos-latest - windows-latest runs-on: ${{ matrix.os }} @@ -58,7 +58,7 @@ jobs: matrix: os: - ubuntu-latest - # - macos-11 + # - macos-latest # - windows-latest runs-on: ${{ matrix.os }} env: