Custom Digest-md5 implementation

Author: Kimahriman

Cargo.lock

@@ -43,21 +43,20 @@ All other settings are generally assumed to be the defaults currently. For insta
 
 ### Mac
 ```
-brew install gsasl krb5
+brew install krb5
 # You might need these env vars on newer Macs
 export BINDGEN_EXTRA_CLANG_ARGS="-I/opt/homebrew/include"
 export LIBRARY_PATH=/opt/homebrew/lib
-cargo build --features token,kerberos
+cargo build --features kerberos
 ```
 
 ### Ubuntu
 ```
-apt-get install clang libkrb5-dev libgsasl-dev
-cargo build --features token,kerberos
+apt-get install clang libkrb5-dev
+cargo build --features kerberos
 ```
 
 ## Crate features
-- `token` - enables token based DIGEST-MD5 authentication support. This uses the `gsasl` native library and only supports authentication, not integrity or confidentiality
 - `kerberos` - enables kerberos GSSAPI authentication support. This uses the `libgssapi` crate and supports integrity as well as confidentiality
 
 ## Object store implementation
@@ -67,7 +66,7 @@ An object_store implementation for HDFS is provided in the [hdfs-native-object-s
 The tests are mostly integration tests that utilize a small Java application in `rust/mindifs/` that runs a custom `MiniDFSCluster`. To run the tests, you need to have Java, Maven, Hadoop binaries, and Kerberos tools available and on your path. Any Java version between 8 and 17 should work.
 
 ```bash
-cargo test -p hdfs-native --features token,kerberos,intergation-test
+cargo test -p hdfs-native --features kerberos,intergation-test
 ```
 
 ### Python tests

README.md

@@ -29,6 +29,5 @@ which = "4"
 
 [features]
 kerberos = ["hdfs-native/kerberos"]
-token = ["hdfs-native/token"]
 
 integration-test = ["hdfs-native/integration-test"]

crates/hdfs-native-object-store/Cargo.toml

@@ -17,14 +17,16 @@ chrono = { workspace = true }
 crc = "3.1.0-beta.1"
 futures = { workspace = true }
 g2p = "1"
-gsasl-sys = { version = "0.2", default-features = false, optional = true }
 libc = "0.2"
 libgssapi = { version = "0.7", default-features = false, optional = true }
 log = "0.4"
+md5 = "0.7"
 num-traits = "0.2"
 once_cell = "1"
 prost = "0.12"
 prost-types = "0.12"
+rand = "0.8"
+regex = "1"
 roxmltree = "0.18"
 socket2 = "0.5"
 thiserror = "1"
@@ -48,7 +50,6 @@ which = "4"
 
 [features]
 kerberos = ["libgssapi"]
-token = ["gsasl-sys"]
 
 generate-protobuf = ["prost-build", "protobuf-src"]
 integration-test = ["which"]
@@ -64,4 +65,4 @@ harness = false
 
 [[bench]]
 name = "rpc"
-harness = false
+harness = false

crates/hdfs-native/Cargo.toml

@@ -1,9 +1,6 @@
 use std::io::Result;
 
 fn main() -> Result<()> {
-    #[cfg(feature = "token")]
-    println!("cargo:rustc-link-lib=gsasl");
-
     #[cfg(feature = "generate-protobuf")]
     {
         std::env::set_var("PROTOC", protobuf_src::protoc());

crates/hdfs-native/build.rs

@@ -64,9 +64,7 @@ public static void main(String args[]) throws Exception {
             conf.set(DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, "hdfs/localhost@" + kdc.getRealm());
             conf.set(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, "true");
             conf.set(DFSConfigKeys.IGNORE_SECURE_PORTS_FOR_TESTING_KEY, "true");
-            if (flags.contains("data_transfer_security")) {
-                conf.set(DFS_DATA_TRANSFER_PROTECTION_KEY, "authentication");
-            }
+            conf.set(DFS_DATA_TRANSFER_PROTECTION_KEY, "authentication");
         }
 
         HdfsConfiguration hdfsConf = new HdfsConfiguration(conf);

crates/hdfs-native/minidfs/src/main/java/main/Main.java

@@ -28,13 +28,11 @@ use crate::proto::common::rpc_response_header_proto::RpcStatusProto;
 use crate::proto::common::TokenProto;
 use crate::proto::hdfs::DatanodeIdProto;
 use crate::proto::{common, hdfs};
+use crate::security::sasl::SaslDatanodeConnection;
 use crate::security::sasl::{SaslReader, SaslRpcClient, SaslWriter};
 use crate::security::user::UserInfo;
 use crate::{HdfsError, Result};
 
-#[cfg(feature = "token")]
-use crate::security::sasl::SaslDatanodeConnection;
-
 const PROTOCOL: &str = "org.apache.hadoop.hdfs.protocol.ClientProtocol";
 const DATA_TRANSFER_VERSION: u16 = 28;
 const MAX_PACKET_HEADER_SIZE: usize = 33;
@@ -531,7 +529,6 @@ impl DatanodeConnection {
         let stream = connect(&url).await?;
 
         // If the token has an identifier, we can do SASL negotation
-        #[cfg(feature = "token")]
         let stream = if token.identifier.is_empty() {
             stream
         } else {

crates/hdfs-native/src/hdfs/connection.rs

@@ -10,7 +10,6 @@ use which::which;
 #[derive(PartialEq, Eq, Hash, Debug)]
 pub enum DfsFeatures {
     Security,
-    DataTransferSecurity,
     Token,
     Privacy,
     HA,
@@ -27,7 +26,6 @@ impl DfsFeatures {
             DfsFeatures::ViewFS => "viewfs",
             DfsFeatures::Privacy => "privacy",
             DfsFeatures::Security => "security",
-            DfsFeatures::DataTransferSecurity => "data_transfer_security",
             DfsFeatures::Token => "token",
             DfsFeatures::RBF => "rbf",
         }
@@ -58,11 +56,6 @@ impl MiniDfs {
         for feature in features.iter() {
             feature_args.push(feature.as_str());
         }
-        // If the `token` feature is enabled, we need to force the data transfer protection
-        #[cfg(feature = "token")]
-        if !features.contains(&DfsFeatures::DataTransferSecurity) {
-            feature_args.push(DfsFeatures::DataTransferSecurity.as_str());
-        }
 
         let mut child = Command::new(mvn_exec)
             .args([