Skip to content

Commit 5e0f8fb

Browse files
arouelarouel
authored
Base64 encode the checksum (#639)
1 parent 6e5c5ad commit 5e0f8fb

12 files changed

+145
-23
lines changed

‎src/integrationTest/java/software/amazon/nio/spi/s3/Containers.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ abstract class Containers {
2323

2424
static {
2525
LOCAL_STACK_CONTAINER = new LocalStackContainer(
26-
DockerImageName.parse("localstack/localstack:2.3.2")
27-
).withServices(S3).withEnv("PROVIDER_OVERRIDE_S3", "v3");
26+
DockerImageName.parse("localstack/localstack:4.2")
27+
).withServices(S3);
2828
LOCAL_STACK_CONTAINER.start();
2929
System.setProperty(S3_SPI_ENDPOINT_PROTOCOL_PROPERTY, "http");
3030
}

‎src/integrationTest/java/software/amazon/nio/spi/s3/FilesNewByteChannelTest.java

+33-11
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,6 @@
2121
import org.junit.jupiter.api.Test;
2222
import org.junit.jupiter.api.TestInstance;
2323

24-
import software.amazon.nio.spi.s3.config.S3NioSpiConfiguration;
25-
2624
@DisplayName("Files$newByteChannel* should read and write on S3")
2725
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
2826
public class FilesNewByteChannelTest {
@@ -76,34 +74,58 @@ public void newByteChannel_READ_WRITE() throws IOException {
7674
assertThat(path).hasContent(text);
7775
}
7876

77+
@Test
78+
@DisplayName("newByteChannel with CRC32 integrity check")
79+
public void newByteChannel_withIntegrityCheck_CRC32() throws Exception {
80+
String text = "we test the integrity check when closing the byte channel";
81+
82+
withEnvironmentVariable("S3_INTEGRITY_CHECK_ALGORITHM", "CRC32").execute(() -> {
83+
var path = Paths.get(URI.create(localStackConnectionEndpoint() + "/" + bucketName + "/bc-integrity-check.txt"));
84+
try (var channel = Files.newByteChannel(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
85+
channel.write(ByteBuffer.wrap(text.getBytes()));
86+
}
87+
88+
assertThat(path).hasContent(text);
89+
});
90+
}
91+
7992
@Test
8093
@DisplayName("newByteChannel with CRC32C integrity check")
8194
public void newByteChannel_withIntegrityCheck_CRC32C() throws Exception {
82-
var path = Paths.get(URI.create(localStackConnectionEndpoint() + "/" + bucketName + "/bc-integrity-check.txt"));
83-
8495
String text = "we test the integrity check when closing the byte channel";
85-
withEnvironmentVariable(S3NioSpiConfiguration.S3_INTEGRITY_CHECK_ALGORITHM_PROPERTY, "CRC32C").execute(() -> {
96+
97+
withEnvironmentVariable("S3_INTEGRITY_CHECK_ALGORITHM", "CRC32C").execute(() -> {
98+
var path = Paths.get(URI.create(localStackConnectionEndpoint() + "/" + bucketName + "/bc-integrity-check.txt"));
8699
try (var channel = Files.newByteChannel(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
87100
channel.write(ByteBuffer.wrap(text.getBytes()));
88101
}
89-
});
90102

91-
assertThat(path).hasContent(text);
103+
assertThat(path).hasContent(text);
104+
});
92105
}
93106

94107
@Test
95108
@DisplayName("newByteChannel with CRC64NVME integrity check")
96109
public void newByteChannel_withIntegrityCheck_CRC64NVME() throws Exception {
97-
var path = Paths.get(URI.create(localStackConnectionEndpoint() + "/" + bucketName + "/bc-integrity-check.txt"));
98-
99110
String text = "we test the integrity check when closing the byte channel";
100-
withEnvironmentVariable(S3NioSpiConfiguration.S3_INTEGRITY_CHECK_ALGORITHM_PROPERTY, "CRC64NVME").execute(() -> {
111+
112+
withEnvironmentVariable("S3_INTEGRITY_CHECK_ALGORITHM", "CRC64NVME").execute(() -> {
113+
var path = (S3Path) Paths.get(URI.create(localStackConnectionEndpoint() + "/" + bucketName + "/bc-integrity-check.txt"));
101114
try (var channel = Files.newByteChannel(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
102115
channel.write(ByteBuffer.wrap(text.getBytes()));
103116
}
117+
118+
assertThat(path).hasContent(text);
104119
});
120+
}
105121

106-
assertThat(path).hasContent(text);
122+
@Test
123+
@DisplayName("newByteChannel with invalid integrity check")
124+
public void newByteChannel_withIntegrityCheck_invalid() throws Exception {
125+
withEnvironmentVariable("S3_INTEGRITY_CHECK_ALGORITHM", "invalid").execute(() -> {
126+
var path = Paths.get(URI.create(localStackConnectionEndpoint() + "/" + bucketName + "/int-check-algo-test.txt"));
127+
assertThatThrownBy(() -> Files.newByteChannel(path)).hasMessage("unknown integrity check algorithm 'invalid'");
128+
});
107129
}
108130

109131
}

‎src/main/java/software/amazon/nio/spi/s3/Crc32FileIntegrityCheck.java

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package software.amazon.nio.spi.s3;
7+
8+
import java.io.IOException;
9+
import java.io.UncheckedIOException;
10+
import java.nio.ByteBuffer;
11+
import java.nio.file.Files;
12+
import java.nio.file.Path;
13+
import software.amazon.awssdk.crt.checksums.CRC32;
14+
import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm;
15+
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
16+
import software.amazon.awssdk.utils.BinaryUtils;
17+
18+
class Crc32FileIntegrityCheck implements S3ObjectIntegrityCheck {
19+
private final byte[] buffer = new byte[16 * 1024];
20+
private final CRC32 checksum = new CRC32();
21+
private final ByteBuffer checksumBuffer = ByteBuffer.allocate(Integer.BYTES);
22+
23+
@Override
24+
public void addChecksumToRequest(Path file, PutObjectRequest.Builder builder) {
25+
checksum.reset();
26+
checksumBuffer.clear();
27+
try (var in = Files.newInputStream(file)) {
28+
int len;
29+
while ((len = in.read(buffer)) != -1) {
30+
checksum.update(buffer, 0, len);
31+
}
32+
checksumBuffer.putInt((int) checksum.getValue());
33+
builder.checksumAlgorithm(ChecksumAlgorithm.CRC32);
34+
builder.checksumCRC32(BinaryUtils.toBase64(checksumBuffer.array()));
35+
} catch (IOException cause) {
36+
throw new UncheckedIOException(cause);
37+
}
38+
}
39+
}

‎src/main/java/software/amazon/nio/spi/s3/Crc32cFileIntegrityCheck.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
import software.amazon.awssdk.crt.checksums.CRC32C;
1414
import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm;
1515
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
16-
import software.amazon.awssdk.utils.internal.Base16;
16+
import software.amazon.awssdk.utils.BinaryUtils;
1717

1818
class Crc32cFileIntegrityCheck implements S3ObjectIntegrityCheck {
1919
private final byte[] buffer = new byte[16 * 1024];
@@ -31,7 +31,7 @@ public void addChecksumToRequest(Path file, PutObjectRequest.Builder builder) {
3131
}
3232
checksumBuffer.putInt((int) checksum.getValue());
3333
builder.checksumAlgorithm(ChecksumAlgorithm.CRC32_C);
34-
builder.checksumCRC32C(Base16.encodeAsString(checksumBuffer.array()));
34+
builder.checksumCRC32C(BinaryUtils.toBase64(checksumBuffer.array()));
3535
} catch (IOException cause) {
3636
throw new UncheckedIOException(cause);
3737
}

‎src/main/java/software/amazon/nio/spi/s3/Crc64nvmeFileIntegrityCheck.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
import software.amazon.awssdk.crt.checksums.CRC64NVME;
1414
import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm;
1515
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
16-
import software.amazon.awssdk.utils.internal.Base16;
16+
import software.amazon.awssdk.utils.BinaryUtils;
1717

1818
class Crc64nvmeFileIntegrityCheck implements S3ObjectIntegrityCheck {
1919
private final byte[] buffer = new byte[16 * 1024];
@@ -31,7 +31,7 @@ public void addChecksumToRequest(Path file, PutObjectRequest.Builder builder) {
3131
}
3232
checksumBuffer.putLong(checksum.getValue());
3333
builder.checksumAlgorithm(ChecksumAlgorithm.CRC64_NVME);
34-
builder.checksumCRC64NVME(Base16.encodeAsString(checksumBuffer.array()));
34+
builder.checksumCRC64NVME(BinaryUtils.toBase64(checksumBuffer.array()));
3535
} catch (IOException cause) {
3636
throw new UncheckedIOException(cause);
3737
}

‎src/main/java/software/amazon/nio/spi/s3/S3FileSystem.java

+3
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,9 @@ public void close() throws IOException {
127127
*/
128128
S3ObjectIntegrityCheck integrityCheck() {
129129
var algorithm = configuration.getIntegrityCheckAlgorithm();
130+
if (algorithm.equalsIgnoreCase("CRC32")) {
131+
return new Crc32FileIntegrityCheck();
132+
}
130133
if (algorithm.equalsIgnoreCase("CRC32C")) {
131134
return new Crc32cFileIntegrityCheck();
132135
}

‎src/main/java/software/amazon/nio/spi/s3/S3FileSystemProvider.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -898,7 +898,9 @@ boolean exists(S3AsyncClient s3Client, S3Path path) throws InterruptedException,
898898
s3Client.headObject(HeadObjectRequest.builder().bucket(path.bucketName()).key(path.getKey()).build())
899899
.get(configuration.getTimeoutLow(), MINUTES);
900900
return true;
901-
} catch (ExecutionException | NoSuchKeyException e) {
901+
} catch (NoSuchKeyException e) {
902+
return false;
903+
} catch (ExecutionException e) {
902904
logger.debug("Could not retrieve object head information", e);
903905
return false;
904906
}

‎src/main/java/software/amazon/nio/spi/s3/config/S3NioSpiConfiguration.java

+16-1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import java.util.Objects;
1313
import java.util.Optional;
1414
import java.util.Properties;
15+
import java.util.Set;
1516
import java.util.regex.Pattern;
1617
import org.slf4j.Logger;
1718
import org.slf4j.LoggerFactory;
@@ -106,6 +107,11 @@ public class S3NioSpiConfiguration extends HashMap<String, Object> {
106107
* The default value of the S3 object integrity check property
107108
*/
108109
public static final String S3_INTEGRITY_CHECK_ALGORITHM_DEFAULT = "disabled";
110+
/**
111+
* Allowed algorithms of the S3 object integrity check property
112+
*/
113+
public static final Set<String> S3_INTEGRITY_CHECK_ALGORITHM_ALLOWED = Set.of(
114+
"CRC32", "CRC32C", "CRC64NVME", S3_INTEGRITY_CHECK_ALGORITHM_DEFAULT.toUpperCase());
109115

110116
private static final Pattern ENDPOINT_REGEXP = Pattern.compile("(\\w[\\w\\-\\.]*)?(:(\\d+))?");
111117

@@ -403,6 +409,7 @@ public S3NioSpiConfiguration withIntegrityCheckAlgorithm(String algorithm) {
403409
} else {
404410
put(S3_INTEGRITY_CHECK_ALGORITHM_PROPERTY, algorithm);
405411
}
412+
validateIntegrityAlgorithm(algorithm);
406413

407414
return this;
408415
}
@@ -542,7 +549,15 @@ public Long getTimeoutHigh() {
542549
* @return the configured value or the default if not overridden
543550
*/
544551
public String getIntegrityCheckAlgorithm() {
545-
return (String) getOrDefault(S3_INTEGRITY_CHECK_ALGORITHM_PROPERTY, S3_INTEGRITY_CHECK_ALGORITHM_DEFAULT);
552+
String algorithm = (String) getOrDefault(S3_INTEGRITY_CHECK_ALGORITHM_PROPERTY, S3_INTEGRITY_CHECK_ALGORITHM_DEFAULT);
553+
validateIntegrityAlgorithm(algorithm);
554+
return algorithm;
555+
}
556+
557+
private void validateIntegrityAlgorithm(String algorithm) {
558+
if (!S3_INTEGRITY_CHECK_ALGORITHM_ALLOWED.contains(algorithm.toUpperCase())) {
559+
throw new UnsupportedOperationException("unknown integrity check algorithm '" + algorithm + "'");
560+
}
546561
}
547562

548563
/**

‎src/test/java/software/amazon/nio/spi/s3/Crc32FileIntegrityCheckTest.java

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package software.amazon.nio.spi.s3;
7+
8+
import static java.nio.file.StandardOpenOption.*;
9+
import static org.assertj.core.api.Assertions.*;
10+
11+
import java.io.IOException;
12+
import java.nio.file.Files;
13+
import java.nio.file.Path;
14+
15+
import org.junit.jupiter.api.Test;
16+
import org.junit.jupiter.api.io.TempDir;
17+
18+
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
19+
20+
class Crc32FileIntegrityCheckTest {
21+
22+
@Test
23+
void test(@TempDir Path tempDir) throws IOException {
24+
var integrityCheck = new Crc32FileIntegrityCheck();
25+
var file = tempDir.resolve("test");
26+
Files.writeString(file, "hello world!", CREATE_NEW);
27+
var putObjectRequest = PutObjectRequest.builder();
28+
integrityCheck.addChecksumToRequest(file, putObjectRequest);
29+
assertThat(putObjectRequest.build().checksumCRC32()).isEqualTo("A7TCbQ==");
30+
}
31+
32+
}

‎src/test/java/software/amazon/nio/spi/s3/Crc32cFileIntegrityCheckTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ void test(@TempDir Path tempDir) throws IOException {
2626
Files.writeString(file, "hello world!", CREATE_NEW);
2727
var putObjectRequest = PutObjectRequest.builder();
2828
integrityCheck.addChecksumToRequest(file, putObjectRequest);
29-
assertThat(putObjectRequest.build().checksumCRC32C()).isEqualTo("49CB5777");
29+
assertThat(putObjectRequest.build().checksumCRC32C()).isEqualTo("SctXdw==");
3030
}
3131

3232
}

‎src/test/java/software/amazon/nio/spi/s3/Crc64nvmeFileIntegrityCheckTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ void test(@TempDir Path tempDir) throws IOException {
2626
Files.writeString(file, "hello world!", CREATE_NEW);
2727
var putObjectRequest = PutObjectRequest.builder();
2828
integrityCheck.addChecksumToRequest(file, putObjectRequest);
29-
assertThat(putObjectRequest.build().checksumCRC64NVME()).isEqualTo("D9160D1FA8E418E3");
29+
assertThat(putObjectRequest.build().checksumCRC64NVME()).isEqualTo("2RYNH6jkGOM=");
3030
}
3131

3232
}

‎src/test/java/software/amazon/nio/spi/s3/config/S3NioSpiConfigurationTest.java

+11-2
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import static org.assertj.core.api.BDDAssertions.entry;
1616

1717
import static org.assertj.core.api.BDDAssertions.then;
18+
import static org.assertj.core.api.BDDAssertions.thenExceptionOfType;
1819
import static org.junit.jupiter.api.Assertions.assertThrows;
1920
import org.junit.jupiter.api.BeforeEach;
2021
import org.junit.jupiter.api.DisplayName;
@@ -290,16 +291,24 @@ public void withAndGetIntegrityCheckAlgorithm() throws Exception {
290291
then(config.withIntegrityCheckAlgorithm("CRC64NVME").getIntegrityCheckAlgorithm()).isEqualTo("CRC64NVME");
291292

292293
var map = new HashMap<String, String>();
293-
map.put(S3_INTEGRITY_CHECK_ALGORITHM_PROPERTY, "1212");
294+
map.put(S3_INTEGRITY_CHECK_ALGORITHM_PROPERTY, "invalid");
294295
var c = new S3NioSpiConfiguration(map);
295296

296-
then(c.getIntegrityCheckAlgorithm()).isEqualTo("1212");
297+
thenExceptionOfType(UnsupportedOperationException.class)
298+
.isThrownBy(() -> c.getIntegrityCheckAlgorithm())
299+
.withMessage("unknown integrity check algorithm 'invalid'");
300+
301+
thenExceptionOfType(UnsupportedOperationException.class)
302+
.isThrownBy(() -> c.withIntegrityCheckAlgorithm("unknown algorithm"))
303+
.withMessage("unknown integrity check algorithm 'unknown algorithm'");
297304

298305
withEnvironmentVariable("S3_INTEGRITY_CHECK_ALGORITHM", "CRC32C")
299306
.execute(() -> then(new S3NioSpiConfiguration().getIntegrityCheckAlgorithm()).isEqualTo("CRC32C"));
300307

301308
withEnvironmentVariable("S3_INTEGRITY_CHECK_ALGORITHM", "CRC64NVME")
302309
.execute(() -> then(new S3NioSpiConfiguration().getIntegrityCheckAlgorithm()).isEqualTo("CRC64NVME"));
310+
311+
then(new S3NioSpiConfiguration(Map.of(S3_INTEGRITY_CHECK_ALGORITHM_PROPERTY, "CRC32")).getIntegrityCheckAlgorithm()).isEqualTo("CRC32");
303312
}
304313

305314
}

0 commit comments

Comments
 (0)