Skip to content

Commit 1344465

Browse files
arouelarouel
committed
Use a temporary directory per file system
Use a temporary directory per file system and preserve the directory structure locally as it is remote.
1 parent 9fd29c5 commit 1344465

File tree

5 files changed

+106
-25
lines changed

5 files changed

+106
-25
lines changed

‎src/main/java/software/amazon/nio/spi/s3/S3FileSystem.java

+24-1
Original file line numberDiff line numberDiff line change
@@ -56,14 +56,16 @@ public class S3FileSystem extends FileSystem {
5656

5757
// private S3AsyncClient client;
5858
private final S3NioSpiConfiguration configuration;
59+
private final Path temporaryDirectory;
5960

6061
/**
6162
* Create a filesystem that represents the bucket specified by the URI
6263
*
6364
* @param provider the provider to be used with this fileSystem
6465
* @param config the configuration to use; can be null to use a default configuration
66+
* @param temporaryDirectory the local temporary directory for the S3 object
6567
*/
66-
S3FileSystem(S3FileSystemProvider provider, S3NioSpiConfiguration config) {
68+
S3FileSystem(S3FileSystemProvider provider, S3NioSpiConfiguration config, Path temporaryDirectory) {
6769
configuration = (config == null) ? new S3NioSpiConfiguration() : config;
6870
bucketName = configuration.getBucketName();
6971

@@ -82,6 +84,7 @@ public class S3FileSystem extends FileSystem {
8284

8385
clientProvider = new S3ClientProvider(configuration);
8486
this.provider = provider;
87+
this.temporaryDirectory = temporaryDirectory;
8588
}
8689

8790
/**
@@ -488,6 +491,26 @@ boolean deregisterClosedChannel(S3SeekableByteChannel closedChannel) {
488491
return openChannels.remove(closedChannel);
489492
}
490493

494+
/**
495+
* Creates a new local temporary file for the specified S3 path to a file living underneath the
496+
* {@link #temporaryDirectory()}.
497+
*
498+
* @return new local temporary file
499+
*/
500+
Path createTempFile(S3Path path) throws IOException {
501+
if (path.isDirectory()) {
502+
throw new IllegalArgumentException("path must be a file");
503+
}
504+
if (path.getNameCount() == 1) {
505+
Path newPath = temporaryDirectory.resolve(path.getFileName().toString());
506+
return Files.createFile(newPath);
507+
}
508+
Path parent = temporaryDirectory.resolve(path.getParent().toString());
509+
Files.createDirectories(parent);
510+
Path newPath = parent.resolve(path.getFileName().toString());
511+
return Files.createFile(newPath);
512+
}
513+
491514
/**
492515
* Tests if two S3 filesystems are equal
493516
*

‎src/main/java/software/amazon/nio/spi/s3/S3FileSystemProvider.java

+8-1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import static software.amazon.nio.spi.s3.util.TimeOutUtils.logAndGenerateExceptionOnTimeOut;
1212

1313
import java.io.IOException;
14+
import java.io.UncheckedIOException;
1415
import java.net.URI;
1516
import java.nio.channels.AsynchronousFileChannel;
1617
import java.nio.channels.FileChannel;
@@ -253,7 +254,13 @@ public FileSystem getFileSystem(URI uri) {
253254
if (info.accessKey() != null) {
254255
config.withCredentials(info.accessKey(), info.accessSecret());
255256
}
256-
return new S3FileSystem(this, config);
257+
Path temporaryDirectory;
258+
try {
259+
temporaryDirectory = Files.createTempDirectory("aws-s3-nio-");
260+
} catch (IOException cause) {
261+
throw new UncheckedIOException(cause);
262+
}
263+
return new S3FileSystem(this, config, temporaryDirectory);
257264
});
258265
}
259266

‎src/main/java/software/amazon/nio/spi/s3/S3WritableByteChannel.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ class S3WritableByteChannel implements SeekableByteChannel {
5353
throw new NoSuchFileException("File at path:" + path + " does not exist yet");
5454
}
5555

56-
tempFile = Files.createTempFile("aws-s3-nio-", ".tmp");
56+
tempFile = path.getFileSystem().createTempFile(path);
5757
if (exists) {
5858
s3TransferUtil.downloadToLocalFile(path, tempFile);
5959
}

‎src/test/java/software/amazon/nio/spi/s3/S3FileSystemTest.java

+38-6
Original file line numberDiff line numberDiff line change
@@ -6,28 +6,27 @@
66
package software.amazon.nio.spi.s3;
77

88
import static org.assertj.core.api.BDDAssertions.then;
9+
import static org.assertj.core.api.BDDAssertions.thenThrownBy;
910
import static org.junit.jupiter.api.Assertions.assertEquals;
1011
import static org.junit.jupiter.api.Assertions.assertFalse;
1112
import static org.junit.jupiter.api.Assertions.assertNotNull;
1213
import static org.junit.jupiter.api.Assertions.assertThrows;
1314
import static org.junit.jupiter.api.Assertions.assertTrue;
14-
import static org.mockito.Mockito.lenient;
1515
import static software.amazon.nio.spi.s3.Constants.PATH_SEPARATOR;
16-
import static software.amazon.nio.spi.s3.S3Matchers.anyConsumer;
1716

1817
import java.io.IOException;
1918
import java.net.URI;
2019
import java.nio.file.FileSystems;
20+
import java.nio.file.Files;
21+
import java.nio.file.Path;
2122
import java.util.Collections;
22-
import java.util.concurrent.CompletableFuture;
2323
import org.junit.jupiter.api.AfterEach;
2424
import org.junit.jupiter.api.BeforeEach;
2525
import org.junit.jupiter.api.Test;
2626
import org.junit.jupiter.api.extension.ExtendWith;
2727
import org.mockito.Mock;
2828
import org.mockito.junit.jupiter.MockitoExtension;
2929
import software.amazon.awssdk.services.s3.S3AsyncClient;
30-
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
3130

3231
@ExtendWith(MockitoExtension.class)
3332
public class S3FileSystemTest {
@@ -43,8 +42,6 @@ public void init() {
4342
provider = new S3FileSystemProvider();
4443
s3FileSystem = (S3FileSystem) provider.getFileSystem(s3Uri);
4544
s3FileSystem.clientProvider = new FixedS3ClientProvider(mockClient);
46-
lenient().when(mockClient.headObject(anyConsumer())).thenReturn(
47-
CompletableFuture.supplyAsync(() -> HeadObjectResponse.builder().contentLength(100L).build()));
4845
}
4946

5047
@AfterEach
@@ -124,6 +121,41 @@ public void getPathMatcher() {
124121
s3FileSystem.getPathMatcher("glob:*.*").getClass());
125122
}
126123

124+
@Test
125+
void createTempFile() throws IOException {
126+
var temporaryDirectory = s3FileSystemTemporaryDirectory();
127+
128+
thenThrownBy(() -> s3FileSystem.createTempFile(S3Path.getPath(s3FileSystem, "/dir/")))
129+
.isInstanceOf(IllegalArgumentException.class)
130+
.hasMessage("path must be a file");
131+
thenThrownBy(() -> s3FileSystem.createTempFile(S3Path.getPath(s3FileSystem, "/dir1/dir2/dir3/")))
132+
.isInstanceOf(IllegalArgumentException.class)
133+
.hasMessage("path must be a file");
134+
135+
var key1 = "file1";
136+
var tempFile1 = s3FileSystem.createTempFile(S3Path.getPath(s3FileSystem, key1));
137+
then(tempFile1).exists().isEqualTo(temporaryDirectory.resolve(key1));
138+
139+
var key2 = "/file2";
140+
var tempFile2 = s3FileSystem.createTempFile(S3Path.getPath(s3FileSystem, key2));
141+
then(tempFile2).exists().isEqualTo(temporaryDirectory.resolve(key2.substring(1)));
142+
143+
var key3 = "/dir1/dir2/file3";
144+
var tempFile3 = s3FileSystem.createTempFile(S3Path.getPath(s3FileSystem, key3));
145+
then(tempFile3).exists().isEqualTo(temporaryDirectory.resolve(key3.substring(1)));
146+
147+
var key4 = "dir1/dir2/file4";
148+
var tempFile4 = s3FileSystem.createTempFile(S3Path.getPath(s3FileSystem, key4));
149+
then(tempFile4).exists().isEqualTo(temporaryDirectory.resolve(key4));
150+
}
151+
152+
private Path s3FileSystemTemporaryDirectory() throws IOException {
153+
var tempFile0 = s3FileSystem.createTempFile(S3Path.getPath(s3FileSystem, "file0"));
154+
var temporaryDirectory = tempFile0.getParent();
155+
Files.delete(tempFile0);
156+
return temporaryDirectory;
157+
}
158+
127159
@Test
128160
public void testGetOpenChannelsIsNotModifiable() {
129161
//

‎src/test/java/software/amazon/nio/spi/s3/S3WritableByteChannelTest.java

+35-16
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,13 @@
1616
import java.io.IOException;
1717
import java.nio.ByteBuffer;
1818
import java.nio.file.FileAlreadyExistsException;
19+
import java.nio.file.FileVisitResult;
1920
import java.nio.file.Files;
2021
import java.nio.file.NoSuchFileException;
2122
import java.nio.file.Path;
23+
import java.nio.file.SimpleFileVisitor;
2224
import java.nio.file.StandardOpenOption;
25+
import java.nio.file.attribute.BasicFileAttributes;
2326
import java.util.Set;
2427
import java.util.concurrent.TimeoutException;
2528
import java.util.stream.Stream;
@@ -43,12 +46,13 @@ class S3WritableByteChannelTest {
4346

4447
@Test
4548
@DisplayName("when file exists and constructor is invoked with option `CREATE_NEW` should throw FileAlreadyExistsException")
46-
void whenFileExistsAndCreateNewShouldThrowFileAlreadyExistsException() throws InterruptedException, TimeoutException {
49+
void whenFileExistsAndCreateNewShouldThrowFileAlreadyExistsException() throws InterruptedException, TimeoutException, IOException {
4750
S3FileSystemProvider provider = mock();
4851
when(provider.exists(any(S3AsyncClient.class), any())).thenReturn(true);
4952

5053
S3FileSystem fs = mock();
5154
when(fs.provider()).thenReturn(provider);
55+
when(fs.createTempFile(any(S3Path.class))).thenReturn(Files.createTempFile("", ""));
5256

5357
var file = S3Path.getPath(fs, "somefile");
5458
assertThatThrownBy(() -> new S3WritableByteChannel(file, mock(), mock(), Set.of(CREATE_NEW)))
@@ -77,6 +81,7 @@ void shouldBeSeekable() throws InterruptedException, TimeoutException, IOExcepti
7781

7882
S3FileSystem fs = mock();
7983
when(fs.provider()).thenReturn(provider);
84+
when(fs.createTempFile(any(S3Path.class))).thenReturn(Files.createTempFile("", ""));
8085

8186
var file = S3Path.getPath(fs, "somefile");
8287
var channel = new S3WritableByteChannel(file, mock(), mock(), Set.of(READ, WRITE));
@@ -103,6 +108,7 @@ void shouldNotThrowWhen(boolean fileExists, Set<StandardOpenOption> openOptions)
103108

104109
S3FileSystem fs = mock();
105110
when(fs.provider()).thenReturn(provider);
111+
when(fs.createTempFile(any(S3Path.class))).thenReturn(Files.createTempFile("", ""));
106112

107113
var file = S3Path.getPath(fs, "somefile");
108114
new S3WritableByteChannel(file, mock(), mock(), openOptions).close();
@@ -116,6 +122,7 @@ void shouldBeOpenBeforeClose() throws InterruptedException, TimeoutException, IO
116122

117123
S3FileSystem fs = mock();
118124
when(fs.provider()).thenReturn(provider);
125+
when(fs.createTempFile(any(S3Path.class))).thenReturn(Files.createTempFile("", ""));
119126

120127
var file = S3Path.getPath(fs, "somefile");
121128
try(var channel = new S3WritableByteChannel(file, mock(), mock(), Set.of(CREATE))){
@@ -131,6 +138,7 @@ void shouldBeNotOpenAfterClose() throws InterruptedException, TimeoutException,
131138

132139
S3FileSystem fs = mock();
133140
when(fs.provider()).thenReturn(provider);
141+
when(fs.createTempFile(any(S3Path.class))).thenReturn(Files.createTempFile("", ""));
134142

135143
var file = S3Path.getPath(fs, "somefile");
136144
var channel = new S3WritableByteChannel(file, mock(), mock(), Set.of(CREATE));
@@ -143,16 +151,20 @@ void shouldBeNotOpenAfterClose() throws InterruptedException, TimeoutException,
143151
void tmpFileIsCleanedUpAfterClose(@TempDir Path tempDir) throws InterruptedException, TimeoutException, IOException {
144152
S3FileSystemProvider provider = mock();
145153
when(provider.exists(any(S3AsyncClient.class), any())).thenReturn(false);
146-
S3FileSystem fs = mock();
147-
when(fs.provider()).thenReturn(provider);
148-
var file = S3Path.getPath(fs, "somefile");
149-
150-
var channel = new S3WritableByteChannel(file, mock(), mock(), Set.of(CREATE));
151-
152-
var countAfterOpening = countTemporaryFiles(tempDir);
153-
channel.close();
154-
var countAfterClosing = countTemporaryFiles(tempDir);
155-
assertThat(countAfterClosing).isLessThan(countAfterOpening);
154+
var fs = new S3FileSystem(provider, null, tempDir);
155+
var file1 = S3Path.getPath(fs, "file1");
156+
var file2 = S3Path.getPath(fs, "dir1/file2");
157+
var file3 = S3Path.getPath(fs, "dir1/dir2/file3");
158+
159+
var channel1 = new S3WritableByteChannel(file1, mock(), mock(), Set.of(CREATE));
160+
var channel2 = new S3WritableByteChannel(file2, mock(), mock(), Set.of(CREATE));
161+
var channel3 = new S3WritableByteChannel(file3, mock(), mock(), Set.of(CREATE));
162+
163+
assertThat(countTemporaryFiles(tempDir)).isEqualTo(3);
164+
channel1.close();
165+
channel2.close();
166+
channel3.close();
167+
assertThat(countTemporaryFiles(tempDir)).isZero();
156168
}
157169

158170
@Test
@@ -162,6 +174,7 @@ void secondCloseIsNoOp() throws InterruptedException, TimeoutException, IOExcept
162174
when(provider.exists(any(S3AsyncClient.class), any())).thenReturn(false);
163175
S3FileSystem fs = mock();
164176
when(fs.provider()).thenReturn(provider);
177+
when(fs.createTempFile(any(S3Path.class))).thenReturn(Files.createTempFile("", ""));
165178
var file = S3Path.getPath(fs, "somefile");
166179

167180
S3TransferUtil utilMock = mock();
@@ -174,11 +187,17 @@ void secondCloseIsNoOp() throws InterruptedException, TimeoutException, IOExcept
174187
}
175188

176189
private long countTemporaryFiles(Path tempDir) throws IOException {
177-
try (var list = Files.list(tempDir.getParent())) {
178-
return list
179-
.filter((path) -> path.getFileName().toString().contains("aws-s3-nio-"))
180-
.count();
181-
}
190+
var visitor = new SimpleFileVisitor<Path>() {
191+
int fileCount = 0;
192+
193+
@Override
194+
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
195+
fileCount++;
196+
return super.visitFile(file, attrs);
197+
}
198+
};
199+
Files.walkFileTree(tempDir, visitor);
200+
return visitor.fileCount;
182201
}
183202

184203
private Stream<Arguments> acceptedFileExistsAndOpenOptions() {

0 commit comments

Comments
 (0)