From 51bbbadf4f9882d65f56cc3005b4a8ecba62c016 Mon Sep 17 00:00:00 2001 From: bboggess Date: Wed, 19 Feb 2020 18:20:24 -0800 Subject: [PATCH] add support for transparent server side encryption --- .../com/upplication/s3fs/AmazonS3Factory.java | 7 +- .../com/upplication/s3fs/S3FileSystem.java | 13 +- .../s3fs/S3FileSystemProvider.java | 121 ++++++++++++++---- .../s3fs/S3SeekableByteChannel.java | 14 +- .../s3fs/FilesServerSideEncryptIT.java | 69 ++++++++++ 5 files changed, 192 insertions(+), 32 deletions(-) create mode 100644 src/test/java/com/upplication/s3fs/FilesServerSideEncryptIT.java diff --git a/src/main/java/com/upplication/s3fs/AmazonS3Factory.java b/src/main/java/com/upplication/s3fs/AmazonS3Factory.java index 3de65575..c668072b 100644 --- a/src/main/java/com/upplication/s3fs/AmazonS3Factory.java +++ b/src/main/java/com/upplication/s3fs/AmazonS3Factory.java @@ -1,5 +1,8 @@ package com.upplication.s3fs; +import java.net.URI; +import java.util.Properties; + import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; import com.amazonaws.auth.AWSCredentialsProvider; @@ -10,9 +13,6 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.S3ClientOptions; -import java.net.URI; -import java.util.Properties; - /** * Factory base class to create a new AmazonS3 instance. @@ -38,6 +38,7 @@ public abstract class AmazonS3Factory { public static final String USER_AGENT = "s3fs_user_agent"; public static final String SIGNER_OVERRIDE = "s3fs_signer_override"; public static final String PATH_STYLE_ACCESS = "s3fs_path_style_access"; + public static final String ENCRYPT_SSE_S3 = "s3fs_encrypt_sse_s3"; /** * Build a new Amazon S3 instance with the URI and the properties provided diff --git a/src/main/java/com/upplication/s3fs/S3FileSystem.java b/src/main/java/com/upplication/s3fs/S3FileSystem.java index 96ed9e4b..6b5283f1 100644 --- a/src/main/java/com/upplication/s3fs/S3FileSystem.java +++ b/src/main/java/com/upplication/s3fs/S3FileSystem.java @@ -28,8 +28,9 @@ public class S3FileSystem extends FileSystem implements Comparable private final AmazonS3 client; private final String endpoint; private int cache; + private boolean serverSideEncryption; - public S3FileSystem(S3FileSystemProvider provider, String key, AmazonS3 client, String endpoint) { + public S3FileSystem(S3FileSystemProvider provider, String key, AmazonS3 client, String endpoint) { this.provider = provider; this.key = key; this.client = client; @@ -37,6 +38,12 @@ public S3FileSystem(S3FileSystemProvider provider, String key, AmazonS3 client, this.cache = 60000; // 1 minute cache for the s3Path } + public S3FileSystem(S3FileSystemProvider provider, String key, AmazonS3 client, String endpoint, + boolean serverSideEncryption) { + this(provider, key, client, endpoint); + this.serverSideEncryption = serverSideEncryption; + } + @Override public S3FileSystemProvider provider() { return provider; @@ -176,4 +183,8 @@ public int compareTo(S3FileSystem o) { public int getCache() { return cache; } + + public boolean isServerSideEncryption() { + return serverSideEncryption; + } } \ No newline at end of file diff --git a/src/main/java/com/upplication/s3fs/S3FileSystemProvider.java b/src/main/java/com/upplication/s3fs/S3FileSystemProvider.java index 77aca376..8b4d030a 100644 --- a/src/main/java/com/upplication/s3fs/S3FileSystemProvider.java +++ b/src/main/java/com/upplication/s3fs/S3FileSystemProvider.java @@ -1,11 +1,78 @@ package com.upplication.s3fs; +import static com.google.common.collect.Sets.difference; +import static com.upplication.s3fs.AmazonS3Factory.ACCESS_KEY; +import static com.upplication.s3fs.AmazonS3Factory.CONNECTION_TIMEOUT; +import static com.upplication.s3fs.AmazonS3Factory.MAX_CONNECTIONS; +import static com.upplication.s3fs.AmazonS3Factory.MAX_ERROR_RETRY; +import static com.upplication.s3fs.AmazonS3Factory.PATH_STYLE_ACCESS; +import static com.upplication.s3fs.AmazonS3Factory.PROTOCOL; +import static com.upplication.s3fs.AmazonS3Factory.PROXY_DOMAIN; +import static com.upplication.s3fs.AmazonS3Factory.PROXY_HOST; +import static com.upplication.s3fs.AmazonS3Factory.PROXY_PASSWORD; +import static com.upplication.s3fs.AmazonS3Factory.PROXY_PORT; +import static com.upplication.s3fs.AmazonS3Factory.PROXY_USERNAME; +import static com.upplication.s3fs.AmazonS3Factory.PROXY_WORKSTATION; +import static com.upplication.s3fs.AmazonS3Factory.REQUEST_METRIC_COLLECTOR_CLASS; +import static com.upplication.s3fs.AmazonS3Factory.SECRET_KEY; +import static com.upplication.s3fs.AmazonS3Factory.SIGNER_OVERRIDE; +import static com.upplication.s3fs.AmazonS3Factory.SOCKET_RECEIVE_BUFFER_SIZE_HINT; +import static com.upplication.s3fs.AmazonS3Factory.SOCKET_SEND_BUFFER_SIZE_HINT; +import static com.upplication.s3fs.AmazonS3Factory.SOCKET_TIMEOUT; +import static com.upplication.s3fs.AmazonS3Factory.USER_AGENT; +import static com.upplication.s3fs.AmazonS3Factory.ENCRYPT_SSE_S3; + +import static java.lang.String.format; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.channels.FileChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.AccessMode; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.CopyOption; +import java.nio.file.DirectoryNotEmptyException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileStore; +import java.nio.file.FileSystem; +import java.nio.file.FileSystemAlreadyExistsException; +import java.nio.file.FileSystemNotFoundException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.NoSuchFileException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.FileAttributeView; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFileAttributes; +import java.nio.file.spi.FileSystemProvider; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.Headers; import com.amazonaws.services.s3.internal.Constants; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.Bucket; +import com.amazonaws.services.s3.model.CopyObjectRequest; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.SSEAlgorithm; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -18,23 +85,6 @@ import com.upplication.s3fs.util.Cache; import com.upplication.s3fs.util.S3Utils; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.nio.channels.FileChannel; -import java.nio.channels.SeekableByteChannel; -import java.nio.file.*; -import java.nio.file.attribute.*; -import java.nio.file.spi.FileSystemProvider; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -import static com.google.common.collect.Sets.difference; -import static com.upplication.s3fs.AmazonS3Factory.*; -import static java.lang.String.format; - /** * Spec: *

@@ -71,12 +121,13 @@ public class S3FileSystemProvider extends FileSystemProvider { private static final ConcurrentMap fileSystems = new ConcurrentHashMap<>(); private static final List PROPS_TO_OVERLOAD = Arrays.asList(ACCESS_KEY, SECRET_KEY, REQUEST_METRIC_COLLECTOR_CLASS, CONNECTION_TIMEOUT, MAX_CONNECTIONS, MAX_ERROR_RETRY, PROTOCOL, PROXY_DOMAIN, PROXY_HOST, PROXY_PASSWORD, PROXY_PORT, PROXY_USERNAME, PROXY_WORKSTATION, SOCKET_SEND_BUFFER_SIZE_HINT, SOCKET_RECEIVE_BUFFER_SIZE_HINT, SOCKET_TIMEOUT, - USER_AGENT, AMAZON_S3_FACTORY_CLASS, SIGNER_OVERRIDE, PATH_STYLE_ACCESS); + USER_AGENT, AMAZON_S3_FACTORY_CLASS, SIGNER_OVERRIDE, PATH_STYLE_ACCESS, ENCRYPT_SSE_S3); private S3Utils s3Utils = new S3Utils(); private Cache cache = new Cache(); - - @Override + private boolean serverSideEncryption = false; + + @Override public String getScheme() { return "s3"; } @@ -414,12 +465,18 @@ public void copy(Path source, Path target, CopyOption... options) throws IOExcep String keySource = s3Source.getKey(); String bucketNameTarget = s3Target.getFileStore().name(); String keyTarget = s3Target.getKey(); - s3Source.getFileSystem() - .getClient().copyObject( - bucketNameOrigin, - keySource, - bucketNameTarget, - keyTarget); + + CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucketNameOrigin, keySource, + bucketNameTarget, keyTarget); + + if (s3Target.getFileSystem().isServerSideEncryption()) { + // force server side encrypt + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setHeader(Headers.SERVER_SIDE_ENCRYPTION, SSEAlgorithm.getDefault().name()); + copyObjectRequest.setNewObjectMetadata(metadata); + } + + s3Source.getFileSystem().getClient().copyObject(copyObjectRequest); } @Override @@ -550,7 +607,9 @@ public void setAttribute(Path path, String attribute, Object value, LinkOption.. * @return S3FileSystem never null */ public S3FileSystem createFileSystem(URI uri, Properties props) { - return new S3FileSystem(this, getFileSystemKey(uri, props), getAmazonS3(uri, props), uri.getHost()); + String encryptPropertyValue = props.getProperty(ENCRYPT_SSE_S3); + boolean encrypt = encryptPropertyValue != null && Boolean.valueOf(encryptPropertyValue); + return new S3FileSystem(this, getFileSystemKey(uri, props), getAmazonS3(uri, props), uri.getHost(), encrypt); } protected AmazonS3 getAmazonS3(URI uri, Properties props) { @@ -634,4 +693,12 @@ public Cache getCache() { public void setCache(Cache cache) { this.cache = cache; } + + public boolean isServerSideEncryption() { + return serverSideEncryption; + } + + public void setServerSideEncryption(boolean serverSideEncryption) { + this.serverSideEncryption = serverSideEncryption; + } } \ No newline at end of file diff --git a/src/main/java/com/upplication/s3fs/S3SeekableByteChannel.java b/src/main/java/com/upplication/s3fs/S3SeekableByteChannel.java index 3cb9dbf3..4de6bcb2 100644 --- a/src/main/java/com/upplication/s3fs/S3SeekableByteChannel.java +++ b/src/main/java/com/upplication/s3fs/S3SeekableByteChannel.java @@ -7,15 +7,23 @@ import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.channels.SeekableByteChannel; -import java.nio.file.*; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.util.Collections; import java.util.HashSet; import java.util.Set; import org.apache.tika.Tika; +import com.amazonaws.services.s3.Headers; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.SSEAlgorithm; public class S3SeekableByteChannel implements SeekableByteChannel { @@ -107,6 +115,10 @@ protected void sync() throws IOException { metadata.setContentType(new Tika().detect(stream, path.getFileName().toString())); } + if (path.getFileSystem().isServerSideEncryption()) { + metadata.setHeader(Headers.SERVER_SIDE_ENCRYPTION, SSEAlgorithm.getDefault().name()); + } + String bucket = path.getFileStore().name(); String key = path.getKey(); path.getFileSystem().getClient().putObject(bucket, key, stream, metadata); diff --git a/src/test/java/com/upplication/s3fs/FilesServerSideEncryptIT.java b/src/test/java/com/upplication/s3fs/FilesServerSideEncryptIT.java new file mode 100644 index 00000000..df0939c2 --- /dev/null +++ b/src/test/java/com/upplication/s3fs/FilesServerSideEncryptIT.java @@ -0,0 +1,69 @@ +package com.upplication.s3fs; + +import static com.upplication.s3fs.util.S3EndpointConstant.S3_GLOBAL_URI_IT; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystemNotFoundException; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import org.junit.Before; +import org.junit.Test; + +import com.amazonaws.services.s3.Headers; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.SSEAlgorithm; +import com.github.marschall.memoryfilesystem.MemoryFileSystemBuilder; +import com.upplication.s3fs.util.EnvironmentBuilder; + +public class FilesServerSideEncryptIT { + + private static final String bucket = EnvironmentBuilder.getBucket(); + private static final URI uriGlobal = EnvironmentBuilder.getS3URI(S3_GLOBAL_URI_IT); + + private FileSystem fileSystemAmazon; + + @Before + public void setup() throws IOException { + System.clearProperty(S3FileSystemProvider.AMAZON_S3_FACTORY_CLASS); + fileSystemAmazon = build(); + } + + private static FileSystem build() throws IOException { + try { + FileSystems.getFileSystem(uriGlobal).close(); + return createNewFileSystem(); + } catch (FileSystemNotFoundException e) { + return createNewFileSystem(); + } + } + + private static FileSystem createNewFileSystem() throws IOException { + Map env = new HashMap(); + env.putAll(EnvironmentBuilder.getRealEnv()); + env.put(AmazonS3Factory.ENCRYPT_SSE_S3, "true"); + + return FileSystems.newFileSystem(uriGlobal, env); + } + + @Test + public void amazonCopyVerifyEncryption() throws IOException { + try (FileSystem linux = MemoryFileSystemBuilder.newLinux().build("linux")) { + Path htmlFile = Files.write(linux.getPath("/index.html"), "html file".getBytes()); + + Path result = fileSystemAmazon.getPath(bucket, UUID.randomUUID().toString() + htmlFile.getFileName().toString()); + Files.copy(htmlFile, result); + + S3Path resultS3 = (S3Path) result; + ObjectMetadata metadata = resultS3.getFileSystem().getClient().getObjectMetadata(resultS3.getFileStore().name(), resultS3.getKey()); + assertEquals(metadata.getRawMetadataValue(Headers.SERVER_SIDE_ENCRYPTION), SSEAlgorithm.getDefault().name()); + } + } +}