Skip to content

Commit

Permalink
feat: Allow specifying a different protocol for proxies. (#719)
Browse files Browse the repository at this point in the history 
Co-authored-by: Steve Todorov <[email protected]>
  • Loading branch information
@stanakaj @steve-todorov
stanakaj and steve-todorov authored Jul 26, 2023
1 parent a28f66f commit 47139ca
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 42 deletions.
3 changes: 2 additions & 1 deletion docs/content/reference/configuration-options.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
A complete list of environment variables which can be set to configure the client.

| Key | Default | Description |
| ------------------------------------------|---------|-------------------------------------------------------------------------------------------------------------------------|
|-------------------------------------------|---------|-------------------------------------------------------------------------------------------------------------------------|
| s3fs.access.key | none | <small>AWS access key, used to identify the user interacting with AWS</small> |
| s3fs.secret.key | none | <small>AWS secret access key, used to authenticate the user interacting with AWS</small> |
| s3fs.request.metric.collector.class | TODO | <small>Fully-qualified class name to instantiate an AWS SDK request/response metric collector</small> |
Expand All @@ -14,6 +14,7 @@ A complete list of environment variables which can be set to configure the clien
| s3fs.max.retry.error | TODO | <small>Maximum number of times that a single request should be retried, assuming it fails for a retryable error</small> |
| s3fs.protocol | TODO | <small>Protocol (HTTP or HTTPS) to use when connecting to AWS</small> |
| s3fs.proxy.domain | none | <small>For NTLM proxies: The Windows domain name to use when authenticating with the proxy</small> |
| s3fs.proxy.protocol | none | <small>Proxy connection protocol.</small> |
| s3fs.proxy.host | none | <small>Proxy host name either from the configured endpoint or from the "http.proxyHost" system property</small> |
| s3fs.proxy.password | none | <small>The password to use when connecting through a proxy</small> |
| s3fs.proxy.port | none | <small>Proxy port either from the configured endpoint or from the "http.proxyPort" system property</small> |
Expand Down
18 changes: 17 additions & 1 deletion src/main/java/org/carlspring/cloud/storage/s3fs/S3Factory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,11 @@ public abstract class S3Factory

public static final String PROXY_WORKSTATION = "s3fs.proxy.workstation";

/**
* Allows you to specify the proxy protocol (http, https, etc)
*/
public static final String PROXY_PROTOCOL = "s3fs.proxy.protocol";

/**
* @deprecated Not supported according to https://github.com/aws/aws-sdk-java-v2/blob/master/docs/LaunchChangelog.md#133-client-override-configuration
*/
Expand Down Expand Up @@ -329,7 +334,18 @@ protected ProxyConfiguration getProxyConfiguration(final Properties props)
printWarningMessage(props, PROXY_PORT);
}

final URI uri = getEndpointUri(host, port, props);
// Calls the getEndpointUri method after setting the PROTOCOL property to the value of PROXY_PROTOCOL.
final Properties propsCopy = new Properties();
for (String key : props.stringPropertyNames()) {
propsCopy.setProperty(key, props.getProperty(key));
}

if (propsCopy.getProperty(PROXY_PROTOCOL) != null)
{
propsCopy.setProperty(PROTOCOL, props.getProperty(PROXY_PROTOCOL));
}

final URI uri = getEndpointUri(host, port, propsCopy);
builder.endpoint(uri);
}

Expand Down
81 changes: 41 additions & 40 deletions src/test/java/org/carlspring/cloud/storage/s3fs/S3ClientFactoryTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,19 @@
package org.carlspring.cloud.storage.s3fs;

import org.carlspring.cloud.storage.s3fs.util.ExposingS3Client;
import org.carlspring.cloud.storage.s3fs.util.ExposingS3ClientFactory;
import org.carlspring.cloud.storage.s3fs.util.S3EndpointConstant;
import static org.assertj.core.api.Assertions.assertThat;
import static org.carlspring.cloud.storage.s3fs.S3Factory.*;
import static org.junit.jupiter.api.Assertions.*;
import static software.amazon.awssdk.core.client.config.SdkAdvancedClientOption.*;
import static software.amazon.awssdk.core.client.config.SdkClientOption.*;

import java.net.URI;
import java.util.Properties;

import org.carlspring.cloud.storage.s3fs.util.ExposingS3Client;
import org.carlspring.cloud.storage.s3fs.util.ExposingS3ClientFactory;
import org.carlspring.cloud.storage.s3fs.util.S3EndpointConstant;
import org.junit.jupiter.api.Test;

import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.core.Protocol;
Expand All @@ -16,35 +22,6 @@
import software.amazon.awssdk.core.client.config.SdkClientConfiguration;
import software.amazon.awssdk.http.apache.ProxyConfiguration;
import software.amazon.awssdk.services.s3.S3Configuration;
import static org.carlspring.cloud.storage.s3fs.S3Factory.ACCESS_KEY;
import static org.carlspring.cloud.storage.s3fs.S3Factory.CONNECTION_TIMEOUT;
import static org.carlspring.cloud.storage.s3fs.S3Factory.MAX_CONNECTIONS;
import static org.carlspring.cloud.storage.s3fs.S3Factory.MAX_ERROR_RETRY;
import static org.carlspring.cloud.storage.s3fs.S3Factory.PATH_STYLE_ACCESS;
import static org.carlspring.cloud.storage.s3fs.S3Factory.PROTOCOL;
import static org.carlspring.cloud.storage.s3fs.S3Factory.PROXY_DOMAIN;
import static org.carlspring.cloud.storage.s3fs.S3Factory.PROXY_HOST;
import static org.carlspring.cloud.storage.s3fs.S3Factory.PROXY_PASSWORD;
import static org.carlspring.cloud.storage.s3fs.S3Factory.PROXY_PORT;
import static org.carlspring.cloud.storage.s3fs.S3Factory.PROXY_USERNAME;
import static org.carlspring.cloud.storage.s3fs.S3Factory.PROXY_WORKSTATION;
import static org.carlspring.cloud.storage.s3fs.S3Factory.REGION;
import static org.carlspring.cloud.storage.s3fs.S3Factory.REQUEST_METRIC_COLLECTOR_CLASS;
import static org.carlspring.cloud.storage.s3fs.S3Factory.SECRET_KEY;
import static org.carlspring.cloud.storage.s3fs.S3Factory.SIGNER_OVERRIDE;
import static org.carlspring.cloud.storage.s3fs.S3Factory.SOCKET_RECEIVE_BUFFER_SIZE_HINT;
import static org.carlspring.cloud.storage.s3fs.S3Factory.SOCKET_SEND_BUFFER_SIZE_HINT;
import static org.carlspring.cloud.storage.s3fs.S3Factory.SOCKET_TIMEOUT;
import static org.carlspring.cloud.storage.s3fs.S3Factory.USER_AGENT;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static software.amazon.awssdk.core.client.config.SdkAdvancedClientOption.SIGNER;
import static software.amazon.awssdk.core.client.config.SdkAdvancedClientOption.USER_AGENT_PREFIX;
import static software.amazon.awssdk.core.client.config.SdkClientOption.ENDPOINT;

class S3ClientFactoryTest
{
Expand All @@ -56,8 +33,7 @@ void neverTrustTheDefaults()
Properties props = new Properties();
props.setProperty(ACCESS_KEY, "some_access_key");
props.setProperty(SECRET_KEY, "super_secret_key");
props.setProperty(REQUEST_METRIC_COLLECTOR_CLASS,
"org.carlspring.cloud.storage.s3fs.util.NoOpRequestMetricCollector");
props.setProperty(REQUEST_METRIC_COLLECTOR_CLASS, "org.carlspring.cloud.storage.s3fs.util.NoOpRequestMetricCollector");
props.setProperty(CONNECTION_TIMEOUT, "10");
props.setProperty(MAX_CONNECTIONS, "50");
props.setProperty(MAX_ERROR_RETRY, "3");
Expand All @@ -68,6 +44,7 @@ void neverTrustTheDefaults()
props.setProperty(PROXY_PORT, "12345");
props.setProperty(PROXY_USERNAME, "proxy_username");
props.setProperty(PROXY_WORKSTATION, "what.does.this.do.localhost");
props.setProperty(PROXY_PROTOCOL, "https");
props.setProperty(SOCKET_SEND_BUFFER_SIZE_HINT, "48000");
props.setProperty(SOCKET_RECEIVE_BUFFER_SIZE_HINT, "49000");
props.setProperty(SOCKET_TIMEOUT, "30");
Expand Down Expand Up @@ -99,12 +76,13 @@ void neverTrustTheDefaults()

ProxyConfiguration proxyConfiguration = clientFactory.getProxyConfiguration(props);

assertEquals("127.0.0.1", proxyConfiguration.host());
assertEquals(12345, proxyConfiguration.port());
assertEquals("proxy_username", proxyConfiguration.username());
assertEquals("proxy_password", proxyConfiguration.password());
assertEquals("localhost", proxyConfiguration.ntlmDomain());
assertEquals("what.does.this.do.localhost", proxyConfiguration.ntlmWorkstation());
assertThat(proxyConfiguration.host()).isEqualTo(props.getProperty(PROXY_HOST));
assertThat(proxyConfiguration.port()).isEqualTo(Integer.valueOf(props.getProperty(PROXY_PORT)));
assertThat(proxyConfiguration.username()).isEqualTo(props.getProperty(PROXY_USERNAME));
assertThat(proxyConfiguration.password()).isEqualTo(props.getProperty(PROXY_PASSWORD));
assertThat(proxyConfiguration.ntlmDomain()).isEqualTo(props.getProperty(PROXY_DOMAIN));
assertThat(proxyConfiguration.ntlmWorkstation()).isEqualTo(props.getProperty(PROXY_WORKSTATION));
assertThat(proxyConfiguration.scheme()).isEqualTo(props.getProperty(PROXY_PROTOCOL));

S3Configuration serviceConfiguration = clientFactory.getServiceConfiguration(props);
assertTrue(serviceConfiguration.pathStyleAccessEnabled());
Expand Down Expand Up @@ -149,6 +127,7 @@ void theDefaults()
assertNull(proxyConfiguration.password());
assertNull(proxyConfiguration.ntlmDomain());
assertNull(proxyConfiguration.ntlmWorkstation());
assertNull(proxyConfiguration.scheme());

S3Configuration serviceConfiguration = clientFactory.getServiceConfiguration(props);
assertFalse(serviceConfiguration.pathStyleAccessEnabled());
Expand Down Expand Up @@ -218,4 +197,26 @@ void overrideHostAndPort()
assertEquals(8001, endpoint.getPort());
}

@Test
void shouldAllowUsingHTTPProxyAndHTTPSProtocolForS3Connections()
{
S3ClientFactory clientFactory = new ExposingS3ClientFactory();

Properties props = new Properties();
props.setProperty(REGION, "eu-central-1");
props.setProperty(PROTOCOL, "https");
props.setProperty(PROXY_DOMAIN, "localhost");
props.setProperty(PROXY_HOST, "127.0.0.1");
props.setProperty(PROXY_PROTOCOL, "http");

ExposingS3Client client =
(ExposingS3Client) clientFactory.getS3Client(S3EndpointConstant.S3_GLOBAL_URI_TEST, props);
final SdkClientConfiguration clientConfiguration = client.getClientConfiguration();

final URI endpoint = clientConfiguration.option(ENDPOINT);
assertEquals("https", endpoint.getScheme());

ProxyConfiguration proxyConfiguration = clientFactory.getProxyConfiguration(props);
assertEquals("http", proxyConfiguration.scheme());
}
}

0 comments on commit 47139ca

Please sign in to comment.