Skip to content

Commit 7d2f8dc

Browse files
ArtyomyuSartiom.darieharshach
authored
Fixes issue-11740: Added support for the om service to connect to AWS RDS using IAM roles (open-metadata#11913)
* ISSUE-11740: Added support for the om service to connect to AWS RDS using IAM roles * ISSUE-11740: Added support for the om service to connect to AWS RDS using IAM roles * ISSUE-11740: Added support for the om service to connect to AWS RDS using IAM roles * ISSUE-11740: Added support for the om service to connect to AWS RDS using IAM roles * ISSUE-11740: Added support for the om service to connect to AWS RDS using IAM roles * ISSUE-11740: Added support for the om service to connect to AWS RDS using IAM roles * ISSUE-11740: Added support for the om service to connect to AWS RDS using IAM roles * ISSUE-11740: Added support for the om service to connect to AWS RDS using IAM roles * ISSUE-11740: Added support for the om service to connect to AWS RDS using IAM roles * ISSUE-11740: Added support for the om service to connect to AWS RDS using IAM roles * ISSUE-11740: Added support for the om service to connect to AWS RDS using IAM roles * ISSUE-11740: Added support for the om service to connect to AWS RDS using IAM roles * ISSUE-11740: Added support for the om service to connect to AWS RDS using IAM roles * ISSUE-11740: Added support for the om service to connect to AWS RDS using IAM roles * ISSUE-11740: Added support for the om service to connect to AWS RDS using IAM roles * ISSUE-11740: Added support for the om service to connect to AWS RDS using IAM roles * ISSUE-11740: Added support for the om service to connect to AWS RDS using IAM roles * ISSUE-11740: Changed intial implementation accordingly. Added better flexibility for different auth prodvider impl * ISSUE-11740: Clean up unnecessary classes * ISSUE-11740: Clean up unnecessary properties * ISSUE-11740: Clean up unnecessary properties * ISSUE-11740: Clean up unnecessary properties * ISSUE-11740: Clean up unnecessary properties * ISSUE-11740: Clean up unnecessary properties * ISSUE-11740: Code formatting * ISSUE-11740: Added support for the om service to connect to AWS RDS using IAM roles * ISSUE-11740: Moved docs to 1.2 version --------- Co-authored-by: artiom.darie <[email protected]> Co-authored-by: Sriharsha Chintalapani <[email protected]>
1 parent 4d9570c commit 7d2f8dc

File tree

12 files changed

+250
-9
lines changed

12 files changed

+250
-9
lines changed

.pre-commit-config.yaml

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
1+
default_language_version:
2+
python: python3
13
repos:
24
- repo: https://github.com/ambv/black
35
rev: 22.3.0
46
hooks:
57
- id: black
6-
language_version: python3.8
78
exclude: ingestion/src/metadata/generated
89
- repo: https://github.com/timothycrosley/isort
910
rev: 5.12.0

conf/openmetadata.yaml

+7-7
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,15 @@ server:
2525
port: ${SERVER_ADMIN_PORT:-8586}
2626

2727
# Above configuration for running http is fine for dev and testing.
28-
# For production setup, where UI app will hit apis through DPS it
28+
# For production setup, where UI app will hit apis through DPS it
2929
# is strongly recommended to run https instead. Note that only
3030
# keyStorePath and keyStorePassword are mandatory properties. Values
3131
# for other properties are defaults
3232
#server:
3333
#applicationConnectors:
3434
# - type: https
3535
# port: 8585
36-
# keyStorePath: ./conf/keystore.jks
36+
# keyStorePath: ./conf/keystore.jks
3737
# keyStorePassword: changeit
3838
# keyStoreType: JKS
3939
# keyStoreProvider:
@@ -57,12 +57,12 @@ server:
5757
# supportedCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
5858
# allowRenegotiation: true
5959
# endpointIdentificationAlgorithm: (none)
60-
60+
6161
#adminConnectors:
6262
# - type: https
6363
# port: 8586
64-
# keyStorePath: ./conf/keystore.jks
65-
# keyStorePassword: changeit
64+
# keyStorePath: ./conf/keystore.jks
65+
# keyStorePassword: changeit
6666
# keyStoreType: JKS
6767
# keyStoreProvider:
6868
# trustStorePath: /path/to/file
@@ -125,7 +125,7 @@ database:
125125
user: ${DB_USER:-openmetadata_user}
126126
password: ${DB_USER_PASSWORD:-openmetadata_password}
127127
# the JDBC URL; the database is called openmetadata_db
128-
url: jdbc:${DB_SCHEME:-mysql}://${DB_HOST:-localhost}:${DB_PORT:-3306}/${OM_DATABASE:-openmetadata_db}?allowPublicKeyRetrieval=true&useSSL=${DB_USE_SSL:-false}&serverTimezone=UTC
128+
url: jdbc:${DB_SCHEME:-mysql}://${DB_HOST:-localhost}:${DB_PORT:-3306}/${OM_DATABASE:-openmetadata_db}?${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
129129

130130
migrationConfiguration:
131131
flywayPath: "./bootstrap/sql/migrations/flyway"
@@ -342,7 +342,7 @@ web:
342342
permission-policy:
343343
enabled: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
344344
option: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
345-
345+
346346

347347
changeEventConfig:
348348
omUri: ${OM_URI:- "http://localhost:8585"} #openmetadata in om uri for eg http://localhost:8585
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
---
2+
title: How to enable AWS RDS IAM Auth on postgresql
3+
slug: /how-to-guides/aws/index.md
4+
---
5+
6+
# Aws resources on Rds IAM Auth
7+
https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.IAMDBAuth.html
8+
9+
# Requirements
10+
1. AWS Rds Cluster with IAM auth enabled
11+
2. User on Db Cluster with iam enabled
12+
3. IAM policy with permission on rds connect
13+
4. Role with IAM policy attached
14+
5. IAM role attached to ec2 instance on which openmetadata is deployed or ServiceAccount/Kube2Iam role attached to pod
15+
16+
# How to enable ADS RDS IAM Auth on postgresql
17+
18+
Set environment variables
19+
```Commandline
20+
AWS_ENABLE_IAM_DATABASE_AUTHENTICATION: true
21+
AWS_REGION: your_region
22+
DB_PARAMS: "allowPublicKeyRetrieval=true&sslmode=require&serverTimezone=UTC"
23+
```
24+
Either through helm (if deployed in kubernetes) or as env vars
25+
26+
# Note
27+
The `DB_USER_PASSWORD` is still required and cannot be empty. Set it to a random/dummy string.

openmetadata-docs/content/v1.2.x-SNAPSHOT/deployment/configuration.md

-1
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,6 @@ database:
4747
# the JDBC URL; the database is called openmetadata_db
4848
url: jdbc:mysql://localhost/openmetadata_db?useSSL=false&serverTimezone=UTC
4949

50-
5150
elasticsearch:
5251
host: localhost
5352
port: 9200
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
---
2+
title: How to enable AWS RDS IAM Auth on postgresql
3+
slug: /how-to-guides/aws/index.md
4+
---
5+
6+
# Aws resources on Rds IAM Auth
7+
https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.IAMDBAuth.html
8+
9+
# Requirements
10+
1. AWS Rds Cluster with IAM auth enabled
11+
2. User on Db Cluster with iam enabled
12+
3. IAM policy with permission on rds connect
13+
4. Role with IAM policy attached
14+
5. IAM role attached to ec2 instance on which openmetadata is deployed or ServiceAccount/Kube2Iam role attached to pod
15+
16+
# How to enable ADS RDS IAM Auth on postgresql
17+
18+
Set environment variables
19+
```Commandline
20+
DB_USER_PASSWORD: "dummy"
21+
DB_PARAMS: "awsRegion=eu-west-1&allowPublicKeyRetrieval=true&sslmode=require&serverTimezone=UTC"
22+
```
23+
Either through helm (if deployed in kubernetes) or as env vars
24+
25+
# Note
26+
The `DB_USER_PASSWORD` is still required and cannot be empty. Set it to a random/dummy string.

openmetadata-service/pom.xml

+10
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,16 @@
247247
<artifactId>ssm</artifactId>
248248
<version>${awssdk.version}</version>
249249
</dependency>
250+
<dependency>
251+
<groupId>software.amazon.awssdk</groupId>
252+
<artifactId>rds</artifactId>
253+
<version>${awssdk.version}</version>
254+
</dependency>
255+
<dependency>
256+
<groupId>software.amazon.awssdk</groupId>
257+
<artifactId>sts</artifactId>
258+
<version>${awssdk.version}</version>
259+
</dependency>
250260

251261
<dependency>
252262
<groupId>io.dropwizard.modules</groupId>

openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java

+11
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@
121121
import org.openmetadata.service.socket.SocketAddressFilter;
122122
import org.openmetadata.service.socket.WebSocketManager;
123123
import org.openmetadata.service.util.MicrometerBundleSingleton;
124+
import org.openmetadata.service.util.jdbi.DatabaseAuthenticationProviderFactory;
124125
import org.openmetadata.service.workflows.searchIndex.SearchIndexEvent;
125126
import org.quartz.SchedulerException;
126127

@@ -273,6 +274,16 @@ private void registerSamlHandlers(OpenMetadataApplicationConfig catalogConfig, E
273274
}
274275

275276
private Jdbi createAndSetupJDBI(Environment environment, DataSourceFactory dbFactory) {
277+
// Check for db auth providers.
278+
DatabaseAuthenticationProviderFactory.get(dbFactory.getUrl())
279+
.ifPresent(
280+
databaseAuthenticationProvider -> {
281+
String token =
282+
databaseAuthenticationProvider.authenticate(
283+
dbFactory.getUrl(), dbFactory.getUser(), dbFactory.getPassword());
284+
dbFactory.setPassword(token);
285+
});
286+
276287
Jdbi jdbi = new JdbiFactory().build(environment, dbFactory, "database");
277288
SqlLogger sqlLogger =
278289
new SqlLogger() {

openmetadata-service/src/main/java/org/openmetadata/service/util/TablesInitializer.java

+13
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
import org.openmetadata.service.search.SearchClient;
5454
import org.openmetadata.service.search.SearchIndexDefinition;
5555
import org.openmetadata.service.secrets.SecretsManagerFactory;
56+
import org.openmetadata.service.util.jdbi.DatabaseAuthenticationProviderFactory;
5657

5758
public final class TablesInitializer {
5859
private static final String DEBUG_MODE_ENABLED = "debug_mode";
@@ -183,9 +184,21 @@ public static void main(String[] args) throws Exception {
183184
if (dataSourceFactory == null) {
184185
throw new RuntimeException("No database in config file");
185186
}
187+
188+
// Check for db auth providers.
189+
DatabaseAuthenticationProviderFactory.get(dataSourceFactory.getUrl())
190+
.ifPresent(
191+
databaseAuthenticationProvider -> {
192+
String token =
193+
databaseAuthenticationProvider.authenticate(
194+
dataSourceFactory.getUrl(), dataSourceFactory.getUser(), dataSourceFactory.getPassword());
195+
dataSourceFactory.setPassword(token);
196+
});
197+
186198
String jdbcUrl = dataSourceFactory.getUrl();
187199
String user = dataSourceFactory.getUser();
188200
String password = dataSourceFactory.getPassword();
201+
189202
boolean disableValidateOnMigrate = commandLine.hasOption(DISABLE_VALIDATE_ON_MIGRATE);
190203
if (disableValidateOnMigrate) {
191204
printToConsoleInDebug("Disabling validation on schema migrate");
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package org.openmetadata.service.util.jdbi;
2+
3+
import java.io.UnsupportedEncodingException;
4+
import java.net.MalformedURLException;
5+
import java.net.URI;
6+
import java.net.URL;
7+
import java.net.URLDecoder;
8+
import java.nio.charset.StandardCharsets;
9+
import java.util.LinkedHashMap;
10+
import java.util.Map;
11+
import java.util.Objects;
12+
import org.jetbrains.annotations.NotNull;
13+
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
14+
import software.amazon.awssdk.regions.Region;
15+
import software.amazon.awssdk.services.rds.RdsUtilities;
16+
import software.amazon.awssdk.services.rds.model.GenerateAuthenticationTokenRequest;
17+
18+
/**
19+
* {@link DatabaseAuthenticationProvider} implementation for AWS RDS IAM Auth.
20+
*
21+
* @see <a href="https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.IAMDBAuth.Enabling.html"></a>
22+
*/
23+
public class AwsRdsDatabaseAuthenticationProvider implements DatabaseAuthenticationProvider {
24+
25+
public static final String AWS_REGION = "awsRegion";
26+
public static final String ALLOW_PUBLIC_KEY_RETRIEVAL = "allowPublicKeyRetrieval";
27+
public static final String PROTOCOL = "https://";
28+
29+
@Override
30+
public String authenticate(String jdbcUrl, String username, String password) {
31+
// !!
32+
try {
33+
// Prepare
34+
URI uri = URI.create(PROTOCOL + removeProtocolFrom(jdbcUrl));
35+
Map<String, String> queryParams = parseQueryParams(uri.toURL());
36+
37+
// Set
38+
String awsRegion = queryParams.get(AWS_REGION);
39+
String allowPublicKeyRetrieval = queryParams.get(ALLOW_PUBLIC_KEY_RETRIEVAL);
40+
41+
// Validate
42+
Objects.requireNonNull(awsRegion, "Parameter `awsRegion` shall be provided in the jdbc url.");
43+
Objects.requireNonNull(
44+
allowPublicKeyRetrieval, "Parameter `allowPublicKeyRetrieval` shall be provided in the jdbc url.");
45+
46+
// Prepare request
47+
GenerateAuthenticationTokenRequest request =
48+
GenerateAuthenticationTokenRequest.builder()
49+
.credentialsProvider(DefaultCredentialsProvider.create())
50+
.hostname(uri.getHost())
51+
.port(uri.getPort())
52+
.username(username)
53+
.build();
54+
55+
// Return token
56+
return RdsUtilities.builder().region(Region.of(awsRegion)).build().generateAuthenticationToken(request);
57+
58+
} catch (MalformedURLException | UnsupportedEncodingException e) {
59+
// Throw
60+
throw new DatabaseAuthenticationProviderException(e);
61+
}
62+
}
63+
64+
@NotNull
65+
private static String removeProtocolFrom(String jdbcUrl) {
66+
return jdbcUrl.substring(jdbcUrl.indexOf("://") + 3);
67+
}
68+
69+
private Map<String, String> parseQueryParams(URL url) throws UnsupportedEncodingException {
70+
// Prepare
71+
Map<String, String> query_pairs = new LinkedHashMap<>();
72+
String query = url.getQuery();
73+
String[] pairs = query.split("&");
74+
75+
// Loop
76+
for (String pair : pairs) {
77+
int idx = pair.indexOf("=");
78+
// Add
79+
query_pairs.put(
80+
URLDecoder.decode(pair.substring(0, idx), StandardCharsets.UTF_8),
81+
URLDecoder.decode(pair.substring(idx + 1), StandardCharsets.UTF_8));
82+
}
83+
// Return
84+
return query_pairs;
85+
}
86+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
package org.openmetadata.service.util.jdbi;
2+
3+
/**
4+
* Database authentication provider is the main interface responsible for all implementation that requires additional
5+
* authentication steps required by the database in order to authorize a user to be able to operate on it.
6+
*
7+
* <p>For example if a jdbc url requires to retrieve and authorized token this interface shall be implemented to
8+
* retrieve the token.
9+
*/
10+
public interface DatabaseAuthenticationProvider {
11+
12+
/**
13+
* Authenticate a user for the given jdbc url.
14+
*
15+
* @return authorization token
16+
*/
17+
String authenticate(String jdbcUrl, String username, String password);
18+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package org.openmetadata.service.util.jdbi;
2+
3+
/** Database authentication provider exception responsible to all generic exception thrown by this layer. */
4+
public class DatabaseAuthenticationProviderException extends RuntimeException {
5+
public DatabaseAuthenticationProviderException() {}
6+
7+
public DatabaseAuthenticationProviderException(String message) {
8+
super(message);
9+
}
10+
11+
public DatabaseAuthenticationProviderException(String message, Throwable cause) {
12+
super(message, cause);
13+
}
14+
15+
public DatabaseAuthenticationProviderException(Throwable cause) {
16+
super(cause);
17+
}
18+
19+
public DatabaseAuthenticationProviderException(
20+
String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
21+
super(message, cause, enableSuppression, writableStackTrace);
22+
}
23+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package org.openmetadata.service.util.jdbi;
2+
3+
import java.util.Optional;
4+
5+
/** Factory class for {@link DatabaseAuthenticationProvider}. */
6+
public class DatabaseAuthenticationProviderFactory {
7+
/** C'tor */
8+
private DatabaseAuthenticationProviderFactory() {}
9+
10+
/**
11+
* Get auth provider based on the given jdbc url.
12+
*
13+
* @param jdbcURL the jdbc url.
14+
* @return instance of {@link DatabaseAuthenticationProvider}.
15+
*/
16+
public static Optional<DatabaseAuthenticationProvider> get(String jdbcURL) {
17+
// Check
18+
if (jdbcURL.contains(AwsRdsDatabaseAuthenticationProvider.AWS_REGION)
19+
&& jdbcURL.contains(AwsRdsDatabaseAuthenticationProvider.ALLOW_PUBLIC_KEY_RETRIEVAL)) {
20+
// Return AWS RDS Auth provider
21+
return Optional.of(new AwsRdsDatabaseAuthenticationProvider());
22+
}
23+
24+
// Return empty
25+
return Optional.empty();
26+
}
27+
}

0 commit comments

Comments
 (0)