Update renewed Jira refresh token back to the secrets store (#5324)

* Introduced PluginConfigVariable interaface to provide ability for the plugins to get access to their underlying aws secrets store member to be able to update if needed Signed-off-by: Santhosh Gandhe <[email protected]> * renewed access token and refresh tokens are now updated back in the secrets store Signed-off-by: Santhosh Gandhe <[email protected]> * better naming Signed-off-by: Santhosh Gandhe <[email protected]> * fixing the test cases based on the new PluginConfigVariable attribute used for refreshToken Signed-off-by: Santhosh Gandhe <[email protected]> * improving the coverage Signed-off-by: Santhosh Gandhe <[email protected]> * Keeping the existing values in the secret. Just updating an existing key Signed-off-by: Santhosh Gandhe <[email protected]> * Allowing secrets manager update without a key and also some additional test coverage Signed-off-by: Santhosh Gandhe <[email protected]> * isUpdatable boolean is introduced and its corresponding tests Signed-off-by: Santhosh Gandhe <[email protected]> * additional coverage Signed-off-by: Santhosh Gandhe <[email protected]> * implementing newly added method Signed-off-by: Santhosh Gandhe <[email protected]> * switching PluginConfigVariable from refreshToken to accessToken Signed-off-by: Santhosh Gandhe <[email protected]> * Only the master node is responsible for Token refresh Signed-off-by: Santhosh Gandhe <[email protected]> * Added addition parameter in the API to accept the secrets version to set that helps with enforcing idempotency while updating the secret store multiple times Signed-off-by: Santhosh Gandhe <[email protected]> * better naming Signed-off-by: Santhosh Gandhe <[email protected]> * removing setting a versionId for idempotency Signed-off-by: Santhosh Gandhe <[email protected]> * removed constructor argument to PluginConfigVariable Signed-off-by: Santhosh Gandhe <[email protected]> --------- Signed-off-by: Santhosh Gandhe <[email protected]>
opensearch-project · Jan 17, 2025 · 83e949d · 83e949d
1 parent 3b12ca1
commit 83e949d
Show file tree

Hide file tree

Showing 28 changed files with 753 additions and 73 deletions.
diff --git a/...ava/org/opensearch/dataprepper/model/plugin/FailedToUpdatePluginConfigValueException.java b/...ava/org/opensearch/dataprepper/model/plugin/FailedToUpdatePluginConfigValueException.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+package org.opensearch.dataprepper.model.plugin;
+
+/**
+ * Exception thrown when a secret could not be updated.
+ *
+ * @since 2.11
+ */
+public class FailedToUpdatePluginConfigValueException extends RuntimeException {
+
+    public FailedToUpdatePluginConfigValueException(final String message) {
+        super(message);
+    }
+
+    public FailedToUpdatePluginConfigValueException(final String message, Throwable e) {
+        super(message, e);
+    }
+}
diff --git a/...pi/src/main/java/org/opensearch/dataprepper/model/plugin/PluginConfigValueTranslator.java b/...pi/src/main/java/org/opensearch/dataprepper/model/plugin/PluginConfigValueTranslator.java
@@ -1,7 +1,48 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
 package org.opensearch.dataprepper.model.plugin;
 
+/**
+ * Interface for a Plugin configuration value translator.
+ * It translates a string expression that is describing a secret store Id and secret Key in to a secretValue
+ * extracted from corresponding secret store.
+ *
+ * @since 2.0
+ */
+
 public interface PluginConfigValueTranslator {
+    /**
+     * Translates a string expression that is describing a secret store Id and secret Key in to a secretValue
+     * extracted from corresponding secret store.
+     * Example expression:  ${{aws_secrets:secretId:secretKey}}
+     *
+     * @param value the string value to translate
+     * @return the translated object
+     */
     Object translate(final String value);
 
+    /**
+     * Returns the prefix for this translator.
+     *
+     * @return the prefix for this translator
+     */
     String getPrefix();
+
+    /**
+     * Translates a string expression that is describing a secret store Id and secret Key in to an instance
+     * of PluginConfigVariable with secretValue extracted from corresponding secret store. Additionally,
+     * this PluginConfigVariable helps with updating the secret value in the secret store, if required.
+     * Example expression:  ${{aws_secrets:secretId:secretKey}}
+     *
+     * @param value the string value to translate
+     * @return the translated object
+     */
+    PluginConfigVariable translateToPluginConfigVariable(final String value);
 }
diff --git a/...epper-api/src/main/java/org/opensearch/dataprepper/model/plugin/PluginConfigVariable.java b/...epper-api/src/main/java/org/opensearch/dataprepper/model/plugin/PluginConfigVariable.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+package org.opensearch.dataprepper.model.plugin;
+
+/**
+ * Interface for a Extension Plugin configuration variable.
+ * It gives access to the details of a defined extension variable.
+ *
+ * @since 2.11
+ */
+public interface PluginConfigVariable {
+
+    /**
+     * Returns the value of this variable.
+     *
+     * @return the value of this variable
+     */
+    Object getValue();
+
+    /**
+     * If this variable is updatable, this method helps to set a new value for this variable
+     *
+     * @param updatedValue the new value to set
+     */
+    void setValue(Object updatedValue);
+
+    /**
+     * Returns if the variable is updatable.
+     *
+     * @return true if this variable is updatable, false otherwise
+     */
+    boolean isUpdatable();
+}
diff --git a/...org/opensearch/dataprepper/model/plugin/FailedToUpdatePluginConfigValueExceptionTest.java b/...org/opensearch/dataprepper/model/plugin/FailedToUpdatePluginConfigValueExceptionTest.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+package org.opensearch.dataprepper.model.plugin;
+
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.UUID;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+public class FailedToUpdatePluginConfigValueExceptionTest extends RuntimeException {
+    private String message;
+
+    @BeforeEach
+    void setUp() {
+        message = UUID.randomUUID().toString();
+    }
+
+    @Test
+    void testGetMessage_should_return_correct_message() {
+        FailedToUpdatePluginConfigValueException failedToUpdateSecretException = new FailedToUpdatePluginConfigValueException(message);
+        assertThat(failedToUpdateSecretException.getMessage(), equalTo(message));
+    }
+
+    @Test
+    void testGetMessage_should_return_correct_message_with_throwable() {
+        RuntimeException cause = new RuntimeException("testException");
+        FailedToUpdatePluginConfigValueException failedToUpdateSecretException = new FailedToUpdatePluginConfigValueException(message, cause);
+        assertThat(failedToUpdateSecretException.getMessage(), equalTo(message));
+        assertThat(failedToUpdateSecretException.getCause(), equalTo(cause));
+    }
+
+}
diff --git a/...-framework/src/main/java/org/opensearch/dataprepper/plugin/ObjectMapperConfiguration.java b/...-framework/src/main/java/org/opensearch/dataprepper/plugin/ObjectMapperConfiguration.java
@@ -10,6 +10,7 @@
 import com.fasterxml.jackson.databind.module.SimpleModule;
 import org.opensearch.dataprepper.model.event.EventKey;
 import org.opensearch.dataprepper.model.event.EventKeyFactory;
+import org.opensearch.dataprepper.model.plugin.PluginConfigVariable;
 import org.opensearch.dataprepper.model.types.ByteCount;
 import org.opensearch.dataprepper.pipeline.parser.ByteCountDeserializer;
 import org.opensearch.dataprepper.pipeline.parser.DataPrepperDurationDeserializer;
@@ -28,7 +29,7 @@
 public class ObjectMapperConfiguration {
     static final Set<Class> TRANSLATE_VALUE_SUPPORTED_JAVA_TYPES = Set.of(
             String.class, Number.class, Long.class, Short.class, Integer.class, Double.class, Float.class,
-            Boolean.class, Character.class);
+            Boolean.class, Character.class, PluginConfigVariable.class);
 
     @Bean(name = "extensionPluginConfigObjectMapper")
     ObjectMapper extensionPluginConfigObjectMapper() {

diff --git a/...er-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/VariableExpander.java b/...er-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/VariableExpander.java
@@ -9,6 +9,7 @@
 import com.fasterxml.jackson.core.JsonToken;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.opensearch.dataprepper.model.plugin.PluginConfigValueTranslator;
+import org.opensearch.dataprepper.model.plugin.PluginConfigVariable;
 
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -29,8 +30,7 @@ public class VariableExpander {
 
     @Inject
     public VariableExpander(
-            @Named("extensionPluginConfigObjectMapper")
-            final ObjectMapper objectMapper,
+            @Named("extensionPluginConfigObjectMapper") final ObjectMapper objectMapper,
             final Set<PluginConfigValueTranslator> pluginConfigValueTranslators) {
         this.objectMapper = objectMapper;
         patternPluginConfigValueTranslatorMap = pluginConfigValueTranslators.stream().collect(Collectors.toMap(
@@ -48,8 +48,13 @@ public <T> T translate(final JsonParser jsonParser, final Class<T> destinationTy
                     .filter(entry -> entry.getKey().matches())
                     .map(entry -> {
                         final String valueReferenceKey = entry.getKey().group(VALUE_REFERENCE_KEY);
-                        return objectMapper.convertValue(
-                                entry.getValue().translate(valueReferenceKey), destinationType);
+                        if (destinationType.equals(PluginConfigVariable.class)) {
+                            return (T) entry.getValue().translateToPluginConfigVariable(valueReferenceKey);
+                        } else {
+                            return objectMapper.convertValue(
+                                    entry.getValue().translate(valueReferenceKey), destinationType);
+                        }
+
                     })
                     .findFirst()
                     .orElseGet(() -> objectMapper.convertValue(rawValue, destinationType));

diff --git a/...lugin-framework/src/test/java/org/opensearch/dataprepper/plugin/VariableExpanderTest.java b/...lugin-framework/src/test/java/org/opensearch/dataprepper/plugin/VariableExpanderTest.java
@@ -19,6 +19,7 @@
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.opensearch.dataprepper.model.plugin.PluginConfigValueTranslator;
+import org.opensearch.dataprepper.model.plugin.PluginConfigVariable;
 
 import java.io.IOException;
 import java.math.BigDecimal;
@@ -30,6 +31,8 @@
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.when;
 
@@ -43,6 +46,32 @@ class VariableExpanderTest {
 
     private VariableExpander objectUnderTest;
 
+    private static Stream<Arguments> getNonStringTypeArguments() {
+        return Stream.of(Arguments.of(Boolean.class, "true", true),
+                Arguments.of(Short.class, "2", (short) 2),
+                Arguments.of(Integer.class, "10", 10),
+                Arguments.of(Long.class, "200", 200L),
+                Arguments.of(Double.class, "1.23", 1.23d),
+                Arguments.of(Float.class, "2.15", 2.15f),
+                Arguments.of(BigDecimal.class, "2.15", BigDecimal.valueOf(2.15)),
+                Arguments.of(Map.class, "{}", Collections.emptyMap()));
+    }
+
+    private static Stream<Arguments> getStringTypeArguments() {
+        final String testRandomValue = "non-secret-prefix-" + RandomStringUtils.randomAlphabetic(5);
+        return Stream.of(Arguments.of(String.class, String.format("\"%s\"", testRandomValue),
+                        testRandomValue),
+                Arguments.of(Duration.class, "\"PT15M\"", Duration.parse("PT15M")),
+                Arguments.of(Boolean.class, "\"true\"", true),
+                Arguments.of(Short.class, "\"2\"", (short) 2),
+                Arguments.of(Integer.class, "\"10\"", 10),
+                Arguments.of(Long.class, "\"200\"", 200L),
+                Arguments.of(Double.class, "\"1.23\"", 1.23d),
+                Arguments.of(Float.class, "\"2.15\"", 2.15f),
+                Arguments.of(BigDecimal.class, "\"2.15\"", BigDecimal.valueOf(2.15)),
+                Arguments.of(Character.class, "\"c\"", 'c'));
+    }
+
     @BeforeEach
     void setUp() {
         objectUnderTest = new VariableExpander(OBJECT_MAPPER, Set.of(pluginConfigValueTranslator));
@@ -107,29 +136,53 @@ void testTranslateJsonParserWithStringValue_translate_success(
         assertThat(actualResult, equalTo(expectedResult));
     }
 
-    private static Stream<Arguments> getNonStringTypeArguments() {
-        return Stream.of(Arguments.of(Boolean.class, "true", true),
-                Arguments.of(Short.class, "2", (short) 2),
-                Arguments.of(Integer.class, "10", 10),
-                Arguments.of(Long.class, "200", 200L),
-                Arguments.of(Double.class, "1.23", 1.23d),
-                Arguments.of(Float.class, "2.15", 2.15f),
-                Arguments.of(BigDecimal.class, "2.15", BigDecimal.valueOf(2.15)),
-                Arguments.of(Map.class, "{}", Collections.emptyMap()));
+    @Test
+    void testTranslateJsonParserWithSPluginConfigVariableValue_translate_success() throws IOException {
+        final String testSecretKey = "testSecretKey";
+        final String testTranslatorKey = "test_prefix";
+        final String testSecretReference = String.format("${{%s:%s}}", testTranslatorKey, testSecretKey);
+        final JsonParser jsonParser = JSON_FACTORY.createParser(String.format("\"%s\"", testSecretReference));
+        jsonParser.nextToken();
+        PluginConfigVariable mockPluginConfigVariable = new PluginConfigVariable() {
+
+            String secretValue = "samplePluginConfigValue";
+
+            @Override
+            public Object getValue() {
+                return secretValue;
+            }
+
+            @Override
+            public void setValue(Object updatedValue) {
+                this.secretValue = updatedValue.toString();
+            }
+
+            @Override
+            public boolean isUpdatable() {
+                return true;
+            }
+        };
+        when(pluginConfigValueTranslator.getPrefix()).thenReturn(testTranslatorKey);
+        when(pluginConfigValueTranslator.translateToPluginConfigVariable(eq(testSecretKey)))
+                .thenReturn(mockPluginConfigVariable);
+        objectUnderTest = new VariableExpander(OBJECT_MAPPER, Set.of(pluginConfigValueTranslator));
+        final Object actualResult = objectUnderTest.translate(jsonParser, PluginConfigVariable.class);
+        assertNotNull(actualResult);
+        assertThat(actualResult, equalTo(mockPluginConfigVariable));
     }
 
-    private static Stream<Arguments> getStringTypeArguments() {
-        final String testRandomValue = "non-secret-prefix-" + RandomStringUtils.randomAlphabetic(5);
-        return Stream.of(Arguments.of(String.class, String.format("\"%s\"", testRandomValue),
-                        testRandomValue),
-                Arguments.of(Duration.class, "\"PT15M\"", Duration.parse("PT15M")),
-                Arguments.of(Boolean.class, "\"true\"", true),
-                Arguments.of(Short.class, "\"2\"", (short) 2),
-                Arguments.of(Integer.class, "\"10\"", 10),
-                Arguments.of(Long.class, "\"200\"", 200L),
-                Arguments.of(Double.class, "\"1.23\"", 1.23d),
-                Arguments.of(Float.class, "\"2.15\"", 2.15f),
-                Arguments.of(BigDecimal.class, "\"2.15\"", BigDecimal.valueOf(2.15)),
-                Arguments.of(Character.class, "\"c\"", 'c'));
+    @Test
+    void testTranslateJsonParserWithSPluginConfigVariableValue_translate_failure() throws IOException {
+        final String testSecretKey = "testSecretKey";
+        final String testTranslatorKey = "test_prefix";
+        final String testSecretReference = String.format("${{%s:%s}}", testTranslatorKey, testSecretKey);
+        final JsonParser jsonParser = JSON_FACTORY.createParser(String.format("\"%s\"", testSecretReference));
+        jsonParser.nextToken();
+        when(pluginConfigValueTranslator.getPrefix()).thenReturn(testTranslatorKey);
+        when(pluginConfigValueTranslator.translateToPluginConfigVariable(eq(testSecretKey)))
+                .thenThrow(IllegalArgumentException.class);
+        objectUnderTest = new VariableExpander(OBJECT_MAPPER, Set.of(pluginConfigValueTranslator));
+        assertThrows(IllegalArgumentException.class,
+                () -> objectUnderTest.translate(jsonParser, PluginConfigVariable.class));
     }
 }
diff --git a/data-prepper-plugins/aws-plugin/build.gradle b/data-prepper-plugins/aws-plugin/build.gradle
@@ -1,4 +1,3 @@
-
 dependencies {
     implementation project(':data-prepper-api')
     implementation project(':data-prepper-plugins:aws-plugin-api')
@@ -11,6 +10,7 @@ dependencies {
     implementation 'org.hibernate.validator:hibernate-validator:8.0.2.Final'
     testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
     testImplementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'
+    testImplementation project(':data-prepper-test-common')
 }
 
 test {