fix(spring): refactor spring configuration (datahub-project#10290)

Author: david-leifker

build.gradle

@@ -2,12 +2,14 @@ buildscript {
   ext.jdkVersionDefault = 17
   ext.javaClassVersionDefault = 11
 
+  def springModules = ['mae-consumer', 'mce-consumer', 'pe-consumer']
+
   ext.jdkVersion = { p ->
     // If Spring 6 is present, hard dependency on jdk17
     if (p.configurations.any { it.getDependencies().any{
       (it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6."))
       || (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test"))
-    }}) {
+    }} || springModules.contains(p.name)) {
       return 17
     } else {
       // otherwise we can use the preferred default which can be overridden with a property: -PjdkVersionDefault
@@ -20,7 +22,7 @@ buildscript {
     if (p.configurations.any { it.getDependencies().any {
       (it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6."))
               || (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test"))
-    }}) {
+    }} || springModules.contains(p.name)) {
       return 17
     } else {
       // otherwise we can use the preferred default which can be overridden with a property: -PjavaClassVersionDefault
@@ -158,6 +160,7 @@ project.ext.externalDependency = [
     'javatuples': 'org.javatuples:javatuples:1.2',
     'javaxInject' : 'javax.inject:javax.inject:1',
     'javaxValidation' : 'javax.validation:validation-api:2.0.1.Final',
+    'jakartaValidation': 'jakarta.validation:jakarta.validation-api:3.1.0-M2',
     'jerseyCore': 'org.glassfish.jersey.core:jersey-client:2.41',
     'jerseyGuava': 'org.glassfish.jersey.bundles.repackaged:jersey-guava:2.25.1',
     'jettyJaas': "org.eclipse.jetty:jetty-jaas:$jettyVersion",
@@ -266,6 +269,7 @@ project.ext.externalDependency = [
     'jline':'jline:jline:1.4.1',
     'jetbrains':' org.jetbrains.kotlin:kotlin-stdlib:1.6.0',
     'annotationApi': 'javax.annotation:javax.annotation-api:1.3.2',
+    'jakartaAnnotationApi': 'jakarta.annotation:jakarta.annotation-api:3.0.0',
     'classGraph': 'io.github.classgraph:classgraph:4.8.168',
 ]
 

datahub-frontend/app/config/ConfigurationProvider.java

@@ -6,16 +6,19 @@
 import com.linkedin.metadata.config.kafka.KafkaConfiguration;
 import com.linkedin.metadata.spring.YamlPropertySourceFactory;
 import lombok.Data;
+import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.PropertySource;
+import org.springframework.stereotype.Component;
 
 /**
  * Minimal sharing between metadata-service and frontend Does not use the factories module to avoid
  * transitive dependencies.
  */
 @EnableConfigurationProperties
-@PropertySource(value = "application.yml", factory = YamlPropertySourceFactory.class)
+@PropertySource(value = "classpath:/application.yaml", factory = YamlPropertySourceFactory.class)
 @ConfigurationProperties
 @Data
 public class ConfigurationProvider {

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeCliApplication.java

@@ -5,7 +5,6 @@
 import com.linkedin.gms.factory.graphql.GraphQLEngineFactory;
 import com.linkedin.gms.factory.kafka.KafkaEventConsumerFactory;
 import com.linkedin.gms.factory.kafka.SimpleKafkaConsumerFactory;
-import com.linkedin.gms.factory.kafka.schemaregistry.InternalSchemaRegistryFactory;
 import com.linkedin.gms.factory.telemetry.ScheduledAnalyticsFactory;
 import org.springframework.boot.WebApplicationType;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -31,7 +30,6 @@
             DataHubAuthorizerFactory.class,
             SimpleKafkaConsumerFactory.class,
             KafkaEventConsumerFactory.class,
-            InternalSchemaRegistryFactory.class,
             GraphQLEngineFactory.class
           })
     })

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java

@@ -6,9 +6,9 @@
 import com.linkedin.datahub.upgrade.system.SystemUpdateBlocking;
 import com.linkedin.datahub.upgrade.system.SystemUpdateNonBlocking;
 import com.linkedin.datahub.upgrade.system.elasticsearch.steps.DataHubStartupStep;
-import com.linkedin.gms.factory.common.TopicConventionFactory;
 import com.linkedin.gms.factory.config.ConfigurationProvider;
 import com.linkedin.gms.factory.kafka.DataHubKafkaProducerFactory;
+import com.linkedin.gms.factory.kafka.common.TopicConventionFactory;
 import com.linkedin.gms.factory.kafka.schemaregistry.InternalSchemaRegistryFactory;
 import com.linkedin.gms.factory.kafka.schemaregistry.SchemaRegistryConfig;
 import com.linkedin.metadata.config.kafka.KafkaConfiguration;
@@ -110,4 +110,14 @@ protected KafkaEventProducer kafkaEventProducer(
       @Qualifier("duheKafkaEventProducer") KafkaEventProducer kafkaEventProducer) {
     return kafkaEventProducer;
   }
+
+  @Primary
+  @Bean(name = "schemaRegistryConfig")
+  @ConditionalOnProperty(
+      name = "kafka.schemaRegistry.type",
+      havingValue = InternalSchemaRegistryFactory.TYPE)
+  protected SchemaRegistryConfig schemaRegistryConfig(
+      @Qualifier("duheSchemaRegistryConfig") SchemaRegistryConfig duheSchemaRegistryConfig) {
+    return duheSchemaRegistryConfig;
+  }
 }

datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNoSchemaRegistryTest.java

@@ -1,13 +1,22 @@
 package com.linkedin.datahub.upgrade;
 
+import static com.linkedin.metadata.EventUtils.RENAMED_MCL_AVRO_SCHEMA;
+import static com.linkedin.metadata.boot.kafka.MockSystemUpdateSerializer.topicToSubjectName;
 import static org.mockito.Mockito.mock;
-import static org.testng.AssertJUnit.assertEquals;
-import static org.testng.AssertJUnit.assertNotNull;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
 
 import com.linkedin.datahub.upgrade.system.SystemUpdate;
+import com.linkedin.gms.factory.kafka.schemaregistry.SchemaRegistryConfig;
+import com.linkedin.metadata.boot.kafka.MockSystemUpdateDeserializer;
+import com.linkedin.metadata.boot.kafka.MockSystemUpdateSerializer;
 import com.linkedin.metadata.dao.producer.KafkaEventProducer;
 import com.linkedin.metadata.entity.EntityServiceImpl;
+import com.linkedin.mxe.Topics;
+import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient;
+import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException;
 import io.datahubproject.metadata.context.OperationContext;
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -23,8 +32,8 @@
     classes = {UpgradeCliApplication.class, UpgradeCliApplicationTestConfiguration.class},
     properties = {
       "kafka.schemaRegistry.type=INTERNAL",
-      "DATAHUB_UPGRADE_HISTORY_TOPIC_NAME=test_due_topic",
-      "METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=test_mcl_versioned_topic"
+      "DATAHUB_UPGRADE_HISTORY_TOPIC_NAME=" + Topics.DATAHUB_UPGRADE_HISTORY_TOPIC_NAME,
+      "METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=" + Topics.METADATA_CHANGE_LOG_VERSIONED,
     },
     args = {"-u", "SystemUpdate"})
 public class DatahubUpgradeNoSchemaRegistryTest extends AbstractTestNGSpringContextTests {
@@ -43,15 +52,29 @@ public class DatahubUpgradeNoSchemaRegistryTest extends AbstractTestNGSpringCont
 
   @Autowired private EntityServiceImpl entityService;
 
+  @Autowired
+  @Named("schemaRegistryConfig")
+  private SchemaRegistryConfig schemaRegistryConfig;
+
   @Test
   public void testSystemUpdateInit() {
     assertNotNull(systemUpdate);
   }
 
   @Test
-  public void testSystemUpdateKafkaProducerOverride() {
+  public void testSystemUpdateKafkaProducerOverride() throws RestClientException, IOException {
+    assertEquals(schemaRegistryConfig.getDeserializer(), MockSystemUpdateDeserializer.class);
+    assertEquals(schemaRegistryConfig.getSerializer(), MockSystemUpdateSerializer.class);
     assertEquals(kafkaEventProducer, duheKafkaEventProducer);
     assertEquals(entityService.getProducer(), duheKafkaEventProducer);
+
+    MockSystemUpdateSerializer serializer = new MockSystemUpdateSerializer();
+    serializer.configure(schemaRegistryConfig.getProperties(), false);
+    SchemaRegistryClient registry = serializer.getSchemaRegistryClient();
+    assertEquals(
+        registry.getId(
+            topicToSubjectName(Topics.METADATA_CHANGE_LOG_VERSIONED), RENAMED_MCL_AVRO_SCHEMA),
+        2);
   }
 
   @Test

datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java

@@ -4,14 +4,21 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
+import static org.testng.Assert.assertEquals;
 import static org.testng.AssertJUnit.assertNotNull;
 
 import com.linkedin.datahub.upgrade.impl.DefaultUpgradeManager;
 import com.linkedin.datahub.upgrade.system.SystemUpdateNonBlocking;
 import com.linkedin.datahub.upgrade.system.vianodes.ReindexDataJobViaNodesCLL;
+import com.linkedin.gms.factory.kafka.schemaregistry.SchemaRegistryConfig;
+import com.linkedin.metadata.boot.kafka.MockSystemUpdateDeserializer;
+import com.linkedin.metadata.boot.kafka.MockSystemUpdateSerializer;
+import com.linkedin.metadata.dao.producer.KafkaEventProducer;
 import com.linkedin.metadata.entity.AspectDao;
 import com.linkedin.metadata.entity.EntityService;
+import com.linkedin.metadata.entity.EntityServiceImpl;
 import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs;
+import com.linkedin.mxe.Topics;
 import io.datahubproject.test.metadata.context.TestOperationContexts;
 import java.util.List;
 import javax.inject.Named;
@@ -27,8 +34,8 @@
     properties = {
       "BOOTSTRAP_SYSTEM_UPDATE_DATA_JOB_NODE_CLL_ENABLED=true",
       "kafka.schemaRegistry.type=INTERNAL",
-      "DATAHUB_UPGRADE_HISTORY_TOPIC_NAME=test_due_topic",
-      "METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=test_mcl_versioned_topic"
+      "DATAHUB_UPGRADE_HISTORY_TOPIC_NAME=" + Topics.DATAHUB_UPGRADE_HISTORY_TOPIC_NAME,
+      "METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=" + Topics.METADATA_CHANGE_LOG_VERSIONED,
     },
     args = {"-u", "SystemUpdateNonBlocking"})
 public class DatahubUpgradeNonBlockingTest extends AbstractTestNGSpringContextTests {
@@ -38,9 +45,28 @@ public class DatahubUpgradeNonBlockingTest extends AbstractTestNGSpringContextTe
   private SystemUpdateNonBlocking systemUpdateNonBlocking;
 
   @Autowired
+  @Named("schemaRegistryConfig")
+  private SchemaRegistryConfig schemaRegistryConfig;
+
+  @Autowired
+  @Named("duheKafkaEventProducer")
+  private KafkaEventProducer duheKafkaEventProducer;
+
+  @Autowired
+  @Named("kafkaEventProducer")
+  private KafkaEventProducer kafkaEventProducer;
+
+  @Autowired private EntityServiceImpl entityService;
+
   @Test
   public void testSystemUpdateNonBlockingInit() {
     assertNotNull(systemUpdateNonBlocking);
+
+    // Expected system update configuration and producer
+    assertEquals(schemaRegistryConfig.getDeserializer(), MockSystemUpdateDeserializer.class);
+    assertEquals(schemaRegistryConfig.getSerializer(), MockSystemUpdateSerializer.class);
+    assertEquals(duheKafkaEventProducer, kafkaEventProducer);
+    assertEquals(entityService.getProducer(), duheKafkaEventProducer);
   }
 
   @Test

datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java

@@ -1,18 +1,15 @@
 package com.linkedin.datahub.upgrade;
 
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
 import com.linkedin.gms.factory.auth.SystemAuthenticationFactory;
 import com.linkedin.metadata.graph.GraphService;
 import com.linkedin.metadata.models.registry.ConfigEntityRegistry;
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import com.linkedin.metadata.registry.SchemaRegistryService;
+import com.linkedin.metadata.registry.SchemaRegistryServiceImpl;
 import com.linkedin.metadata.search.SearchService;
 import com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilders;
+import com.linkedin.mxe.TopicConventionImpl;
 import io.ebean.Database;
-import java.util.Optional;
 import org.springframework.boot.test.context.TestConfiguration;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.context.annotation.Bean;
@@ -38,8 +35,6 @@ public class UpgradeCliApplicationTestConfiguration {
 
   @Bean
   public SchemaRegistryService schemaRegistryService() {
-    SchemaRegistryService mockService = mock(SchemaRegistryService.class);
-    when(mockService.getSchemaIdForTopic(anyString())).thenReturn(Optional.of(0));
-    return mockService;
+    return new SchemaRegistryServiceImpl(new TopicConventionImpl());
   }
 }

docker/build.gradle

@@ -42,6 +42,7 @@ tasks.register('quickstart') {}
 tasks.register('quickstartSlim') {}
 tasks.register('quickstartDebug') {}
 tasks.register('quickstartPg') {}
+tasks.register('quickstartStorage') {}
 
 tasks.register('quickstartNuke') {
     doFirst {
@@ -133,6 +134,19 @@ dockerCompose {
         stopContainers = false
         removeVolumes = false
     }
+
+    quickstartStorage {
+        isRequiredBy(tasks.named('quickstartStorage'))
+        composeAdditionalArgs = ['--profile', 'quickstart-storage']
+
+        useComposeFiles = ['profiles/docker-compose.yml']
+        projectName = 'datahub'
+        projectNamePrefix = ''
+        buildBeforeUp = false
+        buildBeforePull = false
+        stopContainers = false
+        removeVolumes = false
+    }
 }
 tasks.getByName('quickstartComposeUp').dependsOn(
         quickstart_modules.collect { it + ':dockerTag' })

docs/authentication/concepts.md

@@ -75,7 +75,7 @@ for configuring multiple ways to authenticate a given request, for example via L
 
 Only if each Authenticator within the chain fails to authenticate a request will it be rejected.
 
-The Authenticator Chain can be configured in the `application.yml` file under `authentication.authenticators`:
+The Authenticator Chain can be configured in the `application.yaml` file under `authentication.authenticators`:
 
 ```
 authentication:

docs/authentication/introducing-metadata-service-authentication.md

@@ -53,7 +53,7 @@ To enable Metadata Service Authentication:
 
 OR
 
-- change the Metadata Service `application.yml` configuration file to set `authentication.enabled` to "true" AND
+- change the Metadata Service `application.yaml` configuration file to set `authentication.enabled` to "true" AND
 - change the Frontend Proxy Service `application.config` configuration file to set `metadataService.auth.enabled` to "true"
 
 After setting the configuration flag, simply restart the Metadata Service to start enforcing Authentication. 
@@ -116,7 +116,7 @@ These changes represent the first milestone in Metadata Service Authentication.
 
 That's perfectly fine, for now. Metadata Service Authentication is disabled by default, only enabled if you provide the 
 environment variable `METADATA_SERVICE_AUTH_ENABLED` to the `datahub-gms` container or change the `authentication.enabled` to "true"
-inside your DataHub Metadata Service configuration (`application.yml`). 
+inside your DataHub Metadata Service configuration (`application.yaml`). 
 
 That being said, we will be recommending that you enable Authentication for production use cases, to prevent
 arbitrary actors from ingesting metadata into DataHub. 
@@ -141,7 +141,7 @@ the root "datahub" user account.
 
 ### I want to authenticate requests using a custom Authenticator? How do I do this? 
 
-You can configure DataHub to add your custom **Authenticator** to the **Authentication Chain** by changing the `application.yml` configuration file for the Metadata Service:
+You can configure DataHub to add your custom **Authenticator** to the **Authentication Chain** by changing the `application.yaml` configuration file for the Metadata Service:
 
 ```yml
 authentication:

docs/deploy/aws.md

@@ -332,7 +332,7 @@ and [here](../../metadata-service/factories/src/main/java/com/linkedin/gms/facto
 .
 
 A mapping between the property name used in the above two files and the name used in docker/env file can be
-found [here](../../metadata-service/configuration/src/main/resources/application.yml).
+found [here](../../metadata-service/configuration/src/main/resources/application.yaml).
 
 ### Managed Streaming for Apache Kafka (MSK)
 

docs/how/updating-datahub.md

@@ -328,7 +328,7 @@ Helm with `--atomic`: In general, it is recommended to not use the `--atomic` se
 
 - #6243 apache-ranger authorizer is no longer the core part of DataHub GMS, and it is shifted as plugin. Please refer updated documentation [Configuring Authorization with Apache Ranger](./configuring-authorization-with-apache-ranger.md#configuring-your-datahub-deployment) for configuring `apache-ranger-plugin` in DataHub GMS.
 - #6243 apache-ranger authorizer as plugin is not supported in DataHub Kubernetes deployment.
-- #6243 Authentication and Authorization plugins configuration are removed from [application.yml](../../metadata-service/configuration/src/main/resources/application.yml). Refer documentation [Migration Of Plugins From application.yml](../plugins.md#migration-of-plugins-from-applicationyml) for migrating any existing custom plugins.
+- #6243 Authentication and Authorization plugins configuration are removed from [application.yaml](../../metadata-service/configuration/src/main/resources/application.yaml). Refer documentation [Migration Of Plugins From application.yaml](../plugins.md#migration-of-plugins-from-applicationyml) for migrating any existing custom plugins.
 - `datahub check graph-consistency` command has been removed. It was a beta API that we had considered but decided there are better solutions for this. So removing this.
 - `graphql_url` option of `powerbi-report-server` source deprecated as the options is not used.
 - #6789 BigQuery ingestion: If `enable_legacy_sharded_table_support` is set to False, sharded table names will be suffixed with \_yyyymmdd to make sure they don't clash with non-sharded tables. This means if stateful ingestion is enabled then old sharded tables will be recreated with a new id and attached tags/glossary terms/etc will need to be added again. _This behavior is not enabled by default yet, but will be enabled by default in a future release._

docs/plugins.md

@@ -252,11 +252,11 @@ All other access are forbidden for the plugin.
 
 > Disclaimer: In BETA version your plugin can access any port and can read/write to any location on file system, however you should implement the plugin as per above access permission to keep your plugin compatible with upcoming release of DataHub.
 
-## Migration Of Plugins From application.yml
-If you have any custom Authentication or Authorization plugin define in `authorization` or `authentication` section of  [application.yml](../metadata-service/configuration/src/main/resources/application.yml) then migrate them as per below steps.
+## Migration Of Plugins From application.yaml
+If you have any custom Authentication or Authorization plugin define in `authorization` or `authentication` section of  [application.yaml](../metadata-service/configuration/src/main/resources/application.yaml) then migrate them as per below steps.
 
 1. Implement Plugin: For Authentication Plugin follow steps of [Implementing an Authentication Plugin](#implementing-an-authentication-plugin) and for Authorization Plugin follow steps of [Implementing an Authorization Plugin](#implementing-an-authorization-plugin)
-2. Install Plugin: Install the plugins as per steps mentioned in [Plugin Installation](#plugin-installation). Here you need to map the configuration from [application.yml](../metadata-service/configuration/src/main/resources/application.yml) to configuration in `config.yml`. This mapping from `application.yml` to `config.yml` is described below 
+2. Install Plugin: Install the plugins as per steps mentioned in [Plugin Installation](#plugin-installation). Here you need to map the configuration from [application.yaml](../metadata-service/configuration/src/main/resources/application.yaml) to configuration in `config.yml`. This mapping from `application.yaml` to `config.yml` is described below 
 
    **Mapping for Authenticators**