fix(XServiceProvider): fix ebean framework race condition (datahub-project#11378)

Author: david-leifker

.github/workflows/docker-unified.yml

@@ -990,7 +990,7 @@ jobs:
           DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
           DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_INGESTION_IMAGE }}
           ACTIONS_VERSION: ${{ needs.datahub_ingestion_slim_build.outputs.tag || 'head-slim' }}
-          ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor]==0.0.13 acryl-datahub-actions==0.0.13 acryl-datahub==0.10.5"
+          ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor] acryl-datahub-actions"
           ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml"
         run: |
           ./smoke-test/run-quickstart.sh

docker/datahub-ingestion-base/Dockerfile

@@ -43,7 +43,9 @@ RUN apt-get update && apt-get upgrade -y \
     krb5-user \
     krb5-config \
     libkrb5-dev \
+    librdkafka-dev \
     wget \
+    curl \
     zip \
     unzip \
     ldap-utils \

metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/restli/RestliServletConfig.java

@@ -2,6 +2,7 @@
 
 import com.datahub.auth.authentication.filter.AuthenticationFilter;
 import com.linkedin.gms.factory.auth.SystemAuthenticationFactory;
+import com.linkedin.r2.transport.http.server.RAPJakartaServlet;
 import com.linkedin.restli.server.RestliHandlerServlet;
 import java.util.Collections;
 import org.springframework.beans.factory.annotation.Qualifier;
@@ -36,8 +37,8 @@ public ServletRegistrationBean<RestliHandlerServlet> restliServletRegistration(
   }
 
   @Bean("restliHandlerServlet")
-  public RestliHandlerServlet restliHandlerServlet() {
-    return new RestliHandlerServlet();
+  public RestliHandlerServlet restliHandlerServlet(final RAPJakartaServlet r2Servlet) {
+    return new RestliHandlerServlet(r2Servlet);
   }
 
   @Bean

metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java

@@ -77,6 +77,7 @@ public class AuthenticationFilter implements Filter {
   public void init(FilterConfig filterConfig) throws ServletException {
     SpringBeanAutowiringSupport.processInjectionBasedOnCurrentContext(this);
     buildAuthenticatorChain();
+    log.info("AuthenticationFilter initialized.");
   }
 
   @Override

metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java

@@ -73,6 +73,17 @@ public void onApplicationEvent(@Nonnull ContextRefreshedEvent event) {
         event);
     String schemaRegistryType = provider.getKafka().getSchemaRegistry().getType();
     if (ROOT_WEB_APPLICATION_CONTEXT_ID.equals(event.getApplicationContext().getId())) {
+
+      // Handle race condition, if ebean code is executed while waiting/bootstrapping (i.e.
+      // AuthenticationFilter)
+      try {
+        Class.forName("io.ebean.XServiceProvider");
+      } catch (ClassNotFoundException e) {
+        log.error(
+            "Failure to initialize required class `io.ebean.XServiceProvider` during initialization.");
+        throw new RuntimeException(e);
+      }
+
       if (InternalSchemaRegistryFactory.TYPE.equals(schemaRegistryType)) {
         executorService.submit(isSchemaRegistryAPIServletReady());
       } else {

metadata-service/factories/src/main/java/com/linkedin/restli/server/RAPServletFactory.java

@@ -16,6 +16,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.ApplicationContext;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
@@ -29,8 +30,10 @@ public class RAPServletFactory {
   private int maxSerializedStringLength;
 
   @Bean(name = "restliSpringInjectResourceFactory")
-  public SpringInjectResourceFactory springInjectResourceFactory() {
-    return new SpringInjectResourceFactory();
+  public SpringInjectResourceFactory springInjectResourceFactory(final ApplicationContext ctx) {
+    SpringInjectResourceFactory springInjectResourceFactory = new SpringInjectResourceFactory();
+    springInjectResourceFactory.setApplicationContext(ctx);
+    return springInjectResourceFactory;
   }
 
   @Bean("parseqEngineThreads")

metadata-service/factories/src/main/java/com/linkedin/restli/server/RestliHandlerServlet.java

@@ -1,23 +1,29 @@
 package com.linkedin.restli.server;
 
 import com.linkedin.r2.transport.http.server.RAPJakartaServlet;
+import jakarta.servlet.ServletConfig;
 import jakarta.servlet.ServletException;
 import jakarta.servlet.http.HttpServletRequest;
 import jakarta.servlet.http.HttpServletResponse;
 import java.io.IOException;
 import lombok.AllArgsConstructor;
-import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
 import org.springframework.web.HttpRequestHandler;
 import org.springframework.web.context.support.HttpRequestHandlerServlet;
 
+@Slf4j
 @AllArgsConstructor
-@NoArgsConstructor
-@Component
 public class RestliHandlerServlet extends HttpRequestHandlerServlet implements HttpRequestHandler {
   @Autowired private RAPJakartaServlet _r2Servlet;
 
+  @Override
+  public void init(ServletConfig config) throws ServletException {
+    log.info("Initializing RestliHandlerServlet");
+    this._r2Servlet.init(config);
+    log.info("Initialized RestliHandlerServlet");
+  }
+
   @Override
   public void service(HttpServletRequest req, HttpServletResponse res)
       throws ServletException, IOException {

metadata-service/war/src/main/java/com/linkedin/gms/WebApplicationInitializer.java

@@ -42,18 +42,13 @@ public void onStartup(ServletContext container) {
     // Independent dispatcher
     schemaRegistryServlet(container);
 
-    // Non-Spring servlets
-    healthCheckServlet(container);
-    configServlet(container);
-
-    // Restli non-Dispatcher
-    servletNames.add(restliServlet(rootContext, container));
-
     // Spring Dispatcher servlets
     DispatcherServlet dispatcherServlet = new DispatcherServlet(rootContext);
     servletNames.add(authServlet(rootContext, dispatcherServlet, container));
     servletNames.add(graphQLServlet(rootContext, dispatcherServlet, container));
     servletNames.add(openAPIServlet(rootContext, dispatcherServlet, container));
+    // Restli non-Dispatcher default
+    servletNames.add(restliServlet(rootContext, container));
 
     FilterRegistration.Dynamic filterRegistration =
         container.addFilter("authenticationFilter", AuthenticationFilter.class);
@@ -62,6 +57,10 @@ public void onStartup(ServletContext container) {
         EnumSet.of(DispatcherType.ASYNC, DispatcherType.REQUEST),
         false,
         servletNames.toArray(String[]::new));
+
+    // Non-Spring servlets
+    healthCheckServlet(container);
+    configServlet(container);
   }
 
   /*
@@ -133,10 +132,13 @@ private String restliServlet(
     rootContext.register(RestliServletConfig.class);
 
     ServletRegistration.Dynamic registration =
-        container.addServlet(servletName, new HttpRequestHandlerServlet());
+        container.addServlet(servletName, HttpRequestHandlerServlet.class);
     registration.addMapping("/*");
     registration.setLoadOnStartup(10);
     registration.setAsyncSupported(true);
+    registration.setInitParameter(
+        "org.springframework.web.servlet.FrameworkServlet.ORDER",
+        String.valueOf(Integer.MAX_VALUE - 1));
 
     return servletName;
   }

metadata-service/war/src/main/java/com/linkedin/gms/servlet/RestliServletConfig.java

@@ -7,14 +7,14 @@
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.PropertySource;
-import org.springframework.web.HttpRequestHandler;
+import org.springframework.web.context.support.HttpRequestHandlerServlet;
 
 @ComponentScan(basePackages = {"com.linkedin.restli.server"})
 @PropertySource(value = "classpath:/application.yaml", factory = YamlPropertySourceFactory.class)
 @Configuration
 public class RestliServletConfig {
   @Bean("restliRequestHandler")
-  public HttpRequestHandler restliHandlerServlet(final RAPJakartaServlet r2Servlet) {
+  public HttpRequestHandlerServlet restliHandlerServlet(final RAPJakartaServlet r2Servlet) {
     return new RestliHandlerServlet(r2Servlet);
   }
 }