[AMBARI-25059] SPI Upgrade Improvements For Web Calls And Required Pl…

…ugins (apache#2737)

ambari-agent/pom.xml

@@ -90,6 +90,7 @@
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
+      <version>18.0</version>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>

ambari-project/pom.xml

@@ -291,7 +291,7 @@
       <dependency>
         <groupId>com.google.guava</groupId>
         <artifactId>guava</artifactId>
-        <version>18.0</version>
+        <version>24.1.1-jre</version>
       </dependency>
       <dependency>
         <groupId>com.google.code.findbugs</groupId>

ambari-server-spi/src/main/java/org/apache/ambari/spi/net/HttpURLConnectionProvider.java

@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.spi.net;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The {@link HttpURLConnectionProvider} is used as a way to provide
+ * {@link HttpURLConnection} instances which are backed by Ambari's truststore,
+ * cookie store, and timeout configurations.
+ */
+public interface HttpURLConnectionProvider {
+
+  /**
+   * Gets a {@link HttpURLConnection} which is initialized and ready to read.
+   *
+   * @param url
+   *          the URL to retrieve information from.
+   * @param headers
+   *          the HTTP headers to use in the request.
+   *
+   * @return an iniitalized HTTP connection which is ready to read.
+   * @throws IOException
+   *           if the URL could not be opened.
+   */
+  HttpURLConnection getConnection(String url, Map<String, List<String>> headers) throws IOException;
+
+}

ambari-server-spi/src/main/java/org/apache/ambari/spi/net/package-info.java

@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+/**
+ * Provides classes for working with network connections and streaming.
+ */
+package org.apache.ambari.spi.net;

ambari-server-spi/src/main/java/org/apache/ambari/spi/upgrade/UpgradeCheckRequest.java

@@ -17,11 +17,13 @@
  */
 package org.apache.ambari.spi.upgrade;
 
+import java.net.HttpURLConnection;
 import java.util.HashMap;
 import java.util.Map;
 
 import org.apache.ambari.spi.ClusterInformation;
 import org.apache.ambari.spi.RepositoryVersion;
+import org.apache.ambari.spi.net.HttpURLConnectionProvider;
 
 /**
  * Represents a request to run the upgrade checks before an upgrade begins.
@@ -32,6 +34,7 @@ public class UpgradeCheckRequest {
   private boolean m_revert = false;
   private final RepositoryVersion m_targetRepositoryVersion;
   private final Map<String,String> m_checkConfigurations;
+  private final HttpURLConnectionProvider m_httpURLConnectionProvider;
 
   /**
    * Used for tracking results during a check request.
@@ -51,13 +54,19 @@ public class UpgradeCheckRequest {
    * @param checkConfigurations
    *          any configurations specified in the upgrade pack which can be used
    *          to when
+   * @param httpURLConnectionProvider
+   *          provides a mechanism for an {@link UpgradeCheck} to make URL
+   *          requests while using Ambari's truststore and configured stream
+   *          timeout settings.
    */
   public UpgradeCheckRequest(ClusterInformation clusterInformation, UpgradeType upgradeType,
-      RepositoryVersion targetRepositoryVersion, Map<String,String> checkConfigurations) {
+      RepositoryVersion targetRepositoryVersion, Map<String, String> checkConfigurations,
+      HttpURLConnectionProvider httpURLConnectionProvider) {
     m_clusterInformation = clusterInformation;
     m_upgradeType = upgradeType;
     m_targetRepositoryVersion = targetRepositoryVersion;
     m_checkConfigurations = checkConfigurations;
+    m_httpURLConnectionProvider = httpURLConnectionProvider;
   }
 
   /**
@@ -133,4 +142,14 @@ public void addResult(UpgradeCheckDescription description, UpgradeCheckStatus st
   public UpgradeCheckStatus getResult(UpgradeCheckDescription description) {
     return m_results.get(description);
   }
+
+  /**
+   * Gets a class which can construct {@link HttpURLConnection} instances which
+   * are backed by Ambari's cookie store, truststore, and timeout settings.
+   *
+   * @return the httpURLConnectionProvider an instance of the provider.
+   */
+  public HttpURLConnectionProvider getHttpURLConnectionProvider() {
+    return m_httpURLConnectionProvider;
+  }
 }

ambari-server/src/main/java/org/apache/ambari/server/checks/UpgradeCheckRegistry.java

@@ -17,10 +17,13 @@
  */
 package org.apache.ambari.server.checks;
 
+import java.net.URL;
+import java.net.URLClassLoader;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -41,6 +44,10 @@
 import org.apache.ambari.spi.upgrade.UpgradeType;
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
+import org.reflections.Reflections;
+import org.reflections.scanners.SubTypesScanner;
+import org.reflections.scanners.TypeAnnotationsScanner;
+import org.reflections.util.ConfigurationBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -171,7 +178,18 @@ public Set<String> getFailedPluginClassNames() {
 
   /**
    * Uses the library classloader from the the target stack in order to find any
-   * plugin-in {@link UpgradeCheck}s which are declared in the upgrade pack.
+   * plugin-in {@link UpgradeCheck}s which are declared in the upgrade pack as
+   * well as any upgrade checks which are found in the classloader and marked as
+   * {@link UpgradeCheckInfo#required()} for this {@link UpgradeType}.
+   * <p/>
+   * This method uses a {@link Reflections} instance which has been created
+   * using only the {@link URL}s which the stack library is comprised of. This
+   * means that scanning the path for {@link UpgradeCheck} instances is quick.
+   * However, this also means that the {@link ClassLoader} is unable to load
+   * classes which are defined in the stack but ship with Ambari's
+   * {@link ClassLoader}. For this reason, we must use a different
+   * {@link ClassLoader} for loading explicitly defined classes versus those
+   * which are discovered by {@link Reflections}.
    *
    * @param upgradePack
    *          the upgrade pack which defines the upgrade check classes.
@@ -180,15 +198,19 @@ public Set<String> getFailedPluginClassNames() {
    */
   private void loadPluginUpgradeChecksFromStack(UpgradePack upgradePack,
       PluginUpgradeChecks pluginChecks) throws AmbariException {
-    List<String> pluginCheckClassNames = upgradePack.getPrerequisiteChecks();
+    Set<String> pluginCheckClassNames = new HashSet<>(upgradePack.getPrerequisiteChecks());
     StackId ownerStackId = upgradePack.getOwnerStackId();
     StackInfo stackInfo = metainfoProvider.get().getStack(ownerStackId);
 
-    ClassLoader classLoader = stackInfo.getLibraryClassLoader();
+    URLClassLoader classLoader = stackInfo.getLibraryClassLoader();
     if (null != classLoader) {
+
+      // first find all of the plugins which are explicitely defined in the
+      // upgrade pack and attempt to load and register them
       for (String pluginCheckClassName : pluginCheckClassNames) {
         try {
-          UpgradeCheck upgradeCheck = stackInfo.getLibraryInstance(m_injector, pluginCheckClassName);
+          UpgradeCheck upgradeCheck = stackInfo.getLibraryInstance(m_injector,
+              pluginCheckClassName);
 
           pluginChecks.m_loadedChecks.add(upgradeCheck);
 
@@ -200,10 +222,47 @@ private void loadPluginUpgradeChecksFromStack(UpgradePack upgradePack,
           pluginChecks.m_failedChecks.add(pluginCheckClassName);
         }
       }
+
+      // next find all plugin checks which are required for this upgrade type by
+      // scanning just the classes shipped with the stack's library JAR
+      Reflections reflections = new Reflections(
+          new ConfigurationBuilder()
+            .addClassLoader(classLoader)
+            .addUrls(classLoader.getURLs())
+            .setScanners(new SubTypesScanner(),new TypeAnnotationsScanner()));
+
+      Set<Class<? extends UpgradeCheck>> upgradeChecksFromLoader = reflections.getSubTypesOf(
+          UpgradeCheck.class);
+
+      if(null != upgradeChecksFromLoader && !upgradeChecksFromLoader.isEmpty()) {
+        for (Class<? extends UpgradeCheck> clazz : upgradeChecksFromLoader) {
+          // first check to make sure we didn't already try to load this one if it
+          // was explicitely defined in the upgrade pack (from above)
+          if (pluginCheckClassNames.contains(clazz.getName())) {
+            continue;
+          }
+
+          // see if this check required by inspecting the annotation
+          UpgradeCheckInfo upgradeCheckInfo = clazz.getAnnotation(UpgradeCheckInfo.class);
+          if (null != upgradeCheckInfo && ArrayUtils.contains(upgradeCheckInfo.required(), upgradePack.getType())) {
+            // if the annotation says the check is required, then load it
+            try {
+              pluginChecks.m_loadedChecks.add(clazz.newInstance());
+
+              LOG.info("Registered pre-upgrade check {} for stack {}", clazz, ownerStackId);
+            } catch (Exception exception) {
+              LOG.error("Unable to load the upgrade check {}", clazz, exception);
+
+              // keep track of the failed check
+              pluginChecks.m_failedChecks.add(clazz.getName());
+            }
+          }
+        }
+      }
     } else {
       LOG.error(
           "Unable to perform the following upgrade checks because no libraries could be loaded for the {} stack: {}",
-          ownerStackId, StringUtils.join(pluginCheckClassNames, ","));
+          ownerStackId, StringUtils.join(pluginCheckClassNames, ", "));
 
       pluginChecks.m_failedChecks.addAll(pluginCheckClassNames);
     }

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/PreUpgradeCheckResourceProvider.java

@@ -28,6 +28,7 @@
 import org.apache.ambari.server.checks.UpgradeCheckRegistry;
 import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.internal.URLStreamProvider.AmbariHttpUrlConnectionProvider;
 import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
 import org.apache.ambari.server.controller.spi.NoSuchResourceException;
 import org.apache.ambari.server.controller.spi.Predicate;
@@ -215,7 +216,8 @@ public Set<Resource> getResources(Request request, Predicate predicate) throws S
 
       final UpgradeCheckRequest upgradeCheckRequest = new UpgradeCheckRequest(clusterInformation,
           upgradeType, targetRepositoryVersion,
-          upgradePack.getPrerequisiteCheckConfig().getAllProperties());
+          upgradePack.getPrerequisiteCheckConfig().getAllProperties(),
+          new AmbariHttpUrlConnectionProvider());
 
       if (propertyMap.containsKey(UPGRADE_CHECK_FOR_REVERT_PROPERTY_ID)) {
         Boolean forRevert = BooleanUtils.toBooleanObject(propertyMap.get(UPGRADE_CHECK_FOR_REVERT_PROPERTY_ID).toString());

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/URLStreamProvider.java

@@ -37,6 +37,8 @@
 
 import org.apache.ambari.server.configuration.ComponentSSLConfiguration;
 import org.apache.ambari.server.controller.utilities.StreamProvider;
+import org.apache.ambari.server.proxy.ProxyService;
+import org.apache.ambari.spi.net.HttpURLConnectionProvider;
 import org.apache.commons.io.IOUtils;
 import org.apache.http.HttpStatus;
 import org.slf4j.Logger;
@@ -93,20 +95,20 @@ public URLStreamProvider(int connectionTimeout, int readTimeout,
   public URLStreamProvider(int connectionTimeout, int readTimeout, String trustStorePath,
                            String trustStorePassword, String trustStoreType) {
 
-    this.connTimeout        = connectionTimeout;
+    connTimeout        = connectionTimeout;
     this.readTimeout        = readTimeout;
     this.trustStorePath     = trustStorePath;
     this.trustStorePassword = trustStorePassword;
     this.trustStoreType     = trustStoreType;
-    this.setupTruststoreForHttps = true;
+    setupTruststoreForHttps = true;
   }
 
   public void setSetupTruststoreForHttps(boolean setupTruststoreForHttps) {
     this.setupTruststoreForHttps = setupTruststoreForHttps;
   }
-  
+
   public boolean getSetupTruststoreForHttps() {
-    return this.setupTruststoreForHttps;
+    return setupTruststoreForHttps;
   }
 
   // ----- StreamProvider ----------------------------------------------------
@@ -178,7 +180,7 @@ public HttpURLConnection processURL(String spec, String requestMethod, byte[] bo
       LOG.debug("readFrom spec:{}", spec);
     }
 
-    HttpURLConnection connection = (spec.startsWith("https") && this.setupTruststoreForHttps) ?
+    HttpURLConnection connection = (spec.startsWith("https") && setupTruststoreForHttps) ?
             getSSLConnection(spec) : getConnection(spec);
 
     AppCookieManager appCookieManager = getAppCookieManager();
@@ -323,7 +325,39 @@ protected HttpsURLConnection getSSLConnection(String spec) throws IOException, I
         .openConnection());
 
     connection.setSSLSocketFactory(sslSocketFactory);
- 
+
     return connection;
   }
+
+  /**
+   * A default implementation of {@link HttpURLConnectionProvider}, this class
+   * will use the {@link URLStreamProvider} in order to provide an
+   * {@link HttpURLConnection} which is able to use Ambari's cookie store,
+   * truststore, and timeout values.
+   */
+  public static final class AmbariHttpUrlConnectionProvider implements HttpURLConnectionProvider {
+
+    /**
+     * The stream provider.
+     */
+    private final URLStreamProvider m_streamProvider;
+
+    /**
+     * Constructor.
+     *
+     */
+    public AmbariHttpUrlConnectionProvider() {
+      m_streamProvider = new URLStreamProvider(ProxyService.URL_CONNECT_TIMEOUT,
+          ProxyService.URL_READ_TIMEOUT, ComponentSSLConfiguration.instance());
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public HttpURLConnection getConnection(String url, Map<String, List<String>> headers)
+        throws IOException {
+      return m_streamProvider.processURL(url, "GET", (InputStream) null, headers);
+    }
+  }
 }

ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java

@@ -42,6 +42,8 @@
 
 import org.apache.commons.lang.builder.EqualsBuilder;
 
+import com.google.common.base.MoreObjects;
+
 @Entity
 @Table(name = "clusterconfig",
   uniqueConstraints = {@UniqueConstraint(name = "UQ_config_type_tag", columnNames = {"cluster_id", "type_name", "version_tag"}),
@@ -303,7 +305,7 @@ public void setServiceConfigEntities(Collection<ServiceConfigEntity> serviceConf
    */
   @Override
   public String toString() {
-    return com.google.common.base.Objects.toStringHelper(this)
+    return MoreObjects.toStringHelper(this)
       .add("clusterId", clusterId)
       .add("type", type)
       .add("version", version)

ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java

@@ -41,6 +41,7 @@
 import org.apache.ambari.server.state.MaintenanceState;
 import org.apache.ambari.server.state.State;
 
+import com.google.common.base.MoreObjects;
 import com.google.common.base.Objects;
 
 
@@ -278,7 +279,7 @@ public void setRestartRequired(boolean restartRequired) {
    */
   @Override
   public String toString() {
-    return Objects.toStringHelper(this).add("serviceName", serviceName).add("componentName",
+    return MoreObjects.toStringHelper(this).add("serviceName", serviceName).add("componentName",
         componentName).add("hostId", hostId).add("desiredState", desiredState).toString();
   }
 }