Hot reload of certs for HTTP server (#166)

application/pom.xml

@@ -140,7 +140,7 @@
 
         <dependency>
             <groupId>org.mockito</groupId>
-            <artifactId>mockito-all</artifactId>
+            <artifactId>mockito-core</artifactId>
             <scope>test</scope>
         </dependency>
 
@@ -155,5 +155,22 @@
             <artifactId>equalsverifier</artifactId>
             <scope>test</scope>
         </dependency>
+
+        <!-- Test Spring Boot -->
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>

application/src/main/java/com/ericsson/bss/cassandra/ecchronos/application/spring/TomcatWebServerCustomizer.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2021 Telefonaktiebolaget LM Ericsson
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.ericsson.bss.cassandra.ecchronos.application.spring;
+
+
+import org.apache.coyote.http11.Http11NioProtocol;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory;
+import org.springframework.boot.web.server.WebServerFactoryCustomizer;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+@Component
+@EnableScheduling
+public class TomcatWebServerCustomizer implements WebServerFactoryCustomizer<TomcatServletWebServerFactory>
+{
+    private Http11NioProtocol http11NioProtocol;
+
+    @Value("${server.ssl.enabled:false}")
+    private Boolean sslIsEnabled;
+
+    @Override
+    public void customize(TomcatServletWebServerFactory factory)
+    {
+        if (sslIsEnabled)
+        {
+            factory.addConnectorCustomizers(connector ->
+            {
+                http11NioProtocol = (Http11NioProtocol) connector.getProtocolHandler();
+            });
+        }
+    }
+
+    /**
+     * Reload the {@code SSLHostConfig} if SSL is enabled. Doing so should update ssl settings and fetch certificates from Keystores
+     * It reloads them every 60 seconds by default
+     */
+    @Scheduled (initialDelayString = "${server.ssl.refresh-rate-in-ms:60000}", fixedRateString = "${server.ssl.refresh-rate-in-ms:60000}")
+    public void reloadSslContext()
+    {
+        if (sslIsEnabled && http11NioProtocol != null)
+        {
+            http11NioProtocol.reloadSslHostConfigs();
+        }
+    }
+}

application/src/main/resources/application.yml

@@ -33,4 +33,6 @@ management:
 #    key-store-password: <password>
 #    key-store-type: <keystore type>
 #    key-alias: <key alias>
-#    key-password: <key password>
+#    key-password: <key password>
+# Rate at which certificate are reloaded automatically
+#    refresh-rate-in-ms: 60000

application/src/test/java/com/ericsson/bss/cassandra/ecchronos/application/TestReloadingAuthProvider.java

@@ -50,12 +50,6 @@ public class TestReloadingAuthProvider
     @Mock
     private EndPoint endPoint;
 
-    @Before
-    public void setup()
-    {
-        when(endPoint.resolve()).thenReturn(new InetSocketAddress(0));
-    }
-
     @Test
     public void testCorrectResponse()
     {

application/src/test/java/com/ericsson/bss/cassandra/ecchronos/application/spring/TestTomcatWebServerCustomizer.java

@@ -0,0 +1,169 @@
+/*
+ * Copyright 2021 Telefonaktiebolaget LM Ericsson
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.ericsson.bss.cassandra.ecchronos.application.spring;
+
+import static com.google.common.io.Resources.getResource;
+import static java.net.HttpURLConnection.HTTP_OK;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
+import static org.awaitility.Awaitility.await;
+import static org.mockito.Mockito.atLeast;
+import static org.mockito.Mockito.verify;
+import static org.springframework.test.context.support.TestPropertySourceUtils.addInlinedPropertiesToEnvironment;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.concurrent.TimeUnit;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLHandshakeException;
+
+import com.ericsson.bss.cassandra.ecchronos.connection.JmxConnectionProvider;
+import com.ericsson.bss.cassandra.ecchronos.connection.NativeConnectionProvider;
+import com.ericsson.bss.cassandra.ecchronos.core.repair.state.ReplicationState;
+import com.ericsson.bss.cassandra.ecchronos.core.utils.NodeResolver;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.TrustAllStrategy;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.ssl.SSLContextBuilder;
+import org.awaitility.Duration;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.boot.test.mock.mockito.SpyBean;
+import org.springframework.context.ApplicationContextInitializer;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.core.env.Environment;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+@RunWith (SpringRunner.class)
+@SpringBootTest (webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ContextConfiguration(initializers = TestTomcatWebServerCustomizer.PropertyOverrideContextInitializer.class)
+public class TestTomcatWebServerCustomizer
+{
+    private static final String SERVER_KEYSTORE = "src/test/resources/server/ks.p12";
+    private static final String SERVER_TRUSTSTORE = "src/test/resources/server/ts.p12";
+    private static final String CLIENT_VALID_PATH = "valid/";
+    private static final String CLIENT_EXPIRED_PATH = "expired/";
+    private static final int REFRESH_RATE = 100;
+    private static final int INVOCATION_COUNT = 1;
+
+    private String httpsUrl;
+
+    @MockBean
+    private ECChronos ecChronos;
+
+    @MockBean
+    private NativeConnectionProvider nativeConnectionProvider;
+
+    @MockBean
+    private JmxConnectionProvider jmxConnectionProvider;
+
+    @MockBean
+    private ReplicationState replicationState;
+
+    @MockBean
+    private NodeResolver nodeResolver;
+
+    @MockBean
+    private RepairHistoryBean repairHistoryBean;
+
+    @SpyBean
+    private TomcatWebServerCustomizer tomcatWebServerCustomizer;
+
+    @Autowired
+    private Environment environment;
+
+    @Before
+    public void init()
+    {
+        String httpsPort = environment.getProperty("local.server.port");
+        httpsUrl = "https://localhost:" + httpsPort + "/actuator/health";
+    }
+
+    @Test
+    public void testSuccessfulCertificateReloading()
+    {
+        await().atMost(new Duration(REFRESH_RATE * (INVOCATION_COUNT + 10), TimeUnit.MILLISECONDS))
+                .untilAsserted(() -> verify(tomcatWebServerCustomizer, atLeast(INVOCATION_COUNT)).reloadSslContext());
+    }
+
+    @Test
+    public void testSuccessfulResponseWhenValidCertificate() throws IOException, GeneralSecurityException
+    {
+        HttpResponse response = configureHttpClient(CLIENT_VALID_PATH).execute(new HttpGet(httpsUrl));
+        assertThat(response.getStatusLine().getStatusCode()).isEqualTo(HTTP_OK);
+    }
+
+    @Test
+    public void testExceptionWhenExpiredCertificate() throws IOException, GeneralSecurityException
+    {
+        HttpClient httpClient = configureHttpClient(CLIENT_EXPIRED_PATH);
+        assertThatExceptionOfType(SSLHandshakeException.class)
+                .isThrownBy(() -> httpClient.execute(new HttpGet(httpsUrl)))
+                .withMessageContaining("Received fatal alert: certificate_unknown");
+    }
+
+    private HttpClient configureHttpClient(String storePath) throws IOException, GeneralSecurityException
+    {
+        SSLContext sslContext = SSLContextBuilder.create()
+                .loadKeyMaterial(getResource(storePath + "crt.p12"), "".toCharArray(), "".toCharArray())
+                .loadTrustMaterial(new TrustAllStrategy())
+                .build();
+
+        return HttpClients.custom()
+                .setSSLContext(sslContext)
+                .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE)
+                .build();
+    }
+
+    static class PropertyOverrideContextInitializer implements ApplicationContextInitializer<ConfigurableApplicationContext>
+    {
+        @Rule
+        public TemporaryFolder tempFolder = new TemporaryFolder();
+
+        PropertyOverrideContextInitializer() throws IOException
+        {
+            tempFolder.create();
+        }
+
+        @Override
+        public void initialize(ConfigurableApplicationContext configurableApplicationContext)
+        {
+            addInlinedPropertiesToEnvironment(configurableApplicationContext,
+                    "server.ssl.enabled=true",
+                    "server.ssl.key-store=" + SERVER_KEYSTORE,
+                    "server.ssl.key-store-password=",
+                    "server.ssl.key-store-type=PKCS12",
+                    "server.ssl.key-alias=cert",
+                    "server.ssl.key-password=",
+                    "server.ssl.trust-store=" + SERVER_TRUSTSTORE,
+                    "server.ssl.trust-store-password=",
+                    "server.ssl.trust-store-type=PKCS12",
+                    "server.ssl.client-auth=need",
+                    "server.ssl.refresh-rate-in-ms=" + REFRESH_RATE);
+        }
+    }
+}

pom.xml

@@ -67,12 +67,17 @@
         <cassandra.version>3.0.15</cassandra.version>
         <gson.version>2.8.5</gson.version>
         <osgi.version>1.3.0</osgi.version>
-        <mockito.version>1.10.19</mockito.version>
+        <mockito.all.version>1.10.19</mockito.all.version>
+        <mockito.core.version>2.15.0</mockito.core.version>
         <assertj.version>3.11.1</assertj.version>
         <junit.version>4.13</junit.version>
         <org.apache.commons.io.version>1.3.2</org.apache.commons.io.version>
         <com.fasterxml.jackson.version>2.11.2</com.fasterxml.jackson.version>
 
+        <spring.test.version>5.2.9.RELEASE</spring.test.version>
+        <spring.boot.test.version>2.3.4.RELEASE</spring.boot.test.version>
+        <httpclient.version>4.5.10</httpclient.version>
+
         <pax-exam.version>4.13.1</pax-exam.version>
         <pax-logging-logback.version>1.8.4</pax-logging-logback.version>
         <apache.karaf.version>4.1.4</apache.karaf.version>
@@ -272,7 +277,14 @@
             <dependency>
                 <groupId>org.mockito</groupId>
                 <artifactId>mockito-all</artifactId>
-                <version>${mockito.version}</version>
+                <version>${mockito.all.version}</version>
+                <scope>test</scope>
+            </dependency>
+
+            <dependency>
+                <groupId>org.mockito</groupId>
+                <artifactId>mockito-core</artifactId>
+                <version>${mockito.core.version}</version>
                 <scope>test</scope>
             </dependency>
 
@@ -297,6 +309,26 @@
                 <scope>test</scope>
             </dependency>
 
+            <!-- Test Spring Boot -->
+            <dependency>
+                <groupId>org.springframework</groupId>
+                <artifactId>spring-test</artifactId>
+                <version>${spring.test.version}</version>
+                <scope>test</scope>
+            </dependency>
+            <dependency>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-test</artifactId>
+                <version>${spring.boot.test.version}</version>
+                <scope>test</scope>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.httpcomponents</groupId>
+                <artifactId>httpclient</artifactId>
+                <version>${httpclient.version}</version>
+                <scope>test</scope>
+            </dependency>
+
             <!-- Pax exam -->
             <dependency>
                 <groupId>org.ops4j.pax.exam</groupId>
@@ -520,6 +552,7 @@ limitations under the License.
                             <exclude>src/test/resources/cassandra-rackdc.properties</exclude>
                             <exclude>**/*.cql</exclude>
                             <exclude>**/*.txt</exclude>
+                            <exclude>**/*.p12</exclude>
                             <exclude>**/pom.xml.tag</exclude>
                             <exclude>**/pom.xml.releaseBackup</exclude>
                             <exclude>**/*.options</exclude>
@@ -889,6 +922,12 @@ limitations under the License.
             <organization>Ericsson AB</organization>
             <organizationUrl>http://www.ericsson.com</organizationUrl>
         </developer>
+        <developer>
+            <name>Jan Karlsson</name>
+            <email>[email protected]</email>
+            <organization>Ericsson AB</organization>
+            <organizationUrl>http://www.ericsson.com</organizationUrl>
+        </developer>
     </developers>
 
     <scm>