Add mTLS settings to ApiClient (#1415)

Add SSLContext creation based on the provided mTLS jks Keystore/Truststore.
Add two new config options to http, keystore and truststore. They must be mounted as jks keystore files.
If the keystore is set, Reaper tries to build HttpManagementProxy with mTLS enabled.

.github/files/mutual_auth_ca.pem

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDEzCCAfugAwIBAgIUFKo4A37pUPXWajJdf5/7EnpSNyEwDQYJKoZIhvcNAQEN
+BQAwGDEWMBQGA1UEAwwNTmV0dHlUZXN0Um9vdDAgFw0yNDAxMTkxNjUxNTlaGA8y
+MTIzMTIyNjE2NTE1OVowGDEWMBQGA1UEAwwNTmV0dHlUZXN0Um9vdDCCASIwDQYJ
+KoZIhvcNAQEBBQADggEPADCCAQoCggEBAJzEkIoOIu/A+p6OOvNJc2Zny86gW0sR
+r5UPFT415Z78sH/gd3RYxvdpe3JCtTTMbmKaC5rQBkkYPbE1096+aTfEudzXIAyM
+Zj/q53C4yPCc0FX56Isv24lMSL98yFtd1ZOloFICKOGCA1hrtoASjpmmiTGllI6s
+WceThVr19EllNtNtacDumoNZz5keXHuyoLvrPLWo9gU4ldJP8dziqYDUkigJe0xD
+L0DmVNgNLPLKnuiHLTJviuuPoWQCkUOCdOFlNXooo37Za6PT5no0Hin1WJFAgEWK
+U3da9Hbgn8nwRUQVBPgywuE7+MX0R4nLuBoo02BeKQc/xAvfjnDdgBECAwEAAaNT
+MFEwHQYDVR0OBBYEFFBkDgjEaXUiN3FUgRfmK5oGrGnvMB8GA1UdIwQYMBaAFFBk
+DgjEaXUiN3FUgRfmK5oGrGnvMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEN
+BQADggEBABQ19Qnt9wp6rkPMJ/rPESaPh+uQU313m/Vgfjp/7ExvY8OkyDkOLs4r
+2OtO38mi9pBTmAMwF0UH38a2/OK8g/kNSwHu7zIjp52Ic5BUxSMXt7MWE/FD8LQR
+sd++w32CEYSd5XqaVYNDrNQkYKM/G2QEK8FsHPnEC9JnL4TglH9b2+cEdxqfvraH
+VduamM85xvA2qyR2pgStO2I94pnVbXCpGCSts9AEVMTGXhDhMEyCwTcgpyUgsYfc
+njVfnKhXEM1ODlGMDOsNbzETgKIR8rZRu6auBOI15G07DHiBJTKIlu6OgAwWfxbY
+6+q3zBcb5lbLib+zfxGCm9Pb8gInU0o=
+-----END CERTIFICATE-----

.github/files/mutual_auth_server.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDJDCCAgygAwIBAgIUJTGy6IraiFSQh5LsKwXXFuW5RAQwDQYJKoZIhvcNAQEL
+BQAwGDEWMBQGA1UEAwwNTmV0dHlUZXN0Um9vdDAeFw0yNDAxMTkxNjUxNTlaFw0y
+OTAxMTcxNjUxNTlaMBoxGDAWBgNVBAMMD05ldHR5VGVzdFNlcnZlcjCCASIwDQYJ
+KoZIhvcNAQEBBQADggEPADCCAQoCggEBAN9p7uvZl4LkTVNLZPVaWG0KSXh66ZnV
+YaQB2tsj1BaOHZsmW7xfwptrZp5L+adPQfeAg1wSq3EKqAg2fiatysWPJciVFLnZ
+pnCGho4VFzWtwMfjCpgeJ3JM1u4h4gCXlZKAQeEMrcfdzB8azqqDyF0ip4oydGRn
+NfG3apK1wMVA5yi3MsXrklEQZrbKCOj913JNS/W2pJGFyXS5ljJIME7BPv1aWngv
+RZRLRPWj+1fDzfwvbaJ01+YJW9h3vuy3whBrpCx8koTx5mTgWm9ywD9CXhycM7Sr
+NvYJ3Ip2qfmPcFDbLt6+6mKx9i0400DLuQs7mp4D6q4mXlv1d7sOmIkCAwEAAaNk
+MGIwIAYDVR0RBBkwF4IJbG9jYWxob3N0hwR/AAABhwR/AAACMB0GA1UdDgQWBBSo
+rmVS78ucWg8iOxzl10T63yn1HDAfBgNVHSMEGDAWgBRQZA4IxGl1IjdxVIEX5iua
+Bqxp7zANBgkqhkiG9w0BAQsFAAOCAQEAMgjD6WyeuvJ7695Fz/KkG/dJlf6tf5CO
+Bv6fV1IC4hKQfux34uuidH4Isp4tlou9gscQiX6tX5aUHCJns6slGvIyiV+niMPO
+g681gaInTJ2qEfvsP6Mzcb/liPUYNgS1iG2tQG+eWkEPEAlizOufCUzK+RCJtoIR
+JZgarU4Y5mdE9Yi+0qEqjnuDGqKMr/eyywnkQ7Ys/rMybPEpz75lXPXMpx9iZTub
+c/InE0o4SftTyYBieOb6fzYt9UwwtU8XT5eGNk+I41COWGYJKYbfkgfitXR1xgSM
+4B3aZjXr0L8H0RTbPLk+Zl+jtZnQXe4e40CX1jk+T2+yz9QMrv5FhQ==
+-----END CERTIFICATE-----

.github/files/mutual_auth_server.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDfae7r2ZeC5E1T
+S2T1WlhtCkl4eumZ1WGkAdrbI9QWjh2bJlu8X8Kba2aeS/mnT0H3gINcEqtxCqgI
+Nn4mrcrFjyXIlRS52aZwhoaOFRc1rcDH4wqYHidyTNbuIeIAl5WSgEHhDK3H3cwf
+Gs6qg8hdIqeKMnRkZzXxt2qStcDFQOcotzLF65JREGa2ygjo/ddyTUv1tqSRhcl0
+uZYySDBOwT79Wlp4L0WUS0T1o/tXw838L22idNfmCVvYd77st8IQa6QsfJKE8eZk
+4FpvcsA/Ql4cnDO0qzb2CdyKdqn5j3BQ2y7evupisfYtONNAy7kLO5qeA+quJl5b
+9Xe7DpiJAgMBAAECggEAX637f5BRfeIOZ6S7WQRXKGOmcW2aS6KBzX5p3z3p37r/
+XOnBkdwk+PC3IwYApjGgW8Hy/+SS/beFnPbRcfkuBQu4L74BAVMAH+LND15i8T83
+d6N5bwpC9IWJ6rQiGQTSLsq07A/FAPqxJ4UQiyAjGBf2To4rILC+/oUBpFEofJDx
+VHdUdeIyG/n9dnFEWALU/+jIv5ehmd2k9+cWYaFItj4oQQgSQhd+4Ei8FCA0z6H8
+Au3KaAkzk6586SKXrkRSqXgsFq1RPV3yZf3wEpxjKDLQlkW1DZ+rZeiFoSncCypX
+dAP2aPpUwq4cmcnPPjCb1bYT/R4k/UxdPeM6hioLvQKBgQDxaMBH2FlB/pQRBVD+
+MjYn/0hyuucSaphZfBm/NIzO17hj7rowmM7k0vTYCEH4okpS81vyc5cbxp7zrF+6
+ndKqnsQ6xLi7gciugNzcE9r/lEQ0NY5aeSepxAmrAfun/AkyInjtdDHYIe5D+++M
+C+NX+QhQuY+VsCSI3/ty7G4cdwKBgQDs6r7Jm2A9RqL1U1r4xTeOATxtFlwWd2HC
+aUTDgt2XMTIwC5EvJvOSzejxQ/5a0cwVe1CqDsEr2z6VhoPf7Pixfa+kl9RCLaB4
+BW4XKqLRi646mjTri2OntHNSyDtnqYjPM8xD8XsYnDjhabuJeudk1c06Hs8/3SZl
+LLqUB/cy/wKBgGgGC1TqFQwq9wKel9fxKwTxK/bEMu3soNlucGnD6yj2K65kIDbw
+obZYcMjmRgctZ3l3BBflVyfQ3o2BqX5wo2aCI2xveGrT8fW/8Wuj4qpe6UTWDxFN
+Q5oTr1bQ8+/e2tePoywu9Yt8k38YAJEITl6KINsoGocV76hLW76ZwpjhAoGASZR2
+gKLS9vTzM4IZ5N1Lbb0QiUjCfO/C2uBazOXd8FEFJ0rQYnaUe2mcCSJ24rn5ykos
+OUSv1fEXtNS8LFWqv2UrodDNyWfmSKBXGrbRLodVKzotaWI3JySyFGm/mWhb5w5N
+SGS9OF18hPWTmxUivHxyarSa5VjHzfBgWg1HRv0CgYEAl8zFvOLUlduAnqAkGe6h
+Nm8qbb7CaR8u1WY7jQ1i5oqWVGiBoae6i50gKhqJKVuDcsd+EXNaooTuED8GHVle
+HDv3II0B5lV5VFgvk1+ovWIOFctaPbprAuwIJ2iXfLLvwfcGTtQpjYpi812wrVsn
+J/MMLpSOXJhRIorICkY72TU=
+-----END PRIVATE KEY-----

.github/scripts/configure-ccm.sh

@@ -25,9 +25,9 @@ function set_java_home() {
     else
         export CASSANDRA_USE_JDK11=false
     fi
-    for jdk in /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk/${major_version}*/; 
+    for jdk in /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk/${major_version}*/*/;
     do 
-        export JAVA_HOME="${jdk/}"x64/
+        export JAVA_HOME="${jdk/}"
         echo "JAVA_HOME is set to $JAVA_HOME"
         export JAVA_TOOL_OPTIONS="-Dcom.sun.jndi.rmiURLParsing=legacy"
     done
@@ -71,10 +71,10 @@ case "${TEST_TYPE}" in
         mkdir -p ~/.local
         cp ./.github/files/jmxremote.password ~/.local/jmxremote.password
         chmod 400 ~/.local/jmxremote.password
-        for jdk in /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk/8*/; 
-        do 
-          sudo chmod 777 "${jdk/}"x64/jre/lib/management/jmxremote.access
-          echo "cassandra     readwrite" >> "${jdk/}"x64/jre/lib/management/jmxremote.access
+        for jdk in /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk/8*/*/;
+        do
+          sudo chmod 777 "${jdk/}"jre/lib/management/jmxremote.access
+          echo "cassandra     readwrite" >> "${jdk/}"jre/lib/management/jmxremote.access
         done
         if [[ ! -z $ELASSANDRA_VERSION ]]; then
           ccm create test -v file:elassandra-${ELASSANDRA_VERSION}.tar.gz

.github/scripts/run-tests.sh

@@ -21,9 +21,9 @@ set -xe
 
 function set_java_home() {
     major_version=$1
-    for jdk in /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk/${major_version}*/; 
+    for jdk in /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk/${major_version}*/*/;
     do 
-        export JAVA_HOME="${jdk/}"x64/
+        export JAVA_HOME="${jdk/}"
         echo "JAVA_HOME is set to $JAVA_HOME"
         export JAVA_TOOL_OPTIONS="-Dcom.sun.jndi.rmiURLParsing=legacy"
     done
@@ -118,8 +118,9 @@ case "${TEST_TYPE}" in
             # Stop CCM now so we can restart it with Management API
             ccm stop
             # Start Management API
-            MGMT_API_LOG_DIR=/tmp/log/cassandra1 bash -c 'nohup java -jar /tmp/datastax-mgmtapi-server.jar --db-socket=/tmp/db1.sock --host=unix:///tmp/mgmtapi1.sock --host=http://127.0.0.1:8080 --db-home=`dirname ~/.ccm/test/node1`/node1 &'
-            MGMT_API_LOG_DIR=/tmp/log/cassandra2 bash -c 'nohup java -jar /tmp/datastax-mgmtapi-server.jar --db-socket=/tmp/db2.sock --host=unix:///tmp/mgmtapi2.sock --host=http://127.0.0.2:8080 --db-home=`dirname ~/.ccm/test/node2`/node2 &'
+            CERT_DIR=/home/runner/work/cassandra-reaper/cassandra-reaper/.github/files
+            MGMT_API_LOG_DIR=/tmp/log/cassandra1 MGMT_API_TLS_CA_CERT_FILE=$CERT_DIR/mutual_auth_ca.pem MGMT_API_TLS_CERT_FILE=$CERT_DIR/mutual_auth_server.crt MGMT_API_TLS_KEY_FILE=$CERT_DIR/mutual_auth_server.key bash -c 'nohup java -jar /tmp/datastax-mgmtapi-server.jar --tlscacert=$MGMT_API_TLS_CA_CERT_FILE --tlscert=$MGMT_API_TLS_CERT_FILE --tlskey=$MGMT_API_TLS_KEY_FILE --db-socket=/tmp/db1.sock --host=unix:///tmp/mgmtapi1.sock --host=http://127.0.0.1:8080 --db-home=`dirname ~/.ccm/test/node1`/node1 > /tmp/log/cassandra1/mgmt.out 2>&1 &'
+            MGMT_API_LOG_DIR=/tmp/log/cassandra2 MGMT_API_TLS_CA_CERT_FILE=$CERT_DIR/mutual_auth_ca.pem MGMT_API_TLS_CERT_FILE=$CERT_DIR/mutual_auth_server.crt MGMT_API_TLS_KEY_FILE=$CERT_DIR/mutual_auth_server.key bash -c 'nohup java -jar /tmp/datastax-mgmtapi-server.jar --tlscacert=$MGMT_API_TLS_CA_CERT_FILE --tlscert=$MGMT_API_TLS_CERT_FILE --tlskey=$MGMT_API_TLS_KEY_FILE --db-socket=/tmp/db2.sock --host=unix:///tmp/mgmtapi2.sock --host=http://127.0.0.2:8080 --db-home=`dirname ~/.ccm/test/node2`/node2 > /tmp/log/cassandra2/mgmt.out 2>&1 &'
             # wait for Cassandra to be ready
             for i in `seq 1 30` ; do
                 # keep curl from exiting with non-zero

.github/workflows/ci.yaml

@@ -314,7 +314,23 @@ jobs:
           GRIM_MIN: ${{ matrix.grim-min }}
           CUCUMBER_OPTIONS: ${{ matrix.cucumber-options }}
           JDK_VERSION: ${{ matrix.jdk-version }}
-
+
+      # Upload logs
+      - name: Prepare a unique name for logs
+        if: ${{ failure() }}
+        run: |
+          name=${{ matrix.test-type }}-${{ matrix.cassandra-version }}
+          # The option to enable + in sed regexps differs by OS so we avoid it
+          name=$(echo -n "$name" | sed -e 's/[ \t:\/\\"<>|*?]/-/g' -e 's/--*/-/g')
+          echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV
+
+      - name: Archive logs
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: reaper-logs-${{ env.ARTIFACT_NAME }}
+          path: /tmp/log
+
       # Add a tmate action to debug failures. 
       - name: Setup tmate session
         if: ${{ failure() }}

pom.xml

@@ -200,6 +200,9 @@
                 <exclude>**/*.access</exclude>
                 <exclude>**/*.password</exclude>
                 <exclude>**/*.txt</exclude>
+                <exclude>**/*.crt</exclude>
+                <exclude>**/*.key</exclude>
+                <exclude>**/*.pem</exclude>
                 <exclude>src/packaging/**</exclude>
                 <exclude>codecov.yml</exclude>
               </excludes>

src/packaging/docker-services/reaper/reaper.env

@@ -78,4 +78,6 @@ REAPER_LOGGING_ROOT_LEVEL=DEBUG
 #JAVA_OPTS="-Dwhatever=value"
 CRYPTO_SYSTEM_PROPERTY_SECRET="CRYPTO_SECRET"
 CRYPTO_SECRET="secret"
-REAPER_HTTP_MANAGEMENT_ENABLE="false"
+REAPER_HTTP_MANAGEMENT_ENABLE="false"
+REAPER_HTTP_MANAGEMENT_KEYSTORE_PATH=""
+REAPER_HTTP_MANAGEMENT_TRUSTSTORE_PATH=""

src/server/checkstyle.xml

@@ -83,10 +83,6 @@
           <property name="format" value="[\t]"/>
           <property name="message" value="Tabs in java file"/>
         </module>
-        <module name="RegexpSinglelineJava">
-          <property name="format" value="[=+-]$"/>
-          <property name="message" value="Operators must carry"/>
-        </module>
         <module name="AvoidStarImport"/>
         <module name="OneTopLevelClass"/>
         <module name="NoLineWrap"/>

src/server/src/main/docker/Dockerfile

@@ -80,7 +80,9 @@ ENV REAPER_SEGMENT_COUNT_PER_NODE=64 \
     JAVA_OPTS="" \
     REAPER_MAX_PARALLEL_REPAIRS=2 \
     REAPER_MGMT_API_METRICS_PORT=9000 \
-    REAPER_HTTP_MANAGEMENT_ENABLE="false"
+    REAPER_HTTP_MANAGEMENT_ENABLE="false" \
+    REAPER_HTTP_MANAGEMENT_KEYSTORE_PATH="" \
+    REAPER_HTTP_MANAGEMENT_TRUSTSTORE_PATH=""
 
 # used to run spreaper cli
 RUN apk add --update \

src/server/src/main/docker/cassandra-reaper.yml

@@ -29,7 +29,6 @@ repairManagerSchedulingIntervalSeconds: ${REAPER_REPAIR_MANAGER_SCHEDULING_INTER
 jmxConnectionTimeoutInSeconds: ${REAPER_JMX_CONNECTION_TIMEOUT_IN_SECONDS}
 useAddressTranslator: ${REAPER_USE_ADDRESS_TRANSLATOR}
 maxParallelRepairs: ${REAPER_MAX_PARALLEL_REPAIRS}
-mgmtApiMetricsPort: ${REAPER_MGMT_API_METRICS_PORT}
 
 # datacenterAvailability has three possible values: ALL | LOCAL | EACH
 # the correct value to use depends on whether jmx ports to C* nodes in remote datacenters are accessible
@@ -83,4 +82,7 @@ server:
 
 httpManagement:
   enabled: ${REAPER_HTTP_MANAGEMENT_ENABLE}
+  mgmtApiMetricsPort: ${REAPER_MGMT_API_METRICS_PORT}
+  keystore: ${REAPER_HTTP_MANAGEMENT_KEYSTORE_PATH}
+  truststore: ${REAPER_HTTP_MANAGEMENT_TRUSTSTORE_PATH}
 

src/server/src/main/java/io/cassandrareaper/ReaperApplicationConfiguration.java

@@ -44,6 +44,7 @@
 public final class ReaperApplicationConfiguration extends Configuration {
 
   public static final int DEFAULT_MGMT_API_METRICS_PORT = 9000;
+  private static final int DEFAULT_MGMT_API_PORT = 8080;
   private static final int DEFAULT_SEGMENT_COUNT_PER_NODE = 64;
   private static final Integer DEFAULT_MAX_PENDING_COMPACTIONS = 20;
 
@@ -167,9 +168,6 @@ public final class ReaperApplicationConfiguration extends Configuration {
   @Nullable
   private CryptographFactory cryptograph;
 
-  @JsonProperty
-  private Integer mgmtApiMetricsPort;
-
   public HttpManagement getHttpManagement() {
     return httpManagement;
   }
@@ -509,15 +507,6 @@ public void setCryptograph(@Nullable CryptographFactory cryptograph) {
     this.cryptograph = cryptograph;
   }
 
-  public int getMgmtApiMetricsPort() {
-    return mgmtApiMetricsPort == null ? DEFAULT_MGMT_API_METRICS_PORT : mgmtApiMetricsPort;
-  }
-
-  @JsonProperty("mgmtApiMetricsPort")
-  public void setMgmtApiMetricsPort(int mgmtApiMetricsPort) {
-    this.mgmtApiMetricsPort = mgmtApiMetricsPort;
-  }
-
   public enum DatacenterAvailability {
     /* We require direct JMX access to all nodes across all datacenters */
     ALL,
@@ -721,9 +710,46 @@ public static final class HttpManagement {
     @JsonProperty
     private Boolean enabled = false;
 
+    @JsonProperty
+    private String keystore;
+
+    @JsonProperty
+    private String truststore;
+
+    @JsonProperty
+    private Integer mgmtApiMetricsPort;
+
+    @JsonProperty
+    private Integer managementApiPort;
+
     public Boolean isEnabled() {
       return enabled;
     }
     // TODO: Add ports and root paths here.
+
+    public String getKeystore() {
+      return keystore;
+    }
+
+    public String getTruststore() {
+      return truststore;
+    }
+
+    public int getMgmtApiMetricsPort() {
+      return mgmtApiMetricsPort == null ? DEFAULT_MGMT_API_METRICS_PORT : mgmtApiMetricsPort;
+    }
+
+    public void setManagementApiPort(Integer managementApiPort) {
+      this.managementApiPort = managementApiPort;
+    }
+
+    public Integer getManagementApiPort() {
+      return managementApiPort == null ? DEFAULT_MGMT_API_PORT : managementApiPort;
+    }
+
+    @JsonProperty("mgmtApiMetricsPort")
+    public void setMgmtApiMetricsPort(int mgmtApiMetricsPort) {
+      this.mgmtApiMetricsPort = mgmtApiMetricsPort;
+    }
   }
 }