diff --git a/src/clj/puppetlabs/puppetserver/certificate_authority.clj b/src/clj/puppetlabs/puppetserver/certificate_authority.clj index e78170eac..d25850ccf 100644 --- a/src/clj/puppetlabs/puppetserver/certificate_authority.clj +++ b/src/clj/puppetlabs/puppetserver/certificate_authority.clj @@ -1,34 +1,34 @@ (ns puppetlabs.puppetserver.certificate-authority - (:import (java.io BufferedReader BufferedWriter FileNotFoundException InputStream ByteArrayOutputStream ByteArrayInputStream File Reader StringReader IOException) + (:require [clj-time.coerce :as time-coerce] + [clj-time.core :as time] + [clj-time.format :as time-format] + [clojure.java.io :as io] + [clojure.set :as set] + [clojure.string :as str] + [clojure.tools.logging :as log] + [me.raynes.fs :as fs] + [puppetlabs.i18n.core :as i18n] + [puppetlabs.kitchensink.core :as ks] + [puppetlabs.kitchensink.file :as ks-file] + [puppetlabs.puppetserver.common :as common] + [puppetlabs.puppetserver.ringutils :as ringutils] + [puppetlabs.puppetserver.shell-utils :as shell-utils] + [puppetlabs.ssl-utils.core :as utils] + [schema.core :as schema] + [slingshot.slingshot :as sling]) + (:import (java.io BufferedReader BufferedWriter ByteArrayInputStream ByteArrayOutputStream File FileNotFoundException IOException InputStream Reader StringReader) (java.nio CharBuffer) (java.nio.file Files) (java.nio.file.attribute FileAttribute PosixFilePermissions) (java.security PrivateKey PublicKey) - (java.security.cert X509Certificate CRLException CertPathValidatorException X509CRL) + (java.security.cert CRLException CertPathValidatorException X509CRL X509Certificate) (java.text SimpleDateFormat) (java.time LocalDateTime ZoneId) (java.util Date) (java.util.concurrent.locks ReentrantReadWriteLock) (org.apache.commons.io IOUtils) (org.bouncycastle.pkcs PKCS10CertificationRequest) - (org.joda.time DateTime)) - (:require [me.raynes.fs :as fs] - [schema.core :as schema] - [clojure.string :as str] - [clojure.set :as set] - [clojure.java.io :as io] - [clojure.tools.logging :as log] - [clj-time.core :as time] - [clj-time.format :as time-format] - [clj-time.coerce :as time-coerce] - [slingshot.slingshot :as sling] - [puppetlabs.kitchensink.core :as ks] - [puppetlabs.kitchensink.file :as ks-file] - [puppetlabs.puppetserver.common :as common] - [puppetlabs.puppetserver.ringutils :as ringutils] - [puppetlabs.ssl-utils.core :as utils] - [puppetlabs.puppetserver.shell-utils :as shell-utils] - [puppetlabs.i18n.core :as i18n])) + (org.joda.time DateTime))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Public utilities @@ -718,6 +718,29 @@ (log/trace (i18n/trs "Finish append to serial file. ")))] (ks-file/atomic-write infra-node-serials-path stream-content-fn))))))) +(defn stream-content-to-file + [^String cert-inventory ^String entry ^BufferedWriter writer] + (log/trace (i18n/trs "Begin append to inventory file.")) + (let [copy-buffer (CharBuffer/allocate buffer-copy-size)] + (try + (with-open [^BufferedReader reader (io/reader cert-inventory)] + ;; copy all the existing content + (loop [read-length (.read reader copy-buffer)] + ;; theoretically read can return 0, which means try again + (when (<= 0 read-length) + (when (pos? read-length) + (.write writer (.array copy-buffer) 0 read-length)) + (.clear copy-buffer) + (recur (.read reader copy-buffer))))) + (catch FileNotFoundException _e + (log/trace (i18n/trs "Inventory file not found. Assume empty."))) + (catch Throwable e + (log/error e (i18n/trs "Error while appending to inventory file.")) + (throw e)))) + (.write writer entry) + (.flush writer) + (log/trace (i18n/trs "Finish append to inventory file. "))) + (schema/defn ^:always-validate write-cert-to-inventory! "Writes an entry into Puppet's inventory file for a given certificate. @@ -746,33 +769,45 @@ (format-date-time)) subject (utils/get-subject-from-x509-certificate cert) cert-name (utils/x500-name->CN subject) - entry (str formatted-serial-number " " not-before " " not-after " /" subject "\n") - stream-content-fn (fn [^BufferedWriter writer] - (log/trace (i18n/trs "Begin append to inventory file.")) - (let [copy-buffer (CharBuffer/allocate buffer-copy-size)] - (try - (with-open [^BufferedReader reader (io/reader cert-inventory)] - ;; copy all the existing content - (loop [read-length (.read reader copy-buffer)] - ;; theoretically read can return 0, which means try again - (when (<= 0 read-length) - (when (pos? read-length) - (.write writer (.array copy-buffer) 0 read-length)) - (.clear copy-buffer) - (recur (.read reader copy-buffer))))) - (catch FileNotFoundException _e - (log/trace (i18n/trs "Inventory file not found. Assume empty."))) - (catch Throwable e - (log/error e (i18n/trs "Error while appending to inventory file.")) - (throw e)))) - (.write writer entry) - (.flush writer) - (log/trace (i18n/trs "Finish append to inventory file. ")))] + entry (str formatted-serial-number " " not-before " " not-after " /" subject "\n")] (common/with-safe-write-lock inventory-lock inventory-lock-descriptor inventory-lock-timeout-seconds (log/debug (i18n/trs "Append \"{1}\" to inventory file {0}" cert-inventory entry)) - (ks-file/atomic-write cert-inventory stream-content-fn) + (ks-file/atomic-write cert-inventory (partial stream-content-to-file cert-inventory entry)) (maybe-write-to-infra-serial! serial-number cert-name settings)))) +(schema/defn ^:always-validate + write-cert-to-inventory-unlocked! + "Writes an entry into Puppet's inventory file for a given certificate. + The location of this file is defined by Puppet's 'cert_inventory' setting. + The inventory is a text file where each line represents a certificate in the + following format: + $SN $NB $NA /$S + where: + * $SN = The serial number of the cert. The serial number is formatted as a + hexadecimal number, with a leading 0x, and zero-padded up to four + digits, eg. 0x002f. + * $NB = The 'not before' field of the cert, as a date/timestamp in UTC. + * $NA = The 'not after' field of the cert, as a date/timestamp in UTC. + * $S = The distinguished name of the cert's subject." + [cert :- Certificate + {:keys [cert-inventory] :as settings} :- CaSettings] + (let [serial-number (.getSerialNumber cert) + formatted-serial-number (->> serial-number + (format-serial-number) + (str "0x")) + not-before (-> cert + (.getNotBefore) + (format-date-time)) + not-after (-> cert + (.getNotAfter) + (format-date-time)) + subject (utils/get-subject-from-x509-certificate cert) + cert-name (utils/x500-name->CN subject) + entry (str formatted-serial-number " " not-before " " not-after " /" subject "\n")] + (log/debug (i18n/trs "Append \"{1}\" to inventory file {0}" cert-inventory entry)) + (ks-file/atomic-write cert-inventory (partial stream-content-to-file cert-inventory entry)) + (maybe-write-to-infra-serial! serial-number cert-name settings))) + (schema/defn is-subject-in-inventory-row? :- schema/Bool [cn-subject :- utils/ValidX500Name [_serial _not-before _not-after row-subject] :- [schema/Str]] @@ -797,7 +832,6 @@ ;; lack of an end date means we can't tell if it is expired or not, so assume it isn't. false)) - (defn extract-inventory-row-contents [row] (str/split row #" ")) @@ -1776,7 +1810,6 @@ (validate-csr-signature! csr) (autosign-certificate-request! subject csr settings report-activity))))) - (schema/defn ^:always-validate get-certificate-revocation-list :- schema/Str "Given the value of the 'cacrl' setting from Puppet, @@ -2400,4 +2433,83 @@ (when (and enable-infra-crl (fs/exists? infra-crl-path)) (when (crl-expires-in-n-days? infra-crl-path settings crl-expiration-window-days) (log/info (i18n/trs "infra crl expiring within 30 days, updating.")) - (update-and-sign-crl! infra-crl-path settings)))) \ No newline at end of file + (update-and-sign-crl! infra-crl-path settings)))) + +(schema/defn maybe-sign-one :- (schema/enum :signed :not-signed) + [subject :- schema/Str + csr-path :- schema/Str + cacert :- Certificate + casubject :- schema/Str + ca-private-key :- PrivateKey + {:keys [signeddir ca-ttl allow-auto-renewal allow-subject-alt-names + allow-authorization-extensions auto-renewal-cert-ttl] :as ca-settings} :- CaSettings] + (try + (let [csr (utils/pem->csr csr-path) + renewal-ttl (if (and allow-auto-renewal (supports-auto-renewal? csr)) + auto-renewal-cert-ttl + ca-ttl) + _ (log/debug (i18n/trs "Calculating validity dates from ttl of {0} " renewal-ttl)) + validity (cert-validity-dates renewal-ttl)] + ;; these ensure/validate functions throw exceptions if the criteria isn't met + (ensure-subject-alt-names-allowed! csr allow-subject-alt-names) + (ensure-no-authorization-extensions! csr allow-authorization-extensions) + (validate-extensions! (utils/get-extensions csr)) + (validate-csr-signature! csr) + (let [signed-cert (utils/sign-certificate casubject + ca-private-key + (next-serial-number! ca-settings) + (:not-before validity) + (:not-after validity) + (utils/cn subject) + (utils/get-public-key csr) + (create-agent-extensions csr cacert))] + (write-cert-to-inventory-unlocked! signed-cert ca-settings) + (write-cert signed-cert (path-to-cert signeddir subject)) + (delete-certificate-request! ca-settings subject) + ;; success case, add the host to the set of signed results + :signed)) + (catch Throwable e + (log/debug e (i18n/trs "Failed in bulk signing for entry {0}" subject)) + ;; failure case, add the host to the set of not signed results + :not-signed))) + +(schema/defn ^:always-validate + sign-multiple-certificate-signing-requests! :- {:signed [schema/Str] + :not-signed [schema/Str]} + [subjects :- [schema/Str] + {:keys [cacert cakey csrdir + inventory-lock inventory-lock-timeout-seconds + serial-lock serial-lock-timeout-seconds] :as ca-settings} :- CaSettings + report-activity] + (let [;; if part of a CA bundle, the intermediate CA will be first in the chain + cacert (utils/pem->ca-cert cacert cakey) + casubject (utils/get-subject-from-x509-certificate cacert) + ca-private-key (utils/pem->private-key cakey)] + (when-not (empty? subjects) + ;; since we are going to be manipulating the serial file and the inventory file for multiple entries, + ;; acquire the locks to prevent lock thrashing + (common/with-safe-write-lock serial-lock serial-lock-descriptor serial-lock-timeout-seconds + (common/with-safe-write-lock inventory-lock inventory-lock-descriptor inventory-lock-timeout-seconds + (let [results + ;; loop through the subjects, one at a time, and collect the results for success or failure. + (loop [s subjects + result {:signed [] + :not-signed []}] + (if-not (empty? s) + (let [subject (first s) + csr-path (path-to-cert-request csrdir subject)] + (if (fs/exists? csr-path) + (let [_ (log/trace (i18n/trs "File exists at {0}" csr-path)) + one-result (maybe-sign-one subject csr-path cacert casubject ca-private-key ca-settings)] + ;; one-result is either :signed or :not-signed + (recur (rest s) + (update result one-result conj subject))) + (do + (log/trace (i18n/trs "File does not exist at {0}" csr-path)) + (recur (rest s) + (update result :not-signed conj subject))))) + result))] + ;; submit the signing activity as one entry for all the hosts. + (when-not (empty? (:signed results)) + (report-activity (:signed results) "signed")) + results)))))) \ No newline at end of file diff --git a/test/unit/puppetlabs/puppetserver/certificate_authority_test.clj b/test/unit/puppetlabs/puppetserver/certificate_authority_test.clj index a041678ee..4579bdd00 100644 --- a/test/unit/puppetlabs/puppetserver/certificate_authority_test.clj +++ b/test/unit/puppetlabs/puppetserver/certificate_authority_test.clj @@ -1,34 +1,34 @@ (ns puppetlabs.puppetserver.certificate-authority-test - (:import (java.io StringReader - StringWriter - ByteArrayInputStream - ByteArrayOutputStream) - (com.puppetlabs.ssl_utils SSLUtils) - (java.security PublicKey MessageDigest) - (java.security.cert X509CRL) - (java.time LocalDateTime ZoneOffset) - (java.util Date) - (java.util.concurrent TimeUnit) - (java.util.concurrent.locks ReentrantReadWriteLock) - (org.joda.time DateTime Period) - (org.bouncycastle.asn1.x509 SubjectPublicKeyInfo)) - (:require [clojure.set :as set] + (:require [clj-time.coerce :as time-coerce] + [clj-time.core :as time] + [clojure.java.io :as io] + [clojure.set :as set] + [clojure.string :as string] + [clojure.test :refer [deftest is testing use-fixtures]] + [me.raynes.fs :as fs] + [puppetlabs.kitchensink.core :as ks] + [puppetlabs.kitchensink.file :as ks-file] [puppetlabs.puppetserver.certificate-authority :as ca] - [puppetlabs.trapperkeeper.testutils.logging :refer [logged?] :as logutils] - [puppetlabs.ssl-utils.core :as utils] - [puppetlabs.ssl-utils.simple :as simple] [puppetlabs.services.ca.ca-testutils :as testutils] [puppetlabs.services.jruby.jruby-puppet-testutils :as jruby-testutils] - [puppetlabs.kitchensink.core :as ks] - [puppetlabs.kitchensink.file :as ks-file] - [slingshot.test :refer :all] + [puppetlabs.ssl-utils.core :as utils] + [puppetlabs.ssl-utils.simple :as simple] + [puppetlabs.trapperkeeper.testutils.logging :refer [logged?] :as logutils] [schema.test :as schema-test] - [clojure.test :refer [deftest is testing use-fixtures]] - [clojure.java.io :as io] - [clojure.string :as string] - [clj-time.core :as time] - [clj-time.coerce :as time-coerce] - [me.raynes.fs :as fs])) + [slingshot.test :refer :all]) + (:import (com.puppetlabs.ssl_utils SSLUtils) + (java.io ByteArrayInputStream + ByteArrayOutputStream + StringReader + StringWriter) + (java.security MessageDigest PublicKey) + (java.security.cert X509CRL X509Certificate) + (java.time LocalDateTime ZoneOffset) + (java.util Date) + (java.util.concurrent TimeUnit) + (java.util.concurrent.locks ReentrantReadWriteLock) + (org.bouncycastle.asn1.x509 SubjectPublicKeyInfo) + (org.joda.time DateTime Period))) (use-fixtures :once schema-test/validate-schemas) @@ -2109,18 +2109,18 @@ subject (utils/cn "foo")] (testing "should not support auto-renewal" (is (false? (ca/supports-auto-renewal? (utils/generate-certificate-request keypair subject [] [])))) - (is (false? (ca/supports-auto-renewal? (utils/generate-certificate-request keypair subject [] [{:oid "1.3.6.1.4.1.34380.1.3.2" :value false}])))) - (is (false? (ca/supports-auto-renewal? (utils/generate-certificate-request keypair subject [] [{:oid "1.3.6.1.4.1.34380.1.3.2" :value "false"}]))))) + (is (false? (ca/supports-auto-renewal? (utils/generate-certificate-request keypair subject [] [{:oid ca/pp_auth_auto_renew-attribute :value false}])))) + (is (false? (ca/supports-auto-renewal? (utils/generate-certificate-request keypair subject [] [{:oid ca/pp_auth_auto_renew-attribute :value "false"}]))))) (testing "should support auto-renewal" - (is (true? (ca/supports-auto-renewal? (utils/generate-certificate-request keypair subject [] [{:oid "1.3.6.1.4.1.34380.1.3.2" :value true}])))) - (is (true? (ca/supports-auto-renewal? (utils/generate-certificate-request keypair subject [] [{:oid "1.3.6.1.4.1.34380.1.3.2" :value "true"}]))))))) + (is (true? (ca/supports-auto-renewal? (utils/generate-certificate-request keypair subject [] [{:oid ca/pp_auth_auto_renew-attribute :value true}])))) + (is (true? (ca/supports-auto-renewal? (utils/generate-certificate-request keypair subject [] [{:oid ca/pp_auth_auto_renew-attribute :value "true"}]))))))) (deftest get-csr-attributes-test (testing "extract attribute from CSR" (let [keypair (utils/generate-key-pair) subject (utils/cn "foo") - csr (utils/generate-certificate-request keypair subject [] [{:oid "1.3.6.1.4.1.34380.1.3.2" :value true}])] - (is (= [{:oid "1.3.6.1.4.1.34380.1.3.2", :values ["true"]}] (ca/get-csr-attributes csr)))))) + csr (utils/generate-certificate-request keypair subject [] [{:oid ca/pp_auth_auto_renew-attribute :value true}])] + (is (= [{:oid ca/pp_auth_auto_renew-attribute, :values ["true"]}] (ca/get-csr-attributes csr)))))) (deftest crl-expires-in-n-days?-test (let [settings (testutils/ca-sandbox! cadir)] @@ -2251,3 +2251,131 @@ (doseq [i (range 1 10000)] (is (= (biginteger i) (ca/base-16-str->biginteger (format "0x%x" i))))))) +(defn generate-csr + [settings + extensions + attributes] + (let [keypair (utils/generate-key-pair) + subject-name (ks/rand-str :alpha-digits 8) + subject (utils/cn subject-name) + csr (utils/generate-certificate-request keypair subject extensions attributes) + csr-path (ca/path-to-cert-request (:csrdir settings) subject-name)] + (utils/obj->pem! csr csr-path) + {:subject subject + :subject-name subject-name + :csr csr + :csr-path csr-path})) + +(deftest sign-multiple-certificate-signing-requests!-test + (let [inventory-file (str (ks/temp-file)) + settings (-> (testutils/ca-sandbox! bundle-cadir) + (assoc :cert-inventory inventory-file + :allow-auto-renewal true) + (update :auto-renewal-cert-ttl ca/duration-str->sec)) + report-activity (fn [_a _b] nil)] + (testing "single entry" + (testing "happy path" + (let [csr-info (generate-csr settings [] [{:oid ca/pp_auth_auto_renew-attribute :value true}])] + (testing "csr should exist before the operation" + (is (fs/exists? (:csr-path csr-info)))) + (testing "correctly signs" + (is (= {:signed [(:subject-name csr-info)] + :not-signed []} + (ca/sign-multiple-certificate-signing-requests! [(:subject-name csr-info)] settings report-activity)))) + (testing "csr is removed after routine" + (is (not (fs/exists? (:csr-path csr-info))))) + (testing "signed cert" + (let [cert-path (ca/path-to-cert (:signeddir settings) (:subject-name csr-info))] + (testing "should exist" + (is (fs/exists? cert-path))) + (testing "signed cert should be valid" + (let [^X509Certificate cert (utils/pem->cert cert-path) + capub (-> (:cacert settings) + (utils/pem->certs) + (first) + (.getPublicKey))] + ;; this will throw an exception, causing the test to fail if it isn't valid. It returns void + (.checkValidity cert) + (is (nil? (.verify cert capub))) + (testing "not after date should be ~90 days in the future" + (let [diff (- (.getTime (.getNotAfter cert)) (.getTime (Date.))) + days (.convert TimeUnit/DAYS diff TimeUnit/MILLISECONDS)] + ;; 89 days plus some number of hour, minutes seconds + (is (= 89 days)))))) + ;; certificate should be in inventory + (testing "should be in the inventory" + (let [inventory (slurp inventory-file) + entries (string/split inventory #"\n")] + (is (= (count entries) 1)) + (let [row (string/split (first entries) #" ")] + ;; row is constructed of serial-number, start date, end date and name + ;; sandbox has some used serial numbers + (is (= "0x0003" (first row))) + (is (= (str "/" (:subject csr-info)) (nth row 3)))))))))) + (testing "correctly rejects a non-existent csr" + (let [random-csr-name (ks/rand-str :alpha-digits 8)] + (is (= {:signed [] + :not-signed [random-csr-name]} + (ca/sign-multiple-certificate-signing-requests! [random-csr-name] settings report-activity))))) + (testing "correctly rejects a cert with Subject alternative names" + (let [alt-name-ext {:oid utils/subject-alt-name-oid + :value {:dns-name ["bad-name"]} + :critical false} + csr-info (generate-csr settings [alt-name-ext] [{:oid ca/pp_auth_auto_renew-attribute :value true}])] + (is (= {:signed [] + :not-signed [(:subject-name csr-info)]} + (ca/sign-multiple-certificate-signing-requests! [(:subject-name csr-info)] settings report-activity))))) + (testing "correctly rejects a cert with authorization extensions when disabled" + (let [csr-info (generate-csr settings [{:oid ca/ppAuthCertExt :value "true" :critical false}] [{:oid ca/pp_auth_auto_renew-attribute :value true}])] + (is (= {:signed [] + :not-signed [(:subject-name csr-info)]} + (ca/sign-multiple-certificate-signing-requests! [(:subject-name csr-info)] settings report-activity))))) + (testing "correctly rejects a cert with unapproved extensions" + (let [csr-info (generate-csr settings [{:oid "1.9.9.9.9.9.0" :value "true" :critical false}] [{:oid ca/pp_auth_auto_renew-attribute :value true}])] + (is (= {:signed [] + :not-signed [(:subject-name csr-info)]} + (ca/sign-multiple-certificate-signing-requests! [(:subject-name csr-info)] settings report-activity)))))) + (testing "multiple entry with both bad and good csrs" + (let [count-range (range 0 100) + _ (println "begin generate good csrs") + good-csrs (doall (pmap (fn [_i] (generate-csr settings [] [{:oid ca/pp_auth_auto_renew-attribute :value true}])) + count-range)) + _ (println "begin generate random csr names") + random-csr-names (doall (pmap (fn [_i] (ks/rand-str :alpha-digits 8)) count-range)) + _ (println "begin generate alt-name csrs") + alt-name-ext {:oid utils/subject-alt-name-oid + :value {:dns-name ["bad-name"]} + :critical false} + bad-names (doall (pmap (fn [_i] (generate-csr settings [alt-name-ext] [{:oid ca/pp_auth_auto_renew-attribute :value true}])) + count-range)) + _ (println "begin generate unauthorized csrs") + unauthorized (doall (pmap (fn [_i] (generate-csr settings [{:oid ca/ppAuthCertExt :value "true" :critical false}] [{:oid ca/pp_auth_auto_renew-attribute :value true}])) + count-range)) + _ (println "begin generate unapproved csrs") + unapproved-extensions (doall (pmap (fn [_i] (generate-csr settings [{:oid "1.9.9.9.9.9.0" :value "true" :critical false}] [{:oid ca/pp_auth_auto_renew-attribute :value true}])) + count-range)) + _ (println "combining all of them") + all-csrs (concat good-csrs bad-names unauthorized unapproved-extensions) + _ (println "add in bad names and shuffle") + all-names (shuffle (concat (map :subject-name all-csrs) random-csr-names)) + result (ca/sign-multiple-certificate-signing-requests! all-names settings report-activity) + signed-set (set (:signed result)) + unsigned-set (set (:not-signed result)) + good-csrs-set (set (map :subject-name good-csrs)) + random-names-set (set random-csr-names) + bad-names-set (set (map :subject-name bad-names)) + unauthorized-set (set (map :subject-name unauthorized)) + unapproved-extensions-set (set (map :subject-name unapproved-extensions))] + (testing "all the signed entries should be present" + (is (= good-csrs-set + signed-set)) + (testing "none of the valid csrs should be in the not-signed set" + (is (empty? (clojure.set/intersection unsigned-set good-csrs-set)))) + (testing "all of the random names should be in the not-signed" + (is (= random-names-set (clojure.set/intersection unsigned-set random-names-set)))) + (testing "all of the alt names should be in the not-signed" + (is (= bad-names-set (clojure.set/intersection unsigned-set bad-names-set)))) + (testing "all of the unauthorized names should be in the not-signed" + (is (= unauthorized-set (clojure.set/intersection unsigned-set unauthorized-set)))) + (testing "all of the unapproved names should be in the not-signed" + (is (= unapproved-extensions-set (clojure.set/intersection unsigned-set unapproved-extensions-set))))))))) \ No newline at end of file