From 3de38dc18aee2baee8d33a386221aa47606b7696 Mon Sep 17 00:00:00 2001 From: Vladimir Sitnikov Date: Sun, 8 Jan 2023 21:13:18 +0300 Subject: [PATCH] checksum-dependency: use full pgp fingerprints for verification --- README.md | 3 + plugins/checksum-dependency-plugin/README.md | 3 + .../gradle/checksum/ChecksumDependency.kt | 56 +++--- .../checksum/ChecksumDependencyPlugin.kt | 7 +- .../vlsi/gradle/checksum/FileExtensions.kt | 1 + .../gradle/checksum/SignatureExtensions.kt | 53 +++++- .../checksum/model/DependencyVerification.kt | 34 ++-- .../model/DependencyVerificationStore.kt | 44 +++-- .../vlsi/gradle/checksum/pgp/KeyDownloader.kt | 14 +- .../vlsi/gradle/checksum/pgp/KeyStore.kt | 163 ++++++++++++++---- .../vlsi/gradle/checksum/pgp/PgpKeyId.kt | 71 ++++++++ .../checksum/signatures/KeyDownloaderTest.kt | 5 +- 12 files changed, 358 insertions(+), 96 deletions(-) create mode 100644 plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/pgp/PgpKeyId.kt diff --git a/README.md b/README.md index 8c38bd9..bf4efea 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,9 @@ This library is distributed under terms of Apache License 2.0 Change log ---------- +v1.86 +* checksum-dependency: use full fingerprint for PGP verification + v1.85 * licence-gather: better support for build cache by adding PathSensitivity * checksum-dependency: cache PGP public keys under `%{ROOT_DIR}/gradle/checksum-dependency-plugin/cached-pgp-keys` diff --git a/plugins/checksum-dependency-plugin/README.md b/plugins/checksum-dependency-plugin/README.md index d3ffbf4..54168dc 100644 --- a/plugins/checksum-dependency-plugin/README.md +++ b/plugins/checksum-dependency-plugin/README.md @@ -499,6 +499,9 @@ Verification options Changelog --------- +v1.86 +* Use full fingerprint for PGP verification + v1.85 * Cache public PGP keys under `%{ROOT_DIR}/gradle/checksum-dependency-plugin/cached-pgp-keys` directory * Bump org.bouncycastle:bcpg-jdk15on to 1.70 diff --git a/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/ChecksumDependency.kt b/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/ChecksumDependency.kt index 4e69724..20fc521 100644 --- a/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/ChecksumDependency.kt +++ b/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/ChecksumDependency.kt @@ -219,8 +219,9 @@ class ChecksumDependency( val signatures = art.file.toSignatureList() keysToVerify[art] = signatures for (sign in signatures) { - if (verificationDb.isIgnored(sign.keyID)) { - logger.debug("Public key ${sign.keyID.hexKey} is ignored via , so ${art.id.artifactDependency} is assumed to be not signed with that key") + val signKey = sign.pgpShortKeyId + if (verificationDb.isIgnored(signKey)) { + logger.debug("Public key $signKey is ignored via , so ${art.id.artifactDependency} is assumed to be not signed with that key") continue } } @@ -235,31 +236,35 @@ class ChecksumDependency( logger.debug { "Resolved signature $signatureDependency" } receivedSignatures.add(signatureDependency) for (sign in art.file.toSignatureList()) { - if (verificationDb.isIgnored(sign.keyID)) { - logger.debug("Public key ${sign.keyID.hexKey} is ignored via , so ${art.id.artifactDependency} is assumed to be not signed with that key") + val signKey = sign.pgpShortKeyId + if (verificationDb.isIgnored(signKey)) { + logger.debug("Public key $signKey is ignored via , so ${art.id.artifactDependency} is assumed to be not signed with that key") continue } val verifySignature = keyStore - .getKeyAsync(sign.keyID, signatureDependency, executors) - .thenAcceptAsync({ publicKey -> - if (publicKey == null) { - logger.warn("Public key ${sign.keyID.hexKey} is not found. The key was used to sign ${art.id.artifactDependency}." + + .getKeyAsync(signKey, signatureDependency, executors) + .thenAcceptAsync({ publicKeys -> + if (publicKeys.isEmpty()) { + logger.warn("Public key $signKey is not found. The key was used to sign ${art.id.artifactDependency}." + " Please ask dependency author to publish the PGP key otherwise signature verification is not possibles") - verificationDb.ignoreKey(sign.keyID) + verificationDb.ignoreKey(signKey) return@thenAcceptAsync } - logger.debug { "Verifying signature ${sign.keyID.hexKey} for ${art.id.artifactDependency}" } - val file = originalFiles[dependencyChecksum.id]!! - val validSignature = signatureVerificationTimer(file.length()) { - verifySignature(file, sign, publicKey) - } - if (validSignature) { - synchronized(dependencyChecksum) { - dependencyChecksum.pgpKeys += sign.keyID + for (publicKey in publicKeys) { + val fullKeyId = publicKey.pgpFullKeyId + logger.debug { "Verifying signature $fullKeyId for ${art.id.artifactDependency}" } + val file = originalFiles[dependencyChecksum.id]!! + val validSignature = signatureVerificationTimer(file.length()) { + verifySignature(file, sign, publicKey) + } + if (validSignature) { + synchronized(dependencyChecksum) { + dependencyChecksum.pgpKeys += fullKeyId + } + } + logger.log(if (validSignature) LogLevel.DEBUG else LogLevel.LIFECYCLE) { + "${if (validSignature) "OK" else "KO"}: verification of ${art.id.artifactDependency} via $fullKeyId" } - } - logger.log(if (validSignature) LogLevel.DEBUG else LogLevel.LIFECYCLE) { - "${if (validSignature) "OK" else "KO"}: verification of ${art.id.artifactDependency} via ${publicKey.keyID.hexKey}" } }, executors.cpu) verifyPgpTasks.add(verifySignature) @@ -326,7 +331,14 @@ class ChecksumDependency( } private fun verifySignature(file: File, sign: PGPSignature, publicKey: PGPPublicKey): Boolean { - sign.init(BcPGPContentVerifierBuilderProvider(), publicKey) + try { + sign.init(BcPGPContentVerifierBuilderProvider(), publicKey) + } catch (e: Throwable) { + e.addSuppressed( + Throwable("Verifying $file with key ${publicKey.pgpFullKeyId}, sign ${sign.pgpShortKeyId}") + ) + throw e + } file.forEachBlock { block, size -> sign.update(block, 0, size) } return sign.verify() } @@ -343,7 +355,7 @@ class ChecksumDependency( append(" ").append(violation).appendPlatformLine(":") artifacts .asSequence() - .map { "${it.id.dependencyNotation} (pgp=${it.pgpKeys.hexKeys}, sha512=${it.sha512.ifEmpty { "[computation skipped]" }})" } + .map { "${it.id.dependencyNotation} (pgp=${it.pgpKeys}, sha512=${it.sha512.ifEmpty { "[computation skipped]" }})" } .sorted() .forEach { append(" ").appendPlatformLine(it) diff --git a/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/ChecksumDependencyPlugin.kt b/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/ChecksumDependencyPlugin.kt index c502573..2c3d1a2 100644 --- a/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/ChecksumDependencyPlugin.kt +++ b/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/ChecksumDependencyPlugin.kt @@ -73,6 +73,11 @@ open class ChecksumDependencyPlugin : Plugin { settings.property("checksum.xml", "checksum.xml") val checksums = File(settings.rootDir, checksumFileName) val buildDir = settings.property("checksumBuildDir", "build/checksum") + val cachedKeysTempRoot = + File( + settings.rootDir, + settings.property("checksumCachedPgpKeysTempDir", "build/checksum/key-cache-temp") + ) val buildFolder = File(settings.rootDir, buildDir) val cachedKeysRoot = settings.property("checksumCachedPgpKeysDir", "%{ROOT_DIR}/gradle/checksum-dependency-plugin/cached-pgp-keys") @@ -137,7 +142,7 @@ open class ChecksumDependencyPlugin : Plugin { readTimeout = Duration.ofSeconds(pgpReadTimeout) ) ) - val keyStore = KeyStore(cachedKeysRoot, keyDownloader) + val keyStore = KeyStore(cachedKeysRoot, cachedKeysTempRoot, keyDownloader) val verification = if (checksums.exists()) { DependencyVerificationStore.load(checksums) diff --git a/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/FileExtensions.kt b/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/FileExtensions.kt index d175cdd..e925726 100644 --- a/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/FileExtensions.kt +++ b/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/FileExtensions.kt @@ -43,4 +43,5 @@ internal fun File.sha512(): String { md.update(buffer, 0, bytesRead) } return BigInteger(1, md.digest()).toString(16).toUpperCase() + .padStart(128, '0') } diff --git a/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/SignatureExtensions.kt b/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/SignatureExtensions.kt index 6db2e4e..ff62691 100644 --- a/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/SignatureExtensions.kt +++ b/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/SignatureExtensions.kt @@ -16,11 +16,16 @@ */ package com.github.vlsi.gradle.checksum +import com.github.vlsi.gradle.checksum.pgp.PgpKeyId +import org.bouncycastle.bcpg.ArmoredOutputStream import java.io.File import java.io.InputStream import org.bouncycastle.openpgp.* import org.bouncycastle.openpgp.bc.BcPGPObjectFactory import org.bouncycastle.openpgp.operator.bc.BcKeyFingerprintCalculator +import java.io.ByteArrayOutputStream +import java.io.OutputStream +import java.nio.ByteBuffer fun InputStream.toSignatureList() = buffered() @@ -38,10 +43,56 @@ fun File.toSignatureList() = inputStream().toSignatureList() val PGPSignature.hexKey: String get() = keyID.hexKey -val Iterable.hexKeys: String get() = sorted().joinToString(prefix = "[", postfix = "]") { it.hexKey } +val PGPSignature.pgpShortKeyId: PgpKeyId.Short get() = + PgpKeyId.Short(ByteBuffer.allocate(8).putLong(keyID).array()) + +val PGPPublicKey.pgpFullKeyId: PgpKeyId.Full get() = + PgpKeyId.Full(fingerprint) + +val PGPPublicKey.pgpShortKeyId: PgpKeyId.Short get() = + PgpKeyId.Short(ByteBuffer.allocate(8).putLong(keyID).array()) // `java.lang`.Long.toHexString(this) does not generate leading 0 val Long.hexKey: String get() = "%016x".format(this) fun InputStream.readPgpPublicKeys() = PGPPublicKeyRingCollection(PGPUtil.getDecoderStream(this), BcKeyFingerprintCalculator()) + +fun PGPPublicKeyRingCollection.publicKeysWithId(keyId: PgpKeyId.Short) = + keyRings.asSequence() + .flatMap { keyRing -> + keyRing.asSequence().filter { it.keyID == keyId.keyId } + } + +/** + * Remove all UserIDs and Signatures to avoid storing personally identifiable information. + */ +fun PGPPublicKeyRingCollection.strip() = + PGPPublicKeyRingCollection( + toList() + .map { pgpPublicKeyRing -> + PGPPublicKeyRing( + pgpPublicKeyRing + .map { + it.signatures.asSequence() + .fold(it) { key, signature -> + PGPPublicKey.removeCertification(key, signature) + } + } + .map { + it.rawUserIDs.asSequence() + .fold(it) { key, userId -> + PGPPublicKey.removeCertification(key, userId) + } + } + ) + } + ) + +fun armourEncode(body: (OutputStream) -> Unit)= + ByteArrayOutputStream().apply { + ArmoredOutputStream(this).use { + it.clearHeaders() + body(it) + } + }.toByteArray() diff --git a/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/model/DependencyVerification.kt b/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/model/DependencyVerification.kt index 3935039..cd9c635 100644 --- a/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/model/DependencyVerification.kt +++ b/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/model/DependencyVerification.kt @@ -17,7 +17,7 @@ package com.github.vlsi.gradle.checksum.model import com.github.vlsi.gradle.checksum.debug -import com.github.vlsi.gradle.checksum.hexKeys +import com.github.vlsi.gradle.checksum.pgp.PgpKeyId import org.gradle.api.artifacts.DependencyArtifact import org.gradle.api.logging.Logging @@ -56,7 +56,7 @@ class DependencyChecksum( val id: Id ) { val sha512 = mutableSetOf() - val pgpKeys = mutableSetOf() + val pgpKeys = mutableSetOf() val verificationConfig: VerificationConfig get() = VerificationConfig( @@ -71,19 +71,19 @@ class DependencyChecksum( } override fun toString(): String { - return "DependencyChecksum(sha512=$sha512, pgpKeys=${pgpKeys.hexKeys}" + return "DependencyChecksum(sha512=$sha512, pgpKeys=$pgpKeys" } } class DependencyVerification(val defaultVerificationConfig: VerificationConfig) { - val ignoredKeys = mutableSetOf() + val ignoredKeys = mutableSetOf() - val groupKeys = mutableMapOf>() + val groupKeys = mutableMapOf>() - fun add(group: String, key: Long): Boolean = + fun add(group: String, key: PgpKeyId.Full): Boolean = groupKeys.getOrPut(group) { mutableSetOf() }.add(key) - fun groupKeys(group: String): Set? = groupKeys[group] + fun groupKeys(group: String): Set? = groupKeys[group] val dependencies = mutableMapOf() @@ -100,7 +100,7 @@ class DependencyVerification(val defaultVerificationConfig: VerificationConfig) } override fun toString(): String { - return "DependencyVerification(ignoredKeys=${ignoredKeys.hexKeys}, trustedKeys=${groupKeys.mapValues { it.value.hexKeys }}, dependencies=$dependencies)" + return "DependencyVerification(ignoredKeys=$ignoredKeys, trustedKeys=${groupKeys.mapValues { it.value.toString() }}, dependencies=$dependencies)" } } @@ -123,9 +123,9 @@ class DependencyVerificationDb( fun getConfigFor(id: Id): VerificationConfig = verification.dependencies[id]?.verificationConfig ?: verification.defaultVerificationConfig - fun isIgnored(key: Long) = verification.ignoredKeys.contains(key) + fun isIgnored(key: PgpKeyId) = verification.ignoredKeys.contains(key) - fun ignoreKey(key: Long) { + fun ignoreKey(key: PgpKeyId) { updatedVerification.ignoredKeys += key hasUpdates = true } @@ -153,18 +153,18 @@ class DependencyVerificationDb( val pass = groupKeys.any { dependencyChecksum.pgpKeys.contains(it) } logger.debug { "${if (pass) "OK" else "KO"} PGP group verification for $id." + - " The file was signed via ${dependencyChecksum.pgpKeys.hexKeys}," + - " trusted keys for group ${id.group} are ${groupKeys.hexKeys}" + " The file was signed via ${dependencyChecksum.pgpKeys}," + + " trusted keys for group ${id.group} are $groupKeys" } if (pass) { pgpResult = PgpLevel.GROUP } else if (expected == null && verificationConfig.pgp == PgpLevel.GROUP) { details += - "Trusted PGP keys for group ${id.group} are ${groupKeys.hexKeys}, " + + "Trusted PGP keys for group ${id.group} are $groupKeys, " + if (dependencyChecksum.pgpKeys.isEmpty()) { "however no signature found" } else { - "however artifact is signed by ${dependencyChecksum.pgpKeys.hexKeys} only" + "however artifact is signed by ${dependencyChecksum.pgpKeys} only" } } } @@ -186,13 +186,13 @@ class DependencyVerificationDb( val pass = expected.pgpKeys.any { dependencyChecksum.pgpKeys.contains(it) } logger.debug { "${if (pass) "OK" else "KO"} PGP module verification for $id." + - " The file was signed via ${dependencyChecksum.pgpKeys.hexKeys}," + - " trusted keys for module are ${expected.pgpKeys.hexKeys}" + " The file was signed via ${dependencyChecksum.pgpKeys}," + + " trusted keys for module are ${expected.pgpKeys}" } if (pass) { pgpResult = PgpLevel.MODULE } else { - details += "Expecting one of the following PGP signatures: ${expected.pgpKeys.hexKeys}, but artifact is signed by ${dependencyChecksum.pgpKeys.hexKeys} only" + details += "Expecting one of the following PGP signatures: ${expected.pgpKeys}, but artifact is signed by ${dependencyChecksum.pgpKeys} only" } } if (expected.sha512.isNotEmpty()) { diff --git a/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/model/DependencyVerificationStore.kt b/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/model/DependencyVerificationStore.kt index 7a42fd6..2a231a1 100644 --- a/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/model/DependencyVerificationStore.kt +++ b/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/model/DependencyVerificationStore.kt @@ -16,16 +16,16 @@ */ package com.github.vlsi.gradle.checksum.model -import com.github.vlsi.gradle.checksum.hexKey +import com.github.vlsi.gradle.checksum.pgp.PgpKeyId import groovy.util.XmlSlurper import groovy.util.slurpersupport.GPathResult import groovy.xml.MarkupBuilder -import java.io.File -import java.io.InputStream -import java.io.Writer import org.gradle.api.GradleException import org.gradle.api.artifacts.DependencyArtifact import org.gradle.kotlin.dsl.withGroovyBuilder +import java.io.File +import java.io.InputStream +import java.io.Writer private operator fun GPathResult.get(name: String) = getProperty(name) as GPathResult @@ -41,9 +41,22 @@ private fun GPathResult.requiredAttr(name: String): String = @Suppress("UNCHECKED_CAST") private fun GPathResult.getList(name: String) = getProperty(name) as Iterable -private val String.parseKey: Long +private val String.parseKey: PgpKeyId get() = - `java.lang`.Long.parseUnsignedLong(this, 16) + PgpKeyId(this) + +private fun String.parseFullKey(message: () -> String): PgpKeyId.Full? = + parseKey.let { + if (it !is PgpKeyId.Full) { + println( + "checksum-dependency-plugin: incorrect key $it specified for ${message()}. " + + "The key should have full length (20 or 16 byte fingerprint) rather than short form. " + + "Short key ids are insecure as collisions are possible, so the short key will be ignored" + ) + return null + } + it + } private fun GPathResult.toDependencyChecksum(): DependencyChecksum { val id = Id(requiredAttr("group"), requiredAttr("module"), attr("version"), @@ -51,7 +64,9 @@ private fun GPathResult.toDependencyChecksum(): DependencyChecksum { attr("extension").ifBlank { DependencyArtifact.DEFAULT_TYPE }) return DependencyChecksum(id).apply { sha512 += getList("sha512").map { it.text() } - pgpKeys += getList("pgp").map { it.text().parseKey } + pgpKeys += getList("pgp").mapNotNull { + it.text().parseFullKey { "allowable pgp key for module $id" } + } } } @@ -123,7 +138,14 @@ object DependencyVerificationStore { } } .getList("trusted-key").forEach { - result.add(it.requiredAttr("group"), it.requiredAttr("id").parseKey) + val group = it.requiredAttr("group") + val key = it.requiredAttr("id").parseFullKey { "trusted-key for group $group" } + if (key != null) { + result.add( + group, + key + ) + } } xml["dependencies"] @@ -158,7 +180,7 @@ object DependencyVerificationStore { "trust-requirement"(verification.defaultVerificationConfig.toMap()) "ignored-keys" { verification.ignoredKeys - .map { it.hexKey } + .map { it.toString() } .sorted() .forEach { "ignored-key"(mapOf("id" to it)) @@ -166,7 +188,7 @@ object DependencyVerificationStore { } "trusted-keys" { verification.groupKeys - .flatMap { (group, keys) -> keys.map { group to it.hexKey } } + .flatMap { (group, keys) -> keys.map { group to it.toString() } } .sortedWith(compareBy({ it.first }, { it.second })) .forEach { "trusted-key"(mapOf("id" to it.second, "group" to it.first)) @@ -178,7 +200,7 @@ object DependencyVerificationStore { .sortedBy { it.key.toString() } .forEach { (id, dependency) -> "dependency"(id.toMap()) { - dependency.pgpKeys.map { it.hexKey }.sorted().forEach { + dependency.pgpKeys.map { it.toString() }.sorted().forEach { "pgp"(it) } dependency.sha512.sorted().forEach { diff --git a/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/pgp/KeyDownloader.kt b/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/pgp/KeyDownloader.kt index e008b17..c6d2817 100644 --- a/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/pgp/KeyDownloader.kt +++ b/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/pgp/KeyDownloader.kt @@ -17,7 +17,7 @@ package com.github.vlsi.gradle.checksum.pgp import com.github.vlsi.gradle.checksum.debug -import com.github.vlsi.gradle.checksum.hexKey +import com.github.vlsi.gradle.checksum.info import java.net.InetAddress import java.net.URI import java.time.Duration @@ -60,19 +60,17 @@ class KeyDownloader( ) // Sample URL: https://keyserver.ubuntu.com/pks/lookup?op=vindex&fingerprint=on&search=0xbcf4173966770193 - private fun URI.retrieveKeyUri(keyId: Long, inetAddress: InetAddress) = + private fun URI.retrieveKeyUri(keyId: PgpKeyId, inetAddress: InetAddress) = URI( scheme, userInfo, host, port, "/pks/lookup", - "op=get&options=mr&search=0x${keyId.hexKey}", null + "op=get&options=mr&search=0x$keyId", null ) - fun findKey(keyId: String, comment: String) = findKey(`java.lang`.Long.parseUnsignedLong(keyId, 16), comment) - - fun findKey(keyId: Long, comment: String): ByteArray? = - retry("Downloading key ${keyId.hexKey} for $comment") { + fun findKey(keyId: PgpKeyId, comment: String): ByteArray? = + retry("Downloading key $keyId for $comment") { val url = uri.prepare.retrieveKeyUri(keyId, inetAddress) .toURL() - logger.debug { "Downloading PGP key ${keyId.hexKey} from $inetAddress, url: $url" } + logger.debug { "Downloading PGP key $keyId from $inetAddress, url: $url" } val request = Request.Builder() .url(url) .build() diff --git a/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/pgp/KeyStore.kt b/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/pgp/KeyStore.kt index 1053715..c0f9fc4 100644 --- a/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/pgp/KeyStore.kt +++ b/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/pgp/KeyStore.kt @@ -18,7 +18,15 @@ package com.github.vlsi.gradle.checksum.pgp import com.github.vlsi.gradle.checksum.Executors import com.github.vlsi.gradle.checksum.Stopwatch +import com.github.vlsi.gradle.checksum.armourEncode +import com.github.vlsi.gradle.checksum.pgpFullKeyId +import com.github.vlsi.gradle.checksum.pgpShortKeyId +import com.github.vlsi.gradle.checksum.publicKeysWithId import com.github.vlsi.gradle.checksum.readPgpPublicKeys +import com.github.vlsi.gradle.checksum.strip +import org.bouncycastle.openpgp.PGPPublicKey +import org.bouncycastle.openpgp.PGPPublicKeyRing +import org.gradle.api.logging.Logging import java.io.File import java.nio.file.Files import java.util.concurrent.CompletableFuture @@ -26,39 +34,51 @@ import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.locks.ReentrantReadWriteLock import kotlin.concurrent.read import kotlin.concurrent.write -import org.bouncycastle.openpgp.PGPPublicKey -import org.gradle.api.logging.Logging private val logger = Logging.getLogger(KeyStore::class.java) class KeyStore( val storePath: File, + val cachedKeysTempRoot: File, val keyDownloader: KeyDownloader ) { - private val keys = - object : LinkedHashMap(100, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean { + private val keys = mutableMapOf() + + private val loadRequests = + ConcurrentHashMap>>() + + private val shortToFull = + object : LinkedHashMap>(100, 0.75f, true) { + override fun removeEldestEntry(eldest: Map.Entry>): Boolean { return this.size > 1000 } } - private val loadRequests = - ConcurrentHashMap>() - private val lock = ReentrantReadWriteLock() val downloadTimer = Stopwatch() - fun getKeyAsync(keyId: Long, comment: String, executors: Executors): CompletableFuture { + fun getKeyAsync( + keyId: PgpKeyId.Short, + comment: String, + executors: Executors + ): CompletableFuture> { lock.read { - if (keys.containsKey(keyId)) { - return CompletableFuture.completedFuture(keys[keyId]) + if (shortToFull.containsKey(keyId)) { + return CompletableFuture.completedFuture( + shortToFull.getValue(keyId).map { + keys.getValue(it) + } + ) } return loadRequests.computeIfAbsent(keyId) { CompletableFuture .supplyAsync({ -> - loadKey(keyId, comment).also { pgp -> + loadKey(keyId, comment).also { publicKeys -> lock.write { - keys[keyId] = pgp + for (publicKey in publicKeys) { + keys[publicKey.pgpFullKeyId] = publicKey + } + shortToFull[keyId] = publicKeys.mapTo(mutableSetOf()) { it.pgpFullKeyId } loadRequests.remove(keyId) } } @@ -67,33 +87,108 @@ class KeyStore( } } - private fun loadKey(keyId: Long, comment: String): PGPPublicKey? { + private fun loadShortIndex(indexFile: File) = + indexFile.takeIf { it.exists() }?.readLines() + ?.asSequence() + ?.map { PgpKeyId(it) } + ?.filterIsInstance() + + private fun loadKey(keyId: PgpKeyId.Short, comment: String): List { // try filesystem - val fileName = "%02x/%016x.asc".format(keyId ushr 56, keyId) - val cacheFile = File(storePath, fileName) - val keyStream = if (cacheFile.exists()) { - cacheFile.inputStream().buffered() - } else { - val keyBytes = - downloadTimer { keyDownloader.findKey(keyId, comment) } ?: return null - - File(storePath, "$fileName.tmp").apply { - // It will throw exception should create fail (e.g. permission or something) + val indexFile = File(storePath, "%02x/%s.idx".format(keyId.bytes.last(), keyId)) + // It will throw exception should create fail (e.g. permission or something) + + if (indexFile.exists()) { + val indexed = loadShortIndex(indexFile) + ?.flatMap { + val keyFile = File(storePath, "%02x/%s.asc".format(it.bytes.last(), it)) + Files.newInputStream(keyFile.toPath()).use { stream -> + stream.readPgpPublicKeys() + .publicKeysWithId(keyId) + } + } + ?.toList() + if (indexed != null) { + return indexed + } + } + + val keyBytes = + downloadTimer { keyDownloader.findKey(keyId, comment) } ?: return listOf() + + val cleanedPublicKeys = keyBytes.inputStream() + .readPgpPublicKeys() + .strip() + + for (keyRing in cleanedPublicKeys.keyRings) { + val mainKeyId = keyRing.publicKey.pgpFullKeyId + val resultingName = "%02x/%s.asc".format(mainKeyId.bytes.last(), mainKeyId) + + File(cachedKeysTempRoot, resultingName).apply { Files.createDirectories(parentFile.toPath()) - writeBytes(keyBytes) - if (!renameTo(cacheFile)) { - if (cacheFile.exists()) { - // Another thread (e.g. another build) has already received the same key - // Ignore the error - delete() - } else { - logger.warn("Unable to rename $this to $cacheFile") + + writeBytes( + armourEncode { + keyRing.encode(it) + } + ) + val resultingFile = File(storePath, resultingName) + Files.createDirectories(resultingFile.parentFile.toPath()) + + lock.write { + if (!renameTo(resultingFile)) { + if (resultingFile.exists()) { + // Another thread (e.g. another build) has already received the same key + // Ignore the error + if (resultingFile.length() != length()) { + logger.warn("checksum-dependency-plugin: $resultingFile has different size (${resultingFile.length()}) than the received one $this (${length()}.") + } else { + delete() + } + } else { + logger.warn("Unable to rename $this to $resultingFile") + } } } } - keyBytes.inputStream() + + addPublicKeysToIndex(keyRing) } - return keyStream.readPgpPublicKeys().getPublicKey(keyId) + return cleanedPublicKeys.publicKeysWithId(keyId).toList() + } + + private fun addPublicKeysToIndex( + keyRing: PGPPublicKeyRing + ) { + val mainKeyId = keyRing.publicKey.pgpFullKeyId + lock.write { + for (key in keyRing) { + val shortKeyId = key.pgpShortKeyId + + val indexFileName = "%02x/%s.idx".format(shortKeyId.bytes.last(), shortKeyId) + val indexFile = File(storePath, indexFileName) + val existingKeys = shortToFull[shortKeyId] + ?: loadShortIndex(indexFile)?.toSet() + ?: mutableSetOf() + + File(cachedKeysTempRoot, indexFileName).apply { + val newIndexContents = existingKeys + mainKeyId + shortToFull[shortKeyId] = newIndexContents.toMutableSet() + Files.createDirectories(parentFile.toPath()) + writeText( + newIndexContents + .map { it.toString() } + .sorted() + .joinToString(System.lineSeparator()) + ) + if (indexFile.exists()) { + indexFile.delete() + } + Files.createDirectories(indexFile.parentFile.toPath()) + renameTo(indexFile) + } + } + } } } diff --git a/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/pgp/PgpKeyId.kt b/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/pgp/PgpKeyId.kt new file mode 100644 index 0000000..2140bc4 --- /dev/null +++ b/plugins/checksum-dependency-plugin/src/main/kotlin/com/github/vlsi/gradle/checksum/pgp/PgpKeyId.kt @@ -0,0 +1,71 @@ +/* + * Copyright 2019 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.github.vlsi.gradle.checksum.pgp + +import java.math.BigInteger +import java.nio.ByteBuffer + +sealed class PgpKeyId(val bytes: ByteArray, tmp: Nothing?) { + class Short(bytes: ByteArray): PgpKeyId(bytes, null) { + val keyId = ByteBuffer.wrap(bytes).long + init { + require(bytes.size == 8) { + "Short PGP key ID must be 8 bytes long, but got ${bytes.size} bytes: $this" + } + } + } + + class Full(val fingerprint: ByteArray): PgpKeyId(fingerprint, null) { + init { + require(bytes.size > 16) { + "Full PGP key ID must be 16 (MD5) or 20 (SHA1) bytes long, but got ${bytes.size} bytes: $this" + } + } + } + + override fun toString() = + BigInteger(1, bytes).toString(16) + .padStart(bytes.size * 2, '0') + + override fun hashCode(): Int = bytes.contentHashCode() + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as PgpKeyId + + if (!bytes.contentEquals(other.bytes)) return false + + return true + } +} + +fun PgpKeyId(bytes: ByteArray) = + when (bytes.size) { + 8 -> PgpKeyId.Short(bytes) + else -> PgpKeyId.Full(bytes) + } + +fun PgpKeyId(keyId: String): PgpKeyId { + val bytes = keyId + .chunked(2) + .map { it.toInt(16).toByte() } + .toByteArray() + return PgpKeyId(bytes) +} diff --git a/plugins/checksum-dependency-plugin/src/test/kotlin/com/github/vlsi/gradle/checksum/signatures/KeyDownloaderTest.kt b/plugins/checksum-dependency-plugin/src/test/kotlin/com/github/vlsi/gradle/checksum/signatures/KeyDownloaderTest.kt index 8bf0107..a0bcec4 100644 --- a/plugins/checksum-dependency-plugin/src/test/kotlin/com/github/vlsi/gradle/checksum/signatures/KeyDownloaderTest.kt +++ b/plugins/checksum-dependency-plugin/src/test/kotlin/com/github/vlsi/gradle/checksum/signatures/KeyDownloaderTest.kt @@ -17,6 +17,7 @@ package com.github.vlsi.gradle.checksum.signatures import com.github.vlsi.gradle.checksum.pgp.KeyDownloader +import com.github.vlsi.gradle.checksum.pgp.PgpKeyId import com.github.vlsi.gradle.checksum.readPgpPublicKeys import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test @@ -26,10 +27,10 @@ class KeyDownloaderTest { @Test internal fun goodKey() { - val keyId = `java.lang`.Long.parseUnsignedLong("bcf4173966770193", 16) + val keyId = PgpKeyId("bcf4173966770193") as PgpKeyId.Short val bytes = downloader.findKey(keyId, "KeyDownloaderTest") val keys = bytes!!.inputStream().readPgpPublicKeys() - val basicInfo = keys.getPublicKey(keyId).let { + val basicInfo = keys.getPublicKey(keyId.keyId).let { """ algorithm: ${it.algorithm} bitStrength: ${it.bitStrength}