Update Gitleaks to 8.18.1 and custom rules (#85)

* update to Gitleaks version 8.18.1

* update test expectation for plain file to match new output from Gitleaks

* update config to extend default gitleaks config

* delete rules from gitleaks core v 4.1.1

* add IDs to custom gitleaks rules

* re-add custom generic username rule

* remove comment

* update formatting

* re-add allowlist rules

* fix allow lists

* update email rule

* add rule for suspicious filenames

* add suspicious filenames rule

* re-add generic credential rule

Makefile

@@ -1,7 +1,7 @@
-CAULKING_VERSION=2.0.0 2023-01-26
-GITLEAKS_VERSION=8.8.4
+CAULKING_VERSION=2.0.0 2023-12-01
+GITLEAKS_VERSION=8.18.1
 GITLEAKS_ARTIFACT="gitleaks_${GITLEAKS_VERSION}_darwin_x64.tar.gz"
-GITLEAKS_CHECKSUM=509430dada69ee4314068847a8a424d4102defc23fd5714330d36366796feef7
+GITLEAKS_CHECKSUM=8eaae2aec79175a2b9f1879994c47107752200408ef3bc100ce8f9e56ee0e199
 GITLEAKS_DOWNLOAD_DIR="${HOME}/bin/gitleaks-files"
 NOW=$(shell date)
 ME=$(shell whoami)

caulked.bats

@@ -22,7 +22,7 @@ load test_helper
 @test "leak prevention allows plain text, check 'git config --global -l' on failure" {
     run addFileWithNoSecrets
     [ ${status} -eq 0 ]
-    echo ${lines[6]} | grep -q "no leaks found"
+    echo ${lines[7]} | grep -q "no leaks found"
 }
 
 @test "leak prevention catches unstaged aws secrets in test repo" {

local.toml

@@ -1,27 +1,43 @@
 title = "gitleaks config"
 
+[extend]
+useDefault = true
+
+[allowlist]
+	description = "general allowlists"
+	paths = [
+		'''(.*?)(jpg|gif|doc|pdf|bin|svg)$''',
+		'''development.bats''',
+		'''caulked.bats''',
+		'''local.toml'''
+	]
+	regexes	= ['''CHANGEME|changeme|feedabee|EXAMPLE|23.22.13.113|1234567890''']
+
 # Rules borrowed from GSA
 # https://github.com/GSA/odp-code-repository-commit-rules/blob/master/gitleaks/rules.toml
 # and customized for TTS
 
 # If IPv4 is overbroad, cloud.gov external IPs may all be nonsensitive
 # and we can change match to (10|172|192). in the first octet.
 [[rules]]
+	id = "ipv4"
 	description = "IPv4 addresses"
 	regex = '''\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b'''
 	tags = ["IPv4", "IP", "addresses"]
 	[rules.allowlist]
-		regexes = ['''(169.254.169.254|127.0.0.\d+|23.22.13.113)''', # 23.22.13.113 is gsa.gov
-		'''\b0\.(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){2}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b''' # OK to start w/ 0.
+		regexes = [
+			'''(169.254.169.254|127.0.0.\d+|23.22.13.113)''', # 23.22.13.113 is gsa.gov
+			'''\b0\.(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){2}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b''' # OK to start w/ 0.
 		]
 
-# s3config copied from `leaky-repo.toml` upstream, but uncommented:
 [[rules]]
+	id = "s3config"
 	description = "s3config"
 	regex = '''(?i)(dbpasswd|dbuser|dbname|dbhost|api_key|apikey|key|api|password|user|guid|hostname|pw|auth)(.{0,3})?([0-9a-zA-Z-_\/+!{}=]{4,120})'''
 	path = '''(?i)s3cfg$'''
 
 [[rules]]
+	id = "yaml-secrets"
 	description = "yaml secrets"
 	regex = '''(?i)(password|enc.key|auth.pass):\s+(.*)'''
 	path = '''(?i)(\.yml|\.yaml)'''
@@ -31,199 +47,60 @@ title = "gitleaks config"
 		regexes = ['''(\(\(.*\)\)|\{\{.*\}\})''']
 
 [[rules]]
+	id = "email"
 	description = "Email except non-pii business email"
-	regex = '''(.{0,48}?)[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}'''
+	regex = '''(.{0,48}?)([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4})'''
 	tags = ["email"]
 	[rules.allowlist]
-		regexes = ['''(?i)@(cloud.gov|gsa.gov|github.com)''',
-			'''(?i)(Author|Copyright|Contact)''']
-		paths = ['''Godeps._workspace''', '''(composer.lock)$''']
-
-# Rules from original `leaky-repo.toml` v4.1.1
-# https://raw.githubusercontent.com/zricethezav/gitleaks/master/examples/leaky-repo.toml
-# with the following rule sets removed:
-# - Email
-
-[[rules]]
-	description = "AWS MANAGER ID"
-	regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
-	tags = ["key", "AWS"]
-
-[[rules]]
-	description = "AWS cred file info"
-	regex = '''(?i)(aws_access_key_id|aws_secret_access_key)(.{0,20})?=.[0-9a-zA-Z\/+]{20,40}'''
-	tags = ["AWS"]
-
-[[rules]]
-	description = "AWS Secret Key"
-	regex = '''(?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]'''
-	tags = ["key", "AWS"]
-
-[[rules]]
-	description = "AWS MWS key"
-	regex = '''amzn\.mws\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'''
-	tags = ["key", "AWS", "MWS"]
-
-[[rules]]
-	description = "Facebook Secret Key"
-	regex = '''(?i)(facebook|fb)(.{0,20})?(?-i)['\"][0-9a-f]{32}['\"]'''
-	tags = ["key", "Facebook"]
-
-[[rules]]
-	description = "Facebook Client ID"
-	regex = '''(?i)(facebook|fb)(.{0,20})?['\"][0-9]{13,17}['\"]'''
-	tags = ["key", "Facebook"]
-
-[[rules]]
-	description = "Twitter Secret Key"
-	regex = '''(?i)twitter(.{0,20})?['\"][0-9a-z]{35,44}['\"]'''
-	tags = ["key", "Twitter"]
-
-[[rules]]
-	description = "Twitter Client ID"
-	regex = '''(?i)twitter(.{0,20})?['\"][0-9a-z]{18,25}['\"]'''
-	tags = ["client", "Twitter"]
-
-[[rules]]
-	description = "Github"
-	regex = '''(?i)github(.{0,20})?(?-i)['\"][0-9a-zA-Z]{35,40}['\"]'''
-	tags = ["key", "Github"]
-
-[[rules]]
-	description = "LinkedIn Client ID"
-	regex = '''(?i)linkedin(.{0,20})?(?-i)['\"][0-9a-z]{12}['\"]'''
-	tags = ["client", "LinkedIn"]
-
-[[rules]]
-	description = "LinkedIn Secret Key"
-	regex = '''(?i)linkedin(.{0,20})?['\"][0-9a-z]{16}['\"]'''
-	tags = ["secret", "LinkedIn"]
-
-[[rules]]
-	description = "Slack"
-	regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?'''
-	tags = ["key", "Slack"]
-
-[[rules]]
-	description = "EC"
-	regex = '''-----BEGIN EC PRIVATE KEY-----'''
-	tags = ["key", "EC"]
-
-[[rules]]
-	description = "Google API key"
-	regex = '''AIza[0-9A-Za-z\\-_]{35}'''
-	tags = ["key", "Google"]
-
-
-[[rules]]
-	description = "Heroku API key"
-	regex = '''(?i)heroku(.{0,20})?['"][0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}['"]'''
-	tags = ["key", "Heroku"]
-
-[[rules]]
-	description = "MailChimp API key"
-	regex = '''(?i)(mailchimp|mc)(.{0,20})?['"][0-9a-f]{32}-us[0-9]{1,2}['"]'''
-	tags = ["key", "Mailchimp"]
-
-[[rules]]
-	description = "Mailgun API key"
-	regex = '''(?i)(mailgun|mg)(.{0,20})?['"][0-9a-z]{32}['"]'''
-	tags = ["key", "Mailgun"]
-
-[[rules]]
-	description = "PayPal Braintree access token"
-	regex = '''access_token\$production\$[0-9a-z]{16}\$[0-9a-f]{32}'''
-	tags = ["key", "Paypal"]
-
-[[rules]]
-	description = "Picatic API key"
-	regex = '''sk_live_[0-9a-z]{32}'''
-	tags = ["key", "Picatic"]
-
-[[rules]]
-	description = "Slack Webhook"
-	regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}'''
-	tags = ["key", "slack"]
-
-[[rules]]
-	description = "Stripe API key"
-	regex = '''(?i)stripe(.{0,20})?['\"][sk|rk]_live_[0-9a-zA-Z]{24}'''
-	tags = ["key", "Stripe"]
-
-[[rules]]
-	description = "Square access token"
-	regex = '''sq0atp-[0-9A-Za-z\-_]{22}'''
-	tags = ["key", "square"]
-
-[[rules]]
-	description = "Square OAuth secret"
-	regex = '''sq0csp-[0-9A-Za-z\\-_]{43}'''
-	tags = ["key", "square"]
-
-[[rules]]
-	description = "Twilio API key"
-	regex = '''(?i)twilio(.{0,20})?['\"][0-9a-f]{32}['\"]'''
-	tags = ["key", "twilio"]
-
-[[rules]]
-	description = "Env Var"
-	regex = '''(?i)(apikey|secret|key|api|password|pass|pw|host)=[0-9a-zA-Z-_.{}]{4,120}'''
+		regexes = [
+			'''(?i)@(cloud.gov|gsa.gov|github.com)''',
+			'''(?i)(Author|Copyright|Contact)'''
+			]
+		paths = [
+			'''Godeps._workspace''',
+			'''(composer.lock)$'''
+		]
 
 [[rules]]
+	id = "generic-username"
 	description = "Generic Username"
 	regex = '''(?i)(dbuser|user)(.{0,20})?['"]([0-9a-zA-Z-_\\+!{}@\./=]{4,120})['"]'''
 	tags = ["key", "username", "generic"]
 	[rules.allowlist]
 		description = "A username in a terraform file and programs is not a leak"
-		regexes = ['''\w+?username\w+=''']
-		paths = ['''\.(tf|rb|go|py|js)$''']
+		regexes = [
+			'''\w+?username\w+=''',
+			'''\"type\":''',
+		]
+		paths = [
+			'''\.(tf|rb|go|py|js)$'''
+		]
 
 [[rules]]
+	id = "generic-credential"
 	description = "Generic Credential"
 	regex = '''(?im)(dbpasswd|dbname|dbhost|api_key|apikey|secret|key|api|password|guid|hostname|pw|auth)(.{0,20})(['"](\S{4,120})['"]|[(\\]\s*$)'''
 	tags = ["key", "API", "generic"]
 	# ignore leaks with specific identifiers like slack and aws as these should be detected
 	# by more granular rules
 	[rules.allowlist]
-		regexes = ['''xox[baprs]-([0-9a-zA-Z]{10,48})''',
+		regexes = [
+			'''xox[baprs]-([0-9a-zA-Z]{10,48})''',
 			'''(?i)(.{0,20})?['"][0-9a-f]{32}-us[0-9]{1,2}['"]''',
-			'''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}''']
-		paths = ['''(vendor.github|Godeps._workspace)''', '''(yarn.lock|package-lock.json|pnpm-lock.yaml|composer.lock)$''']
-
-[[rules]]
-	description = "High Entropy"
-	regex = '''[0-9a-zA-Z-_!{}/=]{4,120}'''
-	path = '''(?i)(dump.sql|high-entropy-misc.txt)$'''
-	tags = ["entropy"]
-	entropy = 4.3
-	[rules.allowlist]
+			'''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
+		]
 		paths = [
-			'''(pem|ppk|env)$''',
-			'''(.*)?ssh'''
+			'''(vendor.github|Godeps._workspace)''',
+			'''(yarn.lock|package-lock.json|pnpm-lock.yaml|composer.lock)$'''
 		]
 
 [[rules]]
-	description = "Potential bash var"
-	regex='''(?i)(=)([0-9a-zA-Z-_!{}=]{4,120})'''
-	tags = ["key", "bash", "API", "generic"]
-	entropy = 3.5
-	secretGroup = 1
-
-[[rules]]
-	description = "WP-Config"
-	regex='''define(.{0,20})?(DB_CHARSET|NONCE_SALT|LOGGED_IN_SALT|AUTH_SALT|NONCE_KEY|DB_HOST|DB_PASSWORD|AUTH_KEY|SECURE_AUTH_KEY|LOGGED_IN_KEY|DB_NAME|DB_USER)(.{0,20})?['"].{10,120}['"]'''
-	tags = ["key", "API", "generic"]
-
-[[rules]]
+	id = "suspicious-file-extensions"
 	description = "File name extensions of credentials"
 	path = '''(?i)\.(pgpass|pem|key)'''
 	tags = ["file"]
 
 [[rules]]
+	id = "suspicious-filenames"
 	description = "File names of credentials"
 	path = '''(?i)(id_rsa|passwd|pgpass|shadow)'''
-
-[allowlist]
-	description = "image allowlists"
-	paths = ['''(.*?)(jpg|gif|doc|pdf|bin|svg)$''', '''development.bats''', '''caulked.bats''', '''local.toml''']
-	regexes	= ['''CHANGEME|changeme|feedabee|EXAMPLE|23.22.13.113|1234567890''']