From 7fd35d8049bec3023083e9b0466b4b43a6dbb215 Mon Sep 17 00:00:00 2001 From: Richard Gomez Date: Sun, 15 Dec 2024 21:47:31 -0500 Subject: [PATCH] fix(azuredevops): detector wasn't working --- .../azuredevopspersonalaccesstoken.go | 162 ++++++++++++------ ...opspersonalaccesstoken_integration_test.go | 2 +- .../azuredevopspersonalaccesstoken_test.go | 78 +++++++-- pkg/engine/defaults/defaults.go | 2 +- 4 files changed, 175 insertions(+), 69 deletions(-) diff --git a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken.go b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken.go index 3489d48606e8..0c937f0de4e3 100644 --- a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken.go +++ b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken.go @@ -1,13 +1,17 @@ -package azuredevopspersonalaccesstoken +package azure_devops import ( "context" + "encoding/json" + "errors" "fmt" + "io" "net/http" "strings" regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -21,81 +25,143 @@ type Scanner struct { // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) -var ( - defaultClient = common.SaneHttpClient() - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `\b([0-9a-z]{52})\b`) - orgPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `\b([0-9a-zA-Z][0-9a-zA-Z-]{5,48}[0-9a-zA-Z])\b`) -) +func (s Scanner) Type() detectorspb.DetectorType { + return detectorspb.DetectorType_AzureDevopsPersonalAccessToken +} + +func (s Scanner) Description() string { + return "Azure DevOps is a suite of development tools provided by Microsoft. Personal Access Tokens (PATs) are used to authenticate and authorize access to Azure DevOps services and resources." +} // Keywords are used for efficiently pre-filtering chunks. // Use identifiers in the secret preferably, or the provider name. func (s Scanner) Keywords() []string { - return []string{"azure"} + return []string{"dev.azure.com", "az devops"} } +var ( + // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure", "az", "token", "pat"}) + `\b([a-z0-9]{52}|[a-zA-Z0-9]{84})\b`) + orgPat = regexp.MustCompile(`dev\.azure\.com/([0-9a-zA-Z][0-9a-zA-Z-]{5,48}[0-9a-zA-Z])\b`) + + invalidOrgCache = simple.NewCache[struct{}]() +) + // FromData will find and optionally verify AzureDevopsPersonalAccessToken secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) - matches := keyPat.FindAllStringSubmatch(dataStr, -1) - orgMatches := orgPat.FindAllStringSubmatch(dataStr, -1) - - for _, match := range matches { - if len(match) != 2 { + // Deduplicate results. + keyMatches := make(map[string]struct{}) + for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) { + m := match[1] + if detectors.StringShannonEntropy(m) < 3 { continue } - resMatch := strings.TrimSpace(match[1]) - for _, orgMatch := range orgMatches { - if len(orgMatch) != 2 { - continue - } - resOrgMatch := strings.TrimSpace(orgMatch[1]) + keyMatches[m] = struct{}{} + } + orgMatches := make(map[string]struct{}) + for _, match := range orgPat.FindAllStringSubmatch(dataStr, -1) { + m := match[1] + if invalidOrgCache.Exists(m) { + continue + } + orgMatches[m] = struct{}{} + } - s1 := detectors.Result{ + for key := range keyMatches { + for org := range orgMatches { + r := detectors.Result{ DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken, - Raw: []byte(resMatch), - RawV2: []byte(resMatch + resOrgMatch), + Raw: []byte(key), + RawV2: []byte(fmt.Sprintf(`{"organization":"%s","token":"%s"}`, org, key)), } if verify { - client := s.client - if client == nil { - client = defaultClient - } - req, err := http.NewRequestWithContext(ctx, "GET", "https://dev.azure.com/"+resOrgMatch+"/_apis/projects", nil) - if err != nil { - continue + if s.client == nil { + s.client = common.SaneHttpClient() } - req.SetBasicAuth("", resMatch) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - hasVerifiedRes, _ := common.ResponseContainsSubstring(res.Body, "lastUpdateTime") - if res.StatusCode >= 200 && res.StatusCode < 300 && hasVerifiedRes { - s1.Verified = true - } else if res.StatusCode == 401 { - // The secret is determinately not verified (nothing to do) - } else { - err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) - s1.SetVerificationError(err, resMatch) + + isVerified, extraData, verificationErr := verifyMatch(ctx, s.client, org, key) + r.Verified = isVerified + r.ExtraData = extraData + if verificationErr != nil { + if errors.Is(verificationErr, errInvalidOrg) { + delete(orgMatches, org) + invalidOrgCache.Set(org, struct{}{}) + continue } - } else { - s1.SetVerificationError(err, resMatch) + r.SetVerificationError(verificationErr) } } - results = append(results, s1) + results = append(results, r) } } return results, nil } -func (s Scanner) Type() detectorspb.DetectorType { - return detectorspb.DetectorType_AzureDevopsPersonalAccessToken +var errInvalidOrg = errors.New("invalid organization") + +func verifyMatch(ctx context.Context, client *http.Client, org string, key string) (bool, map[string]string, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "https://dev.azure.com/"+org+"/_apis/projects", nil) + if err != nil { + return false, nil, err + } + + req.SetBasicAuth("", key) + req.Header.Set("Accept", "application/json") + req.Header.Set("Content-Type", "application/json") + res, err := client.Do(req) + if err != nil { + return false, nil, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + // {"count":1,"value":[{"id":"...","name":"Test","url":"https://dev.azure.com/...","state":"wellFormed","revision":11,"visibility":"private","lastUpdateTime":"2024-12-16T02:23:58.86Z"}]} + var projectsRes listProjectsResponse + if json.NewDecoder(res.Body).Decode(&projectsRes) != nil { + return false, nil, err + } + + // Condense a list of organizations + roles. + var ( + extraData map[string]string + projects = make([]string, 0, len(projectsRes.Value)) + ) + for _, p := range projectsRes.Value { + projects = append(projects, p.Name) + } + if len(projects) > 0 { + extraData = map[string]string{ + "projects": strings.Join(projects, ","), + } + } + return true, extraData, nil + case http.StatusUnauthorized: + // The secret is determinately not verified (nothing to do) + return false, nil, nil + case http.StatusNotFound: + // Org doesn't exist. + return false, nil, errInvalidOrg + default: + body, _ := io.ReadAll(res.Body) + return false, nil, fmt.Errorf("unexpected HTTP response: status=%d, body=%q", res.StatusCode, string(body)) + } } -func (s Scanner) Description() string { - return "Azure DevOps is a suite of development tools provided by Microsoft. Personal Access Tokens (PATs) are used to authenticate and authorize access to Azure DevOps services and resources." +type listProjectsResponse struct { + Count int `json:"count"` + Value []projectResponse `json:"value"` +} + +type projectResponse struct { + Id string `json:"id"` + Name string `json:"name"` } diff --git a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.go b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.go index 2b7c68f3fd67..793e6e685b3d 100644 --- a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.go +++ b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.go @@ -1,7 +1,7 @@ //go:build detectors // +build detectors -package azuredevopspersonalaccesstoken +package azure_devops import ( "context" diff --git a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go index b19ef9de3f55..c3d94813da79 100644 --- a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go +++ b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go @@ -1,4 +1,4 @@ -package azuredevopspersonalaccesstoken +package azure_devops import ( "context" @@ -10,19 +10,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -var ( - validPattern = ` - azure: - azure_key: uie5tff7m5h5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8un - azure_org_id: WOkQXnjSxCyioEJRa8R6J39cN4Xfyy8CWl1BZksHYsevxVBFzG - ` - invalidPattern = ` - azure: - azure_key: uie5tff7m5H5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8un - azure_org_id: LOKi - ` -) - func TestAzureDevopsPersonalAccessToken_Pattern(t *testing.T) { d := Scanner{} ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) @@ -32,14 +19,67 @@ func TestAzureDevopsPersonalAccessToken_Pattern(t *testing.T) { input string want []string }{ + // old + { + name: "valid - old token", + input: ` +provider "azuredevops" { + # Configuration options + org_service_url = "https://dev.azure.com/housemd" + personal_access_token = "qkfon5cdjdekin4qnkgfr2nf367h6yjnnqm5upwqepd3rekl4l5a" +}`, + want: []string{"qkfon5cdjdekin4qnkgfr2nf367h6yjnnqm5upwqepd3rekl4l5a:housemd"}, + }, + + // new { - name: "valid pattern", - input: validPattern, - want: []string{"uie5tff7m5h5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8unWOkQXnjSxCyioEJRa8R6J39cN4Xfyy8CWl1BZksHYsevxVBFzG"}, + name: "valid - az devops CLI", + input: ` echo "Tests failed. Creating a bug in Azure DevOps..." + az devops login --organization https://dev.azure.com/TechServicesCorp --token A0us9bS1c6qe5blb6CT4FGRR4JcmPDg7uadVFmw4D65bvtdPcdVdJQQJ99AKACAAAAAPnX9AAAASAhDO4GFB + az boards work-item create --title "Automated Bug: Test Failure" --type $(bugType) --description "Tests failed. See results.log for details." --project "Test"`, + want: []string{"A0us9bS1c6qe5blb6CT4FGRR4JcmPDg7uadVFmw4D65bvtdPcdVdJQQJ99AKACAAAAAPnX9AAAASAhDO4GFB:TechServicesCorp"}, }, { - name: "invalid pattern", - input: invalidPattern, + name: "valid - environment variables", + input: `# Base image: Azure CLI with a lightweight Ubuntu distribution-mcr.microsoft.com/azure-cli:2.52.0 +FROM ubuntu:20.04 + +# Set environment variables for Azure DevOps agent +ENV AZP_URL=https://dev.azure.com/EBOrg21 +ENV AZP_TOKEN=2ZGS1XLyxTU2wXlrXy71ldl1tBKceXM9kl6mVAeQchvWIErzkwtBJQjJ99AKACAAAAAAAAAAAAASAZDO5BA2 +ENV AZP_POOL=TestParty +`, + want: []string{"2ZGS1XLyxTU2wXlrXy71ldl1tBKceXM9kl6mVAeQchvWIErzkwtBJQjJ99AKACAAAAAAAAAAAAASAZDO5BA2:EBOrg21"}, + }, + { + name: "valid - jupyter notebook", + input: ` "4 https://dev.azure.com/SSGL-SMT/10_BG_AU5... " + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "token = r\"49QzGd2ZOLTWdoMc0S3M0cZkVVsBMTua01tlMYOkTUnEwxebgYdheQQJ99AKACAAAAAHsyrdAAASAZDOULjm\"" + ]`, + want: []string{"49QzGd2ZOLTWdoMc0S3M0cZkVVsBMTua01tlMYOkTUnEwxebgYdheQQJ99AKACAAAAAHsyrdAAASAZDOULjm:SSGL-SMT"}, + }, + + // Invalid + { + name: "invalid", + input: `ssh.dev.azure.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7Hr1oTWqNqOlzGJOfGJ4NakVyIzf1rXYd4d7wo6jBlkLvCA4odBlL0mDUyZ0/QUfTTqeu+tm22gOsv+VrVTMk6vwRU75gY/y9ut5Mb3bR5BV58dKXyq9A9UeB5Cakehn5Zgm6x1mKoVyf+FFn26iYqXJRgzIZZcZ5V6hrE0Qg39kZm4az48o0AUbf6Sp4SLdvnuMa2sVNwHBboS7EJkm57XQPVU3/QpyNLHbWDdzwtrlS+ez30S3AdYhLKEOxAG8weOnyrtLJAUen9mTkol8oII1edf7mWWbWVf0nBmly21+nZcmCTISQBtdcyPaEno7fFQMDD26/s0lfKob4Kw8H`, want: nil, }, } diff --git a/pkg/engine/defaults/defaults.go b/pkg/engine/defaults/defaults.go index 0d69fe01373a..92fc5f93b339 100644 --- a/pkg/engine/defaults/defaults.go +++ b/pkg/engine/defaults/defaults.go @@ -894,7 +894,7 @@ func buildDetectorList() []detectors.Detector { &azure_serviceprincipal_v2.Scanner{}, &azure_batch.Scanner{}, &azurecontainerregistry.Scanner{}, - &azuredevopspersonalaccesstoken.Scanner{}, + &azure_devops.Scanner{}, // &azurefunctionkey.Scanner{}, // detector is throwing some FPs &azure_openai.Scanner{}, &azuresearchadminkey.Scanner{},