fix(azuredevops): detector wasn't working

pkg/detectors/azure_devops/personalaccesstoken.go

@@ -0,0 +1,170 @@
+package azure_devops
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	regexp "github.com/wasilibs/go-re2"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct {
+	client *http.Client
+	detectors.DefaultMultiPartCredentialProvider
+}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_AzureDevopsPersonalAccessToken
+}
+
+func (s Scanner) Description() string {
+	return "Azure DevOps is a suite of development tools provided by Microsoft. Personal Access Tokens (PATs) are used to authenticate and authorize access to Azure DevOps services and resources."
+}
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"dev.azure.com", "az devops"}
+}
+
+var (
+	defaultClient = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure", "az", "token", "pat"}) + `\b([a-z0-9]{52}|[a-zA-Z0-9]{84})\b`)
+	orgPat = regexp.MustCompile(`dev\.azure\.com/([0-9a-zA-Z][0-9a-zA-Z-]{5,48}[0-9a-zA-Z])\b`)
+
+	invalidOrgCache = simple.NewCache[struct{}]()
+)
+
+// FromData will find and optionally verify AzureDevopsPersonalAccessToken secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	// Deduplicate results.
+	keyMatches := make(map[string]struct{})
+	for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
+		m := match[1]
+		if detectors.StringShannonEntropy(m) < 3 {
+			continue
+		}
+		keyMatches[m] = struct{}{}
+	}
+	orgMatches := make(map[string]struct{})
+	for _, match := range orgPat.FindAllStringSubmatch(dataStr, -1) {
+		m := match[1]
+		if invalidOrgCache.Exists(m) {
+			continue
+		}
+		orgMatches[m] = struct{}{}
+	}
+
+	for key := range keyMatches {
+		for org := range orgMatches {
+			r := detectors.Result{
+				DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken,
+				Raw:          []byte(key),
+				RawV2:        []byte(fmt.Sprintf(`{"organization":"%s","token":"%s"}`, org, key)),
+			}
+
+			if verify {
+				client := s.client
+				if client == nil {
+					client = defaultClient
+				}
+
+				isVerified, extraData, verificationErr := verifyMatch(ctx, client, org, key)
+				r.Verified = isVerified
+				r.ExtraData = extraData
+				if verificationErr != nil {
+					if errors.Is(verificationErr, errInvalidOrg) {
+						delete(orgMatches, org)
+						invalidOrgCache.Set(org, struct{}{})
+						continue
+					}
+					r.SetVerificationError(verificationErr)
+				}
+			}
+
+			results = append(results, r)
+		}
+	}
+
+	return results, nil
+}
+
+var errInvalidOrg = errors.New("invalid organization")
+
+func verifyMatch(ctx context.Context, client *http.Client, org string, key string) (bool, map[string]string, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", "https://dev.azure.com/"+org+"/_apis/projects", nil)
+	if err != nil {
+		return false, nil, err
+	}
+
+	req.SetBasicAuth("", key)
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("Content-Type", "application/json")
+	res, err := client.Do(req)
+	if err != nil {
+		return false, nil, err
+	}
+	defer func() {
+		_, _ = io.Copy(io.Discard, res.Body)
+		_ = res.Body.Close()
+	}()
+
+	switch res.StatusCode {
+	case http.StatusOK:
+		// {"count":1,"value":[{"id":"...","name":"Test","url":"https://dev.azure.com/...","state":"wellFormed","revision":11,"visibility":"private","lastUpdateTime":"2024-12-16T02:23:58.86Z"}]}
+		var projectsRes listProjectsResponse
+		if json.NewDecoder(res.Body).Decode(&projectsRes) != nil {
+			return false, nil, err
+		}
+
+		// Condense a list of organizations + roles.
+		var (
+			extraData map[string]string
+			projects  = make([]string, 0, len(projectsRes.Value))
+		)
+		for _, p := range projectsRes.Value {
+			projects = append(projects, p.Name)
+		}
+		if len(projects) > 0 {
+			extraData = map[string]string{
+				"projects": strings.Join(projects, ","),
+			}
+		}
+		return true, extraData, nil
+	case http.StatusUnauthorized:
+		// The secret is determinately not verified (nothing to do)
+		return false, nil, nil
+	case http.StatusNotFound:
+		// Org doesn't exist.
+		return false, nil, errInvalidOrg
+	default:
+		body, _ := io.ReadAll(res.Body)
+		return false, nil, fmt.Errorf("unexpected HTTP response: status=%d, body=%q", res.StatusCode, string(body))
+	}
+}
+
+type listProjectsResponse struct {
+	Count int               `json:"count"`
+	Value []projectResponse `json:"value"`
+}
+
+type projectResponse struct {
+	Id   string `json:"id"`
+	Name string `json:"name"`
+}

pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.go → pkg/detectors/azure_devops/personalaccesstoken_integration_test.go

@@ -1,7 +1,7 @@
 //go:build detectors
 // +build detectors
 
-package azuredevopspersonalaccesstoken
+package azure_devops
 
 import (
 	"context"

pkg/detectors/azure_devops/personalaccesstoken_test.go

@@ -0,0 +1,128 @@
+package azure_devops
+
+import (
+	"context"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
+
+func TestAzureDevopsPersonalAccessToken_Pattern(t *testing.T) {
+	d := Scanner{}
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+
+	tests := []struct {
+		name  string
+		input string
+		want  []string
+	}{
+		// old
+		{
+			name: "valid - old token",
+			input: `
+provider "azuredevops" {
+  # Configuration options
+  org_service_url       = "https://dev.azure.com/housemd"
+  personal_access_token = "qkfon5cdjdekin4qnkgfr2nf367h6yjnnqm5upwqepd3rekl4l5a"
+}`,
+			want: []string{"qkfon5cdjdekin4qnkgfr2nf367h6yjnnqm5upwqepd3rekl4l5a:housemd"},
+		},
+
+		// new
+		{
+			name: "valid - az devops CLI",
+			input: `        echo "Tests failed. Creating a bug in Azure DevOps..."
+        az devops login --organization https://dev.azure.com/TechServicesCorp --token A0us9bS1c6qe5blb6CT4FGRR4JcmPDg7uadVFmw4D65bvtdPcdVdJQQJ99AKACAAAAAPnX9AAAASAhDO4GFB
+        az boards work-item create --title "Automated Bug: Test Failure" --type $(bugType) --description "Tests failed. See results.log for details." --project "Test"`,
+			want: []string{"A0us9bS1c6qe5blb6CT4FGRR4JcmPDg7uadVFmw4D65bvtdPcdVdJQQJ99AKACAAAAAPnX9AAAASAhDO4GFB:TechServicesCorp"},
+		},
+		{
+			name: "valid - environment variables",
+			input: `# Base image: Azure CLI with a lightweight Ubuntu distribution-mcr.microsoft.com/azure-cli:2.52.0
+FROM ubuntu:20.04
+
+# Set environment variables for Azure DevOps agent
+ENV AZP_URL=https://dev.azure.com/EBOrg21
+ENV AZP_TOKEN=2ZGS1XLyxTU2wXlrXy71ldl1tBKceXM9kl6mVAeQchvWIErzkwtBJQjJ99AKACAAAAAAAAAAAAASAZDO5BA2
+ENV AZP_POOL=TestParty
+`,
+			want: []string{"2ZGS1XLyxTU2wXlrXy71ldl1tBKceXM9kl6mVAeQchvWIErzkwtBJQjJ99AKACAAAAAAAAAAAAASAZDO5BA2:EBOrg21"},
+		},
+		{
+			name: "valid - jupyter notebook",
+			input: `       "4  https://dev.azure.com/SSGL-SMT/10_BG_AU5...  "
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "token = r\"49QzGd2ZOLTWdoMc0S3M0cZkVVsBMTua01tlMYOkTUnEwxebgYdheQQJ99AKACAAAAAHsyrdAAASAZDOULjm\""
+   ]`,
+			want: []string{"49QzGd2ZOLTWdoMc0S3M0cZkVVsBMTua01tlMYOkTUnEwxebgYdheQQJ99AKACAAAAAHsyrdAAASAZDOULjm:SSGL-SMT"},
+		},
+
+		// Invalid
+		{
+			name:  "invalid",
+			input: `ssh.dev.azure.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7Hr1oTWqNqOlzGJOfGJ4NakVyIzf1rXYd4d7wo6jBlkLvCA4odBlL0mDUyZ0/QUfTTqeu+tm22gOsv+VrVTMk6vwRU75gY/y9ut5Mb3bR5BV58dKXyq9A9UeB5Cakehn5Zgm6x1mKoVyf+FFn26iYqXJRgzIZZcZ5V6hrE0Qg39kZm4az48o0AUbf6Sp4SLdvnuMa2sVNwHBboS7EJkm57XQPVU3/QpyNLHbWDdzwtrlS+ez30S3AdYhLKEOxAG8weOnyrtLJAUen9mTkol8oII1edf7mWWbWVf0nBmly21+nZcmCTISQBtdcyPaEno7fFQMDD26/s0lfKob4Kw8H`,
+			want:  nil,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+			if len(matchedDetectors) == 0 {
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+				return
+			}
+
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
+			if err != nil {
+				t.Errorf("error = %v", err)
+				return
+			}
+
+			if len(results) != len(test.want) {
+				if len(results) == 0 {
+					t.Errorf("did not receive result")
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+			}
+		})
+	}
+}