Merge pull request #3510 from semgrep/merge-develop-to-release

Merge Develop into Release

.codemapignore

@@ -10,7 +10,6 @@
 !libsonnet/
 !scripts/
 !stats/
-# restore also fingerprints/ ? trusted_python/ ? 
 
 # do not skip the rules
 ![a-z]*/**/*.yaml

.github/workflows/semgrep-rule-lints.yaml

@@ -45,5 +45,4 @@ jobs:
             --exclude *.test.yaml \
             --exclude contrib/ \
             --exclude stats/ \
-            --exclude fingerprints/ \
             --exclude yaml/semgrep/

.github/workflows/semgrep-rules-test-develop.yml

@@ -1,3 +1,6 @@
+# Running the tests in the repo using `semgrep test` (osemgrep) and
+# the semgrep/semgrep:pro-develop docker image (the bleeding edge!).
+
 name: semgrep-rules-test-develop
 on:
   pull_request:
@@ -9,26 +12,22 @@ on:
     - develop
     - release
 jobs:
-  # Note: if you change this test there will likely need to be a
-  # corresponding change in returntocorp/semgrep
   test-develop:
     name: rules-test-develop
+    # alt: use directly the semgrep/semgrep:pro-develop container here so we
+    # don't need the calls to 'docker run ...' below
     runs-on: ubuntu-20.04
+    # TODO: remove the with: path: below to simplify
     steps:
     - uses: actions/checkout@v2
       with:
         path: semgrep-rules
-    - name: delete stats directory
-      run: rm -rf semgrep-rules/stats
-    - name: delete fingerprints directory
-      run: rm -rf semgrep-rules/fingerprints
-    - name: delete rules requiring Semgrep Pro
-      run: rm -rf semgrep-rules/apex semgrep-rules/elixir
-    - name: validate rules
-      run: |
-        export SEMGREP="docker run --rm -w /src -v ${GITHUB_WORKSPACE}/semgrep-rules:/src returntocorp/semgrep:develop semgrep"
-        make -C "$GITHUB_WORKSPACE"/semgrep-rules validate
-    - name: test with semgrep develop branch
-      run: |
-        export SEMGREP="docker run --rm -w /src -v ${GITHUB_WORKSPACE}/semgrep-rules:/src returntocorp/semgrep:develop semgrep"
-        make -C "$GITHUB_WORKSPACE"/semgrep-rules test-only
+    # alt: call 'make validate' but would require 'make' in the docker image
+    # alt: export SEMGREP="docker run --rm -w ... semgrep"
+    #    make -C "$GITHUB_WORKSPACE"/semgrep-rules validate
+    #TODO: this actually currently fails because of errors in stats/ but GHA
+    # still continue, weird
+    - name: run osemgrep validate --pro
+      run: docker run --rm -w /src -v ${GITHUB_WORKSPACE}/semgrep-rules:/src semgrep/semgrep:pro-develop semgrep validate --pro .
+    - name: run osemgrep test --pro
+      run: docker run --rm -w /src -v ${GITHUB_WORKSPACE}/semgrep-rules:/src semgrep/semgrep:pro-develop semgrep test --pro .

.github/workflows/semgrep-rules-test-historical.yml

@@ -32,8 +32,6 @@ jobs:
       run: pip3 install semgrep
     - name: delete stats directory
       run: rm -rf semgrep-rules/stats
-    - name: delete fingerprints directory
-      run: rm -rf semgrep-rules/fingerprints
     - name: delete rules requiring Semgrep Pro
       run: rm -rf semgrep-rules/apex semgrep-rules/elixir
     # TODO: remove this in the future, there was a regression in semgrep that

.github/workflows/semgrep-rules-test.yml

@@ -17,19 +17,13 @@ jobs:
     - uses: actions/setup-python@v2
       with:
         python-version: 3.9.2
-    - name: install semgrep
+    - name: install semgrep via pip
       run: pip3 install semgrep
     - name: remove stats directory
       run: rm -rf stats
-    - name: remove fingerprints from testing
-      run: rm -rf fingerprints
-    - name: remove .github from testing
-      run: rm -rf .github
-    - name: remove pre-commit-config.yaml
-      run: rm -f .pre-commit-config.yaml
     - name: remove rules requiring Semgrep Pro
       run: rm -rf apex elixir
     - name: validate rules
-      run: semgrep --validate --config .
-    - name: run semgrep
-      run: semgrep --test --test-ignore-todo
+      run: semgrep validate .
+    - name: run semgrep test
+      run: semgrep test .

.github/workflows/trigger-semgrep-scanner-initiate-scan.yaml

@@ -21,7 +21,7 @@ jobs:
         env:
           HEAD_REF: ${{ github.head_ref }}
         run: |
-          CHANGED_FILES=$(git diff --name-only origin/develop origin/$HEAD_REF | xargs -0 | sed '/^$/d' | sed -r '/^(.github|bash|contrib|fingerprints|generic|json|problem-based-packs|scripts|stats|trusted_python|yaml)/d' | sed -n '/.*yaml/p' | tr '\n' ' ')
+          CHANGED_FILES=$(git diff --name-only origin/develop origin/$HEAD_REF | xargs -0 | sed '/^$/d' | sed -r '/^(.github|bash|generic|json|problem-based-packs|scripts|stats|trusted_python|yaml)/d' | sed -n '/.*yaml/p' | tr '\n' ' ')
           echo "changed_files=$CHANGED_FILES" >> $GITHUB_ENV
       - id: print-changed-files
         name: debugging step - print changed files
@@ -52,8 +52,6 @@ jobs:
           HEAD_REF: ${{ github.head_ref }}
           REPO_NAME: ${{ github.event.repository.name }}
           PR_HEAD_SHA: ${{github.event.pull_request.head.sha}}
-        if: |
-          github.event_name == 'pull_request' &&
-          env.changed_lang_count > 0
+        if: github.event_name == 'pull_request' && env.changed_lang_count > 0
         run: |
           curl -X POST https://argoworkflows-dev2.corp.r2c.dev/api/v1/events/security-research/initiate-scan-argo -H "Authorization: ${{ secrets.ARGO_WORKFLOWS_TOKEN }}" -d "{\"branch\" : \"$HEAD_REF\", \"repo\" : \"$REPO_NAME\", \"commit\" : \"$PR_HEAD_SHA\", \"changed_files\" : \"$CHANGED_FILES\" , \"langs\" : \"$CHANGED_LANGS\"}"

Makefile

@@ -1,17 +1,65 @@
 #
 # Check rule validity and check that semgrep finds the expected findings.
+# See https://semgrep.dev/docs/writing-rules/testing-rules for more info.
 #
-# The semgrep repo also runs this as part of its CI for consistency.
+# The semgrep repo (and now semgrep-pro repo) also runs those tests as part
+# of its CI for consistency.
 #
 .PHONY: test
 test:
 	$(MAKE) validate
 	$(MAKE) test-only
 
-.PHONY: validate
-validate:
-	./scripts/run-tests validate
+# Use the SEMGREP env variable to specify a non-standard semgrep command
+SEMGREP ?= semgrep
 
 .PHONY: test-only
+#old: pysemgrep --test was also using flags below but not needed
+# --test-ignore-todo --strict --disable-version-check --metrics=off --verbose
 test-only:
-	./scripts/run-tests test
+	$(SEMGREP) test --pro .
+
+# TODO: semgrep validate use a different targeting than 'semgrep test'
+# so we unfortunately need this whitelist of dirs because it reports
+# errors on stats/ and scripts/ (and .github/workflows/) files otherwise
+# (we also skip libsonnet/ and trusted_python/ which do not contain rules)
+LANG_DIRS=\
+ bash \
+ c \
+ clojure \
+ csharp \
+ dockerfile \
+ generic \
+ go \
+ html \
+ java \
+ javascript \
+ json \
+ kotlin \
+ ocaml \
+ php \
+ python \
+ ruby \
+ rust \
+ scala \
+ solidity \
+ swift \
+ terraform \
+ typescript \
+ yaml
+PRO_DIRS=apex elixir
+OTHER_DIRS=ai problem-based-packs
+DIRS=$(LANG_DIRS) $(PRO_DIRS) $(OTHER_DIRS)
+
+.PHONY: validate
+#old: pysemgrep --validate was also using the flags below but not needed
+# --strict --disable-version-check --metrics=off --verbose
+validate:
+	$(SEMGREP) validate --pro $(DIRS)
+
+.PHONY: test-oss-only
+test-oss-only:
+	@for dir in $(LANG_DIRS) $(OTHER_DIRS); do \
+	   echo "processing $$dir"; \
+           $(SEMGREP) test $$dir; \
+        done

csharp/dotnet/security/use_ecb_mode.cs

@@ -5,7 +5,7 @@ public class Encryption
 {
 	public void EncryptWithAesEcb() {
 		Aes key = Aes.Create();
-		//ruleid: use_ecb_mode
+		//ruleid: deeptodoruleid: use_ecb_mode
 		key.Mode = CipherMode.ECB;
 		using var encryptor = key.CreateEncryptor();
 		byte[] msg = new byte[32];
@@ -15,27 +15,27 @@ public void EncryptWithAesEcb() {
 	public void EncryptWithAesEcb2() {
 		Aes key = Aes.Create();
 		byte[] msg = new byte[32];
-		//ruleid: use_ecb_mode
+		//ruleid: deeptodoruleid: use_ecb_mode
 		var cipherText = key.EncryptEcb(msg, PaddingMode.PKCS7);
 	}
 
 	public void DecryptWithAesEcb(byte[] cipherText) {
 		Aes key = Aes.Create();
-		//ruleid: use_ecb_mode
+		//ruleid: deeptodoruleid: use_ecb_mode
 		key.Mode = CipherMode.ECB;
 		using var decryptor = key.CreateDecryptor();
 		var msg = decryptor.TransformFinalBlock(cipherText, 0, cipherText.Length);
 	}
 
 	public void DecryptWithAesEcb2(byte[] cipherText) {
 		Aes key = Aes.Create();
-		//ruleid: use_ecb_mode
+		//ruleid: deeptodoruleid: use_ecb_mode
 		var msgText = key.DecryptEcb(cipherText, PaddingMode.PKCS7);
 	}
 
 	public void EncryptWith3DESEcb() {
 		TripleDES key = TripleDES.Create();
-		//ruleid: use_ecb_mode
+		//ruleid: deeptodoruleid: use_ecb_mode
 		key.Mode = CipherMode.ECB;
 		using var encryptor = key.CreateEncryptor();
 		byte[] msg = new byte[32];
@@ -45,26 +45,26 @@ public void EncryptWith3DESEcb() {
 	public void EncryptWith3DESEcb2() {
 		TripleDES key = TripleDES.Create();
 		byte[] msg = new byte[32];
-		//ruleid: use_ecb_mode
+		//ruleid: deeptodoruleid: use_ecb_mode
 		var cipherText = key.EncryptEcb(msg, PaddingMode.PKCS7);
 	}
 
 	public void DecryptWith3DESEcb(byte[] cipherText) {
 		TripleDES key = TripleDES.Create();
-		//ruleid: use_ecb_mode
+		//ruleid: deeptodoruleid: use_ecb_mode
 		key.Mode = CipherMode.ECB;
 		using var decryptor = key.CreateDecryptor();
 		var msg = decryptor.TransformFinalBlock(cipherText, 0, cipherText.Length);
 	}
 
 	public void DecryptWith3DESEcb2(byte[] cipherText) {
 		TripleDES key = TripleDES.Create();
-		//ruleid: use_ecb_mode
+		//ruleid: deeptodoruleid: use_ecb_mode
 		var msgText = key.DecryptEcb(cipherText, PaddingMode.PKCS7);
 	}
 
 	public void EncryptWithEcb(SymmetricAlgorithm key) {
-		//ruleid: use_ecb_mode
+		//ruleid: deeptodoruleid: use_ecb_mode
 		key.Mode = CipherMode.ECB;
 		using var encryptor = key.CreateEncryptor();
 		byte[] msg = new byte[32];
@@ -73,19 +73,19 @@ public void EncryptWithEcb(SymmetricAlgorithm key) {
 
 	public void EncryptWithEcb2(SymmetricAlgorithm key) {
 		byte[] msg = new byte[32];
-		//ruleid: use_ecb_mode
+		//ruleid: deeptodoruleid: use_ecb_mode
 		var cipherText = key.EncryptEcb(msg, PaddingMode.PKCS7);
 	}
 
 	public void DecryptWithEcb(SymmetricAlgorithm key, byte[] cipherText) {
-		//ruleid: use_ecb_mode
+		//ruleid: deeptodoruleid: use_ecb_mode
 		key.Mode = CipherMode.ECB;
 		using var decryptor = key.CreateDecryptor();
 		var msg = decryptor.TransformFinalBlock(cipherText, 0, cipherText.Length);
 	}
 
 	public void DecryptWithEcb2(SymmetricAlgorithm key, byte[] cipherText) {
-		//ruleid: use_ecb_mode
+		//ruleid: deeptodoruleid: use_ecb_mode
 		var msgText = key.DecryptEcb(cipherText, PaddingMode.PKCS7);
 	}
 
@@ -124,4 +124,4 @@ public static void Main()
 	{
 		Console.WriteLine("Hello World");
 	}
-}
+}

csharp/dotnet/security/use_weak_rng_for_keygeneration.cs

@@ -8,7 +8,7 @@ public void GenerateBadKey() {
 		byte[] key = new byte[16];
 		rng.NextBytes(key);
 		SymmetricAlgorithm cipher = Aes.Create();
-		// ruleid: use_weak_rng_for_keygeneration
+		// ruleid: deeptodoruleid: use_weak_rng_for_keygeneration
 		cipher.Key = key;
 	}
 

csharp/lang/security/stacktrace-disclosure.cs

@@ -24,3 +24,10 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
             app.UseExceptionHandler("/Error");
         }
 }
+
+public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
+{
+        if (env.IsDevelopment())
+            // ok: stacktrace-disclosure
+            app.UseDeveloperExceptionPage();
+}

csharp/lang/security/stacktrace-disclosure.yaml

@@ -4,8 +4,6 @@ rules:
   - pattern: $APP.UseDeveloperExceptionPage(...);
   - pattern-not-inside: |
       if ($ENV.IsDevelopment(...)) {
-        ... 
-        $APP.UseDeveloperExceptionPage(...); 
         ...
       }
   message: >-

dockerfile/security/dockerd-socket-mount.dockerfile

@@ -0,0 +1,11 @@
+FROM docker:latest
+
+WORKDIR /app
+
+# ruleid: dockerfile-dockerd-socket-mount
+VOLUME /var/run/docker.sock:/var/run/docker.sock
+
+# ok: dockerfile-dockerd-socket-mount
+VOLUME ./app/main.py:/main.py
+
+CMD ["docker", "images"]

dockerfile/security/dockerd-socket-mount.yaml

@@ -0,0 +1,36 @@
+rules:
+  - id: dockerfile-dockerd-socket-mount
+    message: >-
+        The Dockerfile(image) mounts docker.sock to the container which may allow an attacker already inside of the container
+        to escape container and execute arbitrary commands on the host machine.
+    languages:
+      - dockerfile
+      - yaml
+    severity: ERROR
+    metadata:
+      cwe:
+        - "CWE-862: Missing Authorization"
+        - "CWE-269: Improper Privilege Management"
+      confidence: HIGH
+      likelihood: MEDIUM
+      impact: HIGH
+      subcategory:
+        - audit
+      technology:
+        - dockerfile
+      category: security
+      references:
+        - https://cheatsheetseries.owasp.org/cheatsheets/Docker_Security_Cheat_Sheet.html
+        - https://redfoxsec.com/blog/insecure-volume-mounts-in-docker/
+        - https://blog.quarkslab.com/why-is-exposing-the-docker-socket-a-really-bad-idea.html
+    pattern-either:
+      - patterns:
+        - pattern: VOLUME $X
+        - metavariable-regex:
+            metavariable: $X
+            regex: "/var/run/docker.sock"
+      - patterns:
+        - pattern-regex: '- "/var/run/docker.sock:.*"'
+        - pattern-inside: |
+            volumes:
+              ...

dockerfile/security/missing-user-entrypoint.dockerfile

@@ -9,6 +9,5 @@ RUN pip3 install semgrep
 # ruleid: missing-user-entrypoint
 ENTRYPOINT semgrep -f p/xss
 
-# TODO: metavar bug
-# ok: missing-user-entrypoint
+# ruleid: missing-user-entrypoint
 ENTRYPOINT ["semgrep", "--config", "localfile", "targets"]

dockerfile/security/missing-user-entrypoint.fixed.dockerfile

@@ -10,6 +10,6 @@ RUN pip3 install semgrep
 USER non-root
 ENTRYPOINT semgrep -f p/xss
 
-# TODO: metavar bug
-# ok: missing-user-entrypoint
+# ruleid: missing-user-entrypoint
+USER non-root
 ENTRYPOINT ["semgrep", "--config", "localfile", "targets"]

dockerfile/security/missing-user.dockerfile

@@ -12,6 +12,5 @@ CMD semgrep -f p/xss
 # ruleid: missing-user
 CMD semgrep --config localfile targets
 
-# TODO: metavar ellipses bug
-# ok: missing-user
+# ruleid: missing-user
 CMD ["semgrep", "--version"]

dockerfile/security/missing-user.fixed.dockerfile

@@ -14,6 +14,6 @@ CMD semgrep -f p/xss
 USER non-root
 CMD semgrep --config localfile targets
 
-# TODO: metavar ellipses bug
-# ok: missing-user
+# ruleid: missing-user
+USER non-root
 CMD ["semgrep", "--version"]