feat(ci): add lint and integration/regression test

.github/workflows/integration.yaml

@@ -0,0 +1,67 @@
+---
+name: Integration/Regression
+
+on: [push, pull_request]
+
+jobs:
+  modsecurity-test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+
+    steps:
+      - name: "Checkout repo"
+        uses: actions/checkout@v2
+
+      - name: "Install dependencies"
+        run: |
+          wget https://github.com/coreruleset/coreruleset/archive/refs/tags/nightly.tar.gz -O /tmp/nightly.tar.gz
+          tar -zxvf /tmp/nightly.tar.gz -C /tmp/
+          curl -skLo - https://github.com/coreruleset/go-ftw/releases/latest/download/ftw_Linux_x86_64.tar.gz | tar -xzf - ftw
+
+      - name: "Test plugin on ModSecurity v2 / Apache"
+        run: |
+          touch plugins/placeholder-config.conf
+          touch plugins/placeholder-before.conf
+          touch plugins/placeholder-after.conf
+
+          docker-compose -f tests/integration/docker-compose.yml --project-directory . up -d apache-nightly
+          echo "waiting for the webserver to start"; sleep 10
+
+          ./ftw check -d tests/regression/tests
+          ./ftw run -d tests/regression/tests
+
+          docker-compose -f tests/integration/docker-compose.yml --project-directory . down
+        env:
+          FTW_LOGFILE: './tests/logs/modsec2-apache/error.log'
+          FTW_LOGTYPE_NAME: 'apache'
+          FTW_LOGTYPE_TIMEREGEX: '\[([A-Z][a-z]{2} [A-z][a-z]{2} \d{1,2} \d{1,2}\:\d{1,2}\:\d{1,2}\.\d+? \d{4})\]'
+          FTW_LOGTYPE_TIMEFORMAT: 'ddd MMM DD HH:mm:ss.S YYYY'
+
+      - name: "Test plugin on ModSecurity v3 / Nginx"
+        # test on Nginx temp disabled
+        if: ${{ false }}
+        run: |
+          touch plugins/placeholder-config.conf
+          touch plugins/placeholder-before.conf
+          touch plugins/placeholder-after.conf
+
+          docker-compose -f tests/integration/docker-compose.yml --project-directory . up -d nginx-nightly
+          echo "waiting for the webserver to start"; sleep 10
+
+          docker cp tests/integration/modsec-setup.conf nginx-nightly:/etc/modsecurity.d/setup.conf
+          echo $'SecDefaultAction "phase:1,pass,log,tag:\'modsecurity\'"\nSecDefaultAction "phase:2,pass,log,tag:\'modsecurity\'"\nSecAction "id:900000,phase:1,nolog,pass,t:none,setvar:tx.blocking_paranoia_level=4"\nSecAction "id:900110,phase:1,nolog,pass,t:none,setvar:tx.inbound_anomaly_score_threshold=5,setvar:tx.outbound_anomaly_score_threshold=4"\nSecAction "id:900350,phase:1,nolog,pass,t:none,setvar:tx.combined_file_sizes=65535"\nSecCollectionTimeout 600\nSecAction "id:900990,phase:1,nolog,pass,t:none,setvar:tx.crs_setup_version=400"\nSecRule REQUEST_HEADERS:X-CRS-Test "@rx ^.*$" "id:999999,phase:1,log,msg:\'%{MATCHED_VAR}\',pass,t:none"' > /tmp/crs-setup.conf
+          docker cp /tmp/crs-setup.conf nginx-nightly:/etc/modsecurity.d/owasp-crs/crs-setup.conf
+          docker exec nginx-nightly nginx -s reload
+
+          echo "waiting nginx reload"; sleep 5
+
+          ./ftw check -d tests/regression/tests
+          ./ftw run -d tests/regression/tests
+
+          docker-compose -f tests/integration/docker-compose.yml --project-directory . down
+        env:
+          FTW_LOGFILE: './tests/logs/modsec3-nginx/error.log'
+          FTW_LOGTYPE_NAME: 'apache'
+          FTW_LOGTYPE_TIMEREGEX: '\[([A-Z][a-z]{2} [A-z][a-z]{2} \d{1,2} \d{1,2}\:\d{1,2}\:\d{1,2}\.\d+? \d{4})\]'
+          FTW_LOGTYPE_TIMEFORMAT: 'ddd MMM DD HH:mm:ss.S YYYY'

.github/workflows/lint.yaml

@@ -0,0 +1,36 @@
+---
+name: Lint
+
+on: [push, pull_request]
+
+jobs:
+  check-syntax:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+    # check why is failing and change afterwards
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+
+      - name: Lint Yaml
+        uses: ibiqlik/action-yamllint@v3
+        with:
+          format: github
+          file_or_dir: tests/regression/tests
+          config_file: .yamllint.yml
+
+      - name: Linelint
+        uses: fernandrone/linelint@master
+        id: linelint
+
+      - name: Set up Python 3
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: "Check Plugin syntax"
+        run: |
+          pip install --upgrade setuptools
+          pip install secrules-parsing
+          secrules-parser -c --output-type github -f plugins/*.conf

.gitignore

@@ -0,0 +1,3 @@
+tests/logs/modsec2-apache/*
+tests/logs/modsec3-nginx/*
+!.gitkeep

.linelint.yml

@@ -0,0 +1,12 @@
+rules:
+  # checks if file ends in a newline character
+  end-of-file:
+    # set to true to enable this rule
+    enable: true
+
+    # set to true to disable autofix (if enabled globally)
+    disable-autofix: true
+
+    # will be ignored only by this rule
+    ignore:
+      - .pytest_cache/*

.yamllint.yml

@@ -0,0 +1,18 @@
+extends: default
+
+rules:
+  # Test lines can be big
+  line-length:
+    max: 1024
+    level: warning
+    # These files below have very large lines, needed for the test.
+    # So they will raise warnings every time.
+    #ignore: |
+    #  tests/regression/tests/foo/1234.yaml
+
+  # don't bother me with this rule
+  indentation: disable
+
+  comments:
+    require-starting-space: true # default
+    min-spaces-from-content: 1

tests/integration/docker-compose.yml

@@ -0,0 +1,80 @@
+version: "3.9"
+
+services:
+  apache-nightly:
+    container_name: apache-nightly
+    image: owasp/modsecurity-crs:apache
+    depends_on:
+      - backend
+    environment:
+      SERVERNAME: "_default_"
+      BACKEND: "http://backend:8080"
+      PORT: "80"
+      MODSEC_RULE_ENGINE: "DetectionOnly"
+      BLOCKING_PARANOIA: 4
+      TZ: "${TZ}"
+      ERRORLOG: "/var/log/error.log"
+      ACCESSLOG: "/var/log/access.log"
+      MODSEC_AUDIT_LOG_FORMAT: Native
+      MODSEC_AUDIT_LOG_TYPE: Serial
+      MODSEC_AUDIT_LOG: "/var/log/modsec_audit.log"
+      MODSEC_TMP_DIR: "/tmp"
+      MODSEC_RESP_BODY_ACCESS: "On"
+      MODSEC_RESP_BODY_MIMETYPE: "text/plain text/html text/xml application/json"
+      COMBINED_FILE_SIZES: "65535"
+      CRS_ENABLE_TEST_MARKER: 1
+    ports:
+      - 80:80
+    volumes:
+      - /tmp/coreruleset-nightly/rules:/opt/owasp-crs/rules:ro
+      - /tmp/coreruleset-nightly/crs-setup.conf.example:/etc/modsecurity.d/owasp-crs/crs-setup.conf.example
+      - ./tests/logs/modsec2-apache:/var/log:rw
+      - ./tests/integration/modsec-setup.conf:/etc/modsecurity.d/setup.conf:ro
+      - ./plugins:/etc/modsecurity.d/plugins
+    entrypoint: ["/bin/sh", "-c", "/bin/cp /etc/modsecurity.d/owasp-crs/crs-setup.conf.example /etc/modsecurity.d/owasp-crs/crs-setup.conf && /docker-entrypoint.sh && apachectl -D FOREGROUND"]
+    networks:
+      - crs-plugins-net
+
+  nginx-nightly:
+    container_name: nginx-nightly
+    image: owasp/modsecurity-crs:nginx
+    depends_on:
+      - backend
+    environment:
+      SERVERNAME: _
+      BACKEND: "http://backend:8080"
+      PORT: "80"
+      MODSEC_RULE_ENGINE: DetectionOnly
+      BLOCKING_PARANOIA: 4
+      TZ: "${TZ}"
+      ERRORLOG: "/var/log/nginx/error.log"
+      LOGLEVEL: "info"
+      ACCESSLOG: "/var/log/nginx/access.log"
+      MODSEC_AUDIT_LOG_FORMAT: "Native"
+      MODSEC_AUDIT_LOG_TYPE: "Serial"
+      MODSEC_AUDIT_LOG: "/var/log/modsec_audit.log"
+      MODSEC_RESP_BODY_ACCESS: "On"
+      MODSEC_RESP_BODY_MIMETYPE: "text/plain text/html text/xml application/json"
+      COMBINED_FILE_SIZES: "65535"
+      CRS_ENABLE_TEST_MARKER: 1
+    ports:
+      - 80:80
+    volumes:
+      - /tmp/coreruleset-nightly/rules:/opt/owasp-crs/rules:ro
+      - /tmp/coreruleset-nightly/crs-setup.conf.example:/etc/modsecurity.d/owasp-crs/crs-setup.conf.example
+      - ./tests/logs/modsec3-nginx:/var/log/nginx:rw
+      - ./plugins:/etc/modsecurity.d/plugins
+    #entrypoint: ["/bin/sh", "-c", "/bin/cp /etc/modsecurity.d/owasp-crs/crs-setup.conf.example /etc/modsecurity.d/owasp-crs/crs-setup.conf && /docker-entrypoint.sh && nginx -g 'daemon off;'"]
+    networks:
+      - crs-plugins-net
+
+  backend:
+    image: eexit/mirror-http-server #docker.io/kennethreitz/httpbin
+    networks:
+      - crs-plugins-net
+
+volumes:
+  logs:
+networks:
+  crs-plugins-net:
+    driver: bridge

tests/integration/modsec-setup.conf

@@ -0,0 +1,13 @@
+# Allow custom rules to be specified in:
+# /opt/modsecurity/rules/{before,after}-crs/*.conf
+
+Include /etc/modsecurity.d/modsecurity.conf
+
+Include /etc/modsecurity.d/owasp-crs/crs-setup.conf
+
+Include /etc/modsecurity.d/plugins/*-config.conf
+Include /etc/modsecurity.d/plugins/*-before.conf
+
+Include /etc/modsecurity.d/owasp-crs/rules/*.conf
+
+Include /etc/modsecurity.d/plugins/*-after.conf

tests/regression/tests/before/9507100.yaml

@@ -0,0 +1,39 @@
+---
+meta:
+  author: "Andrea Menin"
+  description: "Test WordPress Exclusion Plugin"
+  enabled: true
+  name: 9507100.yaml
+tests:
+  - test_title: 9507100-1
+    desc: Check FPs on pwd ARGS
+    stages:
+      - stage:
+          input:
+            dest_addr: 127.0.0.1
+            headers:
+              Host: localhost
+              User-Agent: OWASP ModSecurity Core Rule Set
+              Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
+            port: 80
+            method: POST
+            uri: /wp-login.php
+            data: log=admin&pwd=exec%28%2Fbin%2Fbash%29%3B
+          output:
+            no_log_contains: id "932160"
+  - test_title: 9507100-2
+    desc: Check FPs on pwd ARGS
+    stages:
+      - stage:
+          input:
+            dest_addr: 127.0.0.1
+            headers:
+              Host: localhost
+              User-Agent: OWASP ModSecurity Core Rule Set
+              Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
+            port: 80
+            method: POST
+            uri: /wp-login.php
+            data: log=admin&notpwd=exec%28%2Fbin%2Fbash%29%3B
+          output:
+            log_contains: id "932160"

tests/regression/tests/before/9507120.yaml

@@ -0,0 +1,39 @@
+---
+meta:
+  author: "Andrea Menin"
+  description: "Test WordPress Exclusion Plugin"
+  enabled: true
+  name: 9507100.yaml
+tests:
+  - test_title: 9507120-1
+    desc: Check FPs on reset password
+    stages:
+      - stage:
+          input:
+            dest_addr: 127.0.0.1
+            headers:
+              Host: localhost
+              User-Agent: OWASP ModSecurity Core Rule Set
+              Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
+            port: 80
+            method: POST
+            uri: /wp-login.php?action=resetpass
+            data: nopass1=exec%28%2Fbin%2Fbash%29%3B&nopass1-text=exec%28%2Fbin%2Fbash%29%3B&nopass2=exec%28%2Fbin%2Fbash%29%3B
+          output:
+            log_contains: id "932160"
+  - test_title: 9507120-2
+    desc: Check FPs on reset password
+    stages:
+      - stage:
+          input:
+            dest_addr: 127.0.0.1
+            headers:
+              Host: localhost
+              User-Agent: OWASP ModSecurity Core Rule Set
+              Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
+            port: 80
+            method: POST
+            uri: /wp-login.php?action=resetpass
+            data: pass1=exec%28%2Fbin%2Fbash%29%3B&pass1-text=exec%28%2Fbin%2Fbash%29%3B&pass2=exec%28%2Fbin%2Fbash%29%3B
+          output:
+            no_log_contains: id "932160"

tests/regression/tests/before/9507130.yaml

@@ -0,0 +1,37 @@
+---
+meta:
+  author: "Andrea Menin"
+  description: "Test WordPress Exclusion Plugin"
+  enabled: true
+  name: 9507130.yaml
+tests:
+  - test_title: 9507130-1
+    desc: Check FPs on comment
+    stages:
+      - stage:
+          input:
+            dest_addr: 127.0.0.1
+            headers:
+              Host: localhost
+              User-Agent: OWASP ModSecurity Core Rule Set
+              Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
+            port: 80
+            method: GET
+            uri: /?url=ftp://foo.bar
+          output:
+            log_contains: id "931130"
+  - test_title: 9507130-2
+    desc: Check FPs on comment
+    stages:
+      - stage:
+          input:
+            dest_addr: 127.0.0.1
+            headers:
+              Host: localhost
+              User-Agent: OWASP ModSecurity Core Rule Set
+              Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
+            port: 80
+            method: GET
+            uri: /wp-comments-post.php?url=ftp://foo.bar
+          output:
+            no_log_contains: id "931130"