chore: upgrades to CRS 4.1. (#1032)

* chore: upgrades to CRS 4.1.

* make httpbin-emulator more flexible, lists failing tests

* nit/wip: describe failing tests

* updates .ftw.yml

---------

Co-authored-by: Matteo Pace <[email protected]>

Author: jcchavezs

testing/coreruleset/.ftw.yml

@@ -7,5 +7,6 @@ testoverride:
     920270-4: 'Rule works, log contains 920270. Test expects status 400 (Apache behaviour)'
     920272-5: 'Rule works, log contains 920272. Test expects status 400 (Apache behaviour)'
     920290-1: 'Rule works, log contains 920290. Test expects status 400 (Apache behaviour)'
-    920430-8: 'Go/http does no allow HTTP/3.0 - 505 HTTP Version Not Supported'
+    920290-4: 'Go/http returns 400 Bad Request: missing required Host header'
+    920430-8: 'Go/http does not allow HTTP/3.0 - 505 HTTP Version Not Supported'
     932200-13: 'wip'

testing/coreruleset/coreruleset_test.go

@@ -29,8 +29,8 @@ import (
 	"github.com/coreruleset/go-ftw/test"
 	"github.com/rs/zerolog"
 
-	coreruleset "github.com/corazawaf/coraza-coreruleset"
-	crstests "github.com/corazawaf/coraza-coreruleset/tests"
+	coreruleset "github.com/corazawaf/coraza-coreruleset/v4"
+	crstests "github.com/corazawaf/coraza-coreruleset/v4/tests"
 	"github.com/corazawaf/coraza/v3"
 	txhttp "github.com/corazawaf/coraza/v3/http"
 	"github.com/corazawaf/coraza/v3/types"
@@ -234,11 +234,16 @@ SecRule REQUEST_HEADERS:X-CRS-Test "@rx ^.*$" \
 				}
 				urldecodedBody, err := url.QueryUnescape(string(body))
 				if err != nil {
-					t.Fatalf("handler can not unescape urlencoded request body: %v", err)
+					t.Logf("[warning] handler can not unescape urlencoded request body: %v", err)
+					// If the body can't be unescaped, we will keep going with the received body
+					urldecodedBody = string(body)
 				}
-				fmt.Fprintf(w, urldecodedBody)
+				fmt.Fprint(w, urldecodedBody)
 			} else {
 				_, err = w.Write(body)
+				if err != nil {
+					t.Fatalf("handler can not write request body: %v", err)
+				}
 			}
 
 		case strings.HasPrefix(r.URL.Path, "/base64/"):
@@ -247,10 +252,10 @@ SecRule REQUEST_HEADERS:X-CRS-Test "@rx ^.*$" \
 			if err != nil {
 				t.Fatalf("handler can not decode base64: %v", err)
 			}
-			fmt.Fprintf(w, string(b64Decoded))
+			fmt.Fprint(w, string(b64Decoded))
 		default:
 			// Common path "/status/200" defaults here
-			fmt.Fprintf(w, "Hello!")
+			fmt.Fprint(w, "Hello!")
 		}
 	})))
 	defer s.Close()

testing/coreruleset/go.mod

@@ -4,7 +4,7 @@ go 1.20
 
 require (
 	github.com/bmatcuk/doublestar/v4 v4.6.1
-	github.com/corazawaf/coraza-coreruleset v0.0.0-20231103220038-fd5c847140a6
+	github.com/corazawaf/coraza-coreruleset/v4 v4.1.0
 	github.com/corazawaf/coraza/v3 v3.0.4
 	github.com/coreruleset/go-ftw v0.6.4
 	github.com/rs/zerolog v1.32.0

testing/coreruleset/go.sum

@@ -6,8 +6,8 @@ github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZC
 github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
 github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I=
 github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
-github.com/corazawaf/coraza-coreruleset v0.0.0-20231103220038-fd5c847140a6 h1:MjSFYff3j1L4zo3MNuqnQ19Jp5ps/sibntdtS/Kq/yk=
-github.com/corazawaf/coraza-coreruleset v0.0.0-20231103220038-fd5c847140a6/go.mod h1:7rsocqNDkTCira5T0M7buoKR2ehh7YZiPkzxRuAgvVU=
+github.com/corazawaf/coraza-coreruleset/v4 v4.1.0 h1:LJAitZ8DszT5eX+3VbGVg9PfsLEMpzPbARedIFURQSM=
+github.com/corazawaf/coraza-coreruleset/v4 v4.1.0/go.mod h1:RQMGurig+irQq7v21yq7rM/9SAEf1bT6hCSplJ0ByKY=
 github.com/corazawaf/coraza/v3 v3.0.4 h1:Llemgoh0hp2NggCwcWN8lNiV4Pfe+AWzf1oEcasT234=
 github.com/corazawaf/coraza/v3 v3.0.4/go.mod h1:3fTYjY5BZv3nezLpH6NAap0gr3jZfbQWUAu2GF17ET4=
 github.com/corazawaf/libinjection-go v0.1.3 h1:PUplAYho1BBl0tIVbhDsNRuVGIeUYSiCEc9oQpb2rJU=
@@ -18,26 +18,21 @@ github.com/coreruleset/ftw-tests-schema v1.1.0/go.mod h1:gRd9wBxjUI85HypWRDxJzbk
 github.com/coreruleset/go-ftw v0.6.4 h1:EdDNld38Jv4lxqHS+csGOJuHu1/8rpp4TlrFyoijTPk=
 github.com/coreruleset/go-ftw v0.6.4/go.mod h1:IayMjfOmmNNBcqTcZU92e6UZTy79/eFdmJEmRu8tLs4=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI=
 github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
-github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
-github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
-github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c=
 github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/goccy/go-yaml v1.11.3 h1:B3W9IdWbvrUu2OYQGwvU1nZtvMQJPBKgBUuweJjLj6I=
 github.com/goccy/go-yaml v1.11.3/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU=
@@ -63,7 +58,6 @@ github.com/knadh/koanf/v2 v2.1.0/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkX
 github.com/kyokomi/emoji/v2 v2.2.12 h1:sSVA5nH9ebR3Zji1o31wu3yOwD1zKXQA2z0zUyeit60=
 github.com/kyokomi/emoji/v2 v2.2.12/go.mod h1:JUcn42DTdsXJo1SWanHh4HKDEyPaR5CqkmoirZZP9qE=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
-github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg=
 github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -82,12 +76,10 @@ github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4 h1:
 github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0=
 github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
 github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=