Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: Introduce multiple value headers #358

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+526 −419
Open

feat: Introduce multiple value headers #358

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 66 additions & 68 deletions config/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,57 +4,55 @@
package config

import (
"fmt"
"regexp"
"fmt"
"regexp"

schema "github.com/coreruleset/ftw-tests-schema/v2/types/overrides"

"github.com/coreruleset/go-ftw/ftwhttp"
schema "github.com/coreruleset/ftw-tests-schema/v2/types/overrides"
)

// RunMode represents the mode of the test run
type RunMode string

const (
// CloudRunMode is the string that will be used to override the run mode of execution to cloud
CloudRunMode RunMode = "cloud"
// DefaultRunMode is the default execution run mode
DefaultRunMode RunMode = "default"
// DefaultLogMarkerHeaderName is the default log marker header name
DefaultLogMarkerHeaderName string = "X-CRS-Test"
// DefaultMaxMarkerRetries is the default amount of retries that will be attempted to find the log markers
DefaultMaxMarkerRetries uint = 20
// DefaultMaxMarkerLogLines is the default lines we are going read back in a logfile to find the markers
DefaultMaxMarkerLogLines uint = 500
// CloudRunMode is the string that will be used to override the run mode of execution to cloud
CloudRunMode RunMode = "cloud"
// DefaultRunMode is the default execution run mode
DefaultRunMode RunMode = "default"
// DefaultLogMarkerHeaderName is the default log marker header name
DefaultLogMarkerHeaderName string = "X-CRS-Test"
// DefaultMaxMarkerRetries is the default amount of retries that will be attempted to find the log markers
DefaultMaxMarkerRetries uint = 20
// DefaultMaxMarkerLogLines is the default lines we are going read back in a logfile to find the markers
DefaultMaxMarkerLogLines uint = 500
)

// FTWConfiguration FTW global Configuration
type FTWConfiguration struct {
// Logfile is the path to the file that contains the WAF logs to check. The path may be absolute or relative, in which case it will be interpreted as relative to the current working directory.
LogFile string `koanf:"logfile"`
// PlatformOverrides holds platform specific overrides for tests in the test suite
PlatformOverrides PlatformOverrides `koanf:"platformoverrides"`
// TestOverride holds the test overrides that will apply globally
TestOverride FTWTestOverride `koanf:"testoverride"`
// LogMarkerHeaderName is the name of the header that will be used by the test framework to mark positions in the log file
LogMarkerHeaderName string `koanf:"logmarkerheadername"`
// RunMode stores the mode used to interpret test results. See https://github.com/coreruleset/go-ftw#%EF%B8%8F-cloud-mode.
RunMode RunMode `koanf:"mode"`
// MaxMarkerRetries is the maximum number of times the search for log markers will be repeated; each time an additional request is sent to the web server, eventually forcing the log to be flushed
MaxMarkerRetries uint `koanf:"maxmarkerretries"`
// MaxMarkerLogLines is the maximum number of lines to search for a marker before aborting
MaxMarkerLogLines uint `koanf:"maxmarkerloglines"`
// IncludeTests is a list of tests to include (same as --include)
IncludeTests map[*FTWRegexp]string `koanf:"include"`
// ExcludeTests is a list of tests to exclude (same as --exclude)
ExcludeTests map[*FTWRegexp]string `koanf:"exclude"`
// IncludeTags is a list of tags matching tests to run (same as --tag)
IncludeTags map[*FTWRegexp]string `koanf:"include_tags"`
// Logfile is the path to the file that contains the WAF logs to check. The path may be absolute or relative, in which case it will be interpreted as relative to the current working directory.
LogFile string `koanf:"logfile"`
// PlatformOverrides holds platform specific overrides for tests in the test suite
PlatformOverrides PlatformOverrides `koanf:"platformoverrides"`
// TestOverride holds the test overrides that will apply globally
TestOverride FTWTestOverride `koanf:"testoverride"`
// LogMarkerHeaderName is the name of the header that will be used by the test framework to mark positions in the log file
LogMarkerHeaderName string `koanf:"logmarkerheadername"`
// RunMode stores the mode used to interpret test results. See https://github.com/coreruleset/go-ftw#%EF%B8%8F-cloud-mode.
RunMode RunMode `koanf:"mode"`
// MaxMarkerRetries is the maximum number of times the search for log markers will be repeated; each time an additional request is sent to the web server, eventually forcing the log to be flushed
MaxMarkerRetries uint `koanf:"maxmarkerretries"`
// MaxMarkerLogLines is the maximum number of lines to search for a marker before aborting
MaxMarkerLogLines uint `koanf:"maxmarkerloglines"`
// IncludeTests is a list of tests to include (same as --include)
IncludeTests map[*FTWRegexp]string `koanf:"include"`
// ExcludeTests is a list of tests to exclude (same as --exclude)
ExcludeTests map[*FTWRegexp]string `koanf:"exclude"`
// IncludeTags is a list of tags matching tests to run (same as --tag)
IncludeTags map[*FTWRegexp]string `koanf:"include_tags"`
}

type PlatformOverrides struct {
schema.FTWOverrides
OverridesMap map[uint][]*schema.TestOverride
schema.FTWOverrides
OverridesMap map[uint][]*schema.TestOverride
}

// FTWTestOverride holds four lists:
Expand All @@ -64,54 +62,54 @@ type PlatformOverrides struct {
// ForcePass is for tests you want to pass unconditionally. You should add a comment on why you force to pass the test
// ForceFail is for tests you want to fail unconditionally. You should add a comment on why you force to fail the test
type FTWTestOverride struct {
Overrides Overrides `koanf:"input"`
Ignore map[*FTWRegexp]string `koanf:"ignore"`
ForcePass map[*FTWRegexp]string `koanf:"forcepass"`
ForceFail map[*FTWRegexp]string `koanf:"forcefail"`
Overrides Overrides `koanf:"input"`
Ignore map[*FTWRegexp]string `koanf:"ignore"`
ForcePass map[*FTWRegexp]string `koanf:"forcepass"`
ForceFail map[*FTWRegexp]string `koanf:"forcefail"`
}

// Overrides represents the overridden inputs that have to be applied to tests
type Overrides struct {
DestAddr *string `yaml:"dest_addr,omitempty" koanf:"dest_addr,omitempty"`
Port *int `yaml:"port,omitempty" koanf:"port,omitempty"`
Protocol *string `yaml:"protocol,omitempty" koanf:"protocol,omitempty"`
URI *string `yaml:"uri,omitempty" koanf:"uri,omitempty"`
Version *string `yaml:"version,omitempty" koanf:"version,omitempty"`
Headers ftwhttp.Header `yaml:"headers,omitempty" koanf:"headers,omitempty"`
Method *string `yaml:"method,omitempty" koanf:"method,omitempty"`
Data *string `yaml:"data,omitempty" koanf:"data,omitempty"`
SaveCookie *bool `yaml:"save_cookie,omitempty" koanf:"save_cookie,omitempty"`
// Deprecated: replaced with AutocompleteHeaders
StopMagic *bool `yaml:"stop_magic" koanf:"stop_magic,omitempty"`
AutocompleteHeaders *bool `yaml:"autocomplete_headers" koanf:"autocomplete_headers,omitempty"`
EncodedRequest *string `yaml:"encoded_request,omitempty" koanf:"encoded_request,omitempty"`
RAWRequest *string `yaml:"raw_request,omitempty" koanf:"raw_request,omitempty"`
OverrideEmptyHostHeader *bool `yaml:"override_empty_host_header,omitempty" koanf:"override_empty_host_header,omitempty"`
DestAddr *string `yaml:"dest_addr,omitempty" koanf:"dest_addr,omitempty"`
Port *int `yaml:"port,omitempty" koanf:"port,omitempty"`
Protocol *string `yaml:"protocol,omitempty" koanf:"protocol,omitempty"`
URI *string `yaml:"uri,omitempty" koanf:"uri,omitempty"`
Version *string `yaml:"version,omitempty" koanf:"version,omitempty"`
Headers map[string]string `yaml:"headers,omitempty" koanf:"headers,omitempty"`
Method *string `yaml:"method,omitempty" koanf:"method,omitempty"`
Data *string `yaml:"data,omitempty" koanf:"data,omitempty"`
SaveCookie *bool `yaml:"save_cookie,omitempty" koanf:"save_cookie,omitempty"`
// Deprecated: replaced with AutocompleteHeaders
StopMagic *bool `yaml:"stop_magic" koanf:"stop_magic,omitempty"`
AutocompleteHeaders *bool `yaml:"autocomplete_headers" koanf:"autocomplete_headers,omitempty"`
EncodedRequest *string `yaml:"encoded_request,omitempty" koanf:"encoded_request,omitempty"`
RAWRequest *string `yaml:"raw_request,omitempty" koanf:"raw_request,omitempty"`
OverrideEmptyHostHeader *bool `yaml:"override_empty_host_header,omitempty" koanf:"override_empty_host_header,omitempty"`
}

// FTWRegexp is a wrapper around regexp.Regexp that implements the Unmarshaler interface
type FTWRegexp regexp.Regexp

// UnmarshalText implements the Unmarshaler interface
func (r *FTWRegexp) UnmarshalText(b []byte) error {
re, err := regexp.Compile(string(b))
if err != nil {
return fmt.Errorf("invalid regexp: %w", err)
}
*r = FTWRegexp(*re)
return nil
re, err := regexp.Compile(string(b))
if err != nil {
return fmt.Errorf("invalid regexp: %w", err)
}
*r = FTWRegexp(*re)
return nil
}

// MatchString implements the MatchString method of the regexp.Regexp struct
func (r *FTWRegexp) MatchString(s string) bool {
return (*regexp.Regexp)(r).MatchString(s)
return (*regexp.Regexp)(r).MatchString(s)
}

// NewFTWRegexp creates a new FTWRegexp from a string
func NewFTWRegexp(s string) (*FTWRegexp, error) {
re, err := regexp.Compile(s)
if err != nil {
return nil, fmt.Errorf("invalid regexp: %w", err)
}
return (*FTWRegexp)(re), nil
re, err := regexp.Compile(s)
if err != nil {
return nil, fmt.Errorf("invalid regexp: %w", err)
}
return (*FTWRegexp)(re), nil
}
22 changes: 16 additions & 6 deletions ftwhttp/client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,11 @@ func (s *clientTestSuite) TestGetTrackedTime() {
Version: "HTTP/1.1",
}

h := Header{"Accept": "*/*", "User-Agent": "go-ftw test agent", "Host": "localhost"}
h := NewHeader(map[string][]string{
"Accept": {"*/*"},
"User-Agent": {"go-ftw test agent"},
"Host": {"localhost"},
})

data := []byte(`test=me&one=two&one=twice`)
req := NewRequest(rl, h, data, true)
Expand Down Expand Up @@ -165,10 +169,12 @@ func (s *clientTestSuite) TestClientMultipartFormDataRequest() {
Version: "HTTP/1.1",
}

h := Header{
"Accept": "*/*", "User-Agent": "go-ftw test agent", "Host": "localhost",
"Content-Type": "multipart/form-data; boundary=--------397236876",
}
h := NewHeader(map[string][]string{
"Accept": {"*/*"},
"User-Agent": {"go-ftw test agent"},
"Host": {"localhost"},
"Content-Type": {"multipart/form-data; boundary=--------397236876"},
})

data := []byte(`----------397236876
Content-Disposition: form-data; name="fileRap"; filename="test.txt"
Expand Down Expand Up @@ -250,7 +256,11 @@ func (s *clientTestSuite) TestClientRateLimits() {
Version: "HTTP/1.1",
}

h := Header{"Accept": "*/*", "User-Agent": "go-ftw test agent", "Host": "localhost"}
h := NewHeader(map[string][]string{
"Accept": {"*/*"},
"User-Agent": {"go-ftw test agent"},
"Host": {"localhost"},
})
req := NewRequest(rl, h, nil, true)

// We need to do at least 2 calls so there is a wait between both.
Expand Down
6 changes: 5 additions & 1 deletion ftwhttp/connection_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,11 @@ func (s *connectionTestSuite) TestMultipleRequestTypes() {
Version: "HTTP/1.1",
}

h := Header{"Accept": "*/*", "User-Agent": "go-ftw test agent", "Host": "localhost"}
h := NewHeader(map[string][]string{
"Accept": {"User-Agent"},
"User-Agent": {"go-ftw test agent"},
"Host": {"localhost"},
})

data := []byte(`test=me&one=two`)
req = NewRequest(rl, h, data, true)
Expand Down
Loading