diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index ca9e32e..06d38c7 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -20,7 +20,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.18 + go-version: 1.23 - name: Test run: go test -v ./... diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index e0e863e..69fd1b1 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -15,4 +15,4 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: - version: v1.56.2 + version: v1.61 diff --git a/README.md b/README.md index db7f641..4f5ceb4 100644 --- a/README.md +++ b/README.md @@ -28,9 +28,13 @@ You can also combine your CLI arguments and the wizard. All arguments you pass f > **WARNING:** Some passwords or secrets are automatically removed, but this no guarantee, so be careful what you share! -The `--hide` flag can be used multiple times to hide sensitive data, it supports regular expressions. +The `--hide` flag can be used multiple times to hide sensitive data. +As these obfuscators are based on regex, you must add a regex pattern that meets your requirements. -`# support-collector --hide "Secret:.*" --hide "Password:.*"` +`# support-collector --hide "Secret:\s*(.*)"` + +This will replace: +* Values like `Secret: DummyValue` In addition, files and folders that follow a specific pattern are not collected. This affects all files that correspond to the following filters: `.*`, `*~`, `*.key`, `*.csr`, `*.crt`, and `*.pem` diff --git a/go.mod b/go.mod index 778016a..fcd703b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/NETWAYS/support-collector -go 1.20 +go 1.22 require ( github.com/Showmax/go-fqdn v1.0.0 diff --git a/internal/collection/collection.go b/internal/collection/collection.go index d6467c2..55d46b8 100644 --- a/internal/collection/collection.go +++ b/internal/collection/collection.go @@ -3,7 +3,6 @@ package collection import ( "archive/zip" "bytes" - "encoding/json" "fmt" "io" "strings" @@ -142,23 +141,20 @@ func (c *Collection) AddFileYAML(fileName string, data interface{}) { _ = c.AddFileToOutput(file) } +// AddFileJSON will add json data and apply obfuscation func (c *Collection) AddFileJSON(fileName string, data []byte) { - var jsonData interface{} - - err := json.Unmarshal(data, &jsonData) - if err != nil { - c.Log.Debugf("could not unmarshal JSON data for '%s': %s", fileName, err) - } + file := NewFile(fileName) + file.Data = data - prettyJSON, err := json.MarshalIndent(jsonData, "", "") - if err != nil { - c.Log.Debugf("could not marshal JSON data for '%s': %s", fileName, err) - } + _ = c.AddFileToOutput(file) +} +// AddFileJSONRaw will add raw json data without obfuscation +func (c *Collection) AddFileJSONRaw(fileName string, data []byte) { file := NewFile(fileName) - file.Data = prettyJSON + file.Data = data - _ = c.AddFileToOutput(file) + _ = c.AddFileToOutputRaw(file) } func (c *Collection) AddFiles(prefix, source string) { @@ -250,16 +246,19 @@ func (c *Collection) AddJournalLog(fileName, service string) { c.AddCommandOutput(fileName, "journalctl", "-u", service, "-S", c.JournalLoggingInterval) } +// RegisterObfuscator adds the given Obfuscator to the Obfuscators of Collection func (c *Collection) RegisterObfuscator(o *obfuscate.Obfuscator) { c.Obfuscators = append(c.Obfuscators, o) } +// RegisterObfuscators adds the given list of Obfuscator to the Obfuscators of Collection func (c *Collection) RegisterObfuscators(list ...*obfuscate.Obfuscator) { for _, o := range list { c.RegisterObfuscator(o) } } +// ClearObfuscators clears the list of Obfuscators in the Collection func (c *Collection) ClearObfuscators() { c.Obfuscators = c.Obfuscators[:0] } @@ -272,14 +271,17 @@ func (c *Collection) callObfuscators(kind obfuscate.Kind, name string, data []by for _, o := range c.Obfuscators { if o.IsAccepting(kind, name) { - count, out, err = o.Process(data) + count, out, err = o.Process(data, name) if err != nil { return } - if count > 0 { - c.Log.Debugf("Obfuscation replaced %d token in %s", count, name) - } + data = out + } + + if count > 0 { + c.Log.Debugf("ReplacePattern '%s' replaced %d token in %s", o.ReplacePattern, count, name) + count = 0 } } diff --git a/internal/obfuscate/obfuscate.go b/internal/obfuscate/obfuscate.go index 26c75db..a0dc1a1 100644 --- a/internal/obfuscate/obfuscate.go +++ b/internal/obfuscate/obfuscate.go @@ -29,24 +29,24 @@ const ( // Obfuscator provides the basic functionality of an obfuscation engine. // -// Kind filters the variant of resource we want to work on, while Affecting defines a list of regexp.Regexp, to match +// Kind filters the variant of resource we want to work on, while ShouldAffect defines a list of regexp.Regexp, to match // against for the file names, or command. // -// Replacements will be iterated, so all matches or matched groups will be replaced. +// Replacement will be iterated, so all matches or matched groups will be replaced. type Obfuscator struct { Kind - Affecting []*regexp.Regexp - Replacements []*regexp.Regexp - Files uint - Replaced uint + ShouldAffect []*regexp.Regexp + ReplacePattern *regexp.Regexp + ObfuscatedFiles []string + Replaced uint } // New returns a basic Obfuscator with provided regexp.Regexp. func New(kind Kind, affects, replace *regexp.Regexp) *Obfuscator { return &Obfuscator{ - Kind: kind, - Affecting: []*regexp.Regexp{affects}, - Replacements: []*regexp.Regexp{replace}, + Kind: kind, + ShouldAffect: []*regexp.Regexp{affects}, + ReplacePattern: replace, } } @@ -77,27 +77,30 @@ func NewAny(replace string) *Obfuscator { return o } -// WithAffecting adds a new element to the list. +// WithAffecting adds a new pattern to the list where the Obfuscator will be applied func (o *Obfuscator) WithAffecting(a *regexp.Regexp) *Obfuscator { - o.Affecting = append(o.Affecting, a) + o.ShouldAffect = append(o.ShouldAffect, a) return o } -// WithReplacement adds a new element to the list. +// WithReplacement adds a new pattern to the list. func (o *Obfuscator) WithReplacement(r *regexp.Regexp) *Obfuscator { - o.Replacements = append(o.Replacements, r) + o.ReplacePattern = r return o } // IsAccepting checks if we want to work on the resource. +// +// Checks if the Obfuscator.Kind is matching the given kind. If not returns false. +// Checks if the Obfuscator.ShouldAffect is matching the given name. func (o Obfuscator) IsAccepting(kind Kind, name string) bool { if o.Kind != KindAny && o.Kind != kind { return false } - for _, re := range o.Affecting { + for _, re := range o.ShouldAffect { if re.MatchString(name) { return true } @@ -106,16 +109,18 @@ func (o Obfuscator) IsAccepting(kind Kind, name string) bool { return false } -// Process takes data and returns it obfuscated. -func (o *Obfuscator) Process(data []byte) (uint, []byte, error) { - count, out, err := o.ProcessReader(bytes.NewReader(data)) +// Process takes data and returns it obfuscated. Also takes name of file that is obfuscated +// +// Returns count of replaced values as uint, obfuscated data as []byte, and error +func (o *Obfuscator) Process(data []byte, name string) (uint, []byte, error) { + count, out, err := o.ProcessReader(bytes.NewReader(data), name) //goland:noinspection GoNilness return count, out.Bytes(), err } // ProcessReader takes an io.Reader and returns a new one obfuscated. -func (o *Obfuscator) ProcessReader(r io.Reader) (count uint, out bytes.Buffer, err error) { +func (o *Obfuscator) ProcessReader(r io.Reader, name string) (count uint, out bytes.Buffer, err error) { rd := bufio.NewReader(r) var ( @@ -153,7 +158,7 @@ func (o *Obfuscator) ProcessReader(r io.Reader) (count uint, out bytes.Buffer, e // Replace any matches, but skip empty lines if strings.TrimSpace(line) != "" { - line, c = ReplacePatterns(line, o.Replacements) + line, c = ReplacePattern(line, o.ReplacePattern) } // Add line ending back after replacement @@ -168,12 +173,13 @@ func (o *Obfuscator) ProcessReader(r io.Reader) (count uint, out bytes.Buffer, e } if count > 0 { - o.Files++ + o.ObfuscatedFiles = append(o.ObfuscatedFiles, name) } return count, out, err } +/* // ReplacePatterns replaces all the patterns matches in a line. func ReplacePatterns(line string, patterns []*regexp.Regexp) (s string, count uint) { for _, pattern := range patterns { @@ -185,6 +191,7 @@ func ReplacePatterns(line string, patterns []*regexp.Regexp) (s string, count ui return line, count } +*/ // ReplacePattern replaces all matches in a line. func ReplacePattern(line string, pattern *regexp.Regexp) (s string, count uint) { diff --git a/internal/obfuscate/obfuscate_test.go b/internal/obfuscate/obfuscate_test.go index 13927ec..741db7d 100644 --- a/internal/obfuscate/obfuscate_test.go +++ b/internal/obfuscate/obfuscate_test.go @@ -24,7 +24,7 @@ func ExampleObfuscator() { content := []byte(`password = "secret"`) if o.IsAccepting(KindFile, "test.ini") { - count, data, err := o.Process(content) + count, data, err := o.Process(content, "") fmt.Println(err) fmt.Println(count) fmt.Println(string(data)) @@ -64,17 +64,15 @@ func TestObfuscator_IsAccepting(t *testing.T) { func TestObfuscator_Process(t *testing.T) { o := &Obfuscator{ - Replacements: []*regexp.Regexp{ - regexp.MustCompile(`^password\s*=\s*(.*)`), - }, + ReplacePattern: regexp.MustCompile(`^password\s*=\s*(.*)`), } - count, out, err := o.Process([]byte("default content\r\n")) + count, out, err := o.Process([]byte("default content\r\n"), "") assert.NoError(t, err) assert.Equal(t, uint(0), count) assert.Equal(t, "default content\r\n", string(out)) - count, out, err = o.Process([]byte(iniExample)) + count, out, err = o.Process([]byte(iniExample), "") assert.NoError(t, err) assert.Equal(t, uint(1), count) assert.Equal(t, iniResult, string(out)) @@ -83,15 +81,15 @@ func TestObfuscator_Process(t *testing.T) { func TestNewFile(t *testing.T) { o := NewFile(`^password\s*=\s*(.*)`, "conf") assert.Equal(t, KindFile, o.Kind) - assert.Len(t, o.Affecting, 1) - assert.Len(t, o.Replacements, 1) + assert.Len(t, o.ShouldAffect, 1) + assert.NotEmpty(t, o.ShouldAffect) } func TestNewOutput(t *testing.T) { o := NewOutput(`^password\s*=\s*(.*)`, "icinga2", "daemon", "-C") assert.Equal(t, KindOutput, o.Kind) - assert.Len(t, o.Affecting, 1) - assert.Len(t, o.Replacements, 1) + assert.Len(t, o.ShouldAffect, 1) + assert.NotEmpty(t, o.ReplacePattern) assert.True(t, o.IsAccepting(KindOutput, "icinga2 daemon -C")) assert.False(t, o.IsAccepting(KindOutput, "icinga2 daemon")) diff --git a/internal/util/testing.go b/internal/util/testing.go index 070d294..fb3dad1 100644 --- a/internal/util/testing.go +++ b/internal/util/testing.go @@ -19,7 +19,7 @@ func AssertObfuscation(t *testing.T, obfuscators []*obfuscate.Obfuscator, continue } - _, out, err := o.Process([]byte(input)) + _, out, err := o.Process([]byte(input), name) if err != nil { t.Errorf("error during obfuscation: %s", err) return diff --git a/internal/util/util.go b/internal/util/util.go index 31ff33c..73136c4 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -20,6 +20,22 @@ func StringInSlice(a string, list []string) bool { return false } +// DistinctStringSlice returns the given slice with unique values +func DistinctStringSlice(arr []string) []string { + seen := make(map[string]bool) + + var result []string + + for _, val := range arr { + if !seen[val] { + result = append(result, val) + seen[val] = true + } + } + + return result +} + // IsPrivilegedUser returns true when the current user is root. func IsPrivilegedUser() bool { u, err := user.Current() diff --git a/main.go b/main.go index 8a3da96..b583bff 100644 --- a/main.go +++ b/main.go @@ -220,7 +220,7 @@ func main() { c.Log.Warn("cant unmarshal metrics: %w", err) } - c.AddFileJSON("metrics.json", body) + c.AddFileJSONRaw("metrics.json", body) }() if noDetailedCollection { @@ -246,16 +246,19 @@ func main() { c.Log.Infof("Collection complete, took us %.3f seconds", metric.Timings["total"].Seconds()) // Collect obfuscation info - var files, count uint + var ( + count uint + affectedFiles []string + ) for _, o := range c.Obfuscators { - files += o.Files - count += o.Replaced + + affectedFiles = append(affectedFiles, o.ObfuscatedFiles...) } - if files > 0 { - c.Log.Infof("Obfuscation replaced %d token in %d files (%d definitions)", count, files, len(c.Obfuscators)) + if len(affectedFiles) > 0 { + c.Log.Infof("Obfuscation replaced %d token in %d files (%d definitions)", count, len(util.DistinctStringSlice(affectedFiles)), len(c.Obfuscators)) } // get absolute path of outputFile diff --git a/modules/base/kernel.go b/modules/base/kernel.go index b3fe13b..86644df 100644 --- a/modules/base/kernel.go +++ b/modules/base/kernel.go @@ -54,7 +54,7 @@ func CharsToString(chars []int8) string { break } - s[i] = uint8(chars[i]) + s[i] = byte(chars[i]) } return string(s[0:i]) diff --git a/modules/icinga2/api.go b/modules/icinga2/api.go index 98874f6..7a4d2ba 100644 --- a/modules/icinga2/api.go +++ b/modules/icinga2/api.go @@ -60,7 +60,7 @@ func InitAPICollection(c *collection.Collection) error { // endpointIsReachable checks if the given endpoint is reachable within 5 sec func endpointIsReachable(endpoint string) error { - timeout := 5 * time.Second + timeout := 5 * time.Second //nolint:mnd // try to dial tcp connection within 5 seconds conn, err := net.DialTimeout("tcp", endpoint, timeout) @@ -84,7 +84,7 @@ func collectStatus(endpoint string, c *collection.Collection) error { client := &http.Client{Transport: tr} // build context for request - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) //nolint:mnd defer cancel() // build request diff --git a/modules/icinga2/collector.go b/modules/icinga2/collector.go index dc87034..f5f8d90 100644 --- a/modules/icinga2/collector.go +++ b/modules/icinga2/collector.go @@ -1,13 +1,13 @@ package icinga2 import ( + "github.com/NETWAYS/support-collector/internal/obfuscate" "github.com/NETWAYS/support-collector/internal/util" "os" "path/filepath" "regexp" "github.com/NETWAYS/support-collector/internal/collection" - "github.com/NETWAYS/support-collector/internal/obfuscate" ) const ModuleName = "icinga2"