From c30198feef3fd1ad6e1c15799ef90084370dea1c Mon Sep 17 00:00:00 2001 From: Alik-Kold Date: Tue, 7 Mar 2023 13:50:32 +0000 Subject: [PATCH] feat: add urls (http+https) to json report closes #585 --- .gitignore | 2 +- cmd/analyze/Dockerfile | 1 + cmd/analyze/main.go | 34 +++++++++++++ internal/dynamicanalysis/analysis.go | 76 ++++++++++++++++++++++++++-- internal/sandbox/sandbox.go | 20 +++++++- internal/worker/rundynamic.go | 2 +- pkg/api/analysisrun/result.go | 1 + sandboxes/dynamicanalysis/Dockerfile | 3 ++ tools/network/iptables.rules | 11 ++++ 9 files changed, 144 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 3edefd94..9b3a0356 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ *.dll *.so *.dylib - +.idea # Test binary, built with `go test -c` *.test diff --git a/cmd/analyze/Dockerfile b/cmd/analyze/Dockerfile index 10394b12..c3f8dd72 100644 --- a/cmd/analyze/Dockerfile +++ b/cmd/analyze/Dockerfile @@ -21,6 +21,7 @@ RUN apt-get update && apt-get upgrade -y && \ iptables \ iproute2 \ podman \ + sslsplit \ software-properties-common && \ update-alternatives --set iptables /usr/sbin/iptables-legacy && \ update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy diff --git a/cmd/analyze/main.go b/cmd/analyze/main.go index 2f87c36f..1aac74a8 100644 --- a/cmd/analyze/main.go +++ b/cmd/analyze/main.go @@ -6,6 +6,7 @@ import ( "fmt" "net/url" "os" + "os/exec" "strings" "github.com/ossf/package-analysis/internal/analysis" @@ -195,6 +196,11 @@ func main() { "error", err) } + err = generateSSLCerts() + if err != nil { + log.Panic("Error creating certificates") + } + if runMode[analysis.Static] { log.Info("Starting static analysis") staticAnalysis(pkg) @@ -206,3 +212,31 @@ func main() { dynamicAnalysis(pkg) } } + +func generateSSLCerts() error { + log.Debug("Generating SSLSplit CA certificates") + createSslCerts := exec.Command("openssl", "genrsa", "-out", "/proxy/certs/ca.pem", "2048") + createSslCerts.Dir = "/proxy/certs" + err := createSslCerts.Start() + + if err != nil { + return err + } + err = createSslCerts.Wait() + if err != nil { + return err + } + + createSslCerts = exec.Command("openssl", "req", "-new", "-nodes", "-x509", "-sha256", "-out", "ca.crt", "-key", "ca.pem", "-addext", "authorityKeyIdentifier=keyid:always,issuer:always", "-subj", "/O=Package Analysis CA/CN=Package Analysis CA/", "-set_serial", "0", "-days", "3650") + createSslCerts.Dir = "/proxy/certs" + err = createSslCerts.Start() + + if err != nil { + return err + } + err = createSslCerts.Wait() + if err != nil { + return err + } + return nil +} diff --git a/internal/dynamicanalysis/analysis.go b/internal/dynamicanalysis/analysis.go index 79c314ce..ee208557 100644 --- a/internal/dynamicanalysis/analysis.go +++ b/internal/dynamicanalysis/analysis.go @@ -2,7 +2,6 @@ package dynamicanalysis import ( "fmt" - "github.com/ossf/package-analysis/internal/analysis" "github.com/ossf/package-analysis/internal/dnsanalyzer" "github.com/ossf/package-analysis/internal/log" @@ -10,6 +9,10 @@ import ( "github.com/ossf/package-analysis/internal/sandbox" "github.com/ossf/package-analysis/internal/strace" "github.com/ossf/package-analysis/pkg/api/analysisrun" + "io/ioutil" + "os/exec" + "strings" + "syscall" ) const ( @@ -20,6 +23,7 @@ const ( type Result struct { StraceSummary analysisrun.StraceSummary FileWrites analysisrun.FileWrites + URLs []string } var resultError = &Result{ @@ -28,6 +32,24 @@ var resultError = &Result{ }, } +func ParseURLsFromSSlStripOutput(content []byte) []string { + var result []string + ret := strings.Split(string(content), "\n") + + for _, value := range ret { + lineParts := strings.Split(value, " ") + if len(lineParts) < 11 { + continue + } + + schema := lineParts[3] + host := lineParts[8] + path := lineParts[10] + result = append(result, schema+"://"+host+path) + } + return result +} + func Run(sb sandbox.Sandbox, args []string) (*Result, error) { log.Info("Running dynamic analysis", "args", args) @@ -45,11 +67,58 @@ func Run(sb sandbox.Sandbox, args []string) (*Result, error) { // Run the command log.Debug("Running dynamic analysis command", "args", args) + + log.Debug("Reroute all http traffic through sslsplit") + iptables := exec.Command("iptables", "-t", "nat", "-A", "PREROUTING", "-i", "cni-analysis", "-p", "tcp", "--dport", "80", "-j", "REDIRECT", "--to-port", "8081") + err := iptables.Start() + + if err != nil { + log.Fatal(err.Error()) + } + err = iptables.Wait() + if err != nil { + log.Fatal(err.Error()) + } + + log.Debug("Reroute all https traffic through sslsplit") + iptables = exec.Command("iptables", "-t", "nat", "-A", "PREROUTING", "-i", "cni-analysis", "-p", "tcp", "--dport", "443", "-j", "REDIRECT", "--to-port", "8080") + err = iptables.Start() + + if err != nil { + log.Fatal(err.Error()) + } + err = iptables.Wait() + if err != nil { + log.Fatal(err.Error()) + } + + log.Debug("starting sslsplit") + sslsplit := exec.Command("sslsplit", "-d", "-L", "/tmp/ssl.flow", "-l", "/tmp/sslLinks.flow", "-k", "/proxy/certs/ca.pem", "-c", "/proxy/certs/ca.crt", "http", "0.0.0.0", "8081", "https", "0.0.0.0", "8080") + err = sslsplit.Start() + + if err != nil { + log.Fatal(err.Error()) + } + r, err := sb.Run(args...) if err != nil { return resultError, fmt.Errorf("sandbox failed (%w)", err) } + log.Debug("stopping sslsplit") + err = sslsplit.Process.Signal(syscall.SIGINT) + if err != nil { + return nil, err + } + + log.Debug("reading sslsplit results") + body, err1 := ioutil.ReadFile("/tmp/sslLinks.flow") + if err1 != nil { + log.Fatal("unable to read file: %v", err1) + } + + urls := ParseURLsFromSSlStripOutput(body) + log.Debug("Stop the packet capture") pcap.Close() @@ -73,11 +142,11 @@ func Run(sb sandbox.Sandbox, args []string) (*Result, error) { Stderr: lastLines(r.Stderr(), maxOutputLines, maxOutputBytes), }, } - analysisResult.setData(straceResult, dns) + analysisResult.setData(straceResult, dns, urls) return &analysisResult, nil } -func (d *Result) setData(straceResult *strace.Result, dns *dnsanalyzer.DNSAnalyzer) { +func (d *Result) setData(straceResult *strace.Result, dns *dnsanalyzer.DNSAnalyzer, sslSplitResult []string) { for _, f := range straceResult.Files() { d.StraceSummary.Files = append(d.StraceSummary.Files, analysisrun.FileResult{ Path: f.Path, @@ -121,4 +190,5 @@ func (d *Result) setData(straceResult *strace.Result, dns *dnsanalyzer.DNSAnalyz } d.StraceSummary.DNS = append(d.StraceSummary.DNS, c) } + d.URLs = sslSplitResult } diff --git a/internal/sandbox/sandbox.go b/internal/sandbox/sandbox.go index 6c16d93b..eac71d33 100644 --- a/internal/sandbox/sandbox.go +++ b/internal/sandbox/sandbox.go @@ -89,7 +89,7 @@ type Sandbox interface { // The result of the supplied command will be returned in an instance of // RunResult. Run(...string) (*RunResult, error) - + UploadFileToContainer(srcFile string, destFile string) *exec.Cmd // Clean cleans up a Sandbox. // // Once called the Sandbox cannot be used again. @@ -127,6 +127,16 @@ type podmanSandbox struct { volumes []volume } +func (s *podmanSandbox) UploadFileToContainer(srcFile string, destFile string) *exec.Cmd { + destParam := fmt.Sprintf("%s:%s", s.container, destFile) + args := []string{ + "cp", + srcFile, + destParam, + } + return podman(args...) +} + type ( Option interface{ set(*podmanSandbox) } option func(*podmanSandbox) // option implements Option. @@ -428,6 +438,14 @@ func (s *podmanSandbox) Run(args ...string) (*RunResult, error) { } errWriter := io.MultiWriter(errWriters...) + log.Debug("upload certs to container") + uploadCmd := s.UploadFileToContainer("/proxy/certs/ca.crt", "/usr/local/share/ca-certificates/ca.crt") //upload certs to container + uploadCmd.Stdout = logOut + uploadCmd.Stderr = logErr + if err := uploadCmd.Run(); err != nil { + return result, fmt.Errorf("error uploading file to container: %w", err) + } + // Start the container startCmd := s.startContainerCmd(logDir) startCmd.Stdout = logOut diff --git a/internal/worker/rundynamic.go b/internal/worker/rundynamic.go index 9a3de6c3..5a7b3760 100644 --- a/internal/worker/rundynamic.go +++ b/internal/worker/rundynamic.go @@ -77,7 +77,7 @@ func RunDynamicAnalysis(pkg *pkgmanager.Pkg, sbOpts []sandbox.Option) (analysisr results.StraceSummary[phase] = &phaseResult.StraceSummary results.FileWrites[phase] = &phaseResult.FileWrites lastStatus = phaseResult.StraceSummary.Status - + results.URLs = phaseResult.URLs if lastStatus != analysis.StatusCompleted { // Error caused by an issue with the package (probably). // Don't continue with phases if this one did not complete successfully. diff --git a/pkg/api/analysisrun/result.go b/pkg/api/analysisrun/result.go index 16aeb3ae..cf2b465d 100644 --- a/pkg/api/analysisrun/result.go +++ b/pkg/api/analysisrun/result.go @@ -14,6 +14,7 @@ type ( type DynamicAnalysisResults struct { StraceSummary DynamicAnalysisStraceSummary FileWrites DynamicAnalysisFileWrites + URLs []string } type StaticAnalysisResults = json.RawMessage diff --git a/sandboxes/dynamicanalysis/Dockerfile b/sandboxes/dynamicanalysis/Dockerfile index 8bf1e053..e6421de8 100644 --- a/sandboxes/dynamicanalysis/Dockerfile +++ b/sandboxes/dynamicanalysis/Dockerfile @@ -147,6 +147,9 @@ ENV NODE_PATH="/app/node_modules" # Test stuff RUN ruby --version && php --version && python3 --version && pip --version && node --version && npm --version && rustc --version && cargo --version +ENV REQUESTS_CA_BUNDLE=/usr/local/share/ca-certificates/ca.crt +ENV SSL_CERT_FILE=/usr/local/share/ca-certificates/ca.crt + ENTRYPOINT [ "sleep" ] CMD [ "30m" ] diff --git a/tools/network/iptables.rules b/tools/network/iptables.rules index 202c6f09..46aedc37 100644 --- a/tools/network/iptables.rules +++ b/tools/network/iptables.rules @@ -5,6 +5,17 @@ *filter :INPUT ACCEPT [0:0] :CNI-ADMIN - [0:0] + +# SSLSplit Routing +-A INPUT -p tcp --dport 80 -j ACCEPT +-A OUTPUT -p tcp --dport 80 -j ACCEPT +-A INPUT -p tcp --dport 443 -j ACCEPT +-A OUTPUT -p tcp --dport 443 -j ACCEPT +-A INPUT -p tcp --dport 8081 -j ACCEPT +-A OUTPUT -p tcp --dport 8081 -j ACCEPT +-A INPUT -p tcp --dport 8080 -j ACCEPT +-A OUTPUT -p tcp --dport 8080 -j ACCEPT + # Block access to this host from the container network. -A INPUT -s 172.16.16.0/24 -j DROP # Block access to metadata.google.internal/AWS metadata.