Support custom sandbox and analysis script path for dynamic analysis (#…

…744) * plumbing to support custom dynamic analysis sandbox and command Signed-off-by: Max Fisher <[email protected]> * support custom sandbox and analysis command in local analysis Signed-off-by: Max Fisher <[email protected]> * address review comments Signed-off-by: Max Fisher <[email protected]> * fix static analysis args generation Signed-off-by: Max Fisher <[email protected]> * fix nil pointer dereference in download Signed-off-by: Max Fisher <[email protected]> * sandbox.Sandbox: add argument names to Run() and improve documentation comments Signed-off-by: Max Fisher <[email protected]> --------- Signed-off-by: Max Fisher <[email protected]>
ossf · Jun 9, 2023 · db3da7d · db3da7d
1 parent 94a0dab
commit db3da7d
Show file tree

Hide file tree

Showing 15 changed files with 158 additions and 130 deletions.
diff --git a/cmd/analyze/main.go b/cmd/analyze/main.go
@@ -30,6 +30,8 @@ var (
 	staticUpload        = flag.String("upload-static", "", "bucket path for uploading static analysis results")
 	uploadFileWriteInfo = flag.String("upload-file-write-info", "", "bucket path for uploading information from file writes")
 	offline             = flag.Bool("offline", false, "disables sandbox network access")
+	customSandbox       = flag.String("sandbox-image", "", "override default dynamic analysis sandbox with custom image")
+	customAnalysisCmd   = flag.String("analysis-command", "", "override default dynamic analysis script path (use with custom sandbox image)")
 	listModes           = flag.Bool("list-modes", false, "prints out a list of available analysis modes")
 	help                = flag.Bool("help", false, "print help on available options")
 	analysisMode        = utils.CommaSeparatedFlags("mode", []string{"static", "dynamic"},
@@ -103,21 +105,25 @@ func dynamicAnalysis(pkg *pkgmanager.Pkg, resultStores worker.ResultStores) {
 
 	sbOpts := append(worker.DynamicSandboxOptions(), makeSandboxOptions()...)
 
-	data, lastRunPhase, lastStatus, err := worker.RunDynamicAnalysis(pkg, sbOpts)
+	if *customSandbox != "" {
+		sbOpts = append(sbOpts, sandbox.Image(*customSandbox))
+	}
+
+	result, err := worker.RunDynamicAnalysis(pkg, sbOpts, *customAnalysisCmd)
 	if err != nil {
 		log.Error("Dynamic analysis aborted (run error)", "error", err)
 		return
 	}
 
 	// this is only valid if RunDynamicAnalysis() returns nil err
-	if lastStatus != analysis.StatusCompleted {
+	if result.LastStatus != analysis.StatusCompleted {
 		log.Warn("Dynamic analysis phase did not complete successfully",
-			"lastRunPhase", lastRunPhase,
-			"status", lastStatus)
+			"lastRunPhase", result.LastRunPhase,
+			"status", result.LastStatus)
 	}
 
 	ctx := context.Background()
-	if err := worker.SaveDynamicAnalysisData(ctx, pkg, resultStores, data); err != nil {
+	if err := worker.SaveDynamicAnalysisData(ctx, pkg, resultStores, result.AnalysisData); err != nil {
 		log.Error("Upload error", "error", err)
 	}
 }

diff --git a/cmd/worker/main.go b/cmd/worker/main.go
@@ -153,13 +153,14 @@ func handleMessage(ctx context.Context, msg *pubsub.Message, packagesBucket *blo
 	}
 
 	dynamicSandboxOpts := append(worker.DynamicSandboxOptions(), sandboxOpts...)
-	results, _, _, err := worker.RunDynamicAnalysis(pkg, dynamicSandboxOpts)
+	result, err := worker.RunDynamicAnalysis(pkg, dynamicSandboxOpts, "")
 	if err != nil {
 		return err
 	}
 
 	staticSandboxOpts := append(worker.StaticSandboxOptions(), sandboxOpts...)
 	var staticResults analysisrun.StaticAnalysisResults
+	// TODO run static analysis first and remove the if statement below
 	if resultStores.StaticAnalysis != nil {
 		staticResults, _, err = worker.RunStaticAnalysis(pkg, staticSandboxOpts, staticanalysis.All)
 		if err != nil {
@@ -170,8 +171,7 @@ func handleMessage(ctx context.Context, msg *pubsub.Message, packagesBucket *blo
 	if err := worker.SaveStaticAnalysisData(ctx, pkg, resultStores, staticResults); err != nil {
 		return err
 	}
-
-	if err := worker.SaveDynamicAnalysisData(ctx, pkg, resultStores, results); err != nil {
+	if err := worker.SaveDynamicAnalysisData(ctx, pkg, resultStores, result.AnalysisData); err != nil {
 		return err
 	}
 

diff --git a/internal/dynamicanalysis/analysis.go b/internal/dynamicanalysis/analysis.go
@@ -30,7 +30,7 @@ var resultError = &Result{
 	},
 }
 
-func Run(sb sandbox.Sandbox, args []string) (*Result, error) {
+func Run(sb sandbox.Sandbox, command string, args []string) (*Result, error) {
 	log.Info("Running dynamic analysis",
 		"args", args)
 
@@ -46,8 +46,8 @@ func Run(sb sandbox.Sandbox, args []string) (*Result, error) {
 
 	// Run the command
 	log.Debug("Running dynamic analysis command",
-		"args", args)
-	r, err := sb.Run(args...)
+		"command", command, "args", args)
+	r, err := sb.Run(command, args...)
 	if err != nil {
 		return resultError, fmt.Errorf("sandbox failed (%w)", err)
 	}

diff --git a/internal/dynamicanalysis/sandbox_args.go b/internal/dynamicanalysis/sandbox_args.go
@@ -0,0 +1,48 @@
+package dynamicanalysis
+
+import (
+	"github.com/ossf/package-analysis/internal/pkgmanager"
+	"github.com/ossf/package-analysis/pkg/api/analysisrun"
+	"github.com/ossf/package-analysis/pkg/api/pkgecosystem"
+)
+
+// defaultCommand returns the path (in the default sandbox image)
+// of the default dynamic analysis command for the ecosystem
+var defaultCommand = map[pkgecosystem.Ecosystem]string{
+	pkgecosystem.CratesIO:  "/usr/local/bin/analyze-rust.py",
+	pkgecosystem.NPM:       "/usr/local/bin/analyze-node.js",
+	pkgecosystem.Packagist: "/usr/local/bin/analyze-php.php",
+	pkgecosystem.PyPI:      "/usr/local/bin/analyze-python.py",
+	pkgecosystem.RubyGems:  "/usr/local/bin/analyze-ruby.rb",
+}
+
+func DefaultCommand(ecosystem pkgecosystem.Ecosystem) string {
+	cmd := defaultCommand[ecosystem]
+	if cmd == "" {
+		panic("unsupported ecosystem: " + ecosystem)
+	}
+	return cmd
+}
+
+// MakeAnalysisArgs returns the arguments to pass to the dynamic analysis command in the sandbox
+// for the given phase of dynamic analysis on a package. The actual analysis command
+// depends on the ecosystem, see pkgmanager.PkgManager.DynamicAnalysisCommand()
+func MakeAnalysisArgs(p *pkgmanager.Pkg, phase analysisrun.DynamicPhase) []string {
+	args := make([]string, 0)
+
+	if p.IsLocal() {
+		args = append(args, "--local", p.LocalPath())
+	} else if p.Version() != "" {
+		args = append(args, "--version", p.Version())
+	}
+
+	if phase == "" {
+		args = append(args, "all")
+	} else {
+		args = append(args, string(phase))
+	}
+
+	args = append(args, p.Name())
+
+	return args
+}
diff --git a/internal/pkgmanager/crates.io.go b/internal/pkgmanager/crates.io.go
@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/ossf/package-analysis/pkg/api/analysisrun"
 	"github.com/ossf/package-analysis/pkg/api/pkgecosystem"
 )
 
@@ -34,8 +33,5 @@ func getCratesLatest(pkg string) (string, error) {
 
 var cratesPkgManager = PkgManager{
 	ecosystem:     pkgecosystem.CratesIO,
-	image:         combinedDynamicAnalysisImage,
-	command:       "/usr/local/bin/analyze-rust.py",
 	latestVersion: getCratesLatest,
-	dynamicPhases: analysisrun.DefaultDynamicPhases(),
 }
diff --git a/internal/pkgmanager/ecosystem.go b/internal/pkgmanager/ecosystem.go
@@ -4,23 +4,17 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/ossf/package-analysis/pkg/api/analysisrun"
 	"github.com/ossf/package-analysis/pkg/api/pkgecosystem"
 )
 
 // PkgManager represents how packages from a common ecosystem are accessed.
 type PkgManager struct {
 	ecosystem      pkgecosystem.Ecosystem
-	image          string
-	command        string
 	latestVersion  func(string) (string, error)
 	archiveURL     func(string, string) (string, error)
 	extractArchive func(string, string) error
-	dynamicPhases  []analysisrun.DynamicPhase
 }
 
-const combinedDynamicAnalysisImage = "gcr.io/ossf-malware-analysis/dynamic-analysis"
-
 var (
 	supportedPkgManagers = map[pkgecosystem.Ecosystem]*PkgManager{
 		npmPkgManager.ecosystem:       &npmPkgManager,
@@ -40,12 +34,8 @@ func (p *PkgManager) String() string {
 	return string(p.ecosystem)
 }
 
-func (p *PkgManager) DynamicAnalysisImage() string {
-	return p.image
-}
-
-func (p *PkgManager) DynamicPhases() []analysisrun.DynamicPhase {
-	return p.dynamicPhases
+func (p *PkgManager) Ecosystem() pkgecosystem.Ecosystem {
+	return p.ecosystem
 }
 
 func (p *PkgManager) Latest(name string) (*Pkg, error) {
@@ -83,6 +73,10 @@ func (p *PkgManager) DownloadArchive(name, version, directory string) (string, e
 		return "", fmt.Errorf("no directory specified")
 	}
 
+	if p.archiveURL == nil {
+		return "", fmt.Errorf("not yet implemented for %s", p.Ecosystem())
+	}
+
 	downloadURL, err := p.archiveURL(name, version)
 	if err != nil {
 		return "", err

diff --git a/internal/pkgmanager/npm.go b/internal/pkgmanager/npm.go
@@ -8,7 +8,6 @@ import (
 	"strings"
 
 	"github.com/ossf/package-analysis/internal/utils"
-	"github.com/ossf/package-analysis/pkg/api/analysisrun"
 	"github.com/ossf/package-analysis/pkg/api/pkgecosystem"
 )
 
@@ -73,10 +72,7 @@ func getNPMArchiveURL(pkgName, version string) (string, error) {
 
 var npmPkgManager = PkgManager{
 	ecosystem:      pkgecosystem.NPM,
-	image:          combinedDynamicAnalysisImage,
-	command:        "/usr/local/bin/analyze-node.js",
 	latestVersion:  getNPMLatest,
 	archiveURL:     getNPMArchiveURL,
 	extractArchive: utils.ExtractTarGzFile,
-	dynamicPhases:  analysisrun.DefaultDynamicPhases(),
 }
diff --git a/internal/pkgmanager/package.go b/internal/pkgmanager/package.go
@@ -1,7 +1,6 @@
 package pkgmanager
 
 import (
-	"github.com/ossf/package-analysis/pkg/api/analysisrun"
 	"github.com/ossf/package-analysis/pkg/api/pkgecosystem"
 )
 
@@ -39,25 +38,3 @@ func (p *Pkg) Manager() *PkgManager {
 func (p *Pkg) LocalPath() string {
 	return p.local
 }
-
-// Command returns the analysis command for the package.
-func (p *Pkg) Command(phase analysisrun.DynamicPhase) []string {
-	args := make([]string, 0)
-	args = append(args, p.manager.command)
-
-	if p.local != "" {
-		args = append(args, "--local", p.local)
-	} else if p.version != "" {
-		args = append(args, "--version", p.version)
-	}
-
-	if phase == "" {
-		args = append(args, "all")
-	} else {
-		args = append(args, string(phase))
-	}
-
-	args = append(args, p.name)
-
-	return args
-}
diff --git a/internal/pkgmanager/packagist.go b/internal/pkgmanager/packagist.go
@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/ossf/package-analysis/pkg/api/analysisrun"
 	"github.com/ossf/package-analysis/pkg/api/pkgecosystem"
 )
 
@@ -51,8 +50,5 @@ func getPackagistLatest(pkg string) (string, error) {
 
 var packagistPkgManager = PkgManager{
 	ecosystem:     pkgecosystem.Packagist,
-	image:         combinedDynamicAnalysisImage,
-	command:       "/usr/local/bin/analyze-php.php",
 	latestVersion: getPackagistLatest,
-	dynamicPhases: analysisrun.DefaultDynamicPhases(),
 }
diff --git a/internal/pkgmanager/pypi.go b/internal/pkgmanager/pypi.go
@@ -8,7 +8,6 @@ import (
 	"strings"
 
 	"github.com/ossf/package-analysis/internal/utils"
-	"github.com/ossf/package-analysis/pkg/api/analysisrun"
 	"github.com/ossf/package-analysis/pkg/api/pkgecosystem"
 )
 
@@ -80,10 +79,7 @@ func getPyPIArchiveURL(pkgName, version string) (string, error) {
 
 var pypiPkgManager = PkgManager{
 	ecosystem:      pkgecosystem.PyPI,
-	image:          combinedDynamicAnalysisImage,
-	command:        "/usr/local/bin/analyze-python.py",
 	latestVersion:  getPyPILatest,
 	archiveURL:     getPyPIArchiveURL,
 	extractArchive: utils.ExtractTarGzFile,
-	dynamicPhases:  analysisrun.DefaultDynamicPhases(),
 }
diff --git a/internal/pkgmanager/rubygems.go b/internal/pkgmanager/rubygems.go
@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/ossf/package-analysis/pkg/api/analysisrun"
 	"github.com/ossf/package-analysis/pkg/api/pkgecosystem"
 )
 
@@ -32,8 +31,5 @@ func getRubyGemsLatest(pkg string) (string, error) {
 
 var rubygemsPkgManager = PkgManager{
 	ecosystem:     pkgecosystem.RubyGems,
-	image:         combinedDynamicAnalysisImage,
-	command:       "/usr/local/bin/analyze-ruby.rb",
 	latestVersion: getRubyGemsLatest,
-	dynamicPhases: analysisrun.DefaultDynamicPhases(),
 }