dashboard/app: show manager unique coverage

Makefile

@@ -236,6 +236,7 @@ generate_go: format_cpp
 	$(GO) generate ./executor ./pkg/ifuzz ./pkg/build ./pkg/rpcserver
 	$(GO) generate ./vm/proxyapp
 	$(GO) generate ./pkg/coveragedb
+	$(GO) generate ./pkg/covermerger
 
 generate_rpc:
 	flatc -o pkg/flatrpc --warnings-as-errors --gen-object-api --filename-suffix "" --go --gen-onefile --go-namespace flatrpc pkg/flatrpc/flatrpc.fbs

dashboard/app/api.go

@@ -26,7 +26,6 @@ import (
 	"github.com/google/syzkaller/pkg/asset"
 	"github.com/google/syzkaller/pkg/auth"
 	"github.com/google/syzkaller/pkg/coveragedb"
-	"github.com/google/syzkaller/pkg/coveragedb/spannerclient"
 	"github.com/google/syzkaller/pkg/debugtracer"
 	"github.com/google/syzkaller/pkg/email"
 	"github.com/google/syzkaller/pkg/gcs"
@@ -105,6 +104,7 @@ var maxCrashes = func() int {
 func handleJSON(fn JSONHandler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		c := appengine.NewContext(r)
+		c = SetCoverageDBClient(c, coverageDBClient)
 		reply, err := fn(c, r)
 		if err != nil {
 			status := logErrorPrepareStatus(c, err)
@@ -1948,12 +1948,7 @@ func apiSaveCoverage(c context.Context, payload io.Reader) (interface{}, error)
 		sss = service.List()
 		log.Infof(c, "found %d subsystems for %s namespace", len(sss), descr.Namespace)
 	}
-	client, err := spannerclient.NewClient(c, appengine.AppID(context.Background()))
-	if err != nil {
-		return 0, fmt.Errorf("coveragedb.NewClient() failed: %s", err.Error())
-	}
-	defer client.Close()
-	rowsCreated, err := coveragedb.SaveMergeResult(c, client, descr, jsonDec, sss)
+	rowsCreated, err := coveragedb.SaveMergeResult(c, GetCoverageDBClient(c), descr, jsonDec, sss)
 	if err != nil {
 		log.Errorf(c, "error storing coverage for ns %s, date %s: %v",
 			descr.Namespace, descr.DateTo.String(), err)

dashboard/app/batch_coverage.go

@@ -43,7 +43,7 @@ func handleBatchCoverage(w http.ResponseWriter, r *http.Request) {
 		if err != nil {
 			log.Errorf(ctx, "failed nsDataAvailable(%s): %s", ns, err)
 		}
-		periodsMerged, rowsMerged, err := coveragedb.NsDataMerged(ctx, "syzkaller", ns)
+		periodsMerged, rowsMerged, err := coveragedb.NsDataMerged(ctx, coverageDBClient, ns)
 		if err != nil {
 			log.Errorf(ctx, "failed coveragedb.NsDataMerged(%s): %s", ns, err)
 		}
@@ -154,7 +154,7 @@ func nsDataAvailable(ctx context.Context, ns string) ([]coveragedb.TimePeriod, [
 
 func handleBatchCoverageClean(w http.ResponseWriter, r *http.Request) {
 	ctx := context.Background()
-	totalDeleted, err := coveragedb.DeleteGarbage(ctx)
+	totalDeleted, err := coveragedb.DeleteGarbage(ctx, coverageDBClient)
 	if err != nil {
 		errMsg := fmt.Sprintf("failed to coveragedb.DeleteGarbage: %s", err.Error())
 		log.Errorf(ctx, "%s", errMsg)

dashboard/app/config.go

@@ -425,6 +425,7 @@ func installConfig(cfg *GlobalConfig) {
 	initAPIHandlers()
 	initKcidb()
 	initBatchProcessors()
+	initCoverageDB()
 }
 
 var contextConfigKey = "Updated config (to be used during tests). Use only in tests!"

dashboard/app/coverage.go

@@ -8,17 +8,49 @@ import (
 	"fmt"
 	"html/template"
 	"net/http"
+	"os"
 	"slices"
 	"strconv"
 
 	"cloud.google.com/go/civil"
 	"github.com/google/syzkaller/pkg/cover"
 	"github.com/google/syzkaller/pkg/coveragedb"
+	"github.com/google/syzkaller/pkg/coveragedb/spannerclient"
 	"github.com/google/syzkaller/pkg/covermerger"
 	"github.com/google/syzkaller/pkg/validator"
+	"google.golang.org/appengine/v2"
 )
 
-type funcStyleBodyJS func(ctx context.Context, projectID string, scope *cover.SelectScope, sss, managers []string,
+var coverageDBClient spannerclient.SpannerClient
+
+func initCoverageDB() {
+	if !appengine.IsAppEngine() {
+		// It is a test environment.
+		// Use SetCoverageDBClient to specify the coveragedb mock or emulator in every test.
+		return
+	}
+	projectID := os.Getenv("GOOGLE_CLOUD_PROJECT")
+	var err error
+	coverageDBClient, err = spannerclient.NewClient(context.Background(), projectID)
+	if err != nil {
+		panic("spanner.NewClient: " + err.Error())
+	}
+}
+
+var keyCoverageDBClient = "coveragedb client key"
+
+func SetCoverageDBClient(ctx context.Context, client spannerclient.SpannerClient) context.Context {
+	return context.WithValue(ctx, &keyCoverageDBClient, client)
+}
+
+func GetCoverageDBClient(ctx context.Context) spannerclient.SpannerClient {
+	client, _ := ctx.Value(&keyCoverageDBClient).(spannerclient.SpannerClient)
+	return client
+}
+
+type funcStyleBodyJS func(
+	ctx context.Context, client spannerclient.SpannerClient,
+	scope *cover.SelectScope, onlyUnique bool, sss, managers []string,
 ) (template.CSS, template.HTML, template.HTML, error)
 
 func handleCoverageHeatmap(c context.Context, w http.ResponseWriter, r *http.Request) error {
@@ -34,6 +66,10 @@ func handleHeatmap(c context.Context, w http.ResponseWriter, r *http.Request, f
 	if err != nil {
 		return err
 	}
+	nsConfig := getNsConfig(c, hdr.Namespace)
+	if nsConfig.Coverage == nil {
+		return ErrClientNotFound
+	}
 	ss := r.FormValue("subsystem")
 	manager := r.FormValue("manager")
 
@@ -71,16 +107,18 @@ func handleHeatmap(c context.Context, w http.ResponseWriter, r *http.Request, f
 	slices.Sort(managers)
 	slices.Sort(subsystems)
 
+	onlyUnique := r.FormValue("unique-only") == "1"
+
 	var style template.CSS
 	var body, js template.HTML
-	if style, body, js, err = f(c, "syzkaller",
+	if style, body, js, err = f(c, GetCoverageDBClient(c),
 		&cover.SelectScope{
 			Ns:        hdr.Namespace,
 			Subsystem: ss,
 			Manager:   manager,
 			Periods:   periods,
 		},
-		subsystems, managers); err != nil {
+		onlyUnique, subsystems, managers); err != nil {
 		return fmt.Errorf("failed to generate heatmap: %w", err)
 	}
 	return serveTemplate(w, "custom_content.html", struct {
@@ -115,10 +153,14 @@ func handleFileCoverage(c context.Context, w http.ResponseWriter, r *http.Reques
 	periodType := r.FormValue("period")
 	targetCommit := r.FormValue("commit")
 	kernelFilePath := r.FormValue("filepath")
+	manager := r.FormValue("manager")
 	if err := validator.AnyError("input validation failed",
 		validator.TimePeriodType(periodType, "period"),
 		validator.CommitHash(targetCommit, "commit"),
 		validator.KernelFilePath(kernelFilePath, "filepath"),
+		validator.AnyOk(
+			validator.Allowlisted(manager, []string{"", "*"}, "manager"),
+			validator.ManagerName(manager, "manager")),
 	); err != nil {
 		return fmt.Errorf("%w: %w", err, ErrClientBadRequest)
 	}
@@ -130,18 +172,40 @@ func handleFileCoverage(c context.Context, w http.ResponseWriter, r *http.Reques
 	if err != nil {
 		return fmt.Errorf("coveragedb.MakeTimePeriod: %w", err)
 	}
+	onlyUnique := r.FormValue("unique-only") == "1"
 	mainNsRepo, _ := nsConfig.mainRepoBranch()
-	hitCounts, err := coveragedb.ReadLinesHitCount(c, hdr.Namespace, targetCommit, kernelFilePath, tp)
+	client := GetCoverageDBClient(c)
+	if client == nil {
+		return fmt.Errorf("spannerdb client is nil")
+	}
+	hitLines, hitCounts, err := coveragedb.ReadLinesHitCount(
+		c, client, hdr.Namespace, targetCommit, kernelFilePath, manager, tp)
+	covMap := cover.MakeCovMap(hitLines, hitCounts)
 	if err != nil {
-		return fmt.Errorf("coveragedb.ReadLinesHitCount: %w", err)
+		return fmt.Errorf("coveragedb.ReadLinesHitCount(%s): %w", manager, err)
+	}
+	if onlyUnique {
+		// This request is expected to be made second by tests.
+		// Moving it to goroutine don't forget to change multiManagerCovDBFixture.
+		allHitLines, allHitCounts, err := coveragedb.ReadLinesHitCount(
+			c, client, hdr.Namespace, targetCommit, kernelFilePath, "*", tp)
+		if err != nil {
+			return fmt.Errorf("coveragedb.ReadLinesHitCount(*): %w", err)
+		}
+		covMap = cover.UniqCoverage(cover.MakeCovMap(allHitLines, allHitCounts), covMap)
+	}
+
+	webGit := getWebGit(c) // Get mock if available.
+	if webGit == nil {
+		webGit = covermerger.MakeWebGit(makeProxyURIProvider(nsConfig.Coverage.WebGitURI))
 	}
 
 	content, err := cover.RendFileCoverage(
 		mainNsRepo,
 		targetCommit,
 		kernelFilePath,
-		makeProxyURIProvider(nsConfig.Coverage.WebGitURI),
-		&covermerger.MergeResult{HitCounts: hitCounts},
+		webGit,
+		&covermerger.MergeResult{HitCounts: covMap},
 		cover.DefaultHTMLRenderConfig())
 	if err != nil {
 		return fmt.Errorf("cover.RendFileCoverage: %w", err)
@@ -151,19 +215,34 @@ func handleFileCoverage(c context.Context, w http.ResponseWriter, r *http.Reques
 	return nil
 }
 
+var keyWebGit = "file content provider"
+
+func setWebGit(ctx context.Context, provider covermerger.FileVersProvider) context.Context {
+	return context.WithValue(ctx, &keyWebGit, provider)
+}
+
+func getWebGit(ctx context.Context) covermerger.FileVersProvider {
+	res, _ := ctx.Value(&keyWebGit).(covermerger.FileVersProvider)
+	return res
+}
+
 func handleCoverageGraph(c context.Context, w http.ResponseWriter, r *http.Request) error {
 	hdr, err := commonHeader(c, r, w, "")
 	if err != nil {
 		return err
 	}
+	nsConfig := getNsConfig(c, hdr.Namespace)
+	if nsConfig.Coverage == nil {
+		return ErrClientNotFound
+	}
 	periodType := r.FormValue("period")
 	if periodType == "" {
 		periodType = coveragedb.QuarterPeriod
 	}
 	if periodType != coveragedb.QuarterPeriod && periodType != coveragedb.MonthPeriod {
 		return fmt.Errorf("only quarter and month are allowed, but received %s instead", periodType)
 	}
-	hist, err := MergedCoverage(c, hdr.Namespace, periodType)
+	hist, err := MergedCoverage(c, GetCoverageDBClient(c), hdr.Namespace, periodType)
 	if err != nil {
 		return err
 	}