diff --git a/tools/tinygoize/build.go b/tools/tinygoize/build.go index a970331811..37da037cd8 100644 --- a/tools/tinygoize/build.go +++ b/tools/tinygoize/build.go @@ -20,8 +20,14 @@ const ( BuildCodeFatal ) -// Additional tags required for specific commands. Assume command names unique -// despite being in different directories. +type BuildRes struct { + err *exec.ExitError + excluded bool + output []byte +} + +// Additional tags required for specific commands. Assumes command names are +// unique despite being in different directories. var addBuildTags = map[string]string{ "gzip": "noasm", "insmod": "noasm", @@ -33,7 +39,7 @@ var addBuildTags = map[string]string{ "init": "noasm", } -// returns the needed build-tags for a given package +// Returns the needed build-tags for a given package func buildTags(dir string) (tags string) { parts := strings.Split(dir, "/") cmd := parts[len(parts)-1] @@ -65,35 +71,38 @@ func isExcluded(dir string) bool { } // "tinygo build" in directory 'dir' -func build(id int, tinygo *string, dir string) (BuildCode, error) { - +func build(id int, tinygo *string, dir string) (res BuildRes, err error) { wlog := func(format string, args ...interface{}) { log.Printf("[%d] "+format, append([]interface{}{id}, args...)...) } - wlog("%s Building...\n", dir) - tags := []string{"tinygo.enable"} if addTags := buildTags(dir); addTags != "" { tags = append(tags, addTags) } - c := exec.Command(*tinygo, "build", "-tags", strings.Join(tags, ",")) - c.Dir = dir - c.Stdout, c.Stderr = os.Stdout, os.Stderr - c.Env = append(os.Environ(), "GOOS=linux", "GOARCH=amd64") - err := c.Run() + cmd := exec.Command(*tinygo, "build", "-tags", strings.Join(tags, ",")) + cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH=amd64") + cmd.Dir = dir + res.output, err = cmd.CombinedOutput() if err != nil { - berr, ok := err.(*exec.ExitError) + var ok bool + res.err, ok = err.(*exec.ExitError) if !ok { - return BuildCodeFatal, err + return } + err = nil if isExcluded(dir) { wlog("%v EXCLUDED\n", dir) - return BuildCodeExclude, nil + res.excluded = true + return + } + lines := strings.Split(string(res.output), "\n") + for _,line := range lines { + wlog(line) } - wlog("%v FAILED %v\n", dir, berr) - return BuildCodeFailed, nil + wlog("%v FAILED %v\n", dir, res.err) + return } wlog("%v PASS\n", dir) - return BuildCodeSuccess, nil + return } diff --git a/tools/tinygoize/config.go b/tools/tinygoize/config.go new file mode 100644 index 0000000000..b41f7ecfc7 --- /dev/null +++ b/tools/tinygoize/config.go @@ -0,0 +1,14 @@ +// Copyright 2017-2024 the u-root Authors. All rights reserved +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +type Config struct { + pathMD string + tinygo string + nWorkers int + checkOnly bool + verbose bool + dirs []string +} diff --git a/tools/tinygoize/constraints.go b/tools/tinygoize/constraints.go index e95e683d4c..ed47f25f22 100644 --- a/tools/tinygoize/constraints.go +++ b/tools/tinygoize/constraints.go @@ -23,96 +23,118 @@ const ( ) // Modifies, adds, or removes //go:build line as appropriate with '!tinygo || -// tinygo.enable' for all .go files in dir depending on whether it 'builds' or -// not as previously tested. -func fixupConstraints(dir string, builds bool) (err error) { - p := printer.Config{Mode: printer.UseSpaces | printer.TabIndent, Tabwidth: 8} - - files, err := filepath.Glob(filepath.Join(dir, "*")) +// tinygo.enable' +func fixupFileConstraints(file string, builds bool, checkonly bool) (mustDoWork bool, err error) { + log.Printf("Process %s", file) + b, err := os.ReadFile(file) if err != nil { log.Fatal(err) } -nextFile: - for _, file := range files { - if !strings.HasSuffix(file, ".go") { - continue - } - log.Printf("Process %s", file) - b, err := os.ReadFile(file) - if err != nil { - log.Fatal(err) - } - fset := token.NewFileSet() // positions are relative to fset - f, err := parser.ParseFile(fset, file, string(b), parser.ParseComments|parser.SkipObjectResolution) - if err != nil { - log.Fatalf("parsing\n%v\n:%v", string(b), err) - } + fset := token.NewFileSet() // positions are relative to fset + f, err := parser.ParseFile(fset, file, string(b), parser.ParseComments|parser.SkipObjectResolution) + if err != nil { + log.Fatalf("parsing\n%v\n:%v", string(b), err) + } - goBuildPresent := false + goBuildPresent := false - done: - // modify existing //go:build line - for _, cg := range f.Comments { - for _, c := range cg.List { - if !strings.HasPrefix(c.Text, goBuild) { - continue - } - goBuildPresent = true + // modify existing //go:build line + for _, cg := range f.Comments { + for _, c := range cg.List { + if !strings.HasPrefix(c.Text, goBuild) { + continue + } + goBuildPresent = true - contains := strings.Contains(c.Text, constraint) + contains := strings.Contains(c.Text, constraint) - if (builds && !contains) || (!builds && contains) { - log.Printf("Skipped, constraint up-to-date: %s\n", file) - continue nextFile - } + if (builds && !contains) || (!builds && contains) { + log.Printf("Skipped, constraint up-to-date: %s\n", file) + return + } + + if builds { + re := regexp.MustCompile(`\(?\s*!tinygo\s+\|\|\s+tinygo.enable\s*\)?(\s+\&\&)?`) + c.Text = re.ReplaceAllString(c.Text, "") + log.Printf("Stripping build constraint %v\n", file) - if builds { - re := regexp.MustCompile(`\(?\s*!tinygo\s+\|\|\s+tinygo.enable\s*\)?(\s+\&\&)?`) - c.Text = re.ReplaceAllString(c.Text, "") - log.Printf("Stripping build constraint %v\n", file) - - // handle potentially now-empty build constraint - re = regexp.MustCompile(`^\s*//go:build\s*$`) - if re.MatchString(c.Text) { - filtered := []*ast.Comment{} - for _, comment := range cg.List { - if !re.MatchString(comment.Text) { - filtered = append(filtered, comment) - } + // handle potentially now-empty build constraint + re = regexp.MustCompile(`^\s*//go:build\s*$`) + if re.MatchString(c.Text) { + filtered := []*ast.Comment{} + for _, comment := range cg.List { + if !re.MatchString(comment.Text) { + filtered = append(filtered, comment) } - cg.List = filtered } - } else { - c.Text = goBuild + "(" + constraint + ") && (" + c.Text[len(goBuild):] + ")" + cg.List = filtered } - break done + } else { + c.Text = goBuild + "(" + constraint + ") && (" + c.Text[len(goBuild):] + ")" } + break } + } - if !builds && !goBuildPresent { - // no //go:build line found: insert one - var cg ast.CommentGroup - cg.List = append(cg.List, &ast.Comment{Text: goBuild + constraint}) - - if len(f.Comments) > 0 { - // insert //go:build after first comment - // group, assumed copyright. Doesn't seem - // quite right but seems to work. - cg.List[0].Slash = f.Comments[0].List[0].Slash + 1 - f.Comments = append([]*ast.CommentGroup{f.Comments[0], &cg}, f.Comments[1:]...) - } else { - // prepend //go:build - f.Comments = append([]*ast.CommentGroup{&cg}, f.Comments...) - } + // if it doesn't build but no //go:build found, insert one XXX skip space after copyright + if !builds && !goBuildPresent { + // no //go:build line found: insert one + var cg ast.CommentGroup + cg.List = append(cg.List, &ast.Comment{Text: goBuild + constraint}) + + if len(f.Comments) > 0 { + // insert //go:build after first comment + // group, assumed copyright. Doesn't seem + // quite right but seems to work. + cg.List[0].Slash = f.Comments[0].List[0].Slash + 1 + f.Comments = append([]*ast.CommentGroup{f.Comments[0], &cg}, f.Comments[1:]...) + } else { + // prepend //go:build + f.Comments = append([]*ast.CommentGroup{&cg}, f.Comments...) } + } + + // Complete source file. + var buf bytes.Buffer + p := printer.Config{Mode: printer.UseSpaces | printer.TabIndent, Tabwidth: 8} + if err = p.Fprint(&buf, fset, f); err != nil { + log.Fatalf("Printing:%v", err) + } + - // Complete source file. - var buf bytes.Buffer - if err = p.Fprint(&buf, fset, f); err != nil { - log.Fatalf("Printing:%v", err) + if bytes.Equal(b, buf.Bytes()) { + log.Printf("Skipped, constraint up-to-date: %s\n", file) + return + } else { + mustDoWork = true + } + if checkonly { + return + } + + if err := os.WriteFile(file, buf.Bytes(), 0o644); err != nil { + log.Fatal(err) + } + return +} + +// fixup build constraint lines for all .go files in pkg +func fixupPkgConstraints(dir string, builds bool, checkonly bool) (mustDoWork bool, err error) { + files, err := filepath.Glob(filepath.Join(dir, "*")) + if err != nil { + log.Fatal(err) + } + for _, file := range files { + if !strings.HasSuffix(file, ".go") { + continue + } + mdw, err2 := fixupFileConstraints(file, builds, checkonly) + if err2 != nil { + err = err2 + return } - if err := os.WriteFile(file, buf.Bytes(), 0o644); err != nil { - log.Fatal(err) + if mdw { + mustDoWork = true } } return diff --git a/tools/tinygoize/main.go b/tools/tinygoize/main.go index f8f942407e..24961ed769 100644 --- a/tools/tinygoize/main.go +++ b/tools/tinygoize/main.go @@ -14,20 +14,14 @@ import ( "fmt" "log" "os" + "io" "os/exec" - "path/filepath" - "sort" "strings" "sync" "runtime" + "golang.org/x/term" ) -// Track set of passing, failing, and excluded commands -type BuildStatus struct { - passing []string - failing []string - excluded []string -} // return trimmed output of "tinygo version" func tinygoVersion(tinygo *string) (string, error) { @@ -38,44 +32,76 @@ func tinygoVersion(tinygo *string) (string, error) { return strings.TrimSpace(string(out)), err } -type BuildResult struct { - dir string - code BuildCode - err error +func progress(nComplete int, outOf int) { + isTerminal := term.IsTerminal(int(os.Stdin.Fd())) + if isTerminal { + fmt.Printf("\033[2K\r") + } + percent := 100 * ( float64(nComplete) / float64(outOf)) + fmt.Printf("%v/%v %.2f%%", nComplete, outOf, percent) + if !isTerminal || nComplete == outOf { + fmt.Println() + } } -func worker(id int, tinygo *string, tasks <-chan string, results chan<- BuildResult, wg *sync.WaitGroup) { - defer wg.Done() +// Track set of passing, failing, and excluded commands +type BuildStatus struct { + passing []string + failing []string + excluded []string + modified []string +} + +type WorkerResult struct { + dir string + buildRes BuildRes + didWork bool // whether files in the package need(ed) constraint update + err error +} + +func worker(id int, conf *Config, tasks <-chan string, results chan<- WorkerResult, workGroup *sync.WaitGroup) { + defer workGroup.Done() for dir := range tasks { - code, err := build(id, tinygo, dir) - // Send the result back to the main routine - results <- BuildResult { - dir: dir, - code: code, - err: err, + br, err := build(id, &conf.tinygo, dir) + var dw bool + if err == nil && !br.excluded { + dw, err = fixupPkgConstraints(dir, br.err == nil, conf.checkOnly) + } + + // send result back to main routine + results <- WorkerResult { + dir: dir, + buildRes: br, + didWork: dw, + err: err, } } } // "tinygo build" in each of directories 'dirs' -func buildDirs(tinygo *string, nWorkers int, dirs []string) (status BuildStatus, err error) { - if nWorkers <= 0 { +func buildDirs(conf *Config) (status BuildStatus, err error) { + jobs := len(conf.dirs) + nWorkers := conf.nWorkers + if conf.nWorkers <= 0 { nWorkers = runtime.NumCPU() } + if nWorkers > jobs { + nWorkers = jobs + } tasks := make(chan string) - results := make(chan BuildResult) + results := make(chan WorkerResult) var wg sync.WaitGroup // Start workers log.Printf("Spawning %v workers", nWorkers) for id := 0; id < nWorkers; id++ { wg.Add(1) - go worker(id, tinygo, tasks, results, &wg) + go worker(id+1, conf, tasks, results, &wg) } // Assign tasks go func() { - for _, dir := range dirs { + for _, dir := range conf.dirs { tasks <- dir } close(tasks) // close channel signals workers to exit when done @@ -87,99 +113,52 @@ func buildDirs(tinygo *string, nWorkers int, dirs []string) (status BuildStatus, close(results) // close results channel after all workers done }() + nComplete := 0 for result := range results { + nComplete += 1 + progress(nComplete, jobs) + if result.err != nil { break } - switch result.code { - case BuildCodeExclude: - status.excluded = append(status.excluded, result.dir) - case BuildCodeFailed: - status.failing = append(status.failing, result.dir) - case BuildCodeSuccess: - status.passing = append(status.passing, result.dir) - } - } - return -} - -func writeMarkdown(file *os.File, pathMD *string, tgVersion *string, status BuildStatus) (err error) { - // (not string literal because conflict with markdown back-tick) - fmt.Fprintf(file, "---\n\n") - fmt.Fprintf(file, "DO NOT EDIT.\n\n") - fmt.Fprintf(file, "Generated via `go run tools/tinygoize/main.go`\n\n") - fmt.Fprintf(file, "%v\n\n", *tgVersion) - fmt.Fprintf(file, "---\n\n") - - fmt.Fprintf(file, `# Status of u-root + tinygo -This document aims to track the process of enabling all u-root commands -to be built using tinygo. It will be updated as more commands can be built via: - - u-root> go run tools/tinygoize/* cmds/{core,exp,extra}/* - -Commands that cannot be built with tinygo have a \"(!tinygo || tinygo.enable)\" -build constraint. Specify the "tinygo.enable" build tag to attempt to build -them. - - tinygo build -tags tinygo.enable cmds/core/ls - -The list below is the result of building each command for Linux, x86_64. - -The necessary additions to tinygo will be tracked in -[#2979](https://github.com/u-root/u-root/issues/2979). - ---- - -## Commands Build Status -`) - - linkText := func(dir string) string { - // ignoring err here because pathMD already opened(exists) and - // dir already checked - relPath, _ := filepath.Rel(filepath.Dir(*pathMD), dir) - return fmt.Sprintf("[%v](%v)", dir, relPath) - } - - processSet := func(header string, dirs []string) { - fmt.Fprintf(file, "\n### %v (%v commands)\n", header, len(dirs)) - sort.Strings(dirs) - - if len(dirs) == 0 { - fmt.Fprintf(file, "NONE\n") + if result.buildRes.excluded { + status.excluded = append(status.excluded, result.dir) + } else if result.buildRes.err != nil { + status.failing = append(status.failing, result.dir) + } else { + status.passing = append(status.passing, result.dir) } - for _, dir := range dirs { - msg := fmt.Sprintf(" - %v", linkText(dir)) - tags := buildTags(dir) - if len(tags) > 0 { - msg += fmt.Sprintf(" tags: %v", tags) - } - fmt.Fprintf(file, "%v\n", msg) + if result.didWork { + status.modified = append(status.modified, result.dir) } } - - processSet("EXCLUDED", status.excluded) - processSet("FAILING", status.failing) - processSet("PASSING", status.passing) - return } func main() { - pathMD := flag.String("o", "-", "Output file for markdown summary, '-' or '' for STDOUT") - tinygo := flag.String("tinygo", "tinygo", "Path to tinygo") - nWorkers := flag.Int("j", 0, "Allow 'j' jobs at once; NumCPU() jobs with no arg.") + conf := Config{} + flag.StringVar(&conf.pathMD, "o", "-", "Output file for markdown summary, '-' or '' for STDOUT") + flag.StringVar(&conf.tinygo, "tinygo", "tinygo", "Path to tinygo") + flag.IntVar(&conf.nWorkers, "j", 0, "Allow 'j' jobs at once; NumCPU() jobs with no arg.") + flag.BoolVar(&conf.checkOnly, "n", false, "Check-only, do not modify sources") + flag.BoolVar(&conf.verbose, "v", false, "Verbose") flag.Parse() + conf.dirs = flag.Args() + + if !conf.verbose { + log.SetOutput(io.Discard) + } - tgVersion, err := tinygoVersion(tinygo) + tgVersion, err := tinygoVersion(&conf.tinygo) if err != nil { log.Fatal(err) } log.Printf("%s\n", tgVersion) file := os.Stdout - if len(*pathMD) > 0 && *pathMD != "-" { - file, err = os.Create(*pathMD) + if len(conf.pathMD) > 0 && conf.pathMD != "-" { + file, err = os.Create(conf.pathMD) if err != nil { fmt.Printf("Error creating opening file: %v\n", err) os.Exit(1) @@ -188,30 +167,46 @@ func main() { } // generate list of commands that pass / fail / are excluded - status, err := buildDirs(tinygo, *nWorkers, flag.Args()) + status, err := buildDirs(&conf) if nil != err { log.Fatal(err) } // fix-up constraints in failing files - for _, f := range status.failing { - err = fixupConstraints(f, false) - if nil != err { - log.Fatal(err) - } - } - - // fix-up constraints in passing files - for _, f := range status.passing { - err = fixupConstraints(f, true) - if nil != err { - log.Fatal(err) - } - } + // for _, f := range status.failing { + // dw, err := fixupPkgConstraints(f, false, conf.checkOnly) + // if nil != err { + // log.Fatal(err) + // } + // if dw { + // modified = append(modified, f) + // } + // } + + // // fix-up constraints in passing files + // for _, f := range status.passing { + // dw, err := fixupPkgConstraints(f, true, conf.checkOnly) + // if nil != err { + // log.Fatal(err) + // } + // if dw { + // modified = append(modified, f) + // } + // } // write markdown output - err = writeMarkdown(file, pathMD, &tgVersion, status) + err = writeMarkdown(file, &conf.pathMD, &tgVersion, status) if nil != err { log.Fatal(err) } + + if len(status.modified) > 0 { + fmt.Println("Updates required in package(s):") + for _,modded := range status.modified { + fmt.Println(modded) + } + os.Exit(1) + } else { + fmt.Println("Build constraints up to date.") + } } diff --git a/tools/tinygoize/markdown.go b/tools/tinygoize/markdown.go new file mode 100644 index 0000000000..9093197f52 --- /dev/null +++ b/tools/tinygoize/markdown.go @@ -0,0 +1,70 @@ +package main + +import ( + "fmt" + "path/filepath" + "sort" + "os" +) + +func writeMarkdown(file *os.File, pathMD *string, tgVersion *string, status BuildStatus) (err error) { + // (not string literal because conflict with markdown back-tick) + fmt.Fprintf(file, "---\n\n") + fmt.Fprintf(file, "DO NOT EDIT.\n\n") + fmt.Fprintf(file, "Generated via `go run tools/tinygoize/main.go`\n\n") + fmt.Fprintf(file, "%v\n\n", *tgVersion) + fmt.Fprintf(file, "---\n\n") + + fmt.Fprintf(file, `# Status of u-root + tinygo +This document aims to track the process of enabling all u-root commands +to be built using tinygo. It will be updated as more commands can be built via: + + u-root> go run tools/tinygoize/* cmds/{core,exp,extra}/* + +Commands that cannot be built with tinygo have a \"(!tinygo || tinygo.enable)\" +build constraint. Specify the "tinygo.enable" build tag to attempt to build +them. + + tinygo build -tags tinygo.enable cmds/core/ls + +The list below is the result of building each command for Linux, x86_64. + +The necessary additions to tinygo will be tracked in +[#2979](https://github.com/u-root/u-root/issues/2979). + +--- + +## Commands Build Status +`) + + linkText := func(dir string) string { + // ignoring err here because pathMD already opened(exists) and + // dir already checked + relPath, _ := filepath.Rel(filepath.Dir(*pathMD), dir) + return fmt.Sprintf("[%v](%v)", dir, relPath) + } + + processSet := func(header string, dirs []string) { + fmt.Fprintf(file, "\n### %v (%v commands)\n", header, len(dirs)) + sort.Strings(dirs) + + if len(dirs) == 0 { + fmt.Fprintf(file, "NONE\n") + } + for _, dir := range dirs { + msg := fmt.Sprintf(" - %v", linkText(dir)) + tags := buildTags(dir) + if len(tags) > 0 { + msg += fmt.Sprintf(" tags: %v", tags) + } + fmt.Fprintf(file, "%v\n", msg) + } + } + + processSet("EXCLUDED", status.excluded) + processSet("FAILING", status.failing) + processSet("PASSING", status.passing) + + return +} +