From 93afcb20cbdad78ed7121e9f959af64b923e70c0 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 2 Feb 2024 14:12:55 +0100 Subject: [PATCH] fix(script/go.bash): support Ubuntu 22.04 LTS and Debian 11 (#1487) This commit is part of https://github.com/ooni/probe/issues/2664. We use a different strategy for downloading and executing the correct version of Go. We have vendored and adapted to code at `golang.org/dl` such that it works with versions of Go as old as the version of Go used by Ubuntu 22.04 LTS and Debian 11. --- .../workflows/{gofixpath.yml => gobash.yml} | 14 +- Readme.md | 97 ++-- internal/cmd/buildtool/gofixpath.go | 78 --- internal/cmd/buildtool/main.go | 1 - pkg/gobash/.gitignore | 1 + pkg/gobash/go.mod | 3 + pkg/gobash/main.go | 22 + pkg/gobash/signal_notunix.go | 14 + pkg/gobash/signal_unix.go | 15 + pkg/gobash/version.go | 538 ++++++++++++++++++ pkg/gobash/version_test.go | 67 +++ script/go.bash | 10 +- script/nocopyreadall.bash | 6 + 13 files changed, 720 insertions(+), 146 deletions(-) rename .github/workflows/{gofixpath.yml => gobash.yml} (66%) delete mode 100644 internal/cmd/buildtool/gofixpath.go create mode 100644 pkg/gobash/.gitignore create mode 100644 pkg/gobash/go.mod create mode 100644 pkg/gobash/main.go create mode 100644 pkg/gobash/signal_notunix.go create mode 100644 pkg/gobash/signal_unix.go create mode 100644 pkg/gobash/version.go create mode 100644 pkg/gobash/version_test.go diff --git a/.github/workflows/gofixpath.yml b/.github/workflows/gobash.yml similarity index 66% rename from .github/workflows/gofixpath.yml rename to .github/workflows/gobash.yml index 9041b75284..3d801d594d 100644 --- a/.github/workflows/gofixpath.yml +++ b/.github/workflows/gobash.yml @@ -1,9 +1,8 @@ -# Ensures that ./internal/cmd/buildtool gofixpath {command} [arguments] downloads the correct -# version of go and executes {command} with [arguments] with "go" being the right version. +# Ensures that ./script/go.bash works as intended with many versions of "go". # # See https://github.com/ooni/probe/issues/2664. -name: gofixpath +name: gobash on: push: branches: @@ -21,7 +20,14 @@ jobs: build_with_specific_go_version: strategy: matrix: - goversion: ["1.19", "1.20", "1.21"] # when releasing check whether we need to update this array + goversion: + - "1.15" # debian 11 "bullseye" + - "1.16" + - "1.17" + - "1.18" # ubuntu 22.04 LTS + - "1.19" # debian 12 "bookworm" + - "1.20" + - "1.21" system: [ubuntu-latest, macos-latest] runs-on: "${{ matrix.system }}" steps: diff --git a/Readme.md b/Readme.md index 0a351887ba..77b7bb6eba 100644 --- a/Readme.md +++ b/Readme.md @@ -62,13 +62,22 @@ Debian/Ubuntu. Once `ooniprobe` is installed, refer to the ## Developer instructions -This repository _should really_ use the Go version mentioned by the -[GOVERSION](GOVERSION) file (i.e., go1.20.12). Using a later version of -Go _should_ work as intended. Using an older version of Go is -_definitely not recommended_ and _may not even compile_. Here's why: we rely -on packages forked from the standard library; so, it is -more robust to use the same version of Go from which -we forked those packages from. +To setup development for this repository you need Go >= 1.15. The +`./script/go.bash` script will automatically download the expected +version of Go mentioned in the [GOVERSION](GOVERSION) file (i.e., +go1.20.12) and use it for building. + +You can also bypass `./script/go.bash` and build ooniprobe manually using +`go build ...` but, in such a case, note that: + +1. using an older version that the one mentioned in [GOVERSION](GOVERSION) +is _definitely not recommended_ and _may not even compile_; + +2. using later versions _should_ work as intended for core functionality +but extra functionality may be disabled or not working as intended. + +Here's why: we rely on packages forked from the standard library; so, it is +more robust to use the same version of Go from which we forked those packages from. You will also need a C compiler. On Linux and other Unix systems both GCC and Clang will work. If you're using Windows, we @@ -81,8 +90,7 @@ mingw-w64 cross-compiler. The following commands show how to setup a development environment using Debian 12 ("bookworm"). The same instructions -should also work for Debian-based distribution (e.g., Ubuntu) -as long as you're using Go >= 1.19. +should also work for Ubuntu 22.04 LTS. ```bash # install the compilers, git, and the root CA @@ -90,21 +98,12 @@ sudo apt install golang build-essential ca-certificates git # [optional] install mingw-w64 if you're targeting windows sudo apt install mingw-w64 - -# install the required go version binary -go install -v golang.org/dl/go1.20.12@latest - -# fetch the whole go distribution -$HOME/go/bin/go1.20.12 download ``` -You can automate the last two steps by using `./script/go.bash` as your -Go compiler, as explained below. - ### Fedora developer setup The following commands show how to setup a development -environment using Fedora, as long as your Fedora uses Go >= 1.19. +environment using Fedora, as long as your Fedora uses Go >= 1.15. ```bash # install the compilers and git @@ -112,17 +111,8 @@ sudo dnf install golang make gcc gcc-c++ git # [optional] install mingw-w64 if you're targeting windows sudo dnf install mingw64-gcc mingw64-gcc-c++ - -# install the required go version binary -go install -v golang.org/dl/go1.20.12@latest - -# fetch the whole go distribution -$HOME/go/bin/go1.20.12 download ``` -You can automate the last two steps by using `./script/go.bash` as your -Go compiler, as explained below. - ### macOS developer setup The following commands show how to setup a development @@ -135,29 +125,35 @@ Then, you need to follow these instructions: ```bash # install the compiler brew install go +``` -# install the required go version binary -go install -v golang.org/dl/go1.20.12@latest +### The `./script/go.bash` script -# fetch the whole go distribution -$HOME/go/bin/go1.20.12 download +The `./script/go.bash` script requires Go >= 1.15 and automates installing and +using the correct version of Go. Running this script as follows: + +```bash +./script/go.bash build -v -ldflags '-s -w' ./internal/cmd/miniooni ``` -You can automate the last two steps by using `./script/go.bash` as your -Go compiler, as explained below. +Is equivalent to running these commands: + +```bash +go install -v golang.org/dl/go1.20.12@latest +$HOME/go/bin/go1.20.12 download +$HOME/sdk/go1.20.12/bin/go build -v -ldflags '-s -w' ./internal/cmd/miniooni +``` -### Build instructions without using `./script/go.bash` +### Common build targets -This section describe how to manually use the correct version of Go you have -installed. You can alternatively use the `./script/go.bash` wrapper to automatically -use the correct Go version (provided you have Go >= 1.19 installed), as described -in a subsequent section of this file. +This section shows how to build using `./script/go.bash`. If you want to bypass +using this script, just run `go` instead of `./script/go.bash`. Once you have installed the correct Go version and a C compiler, you can compile `ooniprobe` using: ```bash -$HOME/go/bin/go1.20.12 build -v -ldflags '-s -w' ./cmd/ooniprobe +./script/go.bash build -v -ldflags '-s -w' ./cmd/ooniprobe ``` This command will generate a stripped binary called `ooniprobe` @@ -166,7 +162,7 @@ in the toplevel directory. Likewise, you can compile `miniooni` using: ```bash -$HOME/go/bin/go1.20.12 build -v -ldflags '-s -w' ./internal/cmd/miniooni +./script/go.bash build -v -ldflags '-s -w' ./internal/cmd/miniooni ``` This command will generate a stripped binary called `miniooni` @@ -175,29 +171,12 @@ in the toplevel directory. And `oohelperd` using: ```bash -$HOME/go/bin/go1.20.12 build -v -ldflags '-s -w' ./internal/cmd/oohelperd +./script/go.bash build -v -ldflags '-s -w' ./internal/cmd/oohelperd ``` This command will generate a stripped binary called `oohelperd` in the toplevel directory. -### Alternative: using the `./script/go.bash` script - -The `./script/go.bash` script requires Go >= 1.19 and automates installing and -using the correct version of Go. Running this script as follows: - -```bash -./script/go.bash build -v -ldflags '-s -w' ./internal/cmd/miniooni -``` - -Is equivalent to running these commands: - -```bash -go install -v golang.org/dl/go1.20.12@latest -$HOME/go/bin/go1.20.12 download -$HOME/sdk/go1.20.12/bin/go build -v -ldflags '-s -w' ./internal/cmd/miniooni -``` - ## Contributing Please, see [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/internal/cmd/buildtool/gofixpath.go b/internal/cmd/buildtool/gofixpath.go deleted file mode 100644 index db1aeb1099..0000000000 --- a/internal/cmd/buildtool/gofixpath.go +++ /dev/null @@ -1,78 +0,0 @@ -package main - -import ( - "fmt" - "path/filepath" - - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/cmd/buildtool/internal/buildtoolmodel" - "github.com/ooni/probe-cli/v3/internal/must" - "github.com/ooni/probe-cli/v3/internal/runtimex" - "github.com/ooni/probe-cli/v3/internal/shellx" - "github.com/spf13/cobra" -) - -// gofixpathSubcommand returns the gofixpath [cobra.Command]. -func gofixpathSubcommand() *cobra.Command { - return &cobra.Command{ - Use: "gofixpath", - Short: "Executes a command ensuring the expected version of Go comes first in PATH lookup", - Run: func(cmd *cobra.Command, args []string) { - gofixpathMain(&buildDeps{}, args...) - }, - Args: cobra.MinimumNArgs(1), - } -} - -// gofixpathMain ensures the correct version of Go is in path, otherwise -// installs such a version, configure the PATH correctly, and then executes -// whatever argument passed to the command with the correct PATH. -// -// See https://github.com/ooni/probe/issues/2664. -func gofixpathMain(deps buildtoolmodel.Dependencies, args ...string) { - // create empty environment - envp := &shellx.Envp{} - - // install and configure the correct go version if needed - if !golangCorrectVersionCheckP("GOVERSION") { - // read the version of Go we would like to use - expected := string(must.FirstLineBytes(must.ReadFile("GOVERSION"))) - - // install the wrapper command - packageName := fmt.Sprintf("golang.org/dl/go%s@latest", expected) - must.Run(log.Log, "go", "install", "-v", packageName) - - // run the wrapper to download the distribution - gobinproxy := filepath.Join( - string(must.FirstLineBytes(must.RunOutput(log.Log, "go", "env", "GOPATH"))), - "bin", - fmt.Sprintf("go%s", expected), - ) - must.Run(log.Log, gobinproxy, "download") - - // add the path to the SDK binary dir - // - // Note: because gomobile wants to execute "go" we must provide the - // path to a directory that contains a command named "go" and we cannot - // just use the gobinproxy binary - sdkbinpath := filepath.Join( - string(must.FirstLineBytes(must.RunOutput(log.Log, gobinproxy, "env", "GOROOT"))), - "bin", - ) - - // prepend to PATH - envp.Append("PATH", cdepsPrependToPath(sdkbinpath)) - } - - // create shellx configuration - config := &shellx.Config{ - Logger: log.Log, - Flags: shellx.FlagShowStdoutStderr, - } - - // create argv - argv := runtimex.Try1(shellx.NewArgv(args[0], args[1:]...)) // safe because cobra.MinimumNArgs(1) - - // execute the child command - runtimex.Try0(shellx.RunEx(config, argv, envp)) -} diff --git a/internal/cmd/buildtool/main.go b/internal/cmd/buildtool/main.go index 139934b2ec..1012a4b68c 100644 --- a/internal/cmd/buildtool/main.go +++ b/internal/cmd/buildtool/main.go @@ -20,7 +20,6 @@ func main() { root.AddCommand(androidSubcommand()) root.AddCommand(darwinSubcommand()) root.AddCommand(genericSubcommand()) - root.AddCommand(gofixpathSubcommand()) root.AddCommand(iosSubcommand()) root.AddCommand(linuxSubcommand()) root.AddCommand(oohelperdSubcommand()) diff --git a/pkg/gobash/.gitignore b/pkg/gobash/.gitignore new file mode 100644 index 0000000000..6a0281cdc8 --- /dev/null +++ b/pkg/gobash/.gitignore @@ -0,0 +1 @@ +/gobash.exe diff --git a/pkg/gobash/go.mod b/pkg/gobash/go.mod new file mode 100644 index 0000000000..fa4a4a2207 --- /dev/null +++ b/pkg/gobash/go.mod @@ -0,0 +1,3 @@ +module github.com/ooni/probe-cli/pkg/gobash + +go 1.16 diff --git a/pkg/gobash/main.go b/pkg/gobash/main.go new file mode 100644 index 0000000000..8a2788a9b1 --- /dev/null +++ b/pkg/gobash/main.go @@ -0,0 +1,22 @@ +package main + +import ( + "io/ioutil" + "log" +) + +func main() { + // read the content of GOVERSION + data, err := ioutil.ReadFile("GOVERSION") + if err != nil { + log.Fatal(err) + } + + // strip trailing newlines + for len(data) > 0 && data[len(data)-1] == '\r' || data[len(data)-1] == '\n' { + data = data[:len(data)-1] + } + + // run the specified version of go + Run("go" + string(data)) +} diff --git a/pkg/gobash/signal_notunix.go b/pkg/gobash/signal_notunix.go new file mode 100644 index 0000000000..cdb15eb962 --- /dev/null +++ b/pkg/gobash/signal_notunix.go @@ -0,0 +1,14 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build windows +// +build windows + +package main + +import ( + "os" +) + +var signalsToIgnore = []os.Signal{os.Interrupt} diff --git a/pkg/gobash/signal_unix.go b/pkg/gobash/signal_unix.go new file mode 100644 index 0000000000..dc64c79553 --- /dev/null +++ b/pkg/gobash/signal_unix.go @@ -0,0 +1,15 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin || freebsd || linux || openbsd +// +build darwin freebsd linux openbsd + +package main + +import ( + "os" + "syscall" +) + +var signalsToIgnore = []os.Signal{os.Interrupt, syscall.SIGQUIT} diff --git a/pkg/gobash/version.go b/pkg/gobash/version.go new file mode 100644 index 0000000000..190a6aed92 --- /dev/null +++ b/pkg/gobash/version.go @@ -0,0 +1,538 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "crypto/sha256" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "os/exec" + "os/signal" + "os/user" + "path" + "path/filepath" + "runtime" + "strconv" + "strings" + "time" +) + +func init() { + http.DefaultTransport = &userAgentTransport{http.DefaultTransport} +} + +// Run runs the "go" tool of the provided Go version. +func Run(version string) { + log.SetFlags(0) + + root, err := goroot(version) + if err != nil { + log.Fatalf("%s: %v", version, err) + } + + if len(os.Args) == 2 && os.Args[1] == "download" { + if err := install(root, version); err != nil { + log.Fatalf("%s: download failed: %v", version, err) + } + os.Exit(0) + } + + if _, err := os.Stat(filepath.Join(root, unpackedOkay)); err != nil { + log.Fatalf("%s: not downloaded. Run '%s download' to install to %v", version, version, root) + } + + runGo(root) +} + +func runGo(root string) { + gobin := filepath.Join(root, "bin", "go"+exe()) + cmd := exec.Command(gobin, os.Args[1:]...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + newPath := filepath.Join(root, "bin") + if p := os.Getenv("PATH"); p != "" { + newPath += string(filepath.ListSeparator) + p + } + cmd.Env = dedupEnv(caseInsensitiveEnv, append(os.Environ(), "GOROOT="+root, "PATH="+newPath)) + + handleSignals() + + if err := cmd.Run(); err != nil { + // TODO: return the same exit status maybe. + os.Exit(1) + } + os.Exit(0) +} + +func fmtSize(size int64) string { + const ( + byte_unit = 1 << (10 * iota) + kilobyte_unit + megabyte_unit + ) + + unit := "B" + value := float64(size) + + switch { + case size >= megabyte_unit: + unit = "MB" + value = value / megabyte_unit + case size >= kilobyte_unit: + unit = "KB" + value = value / kilobyte_unit + } + formatted := strings.TrimSuffix(strconv.FormatFloat(value, 'f', 1, 64), ".0") + return fmt.Sprintf("%s %s", formatted, unit) +} + +// install installs a version of Go to the named target directory, creating the +// directory as needed. +func install(targetDir, version string) error { + if _, err := os.Stat(filepath.Join(targetDir, unpackedOkay)); err == nil { + log.Printf("%s: already downloaded in %v", version, targetDir) + return nil + } + + if err := os.MkdirAll(targetDir, 0755); err != nil { + return err + } + goURL := versionArchiveURL(version) + res, err := http.Head(goURL) + if err != nil { + return err + } + if res.StatusCode == http.StatusNotFound { + return fmt.Errorf("no binary release of %v for %v/%v at %v", version, getOS(), runtime.GOARCH, goURL) + } + if res.StatusCode != http.StatusOK { + return fmt.Errorf("server returned %v checking size of %v", http.StatusText(res.StatusCode), goURL) + } + base := path.Base(goURL) + archiveFile := filepath.Join(targetDir, base) + if fi, err := os.Stat(archiveFile); err != nil || fi.Size() != res.ContentLength { + if err != nil && !os.IsNotExist(err) { + // Something weird. Don't try to download. + return err + } + if err := copyFromURL(archiveFile, goURL); err != nil { + return fmt.Errorf("error downloading %v: %v", goURL, err) + } + fi, err = os.Stat(archiveFile) + if err != nil { + return err + } + if fi.Size() != res.ContentLength { + return fmt.Errorf("downloaded file %s size %v doesn't match server size %v", archiveFile, fi.Size(), res.ContentLength) + } + } + wantSHA, err := slurpURLToString(goURL + ".sha256") + if err != nil { + return err + } + if err := verifySHA256(archiveFile, strings.TrimSpace(wantSHA)); err != nil { + return fmt.Errorf("error verifying SHA256 of %v: %v", archiveFile, err) + } + log.Printf("Unpacking %v ...", archiveFile) + if err := unpackArchive(targetDir, archiveFile); err != nil { + return fmt.Errorf("extracting archive %v: %v", archiveFile, err) + } + if err := ioutil.WriteFile(filepath.Join(targetDir, unpackedOkay), nil, 0644); err != nil { + return err + } + log.Printf("Success. You may now run '%v'", version) + return nil +} + +// unpackArchive unpacks the provided archive zip or tar.gz file to targetDir, +// removing the "go/" prefix from file entries. +func unpackArchive(targetDir, archiveFile string) error { + switch { + case strings.HasSuffix(archiveFile, ".zip"): + return unpackZip(targetDir, archiveFile) + case strings.HasSuffix(archiveFile, ".tar.gz"): + return unpackTarGz(targetDir, archiveFile) + default: + return errors.New("unsupported archive file") + } +} + +// unpackTarGz is the tar.gz implementation of unpackArchive. +func unpackTarGz(targetDir, archiveFile string) error { + r, err := os.Open(archiveFile) + if err != nil { + return err + } + defer r.Close() + madeDir := map[string]bool{} + zr, err := gzip.NewReader(r) + if err != nil { + return err + } + tr := tar.NewReader(zr) + for { + f, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + if !validRelPath(f.Name) { + return fmt.Errorf("tar file contained invalid name %q", f.Name) + } + rel := filepath.FromSlash(strings.TrimPrefix(f.Name, "go/")) + abs := filepath.Join(targetDir, rel) + + fi := f.FileInfo() + mode := fi.Mode() + switch { + case mode.IsRegular(): + // Make the directory. This is redundant because it should + // already be made by a directory entry in the tar + // beforehand. Thus, don't check for errors; the next + // write will fail with the same error. + dir := filepath.Dir(abs) + if !madeDir[dir] { + if err := os.MkdirAll(filepath.Dir(abs), 0755); err != nil { + return err + } + madeDir[dir] = true + } + wf, err := os.OpenFile(abs, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode.Perm()) + if err != nil { + return err + } + n, err := io.Copy(wf, tr) + if closeErr := wf.Close(); closeErr != nil && err == nil { + err = closeErr + } + if err != nil { + return fmt.Errorf("error writing to %s: %v", abs, err) + } + if n != f.Size { + return fmt.Errorf("only wrote %d bytes to %s; expected %d", n, abs, f.Size) + } + if !f.ModTime.IsZero() { + if err := os.Chtimes(abs, f.ModTime, f.ModTime); err != nil { + // benign error. Gerrit doesn't even set the + // modtime in these, and we don't end up relying + // on it anywhere (the gomote push command relies + // on digests only), so this is a little pointless + // for now. + log.Printf("error changing modtime: %v", err) + } + } + case mode.IsDir(): + if err := os.MkdirAll(abs, 0755); err != nil { + return err + } + madeDir[abs] = true + default: + return fmt.Errorf("tar file entry %s contained unsupported file type %v", f.Name, mode) + } + } + return nil +} + +// unpackZip is the zip implementation of unpackArchive. +func unpackZip(targetDir, archiveFile string) error { + zr, err := zip.OpenReader(archiveFile) + if err != nil { + return err + } + defer zr.Close() + + for _, f := range zr.File { + name := strings.TrimPrefix(f.Name, "go/") + + outpath := filepath.Join(targetDir, name) + if f.FileInfo().IsDir() { + if err := os.MkdirAll(outpath, 0755); err != nil { + return err + } + continue + } + + rc, err := f.Open() + if err != nil { + return err + } + + // File + if err := os.MkdirAll(filepath.Dir(outpath), 0755); err != nil { + return err + } + out, err := os.OpenFile(outpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + return err + } + _, err = io.Copy(out, rc) + rc.Close() + if err != nil { + out.Close() + return err + } + if err := out.Close(); err != nil { + return err + } + } + return nil +} + +// verifySHA256 reports whether the named file has contents with +// SHA-256 of the given wantHex value. +func verifySHA256(file, wantHex string) error { + f, err := os.Open(file) + if err != nil { + return err + } + defer f.Close() + hash := sha256.New() + if _, err := io.Copy(hash, f); err != nil { + return err + } + if fmt.Sprintf("%x", hash.Sum(nil)) != wantHex { + return fmt.Errorf("%s corrupt? does not have expected SHA-256 of %v", file, wantHex) + } + return nil +} + +// slurpURLToString downloads the given URL and returns it as a string. +func slurpURLToString(url_ string) (string, error) { + res, err := http.Get(url_) + if err != nil { + return "", err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return "", fmt.Errorf("%s: %v", url_, res.Status) + } + slurp, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", fmt.Errorf("reading %s: %v", url_, err) + } + return string(slurp), nil +} + +// copyFromURL downloads srcURL to dstFile. +func copyFromURL(dstFile, srcURL string) (err error) { + f, err := os.Create(dstFile) + if err != nil { + return err + } + defer func() { + if err != nil { + f.Close() + os.Remove(dstFile) + } + }() + c := &http.Client{ + Transport: &userAgentTransport{&http.Transport{ + // It's already compressed. Prefer accurate ContentLength. + // (Not that GCS would try to compress it, though) + DisableCompression: true, + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + }}, + } + res, err := c.Get(srcURL) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return errors.New(res.Status) + } + pw := &progressWriter{w: f, total: res.ContentLength, output: os.Stderr} + n, err := io.Copy(pw, res.Body) + if err != nil { + return err + } + if res.ContentLength != -1 && res.ContentLength != n { + return fmt.Errorf("copied %v bytes; expected %v", n, res.ContentLength) + } + pw.update() // 100% + return f.Close() +} + +type progressWriter struct { + w io.Writer + n int64 + total int64 + last time.Time + formatted bool + output io.Writer +} + +func (p *progressWriter) update() { + end := " ..." + if p.n == p.total { + end = "" + } + if p.formatted { + fmt.Fprintf(p.output, "Downloaded %5.1f%% (%s / %s)%s\n", + (100.0*float64(p.n))/float64(p.total), + fmtSize(p.n), fmtSize(p.total), end) + } else { + fmt.Fprintf(p.output, "Downloaded %5.1f%% (%*d / %d bytes)%s\n", + (100.0*float64(p.n))/float64(p.total), + ndigits(p.total), p.n, p.total, end) + } +} + +func ndigits(i int64) int { + var n int + for ; i != 0; i /= 10 { + n++ + } + return n +} + +func (p *progressWriter) Write(buf []byte) (n int, err error) { + n, err = p.w.Write(buf) + p.n += int64(n) + if now := time.Now(); now.Unix() != p.last.Unix() { + p.update() + p.last = now + } + return +} + +// getOS returns runtime.GOOS. It exists as a function just for lazy +// testing of the Windows zip path when running on Linux/Darwin. +func getOS() string { + return runtime.GOOS +} + +// versionArchiveURL returns the zip or tar.gz URL of the given Go version. +func versionArchiveURL(version string) string { + goos := getOS() + + ext := ".tar.gz" + if goos == "windows" { + ext = ".zip" + } + arch := runtime.GOARCH + if goos == "linux" && runtime.GOARCH == "arm" { + arch = "armv6l" + } + return "https://dl.google.com/go/" + version + "." + goos + "-" + arch + ext +} + +const caseInsensitiveEnv = runtime.GOOS == "windows" + +// unpackedOkay is a sentinel zero-byte file to indicate that the Go +// version was downloaded and unpacked successfully. +const unpackedOkay = ".unpacked-success" + +func exe() string { + if runtime.GOOS == "windows" { + return ".exe" + } + return "" +} + +func goroot(version string) (string, error) { + home, err := homedir() + if err != nil { + return "", fmt.Errorf("failed to get home directory: %v", err) + } + return filepath.Join(home, "sdk", version), nil +} + +func homedir() (string, error) { + // This could be replaced with os.UserHomeDir, but it was introduced too + // recently, and we want this to work with go as packaged by Linux + // distributions. Note that user.Current is not enough as it does not + // prioritize $HOME. See also Issue 26463. + switch getOS() { + case "plan9": + return "", fmt.Errorf("%q not yet supported", runtime.GOOS) + case "windows": + if dir := os.Getenv("USERPROFILE"); dir != "" { + return dir, nil + } + return "", errors.New("can't find user home directory; %USERPROFILE% is empty") + default: + if dir := os.Getenv("HOME"); dir != "" { + return dir, nil + } + if u, err := user.Current(); err == nil && u.HomeDir != "" { + return u.HomeDir, nil + } + return "", errors.New("can't find user home directory; $HOME is empty") + } +} + +func validRelPath(p string) bool { + if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") { + return false + } + return true +} + +type userAgentTransport struct { + rt http.RoundTripper +} + +func (uat userAgentTransport) RoundTrip(r *http.Request) (*http.Response, error) { + version := runtime.Version() + if strings.Contains(version, "devel") { + // Strip the SHA hash and date. We don't want spaces or other tokens (see RFC2616 14.43) + version = "devel" + } + r.Header.Set("User-Agent", "golang-x-build-version/"+version) + return uat.rt.RoundTrip(r) +} + +// dedupEnv returns a copy of env with any duplicates removed, in favor of +// later values. +// Items are expected to be on the normal environment "key=value" form. +// If caseInsensitive is true, the case of keys is ignored. +// +// This function is unnecessary when the binary is +// built with Go 1.9+, but keep it around for now until Go 1.8 +// is no longer seen in the wild in common distros. +// +// This is copied verbatim from golang.org/x/build/envutil.Dedup at CL 10301 +// (commit a91ae26). +func dedupEnv(caseInsensitive bool, env []string) []string { + out := make([]string, 0, len(env)) + saw := map[string]int{} // to index in the array + for _, kv := range env { + eq := strings.Index(kv, "=") + if eq < 1 { + out = append(out, kv) + continue + } + k := kv[:eq] + if caseInsensitive { + k = strings.ToLower(k) + } + if dupIdx, isDup := saw[k]; isDup { + out[dupIdx] = kv + } else { + saw[k] = len(out) + out = append(out, kv) + } + } + return out +} + +func handleSignals() { + // Ensure that signals intended for the child process are not handled by + // this process' runtime (e.g. SIGQUIT). See issue #36976. + signal.Notify(make(chan os.Signal, 1), signalsToIgnore...) +} diff --git a/pkg/gobash/version_test.go b/pkg/gobash/version_test.go new file mode 100644 index 0000000000..4b08b9308f --- /dev/null +++ b/pkg/gobash/version_test.go @@ -0,0 +1,67 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestDedupEnv(t *testing.T) { + tests := []struct { + noCase bool + in []string + want []string + }{ + { + noCase: true, + in: []string{"k1=v1", "k2=v2", "K1=v3"}, + want: []string{"K1=v3", "k2=v2"}, + }, + { + noCase: false, + in: []string{"k1=v1", "K1=V2", "k1=v3"}, + want: []string{"k1=v3", "K1=V2"}, + }, + } + for _, tt := range tests { + got := dedupEnv(tt.noCase, tt.in) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Dedup(%v, %q) = %q; want %q", tt.noCase, tt.in, got, tt.want) + } + } +} + +func TestFormatted(t *testing.T) { + var total int64 = 1 + var buff = new(bytes.Buffer) + var units = []string{"B", "KB", "MB"} + for i := 1; i < 4; i++ { + pw := &progressWriter{w: nil, total: total, formatted: true, output: buff} + pw.update() + total *= 1024 + expected := fmt.Sprintf("%d %s", 1, units[i-1]) + if !strings.Contains(buff.String(), expected) { + t.Errorf("expected: %s received: %s", expected, buff.String()) + } + } +} + +func TestUnFormatted(t *testing.T) { + var total int64 = 1 + var buff = new(bytes.Buffer) + for i := 1; i < 4; i++ { + pw := &progressWriter{w: nil, total: total, formatted: false, output: buff} + pw.update() + expected := fmt.Sprintf("%d bytes", total) + if !strings.Contains(buff.String(), expected) { + t.Errorf("expected: %s received: %s", expected, buff.String()) + } + total *= 1024 + } +} diff --git a/script/go.bash b/script/go.bash index 86fff4ec2b..40864a0ae5 100755 --- a/script/go.bash +++ b/script/go.bash @@ -1,7 +1,9 @@ #!/bin/bash set -euxo pipefail -# We invoke ./script/internal/go.bash through the gofixpath subcommand such that -# the "go" binary in PATH is the correct version of go. + +# We use ./pkg/gobash to ensure we execute the correct version of go. # -# See https://github.com/ooni/probe/issues/2664 -go run ./internal/cmd/buildtool gofixpath -- ./script/internal/go.bash "$@" +# See https://github.com/ooni/probe/issues/2664 for context. +(cd ./pkg/gobash && go build -v -o gobash.exe .) +./pkg/gobash/gobash.exe download +./pkg/gobash/gobash.exe "$@" diff --git a/script/nocopyreadall.bash b/script/nocopyreadall.bash index 15299fca4f..748461172a 100755 --- a/script/nocopyreadall.bash +++ b/script/nocopyreadall.bash @@ -89,6 +89,12 @@ for file in $(find . -type f -name \*.go); do continue fi + if [ "$file" = "./pkg/gobash/version.go" ]; then + # We're allowed to use ReadAll and Copy in this file because + # this code is only used by the build system + continue + fi + if grep -q 'io\.ReadAll' $file; then echo "in $file: do not use io.ReadAll, use netxlite.ReadAllContext" 1>&2 exitcode=1