Tidy up Go codebase

.github/workflows/build.yml

@@ -115,10 +115,12 @@ jobs:
         with:
           go-version-file: runner/go.mod
           cache-dependency-path: runner/go.sum
+      - name: Check if go.mod and go.sum are up-to-date
+        run: go mod tidy -diff
       - name: Run golangci-lint
         uses: golangci/golangci-lint-action@v6
         with:
-          version: v1.58
+          version: v1.62.0
           args: --timeout=20m
           working-directory: runner
       - name: Test

.pre-commit-config.yaml

@@ -7,8 +7,9 @@ repos:
         args: ['--fix']
       - id: ruff-format
   - repo: https://github.com/golangci/golangci-lint
-    rev: v1.58.1
+    rev: v1.62.0  # Should match .github/workflows/build.yml
     hooks:
       - id: golangci-lint-full
-        entry: bash -c 'cd runner && golangci-lint run -D depguard --presets import,module,unused "$@"'
+        language_version: 1.23.0  # Should match runner/go.mod
+        entry: bash -c 'cd runner && golangci-lint run'
         stages: [manual]

runner/.golangci-lint.yml → runner/.golangci.yml

@@ -6,19 +6,17 @@ run:
   timeout: 1m
 
   # exit code when at least one issue was found, default is 1
-  issues-exit-code: 0
+  issues-exit-code: 1
 
   # include test files or not, default is true
   tests: false
 
-  skip-dirs:
-    - data
-
 # output configuration options
 output:
   # colored-line-number|line-number|json|tab|checkstyle|code-climate|junit-xml|github-actions
   # default is "colored-line-number"
-  format: colored-line-number
+  formats:
+    - format: colored-line-number
 
 # all available settings of specific linters
 linters-settings:
@@ -40,7 +38,6 @@ linters-settings:
     # default is false: such cases aren't reported by default.
     check-blank: false
 
-
   errorlint:
     # Check whether fmt.Errorf uses the %w verb for formatting errors. See the readme for caveats
     errorf: true
@@ -57,15 +54,6 @@ linters-settings:
     # switch
     default-signifies-exhaustive: false
 
-  exhaustivestruct:
-    # Struct Patterns is list of expressions to match struct packages and names
-    # The struct packages have the form example.com/package.ExampleStruct
-    # The matching patterns can use matching syntax from https://pkg.go.dev/path#Match
-    # If this list is empty, all structs are tested.
-    struct-patterns:
-      - '*.Test'
-      - 'example.com/package.ExampleStruct'
-
   funlen:
     lines: 60
     statements: 40
@@ -157,10 +145,6 @@ linters-settings:
     # it's a comma-separated list of prefixes
     local-prefixes: github.com/dstackai/dstackai
 
-  golint:
-    # minimal confidence for issues, default is 0.8
-    min-confidence: 0.8
-
   gosec:
     # To select a subset of rules to run.
     # Available rules: https://github.com/securego/gosec#available-rules
@@ -186,15 +170,10 @@ linters-settings:
         truncate: "32"
 
   gosimple:
-    # Select the Go version to target. The default is '1.13'.
-    go: "1.18"
     # https://staticcheck.io/docs/options#checks
     checks: [ "all" ]
 
   govet:
-    # report about shadowed variables
-    check-shadowing: true
-
     # settings per analyzer
     settings:
       printf: # analyzer name, run `go tool vet help` to see all analyzers
@@ -206,19 +185,10 @@ linters-settings:
 
     # enable or disable analyzers by name
     # run `go tool vet help` to see all analyzers
-    enable:
-      - atomicalign
-    enable-all: false
+    enable-all: true
     disable:
       - shadow
-    disable-all: false
-
-  ifshort:
-    # Maximum length of variable declaration measured in number of lines, after which linter won't suggest using short syntax.
-    # Has higher priority than max-decl-chars.
-    max-decl-lines: 1
-    # Maximum length of variable declaration measured in number of characters, after which linter won't suggest using short syntax.
-    max-decl-chars: 30
+      - fieldalignment
 
   importas:
     # if set to `true`, force to use alias.
@@ -245,14 +215,10 @@ linters-settings:
     tab-width: 1
 
   staticcheck:
-    # Select the Go version to target. The default is '1.13'.
-    go: "1.18"
     # https://staticcheck.io/docs/options#checks
     checks: [ "all" ]
 
   stylecheck:
-    # Select the Go version to target. The default is '1.13'.
-    go: "1.18"
     # https://staticcheck.io/docs/options#checks
     checks: [ "all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022" ]
     # https://staticcheck.io/docs/options#dot_import_whitelist
@@ -298,25 +264,22 @@ linters-settings:
       name: true
       begin: true
 
-  unused:
-    # Select the Go version to target. The default is '1.13'.
-    go: "1.18"
-
   whitespace:
     multi-if: false   # Enforces newlines (or comments) after every multi-line if statement
     multi-func: false # Enforces newlines (or comments) after every multi-line function signatur
 
 linters:
-  enable:
-    - megacheck
-    - govet
-  disable:
-    - maligned
-    - prealloc
-  disable-all: false
   presets:
     - bugs
     - unused
+    - import
+    - module
+    - format
+  disable:
+    - depguard
+    - gomodguard
+    - unparam
+    - contextcheck
   fast: false
 
 

runner/cmd/shim/main.go

@@ -137,7 +137,7 @@ func main() {
 					log.Printf("Config Runner: %+v\n", args.Runner)
 					log.Printf("Config Docker: %+v\n", args.Docker)
 
-					dockerRunner, err := shim.NewDockerRunner(args)
+					dockerRunner, err := shim.NewDockerRunner(&args)
 					if err != nil {
 						return cli.Exit(err, 1)
 					}

runner/go.mod

@@ -1,24 +1,25 @@
 module github.com/dstackai/dstack/runner
 
-go 1.21
-
-toolchain go1.21.9
+go 1.23
 
 require (
 	github.com/alexellis/go-execute/v2 v2.2.1
 	github.com/bluekeyes/go-gitdiff v0.7.2
 	github.com/creack/pty v1.1.21
 	github.com/docker/docker v26.0.0+incompatible
 	github.com/docker/go-connections v0.5.0
+	github.com/docker/go-units v0.5.0
 	github.com/go-git/go-git/v5 v5.12.0
 	github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f
 	github.com/icza/backscanner v0.0.0-20240328210400-b40c3a86dec5
 	github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf
+	github.com/shirou/gopsutil/v3 v3.24.3
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.9.0
 	github.com/urfave/cli/v2 v2.27.1
 	github.com/ztrue/tracerr v0.4.0
 	golang.org/x/crypto v0.22.0
+	golang.org/x/sys v0.19.0
 )
 
 require (
@@ -31,20 +32,21 @@ require (
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/distribution/reference v0.6.0 // indirect
-	github.com/docker/go-units v0.5.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.5.0 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/h2non/filetype v1.1.3 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/juju/errors v1.0.0 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/klauspost/compress v1.17.8 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/term v0.5.0 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
@@ -53,13 +55,16 @@ require (
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
-	github.com/shirou/gopsutil/v3 v3.24.3 // indirect
 	github.com/skeema/knownhosts v1.2.2 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/ulikunitz/xz v0.5.12 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0 // indirect
 	go.opentelemetry.io/otel v1.25.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.25.0 // indirect
@@ -69,7 +74,6 @@ require (
 	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/net v0.24.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/tools v0.20.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect

runner/go.sum

@@ -70,6 +70,7 @@ github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
 github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-stack/stack v1.6.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -120,6 +121,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
 github.com/magiconair/properties v1.7.4-0.20170902060319-8d7837e64d3c/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mattn/go-colorable v0.0.10-0.20170816031813-ad5389df28cd/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
@@ -144,6 +146,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
 github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
@@ -177,7 +180,9 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
 github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
+github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
 github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
 github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
 github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
@@ -190,6 +195,7 @@ github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQut
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 github.com/ztrue/tracerr v0.4.0 h1:vT5PFxwIGs7rCg9ZgJ/y0NmOpJkPCPFK8x0vVIYzd04=
 github.com/ztrue/tracerr v0.4.0/go.mod h1:PaFfYlas0DfmXNpo7Eay4MFhZUONqvXM+T2HyGPpngk=

runner/internal/executor/executor.go

@@ -263,11 +263,11 @@ func (ex *RunExecutor) setupCredentials(ctx context.Context) (func(), error) {
 		if _, err := os.Stat(keyPath); err == nil {
 			return nil, gerrors.New("private key already exists")
 		}
-		if err := os.MkdirAll(filepath.Dir(keyPath), 0700); err != nil {
+		if err := os.MkdirAll(filepath.Dir(keyPath), 0o700); err != nil {
 			return nil, gerrors.Wrap(err)
 		}
 		log.Info(ctx, "Writing private key", "path", keyPath)
-		if err := os.WriteFile(keyPath, []byte(*ex.repoCredentials.PrivateKey), 0600); err != nil {
+		if err := os.WriteFile(keyPath, []byte(*ex.repoCredentials.PrivateKey), 0o600); err != nil {
 			return nil, gerrors.Wrap(err)
 		}
 		return func() {
@@ -282,7 +282,7 @@ func (ex *RunExecutor) setupCredentials(ctx context.Context) (func(), error) {
 		if _, err := os.Stat(hostsPath); err == nil {
 			return nil, gerrors.New("hosts.yml file already exists")
 		}
-		if err := os.MkdirAll(filepath.Dir(hostsPath), 0700); err != nil {
+		if err := os.MkdirAll(filepath.Dir(hostsPath), 0o700); err != nil {
 			return nil, gerrors.Wrap(err)
 		}
 		log.Info(ctx, "Writing OAuth token", "path", hostsPath)
@@ -291,7 +291,7 @@ func (ex *RunExecutor) setupCredentials(ctx context.Context) (func(), error) {
 			return nil, gerrors.Wrap(err)
 		}
 		ghHost := fmt.Sprintf("%s:\n  oauth_token: \"%s\"\n", cloneURL.Hostname(), *ex.repoCredentials.OAuthToken)
-		if err := os.WriteFile(hostsPath, []byte(ghHost), 0644); err != nil {
+		if err := os.WriteFile(hostsPath, []byte(ghHost), 0o600); err != nil {
 			return nil, gerrors.Wrap(err)
 		}
 		return func() {
@@ -305,16 +305,15 @@ func (ex *RunExecutor) setupCredentials(ctx context.Context) (func(), error) {
 func isPtyError(err error) bool {
 	/* read /dev/ptmx: input/output error */
 	var e *os.PathError
-	return errors.As(err, &e) && e.Err == syscall.EIO
+	return errors.As(err, &e) && errors.Is(e.Err, syscall.EIO)
 }
 
 func buildLDLibraryPathEnv() (string, error) {
 	// Execute shell command to get Python prefix
 	cmd := exec.Command("bash", "-i", "-c", "python3-config --prefix")
 	output, err := cmd.Output()
-
 	if err != nil {
-		return "", fmt.Errorf("error executing command: %v", err)
+		return "", fmt.Errorf("error executing command: %w", err)
 	}
 
 	// Extract and trim the prefix path

runner/internal/executor/executor_test.go

@@ -121,7 +121,7 @@ func TestExecutor_RemoteRepo(t *testing.T) {
 		RepoConfigEmail: "[email protected]",
 	}
 	ex.jobSpec.Commands = append(ex.jobSpec.Commands, "git rev-parse HEAD && git config user.name && git config user.email")
-	err := os.WriteFile(ex.codePath, []byte{}, 0600) // empty diff
+	err := os.WriteFile(ex.codePath, []byte{}, 0o600) // empty diff
 	require.NoError(t, err)
 
 	err = ex.setupRepo(context.TODO())
@@ -164,11 +164,11 @@ func makeTestExecutor(t *testing.T) *RunExecutor {
 	}
 
 	temp := filepath.Join(baseDir, "temp")
-	_ = os.Mkdir(temp, 0700)
+	_ = os.Mkdir(temp, 0o700)
 	home := filepath.Join(baseDir, "home")
-	_ = os.Mkdir(home, 0700)
+	_ = os.Mkdir(home, 0o700)
 	repo := filepath.Join(baseDir, "repo")
-	_ = os.Mkdir(repo, 0700)
+	_ = os.Mkdir(repo, 0o700)
 	ex := NewRunExecutor(temp, home, repo)
 	ex.SetJob(body)
 	ex.SetCodePath(filepath.Join(baseDir, "code")) // note: create file before run
@@ -187,7 +187,7 @@ func makeCodeTar(t *testing.T, path string) {
 	}
 
 	for _, f := range files {
-		hdr := &tar.Header{Name: f.name, Mode: 0600, Size: int64(len(f.body))}
+		hdr := &tar.Header{Name: f.name, Mode: 0o600, Size: int64(len(f.body))}
 		require.NoError(t, tw.WriteHeader(hdr))
 		_, err := tw.Write([]byte(f.body))
 		require.NoError(t, err)

runner/internal/executor/repo.go

@@ -13,7 +13,7 @@ import (
 // setupRepo must be called from Run
 func (ex *RunExecutor) setupRepo(ctx context.Context) error {
 	if _, err := os.Stat(ex.workingDir); err != nil {
-		if err = os.MkdirAll(ex.workingDir, 0777); err != nil {
+		if err = os.MkdirAll(ex.workingDir, 0o777); err != nil {
 			return gerrors.Wrap(err)
 		}
 	}