Change iterable interface to be more flexible

.circleci/config.yml

@@ -91,7 +91,7 @@ jobs:
    build-linux-alt:
      working_directory: ~/please
      docker:
-       - image: thoughtmachine/please_ubuntu_alt:20240906
+       - image: thoughtmachine/please_ubuntu_alt:20240910
      resource_class: large
      environment:
        PLZ_ARGS: "-p -c cover --profile ci-alt"

.github/workflows/golangci-lint.yml

@@ -14,10 +14,10 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v4
         with:
-          go-version: '^1.22'
+          go-version: '^1.23'
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v4
         with:
-          version: v1.60
+          version: v1.61
           args: src/... tools/...
           skip-pkg-cache: true

README.md

@@ -188,10 +188,7 @@ file.
 
 To build Please yourself, run `./bootstrap.sh` in the repo root.
 This will bootstrap a minimal version of Please using Go and then
-rebuild it using itself.
-
-You'll need to have Go 1.21+ installed to build Please although once
-built it can target any recent version (we think back to about 1.8ish).
+rebuild it using itself. You'll need to have Go 1.23+ installed.
 
 Optional dependencies for various tests include Python, Java, clang,
 gold and docker - none of those are required to build components so

go.mod

@@ -115,4 +115,4 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-go 1.21
+go 1.23

src/BUILD.plz

@@ -53,7 +53,7 @@ filegroup(
 if is_platform(os = "linux"):
     gentest(
         name = "static_test",
-        test_cmd = "ldd $DATA && exit 1 || exit 0",
         data = [":please"],
         no_test_output = True,
+        test_cmd = "ldd $DATA && exit 1 || exit 0",
     )

src/cache/BUILD

@@ -1,6 +1,9 @@
 go_library(
     name = "cache",
-    srcs = glob(["*.go"], exclude=["*_test.go"]),
+    srcs = glob(
+        ["*.go"],
+        exclude = ["*_test.go"],
+    ),
     pgo_file = "//:pgo",
     visibility = ["PUBLIC"],
     deps = [

src/fs/BUILD

@@ -1,6 +1,9 @@
 go_library(
     name = "fs",
-    srcs = glob(["*.go"], exclude=["*_test.go"]),
+    srcs = glob(
+        ["*.go"],
+        exclude = ["*_test.go"],
+    ),
     pgo_file = "//:pgo",
     visibility = ["PUBLIC"],
     deps = [
@@ -13,7 +16,13 @@ go_library(
 
 go_test(
     name = "fs_test",
-    srcs = glob(["*_test.go"], exclude=["glob_integration_test.go", "*_benchmark_test.go"]),
+    srcs = glob(
+        ["*_test.go"],
+        exclude = [
+            "glob_integration_test.go",
+            "*_benchmark_test.go",
+        ],
+    ),
     data = ["test_data"],
     deps = [
         ":fs",

src/parse/asp/interpreter.go

@@ -2,6 +2,7 @@ package asp
 
 import (
 	"fmt"
+	"iter"
 	"path/filepath"
 	"reflect"
 	"runtime/debug"
@@ -565,9 +566,7 @@ func (s *scope) interpretIf(stmt *IfStatement) pyObject {
 }
 
 func (s *scope) interpretFor(stmt *ForStatement) pyObject {
-	it := s.iterable(&stmt.Expr)
-	for i, n := 0, it.Len(); i < n; i++ {
-		li := it.Item(i)
+	for li := range s.iterable(&stmt.Expr) {
 		s.unpackNames(stmt.Names, li)
 		if ret := s.interpretStatements(stmt.Statements); ret != nil {
 			if s, ok := ret.(pySentinel); ok && s == continueIteration {
@@ -896,8 +895,8 @@ func (s *scope) interpretList(expr *List) pyList {
 		return pyList(s.evaluateExpressions(expr.Values))
 	}
 	cs := s.NewScope(s.filename, s.mode)
-	it := s.iterable(expr.Comprehension.Expr)
-	ret := make(pyList, 0, it.Len())
+	it, l := s.iterableLen(expr.Comprehension.Expr)
+	ret := make(pyList, 0, l)
 	cs.evaluateComprehension(it, expr.Comprehension, func(li pyObject) {
 		if len(expr.Values) == 1 {
 			ret = append(ret, cs.interpretExpression(expr.Values[0]))
@@ -917,8 +916,8 @@ func (s *scope) interpretDict(expr *Dict) pyObject {
 		return d
 	}
 	cs := s.NewScope(s.filename, s.mode)
-	it := s.iterable(expr.Comprehension.Expr)
-	ret := make(pyDict, it.Len())
+	it, l := s.iterableLen(expr.Comprehension.Expr)
+	ret := make(pyDict, l)
 	cs.evaluateComprehension(it, expr.Comprehension, func(li pyObject) {
 		ret.IndexAssign(cs.interpretExpression(&expr.Items[0].Key), cs.interpretExpression(&expr.Items[0].Value))
 	})
@@ -927,22 +926,18 @@ func (s *scope) interpretDict(expr *Dict) pyObject {
 
 // evaluateComprehension handles iterating a comprehension's loops.
 // The provided callback function is called with each item to be added to the result.
-func (s *scope) evaluateComprehension(it iterable, comp *Comprehension, callback func(pyObject)) {
+func (s *scope) evaluateComprehension(it iter.Seq[pyObject], comp *Comprehension, callback func(pyObject)) {
 	if comp.Second != nil {
-		for i, n := 0, it.Len(); i < n; i++ {
-			li := it.Item(i)
+		for li := range it {
 			s.unpackNames(comp.Names, li)
-			it2 := s.iterable(comp.Second.Expr)
-			for j, n := 0, it2.Len(); j < n; j++ {
-				li2 := it2.Item(j)
+			for li2 := range s.iterable(comp.Second.Expr) {
 				if s.evaluateComprehensionExpression(comp, comp.Second.Names, li2) {
 					callback(li2)
 				}
 			}
 		}
 	} else {
-		for i, n := 0, it.Len(); i < n; i++ {
-			li := it.Item(i)
+		for li := range it {
 			if s.evaluateComprehensionExpression(comp, comp.Names, li) {
 				callback(li)
 			}
@@ -972,11 +967,22 @@ func (s *scope) unpackNames(names []string, obj pyObject) {
 }
 
 // iterable returns the result of the given expression as an iterable object.
-func (s *scope) iterable(expr *Expression) iterable {
+func (s *scope) iterable(expr *Expression) iter.Seq[pyObject] {
 	o := s.interpretExpression(expr)
 	it, ok := o.(iterable)
 	s.Assert(ok, "Non-iterable type %s", o.Type())
-	return it
+	return it.Iter()
+}
+
+// iterableLen returns the result of the given expression as an iterable object, and a length hint
+func (s *scope) iterableLen(expr *Expression) (iter.Seq[pyObject], int) {
+	o := s.interpretExpression(expr)
+	it, ok := o.(iterable)
+	s.Assert(ok, "Non-iterable type %s", o.Type())
+	if l, ok := it.(indexable); ok {
+		return it.Iter(), l.Len()
+	}
+	return it.Iter(), 4 // arbitrary length hint when we don't know
 }
 
 // evaluateExpressions runs a series of Python expressions in this scope and creates a series of concrete objects from them.

src/parse/asp/objects.go

@@ -3,6 +3,7 @@ package asp
 import (
 	"encoding/json"
 	"fmt"
+	"iter"
 	"slices"
 	"sort"
 	"strconv"
@@ -36,8 +37,11 @@ type freezable interface {
 // An iterable represents an object that can be iterated (the y in `for x in y`).
 // Not all pyObjects implement this.
 type iterable interface {
-	pyObject
-	// This isn't super generic but it works fine for all cases we have right now.
+	Iter() iter.Seq[pyObject]
+}
+
+// An indexable represents an object that knows its length and can be indexed into.
+type indexable interface {
 	Len() int
 	Item(index int) pyObject
 }
@@ -413,6 +417,16 @@ func (l pyList) String() string {
 	return fmt.Sprintf("%s", []pyObject(l))
 }
 
+func (l pyList) Iter() iter.Seq[pyObject] {
+	return func(yield func(pyObject) bool) {
+		for _, x := range l {
+			if !yield(x) {
+				break
+			}
+		}
+	}
+}
+
 // Freeze freezes this list for further updates.
 // Note that this is a "soft" freeze; callers holding the original unfrozen
 // reference can still modify it.
@@ -437,12 +451,12 @@ func (l pyList) Repeat(n pyInt) pyList {
 	return ret
 }
 
-// Len returns the length of this list, implementing iterable.
+// Len returns the length of this list, implementing indexable.
 func (l pyList) Len() int {
 	return len(l)
 }
 
-// Item returns the i'th item of this list, implementing iterable.
+// Item returns the i'th item of this list, implementing indexable.
 func (l pyList) Item(i int) pyObject {
 	return l[i]
 }
@@ -1061,6 +1075,16 @@ func (r *pyRange) Item(index int) pyObject {
 	return r.Start + newPyInt(index)*r.Step
 }
 
+func (r *pyRange) Iter() iter.Seq[pyObject] {
+	return func(yield func(pyObject) bool) {
+		for i := r.Start; i < r.Stop; i += r.Step {
+			if !yield(i) {
+				break
+			}
+		}
+	}
+}
+
 func (r *pyRange) MarshalJSON() ([]byte, error) {
 	return json.Marshal(r.toList(0))
 }

third_party/binary/BUILD

@@ -1,16 +1,10 @@
-GO_CI_LINT_VERSION = "1.60.3"
+GO_CI_LINT_VERSION = "1.61.0"
 
 remote_file(
     name = "golangci-lint",
     binary = True,
     exported_files = ["golangci-lint-%s-${OS}-${ARCH}/golangci-lint" % GO_CI_LINT_VERSION],
     extract = True,
-    hashes = [
-        "faf60366f99bb4010b634a030c45eaf57baae6c0b7e10be151139871e3fef40e", # golangci-lint-1.60.3-darwin-amd64.tar.gz
-        "deb0fbd0b99992d97808614db1214f57d5bdc12b907581e2ef10d3a392aca11f", # golangci-lint-1.60.3-darwin-arm64.tar.gz
-        "4037af8122871f401ed874852a471e54f147ff8ce80f5a304e020503bdb806ef", # golangci-lint-1.60.3-linux-amd64.tar.gz
-        "74782943b2d2edae1208be3701e0cafe62817ba90b9b4cc5ca52bdef26df12f9", # golangci-lint-1.60.3-linux-arm64.tar.gz
-    ],
     url = "https://github.com/golangci/golangci-lint/releases/download/v%s/golangci-lint-%s-%s-%s.tar.gz" % (
         GO_CI_LINT_VERSION,
         GO_CI_LINT_VERSION,

tools/http_cache/BUILD

@@ -11,6 +11,6 @@ go_binary(
 
 sh_cmd(
     name = "run_local",
-    data = [":http_cache"],
     cmd = r"exec \\$DATA -p 1771 -d /tmp/please_http_cache",
+    data = [":http_cache"],
 )

tools/images/ubuntu_alt/Dockerfile

@@ -10,7 +10,7 @@ RUN apt-get update && \
     apt-get clean
 
 # Go
-RUN curl -fsSL https://dl.google.com/go/go1.22.7.linux-amd64.tar.gz | tar -xzC /usr/local
+RUN curl -fsSL https://dl.google.com/go/go1.23.1.linux-amd64.tar.gz | tar -xzC /usr/local
 RUN ln -s /usr/local/go/bin/go /usr/local/bin/go && ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt
 
 # Locale