Add end-to-end tests.

This makes use of the support for collecting code coverage from binaries added
in Go 1.20: https://go.dev/blog/integration-test-coverage

- Add a new "e2e" directory containing end-to-end tests.

- Change the "cover" Make target to run E2E tests and merge the coverage with
  unit tests.

- Change the CI task on GitHub Actions to run those E2E tests, and upload them
  separately to CodeCov with an "e2e" flag. (Unit tests now get the "unit" flag.)

.github/workflows/ci.yml

@@ -44,15 +44,21 @@ jobs:
       run: |
         go test -race ./...
         cd website; go test -race ./...
-    - name: Generate test coverage
+    - name: Generate unit test coverage
       if: matrix.go-version-is == 'new'
-      run: go test -coverprofile=cover -coverpkg=./pkg/... ./pkg/...
+      run: go test -coverprofile=cover-unit.txt -coverpkg=./pkg/... ./pkg/...
+    - name: Generate E2E test coverage
+      if: matrix.go-version-is == 'new'
+      run: |
+        mkdir cover-e2e
+        GOCOVERDIR=$PWD/cover-e2e go test -count 1 ./e2e
+        go tool covdata textfmt -i cover-e2e -o cover-e2e.txt
     - name: Save test coverage
       if: matrix.go-version-is == 'new'
       uses: actions/upload-artifact@v4
       with:
         name: cover-${{ matrix.os == 'ubuntu' && 'linux' || matrix.os }}
-        path: cover
+        path: cover-*.txt
 
   emulated_tests:
     name: Run tests (emulated ${{ matrix.go-arch }})
@@ -110,12 +116,18 @@ jobs:
       uses: actions/download-artifact@v4
       with:
         name: cover-${{ matrix.ostype }}
-    - name: Upload coverage to codecov
+    - name: Upload unit test coverage to codecov
+      uses: codecov/codecov-action@v4
+      with:
+        files: ./cover-unit.txt
+        token: ${{ secrets.CODECOV_TOKEN }}
+        flags: ${{ matrix.ostype }},unit
+    - name: Upload E2E test coverage to codecov
       uses: codecov/codecov-action@v4
       with:
-        files: ./cover
+        files: ./cover-e2e.txt
         token: ${{ secrets.CODECOV_TOKEN }}
-        flags: ${{ matrix.ostype }}
+        flags: ${{ matrix.ostype }},e2e
 
   checks:
     name: Run checks

Makefile

@@ -25,10 +25,18 @@ test:
 # Generate a basic test coverage report, and open it in the browser. The report
 # is an approximation of https://app.codecov.io/gh/elves/elvish/.
 cover:
-	go test -coverprofile=cover -coverpkg=./pkg/... ./pkg/...
-	./tools/prune-cover.sh .codecov.yml cover
-	go tool cover -html=cover
-	go tool cover -func=cover | tail -1 | awk '{ print "Overall coverage:", $$NF }'
+	mkdir -p _cover/unit _cover/e2e
+	# Generate coverage from unit tests. We could generate text profiles
+	# directly with -coverprofile, but there's no support for merging multiple
+	# text profiles. So we generate binary profiles instead.
+	go test -coverpkg=./pkg/... ./pkg/... -test.gocoverdir $$PWD/_cover/unit
+	# Generate coverage from E2E tests, using -count to skip the cache.
+	env GOCOVERDIR=$$PWD/_cover/e2e go test -count 1 ./e2e
+	# Merge and convert binary profiles to a single text profile.
+	go tool covdata textfmt -i _cover/unit,_cover/e2e -o _cover/merged.txt
+	./tools/prune-cover.sh .codecov.yml < _cover/merged.txt > _cover/pruned.txt
+	go tool cover -html _cover/pruned.txt
+	go tool cover -func _cover/pruned.txt | tail -1 | awk '{ print "Overall coverage:", $$NF }'
 
 # All the checks except check-gen.sh, which is not always convenient to run as
 # it requires a clean working tree.

e2e/e2e_test.go

@@ -0,0 +1,35 @@
+package e2e_test
+
+import (
+	"embed"
+	"testing"
+
+	"src.elv.sh/pkg/eval"
+	"src.elv.sh/pkg/eval/evaltest"
+	"src.elv.sh/pkg/eval/vars"
+	"src.elv.sh/pkg/parse"
+	"src.elv.sh/pkg/testutil"
+)
+
+//go:embed *.elvts
+var transcripts embed.FS
+
+const buildScript = `
+go build (if (not-eq $E:GOCOVERDIR '') { put -cover }) -o $workdir/elvish src.elv.sh/cmd/elvish
+`
+
+func TestTranscripts(t *testing.T) {
+	workdir := t.TempDir()
+	err := eval.NewEvaler().Eval(
+		parse.Source{Name: "[build]", Code: buildScript},
+		eval.EvalCfg{
+			Global: eval.BuildNs().AddVars(map[string]vars.Var{
+				"workdir": vars.NewReadOnly(workdir),
+			}).Ns()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	testutil.Chdir(t, workdir)
+
+	evaltest.TestTranscriptsInFS(t, transcripts)
+}

e2e/shell_test.elvts

@@ -0,0 +1,12 @@
+//////////////
+# run script #
+//////////////
+
+# with -c #
+~> ./elvish -c 'echo hello from -c'
+hello from -c
+
+# with file #
+~> echo 'echo hello from file' > script.elv
+   ./elvish script.elv
+hello from file

tools/prune-cover.sh

@@ -3,15 +3,14 @@
 # Prune the same objects from the "make cover" report that we tell Codecov
 # (https://codecov.io/gh/elves/elvish/) to ignore.
 
-if test $# != 2
+if test $# != 1
 then
-    echo 'Usage: cover_prune.sh ${codecov.yml} $cover' >&2
+    echo 'Usage: cover_prune.sh ${codecov.yml}' >&2
     exit 1
 fi
 yaml="$1"
 data="$2"
 
 sed -En '/^ignore:/,/^[^ ]/s/^  *- "(.*)"/src.elv.sh\/\1/p' $yaml > $yaml.ignore
-grep -F -v -f $yaml.ignore $data > $data.pruned
-mv $data.pruned $data
+grep -F -v -f $yaml.ignore
 rm $yaml.ignore