diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7135f7f47..777d24e95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,15 +44,21 @@ jobs: run: | go test -race ./... cd website; go test -race ./... - - name: Generate test coverage + - name: Generate unit test coverage if: matrix.go-version-is == 'new' - run: go test -coverprofile=cover -coverpkg=./pkg/... ./pkg/... + run: go test -coverprofile=cover-unit.txt -coverpkg=./pkg/... ./pkg/... + - name: Generate E2E test coverage + if: matrix.go-version-is == 'new' + run: | + mkdir cover-e2e + GOCOVERDIR=$PWD/cover-e2e go test -count 1 ./e2e + go tool covdata textfmt -i cover-e2e -o cover-e2e.txt - name: Save test coverage if: matrix.go-version-is == 'new' uses: actions/upload-artifact@v4 with: name: cover-${{ matrix.os == 'ubuntu' && 'linux' || matrix.os }} - path: cover + path: cover-*.txt emulated_tests: name: Run tests (emulated ${{ matrix.go-arch }}) @@ -110,12 +116,18 @@ jobs: uses: actions/download-artifact@v4 with: name: cover-${{ matrix.ostype }} - - name: Upload coverage to codecov + - name: Upload unit test coverage to codecov + uses: codecov/codecov-action@v4 + with: + files: ./cover-unit.txt + token: ${{ secrets.CODECOV_TOKEN }} + flags: ${{ matrix.ostype }},unit + - name: Upload E2E test coverage to codecov uses: codecov/codecov-action@v4 with: - files: ./cover + files: ./cover-e2e.txt token: ${{ secrets.CODECOV_TOKEN }} - flags: ${{ matrix.ostype }} + flags: ${{ matrix.ostype }},e2e checks: name: Run checks diff --git a/Makefile b/Makefile index e0d494443..acd10679b 100644 --- a/Makefile +++ b/Makefile @@ -25,10 +25,18 @@ test: # Generate a basic test coverage report, and open it in the browser. The report # is an approximation of https://app.codecov.io/gh/elves/elvish/. cover: - go test -coverprofile=cover -coverpkg=./pkg/... ./pkg/... - ./tools/prune-cover.sh .codecov.yml cover - go tool cover -html=cover - go tool cover -func=cover | tail -1 | awk '{ print "Overall coverage:", $$NF }' + mkdir -p _cover/unit _cover/e2e + # Generate coverage from unit tests. We could generate text profiles + # directly with -coverprofile, but there's no support for merging multiple + # text profiles. So we generate binary profiles instead. + go test -coverpkg=./pkg/... ./pkg/... -test.gocoverdir $$PWD/_cover/unit + # Generate coverage from E2E tests, using -count to skip the cache. + env GOCOVERDIR=$$PWD/_cover/e2e go test -count 1 ./e2e + # Merge and convert binary profiles to a single text profile. + go tool covdata textfmt -i _cover/unit,_cover/e2e -o _cover/merged.txt + ./tools/prune-cover.sh .codecov.yml < _cover/merged.txt > _cover/pruned.txt + go tool cover -html _cover/pruned.txt + go tool cover -func _cover/pruned.txt | tail -1 | awk '{ print "Overall coverage:", $$NF }' # All the checks except check-gen.sh, which is not always convenient to run as # it requires a clean working tree. diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go new file mode 100644 index 000000000..e1908348e --- /dev/null +++ b/e2e/e2e_test.go @@ -0,0 +1,35 @@ +package e2e_test + +import ( + "embed" + "testing" + + "src.elv.sh/pkg/eval" + "src.elv.sh/pkg/eval/evaltest" + "src.elv.sh/pkg/eval/vars" + "src.elv.sh/pkg/parse" + "src.elv.sh/pkg/testutil" +) + +//go:embed *.elvts +var transcripts embed.FS + +const buildScript = ` +go build (if (not-eq $E:GOCOVERDIR '') { put -cover }) -o $workdir/elvish src.elv.sh/cmd/elvish +` + +func TestTranscripts(t *testing.T) { + workdir := t.TempDir() + err := eval.NewEvaler().Eval( + parse.Source{Name: "[build]", Code: buildScript}, + eval.EvalCfg{ + Global: eval.BuildNs().AddVars(map[string]vars.Var{ + "workdir": vars.NewReadOnly(workdir), + }).Ns()}) + if err != nil { + t.Fatal(err) + } + testutil.Chdir(t, workdir) + + evaltest.TestTranscriptsInFS(t, transcripts) +} diff --git a/e2e/shell_test.elvts b/e2e/shell_test.elvts new file mode 100644 index 000000000..24afb3bbb --- /dev/null +++ b/e2e/shell_test.elvts @@ -0,0 +1,12 @@ +////////////// +# run script # +////////////// + +# with -c # +~> ./elvish -c 'echo hello from -c' +hello from -c + +# with file # +~> echo 'echo hello from file' > script.elv + ./elvish script.elv +hello from file diff --git a/tools/prune-cover.sh b/tools/prune-cover.sh index 509ce5c38..399ac39aa 100755 --- a/tools/prune-cover.sh +++ b/tools/prune-cover.sh @@ -3,15 +3,14 @@ # Prune the same objects from the "make cover" report that we tell Codecov # (https://codecov.io/gh/elves/elvish/) to ignore. -if test $# != 2 +if test $# != 1 then - echo 'Usage: cover_prune.sh ${codecov.yml} $cover' >&2 + echo 'Usage: cover_prune.sh ${codecov.yml}' >&2 exit 1 fi yaml="$1" data="$2" sed -En '/^ignore:/,/^[^ ]/s/^ *- "(.*)"/src.elv.sh\/\1/p' $yaml > $yaml.ignore -grep -F -v -f $yaml.ignore $data > $data.pruned -mv $data.pruned $data +grep -F -v -f $yaml.ignore rm $yaml.ignore