Deprecating escript: Introducing golang only tests (lf-edge#1037)

Instead of using escript to write tests, one now can write tests fully in golang and run those like this

```
cd tests/sec
go test
```

So there's no layered complexity anymore, compared to calling calling forked from golang test binary, which is written by us which actually interprets text files as bash commands, execs those via exec and sometimes substitute some configuration with that's been written in specification below or call arbitratry bash scripts, or compare values from stdout and file using one character command !

For more information refer to lf-edge#1037 

Signed-off-by: Pavel Abramov <[email protected]>

Author: uncleDecart

.github/actions/setup-environment/action.yml

@@ -65,6 +65,7 @@ runs:
           ./eden config set default --key=eve.accel --value=false
           ./eden config set default --key=eve.firmware --value="$(pwd)/firmware/OVMF_CODE.fd $(pwd)/firmware/OVMF_VARS.fd"
         fi
+        ./dist/bin/eden+ports.sh 2223:2223 2224:2224 5912:5902 5911:5901 8027:8027 8028:8028 8029:8029 8030:8030 8031:8031
         ./eden config set default --key=eve.tpm --value=${{ inputs.tpm_enabled }}
         ./eden config set default --key=eve.cpu --value=2
       shell: bash
@@ -116,3 +117,10 @@ runs:
         ./eden setup -v debug --grub-options='set_global dom0_extra_args "$dom0_extra_args eve_install_zfs_with_raid_level "'
       shell: bash
       working-directory: "./eden"
+
+    - name: Start and Onboard
+      run: |
+        ./eden start -v debug
+        ./eden eve onboard -v debug
+      shell: bash
+      working-directory: "./eden"

cmd/certs.go

@@ -20,7 +20,7 @@ func newCertsCmd(cfg *openevec.EdenSetupArgs) *cobra.Command {
 		Short: "manage certs",
 		Long:  `Managed certificates for Adam and EVE.`,
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := eden.GenerateEveCerts(cfg.Eden.CertsDir, cfg.Adam.CertsDomain, cfg.Adam.CertsIP, cfg.Adam.CertsEVEIP, cfg.Eve.CertsUUID, cfg.Eve.DevModel, cfg.Eve.Ssid, cfg.Eve.Password, grubOptions, cfg.Adam.APIv1); err != nil {
+			if err := eden.GenerateEveCerts(cfg.Eden.CertsDir, cfg.Adam.CertsDomain, cfg.Adam.CertsIP, cfg.Adam.CertsEVEIP, cfg.Eve.CertsUUID, cfg.Eve.DevModel, cfg.Eve.Ssid, cfg.Eve.Arch, cfg.Eve.Password, grubOptions, cfg.Adam.APIv1); err != nil {
 				log.Errorf("cannot GenerateEveCerts: %s", err)
 			} else {
 				log.Info("GenerateEveCerts done")

cmd/edenConfig.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"os"
 
 	"github.com/lf-edge/eden/pkg/defaults"
 	"github.com/lf-edge/eden/pkg/openevec"
@@ -10,7 +11,14 @@ import (
 )
 
 func newConfigCmd(configName, verbosity *string) *cobra.Command {
-	cfg := &openevec.EdenSetupArgs{}
+	currentPath, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
+	cfg, err := openevec.GetDefaultConfig(currentPath)
+	if err != nil {
+		log.Fatalf("Failed to generate default config %v\n", err)
+	}
 	var configCmd = &cobra.Command{
 		Use:               "config",
 		Short:             "work with config",
@@ -73,13 +81,13 @@ func newConfigAddCmd(cfg *openevec.EdenSetupArgs) *cobra.Command {
 	configAddCmd.Flags().StringVarP(&cfg.Eve.QemuFileToSave, "qemu-config", "", defaults.DefaultQemuFileToSave, "file to save config")
 	configAddCmd.Flags().IntVarP(&cfg.Eve.QemuCpus, "cpus", "", defaults.DefaultCpus, "cpus")
 	configAddCmd.Flags().IntVarP(&cfg.Eve.QemuMemory, "memory", "", defaults.DefaultMemory, "memory (MB)")
-	configAddCmd.Flags().StringSliceVarP(&cfg.Eve.QemuFirmware, "eve-firmware", "", nil, "firmware path")
-	configAddCmd.Flags().StringVarP(&cfg.Eve.QemuConfigPath, "config-part", "", "", "path for config drive")
-	configAddCmd.Flags().StringVarP(&cfg.Eve.QemuDTBPath, "dtb-part", "", "", "path for device tree drive (for arm)")
+	configAddCmd.Flags().StringSliceVarP(&cfg.Eve.QemuFirmware, "eve-firmware", "", cfg.Eve.QemuFirmware, "firmware path")
+	configAddCmd.Flags().StringVarP(&cfg.Eve.QemuConfigPath, "config-part", "", cfg.Eve.QemuConfigPath, "path for config drive")
+	configAddCmd.Flags().StringVarP(&cfg.Eve.QemuDTBPath, "dtb-part", "", cfg.Eve.QemuDTBPath, "path for device tree drive (for arm)")
 	configAddCmd.Flags().StringToStringVarP(&cfg.Eve.HostFwd, "eve-hostfwd", "", defaults.DefaultQemuHostFwd, "port forward map")
 	configAddCmd.Flags().StringVar(&cfg.Eve.Ssid, "ssid", "", "set ssid of wifi for rpi")
-	configAddCmd.Flags().StringVar(&cfg.Eve.Arch, "arch", "", "arch of EVE (amd64 or arm64)")
-	configAddCmd.Flags().StringVar(&cfg.Eve.ModelFile, "devmodel-file", "", "File to use for overwrite of model defaults")
+	configAddCmd.Flags().StringVar(&cfg.Eve.Arch, "arch", cfg.Eve.Arch, "arch of EVE (amd64 or arm64)")
+	configAddCmd.Flags().StringVar(&cfg.Eve.ModelFile, "devmodel-file", cfg.Eve.ModelFile, "File to use for overwrite of model defaults")
 	configAddCmd.Flags().BoolVarP(&force, "force", "", false, "force overwrite config file")
 
 	return configAddCmd

cmd/edenSetup.go

@@ -23,9 +23,6 @@ func newSetupCmd(configName, verbosity *string) *cobra.Command {
 		Long:              `Setup harness.`,
 		PersistentPreRunE: preRunViperLoadFunction(cfg, configName, verbosity),
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := openevec.ConfigCheck(*configName); err != nil {
-				log.Fatalf("Config check failed %s", err)
-			}
 			if err := openEVEC.SetupEden(*configName, configDir, softSerial, zedControlURL, ipxeOverride, grubOptions, netboot, installer); err != nil {
 
 				log.Fatalf("Setup eden failed: %s", err)

cmd/edenStop.go

@@ -27,7 +27,7 @@ func newStopCmd(configName, verbosity *string) *cobra.Command {
 				registryRm, eServerRm,
 				cfg.Eve.Remote, cfg.Eve.Pid,
 				swtpmPidFile(cfg), cfg.Sdn.PidFile,
-				cfg.Eve.DevModel, vmName,
+				cfg.Eve.DevModel, vmName, cfg.Sdn.Disable,
 			)
 		},
 	}

cmd/edenTest.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/lf-edge/eden/pkg/defaults"
 	"github.com/lf-edge/eden/pkg/openevec"
-	"github.com/lf-edge/eden/pkg/utils"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
@@ -42,7 +41,7 @@ test <test_dir> -r <regexp> [-t <timewait>] [-v <level>]
 				}
 			}
 
-			vars, err := utils.InitVars()
+			vars, err := openevec.InitVarsFromConfig(cfg)
 
 			if err != nil {
 				return fmt.Errorf("error reading config: %s\n", err)

docs/design-decisions.md

@@ -0,0 +1,73 @@
+# Design decisions
+
+This document is a collection of high-level decisions, that have been made in the project. Why do we have a section about how things use to be? In case if we want to revisit certain concept it will be useful to know why we chose one approach over another.
+
+## Using Domain Specific Language (DSL) vs writing native golang tests
+
+When talking about Eden tests till version 0.9.12, we are talking about `escript`, a Domain Specific Language (DSL) which describes test case and uses Eden to setup, environment. Escript looks like this
+
+```bash
+# 1 Setup environment variables
+{{$port := "2223"}}
+{{$network_name := "n1"}}
+{{$app_name := "eclient"}}
+
+# ...
+
+# 2 run eden commands
+eden -t 1m network create 10.11.12.0/24 -n {{$network_name}}
+
+# 3 run escript commands
+test eden.network.test -test.v -timewait 10m ACTIVATED {{$network_name}}
+
+eden pod deploy -n {{$app_name}} --memory=512MB {{template "eclient_image"}} -p {{$port}}:22 --networks={{$network_name}}
+
+# 4 execute shell script which are defined inside escript file
+exec -t 5m bash ssh.sh
+stdout 'Ubuntu'
+
+# 5 overwrite configuration
+-- eden-config.yml --
+{{/* Test's config. file */}}
+test:
+    controller: adam://{{EdenConfig "adam.ip"}}:{{EdenConfig "adam.port"}}
+    eve:
+      {{EdenConfig "eve.name"}}:
+        onboard-cert: {{EdenConfigPath "eve.cert"}}
+        serial: "{{EdenConfig "eve.serial"}}"
+        model: {{EdenConfig "eve.devmodel"}}
+
+-- ssh.sh --
+EDEN={{EdenConfig "eden.root"}}/{{EdenConfig "eden.bin-dist"}}/{{EdenConfig "eden.eden-bin"}}
+for i in `seq 20`
+do
+  sleep 20
+  # Test SSH-access to container
+  echo $i\) $EDEN sdn fwd eth0 {{$port}} -- {{template "ssh"}} grep Ubuntu /etc/issue
+  $EDEN sdn fwd eth0 {{$port}} -- {{template "ssh"}} grep Ubuntu /etc/issue && break
+done
+```
+
+So you can
+
+1) setup some environment variables
+2) run eden commands
+3) run escript commands with test
+4) execute user-defined shell scripts
+5) overwrite eden configuration
+
+Escript file is fed as input to eden test command, which parses the variables using golang templates to substitute some variables using templates in golang, then it's going to read line by line and execute it via os.exec call in golang. test is actually a compiled golang binary, we compile it inside eden repo and then execute it, under the hood it is a manually forked version of standard golang test package with some added commands.
+
+For more information on specific topics refer to following documents:
+
+- Escript test structure [doc](./escript/test-anatomy-sample.md)
+- Writing eden tasks for escript [doc](./escript/task-writing.md)
+- Running tests [doc](./escript/test-running.md)
+
+So there are several problems with that approach:
+
+**You have to be an escript expert**: when new person comes to a project, they will have some industry skills, like C++ expertise, Rust, Golang,  Computer Networking, etc. but if you never worked on this project, you never heard about escript, its' sole purpose was to be part of Eden and be useful language to describe tests for Eden. One might argue, but that's just bash on steroids. Well, true, but do you know what `!` means? It's not equal parameter in escript. So it is like bash, but not exactly it, and it might take time to figure out other hidden features, that could be solved by proper documentation. But before spiraling down that conversation lets think about tools? Imagine, that you wrote new fancy eden test and it doesn't work. Usual thing, you would say. Test Driven Development is all about  writing failing tests first and then making them work.
+
+**But how do you debug that test?** Because you're running an interpreter, which runs bash commands via os.exec plus you execute golang program, which is written and compiled inside the repository. Debug printing would work, but I find debuggers much more useful and efficient most of the times. It is a matter of taste, but better to debugger at your disposal, than not to have it in this case. Even worse, how do you debug escript problem? You need to know it's internals, there's no Stackoverflow or Google by your side, nor there are books about it.
+
+**Bumping golang is actually manual action**: last but not least, we have to maintain custom test files which a basically copy-paste with one added function, bumping golang turns into some weird dances in that case.

docs/task-writing.md docs/escript/task-writing.md

@@ -0,0 +1,98 @@
+# How to write tests with Eden
+
+The eden golang test SDK for eve consists of two categories of functions:
+
+1. `openevec`: infrastructure manager. `openevec` enables you to deploy a controller, a backend for that controller, deploy an edge node, etc. It does not change anything on an individual edge node.
+2. `evetestkit`: provides useful collection of functions to describe expected state of the system (controller, EVE, AppInstances)
+
+And those functions are used in context of standard golang test library. We do a setup in TestMain (for more info check [this](https://pkg.go.dev/testing#hdr-Main)) and then write test functions which interact with the environment we created. Setting up environment takes couple of minutes, so it makes sense to do it once and run tests within that environment. 
+
+Source for the example below can be found [here](../tests/sec/sec_test.go)
+
+In order to test a feature in Eden you need to
+
+## 1. Create a configuration file and describe your environment
+
+The glue between `openevec` and `evetestkit` is a configuration structure called `EdenSetupArgs`. It contains all the necessary information to setup all the components of the test: controller, EVE, etc. You can fill in the structure manually, however `openevec` provides you with a convenient function to create default configuration structure providing project root path:
+
+```go
+// ...
+func TestMain(m *testing.M) {
+  currentPath, err := os.Getwd()
+  if err != nil {
+    log.Fatal(err)
+  }
+  twoLevelsUp := filepath.Dir(filepath.Dir(currentPath))
+  
+  cfg := openevec.GetDefaultConfig(twoLevelsUp)
+  
+  if err = openevec.ConfigAdd(cfg, cfg.ConfigName, "", false); err != nil {
+    log.Fatal(err)
+  }
+  // ...
+}
+```
+
+Due to backward compatibility with escript as of time of writing, the project root path should be Eden repository folder. So if you add tests in the `eden/tests/my-awesome-test` you need to go two levels up, will be removed with escript
+Also we need to write configuration file to file system, because there are components (like changer) which read configuration from file system, will be removed with escript
+
+## 2. Initialize `openevec` Setup, Start and Onboard EVE node
+
+When configuration you need `openevec` to create all needed certificates, start backend. For that we create `openevec` object based on a configuration provided, it is just a convinient wrapper.
+
+```go
+// ... 
+func TestMain(m *testing.M) {
+	// ...
+	evec := openvec.CreateOpenEVEC(cfg)
+
+	evec.SetupEden(/* ... */)
+	evec.StartEden(/* ... */)
+	evec.OnboardEve(/* ... */)
+
+  // ...
+}
+```
+
+## 3. Initialize `evetestkit` and run test suite
+
+`evetestkit` provides an abstraction over EveNode, which is used to describe expected state of the system. Each EveNode is running within a project You can create a global object within one test file to use it across multiple tests. Note that EveNode is not threadsafe, since controller is stateful, so tests should be run consequently (no t.Parallel())
+
+```go
+const projectName = "security-test"
+var eveNode *evetestkit.EveNode
+
+func TestMain(m *testing.M) {
+  // ...
+	node, err := evetestkit.InitializeTestFromConfig(projectName, cfg, evetestkit.WithControllerVerbosity("debug"))
+	if err != nil {
+		log.Fatalf("Failed to initialize test: %v", err)
+	}
+
+	eveNode = node
+	res := m.Run()
+	os.Exit(res)  
+}
+```
+
+## 4. Write your test
+
+Below is an example of test, which check if AppArmor is enabled (specific file on EVE exists). It uses `EveReadFile` function from `evetestkit`
+
+```go
+const appArmorStatus = "/sys/module/apparmor/parameters/enabled"
+// ...
+func TestAppArmorEnabled(t *testing.T) {
+	log.Println("TestAppArmorEnabled started")
+	defer log.Println("TestAppArmorEnabled finished")
+
+	out, err := eveNode.EveReadFile(appArmorStatus)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	exits := strings.TrimSpace(string(out))
+	if exits != "Y" {
+		t.Fatal("AppArmor is not enabled")
+	}
+```