Merge pull request #1297 from openziti/ha-smoketest

Consolidate HA model into simple-transfer

.github/workflows/main.yml

@@ -204,8 +204,6 @@ jobs:
           show: "fail, skip"
         if: ${{ !cancelled() }}
 
-      # END linux-build-steps
-
   fablab-smoketest-teardown:
     name: Teardown SmokeTest
     if: always()
@@ -238,6 +236,105 @@ jobs:
             echo "WARN: No instance archive found for simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz.gpg" >&2
           fi
 
+  fablab-ha-smoketest:
+    name: Fablab HA Smoketest
+    # not applicable to forks. shouldn't run on release build
+    if: github.repository_owner == 'openziti' && github.ref != 'refs/heads/main'
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Git Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Install Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.21.x'
+
+      - name: Install Ziti CI
+        uses: openziti/ziti-ci@v1
+
+      - name: Build and Test
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ziti_ci_gpg_key: ${{ secrets.ZITI_CI_GPG_KEY }}
+          ziti_ci_gpg_key_id: ${{ secrets.ZITI_CI_GPG_KEY_ID }}
+        run: |
+          $(go env GOPATH)/bin/ziti-ci configure-git
+          $(go env GOPATH)/bin/ziti-ci generate-build-info common/version/info_generated.go version
+          pushd zititest && go install ./... && popd
+          go install -tags=all,tests ./...
+
+      - name: Create Test Environment
+        run: |
+          echo "ZITI_ROOT=$(go env GOPATH)/bin" >> "$GITHUB_ENV"
+          $(go env GOPATH)/bin/simple-transfer create -d simple-transfer-ha-${GITHUB_RUN_NUMBER} -n simple-transfer-ha-${GITHUB_RUN_NUMBER} -l ha=true,environment=gh,ziti_version=$($(go env GOPATH)/bin/ziti-ci -q get-current-version)
+          $(go env GOPATH)/bin/simple-transfer up
+
+      #      - name: Test Ziti Command
+      #        run: |
+      #          echo "ZITI_ROOT=$(go env GOPATH)/bin" >> "$GITHUB_ENV"
+      #          pushd zititest && go test -timeout 30m -v ./tests/... 2>&1 | tee test.out && popd
+
+      - name: Create fablab instance archive
+        if: always()
+        env:
+          FABLAB_PASSPHRASE: ${{ secrets.FABLAB_PASSPHRASE }}
+        run: |
+          cp $(go env GOPATH)/bin/simple-transfer simple-transfer-ha-${GITHUB_RUN_NUMBER}
+          cp ~/.fablab/config.yml simple-transfer-ha-${GITHUB_RUN_NUMBER}/
+          tar --exclude *terraform-provider-aws* -czvf - simple-transfer-ha-${GITHUB_RUN_NUMBER}/ | gpg --symmetric --cipher-algo aes256 --batch --quiet --passphrase ${FABLAB_PASSPHRASE} -o simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg
+          aws s3 cp ./simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg s3://ziti-smoketest-fablab-instances/
+          
+          #      - name: Test Report Generation
+          #        if: always()
+          #        run: |
+          #          go install github.com/jstemmer/go-junit-report/v2@latest
+          #$(go env GOPATH)/bin/go-junit-report -in zititest/test.out -out test-report.xml
+
+      #      - name: Test Summary
+      #        uses: test-summary/action@v2
+      #        with:
+      #          paths: |
+      #            test-report.xml
+      #          show: "fail, skip"
+      #        if: always()
+
+      # END linux-build-steps
+
+  fablab-ha-smoketest-teardown:
+    name: Teardown HA SmokeTest
+    if: always()
+    runs-on: ubuntu-20.04
+    needs: [ fablab-ha-smoketest ]
+    steps:
+      # allow time for investigation unless the workflow is cancelled or the smoketest succeeded or was skipped
+      - name: Sleep If Failed
+        if: needs.fablab-ha-smoketest.result != 'success' && needs.fablab-ha-smoketest.result != 'skipped' && needs.fablab-ha-smoketest.result != 'cancelled'
+        run: |
+          sleep 30m
+
+      # release cloud resources if the smoketest succeeded, failed, or was cancelled; unnecessary if skipped
+      - name: Teardown Test Environment
+        if: needs.fablab-smoketest-ha.result != 'skipped'
+        env:
+          FABLAB_PASSPHRASE: ${{ secrets.FABLAB_PASSPHRASE }}
+        run: |
+          if aws s3api head-object \
+            --bucket ziti-smoketest-fablab-instances \
+            --key simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg
+          then
+            aws s3 cp s3://ziti-smoketest-fablab-instances/simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg .
+            gpg --quiet --batch --yes --decrypt --passphrase=${FABLAB_PASSPHRASE} --output simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg
+            tar -xzf simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz 
+            ./simple-transfer-ha-${GITHUB_RUN_NUMBER}/simple-transfer import simple-transfer-ha-${GITHUB_RUN_NUMBER}
+            ./simple-transfer-ha-${GITHUB_RUN_NUMBER}/simple-transfer dispose
+            aws s3 rm s3://ziti-smoketest-fablab-instances/simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg
+          else
+            echo "WARN: No instance archive found for simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg" >&2
+          fi
+
   publish:
     name: Publish Binaries
     # - !cancelled() allows evaluating further conditional expressions even if
@@ -247,9 +344,10 @@ jobs:
       && (needs.mac-os-build.result == 'success' || needs.mac-os-build.result == 'skipped')
       && (needs.windows-build.result == 'success' || needs.windows-build.result == 'skipped')
       && (needs.fablab-smoketest.result == 'success' || needs.fablab-smoketest.result == 'skipped') 
+      && (needs.fablab-ha-smoketest.result == 'success' || needs.fablab-ha-smoketest.result == 'skipped')
       }}
     runs-on: ubuntu-20.04
-    needs: [ linux-build, mac-os-build, windows-build, fablab-smoketest ]
+    needs: [ linux-build, mac-os-build, windows-build, fablab-smoketest, fablab-ha-smoketest ]
     outputs:
       ZITI_VERSION: ${{ steps.get_version.outputs.ZITI_VERSION }}
     steps:

TESTS.md

@@ -20,6 +20,23 @@ $ go test -v ./tests/...
 $ simple-transfer dispose
 ```
 
+The HA smoketest is very similar. There's one additional flag needed when creating the instance.
+
+```
+$ simple-transfer create -l ha=true
+```
+
+So the full set of steps for running the HA smoketest locally are:
+
+```
+$ cd zititest
+$ go install ./...
+$ simple-transfer create -l ha=true
+$ ZITI_VERSION=0.0.0 simple-transfer up
+$ go test -v ./tests/...
+$ simple-transfer dispose
+```
+
 ## Debugging smoketests
 
 When a smoketest runs in Github, the fablab instance data is uploaded to s3. This includes 
@@ -44,3 +61,20 @@ To delete the instance data out of S3, you would run:
 ```
 aws s3 rm s3://ziti-smoketest-fablab-instances/simple-transfer-$1.tar.gz.gpg
 ```
+
+The HA smoketest is very similar. 
+
+```
+aws s3 cp s3://ziti-smoketest-fablab-instances/simple-transfer-ha-$1.tar.gz.gpg ${HOME}/Downloads/simple-transfer-ha-$1.tar.gz.gpg
+FABLAB_PASSPHRASE=<passphrase goes here> simple-transfer import ${HOME}/Downloads/simple-transfer-ha-$1.tar.gz.gpg
+rm ${HOME}/Downloads/simple-transfer-ha-$1.tar.gz.gpg
+```
+You would pass the workflow run number into the script. If you need the GPG passphrase, please ask.
+
+To delete the instance data out of S3, you would run:
+
+```
+aws s3 rm s3://ziti-smoketest-fablab-instances/simple-transfer-ha-$1.tar.gz.gpg
+```
+
+

ziti/cmd/agentcli/agent.go

@@ -11,8 +11,8 @@ import (
 	"github.com/openziti/agent"
 	"github.com/openziti/channel/v2"
 	"github.com/openziti/edge/router/debugops"
-	"github.com/openziti/fabric/controller"
 	"github.com/openziti/fabric/common/pb/mgmt_pb"
+	"github.com/openziti/fabric/controller"
 	"github.com/openziti/fabric/router"
 	"github.com/openziti/identity"
 	"github.com/openziti/ziti/ziti/cmd/common"

ziti/cmd/agentcli/agent_cluster_add.go

@@ -19,8 +19,8 @@ package agentcli
 import (
 	"fmt"
 	"github.com/openziti/channel/v2"
-	"github.com/openziti/fabric/controller"
 	"github.com/openziti/fabric/common/pb/mgmt_pb"
+	"github.com/openziti/fabric/controller"
 	"github.com/openziti/ziti/ziti/cmd/common"
 	cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers"
 	"github.com/spf13/cobra"

ziti/cmd/agentcli/agent_cluster_remove.go

@@ -19,8 +19,8 @@ package agentcli
 import (
 	"fmt"
 	"github.com/openziti/channel/v2"
-	"github.com/openziti/fabric/controller"
 	"github.com/openziti/fabric/common/pb/mgmt_pb"
+	"github.com/openziti/fabric/controller"
 	"github.com/openziti/ziti/ziti/cmd/common"
 	cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers"
 	"github.com/spf13/cobra"

ziti/cmd/agentcli/agent_cluster_transfer_leadership.go

@@ -19,8 +19,8 @@ package agentcli
 import (
 	"fmt"
 	"github.com/openziti/channel/v2"
-	"github.com/openziti/fabric/controller"
 	"github.com/openziti/fabric/common/pb/mgmt_pb"
+	"github.com/openziti/fabric/controller"
 	"github.com/openziti/ziti/ziti/cmd/common"
 	cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers"
 	"github.com/spf13/cobra"

ziti/cmd/edge/list.go

@@ -669,12 +669,12 @@ func outputEnrollments(o *api.Options, children []*gabs.Container, pagingInfo *a
 	t.AppendHeader(table.Row{"ID", "Method", "Identity Id", "Identity Name", "Expires At", "Token", "JWT"})
 
 	for _, entity := range children {
-		id, _ := entity.Path("id").Data().(string)
-		method := entity.Path("method").Data().(string)
-		identityId := entity.Path("identityId").Data().(string)
-		identityName := entity.Path("identity.name").Data().(string)
-		expiresAt := entity.Path("expiresAt").Data().(string)
-		token := entity.Path("token").Data().(string)
+		id := api.GetJsonString(entity, "id")
+		method := api.GetJsonString(entity, "method")
+		identityId := api.GetJsonString(entity, "identityId")
+		identityName := api.GetJsonString(entity, "identity.name")
+		expiresAt := api.GetJsonString(entity, "expiresAt")
+		token := api.GetJsonString(entity, "token")
 		jwt := "See json"
 
 		t.AppendRow(table.Row{id, method, identityId, identityName, expiresAt, token, jwt})

ziti/cmd/fabric/stream_events.go

@@ -20,8 +20,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/openziti/channel/v2"
-	"github.com/openziti/fabric/controller/event"
 	"github.com/openziti/fabric/common/pb/mgmt_pb"
+	"github.com/openziti/fabric/controller/event"
 	"github.com/openziti/ziti/ziti/cmd/api"
 	"github.com/openziti/ziti/ziti/cmd/common"
 	"github.com/pkg/errors"

zititest/go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/openziti/agent v1.0.15
 	github.com/openziti/channel/v2 v2.0.95
 	github.com/openziti/edge v0.24.405
-	github.com/openziti/fablab v0.5.5
+	github.com/openziti/fablab v0.5.12
 	github.com/openziti/fabric v0.24.23
 	github.com/openziti/foundation/v2 v2.0.30
 	github.com/openziti/identity v1.0.61
@@ -95,7 +95,7 @@ require (
 	github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d // indirect
-	github.com/jedib0t/go-pretty/v6 v6.4.6 // indirect
+	github.com/jedib0t/go-pretty/v6 v6.4.7 // indirect
 	github.com/jessevdk/go-flags v1.5.0 // indirect
 	github.com/jinzhu/copier v0.4.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
@@ -141,7 +141,7 @@ require (
 	github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect
 	github.com/parallaxsecond/parsec-client-go v0.0.0-20221025095442-f0a77d263cf9 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
-	github.com/pkg/sftp v1.13.5 // indirect
+	github.com/pkg/sftp v1.13.6 // indirect
 	github.com/pkg/term v1.2.0-beta.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect

zititest/go.sum

@@ -507,8 +507,8 @@ github.com/influxdata/influxdb-client-go/v2 v2.2.2/go.mod h1:fa/d1lAdUHxuc1jedx3
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d h1:/WZQPMZNsjZ7IlCpsLGdQBINg5bxKQ1K1sh6awxLtkA=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
 github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
-github.com/jedib0t/go-pretty/v6 v6.4.6 h1:v6aG9h6Uby3IusSSEjHaZNXpHFhzqMmjXcPq1Rjl9Jw=
-github.com/jedib0t/go-pretty/v6 v6.4.6/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs=
+github.com/jedib0t/go-pretty/v6 v6.4.7 h1:lwiTJr1DEkAgzljsUsORmWsVn5MQjt1BPJdPCtJ6KXE=
+github.com/jedib0t/go-pretty/v6 v6.4.7/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
 github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
@@ -726,8 +726,8 @@ github.com/openziti/edge v0.24.405 h1:LXvX91M29YKJP/UujdAvsUK6hVCNCG/+7n3k4Bf370
 github.com/openziti/edge v0.24.405/go.mod h1:BPUCRbYKZ6VEimd7boo6MMACUa+t0owbJFYQVCtplm4=
 github.com/openziti/edge-api v0.25.33 h1:5XaQvUKeG8ZZ3WLhr/8xqZn56p53ZxWmFooR6I/xrvQ=
 github.com/openziti/edge-api v0.25.33/go.mod h1:T+g7OHoj2KQi3SrSdgbbFoQdknLrehEIlZRGxpTYObI=
-github.com/openziti/fablab v0.5.5 h1:wz5rkuGhdDbmNPXC8XtsyHhWk6WJtxCgfhXt5XVWXmY=
-github.com/openziti/fablab v0.5.5/go.mod h1:xkvaXX6C+bGHH5R4G8/vkz1pbBlwS1EzghF37wcNetM=
+github.com/openziti/fablab v0.5.12 h1:9gdqBZGiEsfBMCyOrSEpEPT+OxSuIsanpyV+fkXpIhA=
+github.com/openziti/fablab v0.5.12/go.mod h1:3V2qepsk3JDC6BeMtWEt+4ifokI7uI9k+6yIIucJkOE=
 github.com/openziti/fabric v0.24.23 h1:7fyguWR7EnFYxPrTP9XY2L402B86tvZLLwV5XdtahfY=
 github.com/openziti/fabric v0.24.23/go.mod h1:ALgaBRt1VMTNn9o0m0PYZhPP9V8xz4yNcj+/MSsZfs4=
 github.com/openziti/foundation/v2 v2.0.30 h1:k6mZ9pkmJgrrM5ogHvmyaRecyASHDhAv9ifZAIzuJ2s=
@@ -777,8 +777,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/profile v1.5.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18=
 github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
-github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go=
-github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg=
+github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo=
+github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
 github.com/pkg/term v1.2.0-beta.2 h1:L3y/h2jkuBVFdWiJvNfYfKmzcCnILw7mJWm2JQuMppw=
 github.com/pkg/term v1.2.0-beta.2/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -1056,7 +1056,6 @@ golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
 golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=