diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 35d71c6da..55ec25172 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -204,8 +204,6 @@ jobs:
           show: "fail, skip"
         if: ${{ !cancelled() }}
 
-      # END linux-build-steps
-
   fablab-smoketest-teardown:
     name: Teardown SmokeTest
     if: always()
@@ -238,6 +236,105 @@ jobs:
             echo "WARN: No instance archive found for simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz.gpg" >&2
           fi
 
+  fablab-ha-smoketest:
+    name: Fablab HA Smoketest
+    # not applicable to forks. shouldn't run on release build
+    if: github.repository_owner == 'openziti' && github.ref != 'refs/heads/main'
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Git Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Install Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.21.x'
+
+      - name: Install Ziti CI
+        uses: openziti/ziti-ci@v1
+
+      - name: Build and Test
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ziti_ci_gpg_key: ${{ secrets.ZITI_CI_GPG_KEY }}
+          ziti_ci_gpg_key_id: ${{ secrets.ZITI_CI_GPG_KEY_ID }}
+        run: |
+          $(go env GOPATH)/bin/ziti-ci configure-git
+          $(go env GOPATH)/bin/ziti-ci generate-build-info common/version/info_generated.go version
+          pushd zititest && go install ./... && popd
+          go install -tags=all,tests ./...
+
+      - name: Create Test Environment
+        run: |
+          echo "ZITI_ROOT=$(go env GOPATH)/bin" >> "$GITHUB_ENV"
+          $(go env GOPATH)/bin/simple-transfer create -d simple-transfer-ha-${GITHUB_RUN_NUMBER} -n simple-transfer-ha-${GITHUB_RUN_NUMBER} -l ha=true,environment=gh,ziti_version=$($(go env GOPATH)/bin/ziti-ci -q get-current-version)
+          $(go env GOPATH)/bin/simple-transfer up
+
+      #      - name: Test Ziti Command
+      #        run: |
+      #          echo "ZITI_ROOT=$(go env GOPATH)/bin" >> "$GITHUB_ENV"
+      #          pushd zititest && go test -timeout 30m -v ./tests/... 2>&1 | tee test.out && popd
+
+      - name: Create fablab instance archive
+        if: always()
+        env:
+          FABLAB_PASSPHRASE: ${{ secrets.FABLAB_PASSPHRASE }}
+        run: |
+          cp $(go env GOPATH)/bin/simple-transfer simple-transfer-ha-${GITHUB_RUN_NUMBER}
+          cp ~/.fablab/config.yml simple-transfer-ha-${GITHUB_RUN_NUMBER}/
+          tar --exclude *terraform-provider-aws* -czvf - simple-transfer-ha-${GITHUB_RUN_NUMBER}/ | gpg --symmetric --cipher-algo aes256 --batch --quiet --passphrase ${FABLAB_PASSPHRASE} -o simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg
+          aws s3 cp ./simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg s3://ziti-smoketest-fablab-instances/
+          
+          #      - name: Test Report Generation
+          #        if: always()
+          #        run: |
+          #          go install github.com/jstemmer/go-junit-report/v2@latest
+          #$(go env GOPATH)/bin/go-junit-report -in zititest/test.out -out test-report.xml
+
+      #      - name: Test Summary
+      #        uses: test-summary/action@v2
+      #        with:
+      #          paths: |
+      #            test-report.xml
+      #          show: "fail, skip"
+      #        if: always()
+
+      # END linux-build-steps
+
+  fablab-ha-smoketest-teardown:
+    name: Teardown HA SmokeTest
+    if: always()
+    runs-on: ubuntu-20.04
+    needs: [ fablab-ha-smoketest ]
+    steps:
+      # allow time for investigation unless the workflow is cancelled or the smoketest succeeded or was skipped
+      - name: Sleep If Failed
+        if: needs.fablab-ha-smoketest.result != 'success' && needs.fablab-ha-smoketest.result != 'skipped' && needs.fablab-ha-smoketest.result != 'cancelled'
+        run: |
+          sleep 30m
+
+      # release cloud resources if the smoketest succeeded, failed, or was cancelled; unnecessary if skipped
+      - name: Teardown Test Environment
+        if: needs.fablab-smoketest-ha.result != 'skipped'
+        env:
+          FABLAB_PASSPHRASE: ${{ secrets.FABLAB_PASSPHRASE }}
+        run: |
+          if aws s3api head-object \
+            --bucket ziti-smoketest-fablab-instances \
+            --key simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg
+          then
+            aws s3 cp s3://ziti-smoketest-fablab-instances/simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg .
+            gpg --quiet --batch --yes --decrypt --passphrase=${FABLAB_PASSPHRASE} --output simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg
+            tar -xzf simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz 
+            ./simple-transfer-ha-${GITHUB_RUN_NUMBER}/simple-transfer import simple-transfer-ha-${GITHUB_RUN_NUMBER}
+            ./simple-transfer-ha-${GITHUB_RUN_NUMBER}/simple-transfer dispose
+            aws s3 rm s3://ziti-smoketest-fablab-instances/simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg
+          else
+            echo "WARN: No instance archive found for simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg" >&2
+          fi
+
   publish:
     name: Publish Binaries
     # - !cancelled() allows evaluating further conditional expressions even if
@@ -247,9 +344,10 @@ jobs:
       && (needs.mac-os-build.result == 'success' || needs.mac-os-build.result == 'skipped')
       && (needs.windows-build.result == 'success' || needs.windows-build.result == 'skipped')
       && (needs.fablab-smoketest.result == 'success' || needs.fablab-smoketest.result == 'skipped') 
+      && (needs.fablab-ha-smoketest.result == 'success' || needs.fablab-ha-smoketest.result == 'skipped')
       }}
     runs-on: ubuntu-20.04
-    needs: [ linux-build, mac-os-build, windows-build, fablab-smoketest ]
+    needs: [ linux-build, mac-os-build, windows-build, fablab-smoketest, fablab-ha-smoketest ]
     outputs:
       ZITI_VERSION: ${{ steps.get_version.outputs.ZITI_VERSION }}
     steps:
diff --git a/TESTS.md b/TESTS.md
index 88bbe3577..14660767f 100644
--- a/TESTS.md
+++ b/TESTS.md
@@ -20,6 +20,23 @@ $ go test -v ./tests/...
 $ simple-transfer dispose
 ```
 
+The HA smoketest is very similar. There's one additional flag needed when creating the instance.
+
+```
+$ simple-transfer create -l ha=true
+```
+
+So the full set of steps for running the HA smoketest locally are:
+
+```
+$ cd zititest
+$ go install ./...
+$ simple-transfer create -l ha=true
+$ ZITI_VERSION=0.0.0 simple-transfer up
+$ go test -v ./tests/...
+$ simple-transfer dispose
+```
+
 ## Debugging smoketests
 
 When a smoketest runs in Github, the fablab instance data is uploaded to s3. This includes 
@@ -44,3 +61,20 @@ To delete the instance data out of S3, you would run:
 ```
 aws s3 rm s3://ziti-smoketest-fablab-instances/simple-transfer-$1.tar.gz.gpg
 ```
+
+The HA smoketest is very similar. 
+
+```
+aws s3 cp s3://ziti-smoketest-fablab-instances/simple-transfer-ha-$1.tar.gz.gpg ${HOME}/Downloads/simple-transfer-ha-$1.tar.gz.gpg
+FABLAB_PASSPHRASE=<passphrase goes here> simple-transfer import ${HOME}/Downloads/simple-transfer-ha-$1.tar.gz.gpg
+rm ${HOME}/Downloads/simple-transfer-ha-$1.tar.gz.gpg
+```
+You would pass the workflow run number into the script. If you need the GPG passphrase, please ask.
+
+To delete the instance data out of S3, you would run:
+
+```
+aws s3 rm s3://ziti-smoketest-fablab-instances/simple-transfer-ha-$1.tar.gz.gpg
+```
+
+
diff --git a/ziti/cmd/agentcli/agent.go b/ziti/cmd/agentcli/agent.go
index 6b84f31cb..e8b30f34d 100644
--- a/ziti/cmd/agentcli/agent.go
+++ b/ziti/cmd/agentcli/agent.go
@@ -11,8 +11,8 @@ import (
 	"github.com/openziti/agent"
 	"github.com/openziti/channel/v2"
 	"github.com/openziti/edge/router/debugops"
-	"github.com/openziti/fabric/controller"
 	"github.com/openziti/fabric/common/pb/mgmt_pb"
+	"github.com/openziti/fabric/controller"
 	"github.com/openziti/fabric/router"
 	"github.com/openziti/identity"
 	"github.com/openziti/ziti/ziti/cmd/common"
diff --git a/ziti/cmd/agentcli/agent_cluster_add.go b/ziti/cmd/agentcli/agent_cluster_add.go
index d1fd0a0fc..0516f2500 100644
--- a/ziti/cmd/agentcli/agent_cluster_add.go
+++ b/ziti/cmd/agentcli/agent_cluster_add.go
@@ -19,8 +19,8 @@ package agentcli
 import (
 	"fmt"
 	"github.com/openziti/channel/v2"
-	"github.com/openziti/fabric/controller"
 	"github.com/openziti/fabric/common/pb/mgmt_pb"
+	"github.com/openziti/fabric/controller"
 	"github.com/openziti/ziti/ziti/cmd/common"
 	cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers"
 	"github.com/spf13/cobra"
diff --git a/ziti/cmd/agentcli/agent_cluster_remove.go b/ziti/cmd/agentcli/agent_cluster_remove.go
index 3b871d611..8fdc82c75 100644
--- a/ziti/cmd/agentcli/agent_cluster_remove.go
+++ b/ziti/cmd/agentcli/agent_cluster_remove.go
@@ -19,8 +19,8 @@ package agentcli
 import (
 	"fmt"
 	"github.com/openziti/channel/v2"
-	"github.com/openziti/fabric/controller"
 	"github.com/openziti/fabric/common/pb/mgmt_pb"
+	"github.com/openziti/fabric/controller"
 	"github.com/openziti/ziti/ziti/cmd/common"
 	cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers"
 	"github.com/spf13/cobra"
diff --git a/ziti/cmd/agentcli/agent_cluster_transfer_leadership.go b/ziti/cmd/agentcli/agent_cluster_transfer_leadership.go
index 0705bc470..1e8a440b5 100644
--- a/ziti/cmd/agentcli/agent_cluster_transfer_leadership.go
+++ b/ziti/cmd/agentcli/agent_cluster_transfer_leadership.go
@@ -19,8 +19,8 @@ package agentcli
 import (
 	"fmt"
 	"github.com/openziti/channel/v2"
-	"github.com/openziti/fabric/controller"
 	"github.com/openziti/fabric/common/pb/mgmt_pb"
+	"github.com/openziti/fabric/controller"
 	"github.com/openziti/ziti/ziti/cmd/common"
 	cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers"
 	"github.com/spf13/cobra"
diff --git a/ziti/cmd/edge/list.go b/ziti/cmd/edge/list.go
index fbba3bf18..8f7143e44 100644
--- a/ziti/cmd/edge/list.go
+++ b/ziti/cmd/edge/list.go
@@ -669,12 +669,12 @@ func outputEnrollments(o *api.Options, children []*gabs.Container, pagingInfo *a
 	t.AppendHeader(table.Row{"ID", "Method", "Identity Id", "Identity Name", "Expires At", "Token", "JWT"})
 
 	for _, entity := range children {
-		id, _ := entity.Path("id").Data().(string)
-		method := entity.Path("method").Data().(string)
-		identityId := entity.Path("identityId").Data().(string)
-		identityName := entity.Path("identity.name").Data().(string)
-		expiresAt := entity.Path("expiresAt").Data().(string)
-		token := entity.Path("token").Data().(string)
+		id := api.GetJsonString(entity, "id")
+		method := api.GetJsonString(entity, "method")
+		identityId := api.GetJsonString(entity, "identityId")
+		identityName := api.GetJsonString(entity, "identity.name")
+		expiresAt := api.GetJsonString(entity, "expiresAt")
+		token := api.GetJsonString(entity, "token")
 		jwt := "See json"
 
 		t.AppendRow(table.Row{id, method, identityId, identityName, expiresAt, token, jwt})
diff --git a/ziti/cmd/fabric/stream_events.go b/ziti/cmd/fabric/stream_events.go
index bc21b24db..9e5306a35 100644
--- a/ziti/cmd/fabric/stream_events.go
+++ b/ziti/cmd/fabric/stream_events.go
@@ -20,8 +20,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/openziti/channel/v2"
-	"github.com/openziti/fabric/controller/event"
 	"github.com/openziti/fabric/common/pb/mgmt_pb"
+	"github.com/openziti/fabric/controller/event"
 	"github.com/openziti/ziti/ziti/cmd/api"
 	"github.com/openziti/ziti/ziti/cmd/common"
 	"github.com/pkg/errors"
diff --git a/zititest/go.mod b/zititest/go.mod
index ab82ba9b7..0545d1184 100644
--- a/zititest/go.mod
+++ b/zititest/go.mod
@@ -13,7 +13,7 @@ require (
 	github.com/openziti/agent v1.0.15
 	github.com/openziti/channel/v2 v2.0.95
 	github.com/openziti/edge v0.24.405
-	github.com/openziti/fablab v0.5.5
+	github.com/openziti/fablab v0.5.12
 	github.com/openziti/fabric v0.24.23
 	github.com/openziti/foundation/v2 v2.0.30
 	github.com/openziti/identity v1.0.61
@@ -95,7 +95,7 @@ require (
 	github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d // indirect
-	github.com/jedib0t/go-pretty/v6 v6.4.6 // indirect
+	github.com/jedib0t/go-pretty/v6 v6.4.7 // indirect
 	github.com/jessevdk/go-flags v1.5.0 // indirect
 	github.com/jinzhu/copier v0.4.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
@@ -141,7 +141,7 @@ require (
 	github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect
 	github.com/parallaxsecond/parsec-client-go v0.0.0-20221025095442-f0a77d263cf9 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
-	github.com/pkg/sftp v1.13.5 // indirect
+	github.com/pkg/sftp v1.13.6 // indirect
 	github.com/pkg/term v1.2.0-beta.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
diff --git a/zititest/go.sum b/zititest/go.sum
index f10f73da4..7c4b8c10e 100644
--- a/zititest/go.sum
+++ b/zititest/go.sum
@@ -507,8 +507,8 @@ github.com/influxdata/influxdb-client-go/v2 v2.2.2/go.mod h1:fa/d1lAdUHxuc1jedx3
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d h1:/WZQPMZNsjZ7IlCpsLGdQBINg5bxKQ1K1sh6awxLtkA=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
 github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
-github.com/jedib0t/go-pretty/v6 v6.4.6 h1:v6aG9h6Uby3IusSSEjHaZNXpHFhzqMmjXcPq1Rjl9Jw=
-github.com/jedib0t/go-pretty/v6 v6.4.6/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs=
+github.com/jedib0t/go-pretty/v6 v6.4.7 h1:lwiTJr1DEkAgzljsUsORmWsVn5MQjt1BPJdPCtJ6KXE=
+github.com/jedib0t/go-pretty/v6 v6.4.7/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
 github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
@@ -726,8 +726,8 @@ github.com/openziti/edge v0.24.405 h1:LXvX91M29YKJP/UujdAvsUK6hVCNCG/+7n3k4Bf370
 github.com/openziti/edge v0.24.405/go.mod h1:BPUCRbYKZ6VEimd7boo6MMACUa+t0owbJFYQVCtplm4=
 github.com/openziti/edge-api v0.25.33 h1:5XaQvUKeG8ZZ3WLhr/8xqZn56p53ZxWmFooR6I/xrvQ=
 github.com/openziti/edge-api v0.25.33/go.mod h1:T+g7OHoj2KQi3SrSdgbbFoQdknLrehEIlZRGxpTYObI=
-github.com/openziti/fablab v0.5.5 h1:wz5rkuGhdDbmNPXC8XtsyHhWk6WJtxCgfhXt5XVWXmY=
-github.com/openziti/fablab v0.5.5/go.mod h1:xkvaXX6C+bGHH5R4G8/vkz1pbBlwS1EzghF37wcNetM=
+github.com/openziti/fablab v0.5.12 h1:9gdqBZGiEsfBMCyOrSEpEPT+OxSuIsanpyV+fkXpIhA=
+github.com/openziti/fablab v0.5.12/go.mod h1:3V2qepsk3JDC6BeMtWEt+4ifokI7uI9k+6yIIucJkOE=
 github.com/openziti/fabric v0.24.23 h1:7fyguWR7EnFYxPrTP9XY2L402B86tvZLLwV5XdtahfY=
 github.com/openziti/fabric v0.24.23/go.mod h1:ALgaBRt1VMTNn9o0m0PYZhPP9V8xz4yNcj+/MSsZfs4=
 github.com/openziti/foundation/v2 v2.0.30 h1:k6mZ9pkmJgrrM5ogHvmyaRecyASHDhAv9ifZAIzuJ2s=
@@ -777,8 +777,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/profile v1.5.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18=
 github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
-github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go=
-github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg=
+github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo=
+github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
 github.com/pkg/term v1.2.0-beta.2 h1:L3y/h2jkuBVFdWiJvNfYfKmzcCnILw7mJWm2JQuMppw=
 github.com/pkg/term v1.2.0-beta.2/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -1056,7 +1056,6 @@ golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
 golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
diff --git a/zititest/models/ha/actions/bootstrap.go b/zititest/models/ha/actions/bootstrap.go
deleted file mode 100644
index 4692c7cdb..000000000
--- a/zititest/models/ha/actions/bootstrap.go
+++ /dev/null
@@ -1,53 +0,0 @@
-package actions
-
-import (
-	"time"
-
-	"github.com/openziti/fablab/kernel/lib/actions"
-	"github.com/openziti/fablab/kernel/lib/actions/component"
-	"github.com/openziti/fablab/kernel/lib/actions/host"
-	"github.com/openziti/fablab/kernel/lib/actions/semaphore"
-	"github.com/openziti/fablab/kernel/model"
-	zitilib_actions "github.com/openziti/ziti/zititest/zitilab/actions"
-	"github.com/openziti/ziti/zititest/zitilab/actions/edge"
-	"github.com/openziti/ziti/zititest/zitilab/models"
-)
-
-type bootstrapAction struct{}
-
-func NewBootstrapAction() model.ActionBinder {
-	action := &bootstrapAction{}
-	return action.bind
-}
-
-func (a *bootstrapAction) bind(m *model.Model) model.Action {
-	workflow := actions.Workflow()
-
-	workflow.AddAction(component.Stop(".ctrl"))
-	workflow.AddAction(host.GroupExec("*", 25, "rm -f logs/*"))
-	workflow.AddAction(host.GroupExec(".ctrl", 5, "rf -rf ./fablab/ctrldata"))
-	workflow.AddAction(component.Start(".ctrl"))
-	workflow.AddAction(semaphore.Sleep(2 * time.Second))
-	workflow.AddAction(edge.RaftJoin(".ctrl"))
-	workflow.AddAction(semaphore.Sleep(2 * time.Second))
-	workflow.AddAction(edge.InitRaftController("#ctrl1"))
-	workflow.AddAction(semaphore.Sleep(time.Second))
-
-	workflow.AddAction(edge.Login("#ctrl1"))
-
-	workflow.AddAction(component.StopInParallel(models.EdgeRouterTag, 25))
-	workflow.AddAction(edge.InitEdgeRouters(models.EdgeRouterTag, 2))
-	workflow.AddAction(edge.InitIdentities(models.SdkAppTag, 2))
-
-	workflow.AddAction(zitilib_actions.Edge("create", "service", "echo"))
-
-	workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "echo-servers", "Bind", "--service-roles", "@echo", "--identity-roles", "#service"))
-	workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "echo-client", "Dial", "--service-roles", "@echo", "--identity-roles", "#client"))
-
-	workflow.AddAction(zitilib_actions.Edge("create", "edge-router-policy", "echo-servers", "--edge-router-roles", "@router-east", "--identity-roles", "#service"))
-	workflow.AddAction(zitilib_actions.Edge("create", "edge-router-policy", "echo-clients", "--edge-router-roles", "@router-west", "--identity-roles", "#client"))
-
-	workflow.AddAction(zitilib_actions.Edge("create", "service-edge-router-policy", "echo", "--semantic", "AnyOf", "--service-roles", "@echo", "--edge-router-roles", "#all"))
-
-	return workflow
-}
diff --git a/zititest/models/ha/actions/start.go b/zititest/models/ha/actions/start.go
deleted file mode 100644
index 4b4b232cc..000000000
--- a/zititest/models/ha/actions/start.go
+++ /dev/null
@@ -1,54 +0,0 @@
-package actions
-
-import (
-	"time"
-
-	"github.com/openziti/fablab/kernel/lib/actions"
-	"github.com/openziti/fablab/kernel/lib/actions/component"
-	"github.com/openziti/fablab/kernel/lib/actions/semaphore"
-	"github.com/openziti/fablab/kernel/model"
-	zitilib_actions "github.com/openziti/ziti/zititest/zitilab/actions"
-	"github.com/openziti/ziti/zititest/zitilab/models"
-)
-
-func NewStartAction(metricbeat MetricbeatConfig, consul ConsulConfig) model.ActionBinder {
-	action := &startAction{
-		Metricbeat: metricbeat,
-		Consul:     consul,
-	}
-	return action.bind
-}
-
-func (a *startAction) bind(m *model.Model) model.Action {
-	workflow := actions.Workflow()
-	workflow.AddAction(component.Start("#ctrl"))
-	workflow.AddAction(semaphore.Sleep(2 * time.Second))
-	workflow.AddAction(component.StartInParallel(models.EdgeRouterTag, 25))
-
-	workflow.AddAction(semaphore.Sleep(2 * time.Second))
-	workflow.AddAction(zitilib_actions.StartMetricbeat("*", a.Metricbeat.ConfigPath, a.Metricbeat.DataPath, a.Metricbeat.LogPath))
-	workflow.AddAction(zitilib_actions.StartConsul("*", a.Consul.ServerAddr, a.Consul.ConfigDir, a.Consul.DataPath, a.Consul.LogPath))
-	workflow.AddAction(semaphore.Sleep(2 * time.Second))
-	workflow.AddAction(component.Start("#echo-server"))
-	workflow.AddAction(semaphore.Sleep(2 * time.Second))
-
-	return workflow
-}
-
-type startAction struct {
-	Metricbeat MetricbeatConfig
-	Consul     ConsulConfig
-}
-
-type MetricbeatConfig struct {
-	ConfigPath string
-	DataPath   string
-	LogPath    string
-}
-
-type ConsulConfig struct {
-	ConfigDir  string
-	ServerAddr string
-	DataPath   string
-	LogPath    string
-}
diff --git a/zititest/models/ha/configs/consul.hcl b/zititest/models/ha/configs/consul.hcl
deleted file mode 100644
index 0f333bb3b..000000000
--- a/zititest/models/ha/configs/consul.hcl
+++ /dev/null
@@ -1,24 +0,0 @@
-datacenter = "ziti-build-metrics"
-data_dir = "/opt/consul"
-encrypt = "${encryption_key}"
-advertise_addr="${public_ip}"
-
-
-tls {
-        defaults {
-                verify_incoming = false
-                verify_outgoing = true
-
-                ca_file="consul/consul-agent-ca.pem"
-        }
-}
-
-auto_encrypt {
-        tls = true
-}
-
-acl {
-        enabled = true
-        default_policy = "allow"
-        enable_token_persistence = true
-}
diff --git a/zititest/models/ha/configs/ctrl.yml.tmpl b/zititest/models/ha/configs/ctrl.yml.tmpl
deleted file mode 100644
index 94f690531..000000000
--- a/zititest/models/ha/configs/ctrl.yml.tmpl
+++ /dev/null
@@ -1,196 +0,0 @@
-v: 3
-
-raft:
-  minClusterSize: 3
-  dataDir: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/ctrldata
-
-identity:
-  cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}-server.cert
-  key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/keys/{{ .Component.Id }}-server.key
-  ca: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}-server.chain.pem
-
-# the endpoint that routers will connect to the controller over.
-ctrl:
-  listener: tls:0.0.0.0:6262
-  options:
-    advertiseAddress: tls:{{ .Host.PublicIp }}:6262
-    # (optional) settings
-    # set the maximum number of connect requests that are buffered and waiting to be acknowledged (1 to 5000, default 1000)
-    #maxQueuedConnects:      50
-
-    # the maximum number of connects that have  begun hello synchronization (1 to 1000, default 16)
-    #maxOutstandingConnects: 100
-
-    # the number of milliseconds to wait before a hello synchronization fails and closes the connection (30ms to 60000ms, default: 1000ms)
-    #connectTimeoutMs:       3000
-
-    # Sets the control channel write timeout. A write timeout will close the control channel, so the router will reconnect
-    #writeTimeout: 15s
-
-    # A listener address which will be sent to connecting routers in order to change their configured controller
-    # address. If defined, routers will update address configuration to immediately use the new address for future
-    # connections. The value of newListener must be resolvable both via DNS and validate via certificates
-    #newListener: tls:localhost:6262
-
-#events:
-#  jsonLogger:
-#    subscriptions:
-#      - type: fabric.routers
-#      - type: fabric.terminators
-#      - type: metrics
-#        sourceFilter: .*
-#        metricFilter: .*egress.*m1_rate*
-#      - type: fabric.circuits
-#        include:
-#          - created
-#      - type: edge.sessions
-#        include:
-#          - created
-#      - type: edge.apiSessions
-#      - type: fabric.usage
-#      - type: services
-#      - type: fabric.usage
-#      - type: edge.entityCounts
-#        interval: 5s
-#    handler:
-#      type: file
-#      format: json
-#      path: /tmp/ziti-events.log
-
-healthChecks:
-  boltCheck:
-    # How often to try entering a bolt read tx. Defaults to 30 seconds
-    interval: 30s
-    # When to timeout the check. Defaults to 15 seconds
-    timeout: 15s
-    # How long to wait before starting the check. Defaults to 15 seconds
-    initialDelay: 15s
-
-# By having an 'edge' section defined, the ziti-controller will attempt to parse the edge configuration. Removing this
-# section, commenting out, or altering the name of the section will cause the edge to not run.
-edge:
-  # This section represents the configuration of the Edge API that is served over HTTPS
-  api:
-    #(optional, default 90s) Alters how frequently heartbeat and last activity values are persisted
-    # activityUpdateInterval: 90s
-    #(optional, default 250) The number of API Sessions updated for last activity per transaction
-    # activityUpdateBatchSize: 250
-    # sessionTimeout - optional, default 10m
-    # The number of minutes before an Edge API session will timeout. Timeouts are reset by
-    # API requests and connections that are maintained to Edge Routers
-    sessionTimeout: 30m
-    # address - required
-    # The default address (host:port) to use for enrollment for the Client API. This value must match one of the addresses
-    # defined in a bind point's address field for the `edge-client` API in the web section.
-    address: {{ .Host.PublicIp }}:1280
-  # enrollment - required
-  # A section containing settings pertaining to enrollment.
-  enrollment:
-    # signingCert - required
-    # A Ziti Identity configuration section that specifically makes use of the cert and key fields to define
-    # a signing certificate from the PKI that the Ziti environment is using to sign certificates. The signingCert.cert
-    # will be added to the /.well-known CA store that is used to bootstrap trust with the Ziti Controller.
-    signingCert:
-      cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}.cert
-      key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/keys/{{ .Component.Id }}.key
-
-    # edgeIdentity - optional
-    # A section for identity enrollment specific settings
-    edgeIdentity:
-      # duration - optional, default 5m
-      # The length of time that a Ziti Edge Identity enrollment should remain valid. After
-      # this duration, the enrollment will expire and not longer be usable.
-      duration: 5m
-    # edgeRouter - Optional
-    # A section for edge router enrollment specific settings.
-    edgeRouter:
-      # duration - optional, default 5m
-      # The length of time that a Ziti Edge Router enrollment should remain valid. After
-      # this duration, the enrollment will expire and not longer be usable.
-      duration: 5m
-
-
-# web - optional
-# Defines webListeners that will be hosted by the controller. Each webListener can host many APIs and be bound to many
-# bind points.
-web:
-  # name - required
-  # Provides a name for this listener, used for logging output. Not required to be unique, but is highly suggested.
-  - name: all-apis-localhost
-    # bindPoints - required
-    # One or more bind points are required. A bind point specifies an interface (interface:port string) that defines
-    # where on the host machine the webListener will listen and the address (host:port) that should be used to
-    # publicly address the webListener(i.e. mydomain.com, localhost, 127.0.0.1). This public address may be used for
-    # incoming address resolution as well as used in responses in the API.
-    bindPoints:
-      #interface - required
-      # A host:port string on which network interface to listen on. 0.0.0.0 will listen on all interfaces
-      - interface: 0.0.0.0:1280
-
-        # address - required
-        # The public address that external incoming requests will be able to resolve. Used in request processing and
-        # response content that requires full host:port/path addresses.
-        address: {{ .Host.PublicIp }}:1280
-
-        # newAddress - optional
-        # A host:port string which will be sent out as an HTTP header "ziti-new-address" if specified. If the header
-        # is present, clients should update location configuration to immediately use the new address for future
-        # connections. The value of newAddress must be resolvable both via DNS and validate via certificates
-        #newAddress: localhost:1280
-    # identity - optional
-    # Allows the webListener to have a specific identity instead of defaulting to the root `identity` section.
-    #    identity:
-    #      cert:                 ${ZITI_SOURCE}/ziti/etc/ca/intermediate/certs/ctrl-client.cert.pem
-    #      server_cert:          ${ZITI_SOURCE}/ziti/etc/ca/intermediate/certs/ctrl-server.cert.pem
-    #      key:                  ${ZITI_SOURCE}/ziti/etc/ca/intermediate/private/ctrl.key.pem
-    #      ca:                   ${ZITI_SOURCE}/ziti/etc/ca/intermediate/certs/ca-chain.cert.pem
-    # options - optional
-    # Allows the specification of webListener level options - mainly dealing with HTTP/TLS settings. These options are
-    # used for all http servers started by the current webListener.
-    options:
-      # idleTimeout - optional, default 5000ms
-      # The maximum amount of idle time in milliseconds allowed for pipelined HTTP requests. Setting this too high
-      # can cause resources on the host to be consumed as clients remain connected and idle. Lowering this value
-      # will cause clients to reconnect on subsequent HTTPs requests.
-      idleTimeout: 5000ms  #http timeouts, new
-
-      # readTimeout - optional, default 5000ms
-      # The maximum amount of time in milliseconds http servers will wait to read the first incoming requests. A higher
-      # value risks consuming resources on the host with clients that are acting bad faith or suffering from high latency
-      # or packet loss. A lower value can risk losing connections to high latency/packet loss clients.
-
-      readTimeout: 5000ms
-      # writeTimeout - optional, default 10000ms
-      # The total maximum time in milliseconds that the http server will wait for a single requests to be received and
-      # responded too. A higher value can allow long running requests to consume resources on the host. A lower value
-      # can risk ending requests before the server has a chance to respond.
-
-      writeTimeout: 100000ms
-      # minTLSVersion - optional, default TSL1.2
-      # The minimum version of TSL to support
-
-      minTLSVersion: TLS1.2
-      # maxTLSVersion - optional, default TSL1.3
-      # The maximum version of TSL to support
-
-      maxTLSVersion: TLS1.3
-    # apis - required
-    # Allows one or more APIs to be bound to this webListener
-    apis:
-      # binding - required
-      # Specifies an API to bind to this webListener. Built-in APIs are
-      #   - health-checks
-      #   - edge-management
-      #   - edge-client
-      #   - fabric-management
-      - binding: health-checks
-        options: {}
-      - binding: fabric
-      - binding: edge-management
-        # options - variable optional/required
-        # This section is used to define values that are specified by the API they are associated with.
-        # These settings are per API. The example below is for the `edge-api` and contains both optional values and
-        # required values.
-        options: {}
-      - binding: edge-client
-        options: {}
diff --git a/zititest/models/ha/configs/metricbeat.yml b/zititest/models/ha/configs/metricbeat.yml
deleted file mode 100644
index 35c1cc9f6..000000000
--- a/zititest/models/ha/configs/metricbeat.yml
+++ /dev/null
@@ -1,53 +0,0 @@
-fields_under_root: true
-fields:
-  build_number: ${build_number}
-  ziti_version: ${ziti_version}
-
-processors:
-  - add_cloud_metadata: ~
-
-metricbeat.modules:
-- module: system
-  metricsets:
-    - cpu             # CPU usage
-    - load            # CPU load averages
-    - memory          # Memory usage
-    - network         # Network IO
-  enabled: true
-  period: 30s
-  cpu.metrics:  ["percentages"]  # The other available options are normalized_percentages and ticks.
-  core.metrics: ["percentages"]  # The other available option is ticks.
-#- module: docker
-#  metricsets:
-#    - "container"
-#    - "cpu"
-#    - "diskio"
-#    - "healthcheck"
-#    - "info"
-#    - "image"
-#    - "memory"
-#    - "network"
-#  hosts: ["unix:///var/run/docker.sock"]
-#  period: 30s
-#  enabled: true
-#  # If set to true, replace dots in labels with `_`.
-#  labels.dedot: true
-
-
-output.elasticsearch:
-  # Array of hosts to connect to.
-  hosts: ["${host}"]
-
-  # Optional protocol and basic auth credentials.
-  protocol: "https"
-  username: "${user}"
-  password: "${password}"
-  index: 'zt-%{[agent.version]}-%{+yyyy.MM.dd}'
-
-setup.template.enabled: true
-setup.template.name: 'zt-%{[agent.version]}'
-setup.template.pattern: 'zt-%{[agent.version]}-*'
-setup.template.fields: "/etc/metricbeat/fields.yml"
-setup.template.settings:
-  index.number_of_shards: 1
-  index.codec: best_compression
\ No newline at end of file
diff --git a/zititest/models/ha/configs/router.yml.tmpl b/zititest/models/ha/configs/router.yml.tmpl
deleted file mode 100644
index 4cb9afca5..000000000
--- a/zititest/models/ha/configs/router.yml.tmpl
+++ /dev/null
@@ -1,70 +0,0 @@
-{{$ssh_username := .Model.MustVariable "credentials.ssh.username"}}
-{{$identity := .Component.PublicIdentity}}
-{{$router_ip := .Host.PublicIp}}
-  
-v: 3
-
-enableDebugOps: true
-
-identity:
-  cert:                 /home/{{$ssh_username}}/fablab/cfg/{{$identity}}-client.cert
-  server_cert:          /home/{{$ssh_username}}/fablab/cfg/{{$identity}}-server.cert
-  key:                  /home/{{$ssh_username}}/fablab/cfg/{{$identity}}.key
-  ca:                   /home/{{$ssh_username}}/fablab/cfg/{{$identity}}-server.chain.pem
-
-ctrl:
-  endpoints: {{ range $host := .Model.MustSelectHosts "component.ctrl" 1 }}
-    - tls:{{ $host.PublicIp }}:6262{{end}}
-
-healthChecks:
-  ctrlPingCheck:
-    # How often to ping the controller over the control channel. Defaults to 30 seconds
-    interval: 30s
-    # When to timeout the ping. Defaults to 15 seconds
-    timeout: 15s
-    # How long to wait before pinging the controller. Defaults to 15 seconds
-    initialDelay: 15s
-
-metrics:
-  reportInterval: 15s
-  messageQueueSize: 10
-
-link:
-  listeners:
-    - binding:          transport
-      bind:             tls:0.0.0.0:6000
-      advertise:        tls:{{$router_ip}}:6000
-  dialers:
-    - binding:          transport
-
-listeners:
-{{if .Component.HasTag "tunneler"}}
-  - binding: tunnel
-    options:
-      mode: tproxy
-{{end}}
-  - binding: edge
-    address: tls:0.0.0.0:6262
-    options:
-      # (required) The public hostname and port combination that Ziti SDKs should connect on. Previously this was in the chanIngress section.
-      advertise: {{ .Host.PublicIp }}:6262
-
-# By having an 'edge' section defined, the ziti-router will attempt to parse the edge configuration. Removing this
-# section, commenting out, or altering the name of the section will cause the router to no longer operate as an Edge
-# Router.
-edge:
-  # (required) Information used to generate the initial registration CSR. For documentation on these fields please
-  # refer to the openssl documentation. These values MUST be supplied and have no defaults.
-  csr:
-    country: US
-    province: NC
-    locality: Charlotte
-    organization: NetFoundry
-    organizationalUnit: Ziti
-
-    # (required) SANs that this Gateways certs should contain. At least one IP or DNS SAN should be defined that matches
-    # the edge listeners "advertise" value from the "listeners" section.
-    sans:
-      ip:
-        - {{ .Host.PublicIp }}
-
diff --git a/zititest/models/ha/configs/ziti.hcl b/zititest/models/ha/configs/ziti.hcl
deleted file mode 100644
index 4af438cc0..000000000
--- a/zititest/models/ha/configs/ziti.hcl
+++ /dev/null
@@ -1,9 +0,0 @@
-service {
-    name = "ziti"
-    id = "ziti"
-    port = 6262
-    meta {
-        build_number= "${build_number}"
-        ziti_version= "${ziti_version}"
-    }
-}
\ No newline at end of file
diff --git a/zititest/models/ha/main.go b/zititest/models/ha/main.go
deleted file mode 100644
index 0b38d5017..000000000
--- a/zititest/models/ha/main.go
+++ /dev/null
@@ -1,251 +0,0 @@
-/*
-	(c) Copyright NetFoundry Inc.
-
-	Licensed under the Apache License, Version 2.0 (the "License");
-	you may not use this file except in compliance with the License.
-	You may obtain a copy of the License at
-
-	https://www.apache.org/licenses/LICENSE-2.0
-
-	Unless required by applicable law or agreed to in writing, software
-	distributed under the License is distributed on an "AS IS" BASIS,
-	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-	See the License for the specific language governing permissions and
-	limitations under the License.
-*/
-
-package main
-
-import (
-	"embed"
-	"fmt"
-	"github.com/openziti/fablab"
-	"github.com/openziti/fablab/kernel/lib/actions/component"
-	"github.com/openziti/fablab/kernel/lib/binding"
-	"github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/aws_ssh_key"
-	semaphore0 "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/semaphore"
-	terraform_0 "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/terraform"
-	distribution "github.com/openziti/fablab/kernel/lib/runlevel/3_distribution"
-	"github.com/openziti/fablab/kernel/lib/runlevel/3_distribution/rsync"
-	aws_ssh_key2 "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/aws_ssh_key"
-	"github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/terraform"
-	"github.com/openziti/fablab/kernel/model"
-	"github.com/openziti/fablab/resources"
-	"github.com/openziti/ziti/zititest/models/ha/actions"
-	"github.com/openziti/ziti/zititest/models/test_resources"
-	"github.com/openziti/ziti/zititest/zitilab"
-	"github.com/openziti/ziti/zititest/zitilab/actions/edge"
-	"github.com/sirupsen/logrus"
-	"os"
-	"time"
-)
-
-//go:embed configs
-var configResource embed.FS
-
-func getConfigData(filePath string) []byte {
-	data, err := configResource.ReadFile(fmt.Sprintf("configs/%s", filePath))
-	if err != nil {
-		logrus.Errorf("Unable to read config data from %s: [%s]", filePath, err)
-	}
-	return data
-}
-
-var m = &model.Model{
-	Id: "ha",
-	Scope: model.Scope{
-		Defaults: model.Variables{
-			"environment": "ha-smoketest",
-			"credentials": model.Variables{
-				"ssh": model.Variables{
-					"username": "ubuntu",
-				},
-				"edge": model.Variables{
-					"username": "admin",
-					"password": "admin",
-				},
-			},
-		},
-	},
-
-	Resources: model.Resources{
-		resources.Configs:   resources.SubFolder(configResource, "configs"),
-		resources.Terraform: test_resources.TerraformResources(),
-	},
-
-	Regions: model.Regions{
-		"us-east-1": {
-			Region: "us-east-1",
-			Site:   "us-east-1a",
-			Hosts: model.Hosts{
-				"ctrl1": {
-					InstanceType: "t3.micro",
-					Components: model.Components{
-						"ctrl1": {
-							Scope: model.Scope{Tags: model.Tags{"ctrl"}},
-							Type:  &zitilab.ControllerType{},
-						},
-					},
-				},
-				"ctrl2": {
-					InstanceType: "t3.micro",
-					Components: model.Components{
-						"ctrl2": {
-							Scope: model.Scope{Tags: model.Tags{"ctrl"}},
-							Type:  &zitilab.ControllerType{},
-						},
-					},
-				},
-
-				"router-east": {
-					InstanceType: "t2.micro",
-					Components: model.Components{
-						"router-east": {
-							Scope: model.Scope{Tags: model.Tags{"edge-router", "terminator"}},
-							Type:  &zitilab.RouterType{},
-						},
-						"echo-server": {
-							Scope: model.Scope{Tags: model.Tags{"sdk-app", "service"}},
-							Type:  &zitilab.EchoServerType{},
-						},
-					},
-				},
-			},
-		},
-		"us-west-2": {
-			Region: "us-west-2",
-			Site:   "us-west-2b",
-			Hosts: model.Hosts{
-				"ctrl3": {
-					InstanceType: "t3.micro",
-					Components: model.Components{
-						"ctrl3": {
-							Scope: model.Scope{Tags: model.Tags{"ctrl"}},
-							Type:  &zitilab.ControllerType{},
-						},
-					},
-				},
-
-				"router-west": {
-					Scope:        model.Scope{Tags: model.Tags{}},
-					InstanceType: "t2.micro",
-					Components: model.Components{
-						"router-west": {
-							Scope: model.Scope{Tags: model.Tags{"edge-router", "terminator"}},
-							Type:  &zitilab.RouterType{},
-						},
-						"zcat": {
-							Scope: model.Scope{Tags: model.Tags{"sdk-app"}},
-							Type:  &zitilab.ZCatType{},
-						},
-					},
-				},
-			},
-		},
-	},
-
-	Actions: model.ActionBinders{
-		"bootstrap": actions.NewBootstrapAction(),
-		"start": actions.NewStartAction(actions.MetricbeatConfig{
-			ConfigPath: "metricbeat",
-			DataPath:   "metricbeat/data",
-			LogPath:    "metricbeat/logs",
-		},
-			actions.ConsulConfig{
-				ServerAddr: os.Getenv("CONSUL_ENDPOINT"),
-				ConfigDir:  "consul",
-				DataPath:   "consul/data",
-				LogPath:    "consul/log.out",
-			}),
-		"stop":  model.Bind(component.StopInParallel("*", 15)),
-		"login": model.Bind(edge.Login("#ctrl1")),
-	},
-
-	Infrastructure: model.Stages{
-		aws_ssh_key.Express(),
-		terraform_0.Express(),
-		semaphore0.Ready(time.Minute),
-	},
-
-	Distribution: model.Stages{
-		distribution.DistributeSshKey("*"),
-		distribution.Locations("*", "logs"),
-		distribution.DistributeDataWithReplaceCallbacks(
-			"*",
-			string(getConfigData("metricbeat.yml")),
-			"metricbeat/metricbeat.yml",
-			os.FileMode(0644),
-			map[string]func(*model.Host) string{
-				"${host}": func(h *model.Host) string {
-					return os.Getenv("ELASTIC_ENDPOINT")
-				},
-				"${user}": func(h *model.Host) string {
-					return os.Getenv("ELASTIC_USERNAME")
-				},
-				"${password}": func(h *model.Host) string {
-					return os.Getenv("ELASTIC_PASSWORD")
-				},
-				"${build_number}": func(h *model.Host) string {
-					return os.Getenv("BUILD_NUMBER")
-				},
-				"${ziti_version}": func(h *model.Host) string {
-					return h.MustStringVariable("ziti_version")
-				},
-			},
-		),
-
-		distribution.DistributeDataWithReplaceCallbacks(
-			"*",
-			string(getConfigData("consul.hcl")),
-			"consul/consul.hcl",
-			os.FileMode(0644),
-			map[string]func(*model.Host) string{
-				"${public_ip}": func(h *model.Host) string {
-					return h.PublicIp
-				},
-				"${encryption_key}": func(h *model.Host) string {
-					return os.Getenv("CONSUL_ENCRYPTION_KEY")
-				},
-				"${build_number}": func(h *model.Host) string {
-					return os.Getenv("BUILD_NUMBER")
-				},
-				"${ziti_version}": func(h *model.Host) string {
-					return h.MustStringVariable("ziti_version")
-				},
-			},
-		),
-		distribution.DistributeDataWithReplaceCallbacks(
-			"#ctrl",
-			string(getConfigData("ziti.hcl")),
-			"consul/ziti.hcl",
-			os.FileMode(0644),
-			map[string]func(*model.Host) string{
-				"${build_number}": func(h *model.Host) string {
-					return os.Getenv("BUILD_NUMBER")
-				},
-				"${ziti_version}": func(h *model.Host) string {
-					return h.MustStringVariable("ziti_version")
-				},
-			}),
-		distribution.DistributeData(
-			"*",
-			[]byte(os.Getenv("CONSUL_AGENT_CERT")),
-			"consul/consul-agent-ca.pem"),
-		rsync.RsyncStaged(),
-	},
-
-	Disposal: model.Stages{
-		terraform.Dispose(),
-		aws_ssh_key2.Dispose(),
-	},
-}
-
-func main() {
-	m.AddActivationActions("stop", "bootstrap", "start")
-
-	model.AddBootstrapExtension(binding.AwsCredentialsLoader)
-	model.AddBootstrapExtension(aws_ssh_key.KeyManager)
-
-	fablab.InitModel(m)
-	fablab.Run()
-}
diff --git a/zititest/models/simple/actions/bootstrap.go b/zititest/models/simple/actions/bootstrap.go
index f9c7f4391..20a9efebd 100644
--- a/zititest/models/simple/actions/bootstrap.go
+++ b/zititest/models/simple/actions/bootstrap.go
@@ -18,6 +18,7 @@ package actions
 
 import (
 	"fmt"
+	"github.com/openziti/fablab/kernel/lib/actions/semaphore"
 	"github.com/openziti/ziti/zititest/zitilab"
 	"time"
 
@@ -40,13 +41,28 @@ func NewBootstrapAction() model.ActionBinder {
 func (a *bootstrapAction) bind(m *model.Model) model.Action {
 	workflow := actions.Workflow()
 
+	isHA := len(m.SelectComponents(".ctrl")) > 1
+
+	workflow.AddAction(component.Stop(".ctrl"))
 	workflow.AddAction(host.GroupExec("*", 25, "rm -f logs/*"))
-	workflow.AddAction(component.Stop("#ctrl"))
-	workflow.AddAction(component.Exec("#ctrl", zitilab.ControllerActionInitStandalone))
-	workflow.AddAction(component.Start("#ctrl"))
-	workflow.AddAction(edge.ControllerAvailable("#ctrl", 30*time.Second))
+	workflow.AddAction(host.GroupExec("component.ctrl", 5, "rm -rf ./fablab/ctrldata"))
+
+	if !isHA {
+		workflow.AddAction(component.Exec("#ctrl1", zitilab.ControllerActionInitStandalone))
+	}
+
+	workflow.AddAction(component.Start(".ctrl"))
+
+	if isHA {
+		workflow.AddAction(semaphore.Sleep(2 * time.Second))
+		workflow.AddAction(edge.RaftJoin(".ctrl"))
+		workflow.AddAction(semaphore.Sleep(2 * time.Second))
+		workflow.AddAction(edge.InitRaftController("#ctrl1"))
+	}
+
+	workflow.AddAction(edge.ControllerAvailable("#ctrl1", 30*time.Second))
 
-	workflow.AddAction(edge.Login("#ctrl"))
+	workflow.AddAction(edge.Login("#ctrl1"))
 
 	workflow.AddAction(component.StopInParallel(models.EdgeRouterTag, 25))
 	workflow.AddAction(edge.InitEdgeRouters(models.EdgeRouterTag, 2))
@@ -64,15 +80,29 @@ func (a *bootstrapAction) bind(m *model.Model) model.Action {
 
 	workflow.AddAction(zitilib_actions.Edge("create", "config", "files-host", "host.v1", `
 		{
-			"address" : "ziti-smoketest-files.s3-us-west-1.amazonaws.com", 
-			"port" : 443, 
+			"address" : "ziti-smoketest-files.s3-us-west-1.amazonaws.com",
+			"port" : 443,
 			"protocol" : "tcp"
 		}`))
 
 	workflow.AddAction(zitilib_actions.Edge("create", "config", "iperf-host", "host.v1", `
 		{
-			"address" : "localhost", 
-			"port" : 5201, 
+			"address" : "localhost",
+			"port" : 5201,
+			"protocol" : "tcp"
+		}`))
+
+	workflow.AddAction(zitilib_actions.Edge("create", "config", "ssh-host", "host.v1", `
+		{
+			"address" : "localhost",
+			"port" : 22,
+			"protocol" : "tcp"
+		}`))
+
+	workflow.AddAction(zitilib_actions.Edge("create", "config", "fortio-host", "host.v1", `
+		{
+			"address" : "localhost",
+			"port" : 8080,
 			"protocol" : "tcp"
 		}`))
 
@@ -106,6 +136,26 @@ func (a *bootstrapAction) bind(m *model.Model) model.Action {
 
 			workflow.AddAction(zitilib_actions.Edge("create", "config", iperfConfigName, "intercept.v1", iperfConfigDef))
 
+			sshConfigName := fmt.Sprintf("ssh-intercept-%s%s", hostType, suffix)
+			sshConfigDef := fmt.Sprintf(`
+				{
+					"addresses": ["ssh-%s%s.ziti"],
+					"portRanges" : [ { "low": 2022, "high": 2022 } ],
+					"protocols": ["tcp"]
+				}`, hostType, suffix)
+
+			workflow.AddAction(zitilib_actions.Edge("create", "config", sshConfigName, "intercept.v1", sshConfigDef))
+
+			fortioConfigName := fmt.Sprintf("fortio-intercept-%s%s", hostType, suffix)
+			fortioConfigDef := fmt.Sprintf(`
+				{
+					"addresses": ["fortio-%s%s.ziti"],
+					"portRanges" : [ { "low": 8080, "high": 8080 } ],
+					"protocols": ["tcp"]
+				}`, hostType, suffix)
+
+			workflow.AddAction(zitilib_actions.Edge("create", "config", fortioConfigName, "intercept.v1", fortioConfigDef))
+
 			filesServiceName := fmt.Sprintf("%s-files%s", hostType, suffix)
 			filesConfigs := fmt.Sprintf("files-host,%s", filesConfigName)
 			workflow.AddAction(zitilib_actions.Edge("create", "service", filesServiceName, "-c", filesConfigs, "-e", encryptionFlag, "-a", hostType))
@@ -113,6 +163,14 @@ func (a *bootstrapAction) bind(m *model.Model) model.Action {
 			iperfServiceName := fmt.Sprintf("%s-iperf%s", hostType, suffix)
 			iperfConfigs := fmt.Sprintf("iperf-host,%s", iperfConfigName)
 			workflow.AddAction(zitilib_actions.Edge("create", "service", iperfServiceName, "-c", iperfConfigs, "-e", encryptionFlag, "-a", hostType))
+
+			sshServiceName := fmt.Sprintf("%s-ssh%s", hostType, suffix)
+			sshConfigs := fmt.Sprintf("ssh-host,%s", sshConfigName)
+			workflow.AddAction(zitilib_actions.Edge("create", "service", sshServiceName, "-c", sshConfigs, "-e", encryptionFlag, "-a", hostType))
+
+			fortioServiceName := fmt.Sprintf("%s-fortio%s", hostType, suffix)
+			fortioConfigs := fmt.Sprintf("fortio-host,%s", fortioConfigName)
+			workflow.AddAction(zitilib_actions.Edge("create", "service", fortioServiceName, "-c", fortioConfigs, "-e", encryptionFlag, "-a", hostType))
 		}
 	}
 
diff --git a/zititest/models/simple/actions/start.go b/zititest/models/simple/actions/start.go
index 576267706..a1220c129 100644
--- a/zititest/models/simple/actions/start.go
+++ b/zititest/models/simple/actions/start.go
@@ -38,8 +38,8 @@ func NewStartAction(metricbeat MetricbeatConfig, consul ConsulConfig) model.Acti
 
 func (a *startAction) bind(m *model.Model) model.Action {
 	workflow := actions.Workflow()
-	workflow.AddAction(component.Start("#ctrl"))
-	workflow.AddAction(edge.ControllerAvailable("#ctrl", 30*time.Second))
+	workflow.AddAction(component.Start(".ctrl"))
+	workflow.AddAction(edge.ControllerAvailable("#ctrl1", 30*time.Second))
 	workflow.AddAction(component.StartInParallel(models.EdgeRouterTag, 25))
 	workflow.AddAction(component.StartInParallel(".iperf", 5))
 
diff --git a/zititest/models/simple/configs/ctrl.yml.tmpl b/zititest/models/simple/configs/ctrl.yml.tmpl
index 8e2f46c7b..be10aa95a 100644
--- a/zititest/models/simple/configs/ctrl.yml.tmpl
+++ b/zititest/models/simple/configs/ctrl.yml.tmpl
@@ -1,17 +1,23 @@
 v: 3
 
+{{if .Component.GetFlag "ha"}}
+raft:
+  minClusterSize: 3
+  dataDir: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/ctrldata
+{{else}}
 db: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/ctrl.db
+{{end}}
 
 identity:
-  cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/ctrl/certs/{{ .Component.Id }}-server.cert
-  key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/ctrl/keys/{{ .Component.Id }}-server.key
-  ca: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/ctrl/certs/{{ .Component.Id }}-server.chain.pem
+  cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}-server.cert
+  key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/keys/{{ .Component.Id }}-server.key
+  ca: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}-server.chain.pem
 
 # the endpoint that routers will connect to the controller over.
 ctrl:
   listener: tls:0.0.0.0:6262
   options:
-    advertiseAddress: tls:{{.Host.PublicIp}}:6262
+    advertiseAddress: tls:{{ .Host.PublicIp }}:6262
     # (optional) settings
     # set the maximum number of connect requests that are buffered and waiting to be acknowledged (1 to 5000, default 1000)
     #maxQueuedConnects:      50
@@ -80,7 +86,7 @@ edge:
     # address - required
     # The default address (host:port) to use for enrollment for the Client API. This value must match one of the addresses
     # defined in a bind point's address field for the `edge-client` API in the web section.
-    address: {{.Host.PublicIp}}:1280
+    address: {{ .Host.PublicIp }}:1280
   # enrollment - required
   # A section containing settings pertaining to enrollment.
   enrollment:
@@ -89,8 +95,8 @@ edge:
     # a signing certificate from the PKI that the Ziti environment is using to sign certificates. The signingCert.cert
     # will be added to the /.well-known CA store that is used to bootstrap trust with the Ziti Controller.
     signingCert:
-      cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/ctrl/certs/ctrl.cert
-      key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/ctrl/keys/ctrl.key
+      cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}.cert
+      key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/keys/{{ .Component.Id }}.key
 
     # edgeIdentity - optional
     # A section for identity enrollment specific settings
@@ -128,7 +134,7 @@ web:
         # address - required
         # The public address that external incoming requests will be able to resolve. Used in request processing and
         # response content that requires full host:port/path addresses.
-        address: {{.Host.PublicIp}}:1280
+        address: {{ .Host.PublicIp }}:1280
 
         # newAddress - optional
         # A host:port string which will be sent out as an HTTP header "ziti-new-address" if specified. If the header
diff --git a/zititest/models/simple/configs/router.yml.tmpl b/zititest/models/simple/configs/router.yml.tmpl
index 788799f78..0f5bd0e01 100644
--- a/zititest/models/simple/configs/router.yml.tmpl
+++ b/zititest/models/simple/configs/router.yml.tmpl
@@ -1,6 +1,5 @@
 {{$ssh_username := .Model.MustVariable "credentials.ssh.username"}}
 {{$identity := .Component.Id}}
-{{$ctrl_ip := publicIp "component#ctrl"}}
 {{$router_ip := .Host.PublicIp}}
   
 v: 3
@@ -14,7 +13,8 @@ identity:
   ca:                   /home/{{$ssh_username}}/fablab/cfg/{{$identity}}-server.chain.pem
 
 ctrl:
-  endpoint:             tls:{{$ctrl_ip}}:6262
+  endpoints: {{ range $host := .Model.MustSelectHosts "component.ctrl" 1 }}
+    - tls:{{ $host.PublicIp }}:6262{{end}}
 
 healthChecks:
   ctrlPingCheck:
diff --git a/zititest/models/simple/simple.go b/zititest/models/simple/simple.go
index a156115ec..dee895868 100644
--- a/zititest/models/simple/simple.go
+++ b/zititest/models/simple/simple.go
@@ -31,7 +31,7 @@ import (
 	"github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/terraform"
 	"github.com/openziti/fablab/kernel/model"
 	"github.com/openziti/fablab/resources"
-	actions2 "github.com/openziti/ziti/zititest/models/simple/actions"
+	"github.com/openziti/ziti/zititest/models/simple/actions"
 	"github.com/openziti/ziti/zititest/models/test_resources"
 	"github.com/openziti/ziti/zititest/zitilab"
 	"github.com/openziti/ziti/zititest/zitilab/actions/edge"
@@ -78,6 +78,17 @@ var Model = &model.Model{
 		},
 	},
 
+	StructureFactories: []model.Factory{
+		model.FactoryFunc(func(m *model.Model) error {
+			if val, _ := m.GetBoolVariable("ha"); !val {
+				for _, host := range m.SelectHosts("component.ha") {
+					delete(host.Region.Hosts, host.Id)
+				}
+			}
+			return nil
+		}),
+	},
+
 	Factories: []model.Factory{
 		model.FactoryFunc(func(m *model.Model) error {
 			pfxlog.Logger().Infof("environment [%s]", m.MustStringVariable("environment"))
@@ -102,10 +113,18 @@ var Model = &model.Model{
 			Region: "us-east-1",
 			Site:   "us-east-1a",
 			Hosts: model.Hosts{
-				"ctrl": {
+				"ctrl1": {
 					Components: model.Components{
-						"ctrl": {
-							Scope: model.Scope{Tags: model.Tags{"ctrl", "long-running"}},
+						"ctrl1": {
+							Scope: model.Scope{Tags: model.Tags{"ctrl"}},
+							Type:  &zitilab.ControllerType{},
+						},
+					},
+				},
+				"ctrl2": {
+					Components: model.Components{
+						"ctrl2": {
+							Scope: model.Scope{Tags: model.Tags{"ctrl", "ha"}},
 							Type:  &zitilab.ControllerType{},
 						},
 					},
@@ -114,7 +133,7 @@ var Model = &model.Model{
 					Scope: model.Scope{Tags: model.Tags{"ert-client"}},
 					Components: model.Components{
 						"router-east-1": {
-							Scope: model.Scope{Tags: model.Tags{"edge-router", "terminator", "tunneler", "client", "long-running"}},
+							Scope: model.Scope{Tags: model.Tags{"edge-router", "terminator", "tunneler", "client"}},
 							Type:  &zitilab.RouterType{},
 						},
 						"zcat": {
@@ -126,7 +145,7 @@ var Model = &model.Model{
 				"router-east-2": {
 					Components: model.Components{
 						"router-east-2": {
-							Scope: model.Scope{Tags: model.Tags{"edge-router", "initiator", "long-running"}},
+							Scope: model.Scope{Tags: model.Tags{"edge-router", "initiator"}},
 							Type:  &zitilab.RouterType{},
 						},
 					},
@@ -135,7 +154,7 @@ var Model = &model.Model{
 					Scope: model.Scope{Tags: model.Tags{"zet-client"}},
 					Components: model.Components{
 						"ziti-edge-tunnel-client": {
-							Scope: model.Scope{Tags: model.Tags{"sdk-app", "client", "long-running"}},
+							Scope: model.Scope{Tags: model.Tags{"sdk-app", "client"}},
 							Type: &zitilab.ZitiEdgeTunnelType{
 								Version: "v0.21.4",
 							},
@@ -146,7 +165,7 @@ var Model = &model.Model{
 					Scope: model.Scope{Tags: model.Tags{"ziti-tunnel-client"}},
 					Components: model.Components{
 						"ziti-tunnel-client": {
-							Scope: model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "client", "long-running"}},
+							Scope: model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "client"}},
 							Type:  &zitilab.ZitiTunnelType{},
 						},
 					},
@@ -157,18 +176,27 @@ var Model = &model.Model{
 			Region: "us-west-2",
 			Site:   "us-west-2b",
 			Hosts: model.Hosts{
+				"ctrl3": {
+					Components: model.Components{
+						"ctrl3": {
+							Scope: model.Scope{Tags: model.Tags{"ctrl", "ha"}},
+							Type:  &zitilab.ControllerType{},
+						},
+					},
+				},
+
 				"router-west": {
 					Components: model.Components{
 						"router-west": {
-							Scope: model.Scope{Tags: model.Tags{"edge-router", "tunneler", "host", "ert-host", "long-running"}},
+							Scope: model.Scope{Tags: model.Tags{"edge-router", "tunneler", "host", "ert-host"}},
 							Type:  &zitilab.RouterType{},
 						},
 						"echo-server": {
-							Scope: model.Scope{Tags: model.Tags{"sdk-app", "service", "long-running"}},
+							Scope: model.Scope{Tags: model.Tags{"sdk-app", "service"}},
 							Type:  &zitilab.EchoServerType{},
 						},
 						"iperf-server-ert": {
-							Scope: model.Scope{Tags: model.Tags{"iperf", "service", "long-running"}},
+							Scope: model.Scope{Tags: model.Tags{"iperf", "service"}},
 							Type:  &zitilab.IPerfServerType{},
 						},
 					},
@@ -176,13 +204,13 @@ var Model = &model.Model{
 				"ziti-edge-tunnel-host": {
 					Components: model.Components{
 						"ziti-edge-tunnel-host": {
-							Scope: model.Scope{Tags: model.Tags{"sdk-app", "host", "zet-host", "long-running"}},
+							Scope: model.Scope{Tags: model.Tags{"sdk-app", "host", "zet-host"}},
 							Type: &zitilab.ZitiEdgeTunnelType{
 								Version: "v0.21.4",
 							},
 						},
 						"iperf-server-zet": {
-							Scope: model.Scope{Tags: model.Tags{"iperf", "service", "long-running"}},
+							Scope: model.Scope{Tags: model.Tags{"iperf", "service"}},
 							Type:  &zitilab.IPerfServerType{},
 						},
 					},
@@ -190,13 +218,13 @@ var Model = &model.Model{
 				"ziti-tunnel-host": {
 					Components: model.Components{
 						"ziti-tunnel-host": {
-							Scope: model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "host", "ziti-tunnel-host", "long-running"}},
+							Scope: model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "host", "ziti-tunnel-host"}},
 							Type: &zitilab.ZitiTunnelType{
 								Mode: zitilab.ZitiTunnelModeHost,
 							},
 						},
 						"iperf-server-zt": {
-							Scope: model.Scope{Tags: model.Tags{"iperf", "service", "long-running"}},
+							Scope: model.Scope{Tags: model.Tags{"iperf", "service"}},
 							Type:  &zitilab.IPerfServerType{},
 						},
 					},
@@ -206,20 +234,20 @@ var Model = &model.Model{
 	},
 
 	Actions: model.ActionBinders{
-		"bootstrap": actions2.NewBootstrapAction(),
-		"start": actions2.NewStartAction(actions2.MetricbeatConfig{
+		"bootstrap": actions.NewBootstrapAction(),
+		"start": actions.NewStartAction(actions.MetricbeatConfig{
 			ConfigPath: "metricbeat",
 			DataPath:   "metricbeat/data",
 			LogPath:    "metricbeat/logs",
 		},
-			actions2.ConsulConfig{
+			actions.ConsulConfig{
 				ServerAddr: os.Getenv("CONSUL_ENDPOINT"),
 				ConfigDir:  "consul",
 				DataPath:   "consul/data",
 				LogPath:    "consul/log.out",
 			}),
 		"stop":  model.Bind(component.StopInParallel("*", 15)),
-		"login": model.Bind(edge.Login("#ctrl")),
+		"login": model.Bind(edge.Login("#ctrl1")),
 	},
 
 	Infrastructure: model.Stages{
diff --git a/zititest/scripts/get-ha-smoke-instance b/zititest/scripts/get-ha-smoke-instance
new file mode 100755
index 000000000..c3dc55a0f
--- /dev/null
+++ b/zititest/scripts/get-ha-smoke-instance
@@ -0,0 +1,4 @@
+# You must have a FABLAB_PASSPHRASE environment variable set with the GPG key to decrypt the smoketests
+aws s3 cp s3://ziti-smoketest-fablab-instances/simple-transfer-ha-$1.tar.gz.gpg ${HOME}/Downloads/simple-transfer-ha-$1.tar.gz.gpg
+simple-transfer import ${HOME}/Downloads/simple-transfer-ha-$1.tar.gz.gpg
+rm ${HOME}/Downloads/simple-transfer-ha-$1.tar.gz.gpg
diff --git a/zititest/scripts/get-smoke-instance b/zititest/scripts/get-smoke-instance
new file mode 100755
index 000000000..0c3f4a0df
--- /dev/null
+++ b/zititest/scripts/get-smoke-instance
@@ -0,0 +1,4 @@
+# You must have a FABLAB_PASSPHRASE environment variable set with the GPG key to decrypt the smoketests
+aws s3 cp s3://ziti-smoketest-fablab-instances/simple-transfer-$1.tar.gz.gpg ${HOME}/Downloads/simple-transfer-$1.tar.gz.gpg
+simple-transfer import ${HOME}/Downloads/simple-transfer-$1.tar.gz.gpg
+rm ${HOME}/Downloads/simple-transfer-$1.tar.gz.gpg
diff --git a/zititest/tests/scp_test.go b/zititest/tests/scp_test.go
new file mode 100644
index 000000000..63e60809c
--- /dev/null
+++ b/zititest/tests/scp_test.go
@@ -0,0 +1,127 @@
+/*
+	(c) Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package tests
+
+import (
+	"fmt"
+	"github.com/google/uuid"
+	"github.com/openziti/fablab/kernel/lib"
+	"github.com/openziti/fablab/kernel/model"
+	"github.com/stretchr/testify/require"
+	"testing"
+	"time"
+)
+
+func TestScp(t *testing.T) {
+	allZetHostedFailed := true
+	allZetClientsFailed := true
+
+	t.Run("scp-tests", func(t *testing.T) {
+		t.Run("test-ert-scp", func(t *testing.T) {
+			t.Parallel()
+			for _, hostType := range []string{"ert", "zet", "ziti-tunnel"} {
+				for _, encrypted := range []bool{true, false} {
+					success := testScp(t, "ert", hostType, encrypted)
+					if hostType == "zet" && success {
+						allZetHostedFailed = false
+					}
+				}
+			}
+		})
+
+		t.Run("test-zet-scp", func(t *testing.T) {
+			t.Parallel()
+
+			for _, hostType := range []string{"zet", "ziti-tunnel", "ert"} {
+				for _, encrypted := range []bool{true, false} {
+					success := testScp(t, "zet", hostType, encrypted)
+					if hostType == "zet" && success {
+						allZetHostedFailed = false
+					}
+					if success {
+						allZetClientsFailed = false
+					}
+				}
+			}
+		})
+
+		t.Run("test-ziti-tunnel-scp", func(t *testing.T) {
+			t.Parallel()
+
+			for _, hostType := range []string{"ziti-tunnel", "ert", "zet"} {
+				for _, encrypted := range []bool{true, false} {
+					success := testScp(t, "ziti-tunnel", hostType, encrypted)
+					if hostType == "zet" && success {
+						allZetHostedFailed = false
+					}
+				}
+			}
+		})
+	})
+
+	req := require.New(t)
+	req.False(allZetHostedFailed, "all zet hosted file transfer should not failed, indicates bigger issue")
+
+	// TODO: fix once ZET client tests are working
+	req.True(allZetClientsFailed, "all zet client file transfers should not failed, indicates bigger issue")
+}
+
+func testScp(t *testing.T, hostSelector string, hostType string, encrypted bool) bool {
+	encDesk := "encrypted"
+	if !encrypted {
+		encDesk = "unencrypted"
+	}
+
+	success := false
+
+	t.Run(fmt.Sprintf("(%s->%s)-%v", hostSelector, hostType, encDesk), func(t *testing.T) {
+		if hostSelector == "zet" {
+			t.Skipf("zet is currently failing as client")
+		}
+		host, err := model.GetModel().SelectHost("." + hostSelector + "-client")
+		req := require.New(t)
+		req.NoError(err)
+
+		nameExtra := ""
+		if !encrypted {
+			nameExtra = "-unencrypted"
+		}
+
+		sshConfigFactory := lib.NewSshConfigFactory(host)
+
+		cmds := []string{
+			fmt.Sprintf("scp -o StrictHostKeyChecking=no ssh-%s%s.ziti:./fablab/bin/ziti /tmp/ziti-%s", hostType, nameExtra, uuid.NewString()),
+			fmt.Sprintf("scp -o StrictHostKeyChecking=no ./fablab/bin/ziti ssh-%s%s.ziti:/tmp/ziti-%s", hostType, nameExtra, uuid.NewString()),
+		}
+
+		o, err := lib.RemoteExecAllWithTimeout(sshConfigFactory, 10*time.Second, cmds...)
+		if hostType == "zet" && err != nil {
+			t.Skipf("zet hosted ssh failed [%v]", err.Error())
+			return
+		}
+
+		if hostSelector == "zet" && err != nil {
+			t.Skipf("zet client ssh failed [%v]", err.Error())
+			return
+		}
+
+		t.Log(o)
+		req.NoError(err)
+		success = true
+	})
+	return success
+}
diff --git a/zititest/zitilab/component_common.go b/zititest/zitilab/component_common.go
index 9268d6b91..f30216a9d 100644
--- a/zititest/zitilab/component_common.go
+++ b/zititest/zitilab/component_common.go
@@ -45,10 +45,6 @@ func startZitiComponent(c *model.Component, zitiType string, version string, con
 	configPath := fmt.Sprintf("/home/%s/fablab/cfg/%s", factory.User(), configName)
 	logsPath := fmt.Sprintf("/home/%s/logs/%s.log", factory.User(), c.Id)
 
-	if zitiType == "tunnel" {
-
-	}
-
 	useSudo := ""
 	if zitiType == "tunnel" || c.HasTag("tunneler") {
 		useSudo = "sudo"