Db creation 2 (#1444)

Squashed and merged. Adds db-creation model and workflows for cd/ci to store fairly populated DB and pki on each release to s3.

.github/workflows/fablab-db-creation.yml

@@ -0,0 +1,61 @@
+name: fablab db-creation workflow
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+env:
+  GOFLAGS: "-trimpath"
+  GOX_OUTPUT: "release/{{.Arch}}/{{.OS}}/{{.Dir}}"
+  GOX_TEST_OUTPUT: "test/{{.Arch}}/{{.OS}}/bin/{{.Dir}}"
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_DEFAULT_REGION: "us-east-1"
+  gh_ci_key: ${{ secrets.GH_CI_KEY }}
+  S3_KEY: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  S3_SECRET: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+jobs:
+  build:
+    name: Build and Run
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout ziti
+        uses: actions/checkout@v3
+        with:
+          path: ziti
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.20.x'
+
+      - name: Install Ziti CI
+        uses: openziti/ziti-ci@v1
+
+      - name: Configure Git
+        run: |
+          cd ziti
+          $(go env GOPATH)/bin/ziti-ci configure-git
+
+      - name: Pull ZITI_VERSION and set as $GITHUB_ENV for use with fablab
+        run: |
+          cd ziti
+          version="$($(go env GOPATH)/bin/ziti-ci -q get-current-version)"
+          echo "Ziti Version: $version"
+          echo "ZITI_VERSION=$version" >> $GITHUB_ENV
+
+      - name: Build and Run
+        run: |
+          cd ziti/zititest/models/db-creation
+          go build -o db-creation main.go
+          echo "ZITI_ROOT=$(go env GOPATH)/bin" >> "$GITHUB_ENV"
+          ./db-creation create db-creation
+          ./db-creation up
+
+      - name: Teardown
+        if: always()
+        run: |
+          cd ziti/zititest/models/db-creation          
+          ./db-creation dispose

common/getziti/github.go

@@ -6,6 +6,7 @@ import (
 	"github.com/blang/semver"
 	"github.com/go-resty/resty/v2"
 	"github.com/michaelquigley/pfxlog"
+	"github.com/openziti/foundation/v2/versions"
 	c "github.com/openziti/ziti/ziti/constants"
 	"github.com/pkg/errors"
 	"net/http"
@@ -247,8 +248,17 @@ func InstallGitHubRelease(zitiApp string, release *GitHubReleasesData, binDir st
 		if zitiApp == c.ZITI {
 			count := 0
 			zitiFileName := "ziti-" + version
+			semVer, err := versions.ParseSemVer(version)
+			if err != nil {
+				return err
+			}
+			expectedPath := "ziti"
+			pathChangedVersion := versions.MustParseSemVer("0.29.0")
+			if semVer.CompareTo(pathChangedVersion) < 0 {
+				expectedPath = "ziti/ziti"
+			}
 			err = UnTarGz(fullPath, binDir, func(path string) (string, bool) {
-				if path == "ziti/ziti" {
+				if path == expectedPath {
 					count++
 					return zitiFileName, true
 				}

zititest/models/db-creation/README.md

@@ -0,0 +1,50 @@
+# db-creation model
+
+### This model is designed to be used for GitHub Actions to create a test DB and export the pki/identities/DB to s3 buckets for later testing usage. ###
+
+- Only setup for AWS.
+- Designed to work with the pete-iperf branch of fablab.
+- You will need to supply your own keys/secrets.
+- Infrastructure is configured in the main.go in the model.
+- This is a very alpha release, minimal features.
+
+### There are several files that will likely need to be customized for your setup: ### 
+
+- ziti/zititest/models/db-creation/main.go - mainly used to alter the model and also your Rsync and Disposal Actions (removing Route 53 A Record)
+- ziti/zititest/models/db-creation/actions/bootstrap.go - This is where the meat of the actions take place. Sets up AWS remotely from the GH Runner (using Fablab executable), then runs the DB Creation Script.
+- ziti/zititest/models/db-creation/resources/db_creator_script_external.sh - This is the script that interacts with Ziti and creates all the identities, services and policies.
+- ziti/zititest/models/db-creation/resources/aws_setup.sh - This will default to us-east-1 region and use JSON output, if you want to change those values do that here.
+- ziti/.github/workflows/fablab-db-creation.yml - This is where you will setup your GitHub workflow specifics, inserting your custom secret variable names, etc. As you can see at the end, the following 3 Fablab commands are all that is needed to run this:
+    - ```./db-creation create db-creation```
+    - ```./db-creation up```
+    -  ```./db-creation dispose```
+
+### Once the DB is saved in s3, you will need to pull that and the pki from the proper buckets via the following steps:
+
+#### Non Fablab import (manual) or something designed by you ####
+- Make sure AWS CLI is configured on the machine you want the DB imported to.
+- cd to the /home/ubuntu/fablab directory which is where the DB lies.
+- Stop any existing Ziti processes.
+- Simply delete the old DB file or rename it.
+- Run the following AWS CLI command to import DB:
+    - ```aws s3 cp s3://db-bucket-name/ctrl.db-filename ctrl.db ```
+- Remove the contents of the entire pki directory using the following:
+    - ```cd pki```
+    - ```sudo rm -rf *```
+    - ```cd ..```
+- Run the following to import the pki directory (replacing pki-s3-bucket-name/pki-folder-name with your names) :
+    - ```aws s3 cp --recursive s3://pki-s3-bucket-name/pki-s3-folder-name/ pki/```
+- Run the following command while replacing the ziti version number in filename to start the controller:
+    - ```nohup  /home/ubuntu/fablab/bin/ziti-v0.28.4 controller run --log-formatter pfxlog /home/ubuntu/fablab/cfg/ctrl.yml --cli-agent-alias ctrl > /home/ubuntu/logs/ctrl.log 2>&1 & ```
+
+#### Fablab import ####
+- cd into your local ziti/zititest/models/db-creation/resources folder and then import both the DB and PKI from your s3 buckets:
+    - Command to run for your DB import:
+        - ```aws s3 cp s3://s3-db-bucket-name/s3-ctrl.db-filename ctrl.db```
+    - Commands to run for your PKI import:
+        - ```mkdir pki```
+        - ```aws s3 cp --recursive s3://pki-s3-bucket-name/pki-s3-folder-name/ pki/```
+- Within your main.go for the db-creation model, you should uncomment the 2 following lines within the Distribution portion of the model, around line 123 or so:
+    - ```rsync.NewRsyncHost("#ctrl", "resources/ctrl.db", "/home/ubuntu/fablab/ctrl.db"),```
+    - ```rsync.NewRsyncHost("#ctrl", "resources/pki/", "/home/ubuntu/fablab/pki/"),```
+- Now you should be able to create a fresh db-creation executable by building and run that, which should have the new DB/PKI.