From a02a8ae205de97024d2c30a8cf3c2cd7cb82448c Mon Sep 17 00:00:00 2001 From: schristoff <28318173+schristoff@users.noreply.github.com> Date: Sun, 4 Feb 2024 22:47:03 -0700 Subject: [PATCH 01/10] chore: add reusable integ workflow, add publish variables (#2979) * chore: ADO -> GHA, add more go caching, add docker buildx to jobs, and added reusable workflow for porter integration so each integ gets their own job (stop out of space errors) Signed-off-by: Sarah Christoff <28318173+schristoff@users.noreply.github.com> --------- Signed-off-by: Sarah Christoff <28318173+schristoff@users.noreply.github.com> --- .../workflows/build-publish-from-fork.yaml | 36 ------ .../build_pipelinesrelease_template.yml | 19 ++-- .github/workflows/check-licenses.yaml | 2 +- .github/workflows/close-issues.yaml | 6 +- .github/workflows/codeql.yaml | 2 +- .../workflows/integ-reuseable-workflow.yml | 41 +++++++ .github/workflows/porter-integration.yml | 104 ++++++++++++++++-- .github/workflows/porter.yml | 6 + .github/workflows/trivy.yaml | 2 +- build/testdata/bundles/wordpress/.gitignore | 2 - build/testdata/bundles/wordpressv2/.gitignore | 0 magefile.go | 10 +- tests/integration/archive_test.go | 1 - tests/integration/dependenciesv1_test.go | 1 - tests/integration/dependenciesv2_test.go | 2 +- 15 files changed, 167 insertions(+), 67 deletions(-) delete mode 100644 .github/workflows/build-publish-from-fork.yaml create mode 100644 .github/workflows/integ-reuseable-workflow.yml create mode 100644 build/testdata/bundles/wordpressv2/.gitignore diff --git a/.github/workflows/build-publish-from-fork.yaml b/.github/workflows/build-publish-from-fork.yaml deleted file mode 100644 index 7a2ff2b9f..000000000 --- a/.github/workflows/build-publish-from-fork.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: build-publish-from-fork -on: - workflow_dispatch: {} - push: {} - pull_request: {} - -jobs: - build: - runs-on: ubuntu-latest - if: github.repository != 'getporter/porter' - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Login to DockerHub - if: github.event_name != 'pull_request' - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ github.token }} - - uses: actions/setup-go@v3 - with: - go-version-file: go.mod - cache: true - cache-dependency-path: go.sum - - name: Set up Mage - run: go run mage.go EnsureMage - - name: Publish - if: ${{ github.event_name != 'pull_request' }} - env: - PORTER_REGISTRY: ghcr.io/${{ github.repository }} - run: | - mage -v XBuildAll - mage -v PublishServerMultiArchImages diff --git a/.github/workflows/build_pipelinesrelease_template.yml b/.github/workflows/build_pipelinesrelease_template.yml index c4cc138da..30e9d9fbc 100644 --- a/.github/workflows/build_pipelinesrelease_template.yml +++ b/.github/workflows/build_pipelinesrelease_template.yml @@ -92,7 +92,7 @@ jobs: shell: bash Validate-integration_test: env: - GHCR_IOGETPORTER_DOCKER_REGISTRY: https://ghcr.io + GHCR_IOGETPORTER_DOCKER_REGISTRY: ${{inputs.registry}} GHCR_IOGETPORTER_DOCKER_USERNAME: getporterbot name: Integration Test needs: @@ -140,7 +140,7 @@ jobs: run: go run mage.go ConfigureAgent UseXBuildBinaries - name: Run Smoke Tests run: mage -v TestSmoke - Publish-publish_binaries: + publish_binaries: name: Publish Binaries needs: - Validate-build @@ -168,10 +168,10 @@ jobs: env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" run: mage PublishPorter PublishMixins - Publish-publish_docker: + publish-ghcr: env: - DOCKER_REGISTRY: - DOCKER_USERNAME: + DOCKER_REGISTRY: ${{inputs.registry}} + DOCKER_USERNAME: getporterbot name: Publish Docker Images needs: - Validate-build @@ -195,12 +195,11 @@ jobs: path: bin - name: Setup Bin run: go run mage.go ConfigureAgent UseXBuildBinaries - # Unable to determine registry '${{parameters.registry}}' type. The service connection was not found or the authentication type not supported. - - name: Docker Login + - name: Login to Container Registry uses: docker/login-action@v3.0.0 with: - registry: "${{ env.DOCKER_REGISTRY }}" - username: "${{ env.DOCKER_USERNAME }}" - password: "${{ secrets.DOCKER_PASSWORD }}" + registry: "${{inputs.registry}}" + username: "${{ github.actor }}" + password: "${{ secrets.GITHUB_TOKEN }}" - name: Publish Docker Images to ${{inputs.registry}} run: PORTER_REGISTRY=${{inputs.registry}} mage PublishImages diff --git a/.github/workflows/check-licenses.yaml b/.github/workflows/check-licenses.yaml index 99576c64f..9d306c380 100644 --- a/.github/workflows/check-licenses.yaml +++ b/.github/workflows/check-licenses.yaml @@ -14,7 +14,7 @@ jobs: with: fetch-depth: 0 - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version-file: go.mod cache: true diff --git a/.github/workflows/close-issues.yaml b/.github/workflows/close-issues.yaml index 3487ea91b..702fc94a7 100644 --- a/.github/workflows/close-issues.yaml +++ b/.github/workflows/close-issues.yaml @@ -10,10 +10,10 @@ on: - release/* jobs: - closeIssue: + closeIssueOnPrMergeTrigger: runs-on: ubuntu-latest steps: - name: Closes issues related to a merged pull request. - uses: ldez/gha-mjolnir@v1.1.0 + uses: ldez/gha-mjolnir@v1.0.3 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 6b4fea43c..51c579f2b 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -21,7 +21,7 @@ jobs: with: fetch-depth: 0 # Get all git history - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version-file: go.mod cache: true diff --git a/.github/workflows/integ-reuseable-workflow.yml b/.github/workflows/integ-reuseable-workflow.yml new file mode 100644 index 000000000..d78be7787 --- /dev/null +++ b/.github/workflows/integ-reuseable-workflow.yml @@ -0,0 +1,41 @@ +name: integ reusable workflow + +on: + workflow_call: + inputs: + test_name: + type: string + required: false +env: + GOVERSION: 1.20.7 + PORTER_INTEG_FILE: ${{inputs.test_name}}.go + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + - uses: actions/setup-go@v4 + with: + go-version: "${{ env.GOVERSION }}" + cache: true + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + - name: Docker Login + uses: docker/login-action@v3.0.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Configure Agent + run: go run mage.go build + shell: bash + - name: Integration Test + run: go run mage.go -v TestIntegration + shell: bash diff --git a/.github/workflows/porter-integration.yml b/.github/workflows/porter-integration.yml index 7a177e2f3..041172e44 100644 --- a/.github/workflows/porter-integration.yml +++ b/.github/workflows/porter-integration.yml @@ -1,8 +1,4 @@ name: porter/porter-integration -# this will only run when this is on main branch apparently so use pr for now (temporary) -#on: -# issue_comment: -# types: [created, edited] on: pull_request: branches: @@ -14,8 +10,88 @@ env: GOVERSION: 1.20.7 jobs: - Integration_test: - name: Integration Test + archive_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: archive_test + build_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: build_test + cli_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: cli_test + connection_nix_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: connection_nix_test + copy_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: copy_test + dependenciesv1_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: dependenciesv1_test + dependenciesv2_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: dependenciesv2_test + driver_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: driver_test + install_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: install_test + invoke_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: invoke_test + lint_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: lint_test + migration_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: migration_test + outputs_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: outputs_test + publish_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: publish_test + pull_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: pull_test + registry_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: registry_integration_test + schema_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: schema_test + sensitive_data_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: sensitive_data_test + suppress_output_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: suppress_output_test + telemetry_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: telemetry_test + # Reusable workflows only supports 20 jobs + uninstall_test_integ: runs-on: ubuntu-latest steps: - name: checkout @@ -23,15 +99,25 @@ jobs: - uses: actions/setup-go@v4 with: go-version: "${{ env.GOVERSION }}" + cache: true + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} - name: Docker Login uses: docker/login-action@v3.0.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Native Build - run: go run mage.go Build + - name: Configure Agent + run: go run mage.go build shell: bash - name: Integration Test - run: go run mage.go -v TestIntegration + env: + PORTER_INTEG_FILE: uninstall_test.go + run: go run mage.go -v TestIntegration shell: bash diff --git a/.github/workflows/porter.yml b/.github/workflows/porter.yml index 348c73d3f..14472eefc 100644 --- a/.github/workflows/porter.yml +++ b/.github/workflows/porter.yml @@ -18,6 +18,7 @@ jobs: - uses: actions/setup-go@v4 with: go-version: "${{ env.GOVERSION }}" # The Go version to download (if necessary) and use. + cache: true - run: go version - name: Native Build run: go run mage.go build @@ -37,6 +38,7 @@ jobs: - uses: actions/setup-go@v4 with: go-version: "${{ env.GOVERSION }}" # The Go version to download (if necessary) and use. + cache: true - run: go version - name: Cross Compile run: go run mage.go -v XBuildAll @@ -56,6 +58,7 @@ jobs: - uses: actions/setup-go@v4 with: go-version: "${{ env.GOVERSION }}" # The Go version to download (if necessary) and use. + cache: true - run: go version - name: Unit Test run: go run mage.go -v TestUnit @@ -70,6 +73,7 @@ jobs: - uses: actions/setup-go@v4 with: go-version: "${{ env.GOVERSION }}" # The Go version to download (if necessary) and use. + cache: true - run: go version - name: Vet run: go run mage.go Vet @@ -90,6 +94,7 @@ jobs: - uses: actions/setup-go@v4 with: go-version: "${{ env.GOVERSION }}" # The Go version to download (if necessary) and use. + cache: true - run: go version - name: Download Cross-Compiled Porter Binaries uses: actions/download-artifact@v3.0.1 @@ -117,6 +122,7 @@ jobs: - uses: actions/setup-go@v4 with: go-version: "${{ env.GOVERSION }}" # The Go version to download (if necessary) and use. + cache: true - run: go version - name: Setup Bin run: go run mage.go UseXBuildBinaries diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index 05ac36e4a..d6c0fa8fb 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -21,7 +21,7 @@ jobs: with: fetch-depth: 0 # Get all git history - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version-file: go.mod cache: true diff --git a/build/testdata/bundles/wordpress/.gitignore b/build/testdata/bundles/wordpress/.gitignore index ea5d8c03a..e69de29bb 100644 --- a/build/testdata/bundles/wordpress/.gitignore +++ b/build/testdata/bundles/wordpress/.gitignore @@ -1,2 +0,0 @@ -Dockerfile -.cnab diff --git a/build/testdata/bundles/wordpressv2/.gitignore b/build/testdata/bundles/wordpressv2/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/magefile.go b/magefile.go index 039261c4d..eebe70c24 100644 --- a/magefile.go +++ b/magefile.go @@ -580,7 +580,15 @@ func TestIntegration() { verbose = "-v" } - must.Command("go", "test", verbose, "-timeout=30m", run, "-tags=integration", "./...").CollapseArgs().RunV() + var path string + filename := os.Getenv("PORTER_INTEG_FILE") + if filename == "" { + path = "./..." + } else { + path = "./tests/integration/" + filename + } + + must.Command("go", "test", verbose, "-timeout=30m", run, "-tags=integration", path).CollapseArgs().RunV() } func TestInitWarnings() { diff --git a/tests/integration/archive_test.go b/tests/integration/archive_test.go index d6330e89f..f3b03f5c9 100644 --- a/tests/integration/archive_test.go +++ b/tests/integration/archive_test.go @@ -19,7 +19,6 @@ import ( // Validate that archiving a bundle twice results in the same digest func TestArchive_StableDigest(t *testing.T) { t.Parallel() - p := porter.NewTestPorter(t) defer p.Close() ctx := p.SetupIntegrationTest() diff --git a/tests/integration/dependenciesv1_test.go b/tests/integration/dependenciesv1_test.go index d5065fa67..2732bf987 100644 --- a/tests/integration/dependenciesv1_test.go +++ b/tests/integration/dependenciesv1_test.go @@ -19,7 +19,6 @@ import ( func TestDependenciesLifecycle(t *testing.T) { t.Parallel() - p := porter.NewTestPorter(t) defer p.Close() ctx := p.SetupIntegrationTest() diff --git a/tests/integration/dependenciesv2_test.go b/tests/integration/dependenciesv2_test.go index 328dcb4e8..cb0ada7d2 100644 --- a/tests/integration/dependenciesv2_test.go +++ b/tests/integration/dependenciesv2_test.go @@ -18,8 +18,8 @@ import ( func TestSharedDependencies(t *testing.T) { t.Parallel() - p := porter.NewTestPorter(t) + defer p.Close() ctx := p.SetupIntegrationTest() p.Config.SetExperimentalFlags(experimental.FlagDependenciesV2) From 8bb804e0aa345f11cf106b7709c1400b6558733e Mon Sep 17 00:00:00 2001 From: schristoff <28318173+schristoff@users.noreply.github.com> Date: Mon, 5 Feb 2024 08:39:43 -0700 Subject: [PATCH 02/10] chore: Swap out integration test in release pipeline (#2981) * chore: swap out integration in release pipeline --------- Signed-off-by: Sarah Christoff <28318173+schristoff@users.noreply.github.com> --- .../build_pipelinesrelease_template.yml | 28 ++-------------- .../workflows/integ-reuseable-workflow.yml | 5 ++- .github/workflows/porter-integration.yml | 33 +++++++++++++++++-- 3 files changed, 37 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build_pipelinesrelease_template.yml b/.github/workflows/build_pipelinesrelease_template.yml index 30e9d9fbc..652050c0e 100644 --- a/.github/workflows/build_pipelinesrelease_template.yml +++ b/.github/workflows/build_pipelinesrelease_template.yml @@ -91,34 +91,12 @@ jobs: run: mage TestUnit shell: bash Validate-integration_test: - env: - GHCR_IOGETPORTER_DOCKER_REGISTRY: ${{inputs.registry}} - GHCR_IOGETPORTER_DOCKER_USERNAME: getporterbot name: Integration Test needs: - Validate-build - runs-on: ubuntu-latest - steps: - - name: checkout - uses: actions/checkout@v4.1.0 - - uses: actions/setup-go@v4 - with: - go-version: "${{ inputs.GOVERSION }}" - - name: Download Bin - uses: actions/download-artifact@v4.1.0 - with: - name: build-bin - path: bin - - name: Docker Login - uses: docker/login-action@v3.0.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Configure Agent - run: go run mage.go ConfigureAgent SetBinExecutable - - name: Integration Test - run: mage -v TestIntegration + uses: "./.github/workflows/porter-integration.yml" + with: + registry: ${{inputs.registry}} Validate-smoke_test: name: Run smoke tests on needs: diff --git a/.github/workflows/integ-reuseable-workflow.yml b/.github/workflows/integ-reuseable-workflow.yml index d78be7787..d83d92b99 100644 --- a/.github/workflows/integ-reuseable-workflow.yml +++ b/.github/workflows/integ-reuseable-workflow.yml @@ -6,6 +6,9 @@ on: test_name: type: string required: false + registry: + type: string + required: false env: GOVERSION: 1.20.7 PORTER_INTEG_FILE: ${{inputs.test_name}}.go @@ -30,7 +33,7 @@ jobs: - name: Docker Login uses: docker/login-action@v3.0.0 with: - registry: ghcr.io + registry: ${{inputs.registry}} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Configure Agent diff --git a/.github/workflows/porter-integration.yml b/.github/workflows/porter-integration.yml index 041172e44..1ee2e9dbc 100644 --- a/.github/workflows/porter-integration.yml +++ b/.github/workflows/porter-integration.yml @@ -6,6 +6,13 @@ on: paths-ignore: - 'docs/**' + workflow_call: + inputs: + registry: + type: string + required: false + default: ghcr.io + env: GOVERSION: 1.20.7 @@ -14,83 +21,103 @@ jobs: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: test_name: archive_test + registry: ${{inputs.registry}} build_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: test_name: build_test + registry: ${{inputs.registry}} cli_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: test_name: cli_test + registry: ${{inputs.registry}} connection_nix_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: test_name: connection_nix_test + registry: ${{inputs.registry}} copy_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: - test_name: copy_test + test_name: copy_tests + registry: ${{inputs.registry}} dependenciesv1_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: test_name: dependenciesv1_test + registry: ${{inputs.registry}} dependenciesv2_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: test_name: dependenciesv2_test + registry: ${{inputs.registry}} driver_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: test_name: driver_test + registry: ${{inputs.registry}} install_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: test_name: install_test + registry: ${{inputs.registry}} invoke_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: test_name: invoke_test + registry: ${{inputs.registry}} lint_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: test_name: lint_test + registry: ${{inputs.registry}} migration_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: test_name: migration_test + registry: ${{inputs.registry}} outputs_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: test_name: outputs_test + registry: ${{inputs.registry}} publish_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: test_name: publish_test + registry: ${{inputs.registry}} pull_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: test_name: pull_test + registry: ${{inputs.registry}} registry_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: test_name: registry_integration_test + registry: ${{inputs.registry}} schema_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: test_name: schema_test + registry: ${{inputs.registry}} sensitive_data_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: test_name: sensitive_data_test + registry: ${{inputs.registry}} suppress_output_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: test_name: suppress_output_test + registry: ${{inputs.registry}} telemetry_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: test_name: telemetry_test - # Reusable workflows only supports 20 jobs + registry: ${{inputs.registry}} + # Reusable workflows only supports 20 jobs uninstall_test_integ: runs-on: ubuntu-latest steps: @@ -110,7 +137,7 @@ jobs: - name: Docker Login uses: docker/login-action@v3.0.0 with: - registry: ghcr.io + registry: ${{inputs.registry}} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Configure Agent From e421a1b45906c5126f45201434dc197ea82668d8 Mon Sep 17 00:00:00 2001 From: schristoff <28318173+schristoff@users.noreply.github.com> Date: Mon, 5 Feb 2024 11:55:14 -0700 Subject: [PATCH 03/10] chore: fix typo, sep. integ. pr and release workflow (#2982) * chore: fix typo, sep. integ. pr and release workflow Signed-off-by: Sarah Christoff <28318173+schristoff@users.noreply.github.com> --------- Signed-off-by: Sarah Christoff <28318173+schristoff@users.noreply.github.com> --- .../build_pipelinesrelease_template.yml | 2 +- .../workflows/integ-reuseable-workflow.yml | 5 +- .github/workflows/porter-integration-pr.yml | 123 ++++++++++++++++++ ...ion.yml => porter-integration-release.yml} | 8 +- 4 files changed, 128 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/porter-integration-pr.yml rename .github/workflows/{porter-integration.yml => porter-integration-release.yml} (97%) diff --git a/.github/workflows/build_pipelinesrelease_template.yml b/.github/workflows/build_pipelinesrelease_template.yml index 652050c0e..8ad44d2f3 100644 --- a/.github/workflows/build_pipelinesrelease_template.yml +++ b/.github/workflows/build_pipelinesrelease_template.yml @@ -94,7 +94,7 @@ jobs: name: Integration Test needs: - Validate-build - uses: "./.github/workflows/porter-integration.yml" + uses: "./.github/workflows/porter-integration-release.yml" with: registry: ${{inputs.registry}} Validate-smoke_test: diff --git a/.github/workflows/integ-reuseable-workflow.yml b/.github/workflows/integ-reuseable-workflow.yml index d83d92b99..699e82e65 100644 --- a/.github/workflows/integ-reuseable-workflow.yml +++ b/.github/workflows/integ-reuseable-workflow.yml @@ -5,10 +5,11 @@ on: inputs: test_name: type: string - required: false + required: true registry: type: string - required: false + required: true + default: ghcr.io env: GOVERSION: 1.20.7 PORTER_INTEG_FILE: ${{inputs.test_name}}.go diff --git a/.github/workflows/porter-integration-pr.yml b/.github/workflows/porter-integration-pr.yml new file mode 100644 index 000000000..4dfe1972b --- /dev/null +++ b/.github/workflows/porter-integration-pr.yml @@ -0,0 +1,123 @@ +name: porter/porter-integration +on: + pull_request: + branches: + - main + paths-ignore: + - 'docs/**' + +env: + GOVERSION: 1.20.7 + +jobs: + archive_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: archive_test + build_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: build_test + cli_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: cli_test + connection_nix_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: connection_nix_test + copy_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: copy_test + dependenciesv1_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: dependenciesv1_test + dependenciesv2_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: dependenciesv2_test + driver_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: driver_test + install_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: install_test + invoke_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: invoke_test + lint_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: lint_test + migration_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: migration_test + outputs_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: outputs_test + publish_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: publish_test + pull_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: pull_test + registry_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: registry_integration_test + schema_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: schema_test + sensitive_data_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: sensitive_data_test + suppress_output_integration_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: suppress_output_test + telemetry_test: + uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main + with: + test_name: telemetry_test + # Reusable workflows only supports 20 jobs + uninstall_test_integ: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v4.1.0 + - uses: actions/setup-go@v4 + with: + go-version: "${{ env.GOVERSION }}" + cache: true + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + - name: Docker Login + uses: docker/login-action@v3.0.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Configure Agent + run: go run mage.go build + shell: bash + - name: Integration Test + env: + PORTER_INTEG_FILE: uninstall_test.go + run: go run mage.go -v TestIntegration + shell: bash diff --git a/.github/workflows/porter-integration.yml b/.github/workflows/porter-integration-release.yml similarity index 97% rename from .github/workflows/porter-integration.yml rename to .github/workflows/porter-integration-release.yml index 1ee2e9dbc..e6876b39c 100644 --- a/.github/workflows/porter-integration.yml +++ b/.github/workflows/porter-integration-release.yml @@ -1,11 +1,5 @@ name: porter/porter-integration on: - pull_request: - branches: - - main - paths-ignore: - - 'docs/**' - workflow_call: inputs: registry: @@ -40,7 +34,7 @@ jobs: copy_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main with: - test_name: copy_tests + test_name: copy_test registry: ${{inputs.registry}} dependenciesv1_integration_test: uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main From a54f660c66ec35703f93365ac7ea7014907fbef1 Mon Sep 17 00:00:00 2001 From: schristoff <28318173+schristoff@users.noreply.github.com> Date: Mon, 5 Feb 2024 14:13:06 -0700 Subject: [PATCH 04/10] bug: GHA needs reference to populate tag (#2983) Signed-off-by: Sarah Christoff <28318173+schristoff@users.noreply.github.com> --- .github/workflows/build_pipelinesrelease_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_pipelinesrelease_template.yml b/.github/workflows/build_pipelinesrelease_template.yml index 8ad44d2f3..c230c066b 100644 --- a/.github/workflows/build_pipelinesrelease_template.yml +++ b/.github/workflows/build_pipelinesrelease_template.yml @@ -145,11 +145,13 @@ jobs: - name: Publish Porter Binaries env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + VERSION: ${{github.ref_name}} run: mage PublishPorter PublishMixins publish-ghcr: env: DOCKER_REGISTRY: ${{inputs.registry}} DOCKER_USERNAME: getporterbot + VERSION: ${{github.ref_name}} name: Publish Docker Images needs: - Validate-build From 9b611b8b283f125cd0b01193afa593f4be92276e Mon Sep 17 00:00:00 2001 From: schristoff <28318173+schristoff@users.noreply.github.com> Date: Mon, 5 Feb 2024 15:38:03 -0700 Subject: [PATCH 05/10] bug: publishporter mage cmd git tag has {} (#2984) schristoff_removeregexmaybe Signed-off-by: Sarah Christoff <28318173+schristoff@users.noreply.github.com> --- magefile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magefile.go b/magefile.go index eebe70c24..cd921d9df 100644 --- a/magefile.go +++ b/magefile.go @@ -487,7 +487,7 @@ func PublishPorter() { // Create or update GitHub release for the permalink (canary/latest) with the version's assets (porter binaries, exec binaries and install scripts) if info.ShouldPublishPermalink() { // Move the permalink tag. The existing release automatically points to the tag. - must.RunV("git", "tag", info.Permalink, info.Version+"^{}", "-f") + must.RunV("git", "tag", info.Permalink, info.Version+"-f") must.RunV("git", "push", "-f", remote, info.Permalink) releases.AddFilesToRelease(repo, info.Permalink, porterVersionDir) From 648bd1f670677c6729472da283a20efac101b2ee Mon Sep 17 00:00:00 2001 From: schristoff <28318173+schristoff@users.noreply.github.com> Date: Mon, 5 Feb 2024 18:35:31 -0700 Subject: [PATCH 06/10] bug: git tag -f needs moved, add permissions to publish workflows (#2985) * bug: git tag -f needs moved, add permissions to publish workflows Signed-off-by: Sarah Christoff <28318173+schristoff@users.noreply.github.com> --------- Signed-off-by: Sarah Christoff <28318173+schristoff@users.noreply.github.com> --- .github/workflows/build_pipelinesrelease_template.yml | 2 ++ magefile.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_pipelinesrelease_template.yml b/.github/workflows/build_pipelinesrelease_template.yml index c230c066b..7261baf28 100644 --- a/.github/workflows/build_pipelinesrelease_template.yml +++ b/.github/workflows/build_pipelinesrelease_template.yml @@ -128,6 +128,7 @@ jobs: - Validate-integration_test - Validate-smoke_test runs-on: ubuntu-latest + permissions: write-all if: success() && inputs.shouldPublish steps: - name: checkout @@ -161,6 +162,7 @@ jobs: - Validate-integration_test - Validate-smoke_test runs-on: ubuntu-latest + permissions: write-all if: success() && inputs.shouldPublish steps: - name: checkout diff --git a/magefile.go b/magefile.go index cd921d9df..66475feca 100644 --- a/magefile.go +++ b/magefile.go @@ -487,7 +487,7 @@ func PublishPorter() { // Create or update GitHub release for the permalink (canary/latest) with the version's assets (porter binaries, exec binaries and install scripts) if info.ShouldPublishPermalink() { // Move the permalink tag. The existing release automatically points to the tag. - must.RunV("git", "tag", info.Permalink, info.Version+"-f") + must.RunV("git", "tag", "-f", info.Permalink, info.Version) must.RunV("git", "push", "-f", remote, info.Permalink) releases.AddFilesToRelease(repo, info.Permalink, porterVersionDir) From 07ad7bb25d2e9094d56aed157a696f829270a86c Mon Sep 17 00:00:00 2001 From: schristoff <28318173+schristoff@users.noreply.github.com> Date: Mon, 5 Feb 2024 20:37:11 -0700 Subject: [PATCH 07/10] chore: update getporter/magefiles (#2986) Signed-off-by: Sarah Christoff <28318173+schristoff@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index e44ab6aa8..16e171dea 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ replace ( ) require ( - get.porter.sh/magefiles v0.6.1-0.20230822194230-77c3a69ead68 + get.porter.sh/magefiles v0.6.1 github.com/Masterminds/semver/v3 v3.2.1 github.com/PaesslerAG/jsonpath v0.1.1 github.com/carolynvs/aferox v0.3.0 @@ -89,7 +89,7 @@ require ( github.com/PaesslerAG/gval v1.2.2 // indirect github.com/PuerkitoBio/goquery v1.8.1 // indirect github.com/Shopify/logrus-bugsnag v0.0.0-20230117174420-439a4b8ba167 // indirect - github.com/andybalholm/brotli v1.0.5 // indirect + github.com/andybalholm/brotli v1.1.0 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/aws/aws-sdk-go-v2 v1.18.1 // indirect github.com/aws/aws-sdk-go-v2/config v1.18.27 // indirect @@ -152,7 +152,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/compress v1.17.6 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/pty v1.1.5 // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -185,7 +185,7 @@ require ( github.com/opencontainers/runc v1.1.12 // indirect github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/osteele/tuesday v1.0.3 // indirect - github.com/pierrec/lz4/v4 v4.1.18 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect diff --git a/go.sum b/go.sum index f2ded2730..c99fd0a01 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= get.porter.sh/magefiles v0.6.1-0.20230822194230-77c3a69ead68 h1:DP6jiRuZC6iLFvriQdQBAnV8Iav/vb64Q123dkIEunw= get.porter.sh/magefiles v0.6.1-0.20230822194230-77c3a69ead68/go.mod h1:YsSlQWtGoXCGC4pdD7NxPpvh/FryM1bM0wzMWlJC+Bg= +get.porter.sh/magefiles v0.6.1 h1:JuGYK9us7Y9GJTr172wK3Y8W5rOz9CBnheuZa8yJjGc= +get.porter.sh/magefiles v0.6.1/go.mod h1:YsSlQWtGoXCGC4pdD7NxPpvh/FryM1bM0wzMWlJC+Bg= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 h1:59MxjQVfjXsBpLy+dbd2/ELV5ofnUkUZBvWSC85sheA= @@ -91,6 +93,8 @@ github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 h1:aM1 github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= @@ -536,6 +540,8 @@ github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= +github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= @@ -733,6 +739,8 @@ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= From 35d799701d0f8a87a83ec1e014ec4a2da8eea24b Mon Sep 17 00:00:00 2001 From: Mani Bindra Date: Wed, 7 Feb 2024 10:22:26 +0530 Subject: [PATCH 08/10] Add feature to mount host volumes (#2949) * Add mount host volume action argument - Fixes #475 - Add option to mount host volume --------- Signed-off-by: Maninderjit Bindra --- cmd/porter/installations.go | 1 + pkg/cnab/provider/action.go | 9 ++ pkg/cnab/provider/docker_linux.go | 11 ++ pkg/cnab/provider/driver.go | 65 +++++++++-- pkg/cnab/provider/driver_darwin.go | 11 ++ pkg/cnab/provider/driver_test.go | 173 ++++++++++++++++++++++++++++ pkg/cnab/provider/driver_windows.go | 11 ++ pkg/porter/lifecycle.go | 49 ++++++++ pkg/porter/lifecycle_test.go | 133 +++++++++++++++++++++ 9 files changed, 451 insertions(+), 12 deletions(-) diff --git a/cmd/porter/installations.go b/cmd/porter/installations.go index 2d9daedc2..94105b77b 100644 --- a/cmd/porter/installations.go +++ b/cmd/porter/installations.go @@ -418,6 +418,7 @@ func addBundleActionFlags(f *pflag.FlagSet, actionOpts porter.BundleAction) { addBundlePullFlags(f, &opts.BundlePullOptions) f.BoolVar(&opts.AllowDockerHostAccess, "allow-docker-host-access", false, "Controls if the bundle should have access to the host's Docker daemon with elevated privileges. See https://getporter.org/configuration/#allow-docker-host-access for the full implications of this flag.") + f.StringArrayVar(&opts.HostVolumeMounts, "mount-host-volume", nil, "Mount a host volume into the bundle. Format is :[: