From 75e5e62f311a7c5d98ed4a68ff236a3dba5e2b37 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 16 Apr 2024 13:13:04 -0700 Subject: [PATCH 1/9] Buildx bake with goss test runner orchestration (#718) * Initial commit of docker buildx bake experiment * Separate builds for azure/google specialized images * Specify targets in bake * Implement test stages on base * Implement stage tests for base * Implement stage tests for ppm * Implement stage tests for connect (borked) * Update connect just test to work with test stage * Implement test stage on r-session-complete * Fix/exclude goss tests for buildx bake * Implement separate test for Connect * Remove compose testing and image local justfiles * Increase timeout for checking Jupyter * Convert content builds to use bake * Target content docker-bake file * Print metadata after build * Enable GHA caching * Default to image output * Remove debug line * Suppress DL3006 * Extend timeouts for GCW * Add timeout to r-session-complete jupyter * Cleanup justfiles * Update tagging for bake release builds * Implement bake preview builds * Update if clause for pushing preview images * Install requests package * Provision buildx in preview * Set file on preview builds * Fix target names * Fix CentOS 7 image bad ref * Calculate RSW download URL in bake * Wait for launcher startup in Workbench tests * Add wait time in Workbench test runner to ensure services are up * Do not cache or output tests Use a standard runner for content images * Do not cache or output tests for preview Provide a buildx config for GHA Unify test sleep behavior of Workbench images * Show running processes (debugging) * Remove test targets * Remove test layers from Dockerfiles * Change test running to be orchestrated by a Python script * Order tags narrowest to broadest * Update workflow patterns * Update preview builds for new test script * Output failed targets * Rename jobs and add authentication for registries * Merge build.justfile into Justfile * README updates * Update README.md for bake changes Change `build` targets to aliases Add a `run` target that leverages docker compose Remove duplicated targets from bake definitions * Merge content bake definition into main bake definition * Change CI build target name * Unify functions between bake files * Test separating image lines * Add missing checkout command * Abandon reusing buildx builder instance * Remove bad env.target refs * Declare bash as shell * Implement reuse of base images across jobs * Remove builder references * Switch print for logging * Swap runners based on image load * Add shim for mounting DinD and deps for WGCW Add entrypoint setting to tests * Use `--init` flag on tests * Remove unused base targets Fix tests on WAML and improve logging * Turn down compression level * Update upload artifacts action * Update preview workflow to match release workflow Clean up dockerfiles * Add missing just install * Update setup-just * Adapt action to work with Previews * Move to fully parallel builds * Use bake-test-push action for previews * Improve test filtering * Remove preview base dependency * Second test filtering fix * Alias build to bake * Fix run command * Update README with test instructions --------- Co-authored-by: Chris Ostrouchov --- .github/actions/bake-test-push/action.yml | 119 ++++ .github/workflows/build-bake-preview.yaml | 357 ++++++++++++ .github/workflows/build-bake.yaml | 290 ++++++++++ .github/workflows/build-content.yaml | 215 ------- .github/workflows/build-manual.yaml | 2 +- .github/workflows/build-prerelease.yaml | 144 ----- .github/workflows/build-release.yaml | 360 ------------ .github/workflows/build-workbench-aml.yaml | 127 ----- .github/workflows/build-workbench-gcw.yaml | 99 ---- .gitignore | 2 + Justfile | 157 +++++- NEWS.md | 4 + README.md | 76 ++- ci.Justfile | 66 --- connect-content-init/Dockerfile.ubuntu2204 | 3 +- connect-content-init/Justfile | 50 -- connect-content-init/docker-compose.test.yml | 13 - connect-content-init/test/run_tests.sh | 18 +- connect/Dockerfile.ubuntu2204 | 10 +- connect/Justfile | 90 --- connect/docker-compose.test.yml | 20 - connect/test/goss.yaml | 4 +- connect/test/run_tests.sh | 21 +- content/README.md | 11 +- content/base/Dockerfile.ubuntu1804 | 2 - content/base/Dockerfile.ubuntu2204 | 2 - content/base/Justfile | 47 -- content/base/NEWS.md | 4 + content/base/README.md | 9 - content/build-images.sh | 48 -- content/docker-bake.hcl | 82 +++ content/matrix.json | 23 - content/pro/Dockerfile.ubuntu1804 | 7 +- content/pro/Dockerfile.ubuntu2204 | 7 +- content/pro/Justfile | 48 -- content/pro/NEWS.md | 4 + content/pro/README.md | 9 - docker-bake.hcl | 524 ++++++++++++++++++ docker-bake.preview.hcl | 417 ++++++++++++++ docker-compose.yml | 46 +- hadolint.yaml | 2 + package-manager/Dockerfile.ubuntu2204 | 9 +- package-manager/Justfile | 98 ---- package-manager/docker-compose.test.yml | 19 - package-manager/test/run_tests.sh | 20 +- product/base/Dockerfile.centos7 | 3 +- product/base/Dockerfile.ubuntu2204 | 3 +- product/base/Justfile | 69 --- product/base/docker-compose.test.yml | 20 - product/base/test/run_tests.sh | 17 +- product/pro/Dockerfile.centos7 | 9 +- product/pro/Dockerfile.ubuntu2204 | 9 +- product/pro/Justfile | 79 --- product/pro/docker-compose.test.yml | 22 - product/pro/test/goss.yaml | 7 +- product/pro/test/run_tests.sh | 17 +- r-session-complete/Dockerfile.centos7 | 9 +- r-session-complete/Dockerfile.ubuntu2204 | 9 +- r-session-complete/Justfile | 87 --- r-session-complete/test/goss.yaml | 2 + r-session-complete/test/run_tests.sh | 18 +- share/buildkitd.toml | 1 + share/local_buildkitd.toml | 4 + tools/export_bake_artifacts.py | 69 +++ tools/import_bake_artifacts.py | 35 ++ tools/test_bake_artifacts.py | 114 ++++ .../Dockerfile.ubuntu2004 | 5 +- .../Justfile | 116 ---- .../docker-compose.test.yml | 24 - .../test/goss.yaml | 37 +- .../test/run_tests.sh | 24 +- .../Dockerfile.ubuntu2204 | 17 +- workbench-for-microsoft-azure-ml/Justfile | 78 --- .../docker-compose.test.yml | 20 - .../test/goss.yaml | 1 + .../test/goss_vars.yaml | 1 - .../test/run_tests.sh | 26 +- workbench/Dockerfile.ubuntu2204 | 11 +- workbench/Justfile | 100 ---- workbench/docker-compose.test.yml | 20 - workbench/test/goss.yaml | 1 + workbench/test/run_tests.sh | 23 +- 82 files changed, 2387 insertions(+), 2405 deletions(-) create mode 100644 .github/actions/bake-test-push/action.yml create mode 100644 .github/workflows/build-bake-preview.yaml create mode 100644 .github/workflows/build-bake.yaml delete mode 100644 .github/workflows/build-content.yaml delete mode 100644 .github/workflows/build-prerelease.yaml delete mode 100644 .github/workflows/build-release.yaml delete mode 100644 .github/workflows/build-workbench-aml.yaml delete mode 100644 .github/workflows/build-workbench-gcw.yaml delete mode 100644 connect-content-init/Justfile delete mode 100644 connect-content-init/docker-compose.test.yml delete mode 100644 connect/Justfile delete mode 100644 connect/docker-compose.test.yml delete mode 100755 content/base/Justfile delete mode 100644 content/base/README.md delete mode 100755 content/build-images.sh create mode 100644 content/docker-bake.hcl delete mode 100644 content/matrix.json delete mode 100755 content/pro/Justfile delete mode 100644 content/pro/README.md create mode 100644 docker-bake.hcl create mode 100644 docker-bake.preview.hcl delete mode 100644 package-manager/Justfile delete mode 100644 package-manager/docker-compose.test.yml delete mode 100755 product/base/Justfile delete mode 100644 product/base/docker-compose.test.yml delete mode 100644 product/pro/Justfile delete mode 100644 product/pro/docker-compose.test.yml delete mode 100755 r-session-complete/Justfile create mode 100644 share/buildkitd.toml create mode 100644 share/local_buildkitd.toml create mode 100644 tools/export_bake_artifacts.py create mode 100644 tools/import_bake_artifacts.py create mode 100644 tools/test_bake_artifacts.py delete mode 100644 workbench-for-google-cloud-workstations/Justfile delete mode 100644 workbench-for-google-cloud-workstations/docker-compose.test.yml delete mode 100644 workbench-for-microsoft-azure-ml/Justfile delete mode 100644 workbench-for-microsoft-azure-ml/docker-compose.test.yml delete mode 100644 workbench/Justfile delete mode 100644 workbench/docker-compose.test.yml diff --git a/.github/actions/bake-test-push/action.yml b/.github/actions/bake-test-push/action.yml new file mode 100644 index 00000000..066e0bc9 --- /dev/null +++ b/.github/actions/bake-test-push/action.yml @@ -0,0 +1,119 @@ +name: 'Build/Test/Scan/Push Image' +inputs: + target: + description: Target or group to bake + required: true + type: string + bakefile: + description: Path to the bakefile + required: true + default: "docker-bake.hcl" + type: string + env-vars: + description: A semicolon delimited list of environment variables to set as overrides for the bake definition + required: false + type: string + test-image: + description: Flag to test image once built + default: true + type: boolean + push-image: + description: Flag to push image once built + default: false + type: boolean + ghcr-token: + description: Username for authentication with GHCR.io + required: true + type: string + dockerhub-username: + description: Username for authentication with DockerHub + required: true + type: string + dockerhub-token: + description: Username for authentication with DockerHub + required: true + type: string + gcp-json: + description: JSON for authenticating Google Cloud Platform + default: "" + type: string + +runs: + using: "composite" + steps: + # Setup dependencies and tools + - name: Set up Just + uses: extractions/setup-just@v2 + env: + GITHUB_TOKEN: ${{ inputs.ghcr-token }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + # Authenticate to registries + - name: Login to ghcr.io + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ inputs.ghcr-token }} + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ inputs.dockerhub-username }} + password: ${{ inputs.dockerhub-token }} + + - name: Login to GCAR us-central1 + continue-on-error: true + uses: docker/login-action@v3 + with: + registry: us-central1-docker.pkg.dev + username: _json_key + password: '${{ inputs.gcp-json }}' + + - name: Login to GCAR us + continue-on-error: true + uses: docker/login-action@v3 + with: + registry: us-docker.pkg.dev + username: _json_key + password: '${{ inputs.gcp-json }}' + + - name: Login to GCAR asia + continue-on-error: true + uses: docker/login-action@v3 + with: + registry: asia-docker.pkg.dev + username: _json_key + password: '${{ inputs.gcp-json }}' + + - name: Login to GCAR europe + continue-on-error: true + uses: docker/login-action@v3 + with: + registry: europe-docker.pkg.dev + username: _json_key + password: '${{ inputs.gcp-json }}' + + # Build, test, and push image + - name: Build + id: build + uses: docker/bake-action@v4 + with: + targets: "${{ inputs.target }}" + push: false + files: ${{ inputs.bakefile }} + + - name: Test + shell: bash + run: | + just test "${{ inputs.target }}" "${{ inputs.bakefile }}" + + - name: Push - ${{ inputs.push-image }} + uses: docker/bake-action@v4 + with: + targets: "${{ inputs.target }}" + push: ${{ inputs.push-image }} + files: ${{ inputs.bakefile }} diff --git a/.github/workflows/build-bake-preview.yaml b/.github/workflows/build-bake-preview.yaml new file mode 100644 index 00000000..fce96eff --- /dev/null +++ b/.github/workflows/build-bake-preview.yaml @@ -0,0 +1,357 @@ +on: + schedule: + - cron: '0 8 * * *' + - cron: '0 9 * * *' + push: + branches: + - main + - dev + - dev-rspm + pull_request: + +name: Preview - Build, Test, and Push +jobs: + versions: + name: Fetch product daily/preview versions + runs-on: ubuntu-latest + + concurrency: + group: fetch-versions-${{ github.ref }} + cancel-in-progress: true + + outputs: + WORKBENCH_DAILY_VERSION: ${{ steps.get-version.outputs.WORKBENCH_DAILY_VERSION }} + WORKBENCH_PREVIEW_VERSION: ${{ steps.get-version.outputs.WORKBENCH_PREVIEW_VERSION }} + PACKAGE_MANAGER_DAILY_VERSION: ${{ steps.get-version.outputs.PACKAGE_MANAGER_DAILY_VERSION }} + CONNECT_DAILY_VERSION: ${{ steps.get-version.outputs.CONNECT_DAILY_VERSION }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Just + uses: extractions/setup-just@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install Python dependencies + run: | + pip install requests + + - name: Get Version + id: get-version + run: | + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) + echo "WORKBENCH_DAILY_VERSION=$WORKBENCH_DAILY_VERSION" >> $GITHUB_OUTPUT + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) + echo "WORKBENCH_PREVIEW_VERSION=$WORKBENCH_PREVIEW_VERSION" >> $GITHUB_OUTPUT + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) + echo "PACKAGE_MANAGER_DAILY_VERSION=$PACKAGE_MANAGER_DAILY_VERSION" >> $GITHUB_OUTPUT + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) + echo "CONNECT_DAILY_VERSION=$CONNECT_DAILY_VERSION" >> $GITHUB_OUTPUT + + base: + needs: [versions] + name: Base Image Previews + runs-on: ubuntu-latest-4x + + env: + target: "base-dev-images" + WORKBENCH_DAILY_VERSION: ${{ needs.versions.outputs.WORKBENCH_DAILY_VERSION }} + WORKBENCH_PREVIEW_VERSION: ${{ needs.versions.outputs.WORKBENCH_PREVIEW_VERSION }} + PACKAGE_MANAGER_DAILY_VERSION: ${{ needs.versions.outputs.PACKAGE_MANAGER_DAILY_VERSION }} + CONNECT_DAILY_VERSION: ${{ needs.versions.outputs.CONNECT_DAILY_VERSION }} + BRANCH: ${{ github.head_ref || github.ref_name }} + + concurrency: + group: bake-base-preview-${{ github.ref }} + cancel-in-progress: true + + steps: + - name: Check Out main Branch + if: github.event.schedule == '0 8 * * *' + uses: actions/checkout@v3 + with: + ref: 'main' + + - name: Check Out Repo at Triggered Branch + if: github.event.schedule != '0 8 * * *' + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build, Test, and Push + uses: ./.github/actions/bake-test-push + with: + target: ${{ env.target }} + bakefile: docker-bake.preview.hcl + push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} + ghcr-token: ${{ secrets.GITHUB_TOKEN }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + connect-daily: + needs: [versions] + name: Connect Image - Daily + runs-on: ubuntu-latest-4x + + env: + target: "connect-daily" + WORKBENCH_DAILY_VERSION: ${{ needs.versions.outputs.WORKBENCH_DAILY_VERSION }} + WORKBENCH_PREVIEW_VERSION: ${{ needs.versions.outputs.WORKBENCH_PREVIEW_VERSION }} + PACKAGE_MANAGER_DAILY_VERSION: ${{ needs.versions.outputs.PACKAGE_MANAGER_DAILY_VERSION }} + CONNECT_DAILY_VERSION: ${{ needs.versions.outputs.CONNECT_DAILY_VERSION }} + BRANCH: ${{ github.head_ref || github.ref_name }} + + concurrency: + group: bake-connect-daily-${{ github.ref }} + cancel-in-progress: true + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build, Test, and Push + uses: ./.github/actions/bake-test-push + with: + target: ${{ env.target }} + bakefile: docker-bake.preview.hcl + push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} + ghcr-token: ${{ secrets.GITHUB_TOKEN }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + connect-content-init-daily: + needs: [versions] + name: Connect Content Init - Daily + runs-on: ubuntu-latest-4x + + env: + target: "connect-content-init-daily" + WORKBENCH_DAILY_VERSION: ${{ needs.versions.outputs.WORKBENCH_DAILY_VERSION }} + WORKBENCH_PREVIEW_VERSION: ${{ needs.versions.outputs.WORKBENCH_PREVIEW_VERSION }} + PACKAGE_MANAGER_DAILY_VERSION: ${{ needs.versions.outputs.PACKAGE_MANAGER_DAILY_VERSION }} + CONNECT_DAILY_VERSION: ${{ needs.versions.outputs.CONNECT_DAILY_VERSION }} + BRANCH: ${{ github.head_ref || github.ref_name }} + + concurrency: + group: bake-connect-content-init-daily-${{ github.ref }} + cancel-in-progress: true + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build, Test, and Push + uses: ./.github/actions/bake-test-push + with: + target: ${{ env.target }} + bakefile: docker-bake.preview.hcl + push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} + ghcr-token: ${{ secrets.GITHUB_TOKEN }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + package-manager-daily: + needs: [versions] + name: Package Manager - Daily + runs-on: ubuntu-latest-4x + + env: + target: "package-manager-daily" + WORKBENCH_DAILY_VERSION: ${{ needs.versions.outputs.WORKBENCH_DAILY_VERSION }} + WORKBENCH_PREVIEW_VERSION: ${{ needs.versions.outputs.WORKBENCH_PREVIEW_VERSION }} + PACKAGE_MANAGER_DAILY_VERSION: ${{ needs.versions.outputs.PACKAGE_MANAGER_DAILY_VERSION }} + CONNECT_DAILY_VERSION: ${{ needs.versions.outputs.CONNECT_DAILY_VERSION }} + BRANCH: ${{ github.head_ref || github.ref_name }} + + concurrency: + group: bake-package-manager-daily-${{ github.ref }} + cancel-in-progress: true + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build, Test, and Push + uses: ./.github/actions/bake-test-push + with: + target: ${{ env.target }} + bakefile: docker-bake.preview.hcl + push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} + ghcr-token: ${{ secrets.GITHUB_TOKEN }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + r-session-complete-preview: + needs: [versions] + name: R Session Complete - Preview + runs-on: ubuntu-latest-4x + + env: + target: "r-session-complete-preview" + WORKBENCH_DAILY_VERSION: ${{ needs.versions.outputs.WORKBENCH_DAILY_VERSION }} + WORKBENCH_PREVIEW_VERSION: ${{ needs.versions.outputs.WORKBENCH_PREVIEW_VERSION }} + PACKAGE_MANAGER_DAILY_VERSION: ${{ needs.versions.outputs.PACKAGE_MANAGER_DAILY_VERSION }} + CONNECT_DAILY_VERSION: ${{ needs.versions.outputs.CONNECT_DAILY_VERSION }} + BRANCH: ${{ github.head_ref || github.ref_name }} + + concurrency: + group: bake-r-session-complete-preview-${{ github.ref }} + cancel-in-progress: true + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build, Test, and Push + uses: ./.github/actions/bake-test-push + with: + target: ${{ env.target }} + bakefile: docker-bake.preview.hcl + push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} + ghcr-token: ${{ secrets.GITHUB_TOKEN }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + r-session-complete-daily: + needs: [versions] + name: R Session Complete - Daily + runs-on: ubuntu-latest-4x + + env: + target: "r-session-complete-daily" + WORKBENCH_DAILY_VERSION: ${{ needs.versions.outputs.WORKBENCH_DAILY_VERSION }} + WORKBENCH_PREVIEW_VERSION: ${{ needs.versions.outputs.WORKBENCH_PREVIEW_VERSION }} + PACKAGE_MANAGER_DAILY_VERSION: ${{ needs.versions.outputs.PACKAGE_MANAGER_DAILY_VERSION }} + CONNECT_DAILY_VERSION: ${{ needs.versions.outputs.CONNECT_DAILY_VERSION }} + BRANCH: ${{ github.head_ref || github.ref_name }} + + concurrency: + group: bake-r-session-complete-daily-${{ github.ref }} + cancel-in-progress: true + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build, Test, and Push + uses: ./.github/actions/bake-test-push + with: + target: ${{ env.target }} + bakefile: docker-bake.preview.hcl + push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} + ghcr-token: ${{ secrets.GITHUB_TOKEN }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + workbench-preview: + needs: [versions] + name: Workbench - Preview + runs-on: ubuntu-latest-4x + + env: + target: "workbench-preview" + WORKBENCH_DAILY_VERSION: ${{ needs.versions.outputs.WORKBENCH_DAILY_VERSION }} + WORKBENCH_PREVIEW_VERSION: ${{ needs.versions.outputs.WORKBENCH_PREVIEW_VERSION }} + PACKAGE_MANAGER_DAILY_VERSION: ${{ needs.versions.outputs.PACKAGE_MANAGER_DAILY_VERSION }} + CONNECT_DAILY_VERSION: ${{ needs.versions.outputs.CONNECT_DAILY_VERSION }} + BRANCH: ${{ github.head_ref || github.ref_name }} + + concurrency: + group: bake-workbench-preview-${{ github.ref }} + cancel-in-progress: true + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build, Test, and Push + uses: ./.github/actions/bake-test-push + with: + target: ${{ env.target }} + bakefile: docker-bake.preview.hcl + push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} + ghcr-token: ${{ secrets.GITHUB_TOKEN }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + workbench-daily: + needs: [versions] + name: Workbench - Daily + runs-on: ubuntu-latest-4x + + env: + target: "workbench-daily" + WORKBENCH_DAILY_VERSION: ${{ needs.versions.outputs.WORKBENCH_DAILY_VERSION }} + WORKBENCH_PREVIEW_VERSION: ${{ needs.versions.outputs.WORKBENCH_PREVIEW_VERSION }} + PACKAGE_MANAGER_DAILY_VERSION: ${{ needs.versions.outputs.PACKAGE_MANAGER_DAILY_VERSION }} + CONNECT_DAILY_VERSION: ${{ needs.versions.outputs.CONNECT_DAILY_VERSION }} + BRANCH: ${{ github.head_ref || github.ref_name }} + + concurrency: + group: bake-workbench-daily-${{ github.ref }} + cancel-in-progress: true + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build, Test, and Push + uses: ./.github/actions/bake-test-push + with: + target: ${{ env.target }} + bakefile: docker-bake.preview.hcl + push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} + ghcr-token: ${{ secrets.GITHUB_TOKEN }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml new file mode 100644 index 00000000..734fba76 --- /dev/null +++ b/.github/workflows/build-bake.yaml @@ -0,0 +1,290 @@ +on: + schedule: + - cron: '0 8 * * 1' # Every Monday at 8:00 UTC for automatic rebuilds (pull in OS updates, security patches, etc.) + push: + branches: + - main + - dev + pull_request: + +name: Release - Build, Test, and Push +jobs: + + base: + name: Base Images + runs-on: ubuntu-latest-8x + + concurrency: + group: bake-base-${{ github.ref }} + cancel-in-progress: true + + env: + target: base-images + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build, Test, and Push + uses: ./.github/actions/bake-test-push + with: + target: ${{ env.target }} + push-image: ${{ github.ref == 'refs/heads/main' }} + ghcr-token: ${{ secrets.GITHUB_TOKEN }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + + connect: + name: Connect + runs-on: ubuntu-latest-4x + + concurrency: + group: bake-connect-${{ github.ref }} + cancel-in-progress: true + + env: + target: connect + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build, Test, and Push + uses: ./.github/actions/bake-test-push + with: + target: ${{ env.target }} + push-image: ${{ github.ref == 'refs/heads/main' }} + ghcr-token: ${{ secrets.GITHUB_TOKEN }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + + connect-content-init: + name: Connect Content Init + runs-on: ubuntu-latest-4x + + concurrency: + group: bake-connect-content-init-${{ github.ref }} + cancel-in-progress: true + + env: + target: connect-content-init + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build, Test, and Push + uses: ./.github/actions/bake-test-push + with: + target: ${{ env.target }} + push-image: ${{ github.ref == 'refs/heads/main' }} + ghcr-token: ${{ secrets.GITHUB_TOKEN }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + + content: + name: Content Images + runs-on: ubuntu-latest-8x + + concurrency: + group: bake-content-${{ github.ref }} + cancel-in-progress: true + + env: + target: content-images + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build, Test, and Push + uses: ./.github/actions/bake-test-push + with: + target: ${{ env.target }} + push-image: ${{ github.ref == 'refs/heads/main' }} + ghcr-token: ${{ secrets.GITHUB_TOKEN }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + + package-manager: + name: Package Manager + runs-on: ubuntu-latest-4x + + concurrency: + group: bake-package-manager-${{ github.ref }} + cancel-in-progress: true + + env: + target: package-manager + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build, Test, and Push + uses: ./.github/actions/bake-test-push + with: + target: ${{ env.target }} + push-image: ${{ github.ref == 'refs/heads/main' }} + ghcr-token: ${{ secrets.GITHUB_TOKEN }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + + r-session-complete: + name: R Session Complete + runs-on: ubuntu-latest-8x + + concurrency: + group: bake-r-session-complete-${{ github.ref }} + cancel-in-progress: true + + env: + target: r-session-complete + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build, Test, and Push + uses: ./.github/actions/bake-test-push + with: + target: ${{ env.target }} + push-image: ${{ github.ref == 'refs/heads/main' }} + ghcr-token: ${{ secrets.GITHUB_TOKEN }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + + workbench: + name: Workbench + runs-on: ubuntu-latest-4x + + concurrency: + group: bake-workbench-${{ github.ref }} + cancel-in-progress: true + + env: + target: workbench + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build, Test, and Push + uses: ./.github/actions/bake-test-push + with: + target: ${{ env.target }} + push-image: ${{ github.ref == 'refs/heads/main' }} + ghcr-token: ${{ secrets.GITHUB_TOKEN }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + + workbench-for-google-cloud-workstations: + name: Workbench for Google Cloud Workstations + runs-on: ubuntu-latest-8x + + concurrency: + group: bake-wgcw-${{ github.ref }} + cancel-in-progress: true + + env: + target: workbench-for-google-cloud-workstations + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build, Test, and Push + uses: ./.github/actions/bake-test-push + with: + target: ${{ env.target }} + push-image: ${{ github.ref == 'refs/heads/main' }} + ghcr-token: ${{ secrets.GITHUB_TOKEN }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + + workbench-for-microsoft-azure-ml: + name: Workbench for Microsoft Azure ML + runs-on: ubuntu-latest-8x + + concurrency: + group: bake-waml-${{ github.ref }} + cancel-in-progress: true + + env: + target: waml-images + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build, Test, and Push + uses: ./.github/actions/bake-test-push + with: + target: ${{ env.target }} + push-image: ${{ github.ref == 'refs/heads/main' }} + ghcr-token: ${{ secrets.GITHUB_TOKEN }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' diff --git a/.github/workflows/build-content.yaml b/.github/workflows/build-content.yaml deleted file mode 100644 index 4b7db4e1..00000000 --- a/.github/workflows/build-content.yaml +++ /dev/null @@ -1,215 +0,0 @@ -on: - push: - branches: - - main - - dev - pull_request: - -name: Content Images - Build, Test, Scan, and Push -jobs: - matrix: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@v3 - - id: set-matrix - run: | - MATRIX=$(jq -Mcr < content/matrix.json) - echo "matrix=$MATRIX" >> $GITHUB_OUTPUT - - build: - runs-on: ubuntu-latest - needs: matrix - name: content-base-${{ matrix.config.os }}-r${{ matrix.config.r }}-py${{ matrix.config.py }}--${{ github.ref }} - - permissions: - contents: read - packages: write - - concurrency: - group: content-base-${{ matrix.config.os }}-r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ github.ref }} - cancel-in-progress: true - - strategy: - fail-fast: false - matrix: - config: ${{ fromJson(needs.matrix.outputs.matrix) }} - - steps: - - name: Check Out Repo - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - get-content-args \ - ${{ matrix.config.r }} \ - ${{ matrix.config.py }} \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - get-content-tags \ - content-base \ - ${{ matrix.config.r }} \ - ${{ matrix.config.py }} \ - ${{ matrix.config.os }} \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push content base image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./content/base - os: ${{ matrix.config.os }} - product: content-base - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - test-image: false - push-image: ${{ github.ref == 'refs/heads/main' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push base pro image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./content/base - os: ${{ matrix.config.os }} - product: content-base - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - test-image: false - push-image: ${{ github.ref == 'refs/heads/main' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic - - build-pro: - runs-on: ubuntu-latest - needs: [ matrix, build ] - name: content-pro-${{ matrix.config.os }}-r${{ matrix.config.r }}-py${{ matrix.config.py }}--${{ github.ref }} - concurrency: - group: content-pro-${{ matrix.config.os }}-r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ github.ref }} - cancel-in-progress: true - - strategy: - fail-fast: false - matrix: - config: ${{ fromJson(needs.matrix.outputs.matrix) }} - - steps: - - name: Check Out Repo - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - get-content-args \ - ${{ matrix.config.r }} \ - ${{ matrix.config.py }} \ - ${{ matrix.config.drivers }} \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - get-content-tags \ - content-pro \ - ${{ matrix.config.r }} \ - ${{ matrix.config.py }} \ - ${{ matrix.config.os }} \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push content pro image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./content/pro - os: ${{ matrix.config.os }} - product: content-pro - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - test-image: false - push-image: ${{ github.ref == 'refs/heads/main' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push base pro image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./content/pro - os: ${{ matrix.config.os }} - product: content-pro - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - test-image: false - push-image: ${{ github.ref == 'refs/heads/main' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic diff --git a/.github/workflows/build-manual.yaml b/.github/workflows/build-manual.yaml index 2f1137a0..89cd4429 100644 --- a/.github/workflows/build-manual.yaml +++ b/.github/workflows/build-manual.yaml @@ -75,7 +75,7 @@ jobs: ref: ${{ inputs.branch }} - name: Set up Just - uses: extractions/setup-just@v1 + uses: extractions/setup-just@v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-prerelease.yaml b/.github/workflows/build-prerelease.yaml deleted file mode 100644 index 470470bb..00000000 --- a/.github/workflows/build-prerelease.yaml +++ /dev/null @@ -1,144 +0,0 @@ -on: - schedule: - # every morning at 8am UTC - # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onschedule - # make sure to change the `if` commands below when changing the schedule - - cron: '0 8 * * *' - - cron: '0 9 * * *' - push: - branches: - - main - - dev - - dev-rspm - pull_request: - -name: Prerelease - Build, Test, Scan, and Push -jobs: - build: - runs-on: ubuntu-latest - name: build-${{ matrix.config.type }}-${{ matrix.config.product }}-${{ matrix.config.os }} - - permissions: - contents: read - packages: write - - strategy: - fail-fast: false - matrix: - config: - - {product: "workbench", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: "workbench", type: "preview", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: "connect", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: "connect-content-init", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: "package-manager", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.12.1", py-alternate: "3.11.7"} - - {product: "r-session-complete", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: "r-session-complete", type: "daily", os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {product: "r-session-complete", type: "preview", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: "r-session-complete", type: "preview", os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - concurrency: - group: prerelease-build-${{ matrix.config.type }}-${{ matrix.config.product }}-${{ matrix.config.os }}-${{ github.ref }} - cancel-in-progress: true - - steps: - - name: Check Out main Branch - if: github.event.schedule == '0 8 * * *' - uses: actions/checkout@v3 - with: - ref: 'main' - - - name: Check Out Repo at Triggered Branch - if: github.event.schedule != '0 8 * * *' - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Version - id: get-version - run: | - VERSION=$(just -f ci.Justfile get-version ${{ matrix.config.product }} --type=${{ matrix.config.type }} --local) - echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-prerelease-args \ - ${{ matrix.config.type }} \ - ${{ matrix.config.product }} \ - ${{ matrix.config.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-prerelease-tags \ - ${{ matrix.config.type }} \ - ${{ matrix.config.product }} \ - ${{ matrix.config.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push base pro image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./${{ matrix.config.product }} - os: ${{ matrix.config.os }} - product: ${{ matrix.config.product }} - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push base pro image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./${{ matrix.config.product }} - os: ${{ matrix.config.os }} - product: product-base - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml deleted file mode 100644 index 72eb2cc5..00000000 --- a/.github/workflows/build-release.yaml +++ /dev/null @@ -1,360 +0,0 @@ -on: - schedule: - - cron: '0 12 * * 1' # If updating this value, be sure to update logic for all `push-image` arguments! - push: - branches: - - main - - dev - pull_request: - -name: Release - Build, Test, Scan, and Push -jobs: - build-base: - runs-on: ubuntu-latest - name: product-base-build-${{ matrix.config.os }}-r${{ matrix.config.r-primary }}_${{ matrix.config.r-alternate }}-py${{ matrix.config.py-primary }}_${{ matrix.config.py-alternate }} - - permissions: - contents: read - packages: write - - concurrency: - group: base-build-${{ matrix.config.os }}-r${{ matrix.config.r-primary }}_${{ matrix.config.r-alternate }}-py${{ matrix.config.py-primary }}_${{ matrix.config.py-alternate }}-${{ github.ref }} - cancel-in-progress: true - - strategy: - fail-fast: false - matrix: - config: - - {os: 'centos7', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {os: 'ubuntu2204', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.12.1", py-alternate: "3.11.7"} - - steps: - - name: Check Out Repo - cron main - if: github.event.schedule == '0 12 * * 1' - uses: actions/checkout@v3 - with: - ref: main - - - name: Check Out Repo - if: github.event.schedule != '0 12 * * 1' - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-base-args ${{ matrix.config.os }} product-base \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-base-tags ${{ matrix.config.os }} product-base \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push base image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./product/base - os: ${{ matrix.config.os }} - product: product-base - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.event.schedule == '0 12 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push base pro image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./product/base - os: ${{ matrix.config.os }} - product: product-base - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.event.schedule == '0 12 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic - - build-base-pro: - needs: build-base - runs-on: ubuntu-latest - name: product-base-pro-build-${{ matrix.config.os }}-r${{ matrix.config.r-primary }}_${{ matrix.config.r-alternate }}-py${{ matrix.config.py-primary }}_${{ matrix.config.py-alternate }} - - permissions: - contents: read - packages: write - - strategy: - fail-fast: false - matrix: - config: - - {os: 'centos7', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {os: 'ubuntu2204', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.12.1", py-alternate: "3.11.7"} - - concurrency: - group: build-base-pro-${{ matrix.config.os }}-r${{ matrix.config.r-primary }}_${{ matrix.config.r-alternate }}-py${{ matrix.config.py-primary }}_${{ matrix.config.py-alternate }}-${{ github.ref }} - cancel-in-progress: true - - steps: - - name: Check Out Repo - cron main - if: github.event.schedule == '0 12 * * 1' - uses: actions/checkout@v3 - with: - ref: main - - - name: Check Out Repo - if: github.event.schedule != '0 12 * * 1' - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-base-args ${{ matrix.config.os }} product-base-pro \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-base-tags ${{ matrix.config.os }} product-base-pro \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push base pro image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./product/pro - os: ${{ matrix.config.os }} - product: product-base-pro - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.event.schedule == '0 12 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push base pro image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./product/pro - os: ${{ matrix.config.os }} - product: product-base-pro - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.event.schedule == '0 12 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic - - build-products: - needs: [ build-base, build-base-pro ] - runs-on: ubuntu-latest - name: build-${{ matrix.config.product }}-${{ matrix.config.os }} - - permissions: - contents: read - packages: write - - strategy: - fail-fast: false - matrix: - config: - - {product: 'workbench', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {product: 'connect', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: 'connect-content-init', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: 'package-manager', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.12.1", py-alternate: "3.11.7"} - - {product: 'r-session-complete', os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {product: 'r-session-complete', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - concurrency: - group: build-products-${{ matrix.config.product }}-${{ matrix.config.os }}-${{ github.ref }} - cancel-in-progress: true - - steps: - - name: Check Out Repo - cron main - if: github.event.schedule == '0 12 * * 1' - uses: actions/checkout@v3 - with: - ref: main - - - name: Check Out Repo - if: github.event.schedule != '0 12 * * 1' - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Version - id: get-version - run: | - VERSION=`just -f ci.Justfile get-version ${{ matrix.config.product }} --type=release --local` - echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-product-args \ - ${{ matrix.config.product }} \ - ${{ matrix.config.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-product-tags \ - ${{ matrix.config.product }} \ - ${{ matrix.config.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push product image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./${{ matrix.config.product }} - os: ${{ matrix.config.os }} - product: ${{ matrix.config.product }} - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.event.schedule == '0 12 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push product image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./${{ matrix.config.product }} - os: ${{ matrix.config.os }} - product: ${{ matrix.config.product }} - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.event.schedule == '0 12 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic diff --git a/.github/workflows/build-workbench-aml.yaml b/.github/workflows/build-workbench-aml.yaml deleted file mode 100644 index 9e70c149..00000000 --- a/.github/workflows/build-workbench-aml.yaml +++ /dev/null @@ -1,127 +0,0 @@ -on: - schedule: - - cron: '0 14 * * 1' # If updating this value, be sure to update logic for all `push-image` arguments! - push: - branches: - - main - - dev - pull_request: - -name: Workbench for Azure ML - Build, Test, Scan, and Push -jobs: - build-workbench-for-azure-ml: - env: - product: workbench-for-microsoft-azure-ml - os: ubuntu2204 - r-primary: 4.2.3 - r-alternate: 4.1.3 - py-primary: 3.9.14 - py-alternate: 3.8.15 - runs-on: ubuntu-latest-4x - name: build-workbench-for-azure-ml - - permissions: - contents: read - packages: write - - steps: - - - name: Check Out Repo - cron main - if: github.event.schedule == '0 14 * * 1' - uses: actions/checkout@v3 - with: - ref: main - - - name: Check Out Repo - if: github.event.schedule != '0 14 * * 1' - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Version - id: get-version - run: | - VERSION=$(just -f ci.Justfile get-version ${{ env.product }} --type=release --local) - echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ env.r-primary }} \ - R_VERSION_ALT=${{ env.r-alternate }} \ - PYTHON_VERSION=${{ env.py-primary }} \ - PYTHON_VERSION_ALT=${{ env.py-alternate }} \ - get-product-args \ - ${{ env.product }} \ - ${{ env.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ env.r-primary }} \ - R_VERSION_ALT=${{ env.r-alternate }} \ - PYTHON_VERSION=${{ env.py-primary }} \ - PYTHON_VERSION_ALT=${{ env.py-alternate }} \ - get-product-tags \ - ${{ env.product }} \ - ${{ env.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push product image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./${{ env.product }} - os: ${{ env.os }} - product: ${{ env.product }} - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.event.schedule == '0 14 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push product image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./${{ env.product }} - os: ${{ env.os }} - product: ${{ env.product }} - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.event.schedule == '0 14 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic diff --git a/.github/workflows/build-workbench-gcw.yaml b/.github/workflows/build-workbench-gcw.yaml deleted file mode 100644 index 750ae7bd..00000000 --- a/.github/workflows/build-workbench-gcw.yaml +++ /dev/null @@ -1,99 +0,0 @@ -on: - schedule: - - cron: '0 14 * * 1' # If updating this value, be sure to update logic for all `push-image` arguments! - push: - branches: - - main - - dev - pull_request: - -name: Workbench for GCW - Build, Test, Scan, and Push -jobs: - build-workbench-for-google-cloud-workstations: - runs-on: ubuntu-latest-4x - name: build-workbench-for-google-cloud-workstations - - permissions: - contents: read - packages: write - - steps: - - name: Check Out Repo - cron main - if: github.event.schedule == '0 14 * * 1' - uses: actions/checkout@v3 - with: - ref: main - - - name: Check Out Repo - if: github.event.schedule != '0 14 * * 1' - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Version - id: get-version - run: | - VERSION=$(just workbench-for-google-cloud-workstations/get-version) - echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$(just workbench-for-google-cloud-workstations/get-build-args) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$(just workbench-for-google-cloud-workstations/get-build-tags) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push base pro image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./workbench-for-google-cloud-workstations - os: ubuntu2004 - product: workbench-for-google-cloud-workstations - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.event.schedule == '0 14 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push base pro image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./workbench-for-google-cloud-workstations - os: ubuntu2004 - product: workbench-for-google-cloud-workstations - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.event.schedule == '0 14 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic diff --git a/.gitignore b/.gitignore index c9c09d20..f2b8f966 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ data .idea/ tmp*/ .python-version +.out/ +.cache/ diff --git a/Justfile b/Justfile index 59152788..24f6daba 100644 --- a/Justfile +++ b/Justfile @@ -1,3 +1,4 @@ +#!/usr/bin/env just --justfile set positional-arguments vars := "-i ''" @@ -23,6 +24,127 @@ PYTHON_VERSION_ALT_RHEL := "3.8.15" QUARTO_VERSION := "1.3.450" +export RSC_LICENSE := "" +export RSPM_LICENSE := "" +export RSW_LICENSE := "" + +# Targets for managing the buildx builder for Posit images + +# just create-builder +create-builder: + docker buildx create \ + --name posit-builder \ + --buildkitd-flags '--allow-insecure-entitlement security.insecure' \ + --buildkitd-config "{{justfile_directory()}}/share/local_buildkitd.toml" + +# just delete-builder +delete-builder: + docker buildx rm posit-builder + +# Build and bake + +# just build +alias build := bake +# just bake workbench-images +bake target="default": + just -f {{justfile()}} create-builder || true + docker buildx bake --builder=posit-builder -f docker-bake.hcl {{target}} + +# just preview-bake workbench-images dev +preview-build: + just preview-bake "default" +preview-bake target branch="$(git branch --show-current)": + just -f {{justfile()}} create-builder || true + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) \ + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) \ + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) \ + BRANCH="{{branch}}" \ + docker buildx bake --builder=posit-builder -f docker-bake.preview.hcl {{target}} + +content-bake: + just -f {{justfile()}} create-builder || true + cd {{justfile_directory()}}/content && docker buildx bake --builder=posit-builder + cd {{justfile_directory()}} + +# just plan +plan: + docker buildx bake -f docker-bake.hcl --print + +# just preview-plan +preview-plan branch="$(git branch --show-current)": + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) \ + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) \ + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) \ + BRANCH="{{branch}}" \ + docker buildx bake -f docker-bake.preview.hcl --print + +# Run tests + +# just test workbench +test target="default" file="docker-bake.hcl": + python3 {{justfile_directory()}}/tools/test_bake_artifacts.py --target "{{target}}" --file "{{file}}" + +# just preview-test connect dev +preview-test target="default" branch="$(git branch --show-current)": + #!/bin/bash + if [ -z "$WORKBENCH_DAILY_VERSION" ]; then + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) + fi + if [ -z "$WORKBENCH_PREVIEW_VERSION" ]; then + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) + fi + if [ -z "$PACKAGE_MANAGER_DAILY_VERSION" ]; then + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) + fi + if [ -z "$CONNECT_DAILY_VERSION" ]; then + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) + fi + if [ -z "$BRANCH" ]; then + BRANCH="{{branch}}" + fi + WORKBENCH_DAILY_VERSION="${WORKBENCH_DAILY_VERSION}" \ + WORKBENCH_PREVIEW_VERSION="${WORKBENCH_PREVIEW_VERSION}" \ + PACKAGE_MANAGER_DAILY_VERSION="${PACKAGE_MANAGER_DAILY_VERSION}" \ + CONNECT_DAILY_VERSION="${CONNECT_DAILY_VERSION}" \ + BRANCH="${BRANCH}" \ + python3 {{justfile_directory()}}/tools/test_bake_artifacts.py --file docker-bake.preview.hcl --target "{{target}}" + +# just lint workbench ubuntu2204 +lint $PRODUCT $OS: + #!/usr/bin/env bash + docker run --rm -i -v $PWD/hadolint.yaml:/.config/hadolint.yaml ghcr.io/hadolint/hadolint < $PRODUCT/Dockerfile.$(just _parse-os {{OS}}) + +# Run targets + +run product tag="": + #!/bin/bash + RSC_VERSION="ubuntu2204" + RSW_VERSION="ubuntu2204" + RSPM_VERSION="ubuntu2204" + if [ "{{product}}" = "workbench" ] && [ -z "{{tag}}" ]; then + RSW_VERSION="{{tag}}" + elif [ "{{product}}" = "connect" ] && [ -z "{{tag}}" ]; then + RSC_VERSION="{{tag}}" + elif [ "{{product}}" = "package-manager" ] && [ -z "{{tag}}" ]; then + RSPM_VERSION="{{tag}}" + fi + RSW_VERSION="${RSW_VERSION}" RSC_VERSION="${RSC_VERSION}" RSPM_VERSION="${RSPM_VERSION}" \ + docker compose up \ + --no-build \ + {{product}} + +# Export/import targets + +export-artifacts target build_definition="docker-bake.hcl": + python3 {{justfile_directory()}}/tools/export_bake_artifacts.py --target "{{target}}" --file "{{build_definition}}" + +import-artifacts: + python3 {{justfile_directory()}}/tools/import_bake_artifacts.py + +# Helper targets + # just _get-tag-safe-version 2022.07.2+576.pro12 _get-tag-safe-version $VERSION: #!/usr/bin/env bash @@ -81,6 +203,8 @@ _config-license-persist-volumes TYPE PRODUCT HOST_DIR: echo "-v {{HOST_DIR}}/float:${licensing_state_root_dir}/.TurboFloat" fi +# Version and dependency version management + # just RSW_VERSION=1.2.3 R_VERSION=4.1.0 update-versions update-versions: just \ @@ -112,13 +236,11 @@ update-rsw-versions: sed {{ sed_vars }} "s/RSW_VERSION:.*/RSW_VERSION: {{ RSW_VERSION }}/g" docker-compose.yml sed {{ sed_vars }} "s/rstudio\/rstudio-workbench:.*/rstudio\/rstudio-workbench:$(just _get-clean-version {{ RSW_VERSION }})/g" docker-compose.yml sed {{ sed_vars }} "s/^RSW_VERSION := .*/RSW_VERSION := \"{{ RSW_VERSION }}\"/g" \ - r-session-complete/Justfile \ - workbench/Justfile \ - workbench-for-microsoft-azure-ml/Justfile \ Justfile sed {{ sed_vars }} "s/[0-9]\{4\}\.[0-9]\{1,2\}\.[0-9]\{1,2\}/`just _get-clean-version {{ RSW_VERSION }}`/g" \ workbench/README.md \ r-session-complete/README.md + sed -i '/variable WORKBENCH_VERSION/!b;n;c\ \ \ \ default = "{{ RSW_VERSION }}"' docker-bake.hcl # just RSPM_VERSION=1.2.3 update-rspm-versions update-rspm-versions: @@ -133,6 +255,7 @@ update-rspm-versions: package-manager/Justfile \ Justfile sed {{ sed_vars }} -E "s/[0-9]{4}\.[0-9]{1,2}\.[0-9]{1,2}/`just _get-clean-version {{ RSPM_VERSION }}`/g" package-manager/README.md + sed -i '/variable PACKAGE_MANAGER_VERSION/!b;n;c\ \ \ \ default = "{{ RSPM_VERSION }}"' docker-bake.hcl # just RSC_VERSION=1.2.3 update-rsc-versions update-rsc-versions: @@ -151,9 +274,11 @@ update-rsc-versions: sed {{ sed_vars }} -E "s/[0-9]{4}\.[0-9]{1,2}\.[0-9]{1,2}/`just _get-clean-version {{ RSC_VERSION }}`/g" \ connect/README.md \ connect-content-init/README.md + sed -i '/variable CONNECT_VERSION/!b;n;c\ \ \ \ default = "{{ RSC_VERSION }}"' docker-bake.hcl # just R_VERSION=3.2.1 R_VERSION_ALT=4.1.0 update-r-versions -update-r-versions: +update-r-versions: update-default-r-versions +update-default-r-versions: #!/usr/bin/env bash set -euxo pipefail # Update primary R versions @@ -197,7 +322,8 @@ update-r-versions: ci.Justfile # just PYTHON_VERSION=3.9.5 PYTHON_VERSION_ALT=3.8.10 update-py-versions -update-py-versions: +update-py-versions: update-default-py-versions +update-default-py-versions: #!/usr/bin/env bash set -euxo pipefail # Update primary Python versions @@ -267,6 +393,7 @@ update-drivers-versions: r-session-complete/Justfile \ product/pro/Justfile \ ci.Justfile + sed -i '/variable DRIVERS_VERSION/!b;n;c\ \ \ \ default = "{{ RSC_VERSION }}"' docker-bake.hcl update-quarto-versions: #!/usr/bin/env bash @@ -283,22 +410,4 @@ update-quarto-versions: connect/rstudio-connect.gcfg sed {{ sed_vars }} "s/qver=\${QUARTO_VERSION:-.*}/qver=\${QUARTO_VERSION:-{{ QUARTO_VERSION }}}/g" \ content/base/maybe_install_quarto.sh - - -# just test-image preview workbench 12.0.11-8 tag1 tag2 tag3 ... -test-image $PRODUCT $VERSION +IMAGES: - #!/usr/bin/env bash - set -euxo pipefail - IMAGES="{{IMAGES}}" - read -ra IMAGE_ARRAY <<<"$IMAGES" - just \ - R_VERSION={{R_VERSION}} \ - R_VERSION_ALT={{R_VERSION_ALT}} \ - PYTHON_VERSION={{PYTHON_VERSION}} \ - PYTHON_VERSION_ALT={{PYTHON_VERSION_ALT}} \ - $PRODUCT/test "${IMAGE_ARRAY[0]}" "$VERSION" - -# just lint workbench ubuntu2204 -lint $PRODUCT $OS: - #!/usr/bin/env bash - docker run --rm -i -v $PWD/hadolint.yaml:/.config/hadolint.yaml ghcr.io/hadolint/hadolint < $PRODUCT/Dockerfile.$(just _parse-os {{OS}}) + sed -i '/variable DEFAULT_QUARTO_VERSION/!b;n;c\ \ \ \ default = "{{ QUARTO_VERSION }}"' docker-bake.hcl diff --git a/NEWS.md b/NEWS.md index 2a981260..857ae42d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,10 @@ changed in each image. This file only captures pervasive, repository-wide changes. +# 2024-04-10 + +- Change build orchestration to use `docker buildx bake` for all images. + # 2024-03-14 - Update Professional Drivers to 2024.03.0 diff --git a/README.md b/README.md index f81e8c33..cca79afd 100644 --- a/README.md +++ b/README.md @@ -30,17 +30,20 @@ Docker images for RStudio Professional Products - RStudio Workbench Session Images (requires the launcher) - [R Session Complete](./r-session-complete/) - Product Base Images - - [Base](./product/base) - - [Pro (with Pro Drivers)](./product/pro) + - [Product Base](./product/base) + - [Product Base Pro (includes Pro Drivers)](./product/pro) +- RStudio Connect Session Images (requires the launcher) + - [Content Base Image](./content/base/) + - [Content Base + Pro Driver Image](./content/pro/) + - [Content Init Container](./connect-content-init/) ### Preview Images *IMPORTANT:* Do not use these images. They are in preparation for a future release -- RStudio Connect Session Images (requires the launcher) - - [Content Base Image](./content/base/) - - [Content Base + Pro Driver Image](./content/pro/) - - [Content Init Container](./connect-content-init/) +- [RStudio Workbench Preview and Daily](./workbench/) +- [RStudio Connect Daily](./connect/) +- [RStudio Package Manager Daily](./package-manager/) ### Image Hierarchy And Relationship @@ -49,19 +52,21 @@ Docker images for RStudio Professional Products ```mermaid flowchart TB; + subgraph Product Base Images; + id4("Product Base") + id5("Product Base Pro") + id4-->id5 + end subgraph s1["Connect"]; id1("Connect Image") id2("Connect Content Init") id1-.-id2 + id5-->id1 end; subgraph Package Manager; id3("Package Manager Image") + id4-->id3 end; - subgraph Product Base Images; - id4("Product Base") - id5("Product Base Pro") - id4-->id5 - end subgraph Workbench; id6("Workbench Image") id7("r-session-complete") @@ -115,28 +120,49 @@ After you have cloned [rstudio-docker-products](https://github.com/rstudio/rstud own containers fairly simply with the provided Justfiles. If you're unfamiliar with `just`, please check out [their documentation](https://just.systems/man/en). If you are unable to use `just` in your organization, most targets in each Justfile can be copy/pasted into your shell and ran there with variables replaced where -appropriate. +appropriate. -To build RStudio Workbench: -``` -just workbench/build -``` -To build RStudio Connect: -``` -just connect/build -``` -To build RStudio Package Manager: +We orchestrate all our builds using `docker buildx bake`. You can learn more about the tool on +[Docker's buildx bake page](https://docs.docker.com/build/bake/), however no additional background knowledge is needed on the tool in order +to use it. + +To build all images: +```bash +just build ``` -just package-manager/build +Individual images or groups of images can also be built. For example, you can build `connect` by running: +```bash +just bake connect ``` +Here are the available targets and groups in bake. To build one, use `just bake `: +- `build` - Builds all images +- `base-images` - Builds `product/base` and `product/pro` +- `package-manager` - Builds `package-manager` +- `connect` - Builds `connect` +- `workbench` - Builds `workbench` +- `connect-content-init` - Builds `connect-content-init` +- `r-session-complete` - Builds `r-session-complete` +- `workbench-for-google-cloud-workstations` - Builds `workbench-for-google-cloud-workstations` +- `waml-images` - Builds `workbench-for-microsoft-azure-ml` stack (build, scan, and final) + +Preview images and content images are also available to build through the `docker-bake.preview.hcl` and +`content/docker-bake.hcl` files respectively or use `just preview-bake` and `just content-bake`. + You can alter what exactly is built by changing `workbench/Dockerfile.$OS`, `connect/Dockerfile.$OS`, -and `package-manager/Dockerfile.$OS`. +and `package-manager/Dockerfile.$OS`. Keep in mind that `product/base` or `product/pro` also impact what our default +image builds contain. -You can then run what you've built to test out with the `run` commands. For instance, to run the workbench container +You can then run what you've built to test out with the `run` target. For instance, to run the workbench container you have built: ``` -just workbench/run +just RSW_LICENSE="" run workbench +``` + +You can also run validation tests on the image using [Goss](https://github.com/goss-org/goss). To run the tests, use +the `test` target. For example, to test Connect you can run: +``` +just test connect ``` Note you must have a license in place, and all other instructions in separate sections are still relevant. diff --git a/ci.Justfile b/ci.Justfile index 39261319..06324e8e 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -59,55 +59,6 @@ _get-rsw-download-url TYPE OS: get-version +NARGS: ./tools/get-version.py {{NARGS}} -# just get-base-args ubuntu2204 base|pro -get-base-args $OS $TYPE="base" $BRANCH=`git branch --show`: - #!/usr/bin/env bash - set -euxo pipefail - if [[ $TYPE == "base" || $TYPE == "product-base" ]]; then - SRC_IMAGE_NAME="" - CTX_PATH="./product/base" - FILE_PATH="./product/base/Dockerfile.${OS}" - elif [[ $TYPE == "base-pro" || $TYPE == "pro" || $TYPE == "product-base-pro" ]]; then - SRC_IMAGE_NAME="product-base" - CTX_PATH="./product/pro" - FILE_PATH="./product/pro/Dockerfile.${OS}" - fi - if [[ $BRANCH != "main" ]]; then - SRC_IMAGE_NAME="${SRC_IMAGE_NAME}-dev" - fi - - if [[ "${OS}" == "centos7" ]]; then - _DRIVERS_VERSION="{{ DRIVERS_VERSION_RHEL }}" - else - _DRIVERS_VERSION="{{ DRIVERS_VERSION }}" - fi - - printf "R_VERSION={{ R_VERSION }} - R_VERSION_ALT={{ R_VERSION_ALT }} - PYTHON_VERSION={{ PYTHON_VERSION }} - PYTHON_VERSION_ALT={{ PYTHON_VERSION_ALT }} - QUARTO_VERSION={{ QUARTO_VERSION }} - DRIVERS_VERSION=${_DRIVERS_VERSION} - SRC_IMAGE_NAME=${SRC_IMAGE_NAME}" - -# just get-base-tags ubuntu2204 base|pro -get-base-tags $OS $TYPE="base" $BRANCH=`git branch --show`: - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="" - if [[ $TYPE == "base" || $TYPE == "product-base" ]]; then - IMAGE_NAME="product-base" - elif [[ $TYPE == "base-pro" || $TYPE == "pro" || $TYPE == "product-base-pro" ]]; then - IMAGE_NAME="product-base-pro" - fi - if [[ $BRANCH != "main" ]]; then - IMAGE_NAME="${IMAGE_NAME}-dev" - fi - - echo ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}},\ - ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}-py{{PYTHON_VERSION}},\ - ghcr.io/rstudio/${IMAGE_NAME}:${OS} - # just get-product-args connect ubuntu2204 2023.05.0 get-product-args $PRODUCT $OS $VERSION $USE_S3="false" $BRANCH=`git branch --show` $SHA_SHORT=`git rev-parse --short HEAD`: #!/usr/bin/env bash @@ -286,20 +237,3 @@ get-prerelease-tags $TYPE $PRODUCT $OS $VERSION $BRANCH=`git branch --show`: done tags=$(IFS="," ; echo "${tag_array[*]}") echo "${tags}" - -# just get-content-args 4.2.3 3.9.17 -get-content-args r-ver py-ver drivers-ver="": - #!/usr/bin/env bash - printf "R_VERSION={{r-ver}} - PYTHON_VERSION={{py-ver}} - DRIVERS_VERSION={{drivers-ver}}" - -# just get-content-tags content-base|content-pro 4.2.3 3.9.17 ubuntu2204 -get-content-tags image-name r-ver py-ver os: - #!/usr/bin/env bash - OS=$(just _parse-os {{os}}) - OS_ALT=$(just _rev-parse-os {{os}}) - echo rstudio/{{image-name}}:r{{r-ver}}-py{{py-ver}}-${OS},\ - ghcr.io/rstudio/{{image-name}}:r{{r-ver}}-py{{py-ver}}-${OS},\ - rstudio/{{image-name}}:r{{r-ver}}-py{{py-ver}}-${OS_ALT},\ - ghcr.io/rstudio/{{image-name}}:r{{r-ver}}-py{{py-ver}}-${OS_ALT} diff --git a/connect-content-init/Dockerfile.ubuntu2204 b/connect-content-init/Dockerfile.ubuntu2204 index ac1e8be5..b0dbacae 100644 --- a/connect-content-init/Dockerfile.ubuntu2204 +++ b/connect-content-init/Dockerfile.ubuntu2204 @@ -1,6 +1,5 @@ # Tag from https://hub.docker.com/_/ubuntu/ -FROM ubuntu:22.04 -LABEL maintainer="RStudio Connect " +FROM ubuntu:22.04 as build # Install required tools: # - ca-certificates installs necessary certificates to use cURL with HTTPS websites diff --git a/connect-content-init/Justfile b/connect-content-init/Justfile deleted file mode 100644 index 3c59f2c9..00000000 --- a/connect-content-init/Justfile +++ /dev/null @@ -1,50 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "rstudio-" -PRODUCT := "connect-content-init" -IMAGE_OS := "ubuntu2204" - -RSC_VERSION := "2024.02.0" -RSC_TAG_SAFE_VERSION := replace(RSC_VERSION, "+", "-") -RSC_LICENSE := "" - -R_VERSION := "3.6.2" -R_VERSION_ALT := "4.1.0" - -PYTHON_VERSION := "3.9.5" -PYTHON_VERSION_ALT := "3.8.10" - -DEFAULT_TAG := IMAGE_PREFIX + PRODUCT + ":" + IMAGE_OS + "-" + RSC_TAG_SAFE_VERSION - -# Build Connect content init image - just build ubuntu2204 2022.10.0 rstudio/rstudio-connect-content-init:ubuntu2204-2022.10.0 -build OS=IMAGE_OS VERSION=RSC_VERSION +TAGS=DEFAULT_TAG: - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - tag_array=() - for TAG in {{TAGS}} - do - tag_array+=("-t" $TAG) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSC_VERSION="{{ VERSION }}" \ - --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . - -# Test Connect content init image - just test rstudio/rstudio-connect:ubuntu2204-2022.10.0 2022.10.0 -test TAG=DEFAULT_TAG VERSION=RSC_VERSION CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - RSC_VERSION="{{ VERSION }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test Connect content init image interactively - just test-i rstudio/rstudio-connect:ubuntu2204-2022.10.0 2022.10.0 -test-i TAG=DEFAULT_TAG VERSION=RSC_VERSION: - just test {{ TAG }} {{ VERSION }} bash diff --git a/connect-content-init/docker-compose.test.yml b/connect-content-init/docker-compose.test.yml deleted file mode 100644 index 0ee6998b..00000000 --- a/connect-content-init/docker-compose.test.yml +++ /dev/null @@ -1,13 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - environment: - # uses .env by default - - RSC_VERSION - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" diff --git a/connect-content-init/test/run_tests.sh b/connect-content-init/test/run_tests.sh index ff787bca..840d328a 100755 --- a/connect-content-init/test/run_tests.sh +++ b/connect-content-init/test/run_tests.sh @@ -1,15 +1,17 @@ #!/bin/bash # install goss - -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.8} +GOSS_FILE=${GOSS_FILE:-/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS +if [ -f /etc/debian_version ]; then + OS="ubuntu" +elif [ -f /etc/centos-release ]; then + OS="centos" +else + echo "OS not supported. Exiting" + exit 1 fi # install goss to tmp location and make executable @@ -17,4 +19,4 @@ curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/gos && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +OS=$OS GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/connect/Dockerfile.ubuntu2204 b/connect/Dockerfile.ubuntu2204 index ead6a71d..499957ae 100644 --- a/connect/Dockerfile.ubuntu2204 +++ b/connect/Dockerfile.ubuntu2204 @@ -1,11 +1,5 @@ -ARG R_VERSION=4.2.3 -ARG R_VERSION_ALT=4.1.3 -ARG PYTHON_VERSION=3.9.17 -ARG PYTHON_VERSION_ALT=3.8.17 -ARG SRC_IMAGE_NAME=product-base-pro -ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} -LABEL maintainer="RStudio Docker " +# syntax=docker/dockerfile:1-labs +FROM product-base-pro as build COPY --chmod=0775 startup.sh /usr/local/bin/startup.sh diff --git a/connect/Justfile b/connect/Justfile deleted file mode 100644 index 29503409..00000000 --- a/connect/Justfile +++ /dev/null @@ -1,90 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "rstudio-" -PRODUCT := "connect" -IMAGE_OS := "ubuntu2204" - -RSC_VERSION := "2024.02.0" -RSC_LICENSE := "" -RSC_LICENSE_SERVER := "" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.9.17" -PYTHON_VERSION_ALT := "3.8.17" - -PERSIST_LICENSE := "false" -PERSIST_LICENSE_DIR := join(justfile_directory(), "tmp-lic") - -DEFAULT_TAG := IMAGE_PREFIX + PRODUCT + ":" + IMAGE_OS + "-" + RSC_VERSION - -# Build Connect image - just build ubuntu2204 2022.10.0 rstudio/rstudio-connect:ubuntu2204-2022.10.0 -build OS=IMAGE_OS VERSION=RSC_VERSION +TAGS=DEFAULT_TAG: - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - tag_array=() - for TAG in {{TAGS}} - do - tag_array+=("-t" $TAG) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSC_VERSION="{{ VERSION }}" \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . - -# Test Connect image - just test rstudio/rstudio-connect:ubuntu2204-2022.10.0 2022.10.0 -test TAG=DEFAULT_TAG VERSION=RSC_VERSION CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - RSC_VERSION="{{ VERSION }}" \ - RSC_LICENSE="{{ RSC_LICENSE }}" \ - RSC_LICENSE_SERVER="{{ RSC_LICENSE_SERVER }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test Connect image interactively - just test-i rstudio/rstudio-connect:ubuntu2204-2022.10.0 2022.10.0 -test-i TAG=DEFAULT_TAG VERSION=RSC_VERSION: - just test {{ TAG }} {{ VERSION }} bash - -# Run Connect - just RSC_LICENSE="" run rstudio/rstudio-connect:ubuntu2204-2022.10.0 -run TAG=DEFAULT_TAG CMD="": - #!/usr/bin/env bash - set -euxo pipefail - if [ -z "{{ RSC_LICENSE }}" ] && [ -z "{{ RSC_LICENSE_SERVER }}" ]; then - echo "Please set RSC_LICENSE or RSC_LICENSE_SERVER before running." - exit 1 - fi - - volume_opts=() - if [ {{ PERSIST_LICENSE }} = "true" ]; then - if [ {{RSC_LICENSE}} ]; then - echo "Volumes will be configured to persist license state data for an activation key." - volume_opts=$(just -f ../Justfile _config-license-persist-volumes key {{PRODUCT}} {{PERSIST_LICENSE_DIR}}) - elif [ {{RSC_LICENSE_SERVER}} ]; then - echo "Volumes will be configured to persist license state data for a floating license server." - volume_opts=$(just -f ../Justfile _config-license-persist-volumes float {{PRODUCT}} {{PERSIST_LICENSE_DIR}}) - fi - fi - - docker run -it --privileged \ - ${volume_opts[@]} \ - -p 3939:3939 \ - -e RSC_LICENSE="{{ RSC_LICENSE }}" \ - -e RSC_LICENSE_SERVER="{{ RSC_LICENSE_SERVER }}" \ - "{{ TAG }}" {{ CMD }} diff --git a/connect/docker-compose.test.yml b/connect/docker-compose.test.yml deleted file mode 100644 index 7ac851a5..00000000 --- a/connect/docker-compose.test.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - privileged: true - environment: - # uses .env by default - - RSC_VERSION - - R_VERSION - - R_VERSION_ALT - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - RSC_LICENSE - - RSC_LICENSE_SERVER - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" diff --git a/connect/test/goss.yaml b/connect/test/goss.yaml index e9d49083..57ffaee8 100644 --- a/connect/test/goss.yaml +++ b/connect/test/goss.yaml @@ -46,7 +46,7 @@ file: exists: true /tmp/startup.log: exists: true - contains: + contents: - "!Error: Couldn't read configuration file " # Check that `cracklib-runtime` is present so `chpasswd` works /var/cache/cracklib/cracklib_dict.pwd: @@ -58,7 +58,7 @@ command: title: connect_version_matches exit-status: 0 stdout: [ - "{{ .Env.RSC_VERSION }}" + "{{ .Env.RSC_VERSION }}" ] # Ensure correct R version "/opt/R/{{ .Env.R_VERSION }}/bin/R --version": diff --git a/connect/test/run_tests.sh b/connect/test/run_tests.sh index 250f2161..10dcb7f7 100755 --- a/connect/test/run_tests.sh +++ b/connect/test/run_tests.sh @@ -10,19 +10,22 @@ sleep 15 echo '--> Startup complete' -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} -GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} - -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS +if [ -f /etc/debian_version ]; then + OS="ubuntu" +elif [ -f /etc/centos-release ]; then + OS="centos" +else + echo "OS not supported. Exiting" + exit 1 fi +GOSS_FILE=${GOSS_FILE:-/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} +GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} + # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +OS=$OS GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/content/README.md b/content/README.md index 2ba1fab8..52b52b27 100644 --- a/content/README.md +++ b/content/README.md @@ -89,20 +89,17 @@ DEBUG=1 ./scripts/build-image.sh \ ## Build matrix -The json defined in `matrix.json` is loaded by the Github Action to +The `CONTENT_BUILD_MATRIX` variable defined in [the buildx bake definition](../docker-bake.hcl) is used to determine which combinations of R and Python to use when building our `content-base` and `content-pro` images. To add a new R and Python version combination, simply update the matrix and the Github Action will publish -the new image combinations to our registries upon the next push to `main` +the new image combinations to our registries upon the next push to `main`. -**NOTE** This combination should be kept in sync with the `build-images.sh` script, as the -file `matrix.json` is used by Github Actions as described above, but the `build-images.sh` script -is used when the developer is validating the ability to build the new images locally. +To build content images locally, use `just bake content-images` from the repository root. ## Github Actions -Because of the dependency on the content-base images, the github actions that build the pro images -depend on completion of the base image builds in [build-content](../.github/workflows/build-content.yaml) +Content base and pro images are built in [build-bake](../.github/workflows/build-bake.yaml). # Licensing diff --git a/content/base/Dockerfile.ubuntu1804 b/content/base/Dockerfile.ubuntu1804 index c53d4485..9ef4363e 100644 --- a/content/base/Dockerfile.ubuntu1804 +++ b/content/base/Dockerfile.ubuntu1804 @@ -1,8 +1,6 @@ # Using dated tags from https://hub.docker.com/_/ubuntu/ FROM ubuntu:18.04 -LABEL maintainer="RStudio Docker " - # Locale configuration --------------------------------------------------------# RUN apt-get update \ && apt-get install -y --no-install-recommends locales \ diff --git a/content/base/Dockerfile.ubuntu2204 b/content/base/Dockerfile.ubuntu2204 index 030eab0b..118fc171 100644 --- a/content/base/Dockerfile.ubuntu2204 +++ b/content/base/Dockerfile.ubuntu2204 @@ -1,8 +1,6 @@ # Using dated tags from https://hub.docker.com/_/ubuntu/ FROM ubuntu:22.04 -LABEL maintainer="RStudio Docker " - ARG DEBIAN_FRONTEND=noninteractive ARG DISTRIBUTION=ubuntu-2204 diff --git a/content/base/Justfile b/content/base/Justfile deleted file mode 100755 index a1defbf8..00000000 --- a/content/base/Justfile +++ /dev/null @@ -1,47 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "rstudio/" -PRODUCT := "content-base" -IMAGE_OS := "ubuntu2204" - -R_VERSION := "3.6.3" - -PYTHON_VERSION := "3.9.5" - -QUARTO_VERSION := "1.3.340" - -DEFAULT_TAG := IMAGE_PREFIX + PRODUCT + ":r" + R_VERSION + "-py" + PYTHON_VERSION + "-" + IMAGE_OS - -# Build content base image - just build ubuntu2204 3.6.3 3.9.5 rstudio/content-base:r3.6.3-py3.9.5-bionic -build OS=IMAGE_OS _R_VERSION=R_VERSION _PYTHON_VERSION=PYTHON_VERSION +TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - tag_array=() - if [[ "{{TAGS}}" == "" ]]; then - read -a OS_ALIASES <<< $(just -f {{ justfile_directory() }}/../../ci.Justfile _get-os-alias {{OS}}) - for os_name in ${OS_ALIASES[@]}; - do - tag_array+=( - "-t" "rstudio/{{IMAGE_PREFIX}}{{PRODUCT}}:r{{_R_VERSION}}-py{{_PYTHON_VERSION}}-${os_name}" - ) - done - else - for TAG in {{TAGS}} - do - tag_array+=("-t" $TAG) - done - fi - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg R_VERSION="{{ _R_VERSION }}" \ - --build-arg PYTHON_VERSION="{{ _PYTHON_VERSION }}" \ - --build-arg QUARTO_VERSION="{{ QUARTO_VERSION }}" \ - --file="Dockerfile.{{ OS }}" {{ justfile_directory() }} diff --git a/content/base/NEWS.md b/content/base/NEWS.md index 47839d89..363fa2a8 100644 --- a/content/base/NEWS.md +++ b/content/base/NEWS.md @@ -1,3 +1,7 @@ +# 2024-04-10 + +- Change build orchestration to `buildx bake` + # 2023-04-26 - Use the Quarto release 1.3.340 in jammy images. diff --git a/content/base/README.md b/content/base/README.md deleted file mode 100644 index eb1ef638..00000000 --- a/content/base/README.md +++ /dev/null @@ -1,9 +0,0 @@ -## Build - -```console -# Creates rstudio/content-base:r3.6.3-py3.9.5-bionic -just build - -# Creates rstudio/content-base:r4.0.4-py3.9.2-bionic -just R_VERSION=4.0.4 PYTHON_VERSION=3.9.2 build -``` diff --git a/content/build-images.sh b/content/build-images.sh deleted file mode 100755 index ed812c3e..00000000 --- a/content/build-images.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -if [ $# -eq 0 ] ; then - echo "usage: $0 ..." - echo - echo "Common uses:" - echo " $0 build" - exit 1 -fi - -TARGETS="$@" - -build() { - just R_VERSION=3.1.3 PYTHON_VERSION=2.7.18 IMAGE_OS=ubuntu1804 ${1} - just R_VERSION=3.2.5 PYTHON_VERSION=2.7.18 IMAGE_OS=ubuntu1804 ${1} - just R_VERSION=3.3.3 PYTHON_VERSION=3.6.13 IMAGE_OS=ubuntu1804 ${1} - just R_VERSION=3.3.3 PYTHON_VERSION=3.6.13 IMAGE_OS=ubuntu1804 ${1} - just R_VERSION=3.4.4 PYTHON_VERSION=3.6.13 IMAGE_OS=ubuntu1804 ${1} - just R_VERSION=3.4.4 PYTHON_VERSION=3.7.10 IMAGE_OS=ubuntu1804 ${1} - just R_VERSION=3.5.3 PYTHON_VERSION=2.7.18 IMAGE_OS=ubuntu1804 ${1} - just R_VERSION=3.5.3 PYTHON_VERSION=3.7.10 IMAGE_OS=ubuntu1804 ${1} - just R_VERSION=3.6.3 PYTHON_VERSION=2.7.18 IMAGE_OS=ubuntu1804 ${1} - just R_VERSION=3.6.3 PYTHON_VERSION=3.6.13 IMAGE_OS=ubuntu1804 ${1} - just R_VERSION=3.6.3 PYTHON_VERSION=3.8.8 IMAGE_OS=ubuntu1804 ${1} - just R_VERSION=4.0.5 PYTHON_VERSION=3.6.13 IMAGE_OS=ubuntu1804 ${1} - just R_VERSION=4.0.5 PYTHON_VERSION=3.7.10 IMAGE_OS=ubuntu1804 ${1} - just R_VERSION=4.0.5 PYTHON_VERSION=3.8.8 IMAGE_OS=ubuntu1804 ${1} - just R_VERSION=4.0.5 PYTHON_VERSION=3.9.2 IMAGE_OS=ubuntu1804 ${1} - just R_VERSION=4.1.0 PYTHON_VERSION=3.8.8 IMAGE_OS=ubuntu1804 ${1} - just R_VERSION=4.1.0 PYTHON_VERSION=3.9.2 IMAGE_OS=ubuntu1804 ${1} - just R_VERSION=4.1.3 PYTHON_VERSION=3.10.4 IMAGE_OS=ubuntu1804 ${1} - just R_VERSION=3.6.3 PYTHON_VERSION=3.8.16 IMAGE_OS=ubuntu2204 ${1} - just R_VERSION=4.0.5 PYTHON_VERSION=3.9.16 IMAGE_OS=ubuntu2204 ${1} - just R_VERSION=4.1.3 PYTHON_VERSION=3.10.11 IMAGE_OS=ubuntu2204 ${1} - just R_VERSION=4.2.2 PYTHON_VERSION=3.11.3 IMAGE_OS=ubuntu2204 ${1} -} - -# build content-base -pushd base/ -build "${TARGETS[@]}" -popd - -# build content-pro -pushd pro/ -build "${TARGETS[@]}" -popd diff --git a/content/docker-bake.hcl b/content/docker-bake.hcl new file mode 100644 index 00000000..3205ca24 --- /dev/null +++ b/content/docker-bake.hcl @@ -0,0 +1,82 @@ +variable CONTENT_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu1804", os_alt = "bionic", r = "3.1.3", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.2.5", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.3.3", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.4.4", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.4.4", py = "3.7.10", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.5.3", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.5.3", py = "3.7.10", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.6.3", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.6.3", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.6.3", py = "3.8.8", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.7.10", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.8.8", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.9.2", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.1.0", py = "3.8.8", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.1.0", py = "3.9.2", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.1.3", py = "3.10.4", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu2204", os_alt = "jammy", r = "3.6.3", py = "3.8.16", drivers = "2024.03.0", quarto = "1.3.340"}, + {os = "ubuntu2204", os_alt = "jammy", r = "4.0.5", py = "3.9.16", drivers = "2024.03.0", quarto = "1.3.340"}, + {os = "ubuntu2204", os_alt = "jammy", r = "4.1.3", py = "3.10.11", drivers = "2024.03.0", quarto = "1.3.340"}, + {os = "ubuntu2204", os_alt = "jammy", r = "4.2.2", py = "3.11.3", drivers = "2024.03.0", quarto = "1.3.340"}, + ] + } +} + +group "default" { + targets = ["base", "pro"] +} + +target "base" { + name = "content-base-r${replace(builds.r, ".", "-")}-py${replace(builds.py, ".", "-")}-${builds.os}" + + tags = [ + "ghcr.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os}", + "ghcr.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os_alt}", + "docker.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os}", + "docker.io.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os_alt}", + ] + output = [ + "type=image", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "base" + + matrix = CONTENT_BUILD_MATRIX + args = { + R_VERSION = "${builds.r}" + PYTHON_VERSION = "${builds.py}" + QUARTO_VERSION = "${builds.quarto}" + } +} + +target "pro" { + name = "content-pro-r${replace(builds.r, ".", "-")}-py${replace(builds.py, ".", "-")}-${builds.os}" + + tags = [ + "ghcr.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os}", + "ghcr.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os_alt}", + "docker.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os}", + "docker.io.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os_alt}", + ] + output = [ + "type=image", + ] + + contexts = { + content-base = "target:content-base-r${replace(builds.r, ".", "-")}-py${replace(builds.py, ".", "-")}-${builds.os}" + } + + dockerfile = "Dockerfile.${builds.os}" + context = "pro" + + matrix = CONTENT_BUILD_MATRIX + args = { + R_VERSION = "${builds.r}" + DRIVERS_VERSION = "${builds.drivers}" + } +} diff --git a/content/matrix.json b/content/matrix.json deleted file mode 100644 index 163b8ddd..00000000 --- a/content/matrix.json +++ /dev/null @@ -1,23 +0,0 @@ -[ - {"r": "3.1.3", "py": "2.7.18", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.2.5", "py": "2.7.18", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.3.3", "py": "3.6.13", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.4.4", "py": "3.6.13", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.4.4", "py": "3.7.10", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.5.3", "py": "2.7.18", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.5.3", "py": "3.7.10", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.6.3", "py": "2.7.18", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.6.3", "py": "3.6.13", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.6.3", "py": "3.8.8", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.0.5", "py": "3.6.13", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.0.5", "py": "3.7.10", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.0.5", "py": "3.8.8", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.0.5", "py": "3.9.2", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.1.0", "py": "3.8.8", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.1.0", "py": "3.9.2", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.1.3", "py": "3.10.4", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.6.3", "py": "3.8.16", "drivers": "2024.03.0", "os": "ubuntu2204", "os_alt": "jammy"}, - {"r": "4.0.5", "py": "3.9.16", "drivers": "2024.03.0", "os": "ubuntu2204", "os_alt": "jammy"}, - {"r": "4.1.3", "py": "3.10.11", "drivers": "2024.03.0", "os": "ubuntu2204", "os_alt": "jammy"}, - {"r": "4.2.2", "py": "3.11.3", "drivers": "2024.03.0", "os": "ubuntu2204", "os_alt": "jammy"} -] diff --git a/content/pro/Dockerfile.ubuntu1804 b/content/pro/Dockerfile.ubuntu1804 index e3be3d7f..7b6f34c1 100644 --- a/content/pro/Dockerfile.ubuntu1804 +++ b/content/pro/Dockerfile.ubuntu1804 @@ -1,9 +1,4 @@ -ARG PYTHON_VERSION -ARG R_VERSION -ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/content-base:r${R_VERSION}-py${PYTHON_VERSION}-ubuntu1804 - -LABEL maintainer="RStudio Docker " +FROM content-base # Install RStudio Professional Drivers ----------------------------------------# ARG DRIVERS_VERSION=2024.03.0 diff --git a/content/pro/Dockerfile.ubuntu2204 b/content/pro/Dockerfile.ubuntu2204 index 4a8066b5..7b6f34c1 100644 --- a/content/pro/Dockerfile.ubuntu2204 +++ b/content/pro/Dockerfile.ubuntu2204 @@ -1,9 +1,4 @@ -ARG PYTHON_VERSION -ARG R_VERSION -ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/content-base:r${R_VERSION}-py${PYTHON_VERSION}-ubuntu2204 - -LABEL maintainer="RStudio Docker " +FROM content-base # Install RStudio Professional Drivers ----------------------------------------# ARG DRIVERS_VERSION=2024.03.0 diff --git a/content/pro/Justfile b/content/pro/Justfile deleted file mode 100755 index 1d77ae54..00000000 --- a/content/pro/Justfile +++ /dev/null @@ -1,48 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "rstudio/" -PRODUCT := "content-pro" -IMAGE_OS := "ubuntu2204" - -R_VERSION := "3.6.3" - -PYTHON_VERSION := "3.9.5" - -DRIVERS_VERSION := "2024.03.0" -DRIVERS_VERSION_RHEL := DRIVERS_VERSION + "-1" - -DEFAULT_TAG := IMAGE_PREFIX + PRODUCT + ":r" + R_VERSION + "-py" + PYTHON_VERSION + "-" + IMAGE_OS - -# Build content pro image - just build ubuntu2204 3.6.3 3.9.5 rstudio/content-pro:r3.6.3-py3.9.5-bionic -build OS=IMAGE_OS _R_VERSION=R_VERSION _PYTHON_VERSION=PYTHON_VERSION +TAGS=DEFAULT_TAG: - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - tag_array=() - if [[ "{{TAGS}}" == "" ]]; then - read -a OS_ALIASES <<< $(just -f {{ justfile_directory() }}/../../ci.Justfile _get-os-alias {{OS}}) - for os_name in ${OS_ALIASES[@]}; - do - tag_array+=( - "-t" "rstudio/{{IMAGE_PREFIX}}{{PRODUCT}}:r{{_R_VERSION}}-py{{_PYTHON_VERSION}}-${os_name}" - ) - done - else - for TAG in {{TAGS}} - do - tag_array+=("-t" $TAG) - done - fi - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg R_VERSION="{{ _R_VERSION }}" \ - --build-arg PYTHON_VERSION="{{ _PYTHON_VERSION }}" \ - --build-arg DRIVERS_VERSION="{{ DRIVERS_VERSION }}" \ - --file="Dockerfile.{{ OS }}" {{ justfile_directory() }} diff --git a/content/pro/NEWS.md b/content/pro/NEWS.md index 530a9fc6..8880dd7b 100644 --- a/content/pro/NEWS.md +++ b/content/pro/NEWS.md @@ -1,3 +1,7 @@ +# 2024-04-10 + +- Change build orchestration to `buildx bake` + # 2021-07-19 - Add NEWS.md diff --git a/content/pro/README.md b/content/pro/README.md deleted file mode 100644 index 254301c5..00000000 --- a/content/pro/README.md +++ /dev/null @@ -1,9 +0,0 @@ -## Build - -```console -# Creates rstudio/content-pro:r3.6.3-py3.9.5-bionic -just build - -# Creates rstudio/content-pro:r4.0.4-py3.9.2-bionic -just R_VERSION=4.0.4 PYTHON_VERSION=3.9.2 build -``` diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 00000000..2a521eae --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,524 @@ +### Variable definitions ### +variable CONNECT_VERSION { + default = "2024.02.0" +} + +variable PACKAGE_MANAGER_VERSION { + default = "2023.12.0-13" +} + +variable WORKBENCH_VERSION { + default = "2023.12.1+402.pro1" +} + +variable DRIVERS_VERSION { + default = "2024.03.0" +} + +variable DEFAULT_QUARTO_VERSION { + default = "1.4.553" +} + +function tag_safe_version { + params = [version] + result = replace(version, "+", "-") +} + +function clean_version { + params = [version] + result = regex_replace(version, "[+|-].*", "") +} + +function get_drivers_version { + params = [os] + result = os == "centos7" ? "${DRIVERS_VERSION}-1" : DRIVERS_VERSION +} + +function get_os_alt_name { + params = [os] + result = os == "ubuntu2204" ? "jammy" : os +} + +function get_centos_tags { + params = [os, product, product_version] + result = [ + "ghcr.io/rstudio/${product}:${os}-${tag_safe_version(product_version)}", + "ghcr.io/rstudio/${product}:${os}", + "docker.io/rstudio/${product}:${os}-${tag_safe_version(product_version)}", + "docker.io/rstudio/${product}:${os}", + ] +} + +function get_ubuntu_tags { + params = [os, product, product_version] + result = [ + "ghcr.io/rstudio/${product}:${os}-${tag_safe_version(product_version)}", + "ghcr.io/rstudio/${product}:${get_os_alt_name(os)}-${tag_safe_version(product_version)}", + "ghcr.io/rstudio/${product}:${os}", + "ghcr.io/rstudio/${product}:${get_os_alt_name(os)}", + "docker.io/rstudio/${product}:${get_os_alt_name(os)}-${tag_safe_version(product_version)}", + "docker.io/rstudio/${product}:${os}-${tag_safe_version(product_version)}", + "docker.io/rstudio/${product}:${get_os_alt_name(os)}", + "docker.io/rstudio/${product}:${os}", + ] +} + +# FIXME: There's an obnoxious amount of hardcoding here due to restrictions with what bake can actually interpret from HCL. +function get_tags { + params = [os, product, product_version] + result = os == "ubuntu2204" ? get_ubuntu_tags(os, product, product_version) : get_centos_tags(os, product, product_version) +} + +### Build matrices ### + +variable BASE_BUILD_MATRIX { + default = { + builds = [ + {os = "centos7", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.17", py_alternate = "3.8.17"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.12.1", py_alternate = "3.11.7"}, + ] + } +} + +variable PRO_BUILD_MATRIX { + default = BASE_BUILD_MATRIX +} + +variable PACKAGE_MANAGER_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.12.1", py_alternate = "3.11.7"}, + ] + } +} + +variable CONNECT_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.17", py_alternate = "3.8.17"}, + ] + } +} + +variable CONNECT_CONTENT_INIT_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204"}, + ] + } +} + +variable CONTENT_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu1804", os_alt = "bionic", r = "3.1.3", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.2.5", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.3.3", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.4.4", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.4.4", py = "3.7.10", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.5.3", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.5.3", py = "3.7.10", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.6.3", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.6.3", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.6.3", py = "3.8.8", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.7.10", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.8.8", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.9.2", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.1.0", py = "3.8.8", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.1.0", py = "3.9.2", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.1.3", py = "3.10.4", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu2204", os_alt = "jammy", r = "3.6.3", py = "3.8.16", drivers = "2024.03.0", quarto = "1.3.340"}, + {os = "ubuntu2204", os_alt = "jammy", r = "4.0.5", py = "3.9.16", drivers = "2024.03.0", quarto = "1.3.340"}, + {os = "ubuntu2204", os_alt = "jammy", r = "4.1.3", py = "3.10.11", drivers = "2024.03.0", quarto = "1.3.340"}, + {os = "ubuntu2204", os_alt = "jammy", r = "4.2.2", py = "3.11.3", drivers = "2024.03.0", quarto = "1.3.340"}, + ] + } +} + +variable R_SESSION_COMPLETE_BUILD_MATRIX { + default = { + builds = [ + {os = "centos7", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, + ] + } +} + +variable WORKBENCH_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, + ] + } +} + +variable WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2004", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.7", py_alternate = "3.10.13"}, + ] + } +} + +variable WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, + ] + } +} + +### Group definitions ### +group "default" { + targets = [ + "product-base", + "product-base-pro", + "connect", + "connect-content-init", + "content-base", + "content-pro", + "package-manager", + "r-session-complete", + "workbench", + ] +} + +group "base-images" { + targets = [ + "product-base", + "product-base-pro" + ] +} + +group "content-images" { + targets = [ + "content-base", + "content-pro" + ] +} + +group "waml-images" { + targets = [ + "build-workbench-for-microsoft-azure-ml", + "scan-workbench-for-microsoft-azure-ml", + "workbench-for-microsoft-azure-ml", + ] +} + +### Base Image targets ### +target "base" { + labels = { + "maintainer" = "Posit Docker " + } + output = ["type=image", "type=docker"] +} + +target "product-base" { + inherits = ["base"] + target = "build" + + name = "product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-base:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-base:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "product/base" + + matrix = BASE_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + TINI_VERSION = "0.19.0" + QUARTO_VERSION = "1.3.340" + } +} + +target "product-base-pro" { + inherits = ["base"] + target = "build" + + name = "product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-base-pro:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-base-pro:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "product/pro" + contexts = { + product-base = "target:product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = PRO_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + DRIVERS_VERSION = get_drivers_version(builds.os) + TINI_VERSION = "0.19.0" + QUARTO_VERSION = "1.3.340" + } +} + +### Package Manager targets ### +target "package-manager" { + inherits = ["base"] + target = "build" + + name = "package-manager-${builds.os}-${replace(PACKAGE_MANAGER_VERSION, ".", "-")}" + tags = get_tags(builds.os, "rstudio-package-manager", PACKAGE_MANAGER_VERSION) + + dockerfile = "Dockerfile.${builds.os}" + context = "package-manager" + contexts = { + product-base = "target:product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = PACKAGE_MANAGER_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + RSPM_VERSION = PACKAGE_MANAGER_VERSION + } +} + +### Connect targets ### +target "connect" { + inherits = ["base"] + target = "build" + + name = "connect-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" + tags = get_tags(builds.os, "rstudio-connect", CONNECT_VERSION) + + dockerfile = "Dockerfile.${builds.os}" + context = "connect" + contexts = { + product-base-pro = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = CONNECT_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + RSC_VERSION = CONNECT_VERSION + } +} + +target "connect-content-init" { + inherits = ["base"] + target = "build" + + name = "connect-content-init-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" + tags = get_tags(builds.os, "rstudio-connect-content-init", CONNECT_VERSION) + + dockerfile = "Dockerfile.${builds.os}" + context = "connect-content-init" + + matrix = CONNECT_CONTENT_INIT_BUILD_MATRIX + + args = { + RSC_VERSION = CONNECT_VERSION + } +} + +target "content-base" { + inherits = ["base"] + name = "content-base-r${replace(builds.r, ".", "-")}-py${replace(builds.py, ".", "-")}-${builds.os}" + + tags = [ + "ghcr.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os}", + "ghcr.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os_alt}", + "docker.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os}", + "docker.io.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os_alt}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "content/base" + + matrix = CONTENT_BUILD_MATRIX + args = { + R_VERSION = "${builds.r}" + PYTHON_VERSION = "${builds.py}" + QUARTO_VERSION = "${builds.quarto}" + } +} + +target "content-pro" { + inherits = ["base"] + name = "content-pro-r${replace(builds.r, ".", "-")}-py${replace(builds.py, ".", "-")}-${builds.os}" + + tags = [ + "ghcr.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os}", + "ghcr.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os_alt}", + "docker.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os}", + "docker.io.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os_alt}", + ] + + contexts = { + content-base = "target:content-base-r${replace(builds.r, ".", "-")}-py${replace(builds.py, ".", "-")}-${builds.os}" + } + + dockerfile = "Dockerfile.${builds.os}" + context = "content/pro" + + matrix = CONTENT_BUILD_MATRIX + args = { + R_VERSION = "${builds.r}" + DRIVERS_VERSION = "${builds.drivers}" + } +} + +### Workbench targets ### +target "r-session-complete" { + inherits = ["base"] + target = "build" + + name = "r-session-complete-${builds.os}-${replace(tag_safe_version(WORKBENCH_VERSION), ".", "-")}" + tags = get_tags(builds.os, "r-session-complete", WORKBENCH_VERSION) + + dockerfile = "Dockerfile.${builds.os}" + context = "r-session-complete" + contexts = { + product-base-pro = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = R_SESSION_COMPLETE_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + JUPYTERLAB_VERSION = "3.6.5" + RSW_VERSION = WORKBENCH_VERSION + RSW_NAME = builds.os == "centos7" ? "rstudio-workbench-rhel" : "rstudio-workbench" + RSW_DOWNLOAD_URL = builds.os == "centos7" ? "https://s3.amazonaws.com/rstudio-ide-build/server/centos7/x86_64" : "https://download2.rstudio.org/server/jammy/amd64" + } +} + +target "workbench" { + inherits = ["base"] + + name = "workbench-${builds.os}-${replace(tag_safe_version(WORKBENCH_VERSION), ".", "-")}" + tags = get_tags(builds.os, "rstudio-workbench", WORKBENCH_VERSION) + + dockerfile = "Dockerfile.${builds.os}" + context = "workbench" + contexts = { + product-base-pro = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = WORKBENCH_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + PYTHON_VERSION_JUPYTER = builds.py_alternate + RSW_VERSION = WORKBENCH_VERSION + RSW_NAME = "rstudio-workbench" + RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/jammy/amd64" + } +} + +### Workbench for Google Cloud Workstations targets ### +target "workbench-for-google-cloud-workstations" { + inherits = ["base"] + + name = "workbench-for-google-cloud-workstation-${builds.os}-${replace(tag_safe_version(WORKBENCH_VERSION), ".", "-")}" + tags = [ + "us-central1-docker.pkg.dev/posit-images/cloud-workstations/workbench:${tag_safe_version(WORKBENCH_VERSION)}", + "us-central1-docker.pkg.dev/posit-images/cloud-workstations/workbench:latest", + "us-docker.pkg.dev/posit-images/cloud-workstations/workbench:${tag_safe_version(WORKBENCH_VERSION)}", + "us-docker.pkg.dev/posit-images/cloud-workstations/workbench:latest", + "europe-docker.pkg.dev/posit-images/cloud-workstations/workbench:${tag_safe_version(WORKBENCH_VERSION)}", + "europe-docker.pkg.dev/posit-images/cloud-workstations/workbench:latest", + "asia-docker.pkg.dev/posit-images/cloud-workstations/workbench:${tag_safe_version(WORKBENCH_VERSION)}", + "asia-docker.pkg.dev/posit-images/cloud-workstations/workbench:latest", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "workbench-for-google-cloud-workstations" + + matrix = WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + PYTHON_VERSION_JUPYTER = builds.py_alternate + JUPYTERLAB_VERSION = "3.6.5" + QUARTO_VERSION = DEFAULT_QUARTO_VERSION + DRIVERS_VERSION = get_drivers_version(builds.os) + RSW_VERSION = WORKBENCH_VERSION + RSW_NAME = "rstudio-workbench" + RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/focal/amd64" + } +} + +### Workbench for Microsoft Azure ML targets ### + +target "build-workbench-for-microsoft-azure-ml" { + inherits = ["base"] + target = "build" + + name = "build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(tag_safe_version(WORKBENCH_VERSION), ".", "-")}" + + dockerfile = "Dockerfile.${builds.os}" + context = "workbench-for-microsoft-azure-ml" + contexts = { + product-base-pro = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + PYTHON_VERSION_JUPYTER = builds.py_alternate + JUPYTERLAB_VERSION = "3.6.5" + RSW_VERSION = WORKBENCH_VERSION + RSW_NAME = "rstudio-workbench" + RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/jammy/amd64" + } +} + +target "scan-workbench-for-microsoft-azure-ml" { + inherits = ["build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(tag_safe_version(WORKBENCH_VERSION), ".", "-")}"] + target = "clamav" + + name = "scan-workbench-for-microsoft-azure-ml-${builds.os}-${replace(tag_safe_version(WORKBENCH_VERSION), ".", "-")}" + + contexts = { + build = "target:build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(tag_safe_version(WORKBENCH_VERSION), ".", "-")}" + } + + matrix = WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX +} + +target "workbench-for-microsoft-azure-ml" { + inherits = ["build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(tag_safe_version(WORKBENCH_VERSION), ".", "-")}"] + target = "final" + + name = "workbench-for-microsoft-azure-ml-${builds.os}-${replace(tag_safe_version(WORKBENCH_VERSION), ".", "-")}" + tags = get_tags(builds.os, "rstudio-workbench-for-microsoft-azure-ml", WORKBENCH_VERSION) + + contexts = { + build = "target:build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(tag_safe_version(WORKBENCH_VERSION), ".", "-")}" + clamav = "target:scan-workbench-for-microsoft-azure-ml-${builds.os}-${replace(tag_safe_version(WORKBENCH_VERSION), ".", "-")}" + } + + matrix = WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX +} \ No newline at end of file diff --git a/docker-bake.preview.hcl b/docker-bake.preview.hcl new file mode 100644 index 00000000..d7754bbe --- /dev/null +++ b/docker-bake.preview.hcl @@ -0,0 +1,417 @@ +### Variable definitions ### +variable BRANCH { + default = "dev" +} + +variable CONNECT_DAILY_VERSION { + default = null +} + +variable PACKAGE_MANAGER_DAILY_VERSION { + default = null +} + +variable WORKBENCH_DAILY_VERSION { + default = null +} + +variable WORKBENCH_PREVIEW_VERSION { + default = null +} + +variable DRIVERS_VERSION { + default = "2024.03.0" +} + +variable DEFAULT_QUARTO_VERSION { + default = "1.4.553" +} + +variable RSW_PREVIEW_URL_BASE { + default = "https://s3.amazonaws.com/rstudio-ide-build/server/" +} + +function get_rsw_download_url { + params = [os] + result = os == "centos7" ? "${RSW_PREVIEW_URL_BASE}centos7/x86_64" : "${RSW_PREVIEW_URL_BASE}jammy/amd64" +} + +function tag_safe_version { + params = [version] + result = replace(version, "+", "-") +} + +function clean_version { + params = [version] + result = regex_replace(version, "[+|-].*", "") +} + +function get_tag_prefix { + params = [] + result = BRANCH != "main" ? "${BRANCH}-" : "" +} + +function get_drivers_version { + params = [os] + result = os == "centos7" ? "${DRIVERS_VERSION}-1" : DRIVERS_VERSION +} + +function get_os_alt_name { + params = [os] + result = os == "ubuntu2204" ? "jammy" : os +} + +function get_centos_tags { + params = [os, product, product_version, build_type] + result = [ + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${os}-${tag_safe_version(product_version)}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${os}-${clean_version(product_version)}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${os}-${build_type}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${os}-${tag_safe_version(product_version)}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${os}-${clean_version(product_version)}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${os}-${build_type}", + ] +} + +function get_ubuntu_tags { + params = [os, product, product_version, build_type] + result = [ + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${os}-${tag_safe_version(product_version)}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${get_os_alt_name(os)}-${tag_safe_version(product_version)}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${os}-${clean_version(product_version)}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${get_os_alt_name(os)}-${clean_version(product_version)}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${os}-${build_type}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${get_os_alt_name(os)}-${build_type}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${os}-${tag_safe_version(product_version)}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${get_os_alt_name(os)}-${tag_safe_version(product_version)}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${os}-${clean_version(product_version)}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${get_os_alt_name(os)}-${clean_version(product_version)}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${os}-${build_type}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${get_os_alt_name(os)}-${build_type}", + ] +} + +# FIXME: There's an obnoxious amount of hardcoding in tag-related functions due to restrictions with what bake can actually interpret from HCL. +function get_tags { + params = [os, product, product_version, build_type] + result = os == "ubuntu2204" ? get_ubuntu_tags(os, product, product_version, build_type) : get_centos_tags(os, product, product_version, build_type) +} + +### Build matrices ### +variable BASE_BUILD_MATRIX { + default = { + builds = [ + {os = "centos7", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + ] + } +} + +variable PRO_BUILD_MATRIX { + default = BASE_BUILD_MATRIX +} + +variable PACKAGE_MANAGER_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + ] + } +} + +variable CONNECT_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + ] + } +} + +variable CONNECT_CONTENT_INIT_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204"}, + ] + } +} + +variable R_SESSION_COMPLETE_BUILD_MATRIX { + default = { + builds = [ + {os = "centos7", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + ] + } +} + +variable WORKBENCH_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + ] + } +} + +### Group definitions ### +group "default" { + targets = [ + "product-base-dev", + "product-base-pro-dev", + "connect-daily", + "connect-content-init-daily", + "package-manager-daily", + "r-session-complete-preview", + "r-session-complete-daily", + "workbench-preview", + "workbench-daily", + ] +} + +group "base-dev-images" { + targets = [ + "product-base-dev", + "product-base-pro-dev", + ] +} + +group "r-session-complete-images" { + targets = [ + "r-session-complete-daily", + "r-session-complete-preview", + ] +} + +group "workbench-images" { + targets = [ + "workbench-daily", + "workbench-preview", + ] +} + +### Dev Base Image targets ### +target "base" { + labels = { + "maintainer" = "Posit Docker " + } + output = ["type=image", "type=docker"] +} + +target "product-base-dev" { + inherits = ["base"] + target = "build" + + name = "product-base-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-base-dev:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-base-dev:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "product/base" + + matrix = BASE_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + TINI_VERSION = "0.19.0" + QUARTO_VERSION = "1.3.340" + } +} + +target "product-base-pro-dev" { + inherits = ["base"] + target = "build" + + name = "product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-base-pro-dev:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-base-pro-dev:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "product/pro" + contexts = { + product-base = "target:product-base-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = PRO_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + DRIVERS_VERSION = get_drivers_version(builds.os) + TINI_VERSION = "0.19.0" + QUARTO_VERSION = "1.3.340" + } +} + +### Package Manager targets ### +target "package-manager-daily" { + inherits = ["base"] + target = "build" + + name = "package-manager-daily-${builds.os}-${replace(PACKAGE_MANAGER_DAILY_VERSION, ".", "-")}" + tags = get_tags(builds.os, "rstudio-package-manager-preview", PACKAGE_MANAGER_DAILY_VERSION, "daily") + + dockerfile = "Dockerfile.${builds.os}" + context = "package-manager" + contexts = { + product-base = "target:product-base-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = PACKAGE_MANAGER_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + RSPM_VERSION = PACKAGE_MANAGER_DAILY_VERSION + } +} + +### Connect targets ### +target "connect-daily" { + inherits = ["base"] + target = "build" + + name = "connect-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" + tags = get_tags(builds.os, "rstudio-connect-preview", CONNECT_DAILY_VERSION, "daily") + + dockerfile = "Dockerfile.${builds.os}" + context = "connect" + contexts = { + product-base-pro = "target:product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = CONNECT_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + RSC_VERSION = CONNECT_DAILY_VERSION + } +} + +target "connect-content-init-daily" { + inherits = ["base"] + target = "build" + + name = "connect-content-init-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" + tags = get_tags(builds.os, "rstudio-connect-content-init-preview", CONNECT_DAILY_VERSION, "daily") + + dockerfile = "Dockerfile.${builds.os}" + context = "connect-content-init" + + matrix = CONNECT_CONTENT_INIT_BUILD_MATRIX + + args = { + RSC_VERSION = CONNECT_DAILY_VERSION + } +} + +### Workbench targets ### +target "r-session-complete-daily" { + inherits = ["base"] + target = "build" + + name = "r-session-complete-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}" + tags = get_tags(builds.os, "r-session-complete-preview", WORKBENCH_DAILY_VERSION, "daily") + + dockerfile = "Dockerfile.${builds.os}" + context = "r-session-complete" + contexts = { + product-base-pro = "target:product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = R_SESSION_COMPLETE_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + JUPYTERLAB_VERSION = "3.6.5" + RSW_VERSION = WORKBENCH_DAILY_VERSION + RSW_NAME = builds.os == "centos7" ? "rstudio-workbench-rhel" : "rstudio-workbench" + RSW_DOWNLOAD_URL = get_rsw_download_url(builds.os) + } +} + +target "r-session-complete-preview" { + inherits = ["base"] + target = "build" + + name = "r-session-complete-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" + tags = get_tags(builds.os, "r-session-complete-preview", WORKBENCH_PREVIEW_VERSION, "preview") + + dockerfile = "Dockerfile.${builds.os}" + context = "r-session-complete" + contexts = { + product-base-pro = "target:product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = R_SESSION_COMPLETE_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + JUPYTERLAB_VERSION = "3.6.5" + RSW_VERSION = WORKBENCH_PREVIEW_VERSION + RSW_NAME = builds.os == "centos7" ? "rstudio-workbench-rhel" : "rstudio-workbench" + RSW_DOWNLOAD_URL = get_rsw_download_url(builds.os) + } +} + +target "workbench-daily" { + inherits = ["base"] + + name = "workbench-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}" + tags = get_tags(builds.os, "rstudio-workbench-preview", WORKBENCH_DAILY_VERSION, "daily") + + dockerfile = "Dockerfile.${builds.os}" + context = "workbench" + contexts = { + product-base-pro = "target:product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = WORKBENCH_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + PYTHON_VERSION_JUPYTER = builds.py_alternate + RSW_VERSION = WORKBENCH_DAILY_VERSION + RSW_NAME = "rstudio-workbench" + RSW_DOWNLOAD_URL = get_rsw_download_url(builds.os) + } +} + +target "workbench-preview" { + inherits = ["base"] + + name = "workbench-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" + tags = get_tags(builds.os, "rstudio-workbench-preview", WORKBENCH_PREVIEW_VERSION, "preview") + + dockerfile = "Dockerfile.${builds.os}" + context = "workbench" + contexts = { + product-base-pro = "target:product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = WORKBENCH_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + PYTHON_VERSION_JUPYTER = builds.py_alternate + RSW_VERSION = WORKBENCH_PREVIEW_VERSION + RSW_NAME = "rstudio-workbench" + RSW_DOWNLOAD_URL = get_rsw_download_url(builds.os) + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 149b95df..20bf3ec5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,55 +1,31 @@ -version: "3.5" +version: "3" services: - rstudio-workbench: + workbench: container_name: rstudio-workbench - build: - context: ./workbench - dockerfile: "Dockerfile.${IMAGE_OS:-ubuntu2204}" - args: - RSW_VERSION: 2023.12.1+402.pro1 - image: rstudio/rstudio-workbench:2023.12.0 + image: rstudio/rstudio-workbench:${RSW_VERSION:-ubuntu2204} environment: RSW_LICENSE: ${RSW_LICENSE} LICENSE_SERVER: ${RSW_LICENSE_SERVER} ports: - - 8787:8787 - - 5559:5559 - volumes: - - ./workbench/conf/:/etc/rstudio - - ./data/rsw:/home + - "127.0.0.1:8787:8787" + - "127.0.0.1:5559:5559" - rstudio-connect: + connect: container_name: rstudio-connect - build: - context: ./connect - dockerfile: "Dockerfile.${IMAGE_OS:-ubuntu1804}" - args: - RSC_VERSION: 2024.02.0 - image: rstudio/rstudio-connect:2024.02.0 + image: rstudio/rstudio-connect:${RSC_VERSION:-ubuntu2204} privileged: true environment: RSC_LICENSE: ${RSC_LICENSE} LICENSE_SERVER: ${RSC_LICENSE_SERVER} ports: - - 3939:3939 - volumes: - - ./connect/rstudio-connect.gcfg:/etc/rstudio-connect/rstudio-connect.gcfg - - ./data/rsc:/data + - "127.0.0.1:3939:3939" - rstudio-package-manager: + package-manager: container_name: rstudio-package-manager - build: - context: ./package-manager - dockerfile: "Dockerfile.${IMAGE_OS:-ubuntu2204}" - args: - RSPM_VERSION: 2023.12.0-13 - image: rstudio/rstudio-package-manager:2023.12.0 + image: rstudio/rstudio-package-manager:${RSPM_VERSION:-ubuntu2204} environment: RSPM_LICENSE: ${RSPM_LICENSE} LICENSE_SERVER: ${RSPM_LICENSE_SERVER} ports: - - 4242:4242 - volumes: - - ./package-manager/rstudio-pm.gcfg:/etc/rstudio-pm/rstudio-pm.gcfg - - ./data/rspm:/data + - "127.0.0.1:4242:4242" diff --git a/hadolint.yaml b/hadolint.yaml index bf071e06..6548cc9c 100644 --- a/hadolint.yaml +++ b/hadolint.yaml @@ -1,5 +1,7 @@ failure-threshold: warning ignored: + # Suppresses warning for not explicitly tagging FROM images. Docker buildx bake is handling FROM image injection. + - DL3006 # Suppresses warning for version pinning on apt installs. We currently do not practice this approach. - DL3008 # Suppresses warning for version pinning on yum installs. We currently do not practice this approach. diff --git a/package-manager/Dockerfile.ubuntu2204 b/package-manager/Dockerfile.ubuntu2204 index 06c052be..2a2c2690 100644 --- a/package-manager/Dockerfile.ubuntu2204 +++ b/package-manager/Dockerfile.ubuntu2204 @@ -1,11 +1,4 @@ -ARG R_VERSION=4.2.3 -ARG R_VERSION_ALT=4.1.3 -ARG PYTHON_VERSION=3.12.1 -ARG PYTHON_VERSION_ALT=3.11.7 -ARG SRC_IMAGE_NAME=product-base -ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} -LABEL maintainer="RStudio Docker " +FROM product-base as build ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 diff --git a/package-manager/Justfile b/package-manager/Justfile deleted file mode 100644 index cb03a2f4..00000000 --- a/package-manager/Justfile +++ /dev/null @@ -1,98 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "rstudio-" -PRODUCT := "package-manager" -IMAGE_OS := "ubuntu2204" - -RSPM_VERSION := "2023.12.0-13" -RSPM_LICENSE := "" -RSPM_LICENSE_SERVER := "" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.12.1" -PYTHON_VERSION_ALT := "3.11.7" - -PERSIST_LICENSE := "false" -PERSIST_LICENSE_DIR := join(justfile_directory(), "tmp-lic") - -_make-default-tag OS=IMAGE_OS: - echo "{{IMAGE_PREFIX}}{{PRODUCT}}:{{OS}}-$(just -f ../Justfile _get-tag-safe-version {{RSPM_VERSION}})" - -# Build Package Manager image - just build ubuntu2204 2022.07.2-11 rstudio/rstudio-package-manager:ubuntu2204-2022.07.2-11 -build OS=IMAGE_OS VERSION=RSPM_VERSION *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag {{OS}})) - else - raw_tag_array=("{{TAGS}}") - fi - - tag_array=() - for tag in $raw_tag_array - do - tag_array+=("-t" $tag) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSPM_VERSION="{{ VERSION }}" \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . - -# Test Package Manager image - just test rstudio/rstudio-package-manager:ubuntu2204-2022.07.2-11 2022.07.2-11 -test TAG=`just _make-default-tag` VERSION=RSPM_VERSION CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - RSPM_VERSION="{{ VERSION }}" \ - RSPM_LICENSE="{{ RSPM_LICENSE }}" \ - RSPM_LICENSE_SERVER="{{ RSPM_LICENSE_SERVER }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test Package Manager image interactively - just test-i rstudio/rstudio-package-manager:ubuntu2204-2022.07.2-11 2022.07.2-11 -test-i TAG=`just _make-default-tag` VERSION=RSPM_VERSION: - just test {{ TAG }} {{ VERSION }} bash - -# Run Package Manager - just RSPM_LICENSE="" run rstudio/rstudio-package-manager:ubuntu2204-2022.07.2-11 -run TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - if [ -z "{{ RSPM_LICENSE }}" ] && [ -z "{{ RSPM_LICENSE_SERVER }}" ]; then - echo "Please set RSPM_LICENSE or RSPM_LICENSE_SERVER before running." - exit 1 - fi - - volume_opts=() - if [ {{ PERSIST_LICENSE }} = "true" ]; then - if [ {{RSPM_LICENSE}} ]; then - echo "Volumes will be configured to persist license state data for an activation key." - volume_opts=$(just -f ../Justfile _config-license-persist-volumes key {{PRODUCT}} {{PERSIST_LICENSE_DIR}}) - elif [ {{RSPM_LICENSE_SERVER}} ]; then - echo "Volumes will be configured to persist license state data for a floating license server." - volume_opts=$(just -f ../Justfile _config-license-persist-volumes float {{PRODUCT}} {{PERSIST_LICENSE_DIR}}) - fi - fi - - docker run -it \ - ${volume_opts[@]} \ - -p 4242:4242 \ - -e RSPM_LICENSE="{{ RSPM_LICENSE }}" \ - -e RSPM_LICENSE_SERVER="{{ RSPM_LICENSE_SERVER }}" \ - "{{ TAG }}" {{ CMD }} diff --git a/package-manager/docker-compose.test.yml b/package-manager/docker-compose.test.yml deleted file mode 100644 index d1279b98..00000000 --- a/package-manager/docker-compose.test.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - environment: - # uses .env by default - - RSPM_VERSION - - R_VERSION - - R_VERSION_ALT - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - RSPM_LICENSE - - RSPM_LICENSE_SERVER - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" diff --git a/package-manager/test/run_tests.sh b/package-manager/test/run_tests.sh index 98227a94..4cf48553 100755 --- a/package-manager/test/run_tests.sh +++ b/package-manager/test/run_tests.sh @@ -1,5 +1,6 @@ #!/bin/bash +# start connect trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/startup.log; exit $err' ERR # start package manager @@ -8,16 +9,17 @@ tini -- /usr/local/bin/startup.sh >/tmp/startup.log 2>&1 & echo '--> Waiting for startup' sleep 15 -echo '--> Startup (hopefully) complete' - -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} +GOSS_FILE=${GOSS_FILE:-/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS +if [ -f /etc/debian_version ]; then + OS="ubuntu" +elif [ -f /etc/centos-release ]; then + OS="centos" +else + echo "OS not supported. Exiting" + exit 1 fi # install goss to tmp location and make executable @@ -25,4 +27,4 @@ curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/gos && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +OS=$OS GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/product/base/Dockerfile.centos7 b/product/base/Dockerfile.centos7 index 27341082..f000c432 100644 --- a/product/base/Dockerfile.centos7 +++ b/product/base/Dockerfile.centos7 @@ -1,5 +1,4 @@ -FROM centos:7 -LABEL maintainer="Posit Docker " +FROM centos:7 as build ### ARG declarations ### ARG R_VERSION=4.2.0 diff --git a/product/base/Dockerfile.ubuntu2204 b/product/base/Dockerfile.ubuntu2204 index f326b3e3..8e574e52 100644 --- a/product/base/Dockerfile.ubuntu2204 +++ b/product/base/Dockerfile.ubuntu2204 @@ -1,5 +1,4 @@ -FROM ubuntu:22.04 -LABEL maintainer="Posit Docker " +FROM ubuntu:22.04 as build ### ARG declarations ### ARG DEBIAN_FRONTEND=noninteractive diff --git a/product/base/Justfile b/product/base/Justfile deleted file mode 100755 index 00c60378..00000000 --- a/product/base/Justfile +++ /dev/null @@ -1,69 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -PRODUCT := "product-base" -IMAGE_OS := "ubuntu2204" - -IMAGE_REGISTRY := "rstudio" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.9.17" -PYTHON_VERSION_ALT := "3.8.17" - -TINI_VERSION := "0.19.0" -QUARTO_VERSION := "1.3.340" - -_make-default-tag OS=IMAGE_OS: - echo "{{IMAGE_REGISTRY}}/{{PRODUCT}}:{{OS}}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}}" - -# Build base image - just build ubuntu2204 rstudio/product-base:ubuntu2204 -build OS=IMAGE_OS *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag {{OS}})) - else - raw_tag_array=("{{TAGS}}") - fi - - tag_array=() - for TAG in {{TAGS}} - do - tag_array+=("-t" $TAG) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --build-arg TINI_VERSION="{{ TINI_VERSION }}" \ - --build-arg QUARTO_VERSION="{{ QUARTO_VERSION }}" \ - --file=./Dockerfile.$(just -f ../../Justfile _parse-os {{OS}}) . - -# Test base image - just test rstudio/product-base:ubuntu2204 -test TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - QUARTO_VERSION="{{ QUARTO_VERSION }}" \ - TINI_VERSION="{{ TINI_VERSION }}" \ - OS="{{ IMAGE_OS }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test base image interactively - just test-i rstudio/product-base:ubuntu2204 -test-i TAG=`just _make-default-tag`: - just test {{ TAG }} bash diff --git a/product/base/docker-compose.test.yml b/product/base/docker-compose.test.yml deleted file mode 100644 index bbd42874..00000000 --- a/product/base/docker-compose.test.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - privileged: true - environment: - # uses .env by default - - R_VERSION - - R_VERSION_ALT - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - TINI_VERSION - - QUARTO_VERSION - - OS - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" diff --git a/product/base/test/run_tests.sh b/product/base/test/run_tests.sh index 1f355886..c18d6a8c 100755 --- a/product/base/test/run_tests.sh +++ b/product/base/test/run_tests.sh @@ -5,14 +5,17 @@ trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/start echo '--> Startup complete' -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} +GOSS_FILE=${GOSS_FILE:-/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS +if [ -f /etc/debian_version ]; then + OS="ubuntu" +elif [ -f /etc/centos-release ]; then + OS="centos" +else + echo "OS not supported. Exiting" + exit 1 fi # install goss to tmp location and make executable @@ -20,4 +23,4 @@ curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/gos && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +OS=$OS GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/product/pro/Dockerfile.centos7 b/product/pro/Dockerfile.centos7 index 0214e2ab..936611e8 100644 --- a/product/pro/Dockerfile.centos7 +++ b/product/pro/Dockerfile.centos7 @@ -1,11 +1,4 @@ -ARG R_VERSION=4.2.0 -ARG R_VERSION_ALT=3.6.2 -ARG PYTHON_VERSION=3.9.14 -ARG PYTHON_VERSION_ALT=3.8.15 -ARG SRC_IMAGE_NAME=product-base -ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:centos7-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} -LABEL maintainer="Posit Docker " +FROM product-base as build ARG R_VERSION=4.2.0 ARG R_VERSION_ALT=3.6.2 diff --git a/product/pro/Dockerfile.ubuntu2204 b/product/pro/Dockerfile.ubuntu2204 index 3914cffb..f17dbfc2 100644 --- a/product/pro/Dockerfile.ubuntu2204 +++ b/product/pro/Dockerfile.ubuntu2204 @@ -1,11 +1,4 @@ -ARG R_VERSION=4.2.3 -ARG R_VERSION_ALT=4.1.3 -ARG PYTHON_VERSION=3.9.17 -ARG PYTHON_VERSION_ALT=3.8.17 -ARG SRC_IMAGE_NAME=product-base -ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} -LABEL maintainer="Posit Docker " +FROM product-base as build ARG DEBIAN_FRONTEND=noninteractive ARG R_VERSION=4.2.3 diff --git a/product/pro/Justfile b/product/pro/Justfile deleted file mode 100644 index d07474f0..00000000 --- a/product/pro/Justfile +++ /dev/null @@ -1,79 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -PRODUCT := "product-base-pro" -IMAGE_OS := "ubuntu2204" - -IMAGE_REGISTRY := "rstudio" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.9.17" -PYTHON_VERSION_ALT := "3.8.17" - -TINI_VERSION := "0.19.0" -QUARTO_VERSION := "1.3.340" - -DRIVERS_VERSION := "2024.03.0" -DRIVERS_VERSION_RHEL := DRIVERS_VERSION + "-1" - -_make-default-tag OS=IMAGE_OS: - echo "{{IMAGE_REGISTRY}}/{{PRODUCT}}:{{OS}}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}}" - -# Build base pro image - just build ubuntu2204 rstudio/product-base-pro:ubuntu2204 -build OS=IMAGE_OS *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag {{OS}})) - else - raw_tag_array=("{{TAGS}}") - fi - - if [[ "{{ OS }}" == "centos7" ]]; then - _DRIVERS_VERSION="{{ DRIVERS_VERSION_RHEL }}" - else - _DRIVERS_VERSION="{{ DRIVERS_VERSION }}" - fi - - tag_array=() - for TAG in $raw_tag_array - do - tag_array+=("-t" $TAG) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --build-arg DRIVERS_VERSION="${_DRIVERS_VERSION}" \ - --file=./Dockerfile.$(just -f ../../Justfile _parse-os {{OS}}) . - -# Test base image - just test rstudio/product-base-pro:ubuntu2204 -test TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - - IMAGE_NAME="{{ TAG }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - QUARTO_VERSION="{{ QUARTO_VERSION }}" \ - TINI_VERSION="{{ TINI_VERSION }}" \ - DRIVERS_VERSION="{{ DRIVERS_VERSION }}" \ - OS="{{ IMAGE_OS }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test base image interactively - just test-i rstudio/product-base-pro:ubuntu2204 -test-i TAG=`just _make-default-tag`: - just test {{ TAG }} bash diff --git a/product/pro/docker-compose.test.yml b/product/pro/docker-compose.test.yml deleted file mode 100644 index e34b86fa..00000000 --- a/product/pro/docker-compose.test.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - privileged: true - environment: - # uses .env by default - - R_VERSION - - R_VERSION_ALT - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - TINI_VERSION - - QUARTO_VERSION - - DRIVERS_VERSION - - DRIVERS_VERSION_RHEL - - OS - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" diff --git a/product/pro/test/goss.yaml b/product/pro/test/goss.yaml index 53ea2a34..8670095d 100644 --- a/product/pro/test/goss.yaml +++ b/product/pro/test/goss.yaml @@ -22,7 +22,12 @@ package: rstudio-drivers: installed: true versions: - - {{ trimSuffix "-1" .Env.DRIVERS_VERSION }} # RHEL driver doesn't print the "-1" suffix in the package name + {{if .Env.OS | regexMatch "ubuntu.*"}} + - {{ .Env.DRIVERS_VERSION }} # RHEL driver doesn't print the "-1" suffix in the package name + {{end}} + {{if .Env.OS | regexMatch "centos.*"}} + - {{ .Env.DRIVERS_VERSION }}.el + {{end}} file: /opt/R/{{.Env.R_VERSION}}/bin/R: diff --git a/product/pro/test/run_tests.sh b/product/pro/test/run_tests.sh index 1f355886..c18d6a8c 100755 --- a/product/pro/test/run_tests.sh +++ b/product/pro/test/run_tests.sh @@ -5,14 +5,17 @@ trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/start echo '--> Startup complete' -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} +GOSS_FILE=${GOSS_FILE:-/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS +if [ -f /etc/debian_version ]; then + OS="ubuntu" +elif [ -f /etc/centos-release ]; then + OS="centos" +else + echo "OS not supported. Exiting" + exit 1 fi # install goss to tmp location and make executable @@ -20,4 +23,4 @@ curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/gos && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +OS=$OS GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/r-session-complete/Dockerfile.centos7 b/r-session-complete/Dockerfile.centos7 index ae766c0b..b3ec72ed 100644 --- a/r-session-complete/Dockerfile.centos7 +++ b/r-session-complete/Dockerfile.centos7 @@ -1,11 +1,4 @@ -ARG R_VERSION=4.2.3 -ARG R_VERSION_ALT=4.1.3 -ARG PYTHON_VERSION=3.9.14 -ARG PYTHON_VERSION_ALT=3.8.15 -ARG SRC_IMAGE_NAME=product-base-pro -ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:centos7-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} -LABEL maintainer="RStudio Docker "` +FROM product-base-pro as build ARG R_VERSION=4.2.3 ARG R_VERSION_ALT=4.1.3 diff --git a/r-session-complete/Dockerfile.ubuntu2204 b/r-session-complete/Dockerfile.ubuntu2204 index 1f3018b2..9bef24ac 100644 --- a/r-session-complete/Dockerfile.ubuntu2204 +++ b/r-session-complete/Dockerfile.ubuntu2204 @@ -1,11 +1,4 @@ -ARG R_VERSION=4.2.3 -ARG R_VERSION_ALT=4.1.3 -ARG PYTHON_VERSION=3.9.17 -ARG PYTHON_VERSION_ALT=3.8.17 -ARG SRC_IMAGE_NAME=product-base-pro -ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} -LABEL maintainer="RStudio Docker " +FROM product-base-pro as build ARG DEBIAN_FRONTEND=noninteractive ARG R_VERSION=4.2.3 diff --git a/r-session-complete/Justfile b/r-session-complete/Justfile deleted file mode 100755 index da24d830..00000000 --- a/r-session-complete/Justfile +++ /dev/null @@ -1,87 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "" -PRODUCT := "r-session-complete" -IMAGE_OS := "ubuntu2204" - -RSW_VERSION := "2023.12.1+402.pro1" -RSW_LICENSE := "" - -DRIVERS_VERSION := "2024.03.0" -DRIVERS_VERSION_RHEL := DRIVERS_VERSION + "-1" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.9.14" -PYTHON_VERSION_ALT := "3.8.15" - -_make-default-tag OS=IMAGE_OS: - echo "{{IMAGE_PREFIX}}{{PRODUCT}}:{{OS}}-$(just -f ../Justfile _get-tag-safe-version {{RSW_VERSION}})" - -# Build r-session-complete image - just build ubuntu2204 2022.07.2+576.pro12 rstudio/r-session-complete:ubuntu2204-2022.07.2-576.pro12 -build OS=IMAGE_OS VERSION=RSW_VERSION *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - if [[ "{{ OS }}" == "centos7" ]]; then - _DRIVERS_VERSION="{{ DRIVERS_VERSION_RHEL }}" - else - _DRIVERS_VERSION="{{ DRIVERS_VERSION }}" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag {{OS}})) - else - raw_tag_array=("{{TAGS}}") - fi - - tag_array=() - for tag in $raw_tag_array - do - tag_array+=("-t" $tag) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSW_VERSION="{{ VERSION }}" \ - --build-arg DRIVERS_VERSION="${_DRIVERS_VERSION}" \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . - -# Test r-session-complete image - just test rstudio/r-session-complete:ubuntu2204-2022.07.2-576.pro12 2022.07.2+576.pro12 -test TAG=`just _make-default-tag` VERSION=RSW_VERSION CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - RSW_VERSION="{{ VERSION }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test r-session-complete image interactively - just test-i rstudio/r-session-complete:ubuntu2204-2022.07.2-576.pro12 2022.07.2+576.pro12 -test-i TAG=`just _make-default-tag` VERSION=RSW_VERSION: - just test {{ TAG }} {{ VERSION }} bash - -# Run r-session-complete - just run rstudio/r-session-complete:ubuntu2204-2022.07.2-576.pro12 -run TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - if [ -z "{{ RSW_LICENSE }}" ]; then - echo "Please set RSW_LICENSE to a valid RStudio Workbench license before running." - exit 1 - fi - docker run -it --privileged \ - -p 8788:8788 \ - -e RSW_LICENSE="{{ RSW_LICENSE }}" \ - "{{ TAG }}" {{ CMD }} diff --git a/r-session-complete/test/goss.yaml b/r-session-complete/test/goss.yaml index 94637931..09760e7c 100644 --- a/r-session-complete/test/goss.yaml +++ b/r-session-complete/test/goss.yaml @@ -22,6 +22,7 @@ file: command: "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/{{.Env.PYTHON_VERSION}}/bin/jupyter nbconvert --to notebook --stdin --stdout": title: jupyter_works + timeout: 60000 exit-status: 0 # Ensure correct R version @@ -49,4 +50,5 @@ command: "jupyter --version": title: jupyter_in_path_var + timeout: 60000 exit-status: 0 diff --git a/r-session-complete/test/run_tests.sh b/r-session-complete/test/run_tests.sh index 16568e9c..e292db06 100755 --- a/r-session-complete/test/run_tests.sh +++ b/r-session-complete/test/run_tests.sh @@ -1,14 +1,16 @@ #!/bin/bash -# install goss - -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} +GOSS_FILE=${GOSS_FILE:-/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS +if [ -f /etc/debian_version ]; then + OS="ubuntu" +elif [ -f /etc/centos-release ]; then + OS="centos" +else + echo "OS not supported. Exiting" + exit 1 fi # install goss to tmp location and make executable @@ -16,4 +18,4 @@ curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/gos && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +OS=$OS GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/share/buildkitd.toml b/share/buildkitd.toml new file mode 100644 index 00000000..ac746f6f --- /dev/null +++ b/share/buildkitd.toml @@ -0,0 +1 @@ +insecure-entitlements = ["security.insecure"] \ No newline at end of file diff --git a/share/local_buildkitd.toml b/share/local_buildkitd.toml new file mode 100644 index 00000000..a775e7ff --- /dev/null +++ b/share/local_buildkitd.toml @@ -0,0 +1,4 @@ +insecure-entitlements = ["security.insecure"] + +[worker.oci] + max-parallelism = 4 \ No newline at end of file diff --git a/tools/export_bake_artifacts.py b/tools/export_bake_artifacts.py new file mode 100644 index 00000000..2a970dc5 --- /dev/null +++ b/tools/export_bake_artifacts.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +""" +Export bake artifacts as tar files for later reuse. + +./export_bake_artifacts.py --file --target --output-path +""" + +import argparse +import json +import logging +import subprocess +from pathlib import Path + +LOGGER = logging.getLogger(__name__) +PROJECT_DIR = Path(__file__).resolve().parents[1] + + +parser = argparse.ArgumentParser( + description="Export one or more bake artifacts to tar files for reuse" +) +parser.add_argument("--file", default="docker-bake.hcl") +parser.add_argument("--target", default="default") +parser.add_argument("--output-path", default=PROJECT_DIR / ".out") + + +def get_bake_plan(bake_file="docker-bake.hcl", target="default"): + cmd = ["docker", "buildx", "bake", "-f", str(PROJECT_DIR / bake_file), "--print", target] + p = subprocess.run(cmd, capture_output=True) + if p.returncode != 0: + LOGGER.error(f"Failed to get bake plan: {p.stderr}") + exit(1) + return json.loads(p.stdout.decode("utf-8")) + + +def build_export_command(target_name, target_spec, output_path): + output_file = Path(output_path) / f"{target_name}.tar" + cmd = [ + "docker", + "image", + "save", + "--output", + f"{output_file}", + " ".join(target_spec["tags"]), + ] + return cmd + + +def run_cmd(target_name, cmd): + p = subprocess.run(" ".join(cmd), shell=True) + if p.returncode != 0: + LOGGER.error(f"{target_name} failed to export: {p.returncode}") + return p.returncode + + +def main(): + args = parser.parse_args() + plan = get_bake_plan(args.file, args.target) + output = args.output_path + if not Path(output).exists(): + Path(output).mkdir(parents=True) + LOGGER.info(f"Exporting {len(plan['target'].keys())} targets: {plan['target'].keys()}") + for target_name, target_spec in plan["target"].items(): + LOGGER.info(f"Exporting {target_name}") + cmd = build_export_command(target_name, target_spec, output) + run_cmd(target_name, cmd) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tools/import_bake_artifacts.py b/tools/import_bake_artifacts.py new file mode 100644 index 00000000..5120433b --- /dev/null +++ b/tools/import_bake_artifacts.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +""" +Run tests against bake artifacts by group/target and build definition. + +./test_bake_artifacts.py --file --target +""" + +import argparse +import logging +import subprocess +from pathlib import Path + +LOGGER = logging.getLogger(__name__) +PROJECT_DIR = Path(__file__).resolve().parents[1] + + +parser = argparse.ArgumentParser( + description="Import one or more bake artifacts from tar files for reuse" +) +parser.add_argument("--archive-path", type=Path, default=PROJECT_DIR / ".out") + + +def main(): + args = parser.parse_args() + if args.archive_path: + for archive in args.archive_path.glob("*.tar"): + LOGGER.info(f"Importing {archive}") + cmd = ["docker", "image", "load", "--input", archive] + p = subprocess.run(cmd) + if p.returncode != 0: + LOGGER.error(f"Failed to import {archive}: {p.returncode}") + + +if __name__ == "__main__": + main() diff --git a/tools/test_bake_artifacts.py b/tools/test_bake_artifacts.py new file mode 100644 index 00000000..f664cbaf --- /dev/null +++ b/tools/test_bake_artifacts.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +""" +Run tests against bake artifacts by group/target and build definition. + +./test_bake_artifacts.py --file --target +""" + +import argparse +import json +import logging +import os +import re +import subprocess +import sys +from pathlib import Path + +logging.basicConfig(stream=sys.stdout, level=logging.INFO) +LOGGER = logging.getLogger(__name__) + +PROJECT_DIR = Path(__file__).resolve().parents[1] +SKIP = [ + "^content.*", # Content images don't have tests right now! + "(build|scan)-workbench-for-microsoft.*", # Intermediary WAML layers aren't exported or tested +] + + +parser = argparse.ArgumentParser( + description="Extract a test command from a bake plan" +) +parser.add_argument("--file", default="docker-bake.hcl") +parser.add_argument("--target", default="default") + + +def get_bake_plan(bake_file="docker-bake.hcl", target="default"): + cmd = ["docker", "buildx", "bake", "-f", str(PROJECT_DIR / bake_file), "--print", target] + run_env = os.environ.copy() + p = subprocess.run(cmd, capture_output=True, env=run_env) + if p.returncode != 0: + LOGGER.error(f"Failed to get bake plan: {p.stderr}") + exit(1) + return json.loads(p.stdout.decode("utf-8")) + + +def custom_options(target_name, context_path): + opts = [] + if "workbench-for-google-cloud-workstation" in target_name: + deps_path = context_path / "deps" + opts.extend([ + "--mount=type=bind,source=/var/run/docker.sock,destination=/var/run/docker.sock", + f"--mount=type=bind,source={deps_path},destination=/tmp/deps", + ]) + return opts + + +def build_test_command(target_name, target_spec): + context_path = PROJECT_DIR / target_spec["context"] + test_path = context_path / "test" + cmd = [ + "docker", + "run", + "-t", + "--init", + "--rm", + "--entrypoint=''", + "--privileged", + f"--mount=type=bind,source={test_path},destination=/test", + ] + cmd.extend(custom_options(target_name, context_path)) + for name, value in target_spec["args"].items(): + cmd.extend(["--env", f'{name}="{value}"']) + cmd.append(target_spec["tags"][0]) + cmd.extend(["/test/run_tests.sh"]) + return cmd + + +def run_cmd(target_name, cmd): + LOGGER.info(f"Running tests for {target_name}") + LOGGER.info(f"{' '.join(cmd)}") + p = subprocess.run(" ".join(cmd), shell=True) + if p.returncode != 0: + LOGGER.error(f"{target_name} test failed with exit code {p.returncode}") + return p.returncode + + +def main(): + args = parser.parse_args() + plan = get_bake_plan(args.file, args.target) + result = 0 + skip_targets = [] + failed_targets = [] + targets = {} + for k in plan["group"][args.target]["targets"]: + for target_name, target_spec in plan["target"].items(): + if target_name.startswith(k): + targets[target_name] = target_spec + LOGGER.info(f"Testing {len(targets.keys())} targets: {targets.keys()}") + for target_name, target_spec in targets.items(): + if any(re.search(pattern, target_name) is not None for pattern in SKIP): + LOGGER.info(f"Skipping {target_name}") + skip_targets.append(target_name) + continue + cmd = build_test_command(target_name, target_spec) + LOGGER.debug(" ".join(cmd)) + return_code = run_cmd(target_name, cmd) + if return_code != 0: + failed_targets.append(target_name) + result = 1 + LOGGER.info(f"Skipped targets: {skip_targets}") + LOGGER.info(f"Failed targets: {failed_targets}") + exit(result) + + +if __name__ == "__main__": + main() diff --git a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 index b612a6a9..ddec0969 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 +++ b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 @@ -1,4 +1,4 @@ -FROM us-central1-docker.pkg.dev/cloud-workstations-images/predefined/base:public-image-current +FROM us-central1-docker.pkg.dev/cloud-workstations-images/predefined/base:public-image-current as build ### ARG declarations ### ARG DEBIAN_FRONTEND=noninteractive @@ -147,8 +147,7 @@ RUN /opt/python/"${PYTHON_VERSION_JUPYTER}"/bin/python -m venv /opt/python/jupyt && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip cache purge \ && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip cache purge -RUN curl -L -o /usr/local/bin/wait-for-it.sh https://raw.githubusercontent.com/rstudio/wait-for-it/master/wait-for-it.sh \ - && chmod +x /usr/local/bin/wait-for-it.sh +ADD --chmod=755 https://raw.githubusercontent.com/rstudio/wait-for-it/master/wait-for-it.sh /usr/local/bin/wait-for-it.sh RUN mkdir -p /var/lib/rstudio-server/monitor/log \ && chown -R rstudio-server:rstudio-server /var/lib/rstudio-server/monitor \ diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile deleted file mode 100644 index ce026a86..00000000 --- a/workbench-for-google-cloud-workstations/Justfile +++ /dev/null @@ -1,116 +0,0 @@ -set dotenv-load -set positional-arguments - -BUILDX_PATH := "" - -PRODUCT := "workbench" -IMAGE_OS := "ubuntu2004" - -RSW_LICENSE := "" -RSW_LICENSE_SERVER := "" - -_make-default-tag: - echo "${IMAGE_REGISTRY_NAME}:$(just -f ../Justfile _get-tag-safe-version "${RSW_VERSION}")" - -get-version: - echo "${RSW_VERSION}" - -get-build-args: - #!/usr/bin/env bash - printf "RSW_VERSION=${RSW_VERSION} - R_VERSION=${R_VERSION} - R_VERSION_ALT=${R_VERSION_ALT} - PYTHON_VERSION=${PYTHON_VERSION} - PYTHON_VERSION_ALT=${PYTHON_VERSION_ALT} - PYTHON_VERSION_JUPYTER=${PYTHON_VERSION_JUPYTER} - QUARTO_VERSION=${QUARTO_VERSION} - DRIVERS_VERSION=${DRIVERS_VERSION} - SRC_IMAGE_NAME=${SRC_IMAGE_NAME} - RSW_DOWNLOAD_URL=${RSW_DOWNLOAD_URL}" - -get-build-tags: - #!/usr/bin/env bash - set -eu - regions=("us-central1" "asia" "europe" "us") - tag_array=() - for region in "${regions[@]}"; do - tag_array+=("$region-docker.pkg.dev/posit-images/cloud-workstations/workbench:${RSW_TAG_VERSION}") - tag_array+=("$region-docker.pkg.dev/posit-images/cloud-workstations/workbench:latest") - done - IFS="," - echo "${tag_array[*]}" - -# Build Workbench image - just build 2022.07.2+576.pro12 rstudio/rstudio-workbench-gcw:2022.07.2 -build *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - CACHE_PATH=${GITHUB_WORKSPACE:-/tmp} - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=${CACHE_PATH}/.buildx-cache --cache-to=type=local,dest=${CACHE_PATH}/.buildx-cache-new,mode=max" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag)) - else - raw_tag_array=("{{TAGS}}") - fi - - tag_array=() - for tag in ${raw_tag_array[@]}; - do - tag_array+=("-t" $tag) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSW_VERSION="${RSW_VERSION}" \ - --build-arg R_VERSION="${R_VERSION}" \ - --build-arg R_VERSION_ALT="${R_VERSION_ALT}" \ - --build-arg PYTHON_VERSION="${PYTHON_VERSION}" \ - --build-arg PYTHON_VERSION_ALT="${PYTHON_VERSION_ALT}" \ - --build-arg PYTHON_VERSION_JUPYTER="${PYTHON_VERSION_JUPYTER}" \ - --build-arg QUARTO_VERSION="${QUARTO_VERSION}" \ - --build-arg DRIVERS_VERSION="${DRIVERS_VERSION}" \ - --build-arg RSW_DOWNLOAD_URL="${RSW_DOWNLOAD_URL}" \ - --file=./Dockerfile.ubuntu2004 . - - echo ${raw_tag_array[@]} - -# Test Workbench image - just test rstudio/rstudio-workbench:ubuntu1804-2022.07.2-576.pro12 2022.07.2+576.pro12 -test TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - RSW_VERSION_CLEAN=$(sed "s/daily-/daily+/" <<<"${RSW_VERSION}") - IMAGE_NAME="{{ TAG }}" \ - RSW_VERSION="${RSW_VERSION_CLEAN}" \ - RSW_LICENSE="{{ RSW_LICENSE }}" \ - RSW_LICENSE_SERVER="{{ RSW_LICENSE_SERVER }}" \ - DRIVERS_VERSION="${DRIVERS_VERSION}" \ - R_VERSION="${R_VERSION}" \ - R_VERSION_ALT="${R_VERSION_ALT}" \ - PYTHON_VERSION="${PYTHON_VERSION}" \ - PYTHON_VERSION_ALT="${PYTHON_VERSION_ALT}" \ - PYTHON_VERSION_JUPYTER="${PYTHON_VERSION_JUPYTER}" \ - QUARTO_VERSION="${QUARTO_VERSION}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test Workbench image interactively - just test-i rstudio/rstudio-workbench:ubuntu1804-2022.07.2-576.pro12 2022.07.2+576.pro12 -test-i TAG=`just _make-default-tag`: - just test {{ TAG }} bash - -# Run Workbench - just RSW_LICENSE="" run rstudio/r-session-complete:ubuntu1804-2022.07.2-576.pro12 -run TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euo pipefail - if [ -z "{{ RSW_LICENSE }}" ] && [ -z "{{ RSW_LICENSE_SERVER }}" ]; then - echo "Please set RSW_LICENSE or RSW_LICENSE_SERVER before running." - exit 1 - fi - - docker run -it --privileged \ - ${volume_opts[@]} \ - -p 8787:80 \ - -e RSW_LICENSE="{{ RSW_LICENSE }}" \ - -e RSW_LICENSE_SERVER="{{ RSW_LICENSE_SERVER }}" \ - "{{ TAG }}" {{ CMD }} diff --git a/workbench-for-google-cloud-workstations/docker-compose.test.yml b/workbench-for-google-cloud-workstations/docker-compose.test.yml deleted file mode 100644 index 8de098c6..00000000 --- a/workbench-for-google-cloud-workstations/docker-compose.test.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - environment: - # uses .env by default - - RSW_VERSION - - R_VERSION - - R_VERSION_ALT - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - PYTHON_VERSION_JUPYTER - - QUARTO_VERSION - - DRIVERS_VERSION - - RSW_LICENSE - - RSW_LICENSE_SERVER - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" - - "./deps/py_packages.txt:/tmp/py_packages.txt" - - "./deps/r_packages.txt:/tmp/r_packages.txt" diff --git a/workbench-for-google-cloud-workstations/test/goss.yaml b/workbench-for-google-cloud-workstations/test/goss.yaml index 674665ef..412fae9d 100644 --- a/workbench-for-google-cloud-workstations/test/goss.yaml +++ b/workbench-for-google-cloud-workstations/test/goss.yaml @@ -35,19 +35,14 @@ port: tcp:80: listening: true ip: - - 0.0.0.0 - skip: false - tcp:5559: - listening: true - ip: - - 127.0.0.1 + - 0.0.0.0 skip: false process: rserver: running: true skip: false - rstudio-launche: + rserver-launche: running: true skip: false @@ -88,14 +83,14 @@ file: exists: true /etc/rstudio/jupyter.conf: exists: true - contains: + contents: - --LabApp.token="" - --no-browser - --allow-root - --ip=0.0.0.0 /etc/rstudio/rserver.conf: exists: true - contains: + contents: - www-port=80 - launcher-sessions-callback-address=http://0.0.0.0:80 - auth-proxy=1 @@ -103,21 +98,21 @@ file: - auth-proxy-user-header=x-custom-user-name /etc/rstudio/nginx.site.conf: exists: true - contains: + contents: - proxy_set_header X-CUSTOM-USER-NAME user/google; - proxy_set_header Host $http_x_forwarded_host; /etc/rstudio/vscode.extensions.conf: exists: true - contains: + contents: - quarto.quarto - GoogleCloudTools.cloudcode /tmp/startup.log: exists: true - contains: + contents: - "!Error reading /etc/rstudio/rserver.conf:" /etc/pam.d/common-session: exists: true - contains: + contents: - "/^session required pam_mkhomedir.so skel=/etc/skel umask=0022$/" /etc/sssd/sssd.conf: exists: true @@ -135,7 +130,7 @@ file: group: root mode: "0755" filetype: file - contains: [ + contents: [ "useradd -m user" ] /etc/workstation-startup.d/110_config-jupyter.sh: @@ -144,7 +139,7 @@ file: group: root mode: "0755" filetype: file - contains: [ + contents: [ # Checks that we're setting the Jupyter shell to /bin/bash "echo \"c.ServerApp.terminado_settings = {'shell_command': ['/bin/bash']}\" > /home/user/.jupyter/jupyter_notebook_config.py" ] @@ -154,7 +149,7 @@ file: group: root mode: "0755" filetype: file - contains: [ + contents: [ "/usr/bin/supervisord -c /etc/supervisor/supervisord.conf" ] @@ -182,21 +177,25 @@ command: ] Test Jupyter works: exec: "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/jupyter/bin/jupyter nbconvert --to notebook --stdin --stdout" + timeout: 60000 exit-status: 0 Check primary Python is version {{.Env.PYTHON_VERSION}}: exec: /opt/python/{{.Env.PYTHON_VERSION}}/bin/python --version + timeout: 60000 exit-status: 0 stdout: [ "{{.Env.PYTHON_VERSION}}" ] Check alternate Python is version {{.Env.PYTHON_VERSION_ALT}}: exec: /opt/python/{{.Env.PYTHON_VERSION_ALT}}/bin/python --version + timeout: 60000 exit-status: 0 stdout: [ "{{.Env.PYTHON_VERSION_ALT}}" ] Check Jupyter venv uses Python {{.Env.PYTHON_VERSION_JUPYTER}}: exec: /opt/python/jupyter/bin/python --version + timeout: 60000 exit-status: 0 stdout: [ "{{.Env.PYTHON_VERSION_JUPYTER}}" @@ -209,18 +208,20 @@ command: ] {{ $python_version := .Env.PYTHON_VERSION }} {{ $python_version_alt := .Env.PYTHON_VERSION_ALT }} - {{ $py_package_list := readFile "/tmp/py_packages.txt" | splitList "\n" }} + {{ $py_package_list := readFile "/tmp/deps/py_packages.txt" | splitList "\n" }} {{- range $py_package_list }} Check Python {{ $python_version }} has "{{.}}" installed: exec: /opt/python/{{$python_version}}/bin/pip show {{.}} + timeout: 60000 exit-status: 0 Check Python {{ $python_version_alt }} has "{{.}}" installed: exec: /opt/python/{{$python_version_alt}}/bin/pip show {{.}} + timeout: 60000 exit-status: 0 {{end}} {{ $r_version := .Env.R_VERSION }} {{ $r_version_alt := .Env.R_VERSION_ALT }} - {{ $r_package_list := readFile "/tmp/r_packages.txt" | splitList "\n" }} + {{ $r_package_list := readFile "/tmp/deps/r_packages.txt" | splitList "\n" }} {{- range $r_package_list }} Check R {{ $r_version }} has "{{.}}" installed: exec: /opt/R/{{$r_version}}/bin/R --slave -e "library(\"{{.}}\")" diff --git a/workbench-for-google-cloud-workstations/test/run_tests.sh b/workbench-for-google-cloud-workstations/test/run_tests.sh index 02c2cd04..9e86f128 100755 --- a/workbench-for-google-cloud-workstations/test/run_tests.sh +++ b/workbench-for-google-cloud-workstations/test/run_tests.sh @@ -9,25 +9,29 @@ trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/start # start rstudio-server echo "--> Starting RStudio Workbench" /usr/bin/supervisord -c /etc/supervisor/supervisord.conf > /tmp/startup.log 2>&1 & +sleep 15 + +if [ -f /etc/debian_version ]; then + OS="ubuntu" +elif [ -f /etc/centos-release ]; then + OS="centos" +else + echo "OS not supported. Exiting" + exit 1 +fi echo "--> Waiting for workbench to startup... with RSW_TIMEOUT: $RSW_TIMEOUT" wait-for-it.sh localhost:80 -t $RSW_TIMEOUT wait-for-it.sh localhost:5559 -t $RSW_TIMEOUT echo "--> Startup complete" -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.22} -GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} - -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS -fi +GOSS_FILE=${GOSS_FILE:-/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} +GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-5} # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +OS=$OS GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 b/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 index 4a815694..81ae1f88 100644 --- a/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 +++ b/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 @@ -1,11 +1,4 @@ -ARG R_VERSION=4.2.3 -ARG R_VERSION_ALT=4.1.3 -ARG PYTHON_VERSION=3.9.17 -ARG PYTHON_VERSION_ALT=3.8.17 -ARG SRC_IMAGE_NAME=product-base-pro -ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} AS workbench -LABEL maintainer="RStudio Docker " +FROM product-base-pro AS build ARG DEBIAN_FRONTEND=noninteractive ARG R_VERSION=4.2.3 @@ -115,8 +108,7 @@ RUN /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install -r /tmp/py_packages && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip cache purge \ && rm /tmp/py_packages.txt -RUN curl -L -o /usr/local/bin/wait-for-it.sh https://raw.githubusercontent.com/rstudio/wait-for-it/master/wait-for-it.sh \ - && chmod +x /usr/local/bin/wait-for-it.sh +ADD --chmod=755 https://raw.githubusercontent.com/rstudio/wait-for-it/master/wait-for-it.sh /usr/local/bin/wait-for-it.sh RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash \ && az extension add -n ml -y @@ -131,8 +123,7 @@ ENTRYPOINT [] CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] - -FROM workbench AS clamav +FROM build AS clamav # Set up ClamAV RUN apt-get update \ @@ -169,7 +160,7 @@ RUN clamscan \ # with ClamAV installed. Avoid adding changes in this stage unless they are related # to the ClamAV stage. Since the ClamAV stage may be cached, you can't guarantee # another full scan if you change anything beyond this point. -FROM workbench AS final +FROM build AS final # Copy ClamAV scan logs so the end user can see them. COPY --from=clamav /var/log/clamav/clamscan.log /var/log/clamav/clamupdate.log / diff --git a/workbench-for-microsoft-azure-ml/Justfile b/workbench-for-microsoft-azure-ml/Justfile deleted file mode 100644 index cd9a2fa0..00000000 --- a/workbench-for-microsoft-azure-ml/Justfile +++ /dev/null @@ -1,78 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "rstudio-" -PRODUCT := "workbench" -IMAGE_OS := "ubuntu2204" - -RSW_VERSION := "2023.12.1+402.pro1" -RSW_LICENSE := "" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.9.17" -PYTHON_VERSION_ALT := "3.8.17" - -_make-default-tag OS=IMAGE_OS: - echo "{{IMAGE_PREFIX}}{{PRODUCT}}:{{OS}}-$(just -f ../Justfile _get-tag-safe-version {{RSW_VERSION}})" - -# Build Workbench for Azure ML image - just build ubuntu2204 2022.07.2+576.pro12 rstudio/rstudio-workbench:ubuntu2204-2022.07.2-576.pro12 -build OS=IMAGE_OS VERSION=RSW_VERSION *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag {{OS}})) - else - raw_tag_array=("{{TAGS}}") - fi - - tag_array=() - for tag in $raw_tag_array - do - tag_array+=("-t" $tag) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSW_VERSION="{{ VERSION }}" \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . - -# Test Workbench for Azure ML image - just test rstudio/rstudio-workbench:ubuntu2204-2022.07.2-576.pro12 2022.07.2+576.pro12 -test TAG=`just _make-default-tag` VERSION=RSW_VERSION CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - RSW_VERSION="{{ VERSION }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test Workbench for Azure ML image interactively - just test-i rstudio/rstudio-workbench:ubuntu2204-2022.07.2-576.pro12 2022.07.2+576.pro12 -test-i TAG=`just _make-default-tag` VERSION=RSW_VERSION: - just test {{ TAG }} {{ VERSION }} bash - -# Run Workbench for Azure ML - just RSW_LICENSE="" run rstudio/r-session-complete:ubuntu2204-2022.07.2-576.pro12 -run TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - if [ -z "{{ RSW_LICENSE }}" ]; then - echo "Please set RSW_LICENSE to a valid RStudio Workbench license before running." - exit 1 - fi - docker run -it --privileged \ - -p 8787:8787 \ - -e RSW_LICENSE="{{ RSW_LICENSE }}" \ - "{{ TAG }}" {{ CMD }} diff --git a/workbench-for-microsoft-azure-ml/docker-compose.test.yml b/workbench-for-microsoft-azure-ml/docker-compose.test.yml deleted file mode 100644 index 53a1d86d..00000000 --- a/workbench-for-microsoft-azure-ml/docker-compose.test.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - environment: - # uses .env by default - - RSW_VERSION - - R_VERSION - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - PYTHON_VERSION_JUPYTER - - R_VERSION_ALT - - RSW_LICENSE - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" - - "./test/goss_vars.yaml:/tmp/goss_vars.yaml" diff --git a/workbench-for-microsoft-azure-ml/test/goss.yaml b/workbench-for-microsoft-azure-ml/test/goss.yaml index 05269e7d..2c8ba3fe 100644 --- a/workbench-for-microsoft-azure-ml/test/goss.yaml +++ b/workbench-for-microsoft-azure-ml/test/goss.yaml @@ -101,6 +101,7 @@ command: ] "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/jupyter/bin/jupyter nbconvert --to notebook --stdin --stdout": title: jupyter_works + timeout: 60000 exit-status: 0 # Ensure correct python version "/opt/python/{{.Env.PYTHON_VERSION_ALT}}/bin/python --version": diff --git a/workbench-for-microsoft-azure-ml/test/goss_vars.yaml b/workbench-for-microsoft-azure-ml/test/goss_vars.yaml index 2669ba18..65f2d26c 100644 --- a/workbench-for-microsoft-azure-ml/test/goss_vars.yaml +++ b/workbench-for-microsoft-azure-ml/test/goss_vars.yaml @@ -19,7 +19,6 @@ syspkgs: - libglib2.0-0 - libpq5 - libsm6 - - libssl-dev - openssl - libssl-dev - libuser diff --git a/workbench-for-microsoft-azure-ml/test/run_tests.sh b/workbench-for-microsoft-azure-ml/test/run_tests.sh index 9dc299cb..8c14be69 100755 --- a/workbench-for-microsoft-azure-ml/test/run_tests.sh +++ b/workbench-for-microsoft-azure-ml/test/run_tests.sh @@ -1,7 +1,7 @@ #!/bin/bash set -xe -RSW_TIMEOUT=${RSW_TIMEOUT:-15} +RSW_TIMEOUT=${RSW_TIMEOUT:-60} touch /tmp/startup.log trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/startup.log; exit $err' ERR @@ -9,24 +9,30 @@ trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/start # start rstudio-server echo "--> Starting RStudio Workbench" /usr/bin/supervisord -c /etc/supervisor/supervisord.conf > /tmp/startup.log 2>&1 & +sleep 15 echo "--> Waiting for workbench to startup... with RSW_TIMEOUT: $RSW_TIMEOUT" wait-for-it.sh localhost:8787 -t $RSW_TIMEOUT +wait-for-it.sh localhost:5559 -t $RSW_TIMEOUT echo "--> Startup complete" -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} -GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} - -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS +if [ -f /etc/debian_version ]; then + OS="ubuntu" +elif [ -f /etc/centos-release ]; then + OS="centos" +else + echo "OS not supported. Exiting" + exit 1 fi +GOSS_FILE=${GOSS_FILE:-/test/goss.yaml} +GOSS_VARS_FILE=${GOSS_VARS_FILE:-/test/goss_vars.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} +GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-5} + # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +OS=$OS GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/workbench/Dockerfile.ubuntu2204 b/workbench/Dockerfile.ubuntu2204 index 64edfed1..8234397e 100644 --- a/workbench/Dockerfile.ubuntu2204 +++ b/workbench/Dockerfile.ubuntu2204 @@ -1,11 +1,4 @@ -ARG R_VERSION=4.2.3 -ARG R_VERSION_ALT=4.1.3 -ARG PYTHON_VERSION=3.9.17 -ARG PYTHON_VERSION_ALT=3.8.17 -ARG SRC_IMAGE_NAME=product-base-pro -ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} -LABEL maintainer="RStudio Docker " +FROM product-base-pro as build ARG DEBIAN_FRONTEND=noninteractive ARG R_VERSION=4.2.3 @@ -108,3 +101,5 @@ EXPOSE 5559/tcp ENTRYPOINT [] CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] + + diff --git a/workbench/Justfile b/workbench/Justfile deleted file mode 100644 index 785e0ad7..00000000 --- a/workbench/Justfile +++ /dev/null @@ -1,100 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "rstudio-" -PRODUCT := "workbench" -IMAGE_OS := "ubuntu2204" - -RSW_VERSION := "2023.12.1+402.pro1" -RSW_RELEASE_TYPE := "release" -RSW_LICENSE := "" -RSW_LICENSE_SERVER := "" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.9.17" -PYTHON_VERSION_ALT := "3.8.17" - -PERSIST_LICENSE := "false" -PERSIST_LICENSE_DIR := join(justfile_directory(), "tmp-lic") - -_make-default-tag OS=IMAGE_OS VERSION=RSW_VERSION: - echo "{{IMAGE_PREFIX}}{{PRODUCT}}:{{OS}}-$(just -f ../Justfile _get-tag-safe-version {{VERSION}})" - -# Build Workbench image - just build ubuntu2204 2022.07.2+576.pro12 rstudio/rstudio-workbench:ubuntu2204-2022.07.2-576.pro12 -build OS=IMAGE_OS VERSION=RSW_VERSION *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag {{OS}} {{VERSION}})) - else - raw_tag_array=("{{TAGS}}") - fi - - tag_array=() - for tag in $raw_tag_array - do - tag_array+=("-t" $tag) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSW_VERSION="{{ VERSION }}" \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --build-arg RSW_DOWNLOAD_URL="$(just -f ../ci.Justfile _get-rsw-download-url {{RSW_RELEASE_TYPE}} {{OS}})" \ - --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . - -# Test Workbench image - just test rstudio/rstudio-workbench:ubuntu2204-2022.07.2-576.pro12 2022.07.2+576.pro12 -test TAG=`just _make-default-tag` VERSION=RSW_VERSION CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - RSW_VERSION="{{ VERSION }}" \ - RSW_LICENSE="{{ RSW_LICENSE }}" \ - RSW_LICENSE_SERVER="{{ RSW_LICENSE_SERVER }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test Workbench image interactively - just test-i rstudio/rstudio-workbench:ubuntu2204-2022.07.2-576.pro12 2022.07.2+576.pro12 -test-i TAG=`just _make-default-tag` VERSION=RSW_VERSION: - just test {{ TAG }} {{ VERSION }} bash - -# Run Workbench - just RSW_LICENSE="" run rstudio/r-session-complete:ubuntu2204-2022.07.2-576.pro12 -run TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - if [ -z "{{ RSW_LICENSE }}" ] && [ -z "{{ RSW_LICENSE_SERVER }}" ]; then - echo "Please set RSW_LICENSE or RSW_LICENSE_SERVER before running." - exit 1 - fi - - volume_opts=() - if [ {{ PERSIST_LICENSE }} = "true" ]; then - if [ {{RSW_LICENSE}} ]; then - echo "Volumes will be configured to persist license state data for an activation key." - volume_opts=$(just -f ../Justfile _config-license-persist-volumes key {{PRODUCT}} {{PERSIST_LICENSE_DIR}}) - elif [ {{RSW_LICENSE_SERVER}} ]; then - echo "Volumes will be configured to persist license state data for a floating license server." - volume_opts=$(just -f ../Justfile _config-license-persist-volumes float {{PRODUCT}} {{PERSIST_LICENSE_DIR}}) - fi - fi - - docker run -it --privileged \ - ${volume_opts[@]} \ - -p 8787:8787 \ - -e RSW_LICENSE="{{ RSW_LICENSE }}" \ - -e RSW_LICENSE_SERVER="{{ RSW_LICENSE_SERVER }}" \ - "{{ TAG }}" {{ CMD }} diff --git a/workbench/docker-compose.test.yml b/workbench/docker-compose.test.yml deleted file mode 100644 index db54a879..00000000 --- a/workbench/docker-compose.test.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - environment: - # uses .env by default - - RSW_VERSION - - R_VERSION - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - PYTHON_VERSION_JUPYTER - - R_VERSION_ALT - - RSW_LICENSE - - RSW_LICENSE_SERVER - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" diff --git a/workbench/test/goss.yaml b/workbench/test/goss.yaml index 0a9a78c7..a1339ce3 100644 --- a/workbench/test/goss.yaml +++ b/workbench/test/goss.yaml @@ -112,6 +112,7 @@ command: ] "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/jupyter/bin/jupyter nbconvert --to notebook --stdin --stdout": title: jupyter_works + timeout: 60000 exit-status: 0 # Ensure correct python version "/opt/python/{{.Env.PYTHON_VERSION_ALT}}/bin/python --version": diff --git a/workbench/test/run_tests.sh b/workbench/test/run_tests.sh index 9dc299cb..ccec65e4 100755 --- a/workbench/test/run_tests.sh +++ b/workbench/test/run_tests.sh @@ -9,24 +9,29 @@ trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/start # start rstudio-server echo "--> Starting RStudio Workbench" /usr/bin/supervisord -c /etc/supervisor/supervisord.conf > /tmp/startup.log 2>&1 & +sleep 15 echo "--> Waiting for workbench to startup... with RSW_TIMEOUT: $RSW_TIMEOUT" wait-for-it.sh localhost:8787 -t $RSW_TIMEOUT +wait-for-it.sh localhost:5559 -t $RSW_TIMEOUT echo "--> Startup complete" -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} -GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} - -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS +if [ -f /etc/debian_version ]; then + OS="ubuntu" +elif [ -f /etc/centos-release ]; then + OS="centos" +else + echo "OS not supported. Exiting" + exit 1 fi +GOSS_FILE=${GOSS_FILE:-/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} +GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-5} + # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +OS=$OS GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT From 490e786e9b1a750a6084cf8495100e5b125018b8 Mon Sep 17 00:00:00 2001 From: Cole Arendt Date: Thu, 18 Apr 2024 05:33:20 -0400 Subject: [PATCH 2/9] add the libarchive-dev package to image builds This is important for the installation of the r-lib/archive package, which is used by shinylive, for instance. --- content/base/Dockerfile.ubuntu2204 | 1 + product/base/Dockerfile.ubuntu2204 | 1 + 2 files changed, 2 insertions(+) diff --git a/content/base/Dockerfile.ubuntu2204 b/content/base/Dockerfile.ubuntu2204 index 118fc171..87e44098 100644 --- a/content/base/Dockerfile.ubuntu2204 +++ b/content/base/Dockerfile.ubuntu2204 @@ -39,6 +39,7 @@ RUN apt-get update \ git \ gsfonts \ imagemagick \ + libarchive-dev \ libcairo2-dev \ libcurl4-openssl-dev \ libfontconfig1-dev \ diff --git a/product/base/Dockerfile.ubuntu2204 b/product/base/Dockerfile.ubuntu2204 index 8e574e52..ae5e655e 100644 --- a/product/base/Dockerfile.ubuntu2204 +++ b/product/base/Dockerfile.ubuntu2204 @@ -30,6 +30,7 @@ RUN apt-get update --fix-missing \ gpg-agent \ gsfonts \ imagemagick \ + libarchive-dev \ libcairo2-dev \ libcurl4-openssl-dev \ libev-dev \ From 01586af72ccdf1f35c3c1d031ccc1627cc9e547a Mon Sep 17 00:00:00 2001 From: Cole Arendt Date: Thu, 18 Apr 2024 05:41:10 -0400 Subject: [PATCH 3/9] fix spacing --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index bbe16171..20bf3ec5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ services: connect: container_name: rstudio-connect - image: rstudio/rstudio-connect:${RSC_VERSION:-ubuntu2204} + image: rstudio/rstudio-connect:${RSC_VERSION:-ubuntu2204} privileged: true environment: RSC_LICENSE: ${RSC_LICENSE} From 1cfd3a36a28b365aa671a52df7c87dca03f26699 Mon Sep 17 00:00:00 2001 From: Cole Arendt Date: Thu, 18 Apr 2024 09:35:32 -0400 Subject: [PATCH 4/9] =?UTF-8?q?add=20a=20sleep=20to=20workbench=20startup.?= =?UTF-8?q?..=20=F0=9F=98=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We discussed this offline. Workbench apparently has a race condition where it starts up and provisions files. If it shuts down before complete, you can end up with failing tests and a weird `launcher.pem` file with no data and the wrong file permissions. Hopefully this "fix" will not be long lived. --- workbench/Dockerfile.ubuntu2204 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workbench/Dockerfile.ubuntu2204 b/workbench/Dockerfile.ubuntu2204 index 8234397e..28f40f27 100644 --- a/workbench/Dockerfile.ubuntu2204 +++ b/workbench/Dockerfile.ubuntu2204 @@ -53,6 +53,8 @@ RUN apt-get update \ && gpg --keyserver keys.openpgp.org --recv-keys 51C0B5BB19F92D60 \ && dpkg-sig --verify ./rstudio-workbench.deb \ && apt-get install -yq --no-install-recommends ./rstudio-workbench.deb \ + # a wild hack to ensure that workbench can install _and start_ completely before shutdown + && sleep 30 \ && rm ./rstudio-workbench.deb \ && apt-get autoremove -y \ && apt-get clean \ From 1c0c92fe0e85207e7bbd09c65da85822c8559368 Mon Sep 17 00:00:00 2001 From: Cole Arendt Date: Thu, 18 Apr 2024 09:43:38 -0400 Subject: [PATCH 5/9] add TZ env var to prevent system-looking errors As discussed in #720, these errors are resolved by setting the TZ variable. Setting that variable in the images should ensure that these errors are not seen across a wide array of environments. --- content/base/Dockerfile.ubuntu2204 | 1 + product/base/Dockerfile.ubuntu2204 | 1 + 2 files changed, 2 insertions(+) diff --git a/content/base/Dockerfile.ubuntu2204 b/content/base/Dockerfile.ubuntu2204 index 118fc171..e96fdcb1 100644 --- a/content/base/Dockerfile.ubuntu2204 +++ b/content/base/Dockerfile.ubuntu2204 @@ -18,6 +18,7 @@ RUN apt-get update \ ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 +ENV TZ=UTC # Installation prerequisites --------------------------------------------------# # curl is used to download things. diff --git a/product/base/Dockerfile.ubuntu2204 b/product/base/Dockerfile.ubuntu2204 index 8e574e52..24104fca 100644 --- a/product/base/Dockerfile.ubuntu2204 +++ b/product/base/Dockerfile.ubuntu2204 @@ -129,6 +129,7 @@ RUN localedef -i en_US -f UTF-8 en_US.UTF-8 ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 +ENV TZ=UTC ### Clean up ### RUN apt-get install -yqf --no-install-recommends \ From 0d64e870af1f862dda7d0c7d98586f3dfdf3a2ea Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 18 Apr 2024 14:04:42 -0700 Subject: [PATCH 6/9] Extract common operations to reusable scripts in base images (#716) * Fix/exclude goss tests for buildx bake * Implement separate test for Connect * Implement bake preview builds * Provision buildx in preview * Do not cache or output tests for preview Provide a buildx config for GHA Unify test sleep behavior of Workbench images * Remove test targets * Remove test layers from Dockerfiles * Change test running to be orchestrated by a Python script * Update preview builds for new test script * Create common use scripts in base image for Ubuntu 22.04 images * Create common use scripts in base image for CentOS 7 images * Move deps to subdirectory Fix hadolint issues * Ignore hadolint issue * Update permissions hex for tests * Remove build.justfile * Remove unnecessary buildx setup from build-bake-preview.yaml * Move libarchive-dev to ubuntu2204_packages.txt --- product/base/Dockerfile.centos7 | 65 ++---- product/base/Dockerfile.ubuntu2204 | 114 ++-------- product/base/deps/centos7_packages.txt | 20 ++ product/base/deps/requirements.txt | 2 + product/base/deps/ubuntu2204_packages.txt | 66 ++++++ product/base/scripts/rhel/install_drivers.sh | 21 ++ product/base/scripts/rhel/install_python.sh | 118 ++++++++++ product/base/scripts/rhel/install_quarto.sh | 75 +++++++ product/base/scripts/rhel/install_r.sh | 173 +++++++++++++++ product/base/scripts/rhel/yum.sh | 112 ++++++++++ product/base/scripts/ubuntu/apt.sh | 118 ++++++++++ .../base/scripts/ubuntu/install_drivers.sh | 21 ++ product/base/scripts/ubuntu/install_python.sh | 115 ++++++++++ product/base/scripts/ubuntu/install_quarto.sh | 75 +++++++ product/base/scripts/ubuntu/install_r.sh | 208 ++++++++++++++++++ product/base/test/goss.yaml | 22 ++ product/pro/Dockerfile.centos7 | 17 +- product/pro/Dockerfile.ubuntu2204 | 20 +- product/pro/deps/r_packages.txt | 1 + 19 files changed, 1203 insertions(+), 160 deletions(-) create mode 100644 product/base/deps/centos7_packages.txt create mode 100644 product/base/deps/requirements.txt create mode 100644 product/base/deps/ubuntu2204_packages.txt create mode 100755 product/base/scripts/rhel/install_drivers.sh create mode 100755 product/base/scripts/rhel/install_python.sh create mode 100755 product/base/scripts/rhel/install_quarto.sh create mode 100755 product/base/scripts/rhel/install_r.sh create mode 100755 product/base/scripts/rhel/yum.sh create mode 100755 product/base/scripts/ubuntu/apt.sh create mode 100755 product/base/scripts/ubuntu/install_drivers.sh create mode 100755 product/base/scripts/ubuntu/install_python.sh create mode 100755 product/base/scripts/ubuntu/install_quarto.sh create mode 100755 product/base/scripts/ubuntu/install_r.sh create mode 100644 product/pro/deps/r_packages.txt diff --git a/product/base/Dockerfile.centos7 b/product/base/Dockerfile.centos7 index f000c432..a0c51b31 100644 --- a/product/base/Dockerfile.centos7 +++ b/product/base/Dockerfile.centos7 @@ -8,30 +8,17 @@ ARG PYTHON_VERSION_ALT=3.8.15 ARG TINI_VERSION=0.19.0 ARG QUARTO_VERSION=1.3.340 +ARG SCRIPTS_DIR=/opt/positscripts + +COPY scripts/rhel/* ${SCRIPTS_DIR}/ + ### Update/upgrade system packages ### -RUN yum upgrade -y -q \ - && yum install -y -q \ - epel-release \ - && yum install -y -q \ - bzip2 \ - git \ - gpg \ - gpg-agent \ - libcurl-devel \ - libuser-devel \ - libxml2-devel \ - openssl-devel \ - openssh-clients \ - pandoc \ - perl-Digest-MD5 \ - postgresql-libs \ - rrdtool \ - sudo \ - unixODBC \ - unixODBC-devel \ - wget \ - which \ - && yum clean all +COPY deps/centos7_packages.txt /tmp/packages.txt +# hadolint ignore=SC2046 +RUN ${SCRIPTS_DIR}/yum.sh --update upgrade \ + && ${SCRIPTS_DIR}/yum.sh install epel-release \ + && ${SCRIPTS_DIR}/yum.sh install $(cat /tmp/packages.txt) \ + && ${SCRIPTS_DIR}/yum.sh --clean ### Install tini ### ADD https://cdn.rstudio.com/platform/tini/v${TINI_VERSION}/tini-amd64 /tini @@ -50,37 +37,22 @@ RUN curl -sL "https://yihui.org/tinytex/install-bin-unix.sh" | sh \ && /opt/TinyTeX/bin/*/tlmgr path add ### Install Quarto ### -RUN curl -o quarto.tar.gz -L https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz \ - && mkdir -p /opt/quarto/${QUARTO_VERSION} \ - && tar -zxvf quarto.tar.gz -C "/opt/quarto/${QUARTO_VERSION}" --strip-components=1 \ - && rm -f quarto.tar.gz \ +RUN QUARTO_VERSION=${QUARTO_VERSION} ${SCRIPTS_DIR}/install_quarto.sh \ && ln -s /opt/quarto/${QUARTO_VERSION}/bin/quarto /usr/local/bin/quarto ### Install R versions ### -RUN curl -O https://cdn.rstudio.com/r/centos-7/pkgs/R-${R_VERSION}-1-1.x86_64.rpm \ - && curl -O https://cdn.rstudio.com/r/centos-7/pkgs/R-${R_VERSION_ALT}-1-1.x86_64.rpm \ - && yum install -y R-${R_VERSION}-1-1.x86_64.rpm \ - && yum install -y R-${R_VERSION_ALT}-1-1.x86_64.rpm \ - && yum clean all \ - && rm -rf R-${R_VERSION}-1-1.x86_64.rpm \ - && rm -rf R-${R_VERSION_ALT}-1-1.x86_64.rpm \ +RUN R_VERSION=${R_VERSION} ${SCRIPTS_DIR}/install_r.sh \ + && R_VERSION=${R_VERSION_ALT} ${SCRIPTS_DIR}/install_r.sh \ && ln -s /opt/R/${R_VERSION} /opt/R/default \ && ln -s /opt/R/default/bin/R /usr/local/bin/R \ && ln -s /opt/R/default/bin/Rscript /usr/local/bin/Rscript ### Install Python versions ### -RUN curl -O https://cdn.rstudio.com/python/centos-7/pkgs/python-${PYTHON_VERSION}-1-1.x86_64.rpm \ - && curl -O https://cdn.rstudio.com/python/centos-7/pkgs/python-${PYTHON_VERSION_ALT}-1-1.x86_64.rpm \ - && yum install -y python-${PYTHON_VERSION}-1-1.x86_64.rpm \ - && yum install -y python-${PYTHON_VERSION_ALT}-1-1.x86_64.rpm \ - && yum clean all \ - && rm -rf python-${PYTHON_VERSION}-1-1.x86_64.rpm \ - && rm -rf python-${PYTHON_VERSION_ALT}-1-1.x86_64.rpm \ - && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install 'virtualenv<20' \ - && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --upgrade setuptools \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install 'virtualenv<20' \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --upgrade setuptools \ - && ln -s /opt/python/${PYTHON_VERSION} /opt/python/default +COPY deps/requirements.txt /tmp/requirements.txt +RUN PYTHON_VERSION=${PYTHON_VERSION} ${SCRIPTS_DIR}/install_python.sh -r /tmp/requirements.txt \ + && PYTHON_VERSION=${PYTHON_VERSION_ALT} ${SCRIPTS_DIR}/install_python.sh -r /tmp/requirements.txt \ + && ln -s /opt/python/${PYTHON_VERSION} /opt/python/default \ + && rm -f /tmp/requirements.txt ### Locale configuration ### RUN localedef -i en_US -f UTF-8 en_US.UTF-8 @@ -88,7 +60,6 @@ ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 - LABEL posit.r.version="${R_VERSION}" \ posit.r.version_alt="${R_VERSION_ALT}" \ posit.python.version="${PYTHON_VERSION}" \ diff --git a/product/base/Dockerfile.ubuntu2204 b/product/base/Dockerfile.ubuntu2204 index e848521f..8f21bc19 100644 --- a/product/base/Dockerfile.ubuntu2204 +++ b/product/base/Dockerfile.ubuntu2204 @@ -9,74 +9,18 @@ ARG PYTHON_VERSION_ALT=3.8.17 ARG TINI_VERSION=0.19.0 ARG QUARTO_VERSION=1.3.340 +ARG SCRIPTS_DIR=/opt/positscripts + +COPY scripts/ubuntu/* ${SCRIPTS_DIR}/ + ### Update/upgrade system packages ### -RUN apt-get update --fix-missing \ - && apt-get upgrade -yq \ - && apt-get install -yq --no-install-recommends \ - apt-transport-https \ - ca-certificates \ - cmake \ - cracklib-runtime \ - curl \ - default-jdk \ - dirmngr \ - dpkg-sig \ - g++ \ - gcc \ - gdal-bin \ - gfortran \ - git \ - gpg \ - gpg-agent \ - gsfonts \ - imagemagick \ - libarchive-dev \ - libcairo2-dev \ - libcurl4-openssl-dev \ - libev-dev \ - libfontconfig1-dev \ - libfreetype6-dev \ - libfribidi-dev \ - libgdal-dev \ - libgeos-dev \ - libgl1-mesa-dev \ - libglpk-dev \ - libglu1-mesa-dev \ - libgmp3-dev \ - libharfbuzz-dev \ - libicu-dev \ - libjpeg-dev \ - libmagick++-dev \ - libmysqlclient-dev \ - libopenblas-dev \ - libpaper-utils \ - libpcre2-dev \ - libpng-dev \ - libproj-dev \ - libsodium-dev \ - libssh2-1-dev \ - libssl-dev \ - libtiff-dev \ - libudunits2-dev \ - libv8-dev \ - libxml2-dev \ - locales \ - make \ - openssh-client \ - pandoc \ - perl \ - sudo \ - tcl \ - tk \ - tk-dev \ - tk-table \ - tzdata \ - unixodbc-dev \ - unzip \ - wget \ - zip \ - zlib1g-dev \ - && rm -rf /var/lib/apt/lists/* +COPY deps/ubuntu2204_packages.txt /tmp/apt_packages.txt +# hadolint ignore=SC2046 +RUN ${SCRIPTS_DIR}/apt.sh --update install lsof \ + && ${SCRIPTS_DIR}/apt.sh --update upgrade \ + && ${SCRIPTS_DIR}/apt.sh install $(cat /tmp/apt_packages.txt) \ + && ${SCRIPTS_DIR}/apt.sh --clean \ + && rm -f /tmp/apt_packages.txt ### Install tini ### ADD https://cdn.rstudio.com/platform/tini/v${TINI_VERSION}/tini-amd64 /tini @@ -95,35 +39,22 @@ RUN curl -sL "https://yihui.org/tinytex/install-bin-unix.sh" | sh \ && /opt/TinyTeX/bin/*/tlmgr path add ### Install Quarto ### -RUN curl -o quarto.tar.gz -L https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz \ - && mkdir -p /opt/quarto/${QUARTO_VERSION} \ - && tar -zxvf quarto.tar.gz -C "/opt/quarto/${QUARTO_VERSION}" --strip-components=1 \ - && rm -f quarto.tar.gz \ +RUN QUARTO_VERSION=${QUARTO_VERSION} ${SCRIPTS_DIR}/install_quarto.sh \ && ln -s /opt/quarto/${QUARTO_VERSION}/bin/quarto /usr/local/bin/quarto ### Install R versions ### -RUN curl -O https://cdn.rstudio.com/r/ubuntu-2204/pkgs/r-${R_VERSION}_1_amd64.deb \ - && curl -O https://cdn.rstudio.com/r/ubuntu-2204/pkgs/r-${R_VERSION_ALT}_1_amd64.deb \ - && apt-get install -yq --no-install-recommends ./r-${R_VERSION}_1_amd64.deb \ - && apt-get install -yq --no-install-recommends ./r-${R_VERSION_ALT}_1_amd64.deb \ - && rm -f ./r-${R_VERSION}_1_amd64.deb \ - && rm -f ./r-${R_VERSION_ALT}_1_amd64.deb \ +RUN R_VERSION=${R_VERSION} ${SCRIPTS_DIR}/install_r.sh \ + && R_VERSION=${R_VERSION_ALT} ${SCRIPTS_DIR}/install_r.sh \ && ln -s /opt/R/${R_VERSION} /opt/R/default \ && ln -s /opt/R/default/bin/R /usr/local/bin/R \ && ln -s /opt/R/default/bin/Rscript /usr/local/bin/Rscript ### Install Python versions ### -RUN curl -O https://cdn.rstudio.com/python/ubuntu-2204/pkgs/python-${PYTHON_VERSION}_1_amd64.deb \ - && curl -O https://cdn.rstudio.com/python/ubuntu-2204/pkgs/python-${PYTHON_VERSION_ALT}_1_amd64.deb \ - && apt-get install -yq --no-install-recommends ./python-${PYTHON_VERSION}_1_amd64.deb \ - && apt-get install -yq --no-install-recommends ./python-${PYTHON_VERSION_ALT}_1_amd64.deb \ - && rm -rf python-${PYTHON_VERSION}_1_amd64.deb \ - && rm -rf python-${PYTHON_VERSION_ALT}_1_amd64.deb \ - && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install 'virtualenv<20' \ - && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --upgrade setuptools \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install 'virtualenv<20' \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --upgrade setuptools \ - && ln -s /opt/python/${PYTHON_VERSION} /opt/python/default +COPY deps/requirements.txt /tmp/requirements.txt +RUN PYTHON_VERSION=${PYTHON_VERSION} ${SCRIPTS_DIR}/install_python.sh -r /tmp/requirements.txt \ + && PYTHON_VERSION=${PYTHON_VERSION_ALT} ${SCRIPTS_DIR}/install_python.sh -r /tmp/requirements.txt \ + && ln -s /opt/python/${PYTHON_VERSION} /opt/python/default \ + && rm -f /tmp/requirements.txt ### Locale configuration ### RUN localedef -i en_US -f UTF-8 en_US.UTF-8 @@ -132,13 +63,6 @@ ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 ENV TZ=UTC -### Clean up ### -RUN apt-get install -yqf --no-install-recommends \ - && apt-get autoremove \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - - LABEL posit.r.version="${R_VERSION}" \ posit.r.version_alt="${R_VERSION_ALT}" \ posit.python.version="${PYTHON_VERSION}" \ diff --git a/product/base/deps/centos7_packages.txt b/product/base/deps/centos7_packages.txt new file mode 100644 index 00000000..cd91ab88 --- /dev/null +++ b/product/base/deps/centos7_packages.txt @@ -0,0 +1,20 @@ +bzip2 +curl +git +gpg +gpg-agent +libcurl-devel +libuser-devel +libxml2-devel +openssl-devel +openssh-clients +pandoc +perl-Digest-MD5 +postgresql-libs +redhat-lsb-core +rrdtool +sudo +unixODBC +unixODBC-devel +wget +which \ No newline at end of file diff --git a/product/base/deps/requirements.txt b/product/base/deps/requirements.txt new file mode 100644 index 00000000..0df1a18f --- /dev/null +++ b/product/base/deps/requirements.txt @@ -0,0 +1,2 @@ +virtualenv<20 +setuptools diff --git a/product/base/deps/ubuntu2204_packages.txt b/product/base/deps/ubuntu2204_packages.txt new file mode 100644 index 00000000..3a836e5c --- /dev/null +++ b/product/base/deps/ubuntu2204_packages.txt @@ -0,0 +1,66 @@ +apt-transport-https +build-essential +ca-certificates +cmake +cracklib-runtime +curl +default-jdk +dirmngr +dpkg-sig +g++ +gcc +gdal-bin +gfortran +git +gnupg2 +gpg +gpg-agent +gsfonts +imagemagick +libarchive-dev +libcairo2-dev +libcurl4-openssl-dev +libev-dev +libfontconfig1-dev +libfreetype6-dev +libfribidi-dev +libgdal-dev +libgeos-dev +libgl1-mesa-dev +libglpk-dev +libglu1-mesa-dev +libgmp3-dev +libharfbuzz-dev +libicu-dev +libjpeg-dev +libmagick++-dev +libmysqlclient-dev +libopenblas-dev +libpaper-utils +libpcre2-dev +libpng-dev +libproj-dev +libsodium-dev +libssh2-1-dev +libssl-dev +libtiff-dev +libudunits2-dev +libv8-dev +libxml2-dev +locales +lsb-release +make +openssh-client +pandoc +perl +sudo +tcl +tk +tk-dev +tk-table +tzdata +unixodbc-dev +unzip +wget +zip +zlib1g-dev \ No newline at end of file diff --git a/product/base/scripts/rhel/install_drivers.sh b/product/base/scripts/rhel/install_drivers.sh new file mode 100755 index 00000000..40538b10 --- /dev/null +++ b/product/base/scripts/rhel/install_drivers.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -exo pipefail +export DEBIAN_FRONTEND=noninteractive + +# Output delimiter +d="====" + +if [ -z "$DRIVERS_VERSION" ]; then + echo "$d No DRIVERS_VERSION specified $d" + exit 1 +fi + +echo "$d$d Installing Professional Drivers ${DRIVERS_VERSION} $d$d" + +drivers_url="https://cdn.rstudio.com/drivers/7C152C12/installer/rstudio-drivers-${DRIVERS_VERSION}.el.x86_64.rpm" +curl -sL "$drivers_url" -o "/tmp/rstudio-drivers_${DRIVERS_VERSION}.el.x86_64.rpm" + +yum install -y -q "/tmp/rstudio-drivers_${DRIVERS_VERSION}.el.x86_64.rpm" +cat /opt/rstudio-drivers/odbcinst.ini.sample > /etc/odbcinst.ini + +rm /tmp/rstudio-drivers_${DRIVERS_VERSION}.el.x86_64.rpm diff --git a/product/base/scripts/rhel/install_python.sh b/product/base/scripts/rhel/install_python.sh new file mode 100755 index 00000000..ecf32923 --- /dev/null +++ b/product/base/scripts/rhel/install_python.sh @@ -0,0 +1,118 @@ +#!/bin/bash +set -eo pipefail +export DEBIAN_FRONTEND=noninteractive + +# Output delimiter +d="====" + +usage() { + echo "Usage:" + echo " $0 [OPTIONS]" + echo "" + echo "Examples:" + echo " # Install Python version specified by PYTHON_VERSION environment variable" + echo " $0" + echo " # Install Python 3.10.4" + echo " PYTHON_VERSION=3.10.4 $0" + echo " # Install Python and packages listed in /tmp/pythonh_packages.txt" + echo " PYTHON_VERSION=3.11.1 $0 -r /tmp/r_packages.txt" + echo "" + echo "Options:" + echo " -d, --debug Enable debug output" + echo " -h, --help Print usage and exit" + echo " --prefix Install Python to a custom prefix" + echo " Each version of Python will have its own subdirectory" + echo " Default: /opt/python" + echo " -r, --requirement " + echo " Install python packages from a requirements file" +} + + +# Set defaults +if [ -z "${DISTRO}" ]; then + DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') +fi +if [ -z "${OS_VERSION}" ]; then + OS_VERSION=$(rpm -E %{rhel}) +fi +YUM_ARGS="-y -q" +PREFIX="/opt/python" + +OPTIONS=$(getopt -o hdr: --long help,debug,prefix:,requirement: -- "$@") +# shellcheck disable=SC2181 +if [[ $? -ne 0 ]]; then + exit 1; +fi + +eval set -- "$OPTIONS" +while true; do + case "$1" in + -d | --debug) + set -x + shift + ;; + -h | --help) + usage + shift + exit + ;; + --prefix) + PREFIX="$2" + shift 2 + ;; + -r | --requirement) + PYTHON_PKG_FILE="$2" + shift 2 + ;; + --) shift; + break + ;; + esac +done + +if [ -z "$PYTHON_VERSION" ]; then + usage + exit 1 +fi + +# Set python binary path +PYTHON_BIN="${PREFIX}/${PYTHON_VERSION}/bin/python" + +# Set yum options +YUM_ARGS="-y -q" + +install_python() { + # Check if Python is already installed + if $PYTHON_BIN --version | grep -qE "^Python ${PYTHON_VERSION}" ; then + echo "$d Python $PYTHON_VERSION is already installed in $PREFIX/$PYTHON_VERSION $d" + return + fi + + echo "$d$d Installing Python $PYTHON_VERSION to $PREFIX/$PYTHON_VERSION $d$d" + mkdir -p "$PREFIX" + + local python_url="https://cdn.rstudio.com/python/${DISTRO}-${OS_VERSION}/pkgs/python-${PYTHON_VERSION}-1-1.x86_64.rpm" + curl -sL "$python_url" -o "/tmp/python-${PYTHON_VERSION}.rpm" + + # shellcheck disable=SC2086 + yum install $YUM_ARGS "/tmp/python-${PYTHON_VERSION}.rpm" + rm "/tmp/python-${PYTHON_VERSION}.rpm" + # Upgrade pip to latest version + $PYTHON_BIN -m pip install -U pip +} + +install_python_packages() { + if [ ! -f "$PYTHON_PKG_FILE" ]; then + echo "$d Python package file $PYTHON_PKG_FILE does not exist $d" + exit 1 + fi + + echo "$d$d Installing python-${PYTHON_VERSION} packages from ${PYTHON_PKG_FILE} $d$d" + $PYTHON_BIN -m pip install -U pip + $PYTHON_BIN -m pip install -r "$PYTHON_PKG_FILE" +} + +install_python +if [ -n "$PYTHON_PKG_FILE" ]; then + install_python_packages +fi diff --git a/product/base/scripts/rhel/install_quarto.sh b/product/base/scripts/rhel/install_quarto.sh new file mode 100755 index 00000000..e6071b13 --- /dev/null +++ b/product/base/scripts/rhel/install_quarto.sh @@ -0,0 +1,75 @@ +#!/bin/bash +set -eo pipefail +export DEBIAN_FRONTEND=noninteractive + +# Output delimiter +d="====" + +usage() { + echo "Usage:" + echo " $0 [OPTIONS]" + echo "" + echo "Examples:" + echo " # Install Quarto version specified by QUARTO_VERSION environment variable" + echo " $0" + echo " # Install Quarto 1.3.340" + echo " QUARTO_VERSION=1.3.340 $0" + echo "" + echo "Options:" + echo " -d, --debug Enable debug output" + echo " -h, --help Print usage and exit" + echo " --prefix Install Quarto to a custom prefix" + echo " Each version of Quarto will have its own subdirectory" + echo " Default: /opt/quarto" +} + + +# Set defaults +PREFIX="/opt/quarto" + +OPTIONS=$(getopt -o hdr: --long help,debug,prefix: -- "$@") +# shellcheck disable=SC2181 +if [[ $? -ne 0 ]]; then + exit 1; +fi + +eval set -- "$OPTIONS" +while true; do + case "$1" in + -d | --debug) + set -x + shift + ;; + -h | --help) + usage + shift + exit + ;; + --prefix) + PREFIX="$2" + shift 2 + ;; + --) shift; + break + ;; + esac +done + +if [ -z "$QUARTO_VERSION" ]; then + usage + exit 1 +fi + +install_quarto() { + # Check if Quarto is already installed + # shellcheck disable=SC2086 + if ${PREFIX}/${QUARTO_VERSION}/bin/quarto --version | grep -qE "^${QUARTO_VERSION}" ; then + echo "$d Quarto $QUARTO_VERSION is already installed in $PREFIX/$QUARTO_VERSION $d" + return + fi + + mkdir -p "/opt/quarto/${QUARTO_VERSION}" + wget -q -O - "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz" | tar xzf - -C "/opt/quarto/${QUARTO_VERSION}" --strip-components=1 +} + +install_quarto diff --git a/product/base/scripts/rhel/install_r.sh b/product/base/scripts/rhel/install_r.sh new file mode 100755 index 00000000..b346ea12 --- /dev/null +++ b/product/base/scripts/rhel/install_r.sh @@ -0,0 +1,173 @@ +#!/bin/bash +set -eo pipefail +export DEBIAN_FRONTEND=noninteractive + +# Output delimiter +d="====" + +usage() { + echo "Usage:" + echo " $0 [OPTIONS]" + echo "" + echo "Examples:" + echo " # Install R version specified by R_VERSION environment variable" + echo " $0" + echo " # Install R 4.1.2" + echo " R_VERSION=4.1.2 $0" + echo " # Install R 4.2.3 and packages listed in /tmp/r_packages.txt" + echo " R_VERSION=4.2.3 $0 -r /tmp/r_packages.txt" + echo "" + echo "Options:" + echo " -d, --debug Enable debug output" + echo " -h, --help Print usage and exit" + echo " --prefix Install R to a custom prefix" + echo " Each version of R will have its own subdirectory" + echo " Default: /opt/R" + echo " --r-exists Expect R version to already be installed" + echo " -r, --requirement " + echo " Install R packages from a requirements file" + echo " --with-source Also download the R source code" +} + + +# Set defaults +if [ -z "${DISTRO}" ]; then + DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') +fi +if [ -z "${OS_VERSION}" ]; then + OS_VERSION=$(rpm -E %{rhel}) +fi +YUM_ARGS="-y -q" +PREFIX="/opt/R" +R_EXISTS=0 +WITH_SOURCE=0 + +OPTIONS=$(getopt -o hdr: --long help,debug,distro:,prefix:,r-exists,requirement:,with-source -- "$@") +# shellcheck disable=SC2181 +if [[ $? -ne 0 ]]; then + exit 1; +fi + +eval set -- "$OPTIONS" +while true; do + case "$1" in + -d | --debug) + set -x + shift + ;; + -h | --help) + usage + shift + exit + ;; + --prefix) + PREFIX="$2" + shift 2 + ;; + --r-exists) + R_EXISTS=1 + shift + ;; + -r | --requirement) + R_PKG_FILE="$2" + shift 2 + ;; + --with-source) + WITH_SOURCE=1 + shift + ;; + --) shift; + break + ;; + esac +done + +if [ -z "$R_VERSION" ]; then + usage + exit 1 +fi + +# Set R binary path +R_BIN="${PREFIX}/${R_VERSION}/bin/R" + +# Set yum options +YUM_ARGS="-y -q" + +install_r() { + echo "$d$d Installing R $R_VERSION to $PREFIX/$R_VERSION $d$d" + mkdir -p "$PREFIX" + + local r_url="https://cdn.rstudio.com/r/${DISTRO}-${OS_VERSION}/pkgs/R-${R_VERSION}-1-1.x86_64.rpm" + curl -sL "$r_url" -o "/tmp/r-${R_VERSION}.rpm" + + # shellcheck disable=SC2086 + yum install $YUM_ARGS "/tmp/r-${R_VERSION}.rpm" + rm "/tmp/r-${R_VERSION}.rpm" +} + +install_r_dependencies() { + # RHEL installs are more straightforward than Ubuntu. Installing epel-release should cover us well. + local r_deps="epel-release" + + # Check whether dependencies are already installed + # shellcheck disable=2086 + if rpm -q $r_deps >/dev/null 2>&1 ; then + echo "$d R dependencies already installed $d" + return + fi + + echo "$d$d Installng R depencencies $d$d" + + # Install R dependencies + # shellcheck disable=2086 + yum install $YUM_ARGS epel-release +} + +install_r_packages() { + if [ ! -f "$R_PKG_FILE" ]; then + echo "$d R package file $R_PKG_FILE does not exist $d" + exit 1 + fi + + echo "$d$d Installing R-${R_VERSION} packages from ${R_PKG_FILE} $d$d" + local cran_repo="https://packagemanager.rstudio.com/cran/__linux__/${DISTRO}${OS_VERSION}/latest" + + $R_BIN --vanilla --no-echo < /dev/null +install.packages(readLines("$R_PKG_FILE"), repos = "$cran_repo") +EOF + +} + +get_r_source() { + local r_prefix=${R_VERSION:0:1} + local r_source_dir="/opt/r-sources" + local r_source_url="https://cloud.r-project.org/src/base/R-${r_prefix}/R-${R_VERSION}.tar.gz" + + echo "$d Fetching R-${R_VERSION} source code into $r_source_dir $d" + mkdir -p "$r_source_dir" + + curl -sL "$r_source_url" -o "$r_source_dir/R-${R_VERSION}.tar.gz" +} + + +# Only add the dependencies if we don't expect R to exist +if [ "$R_EXISTS" -eq 0 ]; then + install_r_dependencies +fi + +# Check if R is already installed +if $R_BIN --version | grep -qE "^R version ${R_VERSION}" ; then + echo "$d R $R_VERSION is already installed in $PREFIX/$R_VERSION $d" +elif [ "$R_EXISTS" -eq 1 ]; then + echo "$d R $R_VERSION is not installed in $PREFIX/$R_VERSION $d" + exit 1 +else + install_r +fi + +if [ -n "$R_PKG_FILE" ]; then + install_r_packages +fi +if [ "$WITH_SOURCE" -eq 1 ]; then + get_r_source +fi diff --git a/product/base/scripts/rhel/yum.sh b/product/base/scripts/rhel/yum.sh new file mode 100755 index 00000000..af70f94f --- /dev/null +++ b/product/base/scripts/rhel/yum.sh @@ -0,0 +1,112 @@ +#!/bin/bash +set -eo pipefail + +# Output delimiter +d="====" + +usage() { + echo "Usage:" + echo " $0 [OPTIONS] [COMMAND [ARG...]]" + echo "" + echo "Options:" + echo " -d, --debug Enable debug output" + echo " -h, --help Print usage and exit" + echo " --clean Clean yum cache (yum clean all)" + echo " --update Update yum cache (yum check-update)" + echo "" + echo "Commands:" + echo " install Install packages (yum install)" + echo " upgrade Upgrade all packages (yum upgrade)" +} + +# Set defaults +YUM_ARGS="-y -q" +CLEAN=0 +UPDATE=0 + +OPTIONS=$(getopt -o hd --long debug,help,clean,update -- "$@") +if [[ $? -ne 0 ]]; then + exit 1; +fi + +eval set -- "$OPTIONS" +while true; do + case "$1" in + -d | --debug) + set -x + shift + ;; + -h | --help) + usage + shift + exit + ;; + --clean) + CLEAN=1 + shift + ;; + --update) + UPDATE=1 + shift + ;; + --) + shift + break + ;; + *) + echo "Unexpected option: $1" + exit 1 + ;; + esac +done + +# Wait for yum to be done before continuing +# The Deep learning AMI runs an extremely long yum update-install on first boot. +echo "$d Waiting for yum to finish $d" + +# lsof is not installed by default in all images +# We install lsof if it is not already installed, but we don't want the check +# to fail when we invoke this script to install lsof. Chicken & egg problem. +if command -v lsof >/dev/null; then + lsof_result="\$(lsof /var/run/yum.pid)" +else + lsof_result="" +fi +sleep 10 +while [ -n "$lsof_result" ] && [ "$(lslocks | grep "yum")" != "" ]; do + sleep 10 +done + +# Clean yum cache +if [ "$CLEAN" -eq 1 ]; then + echo "$d Cleaning yum cache $d" + yum clean all $YUM_ARGS +fi + +# Update yum cache +if [ "$UPDATE" -eq 1 ]; then + echo "$d Updating yum cache $d" + yum update $YUM_ARGS +fi + +case "$1" in + install) + shift + echo "$d$d Installing yum packages $d$d" + # shellcheck disable=SC2086 + yum install $YUM_ARGS "$@" + ;; + upgrade) + echo "$d$d Upgrading yum packages $d$d" + # shellcheck disable=SC2086 + yum upgrade $YUM_ARGS + ;; + *) + # Allow clean, update to be used as commands + if [ "$UPDATE" -eq 1 ] || [ "$CLEAN" -eq 1 ]; then + exit 0 + fi + usage + exit 1 + ;; +esac diff --git a/product/base/scripts/ubuntu/apt.sh b/product/base/scripts/ubuntu/apt.sh new file mode 100755 index 00000000..e67877d6 --- /dev/null +++ b/product/base/scripts/ubuntu/apt.sh @@ -0,0 +1,118 @@ +#!/bin/bash +set -eo pipefail +export DEBIAN_FRONTEND=noninteractive + +# Output delimiter +d="====" + +usage() { + echo "Usage:" + echo " $0 [OPTIONS] [COMMAND [ARG...]]" + echo "" + echo "Options:" + echo " -d, --debug Enable debug output" + echo " -h, --help Print usage and exit" + echo " --clean Clean apt cache (apt-get clean)" + echo " --update Update apt cache (apt-get update)" + echo "" + echo "Commands:" + echo " install Install packages (apt-get install)" + echo " upgrade Upgrade all packages (apt-get upgrade)" +} + +# Set defaults +APT_ARGS="-o DPkg::Lock::Timeout=60 -y -qq" +CLEAN=0 +UPDATE=0 + +OPTIONS=$(getopt -o hd --long debug,help,clean,update -- "$@") +if [[ $? -ne 0 ]]; then + exit 1; +fi + +eval set -- "$OPTIONS" +while true; do + case "$1" in + -d | --debug) + set -x + shift + ;; + -h | --help) + usage + shift + exit + ;; + --clean) + CLEAN=1 + shift + ;; + --update) + UPDATE=1 + shift + ;; + --) + shift + break + ;; + *) + echo "Unexpected option: $1" + exit 1 + ;; + esac +done + +# Wait for apt to be done before continuing +# The Deep learning AMI runs an extremely long apt update-install on first boot. +echo "$d Waiting for apt to finish $d" + +# lsof is not installed by default in all images +# We install lsof if it is not already installed, but we don't want the check +# to fail when we invoke this script to install lsof. Chicken & egg problem. +if command -v lsof >/dev/null; then + lsof_result="\$(lsof /var/lib/apt/lists/lock)" +else + lsof_result="" +fi +sleep 10 +while [ -n "$lsof_result" ] && [ "$(lslocks | grep "apt")" != "" ]; do + sleep 10 +done + +# Clean apt cache +if [ "$CLEAN" -eq 1 ]; then + echo "$d Cleaning apt cache $d" + # shellcheck disable=SC2086 + apt-get clean $APT_ARGS + rm -rf /var/lib/apt/lists/* +fi + +# Update apt cache +if [ "$UPDATE" -eq 1 ]; then + echo "$d Updating apt cache $d" + # shellcheck disable=SC2086 + apt-get update --fix-missing $APT_ARGS +fi + +case "$1" in + install) + shift + echo "$d$d Installing apt packages $d$d" + # shellcheck disable=SC2086 + apt-get install $APT_ARGS "$@" + ;; + upgrade) + echo "$d$d Upgrading apt packages $d$d" + # shellcheck disable=SC2086 + apt-get upgrade $APT_ARGS + # shellcheck disable=SC2086 + apt-get dist-upgrade $APT_ARGS + ;; + *) + # Allow clean, update to be used as commands + if [ "$UPDATE" -eq 1 ] || [ "$CLEAN" -eq 1 ]; then + exit 0 + fi + usage + exit 1 + ;; +esac diff --git a/product/base/scripts/ubuntu/install_drivers.sh b/product/base/scripts/ubuntu/install_drivers.sh new file mode 100755 index 00000000..68d332d9 --- /dev/null +++ b/product/base/scripts/ubuntu/install_drivers.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -eo pipefail +export DEBIAN_FRONTEND=noninteractive + +# Output delimiter +d="====" + +if [ -z "$DRIVERS_VERSION" ]; then + echo "$d No DRIVERS_VERSION specified $d" + exit 1 +fi + +echo "$d$d Installing Professional Drivers ${DRIVERS_VERSION} $d$d" + +drivers_url="https://cdn.rstudio.com/drivers/7C152C12/installer/rstudio-drivers_${DRIVERS_VERSION}_amd64.deb" +curl -sL "$drivers_url" -o "/tmp/rstudio-drivers_${DRIVERS_VERSION}_amd64.deb" + +apt-get install -y -qq "/tmp/rstudio-drivers_${DRIVERS_VERSION}_amd64.deb" +cat /opt/rstudio-drivers/odbcinst.ini.sample > /etc/odbcinst.ini + +rm /tmp/rstudio-drivers_${DRIVERS_VERSION}_amd64.deb diff --git a/product/base/scripts/ubuntu/install_python.sh b/product/base/scripts/ubuntu/install_python.sh new file mode 100755 index 00000000..06b6e12c --- /dev/null +++ b/product/base/scripts/ubuntu/install_python.sh @@ -0,0 +1,115 @@ +#!/bin/bash +set -eo pipefail +export DEBIAN_FRONTEND=noninteractive + +# Output delimiter +d="====" + +usage() { + echo "Usage:" + echo " $0 [OPTIONS]" + echo "" + echo "Examples:" + echo " # Install Python version specified by PYTHON_VERSION environment variable" + echo " $0" + echo " # Install Python 3.10.4" + echo " PYTHON_VERSION=3.10.4 $0" + echo " # Install Python and packages listed in /tmp/pythonh_packages.txt" + echo " PYTHON_VERSION=3.11.1 $0 -r /tmp/r_packages.txt" + echo "" + echo "Options:" + echo " -d, --debug Enable debug output" + echo " -h, --help Print usage and exit" + echo " --prefix Install Python to a custom prefix" + echo " Each version of Python will have its own subdirectory" + echo " Default: /opt/python" + echo " -r, --requirement " + echo " Install python packages from a requirements file" +} + + +# Set defaults +APT_ARGS="-o DPkg::Lock::Timeout=60 -y -qq" +PREFIX="/opt/python" + +OPTIONS=$(getopt -o hdr: --long help,debug,prefix:,requirement: -- "$@") +# shellcheck disable=SC2181 +if [[ $? -ne 0 ]]; then + exit 1; +fi + +eval set -- "$OPTIONS" +while true; do + case "$1" in + -d | --debug) + set -x + shift + ;; + -h | --help) + usage + shift + exit + ;; + --prefix) + PREFIX="$2" + shift 2 + ;; + -r | --requirement) + PYTHON_PKG_FILE="$2" + shift 2 + ;; + --) shift; + break + ;; + esac +done + +if [ -z "$PYTHON_VERSION" ]; then + usage + exit 1 +fi + +# Set python binary path +PYTHON_BIN="${PREFIX}/${PYTHON_VERSION}/bin/python" + +# Set apt options +APT_ARGS="-o DPkg::Lock::Timeout=60 -y -qq" + +# Set ubuntu version +UBUNTU_VERSION=$(lsb_release -rs) + +install_python() { + # Check if Python is already installed + if $PYTHON_BIN --version | grep -qE "^Python ${PYTHON_VERSION}" ; then + echo "$d Python $PYTHON_VERSION is already installed in $PREFIX/$PYTHON_VERSION $d" + return + fi + + echo "$d$d Installing Python $PYTHON_VERSION to $PREFIX/$PYTHON_VERSION $d$d" + mkdir -p "$PREFIX" + + local python_url="https://cdn.rstudio.com/python/ubuntu-${UBUNTU_VERSION//./}/pkgs/python-${PYTHON_VERSION}_1_amd64.deb" + curl -sL "$python_url" -o "/tmp/python-${PYTHON_VERSION}.deb" + + # shellcheck disable=SC2086 + apt-get install $APT_ARGS "/tmp/python-${PYTHON_VERSION}.deb" + rm "/tmp/python-${PYTHON_VERSION}.deb" + # Upgrade pip to latest version + $PYTHON_BIN -m pip install -U pip +} + +install_python_packages() { + if [ ! -f "$PYTHON_PKG_FILE" ]; then + echo "$d Python package file $PYTHON_PKG_FILE does not exist $d" + exit 1 + fi + + echo "$d$d Installing python-${PYTHON_VERSION} packages from ${PYTHON_PKG_FILE} $d$d" + $PYTHON_BIN -m pip install -U pip + $PYTHON_BIN -m pip install -r "$PYTHON_PKG_FILE" +} + +install_python +if [ -n "$PYTHON_PKG_FILE" ]; then + install_python_packages +fi diff --git a/product/base/scripts/ubuntu/install_quarto.sh b/product/base/scripts/ubuntu/install_quarto.sh new file mode 100755 index 00000000..e6071b13 --- /dev/null +++ b/product/base/scripts/ubuntu/install_quarto.sh @@ -0,0 +1,75 @@ +#!/bin/bash +set -eo pipefail +export DEBIAN_FRONTEND=noninteractive + +# Output delimiter +d="====" + +usage() { + echo "Usage:" + echo " $0 [OPTIONS]" + echo "" + echo "Examples:" + echo " # Install Quarto version specified by QUARTO_VERSION environment variable" + echo " $0" + echo " # Install Quarto 1.3.340" + echo " QUARTO_VERSION=1.3.340 $0" + echo "" + echo "Options:" + echo " -d, --debug Enable debug output" + echo " -h, --help Print usage and exit" + echo " --prefix Install Quarto to a custom prefix" + echo " Each version of Quarto will have its own subdirectory" + echo " Default: /opt/quarto" +} + + +# Set defaults +PREFIX="/opt/quarto" + +OPTIONS=$(getopt -o hdr: --long help,debug,prefix: -- "$@") +# shellcheck disable=SC2181 +if [[ $? -ne 0 ]]; then + exit 1; +fi + +eval set -- "$OPTIONS" +while true; do + case "$1" in + -d | --debug) + set -x + shift + ;; + -h | --help) + usage + shift + exit + ;; + --prefix) + PREFIX="$2" + shift 2 + ;; + --) shift; + break + ;; + esac +done + +if [ -z "$QUARTO_VERSION" ]; then + usage + exit 1 +fi + +install_quarto() { + # Check if Quarto is already installed + # shellcheck disable=SC2086 + if ${PREFIX}/${QUARTO_VERSION}/bin/quarto --version | grep -qE "^${QUARTO_VERSION}" ; then + echo "$d Quarto $QUARTO_VERSION is already installed in $PREFIX/$QUARTO_VERSION $d" + return + fi + + mkdir -p "/opt/quarto/${QUARTO_VERSION}" + wget -q -O - "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz" | tar xzf - -C "/opt/quarto/${QUARTO_VERSION}" --strip-components=1 +} + +install_quarto diff --git a/product/base/scripts/ubuntu/install_r.sh b/product/base/scripts/ubuntu/install_r.sh new file mode 100755 index 00000000..67914952 --- /dev/null +++ b/product/base/scripts/ubuntu/install_r.sh @@ -0,0 +1,208 @@ +#!/bin/bash +set -eo pipefail +export DEBIAN_FRONTEND=noninteractive + +# Output delimiter +d="====" + +usage() { + echo "Usage:" + echo " $0 [OPTIONS]" + echo "" + echo "Examples:" + echo " # Install R version specified by R_VERSION environment variable" + echo " $0" + echo " # Install R 4.1.2" + echo " R_VERSION=4.1.2 $0" + echo " # Install R 4.2.3 and packages listed in /tmp/r_packages.txt" + echo " R_VERSION=4.2.3 $0 -r /tmp/r_packages.txt" + echo "" + echo "Options:" + echo " -d, --debug Enable debug output" + echo " -h, --help Print usage and exit" + echo " --prefix Install R to a custom prefix" + echo " Each version of R will have its own subdirectory" + echo " Default: /opt/R" + echo " --r-exists Expect R version to already be installed" + echo " -r, --requirement " + echo " Install R packages from a requirements file" + echo " --with-source Also download the R source code" +} + + +# Set defaults +APT_ARGS="-o DPkg::Lock::Timeout=60 -y -qq" +PREFIX="/opt/R" +R_EXISTS=0 +WITH_SOURCE=0 + +OPTIONS=$(getopt -o hdr: --long help,debug,distro:,prefix:,r-exists,requirement:,with-source -- "$@") +# shellcheck disable=SC2181 +if [[ $? -ne 0 ]]; then + exit 1; +fi + +eval set -- "$OPTIONS" +while true; do + case "$1" in + -d | --debug) + set -x + shift + ;; + -h | --help) + usage + shift + exit + ;; + --prefix) + PREFIX="$2" + shift 2 + ;; + --r-exists) + R_EXISTS=1 + shift + ;; + -r | --requirement) + R_PKG_FILE="$2" + shift 2 + ;; + --with-source) + WITH_SOURCE=1 + shift + ;; + --) shift; + break + ;; + esac +done + +if [ -z "$R_VERSION" ]; then + usage + exit 1 +fi + +# Set R binary path +R_BIN="${PREFIX}/${R_VERSION}/bin/R" + +# Set apt options +APT_ARGS="-o DPkg::Lock::Timeout=60 -y -qq" +APT_KEY="0x51716619e084dab9" +APT_KEY_FILE="/usr/share/keyrings/cran-rstudio-keyring.gpg" +APT_FILE="/etc/apt/sources.list.d/cran-rstudio.list" + +UBUNTU_CODENAME=$(lsb_release -cs) +UBUNTU_VERSION=$(lsb_release -rs) + +add_cran_apt_source() { + # Ensure we have gnupg2 & dirmngr installed + # shellcheck disable=SC2086 + apt-get install $APT_ARGS --install-recommends gnupg2 dirmngr + # Create ~/.gnupg and start dirmngr in the background + gpg --list-keys >/dev/null 2>&1 + eval "$(dirmngr --daemon)" + + # Add the apt-key to the keyring + gpg --no-default-keyring --keyring $APT_KEY_FILE --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys $APT_KEY + kill "$(echo "$DIRMNGR_INFO" | cut -d: -f2)" + + echo "deb [signed-by=${APT_KEY_FILE}] https://cran.rstudio.com/bin/linux/ubuntu ${UBUNTU_CODENAME}-cran40/" >> $APT_FILE + echo "deb-src [signed-by=${APT_KEY_FILE}] https://cran.rstudio.com/bin/linux/ubuntu ${UBUNTU_CODENAME}-cran40/" >> $APT_FILE + + # shellcheck disable=SC2086 + apt-get update $APT_ARGS +} + +remove_cran_apt_source() { + rm -f $APT_FILE $APT_KEY_FILE + # shellcheck disable=SC2086 + apt-get update $APT_ARGS +} + +install_r() { + echo "$d$d Installing R $R_VERSION to $PREFIX/$R_VERSION $d$d" + mkdir -p "$PREFIX" + + local r_url="https://cdn.rstudio.com/r/ubuntu-${UBUNTU_VERSION//./}/pkgs/r-${R_VERSION}_1_amd64.deb" + curl -sL "$r_url" -o "/tmp/r-${R_VERSION}.deb" + + # shellcheck disable=SC2086 + apt-get install $APT_ARGS "/tmp/r-${R_VERSION}.deb" + rm "/tmp/r-${R_VERSION}.deb" +} + +install_r_dependencies() { + # There are many dependencies that R users may rely on that we want to + # include in the images. These include things like a functional X server, + # fonts, and other libraries that are commonly used by R packages. + local r_deps="r-base-core r-base-dev" + + # Check whether dependencies are already installed + # shellcheck disable=2086 + if dpkg -s $r_deps >/dev/null 2>&1 ; then + echo "$d R dependencies already installed $d" + return + fi + + echo "$d$d Installng R depencencies $d$d" + # Ensure we have apt-transport-https installed + # shellcheck disable=SC2086 + apt-get install $APT_ARGS apt-transport-https + + # Install R dependencies + # shellcheck disable=2086 + apt-get install $APT_ARGS $r_deps +} + +install_r_packages() { + if [ ! -f "$R_PKG_FILE" ]; then + echo "$d R package file $R_PKG_FILE does not exist $d" + exit 1 + fi + + echo "$d$d Installing R-${R_VERSION} packages from ${R_PKG_FILE} $d$d" + local cran_repo="https://packagemanager.rstudio.com/cran/__linux__/${UBUNTU_CODENAME}/latest" + + $R_BIN --vanilla --no-echo < /dev/null +install.packages(readLines("$R_PKG_FILE"), repos = "$cran_repo") +EOF + +} + +get_r_source() { + local r_prefix=${R_VERSION:0:1} + local r_source_dir="/opt/r-sources" + local r_source_url="https://cloud.r-project.org/src/base/R-${r_prefix}/R-${R_VERSION}.tar.gz" + + echo "$d Fetching R-${R_VERSION} source code into $r_source_dir $d" + mkdir -p "$r_source_dir" + + curl -sL "$r_source_url" -o "$r_source_dir/R-${R_VERSION}.tar.gz" +} + + +# Only add the CRAN apt source & dependencies if we don't expect R to exist +if [ "$R_EXISTS" -eq 0 ]; then + add_cran_apt_source + install_r_dependencies +fi + +# Check if R is already installed +if $R_BIN --version | grep -qE "^R version ${R_VERSION}" ; then + echo "$d R $R_VERSION is already installed in $PREFIX/$R_VERSION $d" +elif [ "$R_EXISTS" -eq 1 ]; then + echo "$d R $R_VERSION is not installed in $PREFIX/$R_VERSION $d" + exit 1 +else + install_r +fi + +if [ -n "$R_PKG_FILE" ]; then + install_r_packages +fi +if [ "$WITH_SOURCE" -eq 1 ]; then + get_r_source +fi + +if [ "$R_EXISTS" -eq 0 ]; then + remove_cran_apt_source +fi diff --git a/product/base/test/goss.yaml b/product/base/test/goss.yaml index 18e338a5..e321ebe0 100644 --- a/product/base/test/goss.yaml +++ b/product/base/test/goss.yaml @@ -15,6 +15,28 @@ package: {{end}} file: + {{if .Env.OS | regexMatch "ubuntu.*"}} + /opt/positscripts/apt.sh: + exists: true + mode: "0755" + {{end}} + {{if .Env.OS | regexMatch "centos.*"}} + /opt/positscripts/yum.sh: + exists: true + mode: "0755" + {{end}} + /opt/positscripts/install_drivers.sh: + exists: true + mode: "0755" + /opt/positscripts/install_python.sh: + exists: true + mode: "0755" + /opt/positscripts/install_quarto.sh: + exists: true + mode: "0755" + /opt/positscripts/install_r.sh: + exists: true + mode: "0755" /opt/R/{{.Env.R_VERSION}}/bin/R: exists: true /opt/R/{{.Env.R_VERSION_ALT}}/bin/R: diff --git a/product/pro/Dockerfile.centos7 b/product/pro/Dockerfile.centos7 index 936611e8..254456dc 100644 --- a/product/pro/Dockerfile.centos7 +++ b/product/pro/Dockerfile.centos7 @@ -6,14 +6,15 @@ ARG PYTHON_VERSION=3.9.14 ARG PYTHON_VERSION_ALT=3.8.15 ARG DRIVERS_VERSION=2024.03.0-1 -RUN yum update -y \ - && yum install -y unixODBC unixODBC-devel \ - && curl -O https://cdn.rstudio.com/drivers/7C152C12/installer/rstudio-drivers-${DRIVERS_VERSION}.el.x86_64.rpm \ - && yum install -y ./rstudio-drivers-${DRIVERS_VERSION}.el.x86_64.rpm \ - && yum clean all \ - && rm -f rstudio-drivers-${DRIVERS_VERSION}.el.x86_64.rpm \ - && cp /opt/rstudio-drivers/odbcinst.ini.sample /etc/odbcinst.ini \ - && "/opt/R/${R_VERSION}/bin/R" -e 'install.packages("odbc", repos="https://packagemanager.rstudio.com/cran/__linux__/centos7/latest")' +ARG SCRIPTS_DIR=/opt/positscripts + +COPY deps/r_packages.txt /tmp/r_packages.txt +RUN ${SCRIPTS_DIR}/yum.sh --update upgrade \ + && ${SCRIPTS_DIR}/yum.sh install unixODBC unixODBC-devel \ + && DRIVERS_VERSION=${DRIVERS_VERSION} ${SCRIPTS_DIR}/install_drivers.sh \ + && ${SCRIPTS_DIR}/yum.sh --clean \ + && R_VERSION=${R_VERSION} ${SCRIPTS_DIR}/install_r.sh -r /tmp/r_packages.txt \ + && rm -f /tmp/r_packages.txt LABEL rstudio.pro-drivers.version="${DRIVERS_VERSION}" diff --git a/product/pro/Dockerfile.ubuntu2204 b/product/pro/Dockerfile.ubuntu2204 index f17dbfc2..7e36f856 100644 --- a/product/pro/Dockerfile.ubuntu2204 +++ b/product/pro/Dockerfile.ubuntu2204 @@ -7,16 +7,16 @@ ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG DRIVERS_VERSION=2024.03.0 -RUN apt-get update \ - && apt-get install -yq --no-install-recommends unixodbc unixodbc-dev \ - && curl -O https://cdn.rstudio.com/drivers/7C152C12/installer/rstudio-drivers_${DRIVERS_VERSION}_amd64.deb \ - && apt-get update \ - && apt-get install -yq --no-install-recommends ./rstudio-drivers_${DRIVERS_VERSION}_amd64.deb \ - && rm -f ./rstudio-drivers_${DRIVERS_VERSION}_amd64.deb \ - && rm -rf /var/lib/apt/lists/* \ - && cp /opt/rstudio-drivers/odbcinst.ini.sample /etc/odbcinst.ini \ - && /opt/R/${R_VERSION}/bin/R -e 'install.packages("odbc", repos="https://packagemanager.rstudio.com/cran/__linux__/jammy/latest")' +ARG SCRIPTS_DIR=/opt/positscripts -LABEL rstudio.pro-drivers.version="${DRIVERS_VERSION}" +COPY deps/r_packages.txt /tmp/r_packages.txt +RUN ${SCRIPTS_DIR}/apt.sh --update upgrade \ + && ${SCRIPTS_DIR}/apt.sh install unixodbc unixodbc-dev \ + && DRIVERS_VERSION=${DRIVERS_VERSION} ${SCRIPTS_DIR}/install_drivers.sh \ + && ${SCRIPTS_DIR}/apt.sh --clean \ + && R_VERSION=${R_VERSION} ${SCRIPTS_DIR}/install_r.sh -r /tmp/r_packages.txt \ + && rm -f /tmp/r_packages.txt + +LABEL posit.pro-drivers.version="${DRIVERS_VERSION}" ENTRYPOINT ["/tini", "--"] diff --git a/product/pro/deps/r_packages.txt b/product/pro/deps/r_packages.txt new file mode 100644 index 00000000..47869868 --- /dev/null +++ b/product/pro/deps/r_packages.txt @@ -0,0 +1 @@ +odbc \ No newline at end of file From c27cd0715b854cc309b8a5b4c71870c5590e8756 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 19 Apr 2024 12:15:23 -0700 Subject: [PATCH 7/9] Upgrade setuptools on Python install (#731) --- product/base/Dockerfile.centos7 | 2 +- product/base/Dockerfile.ubuntu2204 | 2 +- product/base/scripts/rhel/install_python.sh | 3 ++- product/base/scripts/ubuntu/install_python.sh | 3 ++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/product/base/Dockerfile.centos7 b/product/base/Dockerfile.centos7 index a0c51b31..5e667f12 100644 --- a/product/base/Dockerfile.centos7 +++ b/product/base/Dockerfile.centos7 @@ -10,7 +10,7 @@ ARG QUARTO_VERSION=1.3.340 ARG SCRIPTS_DIR=/opt/positscripts -COPY scripts/rhel/* ${SCRIPTS_DIR}/ +COPY --chmod=0755 scripts/rhel/* ${SCRIPTS_DIR}/ ### Update/upgrade system packages ### COPY deps/centos7_packages.txt /tmp/packages.txt diff --git a/product/base/Dockerfile.ubuntu2204 b/product/base/Dockerfile.ubuntu2204 index 8f21bc19..93c0b1bf 100644 --- a/product/base/Dockerfile.ubuntu2204 +++ b/product/base/Dockerfile.ubuntu2204 @@ -11,7 +11,7 @@ ARG QUARTO_VERSION=1.3.340 ARG SCRIPTS_DIR=/opt/positscripts -COPY scripts/ubuntu/* ${SCRIPTS_DIR}/ +COPY --chmod=0755 scripts/ubuntu/* ${SCRIPTS_DIR}/ ### Update/upgrade system packages ### COPY deps/ubuntu2204_packages.txt /tmp/apt_packages.txt diff --git a/product/base/scripts/rhel/install_python.sh b/product/base/scripts/rhel/install_python.sh index ecf32923..8ee26960 100755 --- a/product/base/scripts/rhel/install_python.sh +++ b/product/base/scripts/rhel/install_python.sh @@ -97,7 +97,8 @@ install_python() { # shellcheck disable=SC2086 yum install $YUM_ARGS "/tmp/python-${PYTHON_VERSION}.rpm" rm "/tmp/python-${PYTHON_VERSION}.rpm" - # Upgrade pip to latest version + # Upgrade pip and setuptools to latest version + $PYTHON_BIN -m pip install -U setuptools $PYTHON_BIN -m pip install -U pip } diff --git a/product/base/scripts/ubuntu/install_python.sh b/product/base/scripts/ubuntu/install_python.sh index 06b6e12c..cb84e4e0 100755 --- a/product/base/scripts/ubuntu/install_python.sh +++ b/product/base/scripts/ubuntu/install_python.sh @@ -94,7 +94,8 @@ install_python() { # shellcheck disable=SC2086 apt-get install $APT_ARGS "/tmp/python-${PYTHON_VERSION}.deb" rm "/tmp/python-${PYTHON_VERSION}.deb" - # Upgrade pip to latest version + # Upgrade pip and setuptools to latest version + $PYTHON_BIN -m pip install -U setuptools $PYTHON_BIN -m pip install -U pip } From 5fed7b9834814f77aa8210a6596d00720062f120 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 19 Apr 2024 12:50:35 -0700 Subject: [PATCH 8/9] [CVE-2024-22421] Patch jupyterlab to 3.6.7 (#730) --- docker-bake.hcl | 10 +++++++--- docker-bake.preview.hcl | 8 ++++++-- r-session-complete/Dockerfile.centos7 | 2 +- .../Dockerfile.ubuntu2004 | 2 +- workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 | 2 +- workbench/Dockerfile.ubuntu2204 | 2 +- 6 files changed, 17 insertions(+), 9 deletions(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 2a521eae..6bd1d696 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -19,6 +19,10 @@ variable DEFAULT_QUARTO_VERSION { default = "1.4.553" } +variable DEFAULT_JUPYTERLAB_VERSION { + default = "3.6.7" +} + function tag_safe_version { params = [version] result = replace(version, "+", "-") @@ -400,7 +404,7 @@ target "r-session-complete" { R_VERSION_ALT = builds.r_alternate PYTHON_VERSION = builds.py_primary PYTHON_VERSION_ALT = builds.py_alternate - JUPYTERLAB_VERSION = "3.6.5" + JUPYTERLAB_VERSION = DEFAULT_JUPYTERLAB_VERSION RSW_VERSION = WORKBENCH_VERSION RSW_NAME = builds.os == "centos7" ? "rstudio-workbench-rhel" : "rstudio-workbench" RSW_DOWNLOAD_URL = builds.os == "centos7" ? "https://s3.amazonaws.com/rstudio-ide-build/server/centos7/x86_64" : "https://download2.rstudio.org/server/jammy/amd64" @@ -458,7 +462,7 @@ target "workbench-for-google-cloud-workstations" { PYTHON_VERSION = builds.py_primary PYTHON_VERSION_ALT = builds.py_alternate PYTHON_VERSION_JUPYTER = builds.py_alternate - JUPYTERLAB_VERSION = "3.6.5" + JUPYTERLAB_VERSION = DEFAULT_JUPYTERLAB_VERSION QUARTO_VERSION = DEFAULT_QUARTO_VERSION DRIVERS_VERSION = get_drivers_version(builds.os) RSW_VERSION = WORKBENCH_VERSION @@ -488,7 +492,7 @@ target "build-workbench-for-microsoft-azure-ml" { PYTHON_VERSION = builds.py_primary PYTHON_VERSION_ALT = builds.py_alternate PYTHON_VERSION_JUPYTER = builds.py_alternate - JUPYTERLAB_VERSION = "3.6.5" + JUPYTERLAB_VERSION = DEFAULT_JUPYTERLAB_VERSION RSW_VERSION = WORKBENCH_VERSION RSW_NAME = "rstudio-workbench" RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/jammy/amd64" diff --git a/docker-bake.preview.hcl b/docker-bake.preview.hcl index d7754bbe..171c0287 100644 --- a/docker-bake.preview.hcl +++ b/docker-bake.preview.hcl @@ -27,6 +27,10 @@ variable DEFAULT_QUARTO_VERSION { default = "1.4.553" } +variable DEFAULT_JUPYTERLAB_VERSION { + default = "3.6.7" +} + variable RSW_PREVIEW_URL_BASE { default = "https://s3.amazonaws.com/rstudio-ide-build/server/" } @@ -333,7 +337,7 @@ target "r-session-complete-daily" { R_VERSION_ALT = builds.r_alternate PYTHON_VERSION = builds.py_primary PYTHON_VERSION_ALT = builds.py_alternate - JUPYTERLAB_VERSION = "3.6.5" + JUPYTERLAB_VERSION = DEFAULT_JUPYTERLAB_VERSION RSW_VERSION = WORKBENCH_DAILY_VERSION RSW_NAME = builds.os == "centos7" ? "rstudio-workbench-rhel" : "rstudio-workbench" RSW_DOWNLOAD_URL = get_rsw_download_url(builds.os) @@ -359,7 +363,7 @@ target "r-session-complete-preview" { R_VERSION_ALT = builds.r_alternate PYTHON_VERSION = builds.py_primary PYTHON_VERSION_ALT = builds.py_alternate - JUPYTERLAB_VERSION = "3.6.5" + JUPYTERLAB_VERSION = DEFAULT_JUPYTERLAB_VERSION RSW_VERSION = WORKBENCH_PREVIEW_VERSION RSW_NAME = builds.os == "centos7" ? "rstudio-workbench-rhel" : "rstudio-workbench" RSW_DOWNLOAD_URL = get_rsw_download_url(builds.os) diff --git a/r-session-complete/Dockerfile.centos7 b/r-session-complete/Dockerfile.centos7 index b3ec72ed..6bb34d88 100644 --- a/r-session-complete/Dockerfile.centos7 +++ b/r-session-complete/Dockerfile.centos7 @@ -4,7 +4,7 @@ ARG R_VERSION=4.2.3 ARG R_VERSION_ALT=4.1.3 ARG PYTHON_VERSION=3.9.14 ARG PYTHON_VERSION_ALT=3.8.15 -ARG JUPYTERLAB_VERSION=3.2.9 +ARG JUPYTERLAB_VERSION=3.6.7 ARG RSW_VERSION=2023.12.1+402.pro1 ARG RSW_NAME=rstudio-workbench-rhel ARG RSW_DOWNLOAD_URL=https://s3.amazonaws.com/rstudio-ide-build/server/centos7/x86_64 diff --git a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 index ddec0969..91418fee 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 +++ b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 @@ -7,7 +7,7 @@ ARG R_VERSION_ALT=4.1.3 ARG PYTHON_VERSION=3.11.7 ARG PYTHON_VERSION_ALT=3.10.13 ARG PYTHON_VERSION_JUPYTER=3.10.13 -ARG JUPYTERLAB_VERSION=3.6.5 +ARG JUPYTERLAB_VERSION=3.6.7 ARG QUARTO_VERSION=1.3.450 ARG DRIVERS_VERSION=2023.05.0 ARG RSW_VERSION=2023.12.1+402.pro1 diff --git a/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 b/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 index 81ae1f88..11f93d3d 100644 --- a/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 +++ b/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 @@ -6,7 +6,7 @@ ARG R_VERSION_ALT=4.1.3 ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG PYTHON_VERSION_JUPYTER=3.8.17 -ARG JUPYTERLAB_VERSION=3.6.5 +ARG JUPYTERLAB_VERSION=3.6.7 ARG RSW_VERSION=2023.12.1+402.pro1 ARG RSW_NAME=rstudio-workbench ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/jammy/amd64 diff --git a/workbench/Dockerfile.ubuntu2204 b/workbench/Dockerfile.ubuntu2204 index 28f40f27..ad8dc96b 100644 --- a/workbench/Dockerfile.ubuntu2204 +++ b/workbench/Dockerfile.ubuntu2204 @@ -6,7 +6,7 @@ ARG R_VERSION_ALT=4.1.3 ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG PYTHON_VERSION_JUPYTER=3.8.17 -ARG JUPYTERLAB_VERSION=3.6.5 +ARG JUPYTERLAB_VERSION=3.6.7 ARG RSW_VERSION=2023.12.1+402.pro1 ARG RSW_NAME=rstudio-workbench ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/jammy/amd64 From 4bef13545b9ccee9748ef0aad22b8f7cbe6ab476 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 19 Apr 2024 13:56:29 -0700 Subject: [PATCH 9/9] Move Quarto installation/configuration to downstream images (#727) * Fix/exclude goss tests for buildx bake * Implement separate test for Connect * Implement bake preview builds * Provision buildx in preview * Do not cache or output tests for preview Provide a buildx config for GHA Unify test sleep behavior of Workbench images * Remove test targets * Remove test layers from Dockerfiles * Change test running to be orchestrated by a Python script * Update preview builds for new test script * Create common use scripts in base image for Ubuntu 22.04 images * Create common use scripts in base image for CentOS 7 images * Move deps to subdirectory Fix hadolint issues * Ignore hadolint issue * Remove build.justfile * Remove unnecessary buildx setup from build-bake-preview.yaml * Move Quarto installation downstream Upgrade Connect and WGCW to 1.4.552 Pin remaining products back to 1.3.340 or lower * Add SCRIPTS_DIR to downstream Dockerfiles * Leave original Quarto install for WGCW * Fix hardcoded QUARTO_VERSION in rstudio-connect.gcfg * Remove unnecessary Quarto tests from base-pro tests * Symlink bundled Quarto for Workbench installations * Fix quarto config replacement in Connect * Remove QUARTO_VERSION arg from base in bake definition * Add quarto check command to Connect goss tests * Remove Quarto build arg from Workbench and r-session-complete * Update preview Connect build matrix with Quarto version * Remove Quarto install from CentOS 7 r-session-complete * Remove package check for Quarto in WGCW * Use DEFAULT_QUARTO_VERSION for Connect matrix * Use DEFAULT_QUARTO_VERSION for Connect matrix * Revert Quarto to 1.3.340 for Connect --- connect/Dockerfile.ubuntu2204 | 13 ++- connect/rstudio-connect-float.gcfg | 7 +- connect/rstudio-connect.gcfg | 2 +- connect/test/goss.yaml | 27 ++++++ content/docker-bake.hcl | 82 ------------------- docker-bake.hcl | 12 ++- docker-bake.preview.hcl | 5 +- product/base/Dockerfile.centos7 | 5 -- product/base/Dockerfile.ubuntu2204 | 4 - product/base/test/goss.yaml | 10 --- product/pro/test/goss.yaml | 10 --- r-session-complete/Dockerfile.centos7 | 4 + r-session-complete/Dockerfile.ubuntu2204 | 4 + r-session-complete/test/goss.yaml | 12 ++- .../Dockerfile.ubuntu2004 | 10 +-- .../test/goss.yaml | 14 ++-- .../Dockerfile.ubuntu2204 | 5 +- .../test/goss.yaml | 8 ++ workbench/Dockerfile.ubuntu2204 | 4 + workbench/test/goss.yaml | 8 ++ 20 files changed, 108 insertions(+), 138 deletions(-) delete mode 100644 content/docker-bake.hcl diff --git a/connect/Dockerfile.ubuntu2204 b/connect/Dockerfile.ubuntu2204 index 39c0105b..0aff7034 100644 --- a/connect/Dockerfile.ubuntu2204 +++ b/connect/Dockerfile.ubuntu2204 @@ -8,6 +8,14 @@ ARG R_VERSION_ALT=4.1.3 ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG RSC_VERSION=2024.03.0 +ARG QUARTO_VERSION=1.3.340 +ARG SCRIPTS_DIR=/opt/positscripts + +### Install Quarto ### +RUN QUARTO_VERSION=${QUARTO_VERSION} ${SCRIPTS_DIR}/install_quarto.sh \ + && ln -s /opt/quarto/${QUARTO_VERSION}/bin/quarto /usr/local/bin/quarto + +### Install Connect and additional dependencies ### SHELL [ "/bin/bash", "-o", "pipefail", "-c"] RUN apt-get update --fix-missing \ && apt-get install -yq --no-install-recommends \ @@ -27,13 +35,16 @@ RUN apt-get update --fix-missing \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* + +### Configure Connect ### EXPOSE 3939/tcp ENV RSC_LICENSE "" ENV RSC_LICENSE_SERVER "" ENV STARTUP_DEBUG_MODE 0 COPY rstudio-connect.gcfg /etc/rstudio-connect/rstudio-connect.gcfg RUN sed -i "s/{{PYTHON_VERSION}}/${PYTHON_VERSION}/g" /etc/rstudio-connect/rstudio-connect.gcfg \ - && sed -i "s/{{PYTHON_VERSION_ALT}}/${PYTHON_VERSION_ALT}/g" /etc/rstudio-connect/rstudio-connect.gcfg + && sed -i "s/{{PYTHON_VERSION_ALT}}/${PYTHON_VERSION_ALT}/g" /etc/rstudio-connect/rstudio-connect.gcfg \ + && sed -i "s/{{QUARTO_VERSION}}/${QUARTO_VERSION}/g" /etc/rstudio-connect/rstudio-connect.gcfg VOLUME ["/data"] ENTRYPOINT ["tini", "--"] diff --git a/connect/rstudio-connect-float.gcfg b/connect/rstudio-connect-float.gcfg index a3d797c4..df12fe67 100644 --- a/connect/rstudio-connect-float.gcfg +++ b/connect/rstudio-connect-float.gcfg @@ -34,7 +34,12 @@ Provider = password [Python] Enabled = true -Executable = /opt/python/3.6.5/bin/python +Executable = /opt/python/{{PYTHON_VERSION}}/bin/python +Executable = /opt/python/{{PYTHON_VERSION_ALT}}/bin/python + +[Quarto] +Enabled = true +Executable = /opt/quarto/{{QUARTO_VERSION}}/bin/quarto ;[RPackageRepository "CRAN"] ;URL = https://demo.rstudiopm.com/all/__linux__/jammy/latest diff --git a/connect/rstudio-connect.gcfg b/connect/rstudio-connect.gcfg index 376d08c2..0cf5006d 100644 --- a/connect/rstudio-connect.gcfg +++ b/connect/rstudio-connect.gcfg @@ -35,7 +35,7 @@ Executable = /opt/python/{{PYTHON_VERSION_ALT}}/bin/python [Quarto] Enabled = true -Executable = /opt/quarto/1.3.340/bin/quarto +Executable = /opt/quarto/{{QUARTO_VERSION}}/bin/quarto [RPackageRepository "CRAN"] URL = https://packagemanager.rstudio.com/cran/__linux__/jammy/latest diff --git a/connect/test/goss.yaml b/connect/test/goss.yaml index 57ffaee8..88d87b2d 100644 --- a/connect/test/goss.yaml +++ b/connect/test/goss.yaml @@ -44,6 +44,11 @@ file: exists: true /opt/python/{{.Env.PYTHON_VERSION_ALT}}/bin/python: exists: true + /opt/quarto/{{.Env.QUARTO_VERSION}}/bin/quarto: + exists: true + /usr/local/bin/quarto: + exists: true + filetype: symlink /tmp/startup.log: exists: true contents: @@ -87,3 +92,25 @@ command: stdout: [ "{{ .Env.PYTHON_VERSION_ALT }}" ] + +# Ensure correct Quarto version + "/opt/quarto/{{ .Env.QUARTO_VERSION }}/bin/quarto --version": + title: quarto_version_matches + exit-status: 0 + stdout: [ + "{{ .Env.QUARTO_VERSION }}" + ] + "/usr/local/bin/quarto --version": + title: quarto_symlink_version_matches + exit-status: 0 + stdout: [ + "{{ .Env.QUARTO_VERSION }}" + ] + +# Ensure Quarto works + "/opt/quarto/{{ .Env.QUARTO_VERSION }}/bin/quarto check --quiet": + title: quarto_check + exit-status: 0 + "/usr/local/bin/quarto check --quiet": + title: quarto_check + exit-status: 0 diff --git a/content/docker-bake.hcl b/content/docker-bake.hcl deleted file mode 100644 index 3205ca24..00000000 --- a/content/docker-bake.hcl +++ /dev/null @@ -1,82 +0,0 @@ -variable CONTENT_BUILD_MATRIX { - default = { - builds = [ - {os = "ubuntu1804", os_alt = "bionic", r = "3.1.3", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, - {os = "ubuntu1804", os_alt = "bionic", r = "3.2.5", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, - {os = "ubuntu1804", os_alt = "bionic", r = "3.3.3", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, - {os = "ubuntu1804", os_alt = "bionic", r = "3.4.4", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, - {os = "ubuntu1804", os_alt = "bionic", r = "3.4.4", py = "3.7.10", drivers = "2024.03.0", quarto = "1.0.37"}, - {os = "ubuntu1804", os_alt = "bionic", r = "3.5.3", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, - {os = "ubuntu1804", os_alt = "bionic", r = "3.5.3", py = "3.7.10", drivers = "2024.03.0", quarto = "1.0.37"}, - {os = "ubuntu1804", os_alt = "bionic", r = "3.6.3", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, - {os = "ubuntu1804", os_alt = "bionic", r = "3.6.3", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, - {os = "ubuntu1804", os_alt = "bionic", r = "3.6.3", py = "3.8.8", drivers = "2024.03.0", quarto = "1.0.37"}, - {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, - {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.7.10", drivers = "2024.03.0", quarto = "1.0.37"}, - {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.8.8", drivers = "2024.03.0", quarto = "1.0.37"}, - {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.9.2", drivers = "2024.03.0", quarto = "1.0.37"}, - {os = "ubuntu1804", os_alt = "bionic", r = "4.1.0", py = "3.8.8", drivers = "2024.03.0", quarto = "1.0.37"}, - {os = "ubuntu1804", os_alt = "bionic", r = "4.1.0", py = "3.9.2", drivers = "2024.03.0", quarto = "1.0.37"}, - {os = "ubuntu1804", os_alt = "bionic", r = "4.1.3", py = "3.10.4", drivers = "2024.03.0", quarto = "1.0.37"}, - {os = "ubuntu2204", os_alt = "jammy", r = "3.6.3", py = "3.8.16", drivers = "2024.03.0", quarto = "1.3.340"}, - {os = "ubuntu2204", os_alt = "jammy", r = "4.0.5", py = "3.9.16", drivers = "2024.03.0", quarto = "1.3.340"}, - {os = "ubuntu2204", os_alt = "jammy", r = "4.1.3", py = "3.10.11", drivers = "2024.03.0", quarto = "1.3.340"}, - {os = "ubuntu2204", os_alt = "jammy", r = "4.2.2", py = "3.11.3", drivers = "2024.03.0", quarto = "1.3.340"}, - ] - } -} - -group "default" { - targets = ["base", "pro"] -} - -target "base" { - name = "content-base-r${replace(builds.r, ".", "-")}-py${replace(builds.py, ".", "-")}-${builds.os}" - - tags = [ - "ghcr.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os}", - "ghcr.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os_alt}", - "docker.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os}", - "docker.io.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os_alt}", - ] - output = [ - "type=image", - ] - - dockerfile = "Dockerfile.${builds.os}" - context = "base" - - matrix = CONTENT_BUILD_MATRIX - args = { - R_VERSION = "${builds.r}" - PYTHON_VERSION = "${builds.py}" - QUARTO_VERSION = "${builds.quarto}" - } -} - -target "pro" { - name = "content-pro-r${replace(builds.r, ".", "-")}-py${replace(builds.py, ".", "-")}-${builds.os}" - - tags = [ - "ghcr.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os}", - "ghcr.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os_alt}", - "docker.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os}", - "docker.io.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os_alt}", - ] - output = [ - "type=image", - ] - - contexts = { - content-base = "target:content-base-r${replace(builds.r, ".", "-")}-py${replace(builds.py, ".", "-")}-${builds.os}" - } - - dockerfile = "Dockerfile.${builds.os}" - context = "pro" - - matrix = CONTENT_BUILD_MATRIX - args = { - R_VERSION = "${builds.r}" - DRIVERS_VERSION = "${builds.drivers}" - } -} diff --git a/docker-bake.hcl b/docker-bake.hcl index 6bd1d696..ee996e3c 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -1,6 +1,6 @@ ### Variable definitions ### variable CONNECT_VERSION { - default = "2024.02.0" + default = "2024.03.0" } variable PACKAGE_MANAGER_VERSION { @@ -16,7 +16,7 @@ variable DRIVERS_VERSION { } variable DEFAULT_QUARTO_VERSION { - default = "1.4.553" + default = "1.4.552" } variable DEFAULT_JUPYTERLAB_VERSION { @@ -101,7 +101,7 @@ variable PACKAGE_MANAGER_BUILD_MATRIX { variable CONNECT_BUILD_MATRIX { default = { builds = [ - {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.17", py_alternate = "3.8.17"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.17", py_alternate = "3.8.17", quarto = "1.3.340"}, ] } } @@ -240,7 +240,6 @@ target "product-base" { PYTHON_VERSION = builds.py_primary PYTHON_VERSION_ALT = builds.py_alternate TINI_VERSION = "0.19.0" - QUARTO_VERSION = "1.3.340" } } @@ -268,8 +267,7 @@ target "product-base-pro" { PYTHON_VERSION_ALT = builds.py_alternate DRIVERS_VERSION = get_drivers_version(builds.os) TINI_VERSION = "0.19.0" - QUARTO_VERSION = "1.3.340" - } + } } ### Package Manager targets ### @@ -317,6 +315,7 @@ target "connect" { PYTHON_VERSION = builds.py_primary PYTHON_VERSION_ALT = builds.py_alternate RSC_VERSION = CONNECT_VERSION + QUARTO_VERSION = builds.quarto } } @@ -472,7 +471,6 @@ target "workbench-for-google-cloud-workstations" { } ### Workbench for Microsoft Azure ML targets ### - target "build-workbench-for-microsoft-azure-ml" { inherits = ["base"] target = "build" diff --git a/docker-bake.preview.hcl b/docker-bake.preview.hcl index 171c0287..60694444 100644 --- a/docker-bake.preview.hcl +++ b/docker-bake.preview.hcl @@ -24,7 +24,7 @@ variable DRIVERS_VERSION { } variable DEFAULT_QUARTO_VERSION { - default = "1.4.553" + default = "1.4.552" } variable DEFAULT_JUPYTERLAB_VERSION { @@ -126,7 +126,7 @@ variable PACKAGE_MANAGER_BUILD_MATRIX { variable CONNECT_BUILD_MATRIX { default = { builds = [ - {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14", quarto = "1.3.340"}, ] } } @@ -297,6 +297,7 @@ target "connect-daily" { PYTHON_VERSION = builds.py_primary PYTHON_VERSION_ALT = builds.py_alternate RSC_VERSION = CONNECT_DAILY_VERSION + QUARTO_VERSION = builds.quarto } } diff --git a/product/base/Dockerfile.centos7 b/product/base/Dockerfile.centos7 index 5e667f12..9faf3e6b 100644 --- a/product/base/Dockerfile.centos7 +++ b/product/base/Dockerfile.centos7 @@ -7,7 +7,6 @@ ARG PYTHON_VERSION=3.9.14 ARG PYTHON_VERSION_ALT=3.8.15 ARG TINI_VERSION=0.19.0 ARG QUARTO_VERSION=1.3.340 - ARG SCRIPTS_DIR=/opt/positscripts COPY --chmod=0755 scripts/rhel/* ${SCRIPTS_DIR}/ @@ -36,10 +35,6 @@ RUN curl -sL "https://yihui.org/tinytex/install-bin-unix.sh" | sh \ && /opt/TinyTeX/bin/*/tlmgr option sys_bin /usr/local/bin \ && /opt/TinyTeX/bin/*/tlmgr path add -### Install Quarto ### -RUN QUARTO_VERSION=${QUARTO_VERSION} ${SCRIPTS_DIR}/install_quarto.sh \ - && ln -s /opt/quarto/${QUARTO_VERSION}/bin/quarto /usr/local/bin/quarto - ### Install R versions ### RUN R_VERSION=${R_VERSION} ${SCRIPTS_DIR}/install_r.sh \ && R_VERSION=${R_VERSION_ALT} ${SCRIPTS_DIR}/install_r.sh \ diff --git a/product/base/Dockerfile.ubuntu2204 b/product/base/Dockerfile.ubuntu2204 index 93c0b1bf..850245e8 100644 --- a/product/base/Dockerfile.ubuntu2204 +++ b/product/base/Dockerfile.ubuntu2204 @@ -38,10 +38,6 @@ RUN curl -sL "https://yihui.org/tinytex/install-bin-unix.sh" | sh \ && /opt/TinyTeX/bin/*/tlmgr option sys_bin /usr/local/bin \ && /opt/TinyTeX/bin/*/tlmgr path add -### Install Quarto ### -RUN QUARTO_VERSION=${QUARTO_VERSION} ${SCRIPTS_DIR}/install_quarto.sh \ - && ln -s /opt/quarto/${QUARTO_VERSION}/bin/quarto /usr/local/bin/quarto - ### Install R versions ### RUN R_VERSION=${R_VERSION} ${SCRIPTS_DIR}/install_r.sh \ && R_VERSION=${R_VERSION_ALT} ${SCRIPTS_DIR}/install_r.sh \ diff --git a/product/base/test/goss.yaml b/product/base/test/goss.yaml index e321ebe0..d15aa828 100644 --- a/product/base/test/goss.yaml +++ b/product/base/test/goss.yaml @@ -47,8 +47,6 @@ file: exists: true /tini: exists: true - /opt/quarto/{{.Env.QUARTO_VERSION}}/bin/quarto: - exists: true {{if .Env.OS | regexMatch "ubuntu.*"}} # Check that `cracklib-runtime` is present so `chpasswd` works /var/cache/cracklib/cracklib_dict.pwd: @@ -83,11 +81,3 @@ command: stdout: [ "{{ .Env.PYTHON_VERSION_ALT }}" ] - -# Ensure correct Quarto version - "/opt/quarto/{{ .Env.QUARTO_VERSION }}/bin/quarto --version": - title: quarto_version_matches - exit-status: 0 - stdout: [ - "{{ .Env.QUARTO_VERSION }}" - ] diff --git a/product/pro/test/goss.yaml b/product/pro/test/goss.yaml index 8670095d..e602a7ac 100644 --- a/product/pro/test/goss.yaml +++ b/product/pro/test/goss.yaml @@ -40,8 +40,6 @@ file: exists: true /tini: exists: true - /opt/quarto/{{.Env.QUARTO_VERSION}}/bin/quarto: - exists: true command: @@ -72,11 +70,3 @@ command: stdout: [ "{{ .Env.PYTHON_VERSION_ALT }}" ] - -# Ensure correct Quarto version - "/opt/quarto/{{ .Env.QUARTO_VERSION }}/bin/quarto --version": - title: quarto_version_matches - exit-status: 0 - stdout: [ - "{{ .Env.QUARTO_VERSION }}" - ] diff --git a/r-session-complete/Dockerfile.centos7 b/r-session-complete/Dockerfile.centos7 index 6bb34d88..5a8e0a7c 100644 --- a/r-session-complete/Dockerfile.centos7 +++ b/r-session-complete/Dockerfile.centos7 @@ -8,6 +8,7 @@ ARG JUPYTERLAB_VERSION=3.6.7 ARG RSW_VERSION=2023.12.1+402.pro1 ARG RSW_NAME=rstudio-workbench-rhel ARG RSW_DOWNLOAD_URL=https://s3.amazonaws.com/rstudio-ide-build/server/centos7/x86_64 +ARG SCRIPTS_DIR=/opt/positscripts SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN yum install -y subversion \ @@ -27,6 +28,9 @@ RUN yum install -y subversion \ && yum clean all \ && rm -rf /var/lib/rstudio-server/r-versions +### Install Quarto to PATH ### +RUN ln -s /lib/rstudio-server/bin/quarto/bin/quarto /usr/local/bin/quarto + COPY maybe_install_vs_code.sh /tmp/maybe_install_vs_code.sh RUN /tmp/maybe_install_vs_code.sh \ && rm /tmp/maybe_install_vs_code.sh diff --git a/r-session-complete/Dockerfile.ubuntu2204 b/r-session-complete/Dockerfile.ubuntu2204 index 9bef24ac..be1817c0 100644 --- a/r-session-complete/Dockerfile.ubuntu2204 +++ b/r-session-complete/Dockerfile.ubuntu2204 @@ -9,6 +9,7 @@ ARG JUPYTERLAB_VERSION=3.6.5 ARG RSW_VERSION=2023.12.1+402.pro1 ARG RSW_NAME=rstudio-workbench ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/jammy/amd64 +ARG SCRIPTS_DIR=/opt/positscripts ENV WORKBENCH_JUPYTER_PATH=/usr/local/bin/jupyter @@ -36,6 +37,9 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && rm -rf /var/lib/rstudio-server/r-versions +### Install Quarto to PATH ### +RUN ln -s /lib/rstudio-server/bin/quarto/bin/quarto /usr/local/bin/quarto + COPY maybe_install_vs_code.sh /tmp/maybe_install_vs_code.sh RUN /tmp/maybe_install_vs_code.sh \ && rm /tmp/maybe_install_vs_code.sh diff --git a/r-session-complete/test/goss.yaml b/r-session-complete/test/goss.yaml index 09760e7c..0cf72448 100644 --- a/r-session-complete/test/goss.yaml +++ b/r-session-complete/test/goss.yaml @@ -14,10 +14,13 @@ file: exists: true {{ end }} /opt/rstudio-drivers: - exists: true - filetype: directory + exists: true + filetype: directory /var/lib/rstudio-server/r-versions: exists: false + /usr/local/bin/quarto: + exists: true + filetype: symlink command: "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/{{.Env.PYTHON_VERSION}}/bin/jupyter nbconvert --to notebook --stdin --stdout": @@ -52,3 +55,8 @@ command: title: jupyter_in_path_var timeout: 60000 exit-status: 0 + +# Ensure Quarto works + "/usr/local/bin/quarto check --quiet": + title: quarto_check + exit-status: 0 diff --git a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 index 91418fee..ae566260 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 +++ b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 @@ -8,11 +8,11 @@ ARG PYTHON_VERSION=3.11.7 ARG PYTHON_VERSION_ALT=3.10.13 ARG PYTHON_VERSION_JUPYTER=3.10.13 ARG JUPYTERLAB_VERSION=3.6.7 -ARG QUARTO_VERSION=1.3.450 ARG DRIVERS_VERSION=2023.05.0 ARG RSW_VERSION=2023.12.1+402.pro1 ARG RSW_NAME=rstudio-workbench ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/focal/amd64 +ARG SCRIPTS_DIR=/opt/positscripts ENV STARTUP_DEBUG_MODE 0 ENV RSW_LICENSE "" @@ -83,12 +83,8 @@ ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 -### Install Quarto ### -RUN curl -o quarto-linux-amd64.deb -L https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.deb \ - && apt-get update \ - && apt-get install -yq ./quarto-linux-amd64.deb \ - && rm -rf /var/lib/apt/lists/* \ - && rm quarto-linux-amd64.deb +### Install Quarto to PATH ### +RUN ln -s /lib/rstudio-server/bin/quarto/bin/quarto /usr/local/bin/quarto ### Install Pro Drivers ### RUN apt-get update \ diff --git a/workbench-for-google-cloud-workstations/test/goss.yaml b/workbench-for-google-cloud-workstations/test/goss.yaml index 412fae9d..fcb383d7 100644 --- a/workbench-for-google-cloud-workstations/test/goss.yaml +++ b/workbench-for-google-cloud-workstations/test/goss.yaml @@ -22,10 +22,6 @@ package: installed: true python-{{.Env.PYTHON_VERSION_ALT}}: installed: true - quarto: - installed: true - versions: - - {{.Env.QUARTO_VERSION}} rstudio-drivers: installed: true versions: @@ -152,6 +148,9 @@ file: contents: [ "/usr/bin/supervisord -c /etc/supervisor/supervisord.conf" ] + /usr/local/bin/quarto: + exists: true + filetype: symlink command: @@ -231,4 +230,9 @@ command: exec: /opt/R/{{$r_version_alt}}/bin/R --slave -e "library(\"{{.}}\")" timeout: 60000 exit-status: 0 - {{end}} \ No newline at end of file + {{end}} + +# Ensure Quarto works + "/usr/local/bin/quarto check --quiet": + title: quarto_check + exit-status: 0 diff --git a/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 b/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 index 11f93d3d..384d6466 100644 --- a/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 +++ b/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 @@ -10,7 +10,7 @@ ARG JUPYTERLAB_VERSION=3.6.7 ARG RSW_VERSION=2023.12.1+402.pro1 ARG RSW_NAME=rstudio-workbench ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/jammy/amd64 -ARG DEBIAN_FRONTEND=noninteractive +ARG SCRIPTS_DIR=/opt/positscripts # Set default env values ENV STARTUP_DEBUG_MODE 0 @@ -59,6 +59,9 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && rm -rf /var/lib/rstudio-server/r-versions +### Install Quarto to PATH ### +RUN ln -s /lib/rstudio-server/bin/quarto/bin/quarto /usr/local/bin/quarto + COPY --chmod=0755 license-manager-shim /opt/rstudio-license/license-manager COPY --chmod=0775 startup.sh /usr/local/bin/startup.sh COPY startup/* /startup/base/ diff --git a/workbench-for-microsoft-azure-ml/test/goss.yaml b/workbench-for-microsoft-azure-ml/test/goss.yaml index 2c8ba3fe..b94fae54 100644 --- a/workbench-for-microsoft-azure-ml/test/goss.yaml +++ b/workbench-for-microsoft-azure-ml/test/goss.yaml @@ -86,6 +86,9 @@ file: exists: true contains: - "!Error reading /etc/rstudio/rserver.conf:" + /usr/local/bin/quarto: + exists: true + filetype: symlink command: "su rstudio-server -c 'touch /var/lib/rstudio-server/monitor/log/rstudio-server.log'": @@ -143,3 +146,8 @@ command: - {{ $pkg }} {{end}} {{end}} + +# Ensure Quarto works + "/usr/local/bin/quarto check --quiet": + title: quarto_check + exit-status: 0 diff --git a/workbench/Dockerfile.ubuntu2204 b/workbench/Dockerfile.ubuntu2204 index ad8dc96b..0166a8e3 100644 --- a/workbench/Dockerfile.ubuntu2204 +++ b/workbench/Dockerfile.ubuntu2204 @@ -10,6 +10,7 @@ ARG JUPYTERLAB_VERSION=3.6.7 ARG RSW_VERSION=2023.12.1+402.pro1 ARG RSW_NAME=rstudio-workbench ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/jammy/amd64 +ARG SCRIPTS_DIR=/opt/positscripts ENV STARTUP_DEBUG_MODE 0 ENV RSW_LICENSE "" @@ -62,6 +63,9 @@ RUN apt-get update \ && rm -rf /var/lib/rstudio-server/r-versions \ && rm -rf /var/lib/rstudio-launcher/Local/jobs/buildkitsandbox +### Install Quarto to PATH ### +RUN ln -s /lib/rstudio-server/bin/quarto/bin/quarto /usr/local/bin/quarto + COPY maybe_install_vs_code.sh /tmp/maybe_install_vs_code.sh RUN /tmp/maybe_install_vs_code.sh \ && rm /tmp/maybe_install_vs_code.sh diff --git a/workbench/test/goss.yaml b/workbench/test/goss.yaml index a1339ce3..777a04ee 100644 --- a/workbench/test/goss.yaml +++ b/workbench/test/goss.yaml @@ -94,6 +94,9 @@ file: owner: root group: root mode: "0600" + /usr/local/bin/quarto: + exists: true + filetype: symlink command: "su rstudio-server -c 'touch /var/lib/rstudio-server/monitor/log/rstudio-server.log'": @@ -140,3 +143,8 @@ command: stdout: [ "/usr/bin/openssl" ] + +# Ensure Quarto works + "/usr/local/bin/quarto check --quiet": + title: quarto_check + exit-status: 0