diff --git a/.github/workflows/imagebuild.yml b/.github/workflows/imagebuild.yml new file mode 100644 index 0000000..812bafd --- /dev/null +++ b/.github/workflows/imagebuild.yml @@ -0,0 +1,133 @@ +name: Build and Push Georust Docker Images + +on: + push: + tags-ignore: + - '**' + branches: + - main + - staging + - trying + pull_request: + merge_group: + +env: + LIBPROJ_VERSION: 9.4.0 + MAIN_IMAGE_NAME: libproj-builder + +jobs: + build_main_image: + name: Build the main image for geo and proj + runs-on: ubuntu-latest + + strategy: + matrix: + rust_version: ["1.70", 1.76, 1.77] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set Up Builder + run: | + docker buildx create --name mybuilder + docker buildx use mybuilder + docker buildx inspect --bootstrap + + - name: Build and export main image to Docker + uses: docker/build-push-action@v5 + with: + file: ./dockerfiles/${{ env.MAIN_IMAGE_NAME }} + push: false + load: true + platforms: linux/amd64 + tags: ghcr.io/${{ github.repository_owner }}/${{ env.MAIN_IMAGE_NAME }}:proj-${{ env.LIBPROJ_VERSION }}-rust-${{ matrix.rust_version }} + build-args: | + RUST_VERSION=${{ matrix.rust_version }} + PROJ_VERSION=${{ env.LIBPROJ_VERSION }} + + - name: Push main image + uses: docker/build-push-action@v5 + with: + file: ./dockerfiles/${{ env.MAIN_IMAGE_NAME }} + push: true + load: false + # platforms: linux/amd64,linux/arm64 + tags: ghcr.io/${{ github.repository_owner }}/${{ env.MAIN_IMAGE_NAME }}:proj-${{ env.LIBPROJ_VERSION }}-rust-${{ matrix.rust_version }} + outputs: type=docker,dest=/tmp/${{ env.MAIN_IMAGE_NAME }}-proj-${{ env.LIBPROJ_VERSION }}-rust-${{ matrix.rust_version }}.tar + build-args: | + RUST_VERSION=${{ matrix.rust_version }} + PROJ_VERSION=${{ env.LIBPROJ_VERSION }} + + build_dependent_images: + name: Build dependent images for geo and proj + needs: build_main_image + runs-on: ubuntu-latest + + strategy: + matrix: + subimages: [ + {image: geo-ci, testcmd: "git clone https://github.com/georust/geo && cd geo && cargo test --no-default-features && cargo test && cargo test --all-features"}, + {image: proj-ci, testcmd: "git clone https://github.com/georust/proj && cd proj && cargo test --no-default-features && cargo test --features bundled_proj && cargo test --features network && cd proj-sys && _PROJ_SYS_TEST_EXPECT_BUILD_FROM_SRC=0 cargo test && _PROJ_SYS_TEST_EXPECT_BUILD_FROM_SRC=1 cargo test --features bundled_proj"}, + {image: proj-ci-without-system-proj, testcmd: "git clone https://github.com/georust/proj && cd proj && cargo test --features bundled_proj && cd proj-sys && _PROJ_SYS_TEST_EXPECT_BUILD_FROM_SRC=1 cargo test"} + ] + rust_version: ["1.70", 1.76, 1.77] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set Up Builder + run: | + docker buildx create --name mybuilder + docker buildx use mybuilder + docker buildx inspect --bootstrap + + - name: Build ${{ matrix.subimages.image }} + uses: docker/build-push-action@v5 + with: + file: ./dockerfiles/${{ matrix.subimages.image }} + push: false + load: true + platforms: linux/amd64 + tags: ghcr.io/${{ github.repository_owner }}/${{ matrix.subimages.image }}:proj-${{ env.LIBPROJ_VERSION }}-rust-${{ matrix.rust_version }} + build-args: | + RUST_VERSION=${{ matrix.rust_version }} + PROJ_VERSION=${{ env.LIBPROJ_VERSION }} + + - name: Test ${{ matrix.subimages.image }} + run: | + docker run --rm ghcr.io/${{ github.repository_owner }}/${{ matrix.subimages.image }}:proj-${{ env.LIBPROJ_VERSION }}-rust-${{ matrix.rust_version }} /bin/bash -c "${{ matrix.subimages.testcmd }}" + + - name: Push tested ${{ matrix.subimages.image }} image + uses: docker/build-push-action@v5 + with: + file: ./dockerfiles/${{ matrix.subimages.image }} + push: true + load: false + # platforms: linux/amd64,linux/arm64 + tags: ghcr.io/${{ github.repository_owner }}/${{ matrix.subimages.image }}:proj-${{ env.LIBPROJ_VERSION }}-rust-${{ matrix.rust_version }} + outputs: type=docker,dest=/tmp/${{ matrix.subimages.image }}-proj-${{ env.LIBPROJ_VERSION }}-rust-${{ matrix.rust_version }}.tar + build-args: | + RUST_VERSION=${{ matrix.rust_version }} + PROJ_VERSION=${{ env.LIBPROJ_VERSION }} diff --git a/Makefile b/Makefile deleted file mode 100644 index 55b3a6c..0000000 --- a/Makefile +++ /dev/null @@ -1,80 +0,0 @@ -GEORUST_ROOT=$(CURDIR)/.. -PUSH_RATE_LIMIT_DELAY=5 - -RUST_VERSION ?= $(error RUST_VERSION not set) -PROJ_VERSION ?= $(error PROJ_VERSION not set) -DOCKER_TAG=proj-$(PROJ_VERSION)-rust-$(RUST_VERSION) - -DOCKER_BUILD_CMD=docker build --platform linux/amd64 --build-arg RUST_VERSION=$(RUST_VERSION) --build-arg PROJ_VERSION=$(PROJ_VERSION) -DOCKER_RUN_CMD=docker run -v $(GEORUST_ROOT):/tmp/georust -e CARGO_TARGET_DIR=/tmp/cargo-target -DOCKERFILE_DIR=dockerfiles/ - -# WIP: On macos w/ apple silicon (aarch64), you'll need buildx to output the proper -# platform/arch for CI. -# Currently though, this seems to output a libproj.a that's not usable. -# DOCKER_BUILD_CMD=docker buildx build --platform linux/amd64 - -default: build-all - -build-all: libproj-builder proj-ci-without-system-proj proj-ci geo-ci - -geo-ci: - $(DOCKER_BUILD_CMD) -f $(DOCKERFILE_DIR)geo-ci.Dockerfile -t georust/geo-ci:$(DOCKER_TAG) . - -proj-ci: - $(DOCKER_BUILD_CMD) -f $(DOCKERFILE_DIR)proj-ci.Dockerfile -t georust/proj-ci:$(DOCKER_TAG) . - -proj-ci-without-system-proj: - $(DOCKER_BUILD_CMD) -f $(DOCKERFILE_DIR)proj-ci-without-system-proj.Dockerfile -t georust/proj-ci-without-system-proj:$(DOCKER_TAG) . - -libproj-builder: - $(DOCKER_BUILD_CMD) -f $(DOCKERFILE_DIR)libproj-builder.Dockerfile -t georust/libproj-builder:$(DOCKER_TAG) . - -publish-all: publish-libproj-builder publish-proj-ci-without-system-proj publish-proj-ci publish-geo-ci - -publish-all-latest: DOCKER_TAG=latest -publish-all-latest: publish-all - -publish-geo-ci: - sleep $(PUSH_RATE_LIMIT_DELAY) && \ - docker push georust/geo-ci:$(DOCKER_TAG) - -publish-proj-ci: - sleep $(PUSH_RATE_LIMIT_DELAY) && \ - docker push georust/proj-ci:$(DOCKER_TAG) - -publish-proj-ci-without-system-proj: - sleep $(PUSH_RATE_LIMIT_DELAY) && \ - docker push georust/proj-ci-without-system-proj:$(DOCKER_TAG) - -publish-libproj-builder: - sleep $(PUSH_RATE_LIMIT_DELAY) && \ - docker push georust/libproj-builder:$(DOCKER_TAG) - -geo-ci-shell: - $(DOCKER_RUN_CMD) -ti georust/geo-ci:$(DOCKER_TAG) bash -l - -libproj-builder-shell: - $(DOCKER_RUN_CMD) -ti georust/libproj-builder:$(DOCKER_TAG) bash -l - -proj-ci-without-system-proj-shell: - $(DOCKER_RUN_CMD) -ti georust/proj-ci-without-system-proj:$(DOCKER_TAG) bash -l - -proj-ci-shell: - $(DOCKER_RUN_CMD) -ti -w /tmp/georust/proj georust/proj-ci:$(DOCKER_TAG) bash -l - -test-all: test-proj-ci test-proj-sys-ci test-geo-ci - -test-geo-ci: - echo 1 \ - && $(DOCKER_RUN_CMD) -w /tmp/georust/geo georust/geo-ci:$(DOCKER_TAG) /bin/bash -c "cargo test --no-default-features && cargo test && cargo test --all-features" - -test-proj-ci: - echo 1 \ - && $(DOCKER_RUN_CMD) -w /tmp/georust/proj georust/proj-ci:$(DOCKER_TAG) /bin/bash -c "cargo test --no-default-features && cargo test --features bundled_proj && cargo test --features network" - -test-proj-sys-ci: - echo 1 \ - && $(DOCKER_RUN_CMD) -w /tmp/georust/proj/proj-sys --env "_PROJ_SYS_TEST_EXPECT_BUILD_FROM_SRC=1" georust/proj-ci-without-system-proj:$(DOCKER_TAG) cargo test \ - && $(DOCKER_RUN_CMD) -w /tmp/georust/proj/proj-sys --env "_PROJ_SYS_TEST_EXPECT_BUILD_FROM_SRC=0" georust/proj-ci:$(DOCKER_TAG) cargo test \ - && $(DOCKER_RUN_CMD) -w /tmp/georust/proj/proj-sys --env "_PROJ_SYS_TEST_EXPECT_BUILD_FROM_SRC=1" georust/proj-ci:$(DOCKER_TAG) cargo test --features bundled_proj diff --git a/README.md b/README.md index a61b03a..11ea427 100644 --- a/README.md +++ b/README.md @@ -30,41 +30,21 @@ number tag as well, e.g. `rust:1.45.1`. Similarly we could publish a new tag for each minor patch (`geo-ci:rust-1.50.0`, `geo-ci:rust-1.50.1`, etc.), but running CI against each patch seems like overkill at this point. -## How to Update Rust +**New images are built by Github Actions and published to the Github Container Registry if tests pass** -### Prerequisites +## How to Update the Rust Version -It's assumed that you have a directory structure containing dependent georust repositories like this: - $ ls - docker-containers # this repository - geo # needed to run tests - proj # needed to run tests +### Add a new Rust version -### add a new Rust version +edit the `.github/workflows/imagebuild.yml` file - echo "1.69" >> rust-versions.txt +Add the Rust version you wish to add to the **two** `rust_version` matrices: - # build containers - ./run-rust.sh 1.69 -- make build-all +#### Example - # run some tests on the new containers - ./run-rust.sh 1.69 -- make test-all - - # If everythig looks good, you can publish the new tags - ./run-rust.sh 1.69 -- make publish-all - - # delete any old unsupported rust versions from the file - edit rust-versions.txt - -### To update multiple Rust versions in parallel - -If you have multiple versions of rust you want to build/test/publish: - - ./run-rust.sh 1.49 1.50 -- make build-all - -For more: - ./run-rust.sh --help + Before: rust_version: [1.74, 1.75, 1.76] + After: rust_version: [1.74, 1.75, 1.76, 1.77] ## How to Update Proj @@ -72,17 +52,8 @@ libproj (the cpp lib) is built using the [docker container builder pattern](https://docs.docker.com/develop/develop-images/multistage-build/), and then reused by multiple CI containers. - edit proj-version.txt - - # All containers will need to be rebuilt when updating proj - ./run-rust.sh --all -- make build-all + edit the `LIBPROJ_VERSION` variable in `imagebuild.yml` - # NOTE: I often see intermittent failures when running all the tests at once, but - # haven't found time to dig into why this is happening. - # I typically re-run failed tests by themselves to see if the failures recurs. - ./run-rust.sh --all -- make test-all - - ./run-rust.sh --all -- make publish-all ### Update the `proj` crate @@ -94,14 +65,3 @@ then reused by multiple CI containers. - When the PR has merged and the `proj` crate has been published, you can update [geo](https://github.com/georust/geo) to use the new `proj` crate: -### Publishing docker containers - -To publish all the containers for a version of rust: - - # e.g. to publish geo-ci:rust-1.58, proj-ci:rust-1.58, etc. - ./run-rust.sh 1.58 -- make publish-all - -You can quickly publish multiple versions like this: - - ./run-rust.sh 1.58 1.59 -- make publish-all - diff --git a/dockerfiles/geo-ci.Dockerfile b/dockerfiles/geo-ci similarity index 83% rename from dockerfiles/geo-ci.Dockerfile rename to dockerfiles/geo-ci index 0d11bc3..9c78610 100644 --- a/dockerfiles/geo-ci.Dockerfile +++ b/dockerfiles/geo-ci @@ -1,4 +1,4 @@ -# https://hub.docker.com/orgs/georust/geo-ci +# https://github.com/orgs/georust/packages/container/package/geo-ci # ------------------------------------------------------------------------------ # Final stage @@ -7,7 +7,7 @@ ARG RUST_VERSION ARG PROJ_VERSION -FROM georust/libproj-builder:proj-${PROJ_VERSION}-rust-${RUST_VERSION} as libproj-builder +FROM ghcr.io/georust/libproj-builder:proj-${PROJ_VERSION}-rust-${RUST_VERSION} as libproj-builder FROM rust:$RUST_VERSION-bullseye ARG RUST_VERSION @@ -21,6 +21,7 @@ RUN apt-get update \ && DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \ ca-certificates \ clang \ + cmake \ git \ libtiff-dev \ pkg-config \ diff --git a/dockerfiles/libproj-builder.Dockerfile b/dockerfiles/libproj-builder similarity index 93% rename from dockerfiles/libproj-builder.Dockerfile rename to dockerfiles/libproj-builder index aae32ee..cb34275 100644 --- a/dockerfiles/libproj-builder.Dockerfile +++ b/dockerfiles/libproj-builder @@ -1,4 +1,4 @@ -# https://hub.docker.com/orgs/georust/libproj-builder +# https://github.com/orgs/georust/packages/container/package/libproj-builder # Builds libproj from source @@ -14,6 +14,7 @@ RUN test -n "$PROJ_VERSION" || (echo "PROJ_VERSION ARG not set" && false) RUN apt-get update \ && DEBIAN_FRONTEND="noninteractive" apt-get install -y \ clang \ + git \ libcurl4-gnutls-dev \ libsqlite3-dev \ libtiff-dev \ diff --git a/dockerfiles/proj-ci.Dockerfile b/dockerfiles/proj-ci similarity index 71% rename from dockerfiles/proj-ci.Dockerfile rename to dockerfiles/proj-ci index 4be0467..0549dbd 100644 --- a/dockerfiles/proj-ci.Dockerfile +++ b/dockerfiles/proj-ci @@ -1,8 +1,8 @@ -# https://hub.docker.com/orgs/georust/proj-ci +# https://github.com/orgs/georust/packages/container/package/proj-ci ARG RUST_VERSION ARG PROJ_VERSION -FROM georust/libproj-builder:proj-${PROJ_VERSION}-rust-${RUST_VERSION} +FROM ghcr.io/georust/libproj-builder:proj-${PROJ_VERSION}-rust-${RUST_VERSION} ARG RUST_VERSION ARG PROJ_VERSION @@ -12,6 +12,7 @@ RUN test -n "$PROJ_VERSION" || (echo "PROJ_VERSION ARG not set" && false) RUN apt-get update \ && DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \ clang \ + cmake \ && rm -rf /var/lib/apt/lists/* RUN cp -r /build/usr/* /usr diff --git a/dockerfiles/proj-ci-without-system-proj.Dockerfile b/dockerfiles/proj-ci-without-system-proj similarity index 78% rename from dockerfiles/proj-ci-without-system-proj.Dockerfile rename to dockerfiles/proj-ci-without-system-proj index 37f480c..bdd040f 100644 --- a/dockerfiles/proj-ci-without-system-proj.Dockerfile +++ b/dockerfiles/proj-ci-without-system-proj @@ -1,4 +1,4 @@ -# https://hub.docker.com/orgs/georust/proj-ci-without-system-proj +# https://github.com/orgs/georust/packages/container/package/proj-ci-without-system-proj # This container is based on the libproj-builder container, which has built # libproj, and thus has all the dependencies for building proj from source, but @@ -7,7 +7,7 @@ ARG RUST_VERSION ARG PROJ_VERSION -FROM georust/libproj-builder:proj-${PROJ_VERSION}-rust-${RUST_VERSION} +FROM ghcr.io/georust/libproj-builder:proj-${PROJ_VERSION}-rust-${RUST_VERSION} ARG RUST_VERSION ARG PROJ_VERSION diff --git a/proj-version.txt b/proj-version.txt deleted file mode 100644 index 8148c55..0000000 --- a/proj-version.txt +++ /dev/null @@ -1 +0,0 @@ -9.4.0 diff --git a/run-rust.sh b/run-rust.sh deleted file mode 100755 index d4a040c..0000000 --- a/run-rust.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env zsh - -declare -A pids -declare -A statuses -declare -a rust_versions - -USAGE=$(cat <<-END -# $(basename $0) -Run a command for each version of rust - -Usage to run for all versions: - $(basename $0) --all -- - -Usage for specific versions: - $(basename $0) -- - - -Example: - $(basename $0) 1.56 1.57 -- make - -END -) - -function usage_violation { - echo "error: $@ \n" - echo $USAGE - exit 1 -} - -function expect_cmd { - if [[ $1 == "--" ]]; then - shift - CMD=($@) - else - usage_violation 'expected `--` to start cmd' - fi -} - -case $1 in -"" | --help) - echo "$USAGE" - exit 0 - ;; ---all) - rust_versions=("${(f)"$(