diff --git a/.circleci/config.yml b/.circleci/config.yml index e3216cdf..3a47c92c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -162,9 +162,8 @@ jobs: fi export NEW_ORG=${NEW_ORG:-$COMPUTED_ORG} - export GOSS_FILES_PATH=~/circleci-bundles/shared/goss + export GOSS_FILES_PATH=~/circleci-bundles/$PLATFORM export GOSS_FILES_STRATEGY=cp - export GOSS_OPTS="--color --format documentation --retry-timeout 120s" export GOSS_SLEEP=5s make -j $PLATFORM/publish_images diff --git a/README.md b/README.md index 8cb2ec11..c25df094 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,22 @@ Don't do it. If you have an Ultrabook laptop, it won't be happy. If you really want to do it, look at the `Makefile`. +### Testing +There is automated testing for every variant of every image! + +Tests run with [dgoss](https://github.com/aelsabbahy/goss/tree/master/extras/dgoss). + +Tests are platform-specific (e.g., PHP and Android have their own distinct sets of tests)—see the `goss.yaml` file in each image directory. + +The testing logic is currently in `shared/images/build.sh`, as it is nestled between the existing automated `docker build` and `docker push` functionality. In short, for a particular variant of an image, we make a copy of its Dockerfile, append some Dockerfile syntax to add a custom entrypoint that allows us to execute tests against the running container, build a special, temporary version of the image (added time is insignificant, as most of the Dockerfile steps are cached), and run the tests. + +A particular variant is pushed to Docker Hub only if tests pass; if not, the same process restarts for the next variant, etc. + +#### Remaining work + +- Tests are very bare-bones right now and could use image-specific additions +- It is possible to convert `goss` test output to JUnit XML—it would be great to aggregate all this output and store it via CircleCI's Test Summary section +- The testing code is spread across the repository and is a bit confusing; some refactoring would help ## Limitations * The template language is WIP - it only supports `{{BASE_IMAGE}}` template. We should extend this. diff --git a/shared/goss/goss.yaml b/android/goss.yaml similarity index 100% rename from shared/goss/goss.yaml rename to android/goss.yaml diff --git a/buildpack-deps/goss.yaml b/buildpack-deps/goss.yaml new file mode 100644 index 00000000..4a424262 --- /dev/null +++ b/buildpack-deps/goss.yaml @@ -0,0 +1,13 @@ +file: + # ensure a basic shell exists + /bin/sh: + exists: true + + # ensure basic directory structure created + /home/circleci: + exists: true + +user: + # ensure circleci user exists + circleci: + exists: true diff --git a/clojure/goss.yaml b/clojure/goss.yaml new file mode 100644 index 00000000..4a424262 --- /dev/null +++ b/clojure/goss.yaml @@ -0,0 +1,13 @@ +file: + # ensure a basic shell exists + /bin/sh: + exists: true + + # ensure basic directory structure created + /home/circleci: + exists: true + +user: + # ensure circleci user exists + circleci: + exists: true diff --git a/dynamodb/goss.yaml b/dynamodb/goss.yaml new file mode 100644 index 00000000..5a657da4 --- /dev/null +++ b/dynamodb/goss.yaml @@ -0,0 +1,4 @@ +file: + # ensure a basic shell exists + /bin/sh: + exists: true diff --git a/elixir/goss.yaml b/elixir/goss.yaml new file mode 100644 index 00000000..4a424262 --- /dev/null +++ b/elixir/goss.yaml @@ -0,0 +1,13 @@ +file: + # ensure a basic shell exists + /bin/sh: + exists: true + + # ensure basic directory structure created + /home/circleci: + exists: true + +user: + # ensure circleci user exists + circleci: + exists: true diff --git a/golang/goss.yaml b/golang/goss.yaml new file mode 100644 index 00000000..4a424262 --- /dev/null +++ b/golang/goss.yaml @@ -0,0 +1,13 @@ +file: + # ensure a basic shell exists + /bin/sh: + exists: true + + # ensure basic directory structure created + /home/circleci: + exists: true + +user: + # ensure circleci user exists + circleci: + exists: true diff --git a/jruby/goss.yaml b/jruby/goss.yaml new file mode 100644 index 00000000..4a424262 --- /dev/null +++ b/jruby/goss.yaml @@ -0,0 +1,13 @@ +file: + # ensure a basic shell exists + /bin/sh: + exists: true + + # ensure basic directory structure created + /home/circleci: + exists: true + +user: + # ensure circleci user exists + circleci: + exists: true diff --git a/mariadb/goss.yaml b/mariadb/goss.yaml new file mode 100644 index 00000000..5a657da4 --- /dev/null +++ b/mariadb/goss.yaml @@ -0,0 +1,4 @@ +file: + # ensure a basic shell exists + /bin/sh: + exists: true diff --git a/mongo/goss.yaml b/mongo/goss.yaml new file mode 100644 index 00000000..5a657da4 --- /dev/null +++ b/mongo/goss.yaml @@ -0,0 +1,4 @@ +file: + # ensure a basic shell exists + /bin/sh: + exists: true diff --git a/mysql/goss.yaml b/mysql/goss.yaml new file mode 100644 index 00000000..5a657da4 --- /dev/null +++ b/mysql/goss.yaml @@ -0,0 +1,4 @@ +file: + # ensure a basic shell exists + /bin/sh: + exists: true diff --git a/node/goss.yaml b/node/goss.yaml new file mode 100644 index 00000000..4a424262 --- /dev/null +++ b/node/goss.yaml @@ -0,0 +1,13 @@ +file: + # ensure a basic shell exists + /bin/sh: + exists: true + + # ensure basic directory structure created + /home/circleci: + exists: true + +user: + # ensure circleci user exists + circleci: + exists: true diff --git a/openjdk/goss.yaml b/openjdk/goss.yaml new file mode 100644 index 00000000..4a424262 --- /dev/null +++ b/openjdk/goss.yaml @@ -0,0 +1,13 @@ +file: + # ensure a basic shell exists + /bin/sh: + exists: true + + # ensure basic directory structure created + /home/circleci: + exists: true + +user: + # ensure circleci user exists + circleci: + exists: true diff --git a/php/goss.yaml b/php/goss.yaml new file mode 100644 index 00000000..4a424262 --- /dev/null +++ b/php/goss.yaml @@ -0,0 +1,13 @@ +file: + # ensure a basic shell exists + /bin/sh: + exists: true + + # ensure basic directory structure created + /home/circleci: + exists: true + +user: + # ensure circleci user exists + circleci: + exists: true diff --git a/postgres/goss.yaml b/postgres/goss.yaml new file mode 100644 index 00000000..5a657da4 --- /dev/null +++ b/postgres/goss.yaml @@ -0,0 +1,4 @@ +file: + # ensure a basic shell exists + /bin/sh: + exists: true diff --git a/python/goss.yaml b/python/goss.yaml new file mode 100644 index 00000000..4a424262 --- /dev/null +++ b/python/goss.yaml @@ -0,0 +1,13 @@ +file: + # ensure a basic shell exists + /bin/sh: + exists: true + + # ensure basic directory structure created + /home/circleci: + exists: true + +user: + # ensure circleci user exists + circleci: + exists: true diff --git a/redis/goss.yaml b/redis/goss.yaml new file mode 100644 index 00000000..5a657da4 --- /dev/null +++ b/redis/goss.yaml @@ -0,0 +1,4 @@ +file: + # ensure a basic shell exists + /bin/sh: + exists: true diff --git a/ruby/goss.yaml b/ruby/goss.yaml new file mode 100644 index 00000000..4a424262 --- /dev/null +++ b/ruby/goss.yaml @@ -0,0 +1,13 @@ +file: + # ensure a basic shell exists + /bin/sh: + exists: true + + # ensure basic directory structure created + /home/circleci: + exists: true + +user: + # ensure circleci user exists + circleci: + exists: true diff --git a/rust/goss.yaml b/rust/goss.yaml new file mode 100644 index 00000000..4a424262 --- /dev/null +++ b/rust/goss.yaml @@ -0,0 +1,13 @@ +file: + # ensure a basic shell exists + /bin/sh: + exists: true + + # ensure basic directory structure created + /home/circleci: + exists: true + +user: + # ensure circleci user exists + circleci: + exists: true diff --git a/shared/goss/notes.md b/shared/goss/notes.md deleted file mode 100644 index d71fe9d2..00000000 --- a/shared/goss/notes.md +++ /dev/null @@ -1,42 +0,0 @@ -ideally, we'd set tests on a platform-specific level, since each image is slightly different. here is current status: - -### android - -### buildpack-deps -- container won't start for tests -- possible duplicate entrypoint? - -### clojure - - -### dynamodb - - -### elixir - - -### golang - - -### jruby - - -### mariadb - - -### mongo - - -### mysql - - -### node - - -### openjdk -### php -### postgres -### python -### redis -### ruby -### rust diff --git a/shared/images/build.sh b/shared/images/build.sh index 962915ff..6896f683 100755 --- a/shared/images/build.sh +++ b/shared/images/build.sh @@ -57,36 +57,37 @@ function run_goss_tests() { mkdir $GOSS_DOCKERFILE_PATH echo "----------------------------------------------------------------------------------------------------" - echo "copying Dockerfile to $GOSS_DOCKERFILE_PATH for testing modifications" + echo "copying Dockerfile to $GOSS_DOCKERFILE_PATH for testing modifications..." cp ~/circleci-bundles/$DOCKERFILE_PATH $GOSS_DOCKERFILE_PATH # cat our additions onto the Dockerfile copy echo "----------------------------------------------------------------------------------------------------" - echo "adding the following modifications to copied Dockerfile:" + echo "adding the following modifications to copied Dockerfile..." echo "----------------------------------------------------------------------------------------------------" cat ~/circleci-bundles/shared/goss/goss-add.Dockerfile cat ~/circleci-bundles/shared/goss/goss-add.Dockerfile >> $GOSS_DOCKERFILE_PATH/Dockerfile echo "----------------------------------------------------------------------------------------------------" - echo "copying custom entrypoint for testing:" + echo "copying custom entrypoint for testing..." echo "----------------------------------------------------------------------------------------------------" cat ~/circleci-bundles/shared/goss/goss-entrypoint.sh cp ~/circleci-bundles/shared/goss/goss-entrypoint.sh $GOSS_DOCKERFILE_PATH # build our test image echo "----------------------------------------------------------------------------------------------------" - echo "building modified test image: $IMAGE_NAME-goss" + echo "building modified test image: $IMAGE_NAME-goss..." echo "----------------------------------------------------------------------------------------------------" docker build -t $IMAGE_NAME-goss $GOSS_DOCKERFILE_PATH || (sleep 2; echo "retry building $IMAGE_NAME-goss"; docker build -t $IMAGE_NAME-goss $GOSS_DOCKERFILE_PATH) # run goss tests echo "----------------------------------------------------------------------------------------------------" - echo "running goss tests on $IMAGE_NAME-goss" + echo "running goss tests on $IMAGE_NAME-goss..." echo "----------------------------------------------------------------------------------------------------" dgoss run $IMAGE_NAME-goss echo "----------------------------------------------------------------------------------------------------" - echo "removing goss variant" + echo "removing goss variant..." + echo "----------------------------------------------------------------------------------------------------" docker image rm $IMAGE_NAME-goss echo "----------------------------------------------------------------------------------------------------" }