container_repro_test rule to test for container reproducibility (baze…

…lbuild#955)

* container_repro_test rule to test for container reproducibility

* buildifier and disable tests on buildkite

* test

* test

* buildifier

* address some comments

* test

* test with buildkite

* enable testing toolchain_container targets

* test without remote cache

* address comments

* buildifier

* test

* test

* address comments and use migrated base_image_docker rules

* fix test script

* address comments

.bazelci/presubmit.yml

@@ -66,6 +66,8 @@ platforms:
     build_targets:
     - "--"
     - "..."
+    - "-//tests/contrib:derivative_with_volume_repro_test"
+    - "-//tests/contrib:set_cmd_repro_test"
     build_flags:
     - "--extra_toolchains=@bazel_toolchains//configs/ubuntu16_04_clang/latest:toolchain_docker"
     - "--extra_execution_platforms=@bazel_toolchains//configs/ubuntu16_04_clang/latest:platform_docker"
@@ -81,6 +83,8 @@ platforms:
     - "--"
     - "//:structure_test_at_workspace_root"
     - "//tests/..."
+    - "-//tests/contrib:derivative_with_volume_repro_test"
+    - "-//tests/contrib:set_cmd_repro_test"
     - "-//tests/contrib:test_compare_ids_test_diff_ids_fails"
     - "-//tests/contrib:test_compare_ids_test_invalid_tar_fails"
     - "-//tests/contrib:test_compare_ids_test_invalid_tar_fails_multi_regex"

BUILD

@@ -21,6 +21,8 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
+exports_files(["WORKSPACE"])
+
 gazelle(
     name = "gazelle",
     prefix = "github.com/bazelbuild/rules_docker",

WORKSPACE

@@ -37,7 +37,7 @@ load("//repositories:deps.bzl", container_deps = "deps")
 
 container_deps()
 
-# pip deps are only needed for running  tests.
+# pip deps are only needed for running tests.
 load("//repositories:pip_repositories.bzl", "pip_deps")
 
 pip_deps()
@@ -47,7 +47,7 @@ load(
     "new_container_pull",
 )
 
-# These are for testing the new container push
+# These are for testing the new container pull
 new_container_pull(
     name = "new_alpine_linux_armv6",
     architecture = "arm",
@@ -219,6 +219,14 @@ container_pull(
     repository = "google-appengine/debian9",
 )
 
+# This image is used by tests/contrib tests.
+container_pull(
+    name = "bazel_0271",
+    digest = "sha256:436708ebb76c0089b94c46adac5d3332adb8c98ef8f24cb32274400d01bde9e3",
+    registry = "l.gcr.io",
+    repository = "google/bazel",
+)
+
 # Have the py_image dependencies for testing.
 load(
     "//python:image.bzl",

contrib/BUILD

@@ -25,6 +25,8 @@ exports_files(["compare_ids_test.bzl"])
 
 exports_files(["extract_image_id.py"])
 
+exports_files(["cmp_images.sh.tpl"])
+
 py_binary(
     name = "extract_image_id",
     srcs = [":extract_image_id.py"],

contrib/cmp_images.sh.tpl

@@ -0,0 +1,64 @@
+#!/bin/bash
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Testing script consumed by the container_repro_test rule that compares
+# two images.
+# Two images are considered to be identical when they both have the same
+# digest and ID. On a mismatch of one of those values, the test fails
+# and produces the summary of the images' differences.
+
+set -e
+
+imgs_differ=false
+
+function cmp_sha_files() {
+  local file1="${1}"
+  local file2="${2}"
+  local content_type="${3}"
+
+  local diff_ret=0
+  diff $file1 $file2 || diff_ret=$?
+  echo === Comparing image "${content_type}"s ===
+  if [ $diff_ret = 0 ]; then
+  	echo Both images have the same SHA256 "${content_type}": "$(<$file1)"
+  else
+  	echo Images have different SHA256 "${content_type}"s
+  	echo First image "${content_type}": "$(<$file1)"
+  	echo Reproduced image "${content_type}": "$(<$file2)"
+    imgs_differ=true
+  fi
+}
+
+# Compare image digests.
+img1_digest_file=%{img1_path}/%{img_name}.digest
+img2_digest_file=%{img2_path}/%{img_name}.digest
+cmp_sha_files $img1_digest_file $img2_digest_file "digest"
+
+# Compare image IDs
+img1_id_file="$(readlink -f %{img1_path}/%{img_name}.id)"
+img2_id_file="$(readlink -f %{img2_path}/%{img_name}.id)"
+cmp_sha_files "$img1_id_file" "$img2_id_file" "ID"
+
+# Run the container_diff tool if images differ.
+if [ "$imgs_differ" = true ]; then
+  echo === Images are different. Running container_diff tool ===
+  img1_tar=%{img1_path}/%{img_name}.tar
+  img2_tar=%{img2_path}/%{img_name}.tar
+  %{container_diff_tool} diff $img1_tar $img2_tar %{container_diff_args}
+  exit 1
+fi
+
+exit 0