Remote docker host (bazelbuild#1228)

* Bump httplib2 library to 0.11.3 for DigiCert Per httplib2/httplib2#91 Which adds the DigiCert Global Root G2 serial to cacert.txt Without this push_image() fails to registries trusted by the DigiCert CA. * added email for cla * added email for cla * added travis.ci * local puller * no label * use docker_tool_path in util * docker command * docker command * docker command * docker command * docker command * docker command * docker command * docker commands use quotes * docker commands use quotes * move flags to toolchain config * misplaced arguments * ran buildifier * more buildifier bs * added tests * added tests * proper linting * test confusion * tests for incremental_load and run_and_commit. * tests for incremental_load and run_and_commit. * making other tests run again after my changes * add docker flag docs * use symlink to test incremental_load template * attempt at tests with image.bzl coverage * add .executable to test scripts * pushed wrong workspace * fix compare test * changes per PR * reverted e2e test with executable * template was in run.bzl too! * fix dict order * do docker cp rather than vm * buildifier lint * fix non merging files * duplicate client_config * fix lint and duplicate client_config * checkout unmerged files * fix tests with double outs * fix e2e style tests for new executable. * correct executable names for test * fix executable in loop test Co-authored-by: Yu YI <[email protected]> Co-authored-by: Nicolas Lopez <[email protected]>
kublr · Jan 14, 2020 · fa5f719 · fa5f719
1 parent 5a6965f
commit fa5f719
Show file tree

Hide file tree

Showing 34 changed files with 339 additions and 63 deletions.
diff --git a/.bazelignore b/.bazelignore
@@ -3,3 +3,4 @@
 testing/examples
 testing/java_image
 testing/download_pkgs_at_root
+testing/custom_toolchain_flags
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
@@ -9,6 +9,7 @@
 # Names should be added to this file as:
 #     Name <email address>
 
+David Schile <[email protected]>
 Matthew Moore <[email protected]>
 Nathan Herring <[email protected]>
 Nicolas Lopez <[email protected]>

diff --git a/README.md b/README.md
@@ -150,6 +150,12 @@ docker_toolchain_configure(
   # OPTIONAL: Path to the xz binary.
   # Should be set explcitly for remote execution.
   xz_path="<enter absolute path to the xz binary (in the remote exec env) here>",
+  # OPTIONAL: List of additional flags to pass to the docker command.
+  docker_flags = [
+    "--tls",
+    "--log-level=info",
+  ],
+
 )
 # End of OPTIONAL segment.
 
@@ -2294,6 +2300,7 @@ Here's a (non-exhaustive) list of companies that use `rules_docker` in productio
   * [Evertz](https://evertz.com/)
   * [Jetstack](https://www.jetstack.io/)
   * [Kubernetes Container Image Promoter](https://github.com/kubernetes-sigs/k8s-container-image-promoter)
+  * [Nordstrom](https://nordstrom.com)
   * [Prow](https://github.com/kubernetes/test-infra/tree/master/prow)
   * [Tink](https://www.tink.com)
   * [Wix](https://www.wix.com)
diff --git a/WORKSPACE b/WORKSPACE
@@ -419,9 +419,9 @@ rbe_exec_properties(
     name = "exec_properties",
 )
 
+load("@bazel_skylib//lib:dicts.bzl", "dicts")
 load("@bazel_toolchains//rules:rbe_repo.bzl", "rbe_autoconfig")
 load("@exec_properties//:constants.bzl", "DOCKER_SIBLINGS_CONTAINERS", "NETWORK_ON")
-load("@bazel_skylib//lib:dicts.bzl", "dicts")
 
 rbe_autoconfig(
     name = "buildkite_config",

diff --git a/container/image.bzl b/container/image.bzl
@@ -330,11 +330,12 @@ def _impl(
     cmd = cmd or ctx.attr.cmd
     operating_system = operating_system or ctx.attr.operating_system
     creation_time = creation_time or ctx.attr.creation_time
-    output_executable = output_executable or ctx.outputs.executable
+    build_executable = output_executable or ctx.outputs.build_script
     output_tarball = output_tarball or ctx.outputs.out
     output_digest = output_digest or ctx.outputs.digest
     output_config = output_config or ctx.outputs.config
     output_layer = output_layer or ctx.outputs.layer
+    build_script = ctx.outputs.build_script
     null_cmd = null_cmd or ctx.attr.null_cmd
     null_entrypoint = null_entrypoint or ctx.attr.null_entrypoint
 
@@ -465,10 +466,11 @@ def _impl(
     _incr_load(
         ctx,
         images,
-        output_executable,
+        build_executable,
         run = not ctx.attr.legacy_run_behavior,
         run_flags = docker_run_flags,
     )
+
     _assemble_image(
         ctx,
         images,
@@ -496,7 +498,7 @@ def _impl(
             docker_run_flags = docker_run_flags,
         ),
         DefaultInfo(
-            executable = output_executable,
+            executable = build_executable,
             files = depset([output_layer]),
             runfiles = runfiles,
         ),
@@ -559,6 +561,8 @@ _outputs["digest"] = "%{name}.digest"
 
 _outputs["config"] = "%{name}.json"
 
+_outputs["build_script"] = "%{name}.executable"
+
 image = struct(
     attrs = _attrs,
     outputs = _outputs,

diff --git a/container/incremental_load.sh.tpl b/container/incremental_load.sh.tpl
@@ -32,6 +32,7 @@ function guess_runfiles() {
 RUNFILES="${PYTHON_RUNFILES:-$(guess_runfiles)}"
 
 DOCKER="%{docker_tool_path}"
+DOCKER_FLAGS="%{docker_flags}"
 
 if [[ -z "${DOCKER}" ]]; then
     echo >&2 "error: docker not found; do you need to manually configure the docker toolchain?"
@@ -43,7 +44,7 @@ TEMP_FILES="$(mktemp -t 2>/dev/null || mktemp -t 'rules_docker_files')"
 TEMP_IMAGES="$(mktemp -t 2>/dev/null || mktemp -t 'rules_docker_images')"
 function cleanup() {
   cat "${TEMP_FILES}" | xargs rm -rf> /dev/null 2>&1 || true
-  cat "${TEMP_IMAGES}" | xargs "${DOCKER}" rmi > /dev/null 2>&1 || true
+  cat "${TEMP_IMAGES}" | xargs "${DOCKER}" ${DOCKER_FLAGS} rmi > /dev/null 2>&1 || true
 
   rm -rf "${TEMP_FILES}"
   rm -rf "${TEMP_IMAGES}"
@@ -56,7 +57,7 @@ function load_legacy() {
 
   # docker load has elision of preloaded layers built in.
   echo "Loading legacy tarball base $1..."
-  "${DOCKER}" load -i "${tarball}"
+  "${DOCKER}" ${DOCKER_FLAGS} load -i "${tarball}"
 }
 
 function join_by() {
@@ -97,7 +98,7 @@ EOF
 EOF
 
   set -o pipefail
-  tar c config.json manifest.json | "${DOCKER}" load 2>/dev/null | cut -d':' -f 2- >> "${TEMP_IMAGES}"
+  tar c config.json manifest.json | "${DOCKER}" ${DOCKER_FLAGS} load 2>/dev/null | cut -d':' -f 2- >> "${TEMP_IMAGES}"
 }
 
 function find_diffbase() {
@@ -204,15 +205,15 @@ EOF
   # We minimize reads / writes by symlinking the layers above
   # and then streaming exactly the layers we've established are
   # needed into the Docker daemon.
-  tar cPh "${MISSING[@]}" | tee image.tar | "${DOCKER}" load
+  tar cPh "${MISSING[@]}" | tee image.tar | "${DOCKER}" ${DOCKER_FLAGS} load
 }
 
 function tag_layer() {
   local name="$(cat "${RUNFILES}/$2")"
 
   local TAG="$1"
   echo "Tagging ${name} as ${TAG}"
-  "${DOCKER}" tag sha256:${name} ${TAG}
+  "${DOCKER}" ${DOCKER_FLAGS} tag sha256:${name} ${TAG}
 }
 
 function read_variables() {

diff --git a/container/layer_tools.bzl b/container/layer_tools.bzl
@@ -261,12 +261,13 @@ def incremental_load(
         if run:
             # Args are embedded into the image, so omitted here.
             run_statements += [
-                "\"${DOCKER}\" run %s %s" % (run_flags, tag_reference),
+                "\"${DOCKER}\" ${DOCKER_FLAGS} run %s %s" % (run_flags, tag_reference),
             ]
 
     ctx.actions.expand_template(
         template = ctx.file.incremental_load_template,
         substitutions = {
+            "%{docker_flags}": " ".join(toolchain_info.docker_flags),
             "%{docker_tool_path}": toolchain_info.tool_path,
             "%{load_statements}": "\n".join(load_statements),
             "%{run_statements}": "\n".join(run_statements),

diff --git a/contrib/automatic_container_release/configs_test.bzl b/contrib/automatic_container_release/configs_test.bzl
@@ -69,6 +69,7 @@ def _impl(ctx):
         template = ctx.file._tpl,
         substitutions = {
             "%{cmd_args}": " ".join(cmd_args),
+            "%{docker_flags}": " ".join(toolchain_info.docker_flags),
             "%{docker_path}": toolchain_info.tool_path,
             "%{image_name}": ctx.attr._checker + ":" + ctx.attr.checker_tag,
             "%{spec_container_paths}": " ".join(spec_container_paths),

diff --git a/contrib/automatic_container_release/run_checker.sh.tpl b/contrib/automatic_container_release/run_checker.sh.tpl
@@ -39,14 +39,15 @@ RUNFILES="${PYTHON_RUNFILES:-$(guess_runfiles)}"
 
 # Resolve the docker tool path.
 DOCKER="%{docker_path}"
+DOCKER_FLAGS="%{docker_flags}"
 
 if [[ -z "$DOCKER" ]]; then
     echo >&2 "error: docker not found; do you need to manually configure the docker toolchain?"
     exit 1
 fi
 
 # Create a new docker container that will run the checker.
-container_id=$($DOCKER create %{image_name} %{cmd_args})
+container_id=$($DOCKER $DOCKER_FLAGS create %{image_name} %{cmd_args})
 
 specs=(%{specs})
 spec_container_paths=(%{spec_container_paths})
@@ -59,20 +60,20 @@ if [ ${#specs[@]} -ne ${#spec_container_paths[@]} ]; then
     exit 1
 fi
 for ((i=0;i<${#specs[@]};i++)); do
-    $DOCKER cp -L ${specs[$i]} $container_id:${spec_container_paths[$i]}
+    $DOCKER $DOCKER_FLAGS cp -L ${specs[$i]} $container_id:${spec_container_paths[$i]}
 done
 
 # Start the container that will run the checker logic.
-$DOCKER start $container_id
+$DOCKER $DOCKER_FLAGS start $container_id
 
 # Wait for the checker to finish running.
-retcode=$($DOCKER wait $container_id)
+retcode=$($DOCKER $DOCKER_FLAGS wait $container_id)
 
 # Print all logs generated by the container.
-$DOCKER logs $container_id
+$DOCKER $DOCKER_FLAGS logs $container_id
 
 # Delete the container.
-$DOCKER rm $container_id
+$DOCKER $DOCKER_FLAGS rm $container_id
 
 exit $retcode
 
diff --git a/contrib/compare_ids_test.bzl b/contrib/compare_ids_test.bzl
@@ -40,7 +40,10 @@ compare_ids_test(
 
 # Implementation of compare_ids_test
 def _compare_ids_test_impl(ctx):
-    tar_files = ctx.files.images
+    tar_files = []
+    for file in ctx.files.images:
+        if file.short_path.endswith("tar"):
+            tar_files += [file]
 
     if (len(tar_files) == 0):
         fail("No images provided for test.")

diff --git a/contrib/repro_test.bzl b/contrib/repro_test.bzl
@@ -113,7 +113,7 @@ def _impl(ctx):
         ctx,
         files = [proj_root],
         tars = [proj_tar],
-        output_executable = ctx.actions.declare_file(name + "_load.sh"),
+        output_executable = ctx.outputs.build_script,
         output_tarball = image_tar,
         workdir = "/",
     )

diff --git a/docker/package_managers/apt_key.bzl b/docker/package_managers/apt_key.bzl
@@ -47,7 +47,7 @@ def _impl(
     keys = keys or ctx.files.keys
     image_tar = image_tar or ctx.file.image
     gpg_image = gpg_image or ctx.file.gpg_image
-    output_executable = output_executable or ctx.outputs.executable
+    output_executable = output_executable or ctx.outputs.build_script
     output_tarball = output_tarball or ctx.outputs.out
     output_layer = output_layer or ctx.outputs.layer
     output_digest = output_digest or ctx.outputs.digest
@@ -58,6 +58,7 @@ def _impl(
 
     # If the user specified an alternate base for this, use it.
     # Otherwise use the same base image we want the key in.
+
     if gpg_image == None:
         gpg_image = image_tar
 

diff --git a/docker/package_managers/download_pkgs.bzl b/docker/package_managers/download_pkgs.bzl
@@ -103,6 +103,7 @@ def _impl(ctx, image_tar = None, packages = None, additional_repos = None, outpu
         template = ctx.file._run_download_tpl,
         output = output_script,
         substitutions = {
+            "%{docker_flags}": " ".join(toolchain_info.docker_flags),
             "%{docker_tool_path}": toolchain_info.tool_path,
             "%{download_commands}": _generate_download_commands(ctx, packages, additional_repos),
             "%{image_id_extractor_path}": ctx.executable._extract_image_id.path,
@@ -128,6 +129,7 @@ def _impl(ctx, image_tar = None, packages = None, additional_repos = None, outpu
         template = ctx.file._run_download_tpl,
         output = output_executable,
         substitutions = {
+            "%{docker_flags}": " ".join(toolchain_info.docker_flags),
             "%{docker_tool_path}": toolchain_info.tool_path,
             "%{download_commands}": _generate_download_commands(ctx, packages, additional_repos),
             "%{image_id_extractor_path}": "${RUNFILES}/%s" % runfile(ctx, ctx.executable._extract_image_id),

diff --git a/docker/package_managers/install_pkgs.bzl b/docker/package_managers/install_pkgs.bzl
@@ -93,6 +93,7 @@ def _impl(ctx, image_tar = None, installables_tar = None, installation_cleanup_c
         template = ctx.file._image_util_tpl,
         output = image_util,
         substitutions = {
+            "%{docker_flags}": " ".join(toolchain_info.docker_flags),
             "%{docker_tool_path}": toolchain_info.tool_path,
         },
         is_executable = True,
@@ -104,6 +105,7 @@ def _impl(ctx, image_tar = None, installables_tar = None, installation_cleanup_c
         output = script,
         substitutions = {
             "%{base_image_tar}": image_tar.path,
+            "%{docker_flags}": " ".join(toolchain_info.docker_flags),
             "%{docker_tool_path}": toolchain_info.tool_path,
             "%{image_id_extractor_path}": ctx.executable._extract_image_id.path,
             "%{installables_tar}": installables_tar_path,
@@ -195,6 +197,7 @@ _attrs = {
 
 _outputs = {
     "out": "%{name}.tar",
+    "build_script": "%{name}.build",
 }
 
 # Export install_pkgs rule for other bazel rules to depend on.

diff --git a/docker/package_managers/run_download.sh.tpl b/docker/package_managers/run_download.sh.tpl
@@ -16,6 +16,7 @@ RUNFILES="${PYTHON_RUNFILES:-$(guess_runfiles)}"
 
 # Resolve the docker tool path
 DOCKER="%{docker_tool_path}"
+DOCKER_FLAGS="%{docker_flags}"
 
 if [[ -z "$DOCKER" ]]; then
     echo >&2 "error: docker not found; do you need to manually configure the docker toolchain?"
@@ -24,12 +25,12 @@ fi
 
 # Load the image and remember its name
 image_id=$(%{image_id_extractor_path} %{image_tar})
-$DOCKER load -i %{image_tar}
+$DOCKER $DOCKER_FLAGS load -i %{image_tar}
 
 # Run the builder image.
-cid=$($DOCKER run -w="/" -d --privileged $image_id sh -c $'%{download_commands}')
-$DOCKER attach $cid
-$DOCKER cp $cid:%{installables}_packages.tar %{output}
-$DOCKER cp $cid:%{installables}_metadata.csv %{output_metadata}
+cid=$($DOCKER $DOCKER_FLAGS run -w="/" -d --privileged $image_id sh -c $'%{download_commands}')
+$DOCKER $DOCKER_FLAGS attach $cid
+$DOCKER $DOCKER_FLAGS cp $cid:%{installables}_packages.tar %{output}
+$DOCKER $DOCKER_FLAGS cp $cid:%{installables}_metadata.csv %{output_metadata}
 # Cleanup
-$DOCKER rm $cid
+$DOCKER $DOCKER_FLAGS rm $cid
diff --git a/docker/package_managers/run_install.sh.tpl b/docker/package_managers/run_install.sh.tpl
@@ -3,6 +3,7 @@ set -ex
 
 # Resolve the docker tool path
 DOCKER="%{docker_tool_path}"
+DOCKER_FLAGS="%{docker_flags}"
 
 if [[ -z "$DOCKER" ]]; then
     echo >&2 "error: docker not found; do you need to manually configure the docker toolchain?"
@@ -15,7 +16,7 @@ source %{util_script}
 
 # Load the image and remember its name
 image_id=$(%{image_id_extractor_path} %{base_image_tar})
-$DOCKER load -i %{base_image_tar}
+$DOCKER $DOCKER_FLAGS load -i %{base_image_tar}
 
 # Create a docker volume containing the installer script and the
 # installables TAR file.
@@ -40,18 +41,18 @@ cp -L $(pwd)/%{installer_script} $tmpdir/installer.sh
 # Temporarily create a container so we can mount the named volume
 # and copy files.  It's okay if /bin/true doesn't exist inside the
 # image; we are never going to run the image anyways.
-vid=$($DOCKER volume create)
-cid=$($DOCKER create -v $vid:/tmp/pkginstall $image_id /bin/true)
+vid=$($DOCKER $DOCKER_FLAGS volume create)
+cid=$($DOCKER $DOCKER_FLAGS create -v $vid:/tmp/pkginstall $image_id /bin/true)
 for f in $tmpdir/*; do
-    $DOCKER cp $f $cid:/tmp/pkginstall
+    $DOCKER $DOCKER_FLAGS cp $f $cid:/tmp/pkginstall
 done
-$DOCKER rm $cid
+$DOCKER $DOCKER_FLAGS rm $cid
 
-cid=$($DOCKER run -d -v $vid:/tmp/pkginstall --privileged $image_id /tmp/pkginstall/installer.sh)
+cid=$($DOCKER $DOCKER_FLAGS run -d -v $vid:/tmp/pkginstall --privileged $image_id /tmp/pkginstall/installer.sh)
 
-$DOCKER attach $cid || true
+$DOCKER $DOCKER_FLAGS attach $cid || true
 
 reset_cmd $image_id $cid %{output_image_name}
-$DOCKER save %{output_image_name} > %{output_file_name}
-$DOCKER rm $cid
-$DOCKER volume rm $vid
+$DOCKER $DOCKER_FLAGS save %{output_image_name} > %{output_file_name}
+$DOCKER $DOCKER_FLAGS rm $cid
+$DOCKER $DOCKER_FLAGS volume rm $vid
diff --git a/docker/toolchain_container/toolchain_container.bzl b/docker/toolchain_container/toolchain_container.bzl
@@ -169,7 +169,15 @@ def _language_tool_layer_impl(
         new_base = install_pkgs_out
 
     # Install tars and configure env, symlinks using the container_image rule.
-    result = _container.image.implementation(ctx, base = new_base, symlinks = symlinks, env = env, tars = tars, files = files)
+    result = _container.image.implementation(
+        ctx,
+        base = new_base,
+        symlinks = symlinks,
+        output_executable = ctx.outputs.build_script,
+        env = env,
+        tars = tars,
+        files = files,
+    )
 
     if hasattr(result[1], "runfiles"):
         result_runfiles = result[1].runfiles
@@ -178,6 +186,7 @@ def _language_tool_layer_impl(
 
     return [
         DefaultInfo(
+            executable = ctx.outputs.build_script,
             files = result[1].files,
             runfiles = result_runfiles,
         ),