From 6e133ff36d641d60bde5ee12909c576cc0674ac3 Mon Sep 17 00:00:00 2001 From: Hannes Kaeufler Date: Fri, 18 Oct 2024 22:19:25 +0200 Subject: [PATCH 1/6] Introduce new attribute for `created` datetime Doing a docker inspect --format='{{.Created}}' my/image:latest on an image built with oci_image previously always returned the static beginning of the unix timestamp (1. January 1970 00:00:00). To be more precise, that's what it returns for images that are built "from scratch". When an image is built on a base, then the created date is set to the created date of the base image. Both are fine strategies for reproducible builds, but in stamped builds that are eventually shipped/deployed, you probably want to set that time to the actual build time. To allow setting that, we introduce a separate file label, allowing to pass in a file containing a stamp variable. --- e2e/smoke/BUILD.bazel | 23 +++++++++++++++++++++++ e2e/smoke/expected_created.txt | 1 + e2e/smoke/workspace_status.sh | 5 +++++ oci/defs.bzl | 15 ++++++++++++++- oci/private/image.bzl | 9 +++++++++ oci/private/image.sh | 3 +++ 6 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 e2e/smoke/expected_created.txt create mode 100644 e2e/smoke/workspace_status.sh diff --git a/e2e/smoke/BUILD.bazel b/e2e/smoke/BUILD.bazel index c4918705..77eda18f 100644 --- a/e2e/smoke/BUILD.bazel +++ b/e2e/smoke/BUILD.bazel @@ -1,8 +1,17 @@ +load("@aspect_bazel_lib//lib:diff_test.bzl", "diff_test") load("@aspect_bazel_lib//lib:testing.bzl", "assert_json_matches") load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@container_structure_test//:defs.bzl", "container_structure_test") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load") +load("@aspect_bazel_lib//lib:expand_template.bzl", "expand_template") + +expand_template( + name = "created", + out = "created.txt", + stamp_substitutions = {"2000-01-01T01:02:03Z": "{{BUILD_ISO8601}}"}, + template = ["2000-01-01T01:02:03Z"], +) # SMOKE TEST: oci_image oci_image( @@ -12,6 +21,7 @@ oci_image( "--arg1", "--arg2", ], + created = ":created", entrypoint = ["/custom_bin"], env = { "ENV": "/test", @@ -62,6 +72,19 @@ assert_json_matches( filter1 = ".[0].RepoTags", ) +genrule( + name = "docker_created", + srcs = [":tarball.tar"], + outs = ["docker_created.txt"], + cmd = "docker load -i $(location :tarball.tar) && docker inspect --format='{{{{.Created}}}}' {} > $@".format(tags[0]), +) + +diff_test( + name = "test_created", + file1 = ":docker_created", + file2 = "expected_created.txt", +) + # SMOKE TEST: oci_image from an external repo build_test( name = "test_external", diff --git a/e2e/smoke/expected_created.txt b/e2e/smoke/expected_created.txt new file mode 100644 index 00000000..6acd0303 --- /dev/null +++ b/e2e/smoke/expected_created.txt @@ -0,0 +1 @@ +2000-01-01T01:02:03Z diff --git a/e2e/smoke/workspace_status.sh b/e2e/smoke/workspace_status.sh new file mode 100644 index 00000000..d4b1c58a --- /dev/null +++ b/e2e/smoke/workspace_status.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BUILD_TIMESTAMP=${BUILD_TIMESTAMP:-$(date +%s)} +BUILD_ISO8601=$(date -u -r "$BUILD_TIMESTAMP" +"%Y-%m-%dT%H:%M:%SZ") +echo "BUILD_ISO8601 $BUILD_ISO8601" diff --git a/oci/defs.bzl b/oci/defs.bzl index fab48105..845e9804 100644 --- a/oci/defs.bzl +++ b/oci/defs.bzl @@ -33,7 +33,17 @@ def _write_nl_seperated_file(name, kind, elems, forwarded_kwargs): ) return label -def oci_image(name, labels = None, annotations = None, env = None, cmd = None, entrypoint = None, exposed_ports = None, volumes = None, **kwargs): +def oci_image( + name, + created = None, + labels = None, + annotations = None, + env = None, + cmd = None, + entrypoint = None, + exposed_ports = None, + volumes = None, + **kwargs): """Macro wrapper around [oci_image_rule](#oci_image_rule). Allows labels and annotations to be provided as a dictionary, in addition to a text file. @@ -49,6 +59,8 @@ def oci_image(name, labels = None, annotations = None, env = None, cmd = None, e Args: name: name of resulting oci_image_rule + created: Label to a file containing a single datetime string. + The content of that file is used as the value of the `created` field in the image config. labels: Labels for the image config. May either be specified as a file, as with the documentation above, or a dict of strings to specify values inline. annotations: Annotations for the image config. @@ -136,6 +148,7 @@ def oci_image(name, labels = None, annotations = None, env = None, cmd = None, e oci_image_rule( name = name, + created = created, annotations = annotations, labels = labels, env = env, diff --git a/oci/private/image.bzl b/oci/private/image.bzl index 99558e91..73a0d573 100644 --- a/oci/private/image.bzl +++ b/oci/private/image.bzl @@ -61,6 +61,11 @@ oci_image( """ _attrs = { "base": attr.label(allow_single_file = True, doc = "Label to an oci_image target to use as the base."), + "created": attr.label(allow_single_file = True, doc = """\ + The datetime when the image was created. This can be a file containing a string in the format `YYYY-MM-DDTHH:MM:SS.sssZ` + Typically, you'd provide a file containing a stamp variable replaced by the datetime of the build + when executed with `--stamp`. + """), "tars": attr.label_list(allow_files = _ACCEPTED_TAR_EXTENSIONS, doc = """\ List of tar files to add to the image as layers. Do not sort this list; the order is preserved in the resulting image. @@ -192,6 +197,10 @@ def _oci_image_impl(ctx): # tars are already added as input above. args.add_joined([layer, descriptor], join_with = "=", format_joined = "--layer=%s") + if ctx.attr.created: + args.add(ctx.file.created.path, format = "--created=%s") + inputs.append(ctx.file.created) + # WARNING: entrypoint should always be added before the cmd argument. # This due to implicit behavior which setting entrypoint deletes `cmd`. # See: https://github.com/bazel-contrib/rules_oci/issues/649 diff --git a/oci/private/image.sh b/oci/private/image.sh index 13f1a480..32cb6ee2 100644 --- a/oci/private/image.sh +++ b/oci/private/image.sh @@ -169,6 +169,9 @@ for ARG in "$@"; do --labels=*) CONFIG=$(jq --rawfile labels "${ARG#--labels=}" '.config.Labels += ($labels | split("\n") | map(select(. | length > 0)) | map(. | split("=")) | map({key: .[0], value: .[1:] | join("=")}) | from_entries)' <<<"$CONFIG") ;; + --created=*) + CONFIG=$(jq --rawfile created "${ARG#--created=}" '.created = $created' <<<"$CONFIG") + ;; --annotations=*) get_manifest | jq --rawfile annotations "${ARG#--annotations=}" \ From 30e75dd4b65f5f95b53441b70b7cdd9c32d2a251 Mon Sep 17 00:00:00 2001 From: Hannes Kaeufler Date: Fri, 25 Oct 2024 22:19:21 +0200 Subject: [PATCH 2/6] Update docs with `create` attribute --- docs/image.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/image.md b/docs/image.md index 98522cb8..9fcc7166 100644 --- a/docs/image.md +++ b/docs/image.md @@ -11,8 +11,8 @@ load("@rules_oci//oci:defs.bzl", ...) ## oci_image_rule 
-oci_image_rule(name, annotations, architecture, base, cmd, entrypoint, env, exposed_ports, labels,
-               os, resource_set, tars, user, variant, volumes, workdir)
+oci_image_rule(name, annotations, architecture, base, cmd, created, entrypoint, env, exposed_ports,
+               labels, os, resource_set, tars, user, variant, volumes, workdir)
Build an OCI compatible container image. @@ -72,6 +72,7 @@ oci_image( | architecture | The CPU architecture which the binaries in this image are built to run on. eg: `arm64`, `arm`, `amd64`, `s390x`. See $GOARCH documentation for possible values: https://go.dev/doc/install/source#environment | String | optional | `""` | | base | Label to an oci_image target to use as the base. | Label | optional | `None` | | cmd | A file containing a newline separated list to be used as the `command & args` of the container. These values act as defaults and may be replaced by any specified when creating a container. | Label | optional | `None` | +| created | The datetime when the image was created. This can be a file containing a string in the format `YYYY-MM-DDTHH:MM:SS.sssZ` Typically, you'd provide a file containing a stamp variable replaced by the datetime of the build when executed with `--stamp`. | Label | optional | `None` | | entrypoint | A file containing a newline separated list to be used as the `entrypoint` to execute when the container starts. These values act as defaults and may be replaced by an entrypoint specified when creating a container. NOTE: Setting this attribute will reset the `cmd` attribute | Label | optional | `None` | | env | A file containing the default values for the environment variables of the container. These values act as defaults and are merged with any specified when creating a container. Entries replace the base environment variables if any of the entries has conflicting keys. To merge entries with keys specified in the base, `${KEY}` or `$KEY` syntax may be used. | Label | optional | `None` | | exposed_ports | A file containing a comma separated list of exposed ports. (e.g. 2000/tcp, 3000/udp or 4000. No protocol defaults to tcp). | Label | optional | `None` | @@ -90,7 +91,7 @@ oci_image( ## oci_image 
-oci_image(name, labels, annotations, env, cmd, entrypoint, exposed_ports, volumes, kwargs)
+oci_image(name, created, labels, annotations, env, cmd, entrypoint, exposed_ports, volumes, kwargs)
Macro wrapper around [oci_image_rule](#oci_image_rule). @@ -113,6 +114,7 @@ This is similar to the same-named target created by rules_docker's `container_im | Name | Description | Default Value | | :------------- | :------------- | :------------- | | name | name of resulting oci_image_rule | none | +| created | Label to a file containing a single datetime string. The content of that file is used as the value of the `created` field in the image config. | `None` | | labels | Labels for the image config. May either be specified as a file, as with the documentation above, or a dict of strings to specify values inline. | `None` | | annotations | Annotations for the image config. May either be specified as a file, as with the documentation above, or a dict of strings to specify values inline. | `None` | | env | Environment variables provisioned by default to the running container. May either be specified as a file, as with the documentation above, or a dict of strings to specify values inline. | `None` | From 62f0aa57186559adf63c506c05a1dc10a6bc1901 Mon Sep 17 00:00:00 2001 From: Hannes Kaeufler Date: Fri, 25 Oct 2024 22:26:41 +0200 Subject: [PATCH 3/6] Make clear that workspace_status command is an example for macOS --- e2e/smoke/workspace_status.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/e2e/smoke/workspace_status.sh b/e2e/smoke/workspace_status.sh index d4b1c58a..b1089c13 100644 --- a/e2e/smoke/workspace_status.sh +++ b/e2e/smoke/workspace_status.sh @@ -1,5 +1,7 @@ #!/bin/bash -BUILD_TIMESTAMP=${BUILD_TIMESTAMP:-$(date +%s)} +# This serves as an example of how the workspace_status.sh script can be used to +# stamp the `created` attribute of an image. +BUILD_TIMESTAMP=${BUILD_TIMESTAMP:-$(date +%s)} # macOS specific BUILD_ISO8601=$(date -u -r "$BUILD_TIMESTAMP" +"%Y-%m-%dT%H:%M:%SZ") echo "BUILD_ISO8601 $BUILD_ISO8601" From 3a90fc39bc0cbe79838861ee0bf8785a5a7fde78 Mon Sep 17 00:00:00 2001 From: Hannes Kaeufler Date: Fri, 25 Oct 2024 22:37:54 +0200 Subject: [PATCH 4/6] Allow usage of templating toolchain Registers dependencies as per installation instructions. --- e2e/smoke/WORKSPACE.bazel | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/e2e/smoke/WORKSPACE.bazel b/e2e/smoke/WORKSPACE.bazel index ce239399..29f83b89 100644 --- a/e2e/smoke/WORKSPACE.bazel +++ b/e2e/smoke/WORKSPACE.bazel @@ -14,6 +14,12 @@ http_archive( url = "https://github.com/aspect-build/bazel-lib/releases/download/v2.7.2/bazel-lib-v2.7.2.tar.gz", ) +load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies", "aspect_bazel_lib_register_toolchains") + +aspect_bazel_lib_dependencies() + +aspect_bazel_lib_register_toolchains() + http_archive( name = "container_structure_test", sha256 = "4fd1e0d4974fb95e06d0e94e6ceaae126382bf958524062db4e582232590b863", From f40862e6b0f327aa561f5b39be618cc40415c52e Mon Sep 17 00:00:00 2001 From: Hannes Kaeufler Date: Fri, 25 Oct 2024 22:39:06 +0200 Subject: [PATCH 5/6] Remove workspace_status example It's not used and has portability issues --- e2e/smoke/workspace_status.sh | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 e2e/smoke/workspace_status.sh diff --git a/e2e/smoke/workspace_status.sh b/e2e/smoke/workspace_status.sh deleted file mode 100644 index b1089c13..00000000 --- a/e2e/smoke/workspace_status.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -# This serves as an example of how the workspace_status.sh script can be used to -# stamp the `created` attribute of an image. -BUILD_TIMESTAMP=${BUILD_TIMESTAMP:-$(date +%s)} # macOS specific -BUILD_ISO8601=$(date -u -r "$BUILD_TIMESTAMP" +"%Y-%m-%dT%H:%M:%SZ") -echo "BUILD_ISO8601 $BUILD_ISO8601" From 8f801a8a2ceaba5b999533a2355e739539d81ca9 Mon Sep 17 00:00:00 2001 From: Hannes Kaeufler Date: Fri, 25 Oct 2024 22:41:27 +0200 Subject: [PATCH 6/6] Explain unused stamp identifier --- e2e/smoke/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/smoke/BUILD.bazel b/e2e/smoke/BUILD.bazel index 77eda18f..3079c557 100644 --- a/e2e/smoke/BUILD.bazel +++ b/e2e/smoke/BUILD.bazel @@ -9,7 +9,7 @@ load("@aspect_bazel_lib//lib:expand_template.bzl", "expand_template") expand_template( name = "created", out = "created.txt", - stamp_substitutions = {"2000-01-01T01:02:03Z": "{{BUILD_ISO8601}}"}, + stamp_substitutions = {"2000-01-01T01:02:03Z": "{{BUILD_ISO8601}}"}, # BUILD_ISO8601 is an imaginary stamp var template = ["2000-01-01T01:02:03Z"], )