From 0b2a141d835d1ac06c22f7bdee487c1af30a440b Mon Sep 17 00:00:00 2001 From: Karl Rister Date: Tue, 6 Aug 2024 16:51:27 -0500 Subject: [PATCH 1/8] modify the existing image identification logic to use the local image id rather than the : - some repository sources, such as the "dir:/" option, will not result in a usable : name once pulled locally so using the local image id is the only way to properly identify the image --- workshop.pl | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/workshop.pl b/workshop.pl index 8838c94..8cd915f 100755 --- a/workshop.pl +++ b/workshop.pl @@ -919,6 +919,7 @@ sub delete_proto { } my $container_mount_point; +my $origin_image_id; # acquire the userenv from the origin logger('info', "Looking for container base image...\n"); @@ -928,6 +929,8 @@ sub delete_proto { command_logger('verbose', $command, $rc, $command_output); $command_output = filter_output($command_output); $userenv_json->{'userenv'}{'origin'}{'local_details'} = decode_json($command_output); + + $origin_image_id = $userenv_json->{'userenv'}{'origin'}{'local_details'}[0]{'id'}; } else { command_logger('verbose', $command, $rc, $command_output); logger('info', "Could not find $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}, attempting to download...\n", 1); @@ -936,8 +939,12 @@ sub delete_proto { logger('info', "succeeded\n", 2); command_logger('verbose', $command, $rc, $command_output); + $command_output = filter_output($command_output); + chomp($command_output); + $origin_image_id = $command_output; + logger('info', "Querying for information about the image...\n", 1); - ($command, $command_output, $rc) = run_command("buildah images --json " . delete_proto($userenv_json->{'userenv'}{'origin'}{'image'}) . ":$userenv_json->{'userenv'}{'origin'}{'tag'}"); + ($command, $command_output, $rc) = run_command("buildah images --json " . $origin_image_id); if ($rc == 0) { logger('info', "succeeded\n", 2); command_logger('verbose', $command, $rc, $command_output); @@ -946,7 +953,7 @@ sub delete_proto { } else { logger('info', "failed\n", 2); command_logger('error', $command, $rc, $command_output); - logger('error', "Failed to download/query $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}!\n"); + logger('error', "Failed to download/query $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'} ($origin_image_id)!\n"); exit(get_exit_code('image_query')); } } else { @@ -1093,11 +1100,11 @@ sub delete_proto { # create a new container based on the userenv source logger('info', "Creating temporary container...\n"); -($command, $command_output, $rc) = run_command("buildah from --name $tmp_container $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}"); +($command, $command_output, $rc) = run_command("buildah from --name $tmp_container $origin_image_id"); if ($rc != 0) { logger('info', "failed\n", 1); command_logger('error', $command, $rc, $command_output); - logger('error', "Could not create new container '$tmp_container' from '$userenv_json->{'origin'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}'!\n"); + logger('error', "Could not create new container '$tmp_container' from '$userenv_json->{'origin'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}' ($origin_image_id)!\n"); exit(get_exit_code('create_container')); } else { logger('info', "succeeded\n", 1); From 97efc86f30ad503e3da53505736cd29c8ea497de Mon Sep 17 00:00:00 2001 From: Karl Rister Date: Wed, 7 Aug 2024 09:22:39 -0500 Subject: [PATCH 2/8] add the (optional) ability for the userenv to specify an update policy - if the userenv specifies "missing" (or nothing) for this property then no changes to the generated config information is made - if the userenv specifies "digest" for this property then the origin image's digest is included in the origin image config data -- allowing the caller(s) (ie. rickshaw) to make use of this information when determining if a new image should be built --- schema.json | 10 +++++- workshop.pl | 91 +++++++++++++++++++++++++++++++++++------------------ 2 files changed, 69 insertions(+), 32 deletions(-) diff --git a/schema.json b/schema.json index a044612..ce241da 100644 --- a/schema.json +++ b/schema.json @@ -74,7 +74,8 @@ "2020.04.30", "2022.07.25", "2023.02.16", - "2024.03.22" + "2024.03.22", + "2024.08.07" ] } }, @@ -110,6 +111,13 @@ "tag": { "type": "string", "minLength": 1 + }, + "update-policy": { + "type": "string", + "enum": [ + "missing", + "digest" + ] } }, "required": [ diff --git a/workshop.pl b/workshop.pl index 8cd915f..cafedb5 100755 --- a/workshop.pl +++ b/workshop.pl @@ -149,7 +149,9 @@ sub get_exit_code { 'package_remove' => 93, 'group_remove' => 94, 'architecture_query_failed' => 95, - 'unsupported_platform_architecture' => 96 + 'unsupported_platform_architecture' => 96, + 'skopeo_inspect_failed' => 97, + 'skopeo_digest_missing' => 98 ); if (exists($reasons{$exit_reason})) { @@ -581,6 +583,13 @@ sub delete_proto { } } +if (! exists $userenv_json->{'userenv'}{'origin'}{'update-policy'}) { + # if the loaded json does not include an update policy then + # default to "missing" which results in the same behavior we have + # always had + $userenv_json->{'userenv'}{'origin'}{'update-policy'} = "missing"; +} + if (exists $userenv_json->{'userenv'}{'properties'}{'platform'}) { # the userenv has platform information that indicates what type of # system architecture(s) it supports so validate that what we are @@ -882,10 +891,41 @@ sub delete_proto { } if ($args{'dump-config'} eq 'true') { - logger('info', "Config dump:\n"); - my %config_dump = (); + if ($userenv_json->{'userenv'}{'origin'}{'update-policy'} eq 'digest') { + # get the userenv digest to include in the config dump -- this + # can be used by callers that are analyzing the dumped config + # to know whether or not the origin image has changed + my $image_id = $userenv_json->{'userenv'}{'origin'}{'image'} . ":" . $userenv_json->{'userenv'}{'origin'}{'tag'}; + my $skopeo_url; + if (($image_id =~ /^dir:/) || ($image_id =~ /^docker:\/\//)) { + $skopeo_url = $image_id; + } else { + $skopeo_url = "docker://" . $image_id; + } + logger('info', "Querying for origin image digest...\n", 1); + ($command, $command_output, $rc) = run_command("skopeo inspect " . $skopeo_url); + if ($rc == 0) { + logger('info', "succeeded\n", 2); + command_logger('verbose', $command, $rc, $command_output); + + $command_output = filter_output($command_output); + my $skopeo_json = decode_json($command_output); + if (exists ($skopeo_json->{'Digest'})) { + $userenv_json->{'userenv'}{'origin'}{'digest'} = $skopeo_json->{'Digest'}; + } else { + logger('error', "Query results do not contain a digest"); + exit(get_exit_code('skopeo_digest_missing')); + } + } else { + logger('info', "failed\n", 2); + command_logger('error', $command, $rc, $command_output); + logger('error', "Failed to query " . $skopeo_url); + exit(get_exit_code('skopeo_inspect_failed')); + } + } + # consolidate information to be dumped $config_dump{'userenv'} = $userenv_json; $config_dump{'requirements'} = $active_requirements{'array'}; @@ -899,6 +939,7 @@ sub delete_proto { delete $config_dump{'requirements'}[$i]{'sha256'}; } + logger('info', "Config dump:\n"); logger('info', Data::Dumper->Dump([\%config_dump], [qw(*config_dump)])); exit() @@ -922,46 +963,34 @@ sub delete_proto { my $origin_image_id; # acquire the userenv from the origin -logger('info', "Looking for container base image...\n"); -($command, $command_output, $rc) = run_command("buildah images --json " . delete_proto($userenv_json->{'userenv'}{'origin'}{'image'}) . ":$userenv_json->{'userenv'}{'origin'}{'tag'}"); +logger('info', "Attempting to download the latest version of $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}...\n", 1); +($command, $command_output, $rc) = run_command("buildah pull --quiet --tls-verify=$args{'reg-tls-verify'} $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}"); if ($rc == 0) { - logger('info', "Found $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'} locally\n", 1); + logger('info', "succeeded\n", 2); command_logger('verbose', $command, $rc, $command_output); + $command_output = filter_output($command_output); - $userenv_json->{'userenv'}{'origin'}{'local_details'} = decode_json($command_output); + chomp($command_output); + $origin_image_id = $command_output; - $origin_image_id = $userenv_json->{'userenv'}{'origin'}{'local_details'}[0]{'id'}; -} else { - command_logger('verbose', $command, $rc, $command_output); - logger('info', "Could not find $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}, attempting to download...\n", 1); - ($command, $command_output, $rc) = run_command("buildah pull --quiet --tls-verify=$args{'reg-tls-verify'} $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}"); + logger('info', "Querying for information about the image...\n", 1); + ($command, $command_output, $rc) = run_command("buildah images --json " . $origin_image_id); if ($rc == 0) { logger('info', "succeeded\n", 2); command_logger('verbose', $command, $rc, $command_output); - $command_output = filter_output($command_output); - chomp($command_output); - $origin_image_id = $command_output; - - logger('info', "Querying for information about the image...\n", 1); - ($command, $command_output, $rc) = run_command("buildah images --json " . $origin_image_id); - if ($rc == 0) { - logger('info', "succeeded\n", 2); - command_logger('verbose', $command, $rc, $command_output); - $command_output = filter_output($command_output); - $userenv_json->{'userenv'}{'origin'}{'local_details'} = decode_json($command_output); - } else { - logger('info', "failed\n", 2); - command_logger('error', $command, $rc, $command_output); - logger('error', "Failed to download/query $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'} ($origin_image_id)!\n"); - exit(get_exit_code('image_query')); - } + $userenv_json->{'userenv'}{'origin'}{'local_details'} = decode_json($command_output); } else { logger('info', "failed\n", 2); command_logger('error', $command, $rc, $command_output); - logger('error', "Failed to download $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}!\n"); - exit(get_exit_code('image_origin_pull')); + logger('error', "Failed to query $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'} ($origin_image_id)!\n"); + exit(get_exit_code('image_query')); } +} else { + logger('info', "failed\n", 2); + command_logger('error', $command, $rc, $command_output); + logger('error', "Failed to download $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}!\n"); + exit(get_exit_code('image_origin_pull')); } logger('debug', "Userenv JSON:\n"); From 2bec3e77da0079e2d1e22f3504d29848d306f134 Mon Sep 17 00:00:00 2001 From: Karl Rister Date: Wed, 7 Aug 2024 14:11:44 -0500 Subject: [PATCH 3/8] allow the workshop.pl invocation to override the userenv's defined/default update policy --- workshop.pl | 50 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/workshop.pl b/workshop.pl index cafedb5..36bca51 100755 --- a/workshop.pl +++ b/workshop.pl @@ -47,13 +47,14 @@ BEGIN $args{'param'} = {}; $args{'reg-tls-verify'} = 'true'; -my @cli_args = ( '--log-level', '--requirements', '--skip-update', '--userenv', '--force', '--config', '--dump-config', '--dump-files' ); +my @cli_args = ( '--log-level', '--requirements', '--skip-update', '--userenv', '--force', '--config', '--dump-config', '--dump-files', '--force-update-policy' ); my %log_levels = ( 'info' => 1, 'verbose' => 1, 'debug' => 1 ); my %update_options = ( 'true' => 1, 'false' => 1 ); my %force_options = ( 'true' => 1, 'false' => 1 ); my %dump_config_options = ( 'true' => 1, 'false' => 1); my %dump_files_options = ( 'true' => 1, 'false' => 1); my %reg_tls_verify_options = ( 'true' => 1, 'false' => 1); +my %force_update_policy_options = ( 'missing' => 1, 'digest' => 1 ); my @virtual_fs = ('dev', 'proc', 'sys'); @@ -363,16 +364,17 @@ sub usage { logger("info", "\n"); logger("info", "Optional arguments: (* denotes default)\n"); logger("info", "\n"); - logger("info", "--requirements Requirements file (can be used multiple times)\n"); - logger("info", "--label Label to apply to container image\n"); - logger("info", "--config Container config file\n"); - logger("info", "--log-level Control logging output\n"); - logger("info", "--skip-update Should the container run it's distro update function\n"); - logger("info", "--force Force the container build\n"); - logger("info", "--dump-config Dump the config instead of building the container\n"); - logger("info", "--dump-files Dump the files that are being manually handled\n"); - logger("info", "--param = When is found in the userenv and/or requirements file, substitute for it\n"); - logger("info", "--reg-tls-verify Use TLS for remote registry actions\n"); + logger("info", "--requirements Requirements file (can be used multiple times)\n"); + logger("info", "--label Label to apply to container image\n"); + logger("info", "--config Container config file\n"); + logger("info", "--log-level Control logging output\n"); + logger("info", "--skip-update Should the container run it's distro update function\n"); + logger("info", "--force Force the container build\n"); + logger("info", "--dump-config Dump the config instead of building the container\n"); + logger("info", "--dump-files Dump the files that are being manually handled\n"); + logger("info", "--param = When is found in the userenv and/or requirements file, substitute for it\n"); + logger("info", "--reg-tls-verify Use TLS for remote registry actions\n"); + logger("info", "--force-update-policy Override the userenv's specified update policy\n"); logger("info", "\n"); } @@ -499,6 +501,12 @@ sub arg_handler { } else { die("--reg-tls-verify must be one of 'true' or 'false' [not '$opt_value']"); } + } elsif ($opt_name eq "force-update-policy") { + if (exists ($force_update_policy_options{$opt_value})) { + $args{'force-update-policy'} = $opt_value; + } else { + die("--force-update-policy must be one of 'missing' or 'digest' [not '$opt_value']"); + } } else { die("I'm confused, how did I get here [$opt_name]?"); } @@ -524,7 +532,8 @@ sub delete_proto { "param=s" => \&arg_handler, "dump-config=s" => \&arg_handler, "dump-files=s" => \&arg_handler, - "reg-tls-verify=s" => \&arg_handler)) { + "reg-tls-verify=s" => \&arg_handler, + "force-update-policy=s" => \&arg_handler)) { usage(); die("Error in command line arguments"); } @@ -893,7 +902,22 @@ sub delete_proto { if ($args{'dump-config'} eq 'true') { my %config_dump = (); - if ($userenv_json->{'userenv'}{'origin'}{'update-policy'} eq 'digest') { + my $include_digest = 0; + if (exists ($args{'force-update-policy'})) { + if ($args{'force-update-policy'} eq 'digest') { + $include_digest = 1; + } elsif ($args{'force-update-policy'} eq 'missing') { + $include_digest = 0; + } + } elsif ($userenv_json->{'userenv'}{'origin'}{'update-policy'} eq 'digest') { + $include_digest = 1; + } elsif ($userenv_json->{'userenv'}{'origin'}{'update-policy'} eq 'missing') { + $include_digest = 0; + } else { + die("I'm confused, how did I get here [include_digest]?") + } + + if ($include_digest == 1) { # get the userenv digest to include in the config dump -- this # can be used by callers that are analyzing the dumped config # to know whether or not the origin image has changed From 684afbd03a3a90663e7d881e922743b3e44c712f Mon Sep 17 00:00:00 2001 From: Karl Rister Date: Wed, 7 Aug 2024 14:27:59 -0500 Subject: [PATCH 4/8] update workshop-ci - drop old fedora userenvs tied to a name - create a new fedora-ci userenv that can be updated without renaming it - add update-policy option testing --- .github/workflows/workshop-ci.yaml | 8 +- userenvs/{fedora36-ci.json => fedora-ci.json} | 13 +- userenvs/fedora33.json | 121 ------------------ 3 files changed, 12 insertions(+), 130 deletions(-) rename userenvs/{fedora36-ci.json => fedora-ci.json} (78%) delete mode 100644 userenvs/fedora33.json diff --git a/.github/workflows/workshop-ci.yaml b/.github/workflows/workshop-ci.yaml index 9d8e231..c51897d 100644 --- a/.github/workflows/workshop-ci.yaml +++ b/.github/workflows/workshop-ci.yaml @@ -41,8 +41,10 @@ jobs: run: | export TOOLBOX_HOME=~/toolbox echo - sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --label workshop-ci-dump-config --userenv ./userenvs/fedora36-ci.json --dump-config true + sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --label workshop-ci-dump-config --userenv ./userenvs/fedora-ci.json --dump-config true echo - sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --label workshop-ci-dump-files --userenv ./userenvs/fedora36-ci.json --dump-files true + sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --label workshop-ci-dump-config --userenv ./userenvs/fedora-ci.json --dump-config true --force-update-policy missing echo - sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --label workshop-ci --userenv ./userenvs/fedora36-ci.json --log-level verbose + sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --label workshop-ci-dump-files --userenv ./userenvs/fedora-ci.json --dump-files true + echo + sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --label workshop-ci --userenv ./userenvs/fedora-ci.json --log-level verbose diff --git a/userenvs/fedora36-ci.json b/userenvs/fedora-ci.json similarity index 78% rename from userenvs/fedora36-ci.json rename to userenvs/fedora-ci.json index 3b16fee..2b69a4e 100644 --- a/userenvs/fedora36-ci.json +++ b/userenvs/fedora-ci.json @@ -1,15 +1,16 @@ { "workshop": { "schema": { - "version": "2023.02.16" + "version": "2024.08.07" } }, "userenv": { - "name": "fedora36", - "label": "Fedora 36", + "name": "fedora40", + "label": "Fedora 40", "origin": { "image": "docker.io/library/fedora", - "tag": "36" + "tag": "40", + "update-policy": "digest" }, "properties": { "platform": [ @@ -28,11 +29,11 @@ }, "requirements": [ { - "name": "python39", + "name": "python3", "type": "distro", "distro_info": { "packages": [ - "python39" + "python3" ] } }, diff --git a/userenvs/fedora33.json b/userenvs/fedora33.json deleted file mode 100644 index 4be7d99..0000000 --- a/userenvs/fedora33.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "workshop": { - "schema": { - "version": "2020.03.02" - } - }, - "userenv": { - "name": "fedora33", - "label": "Fedora 33", - "origin": { - "image": "registry.fedoraproject.org/fedora", - "tag": "33" - }, - "properties": { - "packages": { - "type": "rpm", - "manager": "dnf" - } - } - }, - "requirements": [ - { - "name": "python39", - "type": "distro", - "distro_info": { - "packages": [ - "python39", - "python3-pip" - ] - } - }, - { - "name": "utils", - "type": "distro", - "distro_info": { - "packages": [ - "curl", - "tar", - "cpio", - "xz", - "gzip", - "jq", - "git", - "cpio", - "findutils", - "hostname", - "iputils", - "iproute" - ] - } - }, - { - "name": "core-perl", - "type": "distro", - "distro_info": { - "packages": [ - "perl-App-cpanminus" - ] - } - }, - { - "name": "core-node", - "type": "distro", - "distro_info": { - "packages": [ - "nodejs" - ] - } - }, - { - "name": "core-compiling", - "type": "distro", - "distro_info": { - "packages": [ - "diffutils", - "gcc", - "libtool", - "autoconf", - "automake", - "make" - ] - } - }, - { - "name": "my-perl-deps", - "type": "cpan", - "cpan_info": { - "packages": [ - "Coro JSON JSON::XS JSON::Validator Data::Dumper" - ] - } - }, - { - "name": "my-python-deps", - "type": "python3", - "python3_info": { - "packages": [ - "NumPy pandas Keras Pillow" - ] - } - }, - { - "name": "some-node-deps", - "type": "node", - "node_info": { - "packages": [ - "ansi-regex sax react" - ] - } - }, - { - "name": "more-perl-deps", - "type": "cpan", - "cpan_info": { - "packages": [ - "Data::UUID Digest::SHA Digest::MD5 Getopt::Long DBI DBD::SQLite Time::HiRes" - ] - } - } - ] -} From c1db7d8350f69491894d1feb3a2097269b7a3674 Mon Sep 17 00:00:00 2001 From: Karl Rister Date: Wed, 7 Aug 2024 14:54:54 -0500 Subject: [PATCH 5/8] use verbose logging in workshop-ci --- .github/workflows/workshop-ci.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/workshop-ci.yaml b/.github/workflows/workshop-ci.yaml index c51897d..73491e1 100644 --- a/.github/workflows/workshop-ci.yaml +++ b/.github/workflows/workshop-ci.yaml @@ -41,10 +41,10 @@ jobs: run: | export TOOLBOX_HOME=~/toolbox echo - sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --label workshop-ci-dump-config --userenv ./userenvs/fedora-ci.json --dump-config true + sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --log-level verbose --label workshop-ci-dump-config --userenv ./userenvs/fedora-ci.json --dump-config true echo - sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --label workshop-ci-dump-config --userenv ./userenvs/fedora-ci.json --dump-config true --force-update-policy missing + sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --log-level verbose --label workshop-ci-dump-config --userenv ./userenvs/fedora-ci.json --dump-config true --force-update-policy missing echo - sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --label workshop-ci-dump-files --userenv ./userenvs/fedora-ci.json --dump-files true + sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --log-level verbose --label workshop-ci-dump-files --userenv ./userenvs/fedora-ci.json --dump-files true echo - sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --label workshop-ci --userenv ./userenvs/fedora-ci.json --log-level verbose + sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --log-level verbose --label workshop-ci --userenv ./userenvs/fedora-ci.json From 7b86541e6c17f21fc2bb86201b947c5daec4c62f Mon Sep 17 00:00:00 2001 From: Karl Rister Date: Wed, 7 Aug 2024 16:34:08 -0500 Subject: [PATCH 6/8] separate workshop-ci into multiple steps for easier log analysis --- .github/workflows/workshop-ci.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/workshop-ci.yaml b/.github/workflows/workshop-ci.yaml index 73491e1..42e2355 100644 --- a/.github/workflows/workshop-ci.yaml +++ b/.github/workflows/workshop-ci.yaml @@ -37,14 +37,26 @@ jobs: pushd ~/ git clone https://github.com/perftool-incubator/toolbox.git - - name: Run workshop + - name: Run workshop - dump-config run: | export TOOLBOX_HOME=~/toolbox echo sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --log-level verbose --label workshop-ci-dump-config --userenv ./userenvs/fedora-ci.json --dump-config true + + - name: Run workshop - dump-config - force-update-policy missing + run: | + export TOOLBOX_HOME=~/toolbox echo sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --log-level verbose --label workshop-ci-dump-config --userenv ./userenvs/fedora-ci.json --dump-config true --force-update-policy missing + + - name: Run workshop - dump-files + run: | + export TOOLBOX_HOME=~/toolbox echo sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --log-level verbose --label workshop-ci-dump-files --userenv ./userenvs/fedora-ci.json --dump-files true + + - name: Run workshop + run: | + export TOOLBOX_HOME=~/toolbox echo sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --log-level verbose --label workshop-ci --userenv ./userenvs/fedora-ci.json From 755fdaa2b7872d52a940c9eb5ada54c0c6aa1795 Mon Sep 17 00:00:00 2001 From: Karl Rister Date: Thu, 8 Aug 2024 10:56:24 -0500 Subject: [PATCH 7/8] add some useful flags to skopeo and buildah commands - tell skopeo to ignore tags to reduce the payload size (we don't need those) - tell buildah to always pull newer images when building to insure the most recent origin image is used - these flags require newer tools than is currently available on the GitHub ubuntu runners so rework workshop-ci to use a self-hosted runner designed specifically for this workflow --- .github/workflows/workshop-ci.yaml | 23 ++++++++++++++++++----- workshop.pl | 4 ++-- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/.github/workflows/workshop-ci.yaml b/.github/workflows/workshop-ci.yaml index 42e2355..d97457c 100644 --- a/.github/workflows/workshop-ci.yaml +++ b/.github/workflows/workshop-ci.yaml @@ -21,19 +21,28 @@ concurrency: jobs: workshop-ci: - runs-on: ubuntu-latest + runs-on: [ self-hosted, workshop ] steps: - uses: actions/checkout@v4 - - name: Install Perl Dependencies + - name: Install git run: | - sudo apt install libcoro-perl libjson-perl libjson-xs-perl libjson-validator-perl libdata-uuid-perl libdigest-sha-perl libcompress-raw-lzma-perl libcompress-raw-zlib-perl libcompress-raw-bzip2-perl cpanminus - echo - sudo cpanm IO::Uncompress::UnXz IO::Compress::Xz + sudo dnf install -y git + + - name: Install container dependencies + run: | + sudo dnf install -y skopeo podman buildah + + - name: Install Perl dependencies + run: | + sudo dnf install -y perl-Coro perl-JSON perl-JSON-XS perl-JSON-Validator perl-Data-UUID perl-Digest-SHA perl-Archive-Extract-xz-IO-Uncompress-UnXz.noarch - name: Install Toolbox run: | + if [ -d ~/toolbox ]; then + rm -Rf ~/toolbox + fi pushd ~/ git clone https://github.com/perftool-incubator/toolbox.git @@ -60,3 +69,7 @@ jobs: export TOOLBOX_HOME=~/toolbox echo sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --log-level verbose --label workshop-ci --userenv ./userenvs/fedora-ci.json + + - name: Cleanup toolbox + run: | + rm -Rf ~/toolbox diff --git a/workshop.pl b/workshop.pl index 36bca51..6a83ef7 100755 --- a/workshop.pl +++ b/workshop.pl @@ -929,7 +929,7 @@ sub delete_proto { $skopeo_url = "docker://" . $image_id; } logger('info', "Querying for origin image digest...\n", 1); - ($command, $command_output, $rc) = run_command("skopeo inspect " . $skopeo_url); + ($command, $command_output, $rc) = run_command("skopeo inspect --no-tags " . $skopeo_url); if ($rc == 0) { logger('info', "succeeded\n", 2); command_logger('verbose', $command, $rc, $command_output); @@ -988,7 +988,7 @@ sub delete_proto { # acquire the userenv from the origin logger('info', "Attempting to download the latest version of $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}...\n", 1); -($command, $command_output, $rc) = run_command("buildah pull --quiet --tls-verify=$args{'reg-tls-verify'} $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}"); +($command, $command_output, $rc) = run_command("buildah pull --quiet --policy=ifnewer --tls-verify=$args{'reg-tls-verify'} $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}"); if ($rc == 0) { logger('info', "succeeded\n", 2); command_logger('verbose', $command, $rc, $command_output); From 44e1582e35e2601dd27588cd2b0b1e2d2e6dc844 Mon Sep 17 00:00:00 2001 From: Karl Rister Date: Tue, 13 Aug 2024 16:31:58 -0500 Subject: [PATCH 8/8] use 'build-policy' instead of 'update-policy' - also use 'ifnewer' instead of 'digest' to be more consistent with buildah terminology --- .github/workflows/workshop-ci.yaml | 4 ++-- schema.json | 4 ++-- userenvs/fedora-ci.json | 2 +- workshop.pl | 32 +++++++++++++++--------------- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/workshop-ci.yaml b/.github/workflows/workshop-ci.yaml index d97457c..9b9ce80 100644 --- a/.github/workflows/workshop-ci.yaml +++ b/.github/workflows/workshop-ci.yaml @@ -52,11 +52,11 @@ jobs: echo sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --log-level verbose --label workshop-ci-dump-config --userenv ./userenvs/fedora-ci.json --dump-config true - - name: Run workshop - dump-config - force-update-policy missing + - name: Run workshop - dump-config - force-build-policy missing run: | export TOOLBOX_HOME=~/toolbox echo - sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --log-level verbose --label workshop-ci-dump-config --userenv ./userenvs/fedora-ci.json --dump-config true --force-update-policy missing + sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --log-level verbose --label workshop-ci-dump-config --userenv ./userenvs/fedora-ci.json --dump-config true --force-build-policy missing - name: Run workshop - dump-files run: | diff --git a/schema.json b/schema.json index ce241da..0249939 100644 --- a/schema.json +++ b/schema.json @@ -112,11 +112,11 @@ "type": "string", "minLength": 1 }, - "update-policy": { + "build-policy": { "type": "string", "enum": [ "missing", - "digest" + "ifnewer" ] } }, diff --git a/userenvs/fedora-ci.json b/userenvs/fedora-ci.json index 2b69a4e..3b5166d 100644 --- a/userenvs/fedora-ci.json +++ b/userenvs/fedora-ci.json @@ -10,7 +10,7 @@ "origin": { "image": "docker.io/library/fedora", "tag": "40", - "update-policy": "digest" + "build-policy": "ifnewer" }, "properties": { "platform": [ diff --git a/workshop.pl b/workshop.pl index 6a83ef7..896f13e 100755 --- a/workshop.pl +++ b/workshop.pl @@ -47,14 +47,14 @@ BEGIN $args{'param'} = {}; $args{'reg-tls-verify'} = 'true'; -my @cli_args = ( '--log-level', '--requirements', '--skip-update', '--userenv', '--force', '--config', '--dump-config', '--dump-files', '--force-update-policy' ); +my @cli_args = ( '--log-level', '--requirements', '--skip-update', '--userenv', '--force', '--config', '--dump-config', '--dump-files', '--force-build-policy' ); my %log_levels = ( 'info' => 1, 'verbose' => 1, 'debug' => 1 ); my %update_options = ( 'true' => 1, 'false' => 1 ); my %force_options = ( 'true' => 1, 'false' => 1 ); my %dump_config_options = ( 'true' => 1, 'false' => 1); my %dump_files_options = ( 'true' => 1, 'false' => 1); my %reg_tls_verify_options = ( 'true' => 1, 'false' => 1); -my %force_update_policy_options = ( 'missing' => 1, 'digest' => 1 ); +my %force_build_policy_options = ( 'missing' => 1, 'ifnewer' => 1 ); my @virtual_fs = ('dev', 'proc', 'sys'); @@ -374,7 +374,7 @@ sub usage { logger("info", "--dump-files Dump the files that are being manually handled\n"); logger("info", "--param = When is found in the userenv and/or requirements file, substitute for it\n"); logger("info", "--reg-tls-verify Use TLS for remote registry actions\n"); - logger("info", "--force-update-policy Override the userenv's specified update policy\n"); + logger("info", "--force-build-policy Override the userenv's specified build policy\n"); logger("info", "\n"); } @@ -501,11 +501,11 @@ sub arg_handler { } else { die("--reg-tls-verify must be one of 'true' or 'false' [not '$opt_value']"); } - } elsif ($opt_name eq "force-update-policy") { - if (exists ($force_update_policy_options{$opt_value})) { - $args{'force-update-policy'} = $opt_value; + } elsif ($opt_name eq "force-build-policy") { + if (exists ($force_build_policy_options{$opt_value})) { + $args{'force-build-policy'} = $opt_value; } else { - die("--force-update-policy must be one of 'missing' or 'digest' [not '$opt_value']"); + die("--force-build-policy must be one of 'missing' or 'ifnewer' [not '$opt_value']"); } } else { die("I'm confused, how did I get here [$opt_name]?"); @@ -533,7 +533,7 @@ sub delete_proto { "dump-config=s" => \&arg_handler, "dump-files=s" => \&arg_handler, "reg-tls-verify=s" => \&arg_handler, - "force-update-policy=s" => \&arg_handler)) { + "force-build-policy=s" => \&arg_handler)) { usage(); die("Error in command line arguments"); } @@ -592,11 +592,11 @@ sub delete_proto { } } -if (! exists $userenv_json->{'userenv'}{'origin'}{'update-policy'}) { - # if the loaded json does not include an update policy then +if (! exists $userenv_json->{'userenv'}{'origin'}{'build-policy'}) { + # if the loaded json does not include an build policy then # default to "missing" which results in the same behavior we have # always had - $userenv_json->{'userenv'}{'origin'}{'update-policy'} = "missing"; + $userenv_json->{'userenv'}{'origin'}{'build-policy'} = "missing"; } if (exists $userenv_json->{'userenv'}{'properties'}{'platform'}) { @@ -903,15 +903,15 @@ sub delete_proto { my %config_dump = (); my $include_digest = 0; - if (exists ($args{'force-update-policy'})) { - if ($args{'force-update-policy'} eq 'digest') { + if (exists ($args{'force-build-policy'})) { + if ($args{'force-build-policy'} eq 'ifnewer') { $include_digest = 1; - } elsif ($args{'force-update-policy'} eq 'missing') { + } elsif ($args{'force-build-policy'} eq 'missing') { $include_digest = 0; } - } elsif ($userenv_json->{'userenv'}{'origin'}{'update-policy'} eq 'digest') { + } elsif ($userenv_json->{'userenv'}{'origin'}{'build-policy'} eq 'ifnewer') { $include_digest = 1; - } elsif ($userenv_json->{'userenv'}{'origin'}{'update-policy'} eq 'missing') { + } elsif ($userenv_json->{'userenv'}{'origin'}{'build-policy'} eq 'missing') { $include_digest = 0; } else { die("I'm confused, how did I get here [include_digest]?")