diff --git a/.github/workflows/workshop-ci.yaml b/.github/workflows/workshop-ci.yaml index 9d8e231..9b9ce80 100644 --- a/.github/workflows/workshop-ci.yaml +++ b/.github/workflows/workshop-ci.yaml @@ -21,28 +21,55 @@ concurrency: jobs: workshop-ci: - runs-on: ubuntu-latest + runs-on: [ self-hosted, workshop ] steps: - uses: actions/checkout@v4 - - name: Install Perl Dependencies + - name: Install git run: | - sudo apt install libcoro-perl libjson-perl libjson-xs-perl libjson-validator-perl libdata-uuid-perl libdigest-sha-perl libcompress-raw-lzma-perl libcompress-raw-zlib-perl libcompress-raw-bzip2-perl cpanminus - echo - sudo cpanm IO::Uncompress::UnXz IO::Compress::Xz + sudo dnf install -y git + + - name: Install container dependencies + run: | + sudo dnf install -y skopeo podman buildah + + - name: Install Perl dependencies + run: | + sudo dnf install -y perl-Coro perl-JSON perl-JSON-XS perl-JSON-Validator perl-Data-UUID perl-Digest-SHA perl-Archive-Extract-xz-IO-Uncompress-UnXz.noarch - name: Install Toolbox run: | + if [ -d ~/toolbox ]; then + rm -Rf ~/toolbox + fi pushd ~/ git clone https://github.com/perftool-incubator/toolbox.git - - name: Run workshop + - name: Run workshop - dump-config run: | export TOOLBOX_HOME=~/toolbox echo - sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --label workshop-ci-dump-config --userenv ./userenvs/fedora36-ci.json --dump-config true + sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --log-level verbose --label workshop-ci-dump-config --userenv ./userenvs/fedora-ci.json --dump-config true + + - name: Run workshop - dump-config - force-build-policy missing + run: | + export TOOLBOX_HOME=~/toolbox echo - sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --label workshop-ci-dump-files --userenv ./userenvs/fedora36-ci.json --dump-files true + sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --log-level verbose --label workshop-ci-dump-config --userenv ./userenvs/fedora-ci.json --dump-config true --force-build-policy missing + + - name: Run workshop - dump-files + run: | + export TOOLBOX_HOME=~/toolbox echo - sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --label workshop-ci --userenv ./userenvs/fedora36-ci.json --log-level verbose + sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --log-level verbose --label workshop-ci-dump-files --userenv ./userenvs/fedora-ci.json --dump-files true + + - name: Run workshop + run: | + export TOOLBOX_HOME=~/toolbox + echo + sudo --preserve-env=TOOLBOX_HOME ./workshop.pl --log-level verbose --label workshop-ci --userenv ./userenvs/fedora-ci.json + + - name: Cleanup toolbox + run: | + rm -Rf ~/toolbox diff --git a/schema.json b/schema.json index a044612..0249939 100644 --- a/schema.json +++ b/schema.json @@ -74,7 +74,8 @@ "2020.04.30", "2022.07.25", "2023.02.16", - "2024.03.22" + "2024.03.22", + "2024.08.07" ] } }, @@ -110,6 +111,13 @@ "tag": { "type": "string", "minLength": 1 + }, + "build-policy": { + "type": "string", + "enum": [ + "missing", + "ifnewer" + ] } }, "required": [ diff --git a/userenvs/fedora36-ci.json b/userenvs/fedora-ci.json similarity index 78% rename from userenvs/fedora36-ci.json rename to userenvs/fedora-ci.json index 3b16fee..3b5166d 100644 --- a/userenvs/fedora36-ci.json +++ b/userenvs/fedora-ci.json @@ -1,15 +1,16 @@ { "workshop": { "schema": { - "version": "2023.02.16" + "version": "2024.08.07" } }, "userenv": { - "name": "fedora36", - "label": "Fedora 36", + "name": "fedora40", + "label": "Fedora 40", "origin": { "image": "docker.io/library/fedora", - "tag": "36" + "tag": "40", + "build-policy": "ifnewer" }, "properties": { "platform": [ @@ -28,11 +29,11 @@ }, "requirements": [ { - "name": "python39", + "name": "python3", "type": "distro", "distro_info": { "packages": [ - "python39" + "python3" ] } }, diff --git a/userenvs/fedora33.json b/userenvs/fedora33.json deleted file mode 100644 index 4be7d99..0000000 --- a/userenvs/fedora33.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "workshop": { - "schema": { - "version": "2020.03.02" - } - }, - "userenv": { - "name": "fedora33", - "label": "Fedora 33", - "origin": { - "image": "registry.fedoraproject.org/fedora", - "tag": "33" - }, - "properties": { - "packages": { - "type": "rpm", - "manager": "dnf" - } - } - }, - "requirements": [ - { - "name": "python39", - "type": "distro", - "distro_info": { - "packages": [ - "python39", - "python3-pip" - ] - } - }, - { - "name": "utils", - "type": "distro", - "distro_info": { - "packages": [ - "curl", - "tar", - "cpio", - "xz", - "gzip", - "jq", - "git", - "cpio", - "findutils", - "hostname", - "iputils", - "iproute" - ] - } - }, - { - "name": "core-perl", - "type": "distro", - "distro_info": { - "packages": [ - "perl-App-cpanminus" - ] - } - }, - { - "name": "core-node", - "type": "distro", - "distro_info": { - "packages": [ - "nodejs" - ] - } - }, - { - "name": "core-compiling", - "type": "distro", - "distro_info": { - "packages": [ - "diffutils", - "gcc", - "libtool", - "autoconf", - "automake", - "make" - ] - } - }, - { - "name": "my-perl-deps", - "type": "cpan", - "cpan_info": { - "packages": [ - "Coro JSON JSON::XS JSON::Validator Data::Dumper" - ] - } - }, - { - "name": "my-python-deps", - "type": "python3", - "python3_info": { - "packages": [ - "NumPy pandas Keras Pillow" - ] - } - }, - { - "name": "some-node-deps", - "type": "node", - "node_info": { - "packages": [ - "ansi-regex sax react" - ] - } - }, - { - "name": "more-perl-deps", - "type": "cpan", - "cpan_info": { - "packages": [ - "Data::UUID Digest::SHA Digest::MD5 Getopt::Long DBI DBD::SQLite Time::HiRes" - ] - } - } - ] -} diff --git a/workshop.pl b/workshop.pl index 8838c94..896f13e 100755 --- a/workshop.pl +++ b/workshop.pl @@ -47,13 +47,14 @@ BEGIN $args{'param'} = {}; $args{'reg-tls-verify'} = 'true'; -my @cli_args = ( '--log-level', '--requirements', '--skip-update', '--userenv', '--force', '--config', '--dump-config', '--dump-files' ); +my @cli_args = ( '--log-level', '--requirements', '--skip-update', '--userenv', '--force', '--config', '--dump-config', '--dump-files', '--force-build-policy' ); my %log_levels = ( 'info' => 1, 'verbose' => 1, 'debug' => 1 ); my %update_options = ( 'true' => 1, 'false' => 1 ); my %force_options = ( 'true' => 1, 'false' => 1 ); my %dump_config_options = ( 'true' => 1, 'false' => 1); my %dump_files_options = ( 'true' => 1, 'false' => 1); my %reg_tls_verify_options = ( 'true' => 1, 'false' => 1); +my %force_build_policy_options = ( 'missing' => 1, 'ifnewer' => 1 ); my @virtual_fs = ('dev', 'proc', 'sys'); @@ -149,7 +150,9 @@ sub get_exit_code { 'package_remove' => 93, 'group_remove' => 94, 'architecture_query_failed' => 95, - 'unsupported_platform_architecture' => 96 + 'unsupported_platform_architecture' => 96, + 'skopeo_inspect_failed' => 97, + 'skopeo_digest_missing' => 98 ); if (exists($reasons{$exit_reason})) { @@ -361,16 +364,17 @@ sub usage { logger("info", "\n"); logger("info", "Optional arguments: (* denotes default)\n"); logger("info", "\n"); - logger("info", "--requirements Requirements file (can be used multiple times)\n"); - logger("info", "--label Label to apply to container image\n"); - logger("info", "--config Container config file\n"); - logger("info", "--log-level Control logging output\n"); - logger("info", "--skip-update Should the container run it's distro update function\n"); - logger("info", "--force Force the container build\n"); - logger("info", "--dump-config Dump the config instead of building the container\n"); - logger("info", "--dump-files Dump the files that are being manually handled\n"); - logger("info", "--param = When is found in the userenv and/or requirements file, substitute for it\n"); - logger("info", "--reg-tls-verify Use TLS for remote registry actions\n"); + logger("info", "--requirements Requirements file (can be used multiple times)\n"); + logger("info", "--label Label to apply to container image\n"); + logger("info", "--config Container config file\n"); + logger("info", "--log-level Control logging output\n"); + logger("info", "--skip-update Should the container run it's distro update function\n"); + logger("info", "--force Force the container build\n"); + logger("info", "--dump-config Dump the config instead of building the container\n"); + logger("info", "--dump-files Dump the files that are being manually handled\n"); + logger("info", "--param = When is found in the userenv and/or requirements file, substitute for it\n"); + logger("info", "--reg-tls-verify Use TLS for remote registry actions\n"); + logger("info", "--force-build-policy Override the userenv's specified build policy\n"); logger("info", "\n"); } @@ -497,6 +501,12 @@ sub arg_handler { } else { die("--reg-tls-verify must be one of 'true' or 'false' [not '$opt_value']"); } + } elsif ($opt_name eq "force-build-policy") { + if (exists ($force_build_policy_options{$opt_value})) { + $args{'force-build-policy'} = $opt_value; + } else { + die("--force-build-policy must be one of 'missing' or 'ifnewer' [not '$opt_value']"); + } } else { die("I'm confused, how did I get here [$opt_name]?"); } @@ -522,7 +532,8 @@ sub delete_proto { "param=s" => \&arg_handler, "dump-config=s" => \&arg_handler, "dump-files=s" => \&arg_handler, - "reg-tls-verify=s" => \&arg_handler)) { + "reg-tls-verify=s" => \&arg_handler, + "force-build-policy=s" => \&arg_handler)) { usage(); die("Error in command line arguments"); } @@ -581,6 +592,13 @@ sub delete_proto { } } +if (! exists $userenv_json->{'userenv'}{'origin'}{'build-policy'}) { + # if the loaded json does not include an build policy then + # default to "missing" which results in the same behavior we have + # always had + $userenv_json->{'userenv'}{'origin'}{'build-policy'} = "missing"; +} + if (exists $userenv_json->{'userenv'}{'properties'}{'platform'}) { # the userenv has platform information that indicates what type of # system architecture(s) it supports so validate that what we are @@ -882,10 +900,56 @@ sub delete_proto { } if ($args{'dump-config'} eq 'true') { - logger('info', "Config dump:\n"); - my %config_dump = (); + my $include_digest = 0; + if (exists ($args{'force-build-policy'})) { + if ($args{'force-build-policy'} eq 'ifnewer') { + $include_digest = 1; + } elsif ($args{'force-build-policy'} eq 'missing') { + $include_digest = 0; + } + } elsif ($userenv_json->{'userenv'}{'origin'}{'build-policy'} eq 'ifnewer') { + $include_digest = 1; + } elsif ($userenv_json->{'userenv'}{'origin'}{'build-policy'} eq 'missing') { + $include_digest = 0; + } else { + die("I'm confused, how did I get here [include_digest]?") + } + + if ($include_digest == 1) { + # get the userenv digest to include in the config dump -- this + # can be used by callers that are analyzing the dumped config + # to know whether or not the origin image has changed + my $image_id = $userenv_json->{'userenv'}{'origin'}{'image'} . ":" . $userenv_json->{'userenv'}{'origin'}{'tag'}; + my $skopeo_url; + if (($image_id =~ /^dir:/) || ($image_id =~ /^docker:\/\//)) { + $skopeo_url = $image_id; + } else { + $skopeo_url = "docker://" . $image_id; + } + logger('info', "Querying for origin image digest...\n", 1); + ($command, $command_output, $rc) = run_command("skopeo inspect --no-tags " . $skopeo_url); + if ($rc == 0) { + logger('info', "succeeded\n", 2); + command_logger('verbose', $command, $rc, $command_output); + + $command_output = filter_output($command_output); + my $skopeo_json = decode_json($command_output); + if (exists ($skopeo_json->{'Digest'})) { + $userenv_json->{'userenv'}{'origin'}{'digest'} = $skopeo_json->{'Digest'}; + } else { + logger('error', "Query results do not contain a digest"); + exit(get_exit_code('skopeo_digest_missing')); + } + } else { + logger('info', "failed\n", 2); + command_logger('error', $command, $rc, $command_output); + logger('error', "Failed to query " . $skopeo_url); + exit(get_exit_code('skopeo_inspect_failed')); + } + } + # consolidate information to be dumped $config_dump{'userenv'} = $userenv_json; $config_dump{'requirements'} = $active_requirements{'array'}; @@ -899,6 +963,7 @@ sub delete_proto { delete $config_dump{'requirements'}[$i]{'sha256'}; } + logger('info', "Config dump:\n"); logger('info', Data::Dumper->Dump([\%config_dump], [qw(*config_dump)])); exit() @@ -919,42 +984,37 @@ sub delete_proto { } my $container_mount_point; +my $origin_image_id; # acquire the userenv from the origin -logger('info', "Looking for container base image...\n"); -($command, $command_output, $rc) = run_command("buildah images --json " . delete_proto($userenv_json->{'userenv'}{'origin'}{'image'}) . ":$userenv_json->{'userenv'}{'origin'}{'tag'}"); +logger('info', "Attempting to download the latest version of $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}...\n", 1); +($command, $command_output, $rc) = run_command("buildah pull --quiet --policy=ifnewer --tls-verify=$args{'reg-tls-verify'} $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}"); if ($rc == 0) { - logger('info', "Found $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'} locally\n", 1); + logger('info', "succeeded\n", 2); command_logger('verbose', $command, $rc, $command_output); + $command_output = filter_output($command_output); - $userenv_json->{'userenv'}{'origin'}{'local_details'} = decode_json($command_output); -} else { - command_logger('verbose', $command, $rc, $command_output); - logger('info', "Could not find $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}, attempting to download...\n", 1); - ($command, $command_output, $rc) = run_command("buildah pull --quiet --tls-verify=$args{'reg-tls-verify'} $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}"); + chomp($command_output); + $origin_image_id = $command_output; + + logger('info', "Querying for information about the image...\n", 1); + ($command, $command_output, $rc) = run_command("buildah images --json " . $origin_image_id); if ($rc == 0) { logger('info', "succeeded\n", 2); command_logger('verbose', $command, $rc, $command_output); - - logger('info', "Querying for information about the image...\n", 1); - ($command, $command_output, $rc) = run_command("buildah images --json " . delete_proto($userenv_json->{'userenv'}{'origin'}{'image'}) . ":$userenv_json->{'userenv'}{'origin'}{'tag'}"); - if ($rc == 0) { - logger('info', "succeeded\n", 2); - command_logger('verbose', $command, $rc, $command_output); - $command_output = filter_output($command_output); - $userenv_json->{'userenv'}{'origin'}{'local_details'} = decode_json($command_output); - } else { - logger('info', "failed\n", 2); - command_logger('error', $command, $rc, $command_output); - logger('error', "Failed to download/query $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}!\n"); - exit(get_exit_code('image_query')); - } + $command_output = filter_output($command_output); + $userenv_json->{'userenv'}{'origin'}{'local_details'} = decode_json($command_output); } else { logger('info', "failed\n", 2); command_logger('error', $command, $rc, $command_output); - logger('error', "Failed to download $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}!\n"); - exit(get_exit_code('image_origin_pull')); + logger('error', "Failed to query $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'} ($origin_image_id)!\n"); + exit(get_exit_code('image_query')); } +} else { + logger('info', "failed\n", 2); + command_logger('error', $command, $rc, $command_output); + logger('error', "Failed to download $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}!\n"); + exit(get_exit_code('image_origin_pull')); } logger('debug', "Userenv JSON:\n"); @@ -1093,11 +1153,11 @@ sub delete_proto { # create a new container based on the userenv source logger('info', "Creating temporary container...\n"); -($command, $command_output, $rc) = run_command("buildah from --name $tmp_container $userenv_json->{'userenv'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}"); +($command, $command_output, $rc) = run_command("buildah from --name $tmp_container $origin_image_id"); if ($rc != 0) { logger('info', "failed\n", 1); command_logger('error', $command, $rc, $command_output); - logger('error', "Could not create new container '$tmp_container' from '$userenv_json->{'origin'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}'!\n"); + logger('error', "Could not create new container '$tmp_container' from '$userenv_json->{'origin'}{'origin'}{'image'}:$userenv_json->{'userenv'}{'origin'}{'tag'}' ($origin_image_id)!\n"); exit(get_exit_code('create_container')); } else { logger('info', "succeeded\n", 1);