diff --git a/CHANGELOG.md b/CHANGELOG.md index af1941d0..8fbb628f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Updated recommended runc version to 1.1.9 +### Fixed + +- Fixed support for image manifests which are provided by registries as multi-line, not indented JSON + + ## [1.6.0] ### Added diff --git a/CI/src/integration_tests/test_command_pull.py b/CI/src/integration_tests/test_command_pull.py index 44b2a777..31677879 100755 --- a/CI/src/integration_tests/test_command_pull.py +++ b/CI/src/integration_tests/test_command_pull.py @@ -31,10 +31,17 @@ def test_command_pull_with_dockerhub(self): self._test_command_pull("alpine:latest", is_centralized_repository=False) - def test_command_pull_with_buildah_image(self): + # Manifests generated by Buildah do not use a human-friendly JSON formatting (like Docker-generated manifests), + # but are a contiguous string without newlines or indents + def test_command_pull_with_buildah_manifest(self): self._test_command_pull("quay.io/ethcscs/alpine:buildah", is_centralized_repository=False) + # Manifests stored in GitHub Container Registry are not indented + def test_command_pull_with_ghcr_manifest(self): + self._test_command_pull("quay.io/ethcscs/zlib:1.2.13-ghcr-manifest", + is_centralized_repository=False) + def test_command_pull_by_digest(self): self._test_command_pull("quay.io/ethcscs/alpine@sha256:1775bebec23e1f3ce486989bfc9ff3c4e951690df84aa9f926497d82f2ffca9d", is_centralized_repository=False, diff --git a/src/image_manager/SkopeoDriver.cpp b/src/image_manager/SkopeoDriver.cpp index b266045f..6d5ba87f 100644 --- a/src/image_manager/SkopeoDriver.cpp +++ b/src/image_manager/SkopeoDriver.cpp @@ -190,11 +190,9 @@ std::string SkopeoDriver::inspectRaw(const std::string& sourceTransport, const s // The Skopeo debug/warning messages are useful to be embedded in an exception message, // but prevent the output from being converted to JSON. - // Exclude the extra lines from Skopeo and only return the JSON output - inspectOutput = inspectOutput.substr(inspectOutput.rfind("\n{")+1); - - printLog(boost::format("Raw inspect filtered output: %s") % inspectOutput, common::LogLevel::DEBUG); - return inspectOutput; + // Exclude the extra lines from Skopeo and only return the JSON output, assuming it is the last + // substring enclosed by curly braces that can be found in the output string + return filterInspectOutput(inspectOutput); } std::string SkopeoDriver::manifestDigest(const boost::filesystem::path& manifestPath) const { @@ -238,6 +236,24 @@ boost::filesystem::path SkopeoDriver::acquireAuthFile(const common::Config::Auth return authFilePath; } +std::string SkopeoDriver::filterInspectOutput(const std::string& inspectOutput) const { + std::string filteredOutput; + // TODO: this could be refactored into a common utility to match JSON-like strings + boost::smatch matches; + boost::regex jsonLikePattern(R"(\{(?:[^{}]|(?R))*\})"); + if (boost::regex_search(inspectOutput, matches, jsonLikePattern)) { + filteredOutput = matches[matches.size() - 1]; + } + else { + auto message = boost::format("Could not detect any JSON-like pattern after manifest inspect operation. " + "Raw inspect output:\n%s") % inspectOutput; + SARUS_THROW_ERROR(message.str()); + } + + printLog(boost::format("Raw inspect filtered output: %s") % filteredOutput, common::LogLevel::DEBUG); + return filteredOutput; +} + common::CLIArguments SkopeoDriver::generateBaseArgs() const { auto args = common::CLIArguments{skopeoPath.string()}; diff --git a/src/image_manager/SkopeoDriver.hpp b/src/image_manager/SkopeoDriver.hpp index 0f89ef51..c065e444 100644 --- a/src/image_manager/SkopeoDriver.hpp +++ b/src/image_manager/SkopeoDriver.hpp @@ -34,6 +34,7 @@ class SkopeoDriver { std::string inspectRaw(const std::string& sourceTransport, const std::string& sourceReference) const; std::string manifestDigest(const boost::filesystem::path& manifestPath) const; boost::filesystem::path acquireAuthFile(const common::Config::Authentication& auth, const common::ImageReference& reference); + std::string filterInspectOutput(const std::string& inspectOutput) const; common::CLIArguments generateBaseArgs() const; private: diff --git a/src/image_manager/test/expected_manifests/alpine_3.14.json b/src/image_manager/test/expected_manifests/alpine_3.14.json new file mode 100644 index 00000000..3fca258e --- /dev/null +++ b/src/image_manager/test/expected_manifests/alpine_3.14.json @@ -0,0 +1,16 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 1472, + "digest": "sha256:d4ff818577bc193b309b355b02ebc9220427090057b54a59e73b79bdfe139b83" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 2811478, + "digest": "sha256:5843afab387455b37944e709ee8c78d7520df80f8d01cf7f861aae63beeddb6b" + } + ] +} \ No newline at end of file diff --git a/src/image_manager/test/expected_manifests/alpine_buildah.json b/src/image_manager/test/expected_manifests/alpine_buildah.json new file mode 100644 index 00000000..1e28b45e --- /dev/null +++ b/src/image_manager/test/expected_manifests/alpine_buildah.json @@ -0,0 +1 @@ +{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:1d4d0091fb1c2e430b0a9bab558b9a3a929fce0c93f29b0d4c2b9bcdecbfaf48","size":859},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:8dfb4e6dc5179a0adf4a069e14d984216740f28b088c26090c8f16b97e44b222","size":2903015},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:109ba8c8d73216884b3667c38b4f706df81e6ac958041c5827f6718e112c8836","size":3135718}],"annotations":{"org.opencontainers.image.base.digest":"sha256:a777c9c66ba177ccfea23f2a216ff6721e78a662cd17019488c417135299cd89","org.opencontainers.image.base.name":"docker.io/library/alpine:3.15"}} \ No newline at end of file diff --git a/src/image_manager/test/expected_manifests/zlib_ghcr.json b/src/image_manager/test/expected_manifests/zlib_ghcr.json new file mode 100644 index 00000000..1932cf0f --- /dev/null +++ b/src/image_manager/test/expected_manifests/zlib_ghcr.json @@ -0,0 +1,24 @@ +{ +"mediaType":"application/vnd.oci.image.manifest.v1+json", +"schemaVersion":2, +"config":{ +"mediaType":"application/vnd.oci.image.config.v1+json", +"digest":"sha256:89210d0e9eb2eb59718480e05952c5e6e2f281e647a455b56d18e3a3add39006", +"size":1663 +}, +"layers":[ +{ +"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip", +"size":29533422, +"digest":"sha256:3153aa388d026c26a2235e1ed0163e350e451f41a8a313e1804d7e1afb857ab4" +}, +{ +"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip", +"digest":"sha256:527bca282263ba1baafa344a90d2b07664ec9e241503f678b80182e3cda239e7", +"size":153466 +} +], +"annotations":{ +"org.opencontainers.image.description":"zlib@=1.2.13%gcc@=12.2.0+optimize+pic+shared build_system=makefile arch=linux-ubuntu22.10-zen2" +} +} \ No newline at end of file diff --git a/src/image_manager/test/skopeo_debug_lines.txt b/src/image_manager/test/skopeo_debug_lines.txt new file mode 100644 index 00000000..ec7405ac --- /dev/null +++ b/src/image_manager/test/skopeo_debug_lines.txt @@ -0,0 +1,12 @@ +DEBU[0000] Loading registries configuration "/etc/containers/registries.conf" +DEBU[0000] Loading registries configuration "/etc/containers/registries.conf.d/shortnames.conf" +DEBU[0000] Trying to access "docker.io/library/alpine:latest" +DEBU[0000] No credentials for docker.io found +DEBU[0000] Using registries.d directory /etc/containers/registries.d for sigstore configuration +DEBU[0000] No signature storage configuration found for docker.io/library/alpine:latest, using built-in default file:///home/user/.local/share/containers/sigstore +DEBU[0000] Looking for TLS certificates and private keys in /etc/docker/certs.d/docker.io +DEBU[0000] GET https://registry-1.docker.io/v2/ +DEBU[0000] Ping https://registry-1.docker.io/v2/ status 401 +DEBU[0000] GET https://auth.docker.io/token?scope=repository%3Alibrary%2Falpine%3Apull&service=registry.docker.io +DEBU[0000] GET https://registry-1.docker.io/v2/library/alpine/manifests/latest +DEBU[0001] Content-Type from manifest GET is "application/vnd.docker.distribution.manifest.list.v2+json" diff --git a/src/image_manager/test/test_SkopeoDriver.cpp b/src/image_manager/test/test_SkopeoDriver.cpp index 54efbceb..989c35e0 100644 --- a/src/image_manager/test/test_SkopeoDriver.cpp +++ b/src/image_manager/test/test_SkopeoDriver.cpp @@ -64,25 +64,34 @@ TEST(SkopeoDriverTestGroup, copyToOCIImage) { } } -TEST(SkopeoDriverTestGroup, inspectRaw) { - auto configRAII = test_utility::config::makeConfig(); - auto& config = configRAII.config; - - auto driver = image_manager::SkopeoDriver{config}; - - auto imageReference = std::string{"quay.io/ethcscs/alpine:3.14"}; - auto expectedManifest = common::readFile(boost::filesystem::path{__FILE__}.parent_path() / "expected_inspect_raw_manifest.json"); - - auto returnedManifest = driver.inspectRaw("docker", imageReference); +static void filterInspectOutputTestHelper(const image_manager::SkopeoDriver& driver, + const std::string& expectedManifestFilename) { + auto testSourceDir = boost::filesystem::path{__FILE__}.parent_path(); + auto expectedManifestPath = testSourceDir / expectedManifestFilename; + auto expectedManifest = common::readFile(expectedManifestPath); + auto returnedManifest = driver.filterInspectOutput(expectedManifest); CHECK(returnedManifest == expectedManifest); - // Check debug mode does not alter result - common::Logger::getInstance().setLevel(common::LogLevel::DEBUG); - returnedManifest = driver.inspectRaw("docker", imageReference); + // Check debug output does not alter result + auto skopeoDebugLines = common::readFile(testSourceDir / "skopeo_debug_lines.txt"); + returnedManifest = driver.filterInspectOutput(skopeoDebugLines + expectedManifest); CHECK(returnedManifest == expectedManifest); common::Logger::getInstance().setLevel(common::LogLevel::WARN); } +TEST(SkopeoDriverTestGroup, filterInspectOutput) { + auto configRAII = test_utility::config::makeConfig(); + auto& config = configRAII.config; + auto driver = image_manager::SkopeoDriver{config}; + + // Multi-line, indented + filterInspectOutputTestHelper(driver, "expected_manifests/alpine_3.14.json"); + // Single line + filterInspectOutputTestHelper(driver, "expected_manifests/alpine_buildah.json"); + // Multi-line, not indented + filterInspectOutputTestHelper(driver, "expected_manifests/zlib_ghcr.json"); +} + TEST(SkopeoDriverTestGroup, manifestDigest) { auto configRAII = test_utility::config::makeConfig(); auto& config = configRAII.config; @@ -90,7 +99,7 @@ TEST(SkopeoDriverTestGroup, manifestDigest) { auto driver = image_manager::SkopeoDriver{config}; - auto rawManifestPath = boost::filesystem::path{__FILE__}.parent_path() / "expected_inspect_raw_manifest.json"; + auto rawManifestPath = boost::filesystem::path{__FILE__}.parent_path() / "expected_manifests/alpine_3.14.json"; CHECK(driver.manifestDigest(rawManifestPath) == std::string{"sha256:1775bebec23e1f3ce486989bfc9ff3c4e951690df84aa9f926497d82f2ffca9d"}); // OCI image blobs have their own digests as filenames, so it's a useful property for more test cases