Fixed support for non-indented manifests

CHANGELOG.md

@@ -11,6 +11,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Updated recommended runc version to 1.1.9
 
 
+### Fixed
+
+- Fixed support for image manifests which are provided by registries as multi-line, not indented JSON
+
+
 ## [1.6.0]
 
 ### Added

CI/src/integration_tests/test_command_pull.py

@@ -31,10 +31,17 @@ def test_command_pull_with_dockerhub(self):
         self._test_command_pull("alpine:latest",
                                 is_centralized_repository=False)
 
-    def test_command_pull_with_buildah_image(self):
+    # Manifests generated by Buildah do not use a human-friendly JSON formatting (like Docker-generated manifests),
+    # but are a contiguous string without newlines or indents
+    def test_command_pull_with_buildah_manifest(self):
         self._test_command_pull("quay.io/ethcscs/alpine:buildah",
                                 is_centralized_repository=False)
 
+    # Manifests stored in GitHub Container Registry are not indented
+    def test_command_pull_with_ghcr_manifest(self):
+        self._test_command_pull("quay.io/ethcscs/zlib:1.2.13-ghcr-manifest",
+                                is_centralized_repository=False)
+
     def test_command_pull_by_digest(self):
         self._test_command_pull("quay.io/ethcscs/alpine@sha256:1775bebec23e1f3ce486989bfc9ff3c4e951690df84aa9f926497d82f2ffca9d",
                                 is_centralized_repository=False,

src/image_manager/SkopeoDriver.cpp

@@ -190,11 +190,9 @@ std::string SkopeoDriver::inspectRaw(const std::string& sourceTransport, const s
 
     // The Skopeo debug/warning messages are useful to be embedded in an exception message,
     // but prevent the output from being converted to JSON.
-    // Exclude the extra lines from Skopeo and only return the JSON output
-    inspectOutput = inspectOutput.substr(inspectOutput.rfind("\n{")+1);
-
-    printLog(boost::format("Raw inspect filtered output: %s") % inspectOutput, common::LogLevel::DEBUG);
-    return inspectOutput;
+    // Exclude the extra lines from Skopeo and only return the JSON output, assuming it is the last
+    // substring enclosed by curly braces that can be found in the output string
+    return filterInspectOutput(inspectOutput);
 }
 
 std::string SkopeoDriver::manifestDigest(const boost::filesystem::path& manifestPath) const {
@@ -238,6 +236,24 @@ boost::filesystem::path SkopeoDriver::acquireAuthFile(const common::Config::Auth
     return authFilePath;
 }
 
+std::string SkopeoDriver::filterInspectOutput(const std::string& inspectOutput) const {
+    std::string filteredOutput;
+    // TODO: this could be refactored into a common utility to match JSON-like strings
+    boost::smatch matches;
+    boost::regex jsonLikePattern(R"(\{(?:[^{}]|(?R))*\})");
+    if (boost::regex_search(inspectOutput, matches, jsonLikePattern)) {
+        filteredOutput = matches[matches.size() - 1];
+    }
+    else {
+        auto message = boost::format("Could not detect any JSON-like pattern after manifest inspect operation. "
+                                     "Raw inspect output:\n%s") % inspectOutput;
+        SARUS_THROW_ERROR(message.str());
+    }
+
+    printLog(boost::format("Raw inspect filtered output: %s") % filteredOutput, common::LogLevel::DEBUG);
+    return filteredOutput;
+}
+
 common::CLIArguments SkopeoDriver::generateBaseArgs() const {
     auto args = common::CLIArguments{skopeoPath.string()};
 

src/image_manager/SkopeoDriver.hpp

@@ -34,6 +34,7 @@ class SkopeoDriver {
     std::string inspectRaw(const std::string& sourceTransport, const std::string& sourceReference) const;
     std::string manifestDigest(const boost::filesystem::path& manifestPath) const;
     boost::filesystem::path acquireAuthFile(const common::Config::Authentication& auth, const common::ImageReference& reference);
+    std::string filterInspectOutput(const std::string& inspectOutput) const;
     common::CLIArguments generateBaseArgs() const;
 
 private:

src/image_manager/test/expected_manifests/alpine_3.14.json

@@ -0,0 +1,16 @@
+{
+   "schemaVersion": 2,
+   "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+   "config": {
+      "mediaType": "application/vnd.docker.container.image.v1+json",
+      "size": 1472,
+      "digest": "sha256:d4ff818577bc193b309b355b02ebc9220427090057b54a59e73b79bdfe139b83"
+   },
+   "layers": [
+      {
+         "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+         "size": 2811478,
+         "digest": "sha256:5843afab387455b37944e709ee8c78d7520df80f8d01cf7f861aae63beeddb6b"
+      }
+   ]
+}

src/image_manager/test/expected_manifests/alpine_buildah.json

@@ -0,0 +1 @@
+{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:1d4d0091fb1c2e430b0a9bab558b9a3a929fce0c93f29b0d4c2b9bcdecbfaf48","size":859},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:8dfb4e6dc5179a0adf4a069e14d984216740f28b088c26090c8f16b97e44b222","size":2903015},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:109ba8c8d73216884b3667c38b4f706df81e6ac958041c5827f6718e112c8836","size":3135718}],"annotations":{"org.opencontainers.image.base.digest":"sha256:a777c9c66ba177ccfea23f2a216ff6721e78a662cd17019488c417135299cd89","org.opencontainers.image.base.name":"docker.io/library/alpine:3.15"}}

src/image_manager/test/expected_manifests/zlib_ghcr.json

@@ -0,0 +1,24 @@
+{
+"mediaType":"application/vnd.oci.image.manifest.v1+json",
+"schemaVersion":2,
+"config":{
+"mediaType":"application/vnd.oci.image.config.v1+json",
+"digest":"sha256:89210d0e9eb2eb59718480e05952c5e6e2f281e647a455b56d18e3a3add39006",
+"size":1663
+},
+"layers":[
+{
+"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip",
+"size":29533422,
+"digest":"sha256:3153aa388d026c26a2235e1ed0163e350e451f41a8a313e1804d7e1afb857ab4"
+},
+{
+"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip",
+"digest":"sha256:527bca282263ba1baafa344a90d2b07664ec9e241503f678b80182e3cda239e7",
+"size":153466
+}
+],
+"annotations":{
+"org.opencontainers.image.description":"zlib@=1.2.13%gcc@=12.2.0+optimize+pic+shared build_system=makefile arch=linux-ubuntu22.10-zen2"
+}
+}

src/image_manager/test/skopeo_debug_lines.txt

@@ -0,0 +1,12 @@
+DEBU[0000] Loading registries configuration "/etc/containers/registries.conf" 
+DEBU[0000] Loading registries configuration "/etc/containers/registries.conf.d/shortnames.conf" 
+DEBU[0000] Trying to access "docker.io/library/alpine:latest" 
+DEBU[0000] No credentials for docker.io found           
+DEBU[0000] Using registries.d directory /etc/containers/registries.d for sigstore configuration 
+DEBU[0000]  No signature storage configuration found for docker.io/library/alpine:latest, using built-in default file:///home/user/.local/share/containers/sigstore 
+DEBU[0000] Looking for TLS certificates and private keys in /etc/docker/certs.d/docker.io 
+DEBU[0000] GET https://registry-1.docker.io/v2/         
+DEBU[0000] Ping https://registry-1.docker.io/v2/ status 401 
+DEBU[0000] GET https://auth.docker.io/token?scope=repository%3Alibrary%2Falpine%3Apull&service=registry.docker.io 
+DEBU[0000] GET https://registry-1.docker.io/v2/library/alpine/manifests/latest 
+DEBU[0001] Content-Type from manifest GET is "application/vnd.docker.distribution.manifest.list.v2+json"

src/image_manager/test/test_SkopeoDriver.cpp

@@ -64,33 +64,42 @@ TEST(SkopeoDriverTestGroup, copyToOCIImage) {
     }
 }
 
-TEST(SkopeoDriverTestGroup, inspectRaw) {
-    auto configRAII = test_utility::config::makeConfig();
-    auto& config = configRAII.config;
-
-    auto driver = image_manager::SkopeoDriver{config};
-
-    auto imageReference = std::string{"quay.io/ethcscs/alpine:3.14"};
-    auto expectedManifest = common::readFile(boost::filesystem::path{__FILE__}.parent_path() / "expected_inspect_raw_manifest.json");
-
-    auto returnedManifest = driver.inspectRaw("docker", imageReference);
+static void filterInspectOutputTestHelper(const image_manager::SkopeoDriver& driver,
+                                          const std::string& expectedManifestFilename) {
+    auto testSourceDir = boost::filesystem::path{__FILE__}.parent_path();
+    auto expectedManifestPath = testSourceDir / expectedManifestFilename;
+    auto expectedManifest = common::readFile(expectedManifestPath);
+    auto returnedManifest = driver.filterInspectOutput(expectedManifest);
     CHECK(returnedManifest == expectedManifest);
 
-    // Check debug mode does not alter result
-    common::Logger::getInstance().setLevel(common::LogLevel::DEBUG);
-    returnedManifest = driver.inspectRaw("docker", imageReference);
+    // Check debug output does not alter result
+    auto skopeoDebugLines = common::readFile(testSourceDir / "skopeo_debug_lines.txt");
+    returnedManifest = driver.filterInspectOutput(skopeoDebugLines + expectedManifest);
     CHECK(returnedManifest == expectedManifest);
     common::Logger::getInstance().setLevel(common::LogLevel::WARN);
 }
 
+TEST(SkopeoDriverTestGroup, filterInspectOutput) {
+    auto configRAII = test_utility::config::makeConfig();
+    auto& config = configRAII.config;
+    auto driver = image_manager::SkopeoDriver{config};
+
+    // Multi-line, indented
+    filterInspectOutputTestHelper(driver, "expected_manifests/alpine_3.14.json");
+    // Single line
+    filterInspectOutputTestHelper(driver, "expected_manifests/alpine_buildah.json");
+    // Multi-line, not indented
+    filterInspectOutputTestHelper(driver, "expected_manifests/zlib_ghcr.json");
+}
+
 TEST(SkopeoDriverTestGroup, manifestDigest) {
     auto configRAII = test_utility::config::makeConfig();
     auto& config = configRAII.config;
     boost::filesystem::remove_all(config->directories.cache);
 
     auto driver = image_manager::SkopeoDriver{config};
 
-    auto rawManifestPath = boost::filesystem::path{__FILE__}.parent_path() / "expected_inspect_raw_manifest.json";
+    auto rawManifestPath = boost::filesystem::path{__FILE__}.parent_path() / "expected_manifests/alpine_3.14.json";
     CHECK(driver.manifestDigest(rawManifestPath) == std::string{"sha256:1775bebec23e1f3ce486989bfc9ff3c4e951690df84aa9f926497d82f2ffca9d"});
 
     // OCI image blobs have their own digests as filenames, so it's a useful property for more test cases