Merge branch 'swiftlang#116-support-debian-distributions' into develop

Sources/GeneratorCLI/GeneratorCLI.swift

@@ -200,7 +200,8 @@ extension GeneratorCLI {
 
     @Option(
       help: """
-      Linux distribution to use if the target platform is Linux. Available options: `ubuntu`, `rhel`. Default is `ubuntu`.
+      Linux distribution to use if the target platform is Linux.
+      - Available options: `ubuntu`, `debian`, `rhel`. Default is `ubuntu`.
       """,
       transform: LinuxDistribution.Name.init(nameString:)
     )
@@ -209,8 +210,9 @@ extension GeneratorCLI {
     @Option(
       help: """
       Version of the Linux distribution used as a target platform.
-      Available options for Ubuntu: `20.04`, `22.04` (default when `--linux-distribution-name` is `ubuntu`), `24.04`.
-      Available options for RHEL: `ubi9` (default when `--linux-distribution-name` is `rhel`).
+      - Available options for Ubuntu: `20.04`, `22.04` (default when `--linux-distribution-name` is `ubuntu`), `24.04`.
+      - Available options for Debian: `11`, `12` (default when `--linux-distribution-name` is `debian`).
+      - Available options for RHEL: `ubi9` (default when `--linux-distribution-name` is `rhel`).
       """
     )
     var linuxDistributionVersion: String?
@@ -238,6 +240,8 @@ extension GeneratorCLI {
         linuxDistributionDefaultVersion = "ubi9"
       case .ubuntu:
         linuxDistributionDefaultVersion = "22.04"
+      case .debian:
+        linuxDistributionDefaultVersion = "12"
       }
       let linuxDistributionVersion = self.linuxDistributionVersion ?? linuxDistributionDefaultVersion
       let linuxDistribution = try LinuxDistribution(name: linuxDistributionName, version: linuxDistributionVersion)

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift

@@ -72,7 +72,7 @@ extension SwiftSDKGenerator {
         // architecture-specific directories:
         //   https://wiki.ubuntu.com/MultiarchSpec
         // But not in all containers, so don't fail if it does not exist.
-        if case .ubuntu = targetDistribution {
+        if targetDistribution.name == .ubuntu || targetDistribution.name == .debian {
           subpaths += [("\(targetTriple.archName)-linux-gnu", false)]
 
           // Custom subpath for armv7

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift

@@ -21,8 +21,9 @@ import struct Foundation.URL
 
 import struct SystemPackage.FilePath
 
-private let ubuntuAMD64Mirror = "http://gb.archive.ubuntu.com/ubuntu"
-private let ubuntuARM64Mirror = "http://ports.ubuntu.com/ubuntu-ports"
+private let ubuntuMainMirror = "http://gb.archive.ubuntu.com/ubuntu"
+private let ubuntuPortsMirror = "http://ports.ubuntu.com/ubuntu-ports"
+private let debianMirror = "http://deb.debian.org/debian"
 
 extension FilePath {
   var metadataValue: Logger.MetadataValue {
@@ -73,64 +74,113 @@ extension SwiftSDKGenerator {
     ])
   }
 
-  func downloadUbuntuPackages(
+  func getMirrorURL(for linuxDistribution: LinuxDistribution) throws -> String {
+    if linuxDistribution.name == .ubuntu {
+      if targetTriple.arch == .x86_64 {
+        return ubuntuMainMirror
+      } else {
+        return ubuntuPortsMirror
+      }
+    } else if linuxDistribution.name == .debian {
+      return debianMirror
+    } else {
+      throw GeneratorError.distributionSupportsOnlyDockerGenerator(linuxDistribution)
+    }
+  }
+
+  func packagesFileName(isXzAvailable: Bool) -> String {
+    if isXzAvailable {
+      return "Packages.xz"
+    }
+    // Use .gz if xz is not available
+    return "Packages.gz"
+  }
+
+  func downloadDebianPackages(
     _ client: some HTTPClientProtocol,
     _ engine: QueryEngine,
     requiredPackages: [String],
     versionsConfiguration: VersionsConfiguration,
     sdkDirPath: FilePath
   ) async throws {
-    logger.debug("Parsing Ubuntu packages list...")
+    let mirrorURL = try getMirrorURL(for: versionsConfiguration.linuxDistribution)
+    let distributionName = versionsConfiguration.linuxDistribution.name
+    let distributionRelease = versionsConfiguration.linuxDistribution.release
 
     // Find xz path
     let xzPath = try await which("xz")
     if xzPath == nil {
+      // If we don't have xz, it's required for Packages.xz for debian
+      if distributionName == .debian {
+        throw GeneratorError.debianPackagesListDownloadRequiresXz
+      }
+
       logger.warning("""
       The `xz` utility was not found in `PATH`. \
       Consider installing it for more efficient downloading of package lists.
       """)
     }
 
-    async let mainPackages = try await client.parseUbuntuPackagesList(
-      ubuntuRelease: versionsConfiguration.linuxDistribution.release,
-      repository: "main",
-      targetTriple: self.targetTriple,
-      isVerbose: self.isVerbose,
-      xzPath: xzPath
-    )
-
-    async let updatesPackages = try await client.parseUbuntuPackagesList(
-      ubuntuRelease: versionsConfiguration.linuxDistribution.release,
-      releaseSuffix: "-updates",
-      repository: "main",
-      targetTriple: self.targetTriple,
-      isVerbose: self.isVerbose,
-      xzPath: xzPath
-    )
-
-    async let universePackages = try await client.parseUbuntuPackagesList(
-      ubuntuRelease: versionsConfiguration.linuxDistribution.release,
-      releaseSuffix: "-updates",
-      repository: "universe",
-      targetTriple: self.targetTriple,
-      isVerbose: self.isVerbose,
-      xzPath: xzPath
-    )
-
-    let allPackages = try await mainPackages
-      .merging(updatesPackages, uniquingKeysWith: { $1 })
-      .merging(universePackages, uniquingKeysWith: { $1 })
+    logger.info("Downloading and parsing packages lists...", metadata: [
+      "distributionName": .stringConvertible(distributionName), "distributionRelease": .string(distributionRelease)
+    ])
+
+    let allPackages = try await withThrowingTaskGroup(of: [String: URL].self) { group in
+      group.addTask {
+        return try await self.parseDebianPackageList(
+          using: client,
+          mirrorURL: mirrorURL,
+          release: distributionRelease,
+          releaseSuffix: "",
+          repository: "main",
+          targetTriple: self.targetTriple,
+          xzPath: xzPath
+        )
+      }
+      group.addTask {
+        return try await self.parseDebianPackageList(
+          using: client,
+          mirrorURL: mirrorURL,
+          release: distributionRelease,
+          releaseSuffix: "-updates",
+          repository: "main",
+          targetTriple: self.targetTriple,
+          xzPath: xzPath
+        )
+      }
+      if distributionName == .ubuntu {
+        group.addTask {
+          return try await self.parseDebianPackageList(
+            using: client,
+            mirrorURL: mirrorURL,
+            release: distributionRelease,
+            releaseSuffix: "-updates",
+            repository: "universe",
+            targetTriple: self.targetTriple,
+            xzPath: xzPath
+          )
+        }
+      }
+
+      var packages: [String : URL] = [String: URL]()
+      for try await result in group {
+        packages.merge(result, uniquingKeysWith: { $1 })
+      }
+      return packages
+    }
 
     let urls = requiredPackages.compactMap { allPackages[$0] }
 
     guard urls.count == requiredPackages.count else {
-      throw GeneratorError.ubuntuPackagesParsingFailure(
+      throw GeneratorError.packagesListParsingFailure(
         expectedPackages: requiredPackages.count,
         actual: urls.count
       )
     }
 
-    logger.info("Downloading Ubuntu packages...", metadata: ["packageCount": .stringConvertible(urls.count)])
+    logger.info("Downloading packages...", metadata: [
+      "distributionName": .stringConvertible(distributionName), "packageCount": .stringConvertible(urls.count)
+    ])
     try await inTemporaryDirectory { fs, tmpDir in
       let downloadedFiles = try await self.downloadFiles(from: urls, to: tmpDir, client, engine)
       await report(downloadedFiles: downloadedFiles)
@@ -142,6 +192,65 @@ extension SwiftSDKGenerator {
     }
   }
 
+  private func parseDebianPackageList(
+    using client: HTTPClientProtocol,
+    mirrorURL: String,
+    release: String,
+    releaseSuffix: String,
+    repository: String,
+    targetTriple: Triple,
+    xzPath: String?
+  ) async throws -> [String: URL] {
+    var contextLogger = logger
+
+    let packagesListURL = """
+    \(mirrorURL)/dists/\(release)\(releaseSuffix)/\(repository)/binary-\(
+      targetTriple.arch!.debianConventionName
+    )/\(packagesFileName(isXzAvailable: xzPath != nil))
+    """
+    contextLogger[metadataKey: "packagesListURL"] = .string(packagesListURL)
+
+    contextLogger.debug("Downloading packages list...")
+    guard let packages = try await client.downloadDebianPackagesList(
+      from: packagesListURL,
+      unzipWith: xzPath ?? "/usr/bin/gzip", // fallback on gzip if xz not available
+      logger: logger
+    ) else {
+      throw GeneratorError.packagesListDecompressionFailure
+    }
+
+    let packageRef = Reference(Substring.self)
+    let pathRef = Reference(Substring.self)
+
+    let regex = Regex {
+      "Package: "
+
+      Capture(as: packageRef) {
+        OneOrMore(.anyNonNewline)
+      }
+
+      OneOrMore(.any, .reluctant)
+
+      "Filename: "
+
+      Capture(as: pathRef) {
+        OneOrMore(.anyNonNewline)
+      }
+
+      Anchor.endOfLine
+    }
+
+    contextLogger.debug("Processing packages list...")
+    var result = [String: URL]()
+    for match in packages.matches(of: regex) {
+      guard let url = URL(string: "\(mirrorURL)/\(match[pathRef])") else { continue }
+
+      result[String(match[packageRef])] = url
+    }
+
+    return result
+  }
+
   func downloadFiles(
     from urls: [URL],
     to directory: FilePath,
@@ -185,89 +294,15 @@ extension SwiftSDKGenerator {
 }
 
 extension HTTPClientProtocol {
-  private func downloadUbuntuPackagesList(
+  func downloadDebianPackagesList(
     from url: String,
     unzipWith zipPath: String,
-    isVerbose: Bool
+    logger: Logger
   ) async throws -> String? {
-    guard let packages = try await get(url: url).body?.unzip(zipPath: zipPath, isVerbose: isVerbose) else {
+    guard let packages = try await get(url: url).body?.unzip(zipPath: zipPath, logger: logger) else {
       throw FileOperationError.downloadFailed(url)
     }
 
     return String(buffer: packages)
   }
-
-  func packagesFileName(isXzAvailable: Bool) -> String {
-    if isXzAvailable {
-      return "Packages.xz"
-    }
-    // Use .gz if xz is not available
-    return "Packages.gz"
-  }
-
-  func parseUbuntuPackagesList(
-    ubuntuRelease: String,
-    releaseSuffix: String = "",
-    repository: String,
-    targetTriple: Triple,
-    isVerbose: Bool,
-    xzPath: String?
-  ) async throws -> [String: URL] {
-    let mirrorURL: String
-    if targetTriple.arch == .x86_64 {
-      mirrorURL = ubuntuAMD64Mirror
-    } else {
-      mirrorURL = ubuntuARM64Mirror
-    }
-
-    let packagesListURL = """
-    \(mirrorURL)/dists/\(ubuntuRelease)\(releaseSuffix)/\(repository)/binary-\(
-      targetTriple.arch!.debianConventionName
-    )/\(packagesFileName(isXzAvailable: xzPath != nil))
-    """
-
-    guard let packages = try await downloadUbuntuPackagesList(
-      from: packagesListURL,
-      unzipWith: xzPath ?? "/usr/bin/gzip", // fallback on gzip if xz not available
-      isVerbose: isVerbose
-    ) else {
-      throw GeneratorError.ubuntuPackagesDecompressionFailure
-    }
-
-    let packageRef = Reference(Substring.self)
-    let pathRef = Reference(Substring.self)
-
-    let regex = Regex {
-      "Package: "
-
-      Capture(as: packageRef) {
-        OneOrMore(.anyNonNewline)
-      }
-
-      OneOrMore(.any, .reluctant)
-
-      "Filename: "
-
-      Capture(as: pathRef) {
-        OneOrMore(.anyNonNewline)
-      }
-
-      Anchor.endOfLine
-
-      OneOrMore(.any, .reluctant)
-
-      "Description-md5: "
-
-      OneOrMore(.hexDigit)
-    }
-
-    var result = [String: URL]()
-    for match in packages.matches(of: regex) {
-      guard let url = URL(string: "\(mirrorURL)/\(match[pathRef])") else { continue }
-
-      result[String(match[packageRef])] = url
-    }
-
-    return result
-  }
 }