improve build-docker-image

.github/workflows/ci.yml

@@ -186,7 +186,7 @@ jobs:
         id: build-docker-image
         if: steps.prepare-meta.outputs.has-image
         timeout-minutes: 120
-        run: cargo xtask build-docker-image -v "${TARGET}${SUB:+.$SUB}" ${{ matrix.verbose && '-v' || ''  }}
+        run: cargo xtask build-docker-image -v "${TARGET}${SUB:+.$SUB}" ${{ matrix.verbose && '-v' || ''  }}${{ matrix.platform && '--platform ' }}${{ join(matrix.platform) || '' }}
         env:
           TARGET: ${{ matrix.target }}
           SUB: ${{ matrix.sub }}
@@ -246,7 +246,7 @@ jobs:
             github.ref == format('refs/heads/{0}', github.event.repository.default_branch) ||
             startsWith(github.ref, 'refs/tags/v')
           )
-        run: cargo xtask build-docker-image -v --push "${TARGET}${SUB:+.$SUB}"
+        run: cargo xtask build-docker-image -v --push "${TARGET}${SUB:+.$SUB}" ${{ matrix.verbose && '-v' || ''  }}${{ matrix.platform && '--platform ' }}${{ join(matrix.platform) || '' }}
         env:
           TARGET: ${{ matrix.target }}
           SUB: ${{ matrix.sub }}

docker/qemu.sh

@@ -131,7 +131,7 @@ build_static_slirp() {
 
     rm -rf "${td}"
 }
-
+# passed arg 1: $1 = arch, so for example "arm64" or "aarch64"
 main() {
     local version=5.1.0
 

src/docker/image.rs

@@ -162,7 +162,7 @@ impl From<String> for ImageReference {
 /// The architecture/platform to use in the image
 ///
 /// <https://github.com/containerd/containerd/blob/release/1.6/platforms/platforms.go#L63>
-#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize)]
 #[serde(try_from = "String")]
 pub struct ImagePlatform {
     /// CPU architecture, x86_64, aarch64 etc

src/docker/provided_images.rs

@@ -5,7 +5,7 @@ use super::{ImagePlatform, ProvidedImage};
 pub static PROVIDED_IMAGES: &[ProvidedImage] = &[
         ProvidedImage {
             name: "x86_64-unknown-linux-gnu",
-            platforms: &[ImagePlatform::X86_64_UNKNOWN_LINUX_GNU],
+            platforms: &[ImagePlatform::X86_64_UNKNOWN_LINUX_GNU, ImagePlatform::AARCH64_UNKNOWN_LINUX_GNU],
             sub: None
         },
         ProvidedImage {
@@ -20,7 +20,7 @@ pub static PROVIDED_IMAGES: &[ProvidedImage] = &[
         },
         ProvidedImage {
             name: "aarch64-unknown-linux-gnu",
-            platforms: &[ImagePlatform::X86_64_UNKNOWN_LINUX_GNU],
+            platforms: &[ImagePlatform::X86_64_UNKNOWN_LINUX_GNU, ImagePlatform::AARCH64_UNKNOWN_LINUX_GNU],
             sub: None
         },
         ProvidedImage {

src/lib.rs

@@ -72,7 +72,7 @@ pub use self::rustc::{TargetList, VersionMetaExt};
 pub const CROSS_LABEL_DOMAIN: &str = "org.cross-rs";
 
 #[allow(non_camel_case_types)]
-#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Hash)]
+#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Hash, PartialOrd, Ord)]
 #[serde(from = "&str", into = "String")]
 #[serde(rename_all = "snake_case")]
 pub enum TargetTriple {

targets.toml

@@ -11,6 +11,7 @@ deploy = true
 
 [[target]]
 target = "x86_64-unknown-linux-gnu"
+platforms = ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"]
 os = "ubuntu-latest"
 cpp = true
 dylib = true
@@ -47,6 +48,7 @@ runners = "native qemu-user qemu-system"
 
 [[target]]
 target = "aarch64-unknown-linux-gnu"
+platforms = ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"]
 os = "ubuntu-latest"
 cpp = true
 dylib = true
@@ -391,13 +393,15 @@ run = true
 [[target]]
 target = "x86_64-pc-windows-gnu"
 os = "ubuntu-latest"
+platforms = ["x86_64-unknown-linux-gnu"]
 cpp = true
 std = true
 run = true
 
 [[target]]
 target = "i686-pc-windows-gnu"
 os = "ubuntu-latest"
+platforms = ["x86_64-unknown-linux-gnu"]
 cpp = true
 std = true
 run = true
@@ -531,6 +535,7 @@ special = true
 [[target]]
 target = "aarch64-unknown-linux-gnu"
 sub = "centos"
+platforms = ["x86_64-unknown-linux-gnu"]
 os = "ubuntu-latest"
 cpp = true
 dylib = true

xtask/src/build_docker_image.rs

@@ -146,6 +146,12 @@ pub fn build_docker_image(
                 .contents_first(true)
                 .into_iter()
                 .filter_map(|e| e.ok().filter(|f| f.file_type().is_file()))
+                // don't add native
+                .filter(|f| {
+                    !f.file_name()
+                        .to_string_lossy()
+                        .starts_with("Dockerfile.native")
+                })
                 .filter_map(|f| {
                     f.file_name()
                         .to_string_lossy()
@@ -156,6 +162,17 @@ pub fn build_docker_image(
                 .collect();
         }
     }
+
+    // grab platform from the targets.toml file, if it exists.
+    targets.iter_mut().for_each(|t| {
+        if let Some(m) = get_matrix()
+            .iter()
+            .find(|m| m.target == t.name && m.sub == t.sub)
+        {
+            t.platform = m.platforms.clone();
+        }
+    });
+
     let gha = std::env::var("GITHUB_ACTIONS").is_ok();
     let mut progress = progress.map(|x| x.parse().unwrap());
     if gha {
@@ -178,10 +195,17 @@ pub fn build_docker_image(
     let mut results = vec![];
     for (platform, (target, dockerfile)) in targets
         .iter()
-        .flat_map(|t| platforms.iter().map(move |p| (p, t)))
+        .flat_map(|(t, d)| {
+            t.platform
+                .as_deref()
+                .unwrap_or(&platforms)
+                .iter()
+                .map(move |p| (p, (t, d)))
+        })
+        .filter(|(p, _)| platforms.contains(p))
     {
         if gha && targets.len() > 1 {
-            gha_print("::group::Build {target}");
+            gha_print(&format!("::group::Build {target} for {}", platform.target));
         } else {
             msg_info.note(format_args!("Build {target} for {}", platform.target))?;
         }

xtask/src/ci/target_matrix.rs

@@ -2,7 +2,7 @@ use std::process::Command;
 
 use clap::builder::{BoolishValueParser, PossibleValuesParser};
 use clap::Parser;
-use cross::{shell::Verbosity, CommandExt};
+use cross::{docker::ImagePlatform, shell::Verbosity, CommandExt};
 use serde::{Deserialize, Serialize};
 
 use crate::util::{get_matrix, gha_output, gha_print, CiTarget, ImageTarget};
@@ -62,6 +62,7 @@ impl TargetMatrix {
                     runners: vec![],
                     none: false,
                     has_image: true,
+                    platform: vec![],
                     verbose: false,
                     tests: vec!["all".to_owned()],
                 },
@@ -244,7 +245,7 @@ fn process_try_comment(message: &str) -> cross::Result<(bool, TargetMatrixArgs)>
 #[serde(rename_all = "kebab-case")]
 struct TargetMatrixElement<'a> {
     pretty: String,
-    platforms: &'a [String],
+    platforms: &'a [ImagePlatform],
     target: &'a str,
     #[serde(skip_serializing_if = "Option::is_none")]
     sub: Option<&'a str>,
@@ -285,6 +286,8 @@ struct TargetMatrixArgs {
     none: bool,
     #[clap(long)]
     has_image: bool,
+    #[clap(long, num_args = 0..)]
+    platform: Vec<ImagePlatform>,
     #[clap(long, short)]
     verbose: bool,
     #[clap(long, value_parser = PossibleValuesParser::new(&[
@@ -314,6 +317,7 @@ impl Default for TargetMatrixArgs {
             runners: Vec::new(),
             none: false,
             has_image: false,
+            platform: Vec::new(),
             verbose: false,
             tests: vec!["all".to_owned()],
         }
@@ -367,6 +371,22 @@ impl TargetMatrixArgs {
                     .any(|runner| m.runners.as_deref().unwrap_or_default().contains(runner))
             });
         }
+        if !self.platform.is_empty() {
+            matrix.retain(|t| {
+                t.platforms()
+                    .iter()
+                    .any(|platform| self.platform.contains(platform))
+            });
+            matrix.iter_mut().for_each(|t| {
+                t.platforms = Some(
+                    t.platforms()
+                        .iter()
+                        .filter(|&p| self.platform.contains(p))
+                        .cloned()
+                        .collect(),
+                )
+            });
+        }
     }
 
     fn tests(&self) -> Result<serde_json::Value, serde_json::Error> {

xtask/src/codegen.rs

@@ -1,5 +1,4 @@
 use clap::Args;
-use cross::docker::ImagePlatform;
 use eyre::Context;
 use std::fmt::Write;
 
@@ -39,10 +38,7 @@ pub static PROVIDED_IMAGES: &[ProvidedImage] = &["#,
             platform = &image_target
                 .platforms()
                 .iter()
-                .map(|p| {
-                    let image_platform: ImagePlatform =
-                        p.parse().expect("should be a valid platform");
-
+                .map(|image_platform| {
                     image_platform
                         .to_codegen_string()
                         .expect("should be a valid platform")

xtask/src/util.rs

@@ -4,8 +4,8 @@ use std::io::Write;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
-use cross::shell::MessageInfo;
 use cross::{docker, CommandExt, ToUtf8};
+use cross::{docker::ImagePlatform, shell::MessageInfo};
 
 use once_cell::sync::{Lazy, OnceCell};
 use serde::Deserialize;
@@ -46,9 +46,9 @@ pub struct CiTarget {
     /// if `true` publish the generated binaries for cross
     #[serde(default)]
     pub deploy: Option<bool>,
-    /// the platform to build this image for, defaults to `["linux/amd64"]`, takes multiple
+    /// the platform to build this image for, defaults to whatever is needed, takes multiple
     #[serde(skip_serializing_if = "Option::is_none")]
-    platforms: Option<Vec<String>>,
+    pub platforms: Option<Vec<ImagePlatform>>,
     /// if `true` signal that this target requires `-Zbuild-std`
     #[serde(skip_serializing_if = "Option::is_none")]
     pub build_std: Option<bool>,
@@ -82,15 +82,17 @@ impl CiTarget {
         crate::ImageTarget {
             name: self.target.clone(),
             sub: self.sub.clone(),
+            // XXX: This does not align with platforms() by design, as we want to be able to know if the field was set or not.
+            platform: self.platforms.clone(),
         }
     }
 
     pub fn builds_image(&self) -> bool {
         self.os == "ubuntu-latest"
     }
 
-    pub fn platforms(&self) -> &[String] {
-        self.platforms.as_ref().unwrap_or(&DEFAULT_PLATFORMS_STRING)
+    pub fn platforms(&self) -> &[ImagePlatform] {
+        self.platforms.as_deref().unwrap_or(DEFAULT_PLATFORMS)
     }
 }
 
@@ -162,6 +164,7 @@ pub fn pull_image(
 pub struct ImageTarget {
     pub name: String,
     pub sub: Option<String>,
+    pub platform: Option<Vec<ImagePlatform>>,
 }
 
 impl ImageTarget {
@@ -235,13 +238,15 @@ impl std::str::FromStr for ImageTarget {
                 return Ok(ImageTarget {
                     name: target.to_string(),
                     sub: Some(sub.to_string()),
+                    platform: None,
                 });
             }
         }
 
         Ok(ImageTarget {
             name: s.to_string(),
             sub: None,
+            platform: None,
         })
     }
 }
@@ -382,27 +387,31 @@ mod tests {
             ImageTarget {
                 name: "x86_64-unknown-linux-gnu".to_owned(),
                 sub: None,
+                platform: None,
             },
             "x86_64-unknown-linux-gnu".parse().unwrap()
         );
         assert_eq!(
             ImageTarget {
                 name: "x86_64-unknown-linux-gnu".to_owned(),
                 sub: Some("centos".to_owned()),
+                platform: None,
             },
             "x86_64-unknown-linux-gnu.centos".parse().unwrap()
         );
         assert_eq!(
             ImageTarget {
                 name: "thumbv8m.main-none-eabihf".to_owned(),
                 sub: None,
+                platform: None,
             },
             "thumbv8m.main-none-eabihf".parse().unwrap()
         );
         assert_eq!(
             ImageTarget {
                 name: "thumbv8m.main-unknown-linux-gnueabihf".to_owned(),
                 sub: Some("alpine".to_owned()),
+                platform: None,
             },
             "thumbv8m.main-unknown-linux-gnueabihf.alpine"
                 .parse()