repo: default to target arch for stage package cache (#3416)

- Add interface to project to define which arch should
  be used for staging packages.

- Add target_arch parameter to fetch_stage_packages().

- Update apt_cache to support specifying the default
  architecture for the stage package cache.

If running with an existing cache, check if <cache>/etc/apt
is a symlink (how it was done up until now) and unlink it.
Then copy the /etc/apt tree instead of linking it. Finally,
add a config file to specify the default target architecture.
Write that config file out every time the stage cache is
(re)initialized.

- Update all usage of fetch_stage_packages() to comply
  with new interface.

Signed-off-by: Chris Patterson <[email protected]>

snapcraft/internal/pluginhandler/__init__.py

@@ -457,6 +457,7 @@ def _fetch_stage_packages(self):
                     package_names=stage_packages,
                     base=self._project._get_build_base(),
                     stage_packages_path=self.stage_packages_path,
+                    target_arch=self._project._get_stage_packages_target_arch(),
                 )
             except repo.errors.PackageNotFoundError as e:
                 raise errors.StagePackageDownloadError(self.name, e.message)

snapcraft/internal/repo/_base.py

@@ -167,7 +167,12 @@ def get_installed_packages(cls) -> List[str]:
 
     @classmethod
     def fetch_stage_packages(
-        cls, *, package_names: List[str], base: str, stage_packages_path: pathlib.Path
+        cls,
+        *,
+        package_names: List[str],
+        base: str,
+        stage_packages_path: pathlib.Path,
+        target_arch: str,
     ) -> List[str]:
         """Fetch stage packages to stage_packages_path."""
         raise errors.NoNativeBackendError()

snapcraft/internal/repo/_deb.py

@@ -416,14 +416,21 @@ def _install_packages(cls, package_names: List[str]) -> None:
 
     @classmethod
     def fetch_stage_packages(
-        cls, *, package_names: List[str], base: str, stage_packages_path: pathlib.Path
+        cls,
+        *,
+        package_names: List[str],
+        base: str,
+        stage_packages_path: pathlib.Path,
+        target_arch: str,
     ) -> List[str]:
         logger.debug(f"Requested stage-packages: {sorted(package_names)!r}")
 
         installed: Set[str] = set()
 
         stage_packages_path.mkdir(exist_ok=True)
-        with AptCache(stage_cache=_STAGE_CACHE_DIR) as apt_cache:
+        with AptCache(
+            stage_cache=_STAGE_CACHE_DIR, stage_cache_arch=target_arch
+        ) as apt_cache:
             filter_packages = set(get_packages_in_base(base=base))
             apt_cache.update()
             apt_cache.mark_packages(set(package_names))

snapcraft/internal/repo/apt_cache.py

@@ -38,13 +38,19 @@
 class AptCache(ContextDecorator):
     """Transient cache for use with stage-packages, or read-only host-mode for build-packages."""
 
-    def __init__(self, *, stage_cache: Optional[Path] = None) -> None:
+    def __init__(
+        self,
+        *,
+        stage_cache: Optional[Path] = None,
+        stage_cache_arch: Optional[str] = None,
+    ) -> None:
         self.stage_cache = stage_cache
+        self.stage_cache_arch = stage_cache_arch
 
     def __enter__(self) -> "AptCache":
         if self.stage_cache is not None:
             self._configure_apt()
-            self._populate_cache_dir()
+            self._populate_stage_cache_dir()
             self.cache = apt.Cache(rootdir=str(self.stage_cache), memonly=True)
         else:
             # There appears to be a slowdown when using `rootdir` = '/' with
@@ -91,15 +97,35 @@ def _configure_apt(self):
             self.progress.pulse = lambda owner: True
             self.progress._width = 0
 
-    def _populate_cache_dir(self) -> None:
+    def _populate_stage_cache_dir(self) -> None:
+        """Create/refresh cache configuration.
+
+        (1) Delete old-style symlink cache, if symlink.
+        (2) Delete current-style (copied) tree.
+        (3) Copy current host apt configuration.
+        (4) Configure primary arch to target arch.
+        (5) Install dpkg into cache directory to support multi-arch.
+        """
         if self.stage_cache is None:
             return
 
-        # Create /etc inside cache root, and symlink apt to host's.
+        # Copy apt configuration from host.
         cache_etc_apt_path = Path(self.stage_cache, "etc", "apt")
-        if not cache_etc_apt_path.exists():
-            cache_etc_apt_path.parent.mkdir(parents=True, exist_ok=True)
-            os.symlink(Path("/etc/apt"), cache_etc_apt_path)
+
+        # Delete potentially outdated cache configuration.
+        if cache_etc_apt_path.is_symlink():
+            cache_etc_apt_path.unlink()
+        elif cache_etc_apt_path.exists():
+            shutil.rmtree(cache_etc_apt_path)
+
+        # Copy current cache configuration.
+        cache_etc_apt_path.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copytree("/etc/apt", cache_etc_apt_path)
+
+        # Specify default arch (if specified).
+        if self.stage_cache_arch is not None:
+            arch_conf_path = cache_etc_apt_path / "apt.conf.d" / "00default-arch"
+            arch_conf_path.write_text(f'APT::Architecture "{self.stage_cache_arch}";\n')
 
         # dpkg also needs to be in the rootdir in order to support multiarch
         # (apt calls dpkg --print-foreign-architectures).

snapcraft/plugins/v1/_ros/rosdep.py

@@ -100,7 +100,8 @@ def __init__(
         ros_package_path,
         rosdep_path,
         ubuntu_distro,
-        base
+        base,
+        target_arch,
     ):
         self._ros_distro = ros_distro
         self._ros_version = ros_version
@@ -115,6 +116,7 @@ def __init__(
         self._rosdep_install_path = os.path.join(self._rosdep_path, "install")
         self._rosdep_sources_path = os.path.join(self._rosdep_path, "sources.list.d")
         self._rosdep_cache_path = os.path.join(self._rosdep_path, "cache")
+        self._target_arch = target_arch
 
     def setup(self):
         # Make sure we can run multiple times without error, while leaving the
@@ -134,6 +136,7 @@ def setup(self):
             package_names=["python-rosdep"],
             stage_packages_path=self._rosdep_stage_packages_path,
             base=self._base,
+            target_arch=self._target_arch,
         )
         repo.Ubuntu.unpack_stage_packages(
             stage_packages_path=self._rosdep_stage_packages_path,

snapcraft/plugins/v1/_ros/wstool.py

@@ -89,6 +89,7 @@ def setup(self) -> None:
             package_names=["python-wstool"],
             stage_packages_path=self._wstool_stage_packages_path,
             base=self._base,
+            target_arch=self._project._get_stage_packages_target_arch(),
         )
         repo.Ubuntu.unpack_stage_packages(
             stage_packages_path=self._wstool_stage_packages_path,

snapcraft/plugins/v1/catkin.py

@@ -506,6 +506,7 @@ def pull(self):
             rosdep_path=self._rosdep_path,
             ubuntu_distro=_BASE_TO_UBUNTU_RELEASE_MAP[self.project._get_build_base()],
             base=self.project._get_build_base(),
+            target_arch=self.project._get_stage_packages_target_arch(),
         )
         rosdep.setup()
 
@@ -545,6 +546,7 @@ def _setup_apt_dependencies(self, apt_dependencies):
                 package_names=apt_dependencies,
                 stage_packages_path=self.stage_packages_path,
                 base=self.project._get_build_base(),
+                target_arch=self.project._get_stage_packages_target_arch(),
             )
         except repo.errors.PackageNotFoundError as e:
             raise CatkinAptDependencyFetchError(e.message)
@@ -988,6 +990,7 @@ def setup(self):
             package_names=["ros-{}-catkin".format(self._ros_distro)],
             stage_packages_path=self._catkin_stage_packages_path,
             base=self._project._get_build_base(),
+            target_arch=self._project._get_stage_packages_target_arch(),
         )
         repo.Ubuntu.unpack_stage_packages(
             stage_packages_path=self._catkin_stage_packages_path,

snapcraft/plugins/v1/colcon.py

@@ -364,6 +364,7 @@ def pull(self):
             rosdep_path=self._rosdep_path,
             ubuntu_distro=_BASE_TO_UBUNTU_RELEASE_MAP[self.project._get_build_base()],
             base=self.project._get_build_base(),
+            target_arch=self.project._get_stage_packages_target_arch(),
         )
         rosdep.setup()
 
@@ -389,6 +390,7 @@ def _setup_apt_dependencies(self, apt_dependencies):
                 package_names=apt_dependencies,
                 stage_packages_path=self.stage_packages_path,
                 base=self.project._get_build_base(),
+                target_arch=self.project._get_stage_packages_target_arch(),
             )
         except repo.errors.PackageNotFoundError as e:
             raise ColconAptDependencyFetchError(e.message)

snapcraft/plugins/v2/_ros.py

@@ -116,6 +116,8 @@ def _get_stage_runtime_dependencies_commands(self) -> List[str]:
                     "$SNAPCRAFT_PART_INSTALL",
                     "--ros-distro",
                     "$ROS_DISTRO",
+                    "--target-arch",
+                    "$SNAPCRAFT_TARGET_ARCH",
                 ]
             )
         ]
@@ -142,7 +144,10 @@ def plugin_cli():
 @click.option("--part-src", envvar="SNAPCRAFT_PART_SRC", required=True)
 @click.option("--part-install", envvar="SNAPCRAFT_PART_INSTALL", required=True)
 @click.option("--ros-distro", envvar="ROS_DISTRO", required=True)
-def stage_runtime_dependencies(part_src: str, part_install: str, ros_distro: str):
+@click.option("--target-arch", envvar="SNAPCRAFT_TARGET_ARCH", required=True)
+def stage_runtime_dependencies(
+    part_src: str, part_install: str, ros_distro: str, target_arch: str
+):
     click.echo("Staging runtime dependencies...")
     # TODO: support python packages (only apt currently supported)
     apt_packages: Set[str] = set()
@@ -180,6 +185,7 @@ def stage_runtime_dependencies(part_src: str, part_install: str, ros_distro: str
             package_names=package_names,
             base="core20",
             stage_packages_path=stage_packages_path,
+            target_arch=target_arch,
         )
 
         click.echo(f"Unpacking stage packages: {package_names!r}")

snapcraft/project/_project.py

@@ -136,6 +136,19 @@ def _get_global_state_file_path(self) -> str:
 
         return state_file_path
 
+    def _get_stage_packages_target_arch(self) -> str:
+        """Get architecture for staging packages.
+
+        Prior to core20, staging packages has broken behavior in that it will
+        stage native architecture packages by default.
+
+        :return: The appropriate default architecture to stage.
+        """
+        if self._get_build_base() in ["core16", "core18"]:
+            return self.deb_arch
+        else:
+            return self.target_arch
+
     def _get_start_time(self) -> datetime:
         """Returns the timestamp for when a snapcraft project was loaded."""
         return self._start_time

snapcraft/project/_project_options.py

@@ -250,6 +250,10 @@ def _get_provider_content_dirs(self) -> Set[str]:
         """
         return set()
 
+    def _get_stage_packages_target_arch(self) -> str:
+        """Stub for 'Project' interface for tests using ProjectOptions()."""
+        return self.deb_arch
+
     def is_static_base(self, base: str) -> bool:
         """Return True if a base that is intended to be static is used.
 

spread.yaml

@@ -242,6 +242,13 @@ suites:
    - ubuntu-16.04*
    - ubuntu-18.04*
 
+ # General, core suite
+ tests/spread/cross-compile/:
+   summary: tests of supported cross-compile functionality
+   systems:
+   - ubuntu-18.04*
+   - ubuntu-20.04*
+
  # Use of multipass and lxd build providers
  tests/spread/build-providers/:
    summary: tests of snapcraft using build providers

tests/spread/cross-compile/stage-packages/snap/snapcraft.yaml

@@ -0,0 +1,15 @@
+name: cross-compile-stage-package-test
+base: core20
+version: '0.1'
+summary: Test
+description: |
+  Test xcompile stage-packages behavior for core20.
+
+grade: stable
+confinement: strict
+
+parts:
+  my-part:
+    plugin: nil
+    stage-packages:
+    - jq

tests/spread/cross-compile/stage-packages/task.yaml

@@ -0,0 +1,78 @@
+summary: Cross-compiliation stage-package test
+
+systems:
+  - ubuntu-20.04*
+
+environment:
+  SNAP_DIR: .
+
+prepare: |
+  #shellcheck source=tests/spread/tools/snapcraft-yaml.sh
+  . "$TOOLS_DIR/snapcraft-yaml.sh"
+  set_base "$SNAP_DIR/snap/snapcraft.yaml"
+
+  # For gcc and dpkg-architecture.
+  apt-get install -y dpkg-dev gcc
+  apt-mark auto dpkg-dev gcc
+
+  # Add architecture to sources.
+  codename="$(lsb_release -cs)"
+  echo "deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports $codename main restricted universe multiverse" > /etc/apt/sources.list.d/armhf.list
+
+  # Save original sources, but specify to host arch only.
+  host_arch="$(dpkg-architecture  | grep DEB_BUILD_ARCH= | cut -f 2 -d =)"
+  cp /etc/apt/sources.list /etc/apt/sources.list.save
+  sed -i "s|^deb |deb [arch=$host_arch] |g" /etc/apt/sources.list
+
+  # Add armhf arch and update apt cache.
+  dpkg --add-architecture armhf
+  apt-get update
+
+restore: |
+  cd "$SNAP_DIR"
+
+  snapcraft clean
+  rm -f ./*.snap
+
+  # Remove architecture from sources.
+  mv /etc/apt/sources.list.save /etc/apt/sources.list
+  rm -f /etc/apt/sources.list.d/armhf.list
+  rm -f /etc/apt/sources.list.d/main.list
+
+  # Remove arch and update apt cache.
+  dpkg --remove-architecture armhf
+  apt-get update
+
+  #shellcheck source=tests/spread/tools/snapcraft-yaml.sh
+  . "$TOOLS_DIR/snapcraft-yaml.sh"
+  restore_yaml "snap/snapcraft.yaml"
+
+execute: |
+  cd "$SNAP_DIR"
+
+  host_arch="$(dpkg-architecture  | grep DEB_BUILD_ARCH= | cut -f 2 -d =)"
+  host_arch_triplet="$(gcc -dumpmachine)"
+
+  # First build for armhf.
+  snapcraft --target-arch armhf --enable-experimental-target-arch
+
+  [ -f cross-compile-stage-package-test_0.1_armhf.snap ]
+  [ -f prime/usr/lib/arm-linux-gnueabihf/libjq.so.1 ]
+  [ "$(ls prime/usr/lib/)" == "arm-linux-gnueabihf" ]
+  rm cross-compile-stage-package-test_0.1_armhf.snap
+
+  # Re-spin as native.
+  snapcraft
+
+  [ -f "cross-compile-stage-package-test_0.1_$host_arch.snap" ]
+  [ -f "prime/usr/lib/$host_arch_triplet/libjq.so.1" ]
+  [ "$(ls prime/usr/lib/)" == "$host_arch_triplet" ]
+  rm "cross-compile-stage-package-test_0.1_$host_arch.snap"
+
+  # Re-build for armhf.
+  snapcraft --target-arch armhf --enable-experimental-target-arch
+
+  [ -f cross-compile-stage-package-test_0.1_armhf.snap ]
+  [ -f prime/usr/lib/arm-linux-gnueabihf/libjq.so.1 ]
+  [ "$(ls prime/usr/lib/)" == "arm-linux-gnueabihf" ]
+  rm cross-compile-stage-package-test_0.1_armhf.snap

tests/unit/plugins/v1/ros/test_rosdep.py

@@ -38,6 +38,7 @@ def setUp(self):
             rosdep_path="rosdep_path",
             ubuntu_distro="xenial",
             base="core",
+            target_arch=self.project._get_stage_packages_target_arch(),
         )
 
         patcher = mock.patch("snapcraft.repo.Ubuntu")
@@ -63,6 +64,7 @@ def test_setup(self):
                         stage_packages_path=self.rosdep._rosdep_stage_packages_path,
                         package_names=["python-rosdep"],
                         base="core",
+                        target_arch=self.project._get_stage_packages_target_arch(),
                     )
                 ]
             ),

tests/unit/plugins/v1/ros/test_wstool.py

@@ -55,6 +55,7 @@ def test_setup(self):
                         stage_packages_path=self.wstool._wstool_stage_packages_path,
                         package_names=["python-wstool"],
                         base="core",
+                        target_arch=self.project.target_arch,
                     )
                 ]
             ),