Merge pull request #3758 from mr-cal/environmental-variables-2

environment: get LD_LIBRARY_PATH and migrate setup logic

snapcraft/meta/snap_yaml.py

@@ -23,6 +23,7 @@
 from pydantic_yaml import YamlModel
 
 from snapcraft.projects import Project
+from snapcraft.utils import get_ld_library_paths
 
 
 class Socket(YamlModel):
@@ -120,7 +121,7 @@ class Config:  # pylint: disable=too-few-public-methods
         alias_generator = lambda s: s.replace("_", "-")  # noqa: E731
 
 
-def write(project: Project, prime_dir: Path, *, arch: str):
+def write(project: Project, prime_dir: Path, *, arch: str, arch_triplet: str):
     """Create a snap.yaml file."""
     meta_dir = prime_dir / "meta"
     meta_dir.mkdir(parents=True, exist_ok=True)
@@ -173,6 +174,8 @@ def write(project: Project, prime_dir: Path, *, arch: str):
     if project.hooks and any(h for h in project.hooks.values() if h.command_chain):
         assumes.add("command-chain")
 
+    environment = _populate_environment(project.environment, prime_dir, arch_triplet)
+
     snap_metadata = SnapMetadata(
         name=project.name,
         title=project.title,
@@ -188,7 +191,7 @@ def write(project: Project, prime_dir: Path, *, arch: str):
         apps=snap_apps or None,
         confinement=project.confinement,
         grade=project.grade or "stable",
-        environment=project.environment,
+        environment=environment,
         plugs=project.plugs,
         slots=project.slots,
         hooks=project.hooks,
@@ -214,3 +217,38 @@ def _repr_str(dumper, data):
     if "\n" in data:
         return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
     return dumper.represent_scalar("tag:yaml.org,2002:str", data)
+
+
+def _populate_environment(
+    environment: Optional[Dict[str, Optional[str]]], prime_dir: Path, arch_triplet: str
+):
+    """Populate default app environmental variables.
+
+    Three cases for LD_LIBRARY_PATH and PATH variables:
+        - If LD_LIBRARY_PATH or PATH are defined, keep user-defined values.
+        - If LD_LIBRARY_PATH or PATH are not defined, set to default values.
+        - If LD_LIBRARY_PATH or PATH are null, do not use default values.
+    """
+    if environment is None:
+        return {
+            "LD_LIBRARY_PATH": get_ld_library_paths(prime_dir, arch_triplet),
+            "PATH": "$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH",
+        }
+
+    try:
+        if not environment["LD_LIBRARY_PATH"]:
+            environment.pop("LD_LIBRARY_PATH")
+    except KeyError:
+        environment["LD_LIBRARY_PATH"] = get_ld_library_paths(prime_dir, arch_triplet)
+
+    try:
+        if not environment["PATH"]:
+            environment.pop("PATH")
+    except KeyError:
+        environment["PATH"] = "$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH"
+
+    if len(environment):
+        return environment
+
+    # if the environment only contained a null LD_LIBRARY_PATH and a null PATH, return None
+    return None

snapcraft/parts/lifecycle.py

@@ -251,6 +251,7 @@ def _run_command(
             project,
             lifecycle.prime_dir,
             arch=lifecycle.target_arch,
+            arch_triplet=lifecycle.target_arch_triplet,
         )
         emit.message("Generated snap metadata", intermediate=True)
 
@@ -378,7 +379,9 @@ def _set_step_environment(step_info: StepInfo) -> bool:
     return True
 
 
-def _expand_environment(snapcraft_yaml: Dict[str, Any], *, parallel_build_count: int) -> None:
+def _expand_environment(
+    snapcraft_yaml: Dict[str, Any], *, parallel_build_count: int
+) -> None:
     """Expand global variables in the provided dictionary values.
 
     :param snapcraft_yaml: A dictionary containing the contents of the

snapcraft/parts/parts.py

@@ -108,6 +108,11 @@ def target_arch(self) -> str:
         """Return the parts project target architecture."""
         return self._lcm.project_info.target_arch
 
+    @property
+    def target_arch_triplet(self) -> str:
+        """Return the parts project target architecture."""
+        return self._lcm.project_info.arch_triplet
+
     @property
     def project_vars(self) -> Dict[str, str]:
         """Return the value of project variable ``version``."""

snapcraft/projects.py

@@ -388,42 +388,6 @@ def _validate_epoch(cls, epoch):
 
         return epoch
 
-    @pydantic.validator("environment", always=True)
-    @classmethod
-    def _validate_environment(cls, environment):
-        """Validate app environmental variables.
-
-        Three cases for LD_LIBRARY_PATH and PATH variables:
-          - If LD_LIBRARY_PATH or PATH are defined, keep user-defined values.
-          - If LD_LIBRARY_PATH or PATH are not defined, set to default values.
-          - If LD_LIBRARY_PATH or PATH are null, do not use default values.
-        """
-        if environment is None:
-            return {
-                "LD_LIBRARY_PATH": "$SNAP_LIBRARY_PATH:$LD_LIBRARY_PATH",
-                "PATH": "$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH",
-            }
-
-        try:
-            if not environment["LD_LIBRARY_PATH"]:
-                environment.pop("LD_LIBRARY_PATH")
-        except KeyError:
-            environment["LD_LIBRARY_PATH"] = "$SNAP_LIBRARY_PATH:$LD_LIBRARY_PATH"
-
-        try:
-            if not environment["PATH"]:
-                environment.pop("PATH")
-        except KeyError:
-            environment[
-                "PATH"
-            ] = "$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH"
-
-        if len(environment):
-            return environment
-
-        # if the environment only contained a null LD_LIBRARY_PATH and a null PATH, return None
-        return None
-
     @classmethod
     def unmarshal(cls, data: Dict[str, Any]) -> "Project":
         """Create and populate a new ``Project`` object from dictionary data.

snapcraft/utils.py

@@ -15,15 +15,17 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """Utilities for snapcraft."""
-
+import glob
 import multiprocessing
 import os
 import pathlib
 import platform
+import re
 import sys
 from dataclasses import dataclass
 from getpass import getpass
-from typing import Iterable, Optional
+from pathlib import Path
+from typing import Iterable, List, Optional
 
 from craft_cli import emit
 
@@ -276,3 +278,69 @@ def humanize_list(
         humanized += ","
 
     return f"{humanized} {conjunction} {quoted_items[-1]}"
+
+
+def _extract_ld_library_paths(ld_conf_file: str) -> List[str]:
+    # From the ldconfig manpage, paths can be colon-, space-, tab-, newline-,
+    # or comma-separated.
+    path_delimiters = re.compile(r"[:\s,]")
+    comments = re.compile(r"#.*$")
+
+    paths = []
+    with open(ld_conf_file, "r", encoding="utf-8") as ld_config:
+        for line in ld_config:
+            # Remove comments from line
+            line = comments.sub("", line).strip()
+
+            if line:
+                paths.extend(path_delimiters.split(line))
+
+    return paths
+
+
+def _get_configured_ld_library_paths(prime_dir: Path) -> List[str]:
+    """Determine additional library paths needed for the linker loader.
+
+    This is a workaround until full library searching is implemented which
+    works by searching for ld.so.conf in specific hard coded locations
+    within root.
+
+    :param prime_dir str: the directory to search for specific ld.so.conf
+                          entries.
+    :returns: a list of strings of library paths where relevant libraries
+              can be found within prime_dir.
+    """
+    # If more ld.so.conf files need to be supported, add them here.
+    ld_config_globs = {f"{str(prime_dir)}/usr/lib/*/mesa*/ld.so.conf"}
+
+    ld_library_paths = []
+    for this_glob in ld_config_globs:
+        for ld_conf_file in glob.glob(this_glob):
+            ld_library_paths.extend(_extract_ld_library_paths(ld_conf_file))
+
+    return [str(prime_dir / path.lstrip("/")) for path in ld_library_paths]
+
+
+def _get_common_ld_library_paths(prime_dir: Path, arch_triplet: str) -> List[str]:
+    """Return common existing PATH entries for a snap."""
+    paths = [
+        prime_dir / "lib",
+        prime_dir / "usr" / "lib",
+        prime_dir / "lib" / arch_triplet,
+        prime_dir / "usr" / "lib" / arch_triplet,
+    ]
+
+    return [str(p) for p in paths if p.exists()]
+
+
+def get_ld_library_paths(prime_dir: Path, arch_triplet: str) -> str:
+    """Return a usable in-snap LD_LIBRARY_PATH variable."""
+    paths = ["${SNAP_LIBRARY_PATH}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"]
+    # Add the default LD_LIBRARY_PATH
+    paths += _get_common_ld_library_paths(prime_dir, arch_triplet)
+    # Add more specific LD_LIBRARY_PATH from staged packages if necessary
+    paths += _get_configured_ld_library_paths(prime_dir)
+
+    ld_library_path = ":".join(paths)
+
+    return re.sub(str(prime_dir), "$SNAP", ld_library_path)

tests/spread/core22/appstream-desktop/expected_snap.yaml

@@ -23,5 +23,5 @@ apps:
 confinement: strict
 grade: devel
 environment:
-  LD_LIBRARY_PATH: $SNAP_LIBRARY_PATH:$LD_LIBRARY_PATH
+  LD_LIBRARY_PATH: ${SNAP_LIBRARY_PATH}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
   PATH: $SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH

tests/spread/general/unicode-metadata/task.yaml

@@ -12,7 +12,7 @@ prepare: |
   sed -e "s/base: {{BASE}}/base: ${base}/g" expected_snap_tmpl.yaml > expected_snap.yaml
   if [[ "$base" =~ "core22" ]]; then
     # shellcheck disable=SC2016
-    echo -e 'environment:\n  LD_LIBRARY_PATH: $SNAP_LIBRARY_PATH:$LD_LIBRARY_PATH\n  PATH: $SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH' >> expected_snap.yaml
+    echo -e 'environment:\n  LD_LIBRARY_PATH: ${SNAP_LIBRARY_PATH}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}\n  PATH: $SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH' >> expected_snap.yaml
   fi
 
 restore: |