Merge branch 'master' into jbrill-msvs-tests

Manually resolve conflicts in CHANGES.txt.

CHANGES.txt

@@ -59,7 +59,7 @@ RELEASE  VERSION/DATE TO BE FILLED IN LATER
       and nodes from the MSVSProject return values.  Resolves #4613.
     - MSVS: Remove the platform specification (i.e., platform = 'win32') from select
       test script environments.  The platform specification appears superfluous.
-      
+
   From Alex James:
     - On Darwin, PermissionErrors are now handled while trying to access
       /etc/paths.d. This may occur if SCons is invoked in a sandboxed
@@ -112,6 +112,12 @@ RELEASE  VERSION/DATE TO BE FILLED IN LATER
     - Change long-standing irritant in Environment tests - instead of using
       a while loop to pull test info from a list of tests and then delete
       the test, structure the test data as a list of tuples and iterate it.
+    - Skip running a few validation tests if the user is root and the test is
+      not designed to work for the root user.
+    - Clarify documentation of Repository() in manpage and user guide.
+    - Add a tag to each CacheDir to let systems ignore backing it up
+      (per https://bford.info/cachedir/). Update the way a CacheDir
+      is created, since it now has to create two files.
 
 
 RELEASE 4.8.1 -  Tue, 03 Sep 2024 17:22:20 -0700

RELEASE.txt

@@ -33,6 +33,7 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY
   keyword arguments to Builder calls (or manually, through the
   undocumented Override method), were modified not to "leak" on item deletion.
   The item will now not be deleted from the base environment.
+
 - Added support for tracking beamer themes in the LaTeX scanner.
 - MSVS: msvs project files are always generated before the corresponding
   msvs solution files. This changes the behavior of clean for a project
@@ -53,20 +54,26 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY
   in the projects argument list rather than manually removing solution file
   names and nodes from the MSVSProject return values.
 
+- Add a tag to each CacheDir to let systems ignore backing it up
+  (per https://bford.info/cachedir/). Update the way a CacheDir
+  is created, since it now has to create two files.
+
 FIXES
 -----
-
 - PackageVariable now does what the documentation always said it does
   if the variable is used on the command line with one of the enabling
   string as the value: the variable's default value is produced (previously
   it always produced True in this case).
+
 - Temporary files created by TempFileMunge() are now cleaned up on
   scons exit, instead of at the time they're used.  Fixes #4595.
+
 - AddOption now correctly adds short (single-character) options.
   Previously an added short option would always report as unknown,
   while long option names for the same option worked. Short options
   that take a value require the user to specify the value immediately
   following the option, with no spaces (e.g. -j5 and not -j 5).
+
 - Fix a problem with compilation_db component initialization - the
   entries for assembler files were not being set up correctly.
 - On Darwin, PermissionErrors are now handled while trying to access
@@ -90,6 +97,9 @@ FIXES
   written to the solution file.
 - Fix nasm test for missing include file, cleanup.
 
+- Skip running a few validation tests if the user is root and the test is
+  not designed to work for the root user.
+
 IMPROVEMENTS
 ------------
 
@@ -98,13 +108,13 @@ IMPROVEMENTS
   under which they would be observed), or major code cleanups
 
 - For consistency with the optparse "add_option" method, AddOption accepts
-  an SConsOption object as a single argument (this failed previouly).
+  an SConsOption object as a single argument (this failed previously).
   Calling AddOption with the full set of arguments (option names and
   attributes) to set up the option is still the recommended approach.
 
 - Add clang and clang++ to the default tool search orders for POSIX
   and Windows platforms. These will be searched for after gcc and g++,
-  respectively. Does not affect expliclity requested tool lists.  Note:
+  respectively. Does not affect explicitly requested tool lists.  Note:
   on Windows, SCons currently only has builtin support for clang, not
   for clang-cl, the version of the frontend that uses cl.exe-compatible
   command line switches.
@@ -122,8 +132,11 @@ DOCUMENTATION
   the contributor credit)
 
 - Some manpage cleanup for the gettext and pdf/ps builders.
+
 - Some clarifications in the User Guide "Environments" chapter.
 
+- Clarify documentation of Repository() in manpage and user guide.
+
 DEVELOPMENT
 -----------
 

SCons/CacheDir.py

@@ -29,13 +29,20 @@
 import os
 import stat
 import sys
+import tempfile
 import uuid
 
 import SCons.Action
 import SCons.Errors
 import SCons.Warnings
 import SCons.Util
 
+CACHE_PREFIX_LEN = 2  # first two characters used as subdirectory name
+CACHE_TAG = (
+    b"Signature: 8a477f597d28d172789f06886806bc55\n"
+    b"# SCons cache directory - see https://bford.info/cachedir/\n"
+)
+
 cache_enabled = True
 cache_debug = False
 cache_force = False
@@ -67,20 +74,20 @@ def CacheRetrieveFunc(target, source, env) -> int:
         fs.chmod(t.get_internal_path(), stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE)
     return 0
 
-def CacheRetrieveString(target, source, env) -> None:
+def CacheRetrieveString(target, source, env) -> str:
     t = target[0]
     fs = t.fs
     cd = env.get_CacheDir()
     cachedir, cachefile = cd.cachepath(t)
     if t.fs.exists(cachefile):
         return "Retrieved `%s' from cache" % t.get_internal_path()
-    return None
+    return ""
 
 CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString)
 
 CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None)
 
-def CachePushFunc(target, source, env):
+def CachePushFunc(target, source, env) -> None:
     if cache_readonly:
         return
 
@@ -134,8 +141,7 @@ def CachePushFunc(target, source, env):
 class CacheDir:
 
     def __init__(self, path) -> None:
-        """
-        Initialize a CacheDir object.
+        """Initialize a CacheDir object.
 
         The cache configuration is stored in the object. It
         is read from the config file in the supplied path if
@@ -147,53 +153,97 @@ def __init__(self, path) -> None:
         self.path = path
         self.current_cache_debug = None
         self.debugFP = None
-        self.config = dict()
-        if path is None:
-            return
-
-        self._readconfig(path)
+        self.config = {}
+        if path is not None:
+            self._readconfig(path)
+
+    def _add_config(self, path: str) -> None:
+        """Create the cache config file in *path*.
+
+        Locking isn't necessary in the normal case - when the cachedir is
+        being created - because it's written to a unique directory first,
+        before the directory is renamed. But it is legal to call CacheDir
+        with an existing directory, which may be missing the config file,
+        and in that case we do need locking. Simpler to always lock.
+        """
+        config_file = os.path.join(path, 'config')
+        # TODO: this breaks the "unserializable config object" test which
+        #   does some crazy stuff, so for now don't use setdefault. It does
+        #   seem like it would be better to preserve an exisiting value.
+        # self.config.setdefault('prefix_len', CACHE_PREFIX_LEN)
+        self.config['prefix_len'] = CACHE_PREFIX_LEN
+        with SCons.Util.FileLock(config_file, timeout=5, writer=True), open(
+            config_file, "x"
+        ) as config:
+            try:
+                json.dump(self.config, config)
+            except Exception:
+                msg = "Failed to write cache configuration for " + path
+                raise SCons.Errors.SConsEnvironmentError(msg)
 
+        # Add the tag file "carelessly" - the contents are not used by SCons
+        # so we don't care about the chance of concurrent writes.
+        try:
+            tagfile = os.path.join(path, "CACHEDIR.TAG")
+            with open(tagfile, 'xb') as cachedir_tag:
+                cachedir_tag.write(CACHE_TAG)
+        except FileExistsError:
+            pass
 
-    def _readconfig(self, path):
-        """
-        Read the cache config.
+    def _mkdir_atomic(self, path: str) -> bool:
+        """Create cache directory at *path*.
 
-        If directory or config file do not exist, create.  Take advantage
-        of Py3 capability in os.makedirs() and in file open(): just try
-        the operation and handle failure appropriately.
+        Uses directory renaming to avoid races.  If we are actually
+        creating the dir, populate it with the metadata files at the
+        same time as that's the safest way. But it's not illegal to point
+        CacheDir at an existing directory that wasn't a cache previously,
+        so we may have to do that elsewhere, too.
 
-        Omit the check for old cache format, assume that's old enough
-        there will be none of those left to worry about.
+        Returns:
+            ``True`` if it we created the dir, ``False`` if already existed,
 
-        :param path: path to the cache directory
+        Raises:
+            SConsEnvironmentError: if we tried and failed to create the cache.
         """
-        config_file = os.path.join(path, 'config')
+        directory = os.path.abspath(path)
+        if os.path.exists(directory):
+            return False
+
         try:
-            # still use a try block even with exist_ok, might have other fails
-            os.makedirs(path, exist_ok=True)
-        except OSError:
+            tempdir = tempfile.TemporaryDirectory(dir=os.path.dirname(directory))
+        except OSError as e:
             msg = "Failed to create cache directory " + path
-            raise SCons.Errors.SConsEnvironmentError(msg)
+            raise SCons.Errors.SConsEnvironmentError(msg) from e
+        self._add_config(tempdir.name)
+        with tempdir:
+            try:
+                os.rename(tempdir.name, directory)
+                return True
+            except Exception as e:
+                # did someone else get there first?
+                if os.path.isdir(directory):
+                    return False
+                msg = "Failed to create cache directory " + path
+                raise SCons.Errors.SConsEnvironmentError(msg) from e
+
+    def _readconfig(self, path: str) -> None:
+        """Read the cache config from *path*.
 
+        If directory or config file do not exist, create and populate.
+        """
+        config_file = os.path.join(path, 'config')
+        created = self._mkdir_atomic(path)
+        if not created and not os.path.isfile(config_file):
+            # Could have been passed an empty directory
+            self._add_config(path)
         try:
-            with SCons.Util.FileLock(config_file, timeout=5, writer=True), open(
-                config_file, "x"
+            with SCons.Util.FileLock(config_file, timeout=5, writer=False), open(
+                config_file
             ) as config:
-                self.config['prefix_len'] = 2
-                try:
-                    json.dump(self.config, config)
-                except Exception:
-                    msg = "Failed to write cache configuration for " + path
-                    raise SCons.Errors.SConsEnvironmentError(msg)
-        except FileExistsError:
-            try:
-                with SCons.Util.FileLock(config_file, timeout=5, writer=False), open(
-                    config_file
-                ) as config:
-                    self.config = json.load(config)
-            except (ValueError, json.decoder.JSONDecodeError):
-                msg = "Failed to read cache configuration for " + path
-                raise SCons.Errors.SConsEnvironmentError(msg)
+                self.config = json.load(config)
+        except (ValueError, json.decoder.JSONDecodeError):
+            msg = "Failed to read cache configuration for " + path
+            raise SCons.Errors.SConsEnvironmentError(msg)
 
     def CacheDebug(self, fmt, target, cachefile) -> None:
         if cache_debug != self.current_cache_debug:
@@ -252,7 +302,7 @@ def is_enabled(self) -> bool:
     def is_readonly(self) -> bool:
         return cache_readonly
 
-    def get_cachedir_csig(self, node):
+    def get_cachedir_csig(self, node) -> str:
         cachedir, cachefile = self.cachepath(node)
         if cachefile and os.path.exists(cachefile):
             return SCons.Util.hash_file_signature(cachefile, SCons.Node.FS.File.hash_chunksize)