1991 better compilation support

doc/developer_guide/psy_data.rst

@@ -1281,7 +1281,7 @@ which prints the filenames of all modules used in ``tl_testkern_mod``:
 
 .. testoutput::
 
-    Module: argument_mod argument_mod.F90
-    Module: constants_mod constants_mod.F90
-    Module: fs_continuity_mod fs_continuity_mod.F90
-    Module: kernel_mod kernel_mod.F90
+    Module: argument_mod argument_mod.f90
+    Module: constants_mod constants_mod.f90
+    Module: fs_continuity_mod fs_continuity_mod.f90
+    Module: kernel_mod kernel_mod.f90

src/psyclone/domain/lfric/lfric_extract_driver_creator.py

@@ -880,7 +880,7 @@ def collect_all_required_modules(file_container):
 
         :returns: a dictionary, with the required module names as key, and \
             as value a set of all modules required by the key module.
-        :rtype: Set[str]
+        :rtype: Dict[str, Set[str]]
 
         '''
         all_mods = set()

src/psyclone/parse/module_info.py

@@ -88,9 +88,8 @@ def __init__(self, name, filename):
         # of all modules used by this module. Type: Set[str]
         self._used_modules = None
 
-        # This is a dictionary, with the modules from the list of all used
-        # modules as key, and it stores the set of all symbols imported from
-        # this module: Dict[str, Set(str)]
+        # This is a dictionary containing the sets of symbols imported from
+        # each module, indexed by the module names: Dict[str, Set(str)].
         self._used_symbols_from_module = None
 
     # ------------------------------------------------------------------------

src/psyclone/parse/module_manager.py

@@ -37,6 +37,7 @@
 which module is contained in which file (including full location). '''
 
 
+from collections import OrderedDict
 import copy
 import os
 
@@ -69,8 +70,13 @@ def __init__(self):
         if ModuleManager._instance is not None:
             raise InternalError("You need to use 'ModuleManager.get()' "
                                 "to get the singleton instance.")
+        # Cached mapping from module name to filename.
         self._mod_2_filename = {}
-        self._search_paths = []
+
+        # The list of all search paths which have not yet all their files
+        # checked. It is stored as an ordered dict to make it easier to avoid
+        # duplicating entries.
+        self._remaining_search_paths = OrderedDict()
 
     # ------------------------------------------------------------------------
     def add_search_path(self, directories, recursive=True):
@@ -91,16 +97,14 @@ def add_search_path(self, directories, recursive=True):
 
         for directory in directories:
             if not os.access(directory, os.R_OK):
-                raise IOError(f"Directory '{directory}' does not exist.")
-            if directory not in self._search_paths:
-                self._search_paths.append(directory)
-            if not recursive:
-                break
-            for root, dirs, _ in os.walk(directory):
-                for current_dir in dirs:
-                    new_dir = os.path.join(root, current_dir)
-                    if new_dir not in self._search_paths:
-                        self._search_paths.append(new_dir)
+                raise IOError(f"Directory '{directory}' does not exist or "
+                              f"cannot be read.")
+            self._remaining_search_paths[directory] = 1
+            if recursive:
+                for root, dirs, _ in os.walk(directory):
+                    for current_dir in dirs:
+                        new_dir = os.path.join(root, current_dir)
+                        self._remaining_search_paths[new_dir] = 1
 
     # ------------------------------------------------------------------------
     def _add_all_files_from_dir(self, directory):
@@ -109,19 +113,24 @@ def _add_all_files_from_dir(self, directory):
         module names are based on the filename using `get_modules_in_file()`.
         By default it is assumed that `a_mod.f90` contains the module `a_mod`.
 
-        :param str directory: the directory to list all files from
-        '''
+        :param str directory: the directory containing Fortran files \
+            to analyse.
 
+        '''
         with os.scandir(directory) as all_entries:
             for entry in all_entries:
                 _, ext = os.path.splitext(entry.name)
                 if (not entry.is_file()) or \
                         ext not in [".F90", ".f90", ".X90", ".x90"]:
                     continue
                 full_path = os.path.join(directory, entry.name)
+                # Obtain the names of all modules defined in this source file.
                 all_modules = self.get_modules_in_file(full_path)
                 for module in all_modules:
-                    if module not in self._mod_2_filename:
+                    # Pre-processed file should always take precedence
+                    # over non-pre-processed files:
+                    if module not in self._mod_2_filename or \
+                            ext in [".f90", ".x90"]:
                         mod_info = ModuleInfo(module, full_path)
                         self._mod_2_filename[module] = mod_info
 
@@ -141,7 +150,7 @@ def get_module_info(self, module_name):
         '''
         mod_lower = module_name.lower()
 
-        # First check if we already know about this file:
+        # First check if we have already cached this file:
         mod_info = self._mod_2_filename.get(mod_lower, None)
         if mod_info:
             return mod_info
@@ -150,9 +159,9 @@ def get_module_info(self, module_name):
         # the directories, we search directories one at a time, and
         # add the list of all files in that directory to our cache
         # _mod_2_filename
-        while self._search_paths:
+        while self._remaining_search_paths:
             # Get the first element from the search path list:
-            directory = self._search_paths.pop(0)
+            directory, _ = self._remaining_search_paths.popitem(last=False)
             self._add_all_files_from_dir(directory)
             mod_info = self._mod_2_filename.get(mod_lower, None)
             if mod_info:
@@ -163,12 +172,11 @@ def get_module_info(self, module_name):
 
     # ------------------------------------------------------------------------
     def get_modules_in_file(self, filename):
-        # pylint: disable=no-self-use
         '''This function returns the list of modules defined in the specified
-        file. The base implementation uses the coding style: the file
-        `a_mod.f90` implements the module `a_mod`. This function can be
-        implemented in a derived class to actually parse the source file if
-        required.
+        file. The base implementation assumes the use of the LFRic coding
+        style: the file `a_mod.f90` implements the module `a_mod`. This
+        function can be implemented in a derived class to actually parse the
+        source file if required.
 
         :param str filename: the file name for which to find the list \
             of modules it contains.
@@ -179,7 +187,7 @@ def get_modules_in_file(self, filename):
         '''
         basename = os.path.basename(filename)
         root, _ = os.path.splitext(basename)
-        if root[-4:].lower() == "_mod":
+        if root.lower().endswith("_mod"):
             return [root]
 
         return []
@@ -191,18 +199,19 @@ def get_all_dependencies_recursively(self, all_mods):
         add all modules used by any module listed in ``all_mods``,
         and any modules used by the just added modules etc. In the end,
         it will return a dictionary that for each module lists which
-        module this module depends on. This dictionary will be complete,
+        modules it depends on. This dictionary will be complete,
         i.e. all modules that are required for the original set of modules
         (and that could be found) will be a key in the dictionary. It will
         include the original set of modules as well.
 
         If a module cannot be found (e.g. its path was not given to the
         ModuleManager, or it might be a system module for which the sources
         are not available, a message will be printed, and this module will
-        be ignored (i.e. not listed in any dependencies)
+        be ignored (i.e. not listed in any dependencies).
+        # TODO 2120: allow a choice to abort or ignore.
 
-        :param Set[str] all_mods: the set of all modules to collect the \
-            modules they use from.
+        :param Set[str] all_mods: the set of all modules for which to collect
+            module dependencies.
 
         :returns: a dictionary with all modules that are required (directly \
             or indirectly) for the modules in ``all_mods``.
@@ -229,6 +238,7 @@ def get_all_dependencies_recursively(self, all_mods):
                 if module not in not_found:
                     # We don't have any information about this module,
                     # ignore it.
+                    # TODO 2120: allow a choice to abort or ignore.
                     print(f"Could not find module '{module}'.")
                     not_found.add(module)
                     # Remove this module as dependencies from any other
@@ -289,6 +299,7 @@ def sort_modules(module_dependencies):
                 if dep not in todo:
                     print(f"Module '{module}' contains a dependency to "
                           f"'{dep}', for which we have no dependencies.")
+                    # TODO 2120: allow a choice to abort or ignore.
                     dependencies.remove(dep)
 
         while todo:
@@ -302,6 +313,7 @@ def sort_modules(module_dependencies):
                 # is a circular dependency
                 print(f"Circular dependency - cannot sort "
                       f"module dependencies: {todo}")
+                # TODO 2120: allow a choice to abort or ignore.
                 # In this case pick a module with the least number of
                 # dependencies, the best we can do in this case - and
                 # it's better to provide all modules (even if they cannot)

src/psyclone/tests/domain/lfric/lfric_extract_driver_creator_test.py

@@ -62,7 +62,8 @@ def init_module_manager():
     '''
 
     infrastructure_path = get_base_path(API)
-    # Define the path to the read-kernel relative to the infrastructure path:
+    # Define the path to the ReadKernelData module (which contains functions
+    # to read extracted data from a file) relative to the infrastructure path:
     read_mod_path = os.path.join(infrastructure_path, "..", "..", "..", "..",
                                  "..", "lib", "extract", "standalone")
     # Enforce loading of the default ModuleManager
@@ -267,6 +268,7 @@ def test_lfric_driver_simple_test():
                  "call extract_psy_data%ReadVariable('cell_post', cell_post)"]:
         assert line in driver
 
+    # Check that all module dependencies have been inlined:
     for mod in ["read_kernel_data_mod", "constants_mod", "kernel_mod",
                 "argument_mod", "log_mod", "fs_continuity_mod",
                 "testkern_mod"]:

src/psyclone/tests/parse/conftest.py

@@ -0,0 +1,74 @@
+# -----------------------------------------------------------------------------
+# BSD 3-Clause License
+#
+# Copyright (c) 2023, Science and Technology Facilities Council.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# -----------------------------------------------------------------------------
+# Author J. Henrichs, Bureau of Meteorology
+
+
+''' Module which provides a fixture to setup a test environment for
+the module manager.
+'''
+
+import os
+
+import pytest
+
+
+# ----------------------------------------------------------------------------
+@pytest.fixture(scope='function')
+def mod_man_test_setup_directories():
+    '''Sets up a directory and file structure for several of the following
+    tests. The following structure is created - note that each Fortran file
+    declares a module of the same name as the basename of the file:
+    tmp/d1/a_mod.f90       : no dependencies
+    tmp/d1/d3/b_mod.F90    : no dependencies
+    tmp/d1/d3/c_mod.x90    : depends on a_mod/b_mod
+    tmp/d2/d_mod.X90       : depends on c_mod
+    tmp/d2/d4/e_mod.F90    : depends on netcdf
+    tmp/d2/d4/f_mod.ignore
+    '''
+
+    os.makedirs("d1/d3")
+    os.makedirs("d2/d4")
+    for (name, path, dependencies) in [("a_mod.f90", "d1", []),
+                                       ("b_mod.F90", "d1/d3", []),
+                                       ("c_mod.x90", "d1/d3", ["a_mod",
+                                                               "b_mod"]),
+                                       ("d_mod.X90", "d2", ["c_mod"]),
+                                       ("e_mod.F90", "d2/d4", ["netcdf"]),
+                                       ("f_mod.ignore", "d2/d4", [])]:
+        # Create a list of "use a_mod, only: a_mod_symbol" statements
+        uses = "\n".join(f"use {dep}, only: {dep}_symbol"
+                         for dep in dependencies)
+        base, _ = os.path.splitext(name)
+        with open(os.path.join(path, name), "w", encoding="utf-8") as f_out:
+            f_out.write(f"module {base}\n{uses}\nend module {base}")