From 935e8692913857a854698936e3d8d461474b8002 Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Sun, 23 Jul 2023 21:13:58 +0200 Subject: [PATCH 01/10] Add fs mount options check to hpctstlib The filesystem_option_check check if the file systems were properly configured based on their type --- hpctestlib/system/fs/mnt_opts.py | 127 +++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 hpctestlib/system/fs/mnt_opts.py diff --git a/hpctestlib/system/fs/mnt_opts.py b/hpctestlib/system/fs/mnt_opts.py new file mode 100644 index 0000000000..d9c350d85f --- /dev/null +++ b/hpctestlib/system/fs/mnt_opts.py @@ -0,0 +1,127 @@ +# Copyright 2016-2022 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn +import reframe.utility.typecheck as typ + + +def get_mounted_fs(): + result = [] + with open('/proc/mounts','r') as f: + for line in f.readlines(): + mnt_fs = dict() + _, mnt_point, type, options, _, _ = line.split() + result.append((mnt_point, type, options)) + return result + + +@rfm.simple_test +class filesystem_options_check(rfm.RunOnlyRegressionTest): + '''filesystem mount options check + + Check if the mounted filesystems have been configured appropriately + based on their type + ''' + + #: Parameter pack with listing the mounted file systems, mount points + #: type and mount options + #: + #: :type: `Tupe[str, str, str]`. The keys should be + #: 'mount_poinnt' with the filesystem mount point as a str value + #: 'type' with the filesystem type as a str value + #: 'options' with the filesystem options as a List[str] value + #: E.g., '/home', 'xfs', 'rw,nosuid,logbsize=32k' + #: + #: :values: Values are generated using the `/proc/mounts` file + filesystems = parameter(get_mounted_fs(), + fmt=lambda x: x[0], + loggable=True) + + #: Reference mount options + #: + #: :type: `Dict[str, str]`. The key should be the file system type. + #: and the value should be a string with options. + #: E.g., {'xfs: 'nosuid,logbsize=32k'} + #: :default: ``{}`` + fs_ref_opts = variable(typ.Dict, value={}, loggable=True) + + executable = 'stat' + tags = {'system', 'fs'} + + @loggable + @property + def fs_mnt_point(self): + '''The file system mount point + + :type: :class:`str` + ''' + return self.filesystems[0] + + @loggable + @property + def fs_type(self): + '''The file system type + + :type: :class:`str` + ''' + return self.filesystems[1] + + @loggable + @property + def fs_mnt_opts(self): + '''The file system mount options + + :type: :class:`str` + ''' + return self.filesystems[2] + + @run_after('init') + def set_fs_opts(self): + # skip the test if the filesystem is not supported by the test + self.skip_if(self.fs_type not in self.fs_ref_opts, + msg=f'This test does not support filesystem ' + f'type {self.fs_type}') + + self.ref_opts=self.explode_opts_str(self.fs_ref_opts[self.fs_type]) + self.curr_opts=self.explode_opts_str(self.fs_mnt_opts) + + @run_after('init') + def set_executable_opts(self): + self.executable_opts = [self.fs_mnt_point] + + def explode_opts_str(self, opts_str): + result=dict() + for opt in opts_str.split(','): + if opt == '': + continue + opt_splitted = opt.split('=') + keystr = opt_splitted[0] + valstr = opt_splitted[1] if len(opt_splitted) > 1 else '' + result[keystr] = valstr + return result + + @sanity_function + def assert_config(self): + skip_me = sn.extractall('No such file or directory', self.stderr) + self.skip_if(skip_me, msg=f'Skipping test because filesystem ' + f'{self.fs_mnt_point} was not found') + + return sn.all(sn.chain( + sn.map(lambda x: sn.assert_in(x, self.curr_opts, + msg=f'Option {x} not present for ' + f'mount point ' + f'{self.fs_mnt_point}, ' + f'of type {self.fs_type}'), + self.ref_opts), + sn.map(lambda x: sn.assert_eq(x[1], self.curr_opts[x[0]], + msg=f'Value {x[1]} not equal to ' + f'{self.curr_opts[x[0]]} in ' + f'the variable {x[0]} for the ' + f'mount point ' + f'{self.fs_mnt_point}, ' + f'of type {self.fs_type}'), + self.ref_opts.items()), + )) From 13efc6999fa98131af0ac06c9d43dca22899e006 Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Sun, 23 Jul 2023 21:32:45 +0200 Subject: [PATCH 02/10] Fix filesystem parameter comments --- hpctestlib/system/fs/mnt_opts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hpctestlib/system/fs/mnt_opts.py b/hpctestlib/system/fs/mnt_opts.py index d9c350d85f..4b42de4f38 100644 --- a/hpctestlib/system/fs/mnt_opts.py +++ b/hpctestlib/system/fs/mnt_opts.py @@ -29,10 +29,10 @@ class filesystem_options_check(rfm.RunOnlyRegressionTest): #: Parameter pack with listing the mounted file systems, mount points #: type and mount options #: - #: :type: `Tupe[str, str, str]`. The keys should be - #: 'mount_poinnt' with the filesystem mount point as a str value + #: :type: `Tupe[str, str, str]`. The tuple fields represent + #: 'mount_point' with the filesystem mount point as a str value #: 'type' with the filesystem type as a str value - #: 'options' with the filesystem options as a List[str] value + #: 'options' with the filesystem options as a str value #: E.g., '/home', 'xfs', 'rw,nosuid,logbsize=32k' #: #: :values: Values are generated using the `/proc/mounts` file From ed3f0c23060f550a0e9ca4daaf29b604b89275c8 Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Thu, 19 Oct 2023 09:34:42 +0200 Subject: [PATCH 03/10] Address PR remarks and simplify test This newer version of test is not parameterised. This implies that one should see a single test that processes all filesystems. This version also adds to the output the reference options used by the test. --- hpctestlib/system/fs/mnt_opts.py | 160 ++++++++++++++----------------- 1 file changed, 71 insertions(+), 89 deletions(-) diff --git a/hpctestlib/system/fs/mnt_opts.py b/hpctestlib/system/fs/mnt_opts.py index 4b42de4f38..3b0ce0a9f5 100644 --- a/hpctestlib/system/fs/mnt_opts.py +++ b/hpctestlib/system/fs/mnt_opts.py @@ -1,23 +1,17 @@ -# Copyright 2016-2022 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# Copyright 2016-2023 Swiss National Supercomputing Centre (CSCS/ETH Zurich) # ReFrame Project Developers. See the top-level LICENSE file for details. # # SPDX-License-Identifier: BSD-3-Clause +import os + +from string import Template + import reframe as rfm import reframe.utility.sanity as sn import reframe.utility.typecheck as typ -def get_mounted_fs(): - result = [] - with open('/proc/mounts','r') as f: - for line in f.readlines(): - mnt_fs = dict() - _, mnt_point, type, options, _, _ = line.split() - result.append((mnt_point, type, options)) - return result - - @rfm.simple_test class filesystem_options_check(rfm.RunOnlyRegressionTest): '''filesystem mount options check @@ -26,102 +20,90 @@ class filesystem_options_check(rfm.RunOnlyRegressionTest): based on their type ''' - #: Parameter pack with listing the mounted file systems, mount points - #: type and mount options - #: - #: :type: `Tupe[str, str, str]`. The tuple fields represent - #: 'mount_point' with the filesystem mount point as a str value - #: 'type' with the filesystem type as a str value - #: 'options' with the filesystem options as a str value - #: E.g., '/home', 'xfs', 'rw,nosuid,logbsize=32k' - #: - #: :values: Values are generated using the `/proc/mounts` file - filesystems = parameter(get_mounted_fs(), - fmt=lambda x: x[0], - loggable=True) - #: Reference mount options #: #: :type: `Dict[str, str]`. The key should be the file system type. - #: and the value should be a string with options. + #: and the value should be a string with mount options. #: E.g., {'xfs: 'nosuid,logbsize=32k'} #: :default: ``{}`` fs_ref_opts = variable(typ.Dict, value={}, loggable=True) + fs_ref_opts = required - executable = 'stat' - tags = {'system', 'fs'} - - @loggable - @property - def fs_mnt_point(self): - '''The file system mount point - - :type: :class:`str` - ''' - return self.filesystems[0] + #: Fail if the test finds a filesystem type that is not in the + #: reference dictionary + #: + #: :type: `Bool`. + #: :value: ``False`` + fail_unknown_fs = variable(typ.Bool, value=False, loggable=True) - @loggable - @property - def fs_type(self): - '''The file system type + executable = 'cat' + executable_opts = ['/proc/mounts'] + tags = {'system', 'fs'} - :type: :class:`str` + @run_before('sanity') + def process_system_fs_opts(self): + self.filesystem = [] + stdout = os.path.join(self.stagedir, sn.evaluate(self.stdout)) + with open(stdout, 'r') as fp: + for line in fp: + _, mnt_point, type, options, _, _ = line.split() + self.filesystem.append((mnt_point, type, options)) + + @run_before('sanity') + def print_test_variables_to_output(self): ''' - return self.filesystems[1] - - @loggable - @property - def fs_mnt_opts(self): - '''The file system mount options - - :type: :class:`str` + Write the reference mount point options used by the test + at the time of execution. ''' - return self.filesystems[2] - - @run_after('init') - def set_fs_opts(self): - # skip the test if the filesystem is not supported by the test - self.skip_if(self.fs_type not in self.fs_ref_opts, - msg=f'This test does not support filesystem ' - f'type {self.fs_type}') - - self.ref_opts=self.explode_opts_str(self.fs_ref_opts[self.fs_type]) - self.curr_opts=self.explode_opts_str(self.fs_mnt_opts) - - @run_after('init') - def set_executable_opts(self): - self.executable_opts = [self.fs_mnt_point] + stdout = os.path.join(self.stagedir, sn.evaluate(self.stdout)) + with open(stdout, 'a') as fp: + fp.write('\n---- Reference mount options ----\n') + for mnt_type, options in self.fs_ref_opts.items(): + fp.write(f'{mnt_type} {options}\n') def explode_opts_str(self, opts_str): result=dict() for opt in opts_str.split(','): if opt == '': continue - opt_splitted = opt.split('=') - keystr = opt_splitted[0] - valstr = opt_splitted[1] if len(opt_splitted) > 1 else '' + opt_parts = opt.split('=', maxsplit=2) + keystr = opt_parts[0] + valstr = opt_parts[1] if len(opt_parts) > 1 else '' result[keystr] = valstr return result @sanity_function - def assert_config(self): - skip_me = sn.extractall('No such file or directory', self.stderr) - self.skip_if(skip_me, msg=f'Skipping test because filesystem ' - f'{self.fs_mnt_point} was not found') - - return sn.all(sn.chain( - sn.map(lambda x: sn.assert_in(x, self.curr_opts, - msg=f'Option {x} not present for ' - f'mount point ' - f'{self.fs_mnt_point}, ' - f'of type {self.fs_type}'), - self.ref_opts), - sn.map(lambda x: sn.assert_eq(x[1], self.curr_opts[x[0]], - msg=f'Value {x[1]} not equal to ' - f'{self.curr_opts[x[0]]} in ' - f'the variable {x[0]} for the ' - f'mount point ' - f'{self.fs_mnt_point}, ' - f'of type {self.fs_type}'), - self.ref_opts.items()), - )) + def assert_mnt_options(self): + msg = Template('Found filesystem type(s) - "$mnt_type" - that are not ' + 'compatible with this test') + msg1 = Template('The mount point "$mnt_point" of type "$mnt_type" does ' + 'not have the "$opt" option.') + msg2 = Template('The "$variable" variable value of "$value" does not ' + 'match the reference "$ref_value" for the "$mnt_point" ' + 'mount point ("$mnt_type" type)') + + errors = [] + unsupported_types = set() + for mnt_point, mnt_type, options in self.filesystem: + opts = self.explode_opts_str(options) + if mnt_type not in self.fs_ref_opts: + unsupported_types.add(mnt_type) + continue + + ref_opts = self.explode_opts_str(self.fs_ref_opts[mnt_type]) + for ref_opt, ref_value in ref_opts.items(): + if ref_opt not in opts: + errors.append(msg1.substitute(opt=ref_opt, + mnt_point=mnt_point, + mnt_type=mnt_type)) + elif ref_value != opts[ref_opt]: + errors.append(msg2.substitute(value=opts[ref_opt], + ref_value=ref_value, + variable=ref_opt, + mnt_point=mnt_point, + mnt_type=mnt_type)) + + if self.fail_unknown_fs: + errors.append(msg.substitute(mnt_type=', '.join(unsupported_types))) + + return sn.assert_true(errors == [], msg='\n'.join(errors)) From 96c2383a3827438bd6bcd699c220cff8af4d1c90 Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Thu, 19 Oct 2023 09:38:47 +0200 Subject: [PATCH 04/10] Address the coding style guidelines --- hpctestlib/system/fs/mnt_opts.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/hpctestlib/system/fs/mnt_opts.py b/hpctestlib/system/fs/mnt_opts.py index 3b0ce0a9f5..c0b4a25c87 100644 --- a/hpctestlib/system/fs/mnt_opts.py +++ b/hpctestlib/system/fs/mnt_opts.py @@ -76,11 +76,11 @@ def explode_opts_str(self, opts_str): def assert_mnt_options(self): msg = Template('Found filesystem type(s) - "$mnt_type" - that are not ' 'compatible with this test') - msg1 = Template('The mount point "$mnt_point" of type "$mnt_type" does ' - 'not have the "$opt" option.') + msg1 = Template('The mount point "$mnt_point" of type "$mnt_type" does' + ' not have the "$opt" option.') msg2 = Template('The "$variable" variable value of "$value" does not ' - 'match the reference "$ref_value" for the "$mnt_point" ' - 'mount point ("$mnt_type" type)') + 'match the reference "$ref_value" for the "$mnt_point"' + ' mount point ("$mnt_type" type)') errors = [] unsupported_types = set() @@ -104,6 +104,7 @@ def assert_mnt_options(self): mnt_type=mnt_type)) if self.fail_unknown_fs: - errors.append(msg.substitute(mnt_type=', '.join(unsupported_types))) + errors.append(msg.substitute( + mnt_type=', '.join(unsupported_types))) return sn.assert_true(errors == [], msg='\n'.join(errors)) From 7cf3497c4085de2de22a46b105a46ddf358b31c4 Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Thu, 19 Oct 2023 09:49:04 +0200 Subject: [PATCH 05/10] Update the hpctestlib docs --- docs/hpctestlib.rst | 65 ++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/docs/hpctestlib.rst b/docs/hpctestlib.rst index 6d52a425e1..80c762fef2 100644 --- a/docs/hpctestlib.rst +++ b/docs/hpctestlib.rst @@ -6,6 +6,34 @@ ReFrame Test Library (experimental) This is a collection of generic tests that you can either run out-of-the-box by specializing them for your system using the :option:`-S` option or create your site-specific tests by building upon them. +Data Analytics +============== + +.. automodule:: hpctestlib.data_analytics.spark.spark_checks + :members: + :show-inheritance: + + +Interactive Computing +===================== + +.. automodule:: hpctestlib.interactive.jupyter.ipcmagic + :members: + :show-inheritance: + + +Machine Learning +================ + +.. automodule:: hpctestlib.ml.tensorflow.horovod + :members: + :show-inheritance: + +.. automodule:: hpctestlib.ml.pytorch.horovod + :members: + :show-inheritance: + + Microbenchmarks =============== @@ -55,25 +83,6 @@ GPU benchmarks :show-inheritance: -Scientific Applications -======================= - -.. automodule:: hpctestlib.sciapps.amber.nve - :members: - :show-inheritance: - -.. automodule:: hpctestlib.sciapps.gromacs.benchmarks - :members: - :show-inheritance: - -Data Analytics -============== - -.. automodule:: hpctestlib.data_analytics.spark.spark_checks - :members: - :show-inheritance: - - Python ====== @@ -82,21 +91,21 @@ Python :show-inheritance: -Interactive Computing -===================== +Scientific Applications +======================= -.. automodule:: hpctestlib.interactive.jupyter.ipcmagic +.. automodule:: hpctestlib.sciapps.amber.nve :members: :show-inheritance: - -Machine Learning -================ - -.. automodule:: hpctestlib.ml.tensorflow.horovod +.. automodule:: hpctestlib.sciapps.gromacs.benchmarks :members: :show-inheritance: -.. automodule:: hpctestlib.ml.pytorch.horovod + +System +======================= + +.. automodule:: hpctestlib.system.fs.mnt_opts :members: :show-inheritance: From cd8a4a3bacc3750999ab9f7f1f1f5402a8669b4b Mon Sep 17 00:00:00 2001 From: victorusu Date: Mon, 23 Oct 2023 20:18:21 +0200 Subject: [PATCH 06/10] Update hpctestlib/system/fs/mnt_opts.py Co-authored-by: Vasileios Karakasis --- hpctestlib/system/fs/mnt_opts.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hpctestlib/system/fs/mnt_opts.py b/hpctestlib/system/fs/mnt_opts.py index c0b4a25c87..cb85ddba66 100644 --- a/hpctestlib/system/fs/mnt_opts.py +++ b/hpctestlib/system/fs/mnt_opts.py @@ -51,8 +51,7 @@ def process_system_fs_opts(self): @run_before('sanity') def print_test_variables_to_output(self): - ''' - Write the reference mount point options used by the test + '''Write the reference mount point options used by the test at the time of execution. ''' stdout = os.path.join(self.stagedir, sn.evaluate(self.stdout)) From 1dbf7bd2b9723b372acd8c6fee6b79b5650df18c Mon Sep 17 00:00:00 2001 From: victorusu Date: Mon, 23 Oct 2023 20:18:30 +0200 Subject: [PATCH 07/10] Update hpctestlib/system/fs/mnt_opts.py Co-authored-by: Vasileios Karakasis --- hpctestlib/system/fs/mnt_opts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hpctestlib/system/fs/mnt_opts.py b/hpctestlib/system/fs/mnt_opts.py index cb85ddba66..bd33b9f6ac 100644 --- a/hpctestlib/system/fs/mnt_opts.py +++ b/hpctestlib/system/fs/mnt_opts.py @@ -61,7 +61,7 @@ def print_test_variables_to_output(self): fp.write(f'{mnt_type} {options}\n') def explode_opts_str(self, opts_str): - result=dict() + result = {} for opt in opts_str.split(','): if opt == '': continue From 95496cd8aedeb38e857ed0f2b6bef0786d22367f Mon Sep 17 00:00:00 2001 From: victorusu Date: Mon, 23 Oct 2023 20:18:42 +0200 Subject: [PATCH 08/10] Update hpctestlib/system/fs/mnt_opts.py Co-authored-by: Vasileios Karakasis --- hpctestlib/system/fs/mnt_opts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hpctestlib/system/fs/mnt_opts.py b/hpctestlib/system/fs/mnt_opts.py index bd33b9f6ac..40b613a7a2 100644 --- a/hpctestlib/system/fs/mnt_opts.py +++ b/hpctestlib/system/fs/mnt_opts.py @@ -65,6 +65,7 @@ def explode_opts_str(self, opts_str): for opt in opts_str.split(','): if opt == '': continue + opt_parts = opt.split('=', maxsplit=2) keystr = opt_parts[0] valstr = opt_parts[1] if len(opt_parts) > 1 else '' From 889ca3c9563ffddb69861d0c5a84d97263f73539 Mon Sep 17 00:00:00 2001 From: victorusu Date: Mon, 23 Oct 2023 20:18:53 +0200 Subject: [PATCH 09/10] Update hpctestlib/system/fs/mnt_opts.py Co-authored-by: Vasileios Karakasis --- hpctestlib/system/fs/mnt_opts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hpctestlib/system/fs/mnt_opts.py b/hpctestlib/system/fs/mnt_opts.py index 40b613a7a2..5343d382c1 100644 --- a/hpctestlib/system/fs/mnt_opts.py +++ b/hpctestlib/system/fs/mnt_opts.py @@ -70,6 +70,7 @@ def explode_opts_str(self, opts_str): keystr = opt_parts[0] valstr = opt_parts[1] if len(opt_parts) > 1 else '' result[keystr] = valstr + return result @sanity_function From 207384d3b004886f52816daf50b1025945378895 Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Mon, 23 Oct 2023 20:22:39 +0200 Subject: [PATCH 10/10] Address PR remarks --- hpctestlib/system/fs/mnt_opts.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/hpctestlib/system/fs/mnt_opts.py b/hpctestlib/system/fs/mnt_opts.py index 5343d382c1..53a3782166 100644 --- a/hpctestlib/system/fs/mnt_opts.py +++ b/hpctestlib/system/fs/mnt_opts.py @@ -25,9 +25,7 @@ class filesystem_options_check(rfm.RunOnlyRegressionTest): #: :type: `Dict[str, str]`. The key should be the file system type. #: and the value should be a string with mount options. #: E.g., {'xfs: 'nosuid,logbsize=32k'} - #: :default: ``{}`` - fs_ref_opts = variable(typ.Dict, value={}, loggable=True) - fs_ref_opts = required + fs_ref_opts = variable(typ.Dict, loggable=True) #: Fail if the test finds a filesystem type that is not in the #: reference dictionary @@ -70,7 +68,7 @@ def explode_opts_str(self, opts_str): keystr = opt_parts[0] valstr = opt_parts[1] if len(opt_parts) > 1 else '' result[keystr] = valstr - + return result @sanity_function