From 541fb3e6c3d1e8c747725916936620551e2e1546 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Tue, 23 Jul 2024 14:49:19 +0200 Subject: [PATCH 1/3] refactor: use same style for ArgumentParser and the args --- dlopen-notes.py | 50 +++++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/dlopen-notes.py b/dlopen-notes.py index 7d1f4ec..4629dcf 100755 --- a/dlopen-notes.py +++ b/dlopen-notes.py @@ -125,26 +125,36 @@ def make_parser(): add_help=False, epilog='If no option is specifed, --raw is the default.', ) - p.add_argument('-r', '--raw', - action='store_true', - help='Show the original JSON extracted from input files') - p.add_argument('-s', '--sonames', - action='store_true', - help='List all sonames and their priorities, one soname per line') - p.add_argument('-f', '--features', - nargs='?', - const=[], - type=lambda s: s.split(','), - action='extend', - metavar='FEATURE1,FEATURE2', - help='Describe features, can be specified multiple times') - p.add_argument('filenames', - nargs='+', - metavar='filename', - help='Library file to extract notes from') - p.add_argument('-h', '--help', - action='help', - help='Show this help message and exit') + p.add_argument( + '-r', '--raw', + action='store_true', + help='Show the original JSON extracted from input files', + ) + p.add_argument( + '-s', '--sonames', + action='store_true', + help='List all sonames and their priorities, one soname per line', + ) + p.add_argument( + '-f', '--features', + nargs='?', + const=[], + type=lambda s: s.split(','), + action='extend', + metavar='FEATURE1,FEATURE2', + help='Describe features, can be specified multiple times', + ) + p.add_argument( + 'filenames', + nargs='+', + metavar='filename', + help='Library file to extract notes from', + ) + p.add_argument( + '-h', '--help', + action='help', + help='Show this help message and exit', + ) return p def parse_args(): From 2aa0091a65ba78271abe3572b06b7f9b6b748526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Tue, 23 Jul 2024 16:59:54 +0200 Subject: [PATCH 2/3] refactor: turn the code to read notes into a class In preparation for future changes where additional state will be necessary. --- dlopen-notes.py | 111 +++++++++++++++++++++++++----------------------- test/test.py | 4 +- 2 files changed, 61 insertions(+), 54 deletions(-) diff --git a/dlopen-notes.py b/dlopen-notes.py index 4629dcf..cf9aaa9 100755 --- a/dlopen-notes.py +++ b/dlopen-notes.py @@ -24,40 +24,44 @@ def wrap(*args, **kwargs): return list(f(*args, **kwargs)) return functools.update_wrapper(wrap, f) -@listify -def read_dlopen_notes(filename): - elffile = ELFFile(open(filename, 'rb')) - - for section in elffile.iter_sections(): - if not isinstance(section, NoteSection) or section.name != '.note.dlopen': - continue - - for note in section.iter_notes(): - if note['n_type'] != 0x407c0c0a or note['n_name'] != 'FDO': +class ELFFileReader: + def __init__(self, filename): + self.filename = filename + self.elffile = ELFFile(open(filename, 'rb')) + + @functools.cache + @listify + def notes(self): + for section in self.elffile.iter_sections(): + if not isinstance(section, NoteSection) or section.name != '.note.dlopen': continue - note_desc = note['n_desc'] - - try: - # On older Python versions (e.g.: Ubuntu 22.04) we get a string, on - # newer versions a bytestring - if not isinstance(note_desc, str): - text = note_desc.decode('utf-8').rstrip('\0') - else: - text = note_desc.rstrip('\0') - except UnicodeDecodeError as e: - raise ValueError(f'{filename}: Invalid UTF-8 in .note.dlopen n_desc') from e - - try: - j = json.loads(text) - except json.JSONDecodeError as e: - raise ValueError(f'{filename}: Invalid JSON in .note.dlopen note_desc') from e - if not isinstance(j, list): - print(f'{filename}: ignoring .note.dlopen n_desc with JSON that is not a list', - file=sys.stderr) - continue + for note in section.iter_notes(): + if note['n_type'] != 0x407c0c0a or note['n_name'] != 'FDO': + continue + note_desc = note['n_desc'] + + try: + # On older Python versions (e.g.: Ubuntu 22.04) we get a string, on + # newer versions a bytestring + if not isinstance(note_desc, str): + text = note_desc.decode('utf-8').rstrip('\0') + else: + text = note_desc.rstrip('\0') + except UnicodeDecodeError as e: + raise ValueError(f'{self.filename}: Invalid UTF-8 in .note.dlopen n_desc') from e + + try: + j = json.loads(text) + except json.JSONDecodeError as e: + raise ValueError(f'{self.filename}: Invalid JSON in .note.dlopen note_desc') from e + + if not isinstance(j, list): + print(f'{self.filename}: ignoring .note.dlopen n_desc with JSON that is not a list', + file=sys.stderr) + continue - yield from j + yield from j def dictify(f): def wrap(*args, **kwargs): @@ -65,9 +69,9 @@ def wrap(*args, **kwargs): return functools.update_wrapper(wrap, f) @dictify -def group_by_soname(notes): - for note in notes: - for element in note: +def group_by_soname(elffiles): + for elffile in elffiles: + for element in elffile.notes(): priority = element.get('priority', 'recommended') for soname in element['soname']: yield soname, priority @@ -80,7 +84,7 @@ class Priority(enum.Enum): def __lt__(self, other): return self.value < other.value -def group_by_feature(filenames, notes): +def group_by_feature(elffiles): features = {} # We expect each note to be in the format: @@ -92,8 +96,8 @@ def group_by_feature(filenames, notes): # "soname": ["..."], # } # ] - for filename, note_group in zip(filenames, notes): - for note in note_group: + for elffiles in elffiles: + for note in elffiles.notes(): prio = Priority[note.get('priority', 'recommened')] feature_name = note['feature'] @@ -108,7 +112,7 @@ def group_by_feature(filenames, notes): else: # Merge if feature['description'] != note.get('description', ''): - print(f"{filename}: feature {note['feature']!r} found with different description, ignoring", + print(f"{note.filename}: feature {note['feature']!r} found with different description, ignoring", file=sys.stderr) for soname in note['soname']: @@ -118,6 +122,15 @@ def group_by_feature(filenames, notes): return features +def filter_features(features, filter): + if filter is None: + return None + ans = { name:feature for name,feature in features.items() + if name in filter or not filter } + if missing := set(filter) - set(ans): + sys.exit('Some features not found:', ', '.join(missing)) + return ans + def make_parser(): p = argparse.ArgumentParser( description=__doc__, @@ -169,27 +182,21 @@ def parse_args(): if __name__ == '__main__': args = parse_args() - notes = [read_dlopen_notes(filename) for filename in args.filenames] + elffiles = [ELFFileReader(filename) for filename in args.filenames] + features = group_by_feature(elffiles) if args.raw: - for filename, note in zip(args.filenames, notes): - print(f'# {filename}') - print_json(json.dumps(note, indent=2)) - - if args.features is not None: - features = group_by_feature(args.filenames, notes) - - toprint = {name:feature for name,feature in features.items() - if name in args.features or not args.features} - if len(toprint) < len(args.features): - sys.exit('Some features were not found') + for elffile in elffiles: + print(f'# {elffile.filename}') + print_json(json.dumps(elffile.notes(), indent=2)) + if features_to_print := filter_features(features, args.features): print('# grouped by feature') - print_json(json.dumps(toprint, + print_json(json.dumps(features_to_print, indent=2, default=lambda prio: prio.name)) if args.sonames: - sonames = group_by_soname(notes) + sonames = group_by_soname(elffiles) for soname in sorted(sonames.keys()): print(f"{soname} {sonames[soname]}") diff --git a/test/test.py b/test/test.py index 462f476..0a4747b 100644 --- a/test/test.py +++ b/test/test.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: CC0-1.0 -from _notes import read_dlopen_notes, group_by_soname +from _notes import ELFFileReader, group_by_soname def test_notes(): expected = { @@ -11,5 +11,5 @@ def test_notes(): 'libtss2-esys.so.0': 'recommended', 'libtss2-mu.so.0': 'recommended', } - notes = [read_dlopen_notes('notes')] + notes = [ELFFileReader('notes')] assert group_by_soname(notes) == expected From c2d3548b7edd863250027fd95e7c909fdd5d8ac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Wed, 24 Jul 2024 09:42:07 +0200 Subject: [PATCH 3/3] dlopen-notes: add mode to generate rpm Provides/Requires This can be used to insert the appropriate deps into the spec file: $ ./dlopen-notes.py .../libsystemd-shared-256.so --rpm-requires zstd,qrencode --rpm-recommends tpm Requires: libqrencode.so.4()(64bit) Requires: libzstd.so.1()(64bit) Recommends: libtss2-esys.so.0()(64bit) Recommends: libtss2-rc.so.0()(64bit) Recommends: libtss2-mu.so.0()(64bit) Most likely, it needs to be used manually. Rpm doesn't really have a notion of dynamically generated deps at the package level :( (There is the fileattr mechanism, https://rpm-software-management.github.io/rpm/manual/dependency_generators.html, but that is per-file. It has the shortcoming that it assumes that the generated deps depend only on the file itself, and not the subpackage that the file lands in, and there is no notion of package configuration. We _could_ use that, but it seems a poor fit. Also, before rpm 4.20, fileattr generators could not be provided by the package being built, they needed to be installed in the filesystem. I don't think we're at the point where it makes sense to make this "global". 4.15 added %generate_buildrequires. Obviously that only works for BuildRequires. Also, the mechanism with generating src.rpms in a loop is rather convoluted and heavyweight.) I'll open an RFE in rpm. For now, the generator can be used manually in systemd. That is still nicer than the previous method of looking at the sources. --- dlopen-notes.py | 44 +++++++++++++++++++++++++++++++++++++++++++- test/test.py | 28 ++++++++++++++++++++++++---- 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/dlopen-notes.py b/dlopen-notes.py index cf9aaa9..3d5b651 100755 --- a/dlopen-notes.py +++ b/dlopen-notes.py @@ -131,6 +131,18 @@ def filter_features(features, filter): sys.exit('Some features not found:', ', '.join(missing)) return ans +@listify +def generate_rpm(elffiles, stanza, filter): + # Produces output like: + # Requires: libqrencode.so.4()(64bit) + # Requires: libzstd.so.1()(64bit) + for elffile in elffiles: + suffix = '()(64bit)' if elffile.elffile.elfclass == 64 else '' + for note in elffile.notes(): + if note['feature'] in filter or not filter: + soname = next(iter(note['soname'])) # we take the first — most recommended — soname + yield f"{stanza}: {soname}{suffix}" + def make_parser(): p = argparse.ArgumentParser( description=__doc__, @@ -157,6 +169,24 @@ def make_parser(): metavar='FEATURE1,FEATURE2', help='Describe features, can be specified multiple times', ) + p.add_argument( + '--rpm-requires', + nargs='?', + const=[], + type=lambda s: s.split(','), + action='extend', + metavar='FEATURE1,FEATURE2', + help='Generate rpm Requires for listed features', + ) + p.add_argument( + '--rpm-recommends', + nargs='?', + const=[], + type=lambda s: s.split(','), + action='extend', + metavar='FEATURE1,FEATURE2', + help='Generate rpm Recommends for listed features', + ) p.add_argument( 'filenames', nargs='+', @@ -173,7 +203,11 @@ def make_parser(): def parse_args(): args = make_parser().parse_args() - if not args.raw and args.features is None and not args.sonames: + if (not args.raw + and not args.sonames + and args.features is None + and args.rpm_requires is None + and args.rpm_recommends is None): # Make --raw the default if no action is specified. args.raw = True @@ -196,6 +230,14 @@ def parse_args(): indent=2, default=lambda prio: prio.name)) + if args.rpm_requires is not None: + lines = generate_rpm(elffiles, 'Requires', args.rpm_requires) + print('\n'.join(lines)) + + if args.rpm_recommends is not None: + lines = generate_rpm(elffiles, 'Recommends', args.rpm_recommends) + print('\n'.join(lines)) + if args.sonames: sonames = group_by_soname(elffiles) for soname in sorted(sonames.keys()): diff --git a/test/test.py b/test/test.py index 0a4747b..20b49e0 100644 --- a/test/test.py +++ b/test/test.py @@ -1,8 +1,8 @@ # SPDX-License-Identifier: CC0-1.0 -from _notes import ELFFileReader, group_by_soname +from _notes import ELFFileReader, group_by_soname, generate_rpm -def test_notes(): +def test_sonames(): expected = { 'libfido2.so.1': 'required', 'liblz4.so.1': 'recommended', @@ -11,5 +11,25 @@ def test_notes(): 'libtss2-esys.so.0': 'recommended', 'libtss2-mu.so.0': 'recommended', } - notes = [ELFFileReader('notes')] - assert group_by_soname(notes) == expected + notes = ELFFileReader('notes') + assert group_by_soname([notes]) == expected + +def test_requires(): + notes = ELFFileReader('notes') + + expected = { + 32: [ + 'Suggests: libpcre2-8.so.0', + 'Suggests: libtss2-mu.so.0', + 'Suggests: libtss2-esys.so.0', + ], + 64: [ + 'Suggests: libpcre2-8.so.0()(64bit)', + 'Suggests: libtss2-mu.so.0()(64bit)', + 'Suggests: libtss2-esys.so.0()(64bit)', + ], + } + + lines = generate_rpm([notes], 'Suggests', ('pcre2', 'tpm')) + expect = expected[notes.elffile.elfclass] + assert lines == expect