diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..4b39fba --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,25 @@ +--- +# SPDX-License-Identifier: CC0-1.0 +# vi: ts=2 sw=2 et: + +name: Run tests +on: [pull_request] + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-22.04 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + strategy: + fail-fast: false + steps: + - name: Repository checkout + uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt -y update && sudo apt -y install python3-pyelftools python3-pytest + - name: Run tests + run: make check diff --git a/.gitignore b/.gitignore index 6e4a78e..048f5ec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.sw* /*.src.rpm /*.log +/test/notes !/debian/changelog !/debian/control !/debian/copyright diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..13de305 --- /dev/null +++ b/Makefile @@ -0,0 +1,10 @@ +all: + +install: + install -m 755 -D dlopen-notes.py $(DESTDIR)/usr/bin/dlopen-notes + +check: + make -C test check + +clean: + make -C test clean diff --git a/debian/control b/debian/control index 1ca23da..d468b3f 100644 --- a/debian/control +++ b/debian/control @@ -4,6 +4,9 @@ Priority: optional Rules-Requires-Root: no Maintainer: Luca Boccassi Build-Depends: debhelper-compat (= 13), + python3:native , + python3-pyelftools:native , + python3-pytest:native , Standards-Version: 4.6.1 Homepage: https://systemd.io/ELF_PACKAGE_METADATA/ Vcs-Git: https://github.com/systemd/package-notes.git @@ -17,3 +20,17 @@ Depends: ${misc:Depends}, debhelper, Description: Debian Helper for adding package metadata to ELF binaries Provides a linker specs file to add package metadata to the ELF binaries being built. See: https://systemd.io/ELF_PACKAGE_METADATA/ + +Package: dh-dlopenlibdeps +Architecture: all +Multi-Arch: foreign +Enhances: debhelper +Depends: ${misc:Depends}, + ${perl:Depends}, + debhelper, + python3, + python3-pyelftools, +Provides: dh-sequence-dlopenlibdeps, +Description: Debian Helper for parsing dlopen metadata from ELF binaries + Parses dlopen ELF note and generates dependencies from it that can be used + via ${dlopen:Depends|Recommends|Suggests} diff --git a/debian/dh-dlopenlibdeps.install b/debian/dh-dlopenlibdeps.install new file mode 100644 index 0000000..75c3a60 --- /dev/null +++ b/debian/dh-dlopenlibdeps.install @@ -0,0 +1,3 @@ +debian/dlopenlibdeps.pm usr/share/perl5/Debian/Debhelper/Sequence +debian/dh_dlopenlibdeps usr/bin +usr/bin/dlopen-notes diff --git a/debian/dh-dlopenlibdeps.manpages b/debian/dh-dlopenlibdeps.manpages new file mode 100644 index 0000000..87a4db4 --- /dev/null +++ b/debian/dh-dlopenlibdeps.manpages @@ -0,0 +1 @@ +debian/dh_dlopenlibdeps.1 diff --git a/debian/dh_dlopenlibdeps b/debian/dh_dlopenlibdeps new file mode 100755 index 0000000..067618c --- /dev/null +++ b/debian/dh_dlopenlibdeps @@ -0,0 +1,156 @@ +#!/usr/bin/perl +# SPDX-License-Identifier: CC0-1.0 + +=head1 NAME + +dh_dlopenlibdeps - parse dlopen library dependencies from ELF notes + +=cut + +use strict; +use warnings; +use Debian::Debhelper::Dh_Lib; + +our $VERSION = DH_BUILTIN_VERSION; + +=head1 SYNOPSIS + +B [S>] [B<-X>I] + +=head1 DESCRIPTION + +B is a debhelper program that is responsible for calculating +dlopen library dependencies for packages. + +This program follows the dlopen notes metadata specification as defined at +https://systemd.io/ELF_PACKAGE_METADATA/ + +=head1 OPTIONS + +=over 4 + +=item B<-X>I, B<--exclude=>I + +Exclude files that contain F anywhere in their filename from being +parsed. This will make their dependencies be ignored. +This may be useful in some situations, but use it with caution. This option +may be used more than once to exclude more than one thing. + +=back + +=cut + +init(); + +on_pkgs_in_parallel { + my $is_elf_file = sub { + my ($file) = @_; + my @file_args = Debian::Debhelper::Dh_Lib::_internal_optional_file_args(); + my $ff = qx_cmd('file', @file_args, '--brief', '-e', 'apptype', '-e', 'ascii', + '-e', 'encoding', '-e', 'cdf', '-e', 'compress', '-e', 'tar', '--', $file); + return 1 if $ff =~ m/ELF/; + return 0; + }; + + foreach my $package (@_) { + my $tmp = tmpdir($package); + my $ext = pkgext($package); + my (@filelist); + my %required_packages; + my %recommended_packages; + my %suggested_packages; + + # Generate a list of ELF binaries in the package, ignoring any we were told to exclude. + my $find_options=''; + if (defined($dh{EXCLUDE_FIND}) && $dh{EXCLUDE_FIND} ne '') { + $find_options="! \\( $dh{EXCLUDE_FIND} \\)"; + } + next if not -d $tmp; + foreach my $file (split(/\n/, `find $tmp -type f \\( -perm /111 -or -name "*.so*" -or -name "*.cmxs" -or -name "*.node" \\) $find_options -print`)) { + # Prune directories that contain separated debug symbols. + # CAVEAT: There are files in /usr/lib/debug that are not detached debug symbols, which should be processed. (see #865982) + next if $file =~ m!^\Q$tmp\E/usr/lib/debug/(lib|lib64|usr|bin|sbin|opt|dev|emul|\.build-id)/!; + if ($is_elf_file->($file)) { + push @filelist, $file; + } + } + + if (@filelist) { + my $required_sonames = ''; + my $recommended_sonames = ''; + my $suggested_sonames = ''; + + my $sonames = `dlopen-notes --sonames @filelist`; + foreach my $line (split(/\n/, $sonames)) { + my ($soname, $priority) = split(' ', $line, 2); + + if ($priority eq 'required') { + $required_sonames .= " $soname"; + } elsif ($priority eq 'recommended') { + $recommended_sonames .= " $soname"; + } elsif ($priority eq 'suggested') { + $suggested_sonames .= " $soname"; + } else { + warning("Unknown priority $priority for $soname"); + } + } + + if ($required_sonames) { + my $dpkg_query = `dpkg-query --search -- $required_sonames`; + foreach my $line (split(/\n/, $dpkg_query)) { + chomp $line; + if ($line =~ m/^local diversion |^diversion by/) { + next; + } + if ($line =~ m/^([-a-z0-9+]+):/) { + $required_packages{$1} = 1; + } + } + } + + if ($recommended_sonames) { + my $dpkg_query = `dpkg-query --search -- $recommended_sonames`; + foreach my $line (split(/\n/, $dpkg_query)) { + chomp $line; + if ($line =~ m/^local diversion |^diversion by/) { + next; + } + if ($line =~ m/^([-a-z0-9+]+):/) { + $recommended_packages{$1} = 1; + } + } + } + + if ($suggested_sonames) { + my $dpkg_query = `dpkg-query --search -- $suggested_sonames`; + foreach my $line (split(/\n/, $dpkg_query)) { + chomp $line; + if ($line =~ m/^local diversion |^diversion by/) { + next; + } + if ($line =~ m/^([-a-z0-9+]+):/) { + $suggested_packages{$1} = 1; + } + } + } + } + + # Always write the substvars file, even if it's empty, so that the variables are defined and + # there are no warnings when using them in the control file. + open(SV, ">>debian/${ext}substvars") || error("open debian/${ext}substvars: $!"); + print SV "dlopen:Depends=" . join(", ", sort keys %required_packages) . "\n"; + print SV "dlopen:Recommends=" . join(", ", sort keys %recommended_packages) . "\n"; + print SV "dlopen:Suggests=" . join(", ", sort keys %suggested_packages) . "\n"; + close(SV); + } +}; + +=head1 SEE ALSO + +L, L + +=head1 AUTHOR + +Luca Boccassi + +=cut diff --git a/debian/dlopenlibdeps.pm b/debian/dlopenlibdeps.pm new file mode 100644 index 0000000..2d98645 --- /dev/null +++ b/debian/dlopenlibdeps.pm @@ -0,0 +1,7 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use Debian::Debhelper::Dh_Lib; + +insert_after("dh_shlibdeps", "dh_dlopenlibdeps"); diff --git a/debian/rules b/debian/rules index 2d33f6a..123257a 100755 --- a/debian/rules +++ b/debian/rules @@ -2,3 +2,9 @@ %: dh $@ + +execute_after_dh_auto_build: + pod2man --utf8 $(CURDIR)/debian/dh_dlopenlibdeps > $(CURDIR)/debian/dh_dlopenlibdeps.1 + +execute_after_dh_auto_clean: + rm -f $(CURDIR)/debian/dh_dlopenlibdeps.1 diff --git a/dlopen-notes.py b/dlopen-notes.py new file mode 100755 index 0000000..29ea270 --- /dev/null +++ b/dlopen-notes.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: CC0-1.0 + +"""\ +Read .note.dlopen notes from ELF files and report the contents +""" + +import argparse +import enum +import functools +import json +import sys +from elftools.elf.elffile import ELFFile +from elftools.elf.sections import NoteSection + +try: + import rich + print_json = rich.print_json +except ImportError: + print_json = print + +def listify(f): + def wrap(*args, **kwargs): + return list(f(*args, **kwargs)) + return functools.update_wrapper(wrap, f) + +@listify +def read_dlopen_notes(filename): + elffile = ELFFile(open(filename, 'rb')) + + for section in elffile.iter_sections(): + if not isinstance(section, NoteSection) or section.name != '.note.dlopen': + continue + + for note in section.iter_notes(): + if note['n_type'] != 0x407c0c0a or note['n_name'] != 'FDO': + continue + note_desc = note['n_desc'] + + try: + # On older Python versions (e.g.: Ubuntu 22.04) we get a string, on + # newer versions a bytestring + if not isinstance(note_desc, str): + text = note_desc.decode('utf-8').rstrip('\0') + else: + text = note_desc.rstrip('\0') + except UnicodeDecodeError as e: + raise ValueError(f'{filename}: Invalid UTF-8 in .note.dlopen n_desc') from e + + try: + j = json.loads(text) + except json.JSONDecodeError as e: + raise ValueError(f'{filename}: Invalid JSON in .note.dlopen note_desc') from e + + if not isinstance(j, list): + print(f'{filename}: ignoring .note.dlopen n_desc with JSON that is not a list', + file=sys.stderr) + continue + + yield from j + +def dictify(f): + def wrap(*args, **kwargs): + return dict(f(*args, **kwargs)) + return functools.update_wrapper(wrap, f) + +@dictify +def group_by_soname(notes): + for note in notes: + for element in note: + priority = element.get('priority', 'recommended') + for soname in element['soname']: + yield soname, priority + +class Priority(enum.Enum): + suggested = 1 + recommended = 2 + required = 3 + + def __lt__(self, other): + return self.value < other.value + +def group_by_feature(filenames, notes): + features = {} + + # We expect each note to be in the format: + # [ + # { + # "feature": "...", + # "description": "...", + # "priority": "required"|"recommended"|"suggested", + # "soname": ["..."], + # } + # ] + for filename, note_group in zip(filenames, notes): + for note in note_group: + prio = Priority[note.get('priority', 'recommened')] + feature_name = note['feature'] + + try: + feature = features[feature_name] + except KeyError: + # Create new + feature = features[feature_name] = { + 'description': note.get('description', ''), + 'sonames': { soname:prio for soname in note['soname'] }, + } + else: + # Merge + if feature['description'] != note.get('description', ''): + print(f"{filename}: feature {note['feature']!r} found with different description, ignoring", + file=sys.stderr) + + for soname in note['soname']: + highest = max(feature['sonames'].get(soname, Priority.suggested), + prio) + feature['sonames'][soname] = highest + + return features + +def parse_args(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--raw', + action='store_true', + help='show the original JSON extracted from input files') + p.add_argument('--sonames', + action='store_true', + help='list all sonames and their priorities, one soname per line') + p.add_argument('--features', + nargs='?', + const=[], + type=lambda s: s.split(','), + action='extend', + metavar='FEATURE1,FEATURE2', + help='describe features, can be specified multiple times') + p.add_argument('filenames', nargs='+', metavar='filename') + return p.parse_args() + +if __name__ == '__main__': + args = parse_args() + + notes = [read_dlopen_notes(filename) for filename in args.filenames] + + if args.raw: + for filename, note in zip(args.filenames, notes): + print(f'# {filename}') + print_json(json.dumps(note, indent=2)) + + if args.features is not None: + features = group_by_feature(args.filenames, notes) + + toprint = {name:feature for name,feature in features.items() + if name in args.features or not args.features} + if len(toprint) < len(args.features): + sys.exit('Some features were not found') + + print('# grouped by feature') + print_json(json.dumps(toprint, + indent=2, + default=lambda prio: prio.name)) + + if args.sonames: + sonames = group_by_soname(notes) + for soname in sorted(sonames.keys()): + print(f"{soname} {sonames[soname]}") diff --git a/test/Makefile b/test/Makefile new file mode 100644 index 0000000..b91a4ac --- /dev/null +++ b/test/Makefile @@ -0,0 +1,9 @@ +notes: notes.c + $(CC) -o $@ $+ $(CFLAGS) $(LDFLAGS) $(LDLIBS) + +check: notes + python3 -m pytest test.py + +clean: + rm -f notes + rm -rf __pycache__/ diff --git a/test/_notes.py b/test/_notes.py new file mode 120000 index 0000000..88fcc3d --- /dev/null +++ b/test/_notes.py @@ -0,0 +1 @@ +../dlopen-notes.py \ No newline at end of file diff --git a/test/notes.c b/test/notes.c new file mode 100644 index 0000000..5fa9dc1 --- /dev/null +++ b/test/notes.c @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: CC0-1.0 */ + +#include + +#define XCONCATENATE(x, y) x ## y +#define CONCATENATE(x, y) XCONCATENATE(x, y) +#define UNIQ_T(x, uniq) CONCATENATE(__unique_prefix_, CONCATENATE(x, uniq)) +#define UNIQ __COUNTER__ + +#define ELF_NOTE_DLOPEN_VENDOR "FDO" +#define ELF_NOTE_DLOPEN_TYPE 0x407c0c0a + +#define _ELF_NOTE_DLOPEN(module, variable_name) \ + __attribute__((used, section(".note.dlopen"))) _Alignas(sizeof(uint32_t)) static const struct { \ + struct { \ + uint32_t n_namesz, n_descsz, n_type; \ + } nhdr; \ + char name[sizeof(ELF_NOTE_DLOPEN_VENDOR)]; \ + _Alignas(sizeof(uint32_t)) char dlopen_module[sizeof(module)]; \ + } variable_name = { \ + .nhdr = { \ + .n_namesz = sizeof(ELF_NOTE_DLOPEN_VENDOR), \ + .n_descsz = sizeof(module), \ + .n_type = ELF_NOTE_DLOPEN_TYPE, \ + }, \ + .name = ELF_NOTE_DLOPEN_VENDOR, \ + .dlopen_module = module, \ + } + +#define _SONAME_ARRAY1(a) "[\""a"\"]" +#define _SONAME_ARRAY2(a, b) "[\""a"\",\""b"\"]" +#define _SONAME_ARRAY3(a, b, c) "[\""a"\",\""b"\",\""c"\"]" +#define _SONAME_ARRAY4(a, b, c, d) "[\""a"\",\""b"\",\""c"\"",\""d"\"]" +#define _SONAME_ARRAY5(a, b, c, d, e) "[\""a"\",\""b"\",\""c"\"",\""d"\",\""e"\"]" +#define _SONAME_ARRAY_GET(_1,_2,_3,_4,_5,NAME,...) NAME +#define _SONAME_ARRAY(...) _SONAME_ARRAY_GET(__VA_ARGS__, _SONAME_ARRAY5, _SONAME_ARRAY4, _SONAME_ARRAY3, _SONAME_ARRAY2, _SONAME_ARRAY1)(__VA_ARGS__) + +#define ELF_NOTE_DLOPEN(feature, description, priority, ...) \ + _ELF_NOTE_DLOPEN("[{\"feature\":\"" feature "\",\"description\":\"" description "\",\"priority\":\"" priority "\",\"soname\":" _SONAME_ARRAY(__VA_ARGS__) "}]", UNIQ_T(s, UNIQ)) + +#define ELF_NOTE_DLOPEN_DUAL(feature0, priority0, module0, feature1, priority1, module1) \ + _ELF_NOTE_DLOPEN("[{\"feature\":\"" feature0 "\",\"priority\":\"" priority0 "\",\"soname\":[\"" module0 "\"]}, {\"feature\":\"" feature1 "\",\"priority\":\"" priority1 "\",\"soname\":[\"" module1 "\"]}]", UNIQ_T(s, UNIQ)) + +int main(int argc, char **argv) { + ELF_NOTE_DLOPEN("fido2", "Support fido2 for encryption and authentication.", "required", "libfido2.so.1"); + ELF_NOTE_DLOPEN("pcre2", "Support pcre2 for regex", "suggested", "libpcre2-8.so.0","libpcre2-8.so.1"); + ELF_NOTE_DLOPEN("lz4", "Support lz4 decompression in journal and coredump files", "recommended", "liblz4.so.1"); + ELF_NOTE_DLOPEN_DUAL("tpm", "recommended", "libtss2-mu.so.0", "tpm", "recommended", "libtss2-esys.so.0"); + return 0; +} diff --git a/test/test.py b/test/test.py new file mode 100644 index 0000000..462f476 --- /dev/null +++ b/test/test.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: CC0-1.0 + +from _notes import read_dlopen_notes, group_by_soname + +def test_notes(): + expected = { + 'libfido2.so.1': 'required', + 'liblz4.so.1': 'recommended', + 'libpcre2-8.so.0': 'suggested', + 'libpcre2-8.so.1': 'suggested', + 'libtss2-esys.so.0': 'recommended', + 'libtss2-mu.so.0': 'recommended', + } + notes = [read_dlopen_notes('notes')] + assert group_by_soname(notes) == expected