Merge pull request #8 from delphix/master

Merge branch 'master' into '6.0/stage'

.github/workflows/sync-with-master.yml

@@ -0,0 +1,21 @@
+on:
+  push:
+    branches:
+      - master
+  schedule:
+    - cron:  '0 0 * * *'
+
+jobs:
+  sync:
+    strategy:
+      matrix:
+        branch:
+          - 6.0/stage
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: delphix/actions/sync-with-master@master
+        with:
+          branch-to-sync: ${{ matrix.branch }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/sync-with-upstream.yml

@@ -0,0 +1,18 @@
+on:
+  schedule:
+    - cron:  '0 * * * *'
+
+jobs:
+  sync:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          token: ${{ secrets.DEVOPS_AUTOMATION_TOKEN }}
+      - uses: delphix/actions/sync-with-upstream@master
+        with:
+          upstream-repository: https://github.com/sdimitro/savedump.git
+          upstream-branch: master
+          downstream-branch: master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

README.md

@@ -1,8 +1,12 @@
 # savedump
 
-TL;DR; A Python hack I put together that does its best to archive
-crash dumps and core dumps together with their required binaries
-and debug info in Linux.
+![](https://github.com/delphix/savedump/workflows/.github/workflows/main.yml/badge.svg)
+
+TL;DR; A Python script that creates a best-effort self-contained
+archive of a kernel crash dump or userland core dump. The archive
+contains the memory dump coupled together with any required
+binaries and debug information that it could find at the time it
+was invoked.
 
 ### Motivation
 
@@ -35,6 +39,26 @@ dumps. What it does lack though is proper tooling to capture a
 self-contained dump from one system to analyze it in another.
 This is what this utility is attempting to help with.
 
+### Installation
+
+Ensure you have the following dependencies:
+* Python 3.6 or newer
+* [libkdumpfile](https://github.com/ptesarik/libkdumpfile)
+* [drgn](https://github.com/osandov/drgn/)
+* [gdb](https://www.gnu.org/software/gdb/)
+
+Note that `libkdumpfile` and `drgn` are only needed for kernel
+crash dumps. If you only need `savedump` for userland core dumps
+then you only need `python3`. `gdb` is not a hard dependency
+either but it is recommeneded for accurate archival of shared
+objects in userland core dumps.
+
+Once all dependencies are installed clone this repo and
+run the following command from the root of the repo:
+```
+sudo python3 setup.py install
+```
+
 ### How do I use it?
 
 To capture a crash dump or a core dump:
@@ -99,9 +123,7 @@ As mentioned in the TL;DR; the utility is far from perfect but I do
 hope to add more functionality to it as cases arise.
 
 * [verify BuildID and/or SRCVERSION between dumps and binaries](https://github.com/sdimitro/savedump/issues/6)
-* [only package the modules needed in crash dumps](https://github.com/sdimitro/savedump/issues/7)
 * [make the gdb dependency optional](https://github.com/sdimitro/savedump/issues/9)
-* [make the libkdumpfile dependency optional](https://github.com/sdimitro/savedump/issues/8)
 * [support custom paths for binaries and debug info](https://github.com/sdimitro/savedump/issues/5)
 * [generate run-sdb.sh](https://github.com/sdimitro/savedump/issues/10)
 * [support for plain vmcores](https://github.com/sdimitro/savedump/issues/3)

debian/control

@@ -3,13 +3,7 @@ Section: misc
 Priority: optional
 Maintainer: Delphix Engineering <[email protected]>
 Standards-Version: 4.1.2
-Build-Depends: debhelper (>= 9),
-               dh-python,
-               python3,
-               python3-distutils,
-               python3.6-dev,
-               zlib1g-dev
-Suggests: libkdumpfile, drgn
+Build-Depends: debhelper (>= 9), dh-python, python3
 
 Package: savedump
 Architecture: any

savedump/savedump.py

@@ -19,8 +19,6 @@
 """
 
 import argparse
-import distutils
-from distutils import dir_util  # pylint: disable=unused-import
 from enum import Enum
 import os
 import pathlib
@@ -133,16 +131,83 @@ def get_dump_type(path: str) -> Optional[DumpType]:
 """
 
 
+def get_module_paths(osrelease: str, path: str) -> List[str]:
+    """
+    Use drgn on the crash dump specified by `path` and return list
+    of paths from the `osrelease` kernel modules relevant to the
+    crash dump.
+    """
+    #
+    # Similarly to libkdumpfile we import these libraries locally
+    # here so people who don't have drgn can still use savedump
+    # for userland core dumps.
+    #
+    import drgn  # pylint: disable=import-outside-toplevel
+    from drgn.helpers.linux.list import list_for_each_entry  # pylint: disable=import-outside-toplevel
+
+    prog = drgn.program_from_core_dump(path)
+
+    #
+    # First go through all modules in the dump and create a map
+    # of [key: module name] -> (value: module srcversion).
+    #
+    # Note:
+    # It would be prefereable to be able to use the binary's
+    # .build-id to do the matching instead of srcversion.
+    # Unfortunately there doesn't seem to be a straightforward
+    # way to get the build-id section of the ELF files recorded
+    # in the dump. Hopefully that changes in the future.
+    #
+    mod_name_srcvers = {}
+    for mod in list_for_each_entry('struct module',
+                                   prog['modules'].address_of_(), 'list'):
+        mod_name_srcvers[str(mod.name.string_(),
+                             encoding='utf-8')] = str(mod.srcversion.string_(),
+                                                      encoding='utf-8')
+
+    #
+    # Go through all modules in /usr/lib/debug/lib/modules/<osrelease>
+    # and gather the file paths of the ones that are part of our
+    # module name-to-srcversion map.
+    #
+    system_modules = pathlib.Path(
+        f"/usr/lib/debug/lib/modules/{osrelease}/").rglob('*.ko')
+    mod_paths = []
+    for modpath in system_modules:
+        modname = os.path.basename(modpath)[:-3]
+        if not mod_name_srcvers.get(modname):
+            continue
+
+        success, output = shell_cmd(
+            ['modinfo', '--field=srcversion',
+             str(modpath)])
+        if not success:
+            sys.exit(output)
+        output = output.strip()
+
+        if output != mod_name_srcvers[modname]:
+            continue
+
+        mod_paths.append(str(modpath))
+        del mod_name_srcvers[modname]
+
+    print(f"found {len(mod_paths)} relevant modules with their debug info...")
+    print("warning: could not find the debug info of the following modules:")
+    print(f"  {', '.join(mod_name_srcvers.keys())}")
+    return mod_paths
+
+
 def archive_kernel_dump(path: str) -> None:
     """
     Packages the dump together with its vmlinux and modules in a
     gzipped archive in the working directory.
     """
+    # pylint: disable=too-many-locals
     #
-    # We import libkdumpfile specifically here and not
-    # in the top-level to allow users that don't have
+    # We import drgn and libkdumpfile specifically here and
+    # not in the top-level to allow users that don't have
     # it installed to still be able to use savedump for
-    # core files.
+    # userland core files.
     #
     import kdumpfile  # pylint: disable=import-outside-toplevel
 
@@ -156,18 +221,17 @@ def archive_kernel_dump(path: str) -> None:
         sys.exit(f"error: cannot find vmlinux at: {vmlinux_path}")
     print(f"vmlinux found: {vmlinux_path}")
 
-    extra_mod_path = f"/usr/lib/debug/lib/modules/{osrelease}/extra"
-    if not os.path.exists(extra_mod_path):
-        sys.exit(f"error: cannot find extra mod path: {extra_mod_path}")
-    print(f"using module path: {extra_mod_path}")
+    mod_paths = get_module_paths(osrelease, path)
 
     archive_dir = f"{nodename}.archive-{dumpname}"
     pathlib.Path(archive_dir).mkdir(parents=True, exist_ok=True)
     shutil.copy(path, archive_dir)
     shutil.copy(vmlinux_path, archive_dir)
 
-    archive_extra_mod_path = f"{archive_dir}{extra_mod_path}"
-    distutils.dir_util.copy_tree(extra_mod_path, archive_extra_mod_path)
+    for mod_path in mod_paths:
+        archive_mod_path = f"{archive_dir}{mod_path}"
+        os.makedirs(os.path.dirname(archive_mod_path), exist_ok=True)
+        shutil.copy(mod_path, archive_mod_path)
 
     #
     # Generate run-sdb.sh.