Merge remote-tracking branch 'origin/upstreams/develop' into develop

.readthedocs.yml → .readthedocs.yaml

@@ -1,4 +1,8 @@
 version: 2
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3"
 sphinx:
   configuration: docs/conf.py
 python:

_drgn.pyi

@@ -451,15 +451,15 @@ class Program:
             return an :class:`Object`.
         """
         ...
-    def set_core_dump(self, path: Path) -> None:
+    def set_core_dump(self, path: Union[Path, int]) -> None:
         """
         Set the program to a core dump.
 
         This loads the memory segments from the core dump and determines the
         mapped executable and libraries. It does not load any debugging
         symbols; see :meth:`load_default_debug_info()`.
 
-        :param path: Core dump file path.
+        :param path: Core dump file path or open file descriptor.
         """
         ...
     def set_kernel(self) -> None:
@@ -888,12 +888,12 @@ def filename_matches(haystack: Optional[str], needle: Optional[str]) -> bool:
     """
     ...
 
-def program_from_core_dump(path: Path) -> Program:
+def program_from_core_dump(path: Union[Path, int]) -> Program:
     """
     Create a :class:`Program` from a core dump file. The type of program (e.g.,
     userspace or kernel) is determined automatically.
 
-    :param path: Core dump file path.
+    :param path: Core dump file path or open file descriptor.
     """
     ...
 

contrib/ptdrgn.py

@@ -0,0 +1,210 @@
+#!/usr/bin/python3
+# Copyright (c) 2023, Oracle and/or its affiliates.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# SPDX-License-Identifier: LGPL-2.1-or-later
+"""
+Drgn CLI, but using ptpython rather than the standard code.interact()
+
+NOTE: this is definitely a bit of a hack, using implementation details of Drgn
+*and* ptpython. It may break at any time, but it is also quite useful, and this
+makes it worth sharing.
+
+Requires: "pip install ptpython" which brings in pygments and prompt_toolkit
+"""
+import functools
+import importlib
+import os
+import shutil
+import sys
+from typing import Any, Callable, Dict, Optional, Set
+
+from prompt_toolkit.completion import Completion, Completer
+from prompt_toolkit.formatted_text import PygmentsTokens
+from prompt_toolkit.formatted_text import fragment_list_to_text, to_formatted_text
+from ptpython import embed
+from ptpython.completer import DictionaryCompleter
+from ptpython.repl import run_config
+from pygments.lexers.c_cpp import CLexer
+
+import drgn
+import drgn.cli
+
+
+class DummyForRepr:
+    """
+    A dummy class to pass back to _format_result_output() that pretends to have
+    the given repr()
+    """
+
+    def __init__(self, s):
+        self.s = s
+
+    def __repr__(self):
+        return self.s
+
+
+class DummyForPtRepr:
+    """A similar dummy class for the __pt_repr__() method."""
+
+    def __init__(self, s):
+        self.s = s
+
+    def __pt_repr__(self):
+        return self.s
+
+
+def _maybe_c_format(s):
+    """Given a string, try to use pygments to highlight it it as a C string."""
+    try:
+        tokens = CLexer().get_tokens_unprocessed(s)
+        formatted = PygmentsTokens([(tokentype, value) for index, tokentype, value in tokens])
+        to_format = DummyForPtRepr(formatted)
+    except Exception as e:
+        to_format = DummyForRepr(s)
+    return to_format
+
+
+@functools.lru_cache(maxsize=1)
+def _object_fields() -> Set[str]:
+    return set(dir(drgn.Object))
+
+
+class ReorderDrgnObjectCompleter(Completer):
+    """A completer which puts Object member fields above Object defaults"""
+
+    def __init__(self, c: Completer):
+        self.c = c
+
+    def get_completions(self, document, complete_event):
+        completions = list(self.c.get_completions(document, complete_event))
+        if not completions:
+            return completions
+        text = completions[0].text
+        member_fields = []
+        # If the first completion is "absent_", it is *very likely* that we are
+        # now looking at the completion of on Object.  Move the default Object
+        # attributes to the end of the list so that we get the struct attributes
+        if text == "absent_":
+            fields = _object_fields()
+            for i in reversed(range(len(completions))):
+                text = completions[i].text
+                if text not in fields:
+                    member_fields.append(completions[i])
+                    del completions[i]
+            return list(reversed(member_fields)) + completions
+        return completions
+
+
+def configure(repl) -> None:
+    """
+    Muck around with the internals of PythonRepl so that we will special case the
+    drgn data structures, similar to how drgn messes with sys.displayhook. We can
+    do C syntax highlighting too, which is really nice.
+
+    This also automatically runs the default config file:
+    ~/.config/ptpython/config.py
+    """
+    _format_result_output_orig = repl._format_result_output
+
+    def _format_result_output(result: object):
+        if isinstance(result, drgn.Object):
+            s = result.format_(columns=shutil.get_terminal_size((0, 0)).columns)
+            to_format = _maybe_c_format(s)
+        elif isinstance(result, (drgn.StackFrame, drgn.StackTrace)):
+            to_format = DummyForRepr(str(result))
+        elif isinstance(result, drgn.Type):
+            to_format = _maybe_c_format(str(result))
+        else:
+            to_format = result
+        return _format_result_output_orig(to_format)
+
+    repl._format_result_output = _format_result_output
+    run_config(repl)
+    repl._completer = ReorderDrgnObjectCompleter(repl._completer)
+    repl.completer = ReorderDrgnObjectCompleter(repl.completer)
+
+
+def run_interactive(
+    prog: drgn.Program,
+    banner_func: Optional[Callable[[str], str]] = None,
+    globals_func: Optional[Callable[[Dict[str, Any]], Dict[str, Any]]] = None,
+    quiet: bool = False,
+) -> None:
+    """
+    Run drgn's :ref:`interactive-mode` via ptpython
+
+    :param prog: Pre-configured program to run against. Available as a global
+        named ``prog`` in the CLI.
+    :param banner_func: Optional function to modify the printed banner. Called
+        with the default banner, and must return a string to use as the new
+        banner. The default banner does not include the drgn version, which can
+        be retrieved via :func:`version_header()`.
+    :param globals_func: Optional function to modify globals provided to the
+        session. Called with a dictionary of default globals, and must return a
+        dictionary to use instead.
+    :param quiet: Whether to suppress non-fatal warnings.
+    """
+    init_globals: Dict[str, Any] = {
+        "prog": prog,
+        "drgn": drgn,
+        "__name__": "__main__",
+        "__doc__": None,
+    }
+    drgn_globals = [
+        "NULL",
+        "Object",
+        "cast",
+        "container_of",
+        "execscript",
+        "offsetof",
+        "reinterpret",
+        "sizeof",
+    ]
+    for attr in drgn_globals:
+        init_globals[attr] = getattr(drgn, attr)
+
+    banner = f"""\
+For help, type help(drgn).
+>>> import drgn
+>>> from drgn import {", ".join(drgn_globals)}
+>>> from drgn.helpers.common import *"""
+
+    module = importlib.import_module("drgn.helpers.common")
+    for name in module.__dict__["__all__"]:
+        init_globals[name] = getattr(module, name)
+    if prog.flags & drgn.ProgramFlags.IS_LINUX_KERNEL:
+        banner += "\n>>> from drgn.helpers.linux import *"
+        module = importlib.import_module("drgn.helpers.linux")
+        for name in module.__dict__["__all__"]:
+            init_globals[name] = getattr(module, name)
+
+    if banner_func:
+        banner = banner_func(banner)
+    if globals_func:
+        init_globals = globals_func(init_globals)
+
+    old_path = list(sys.path)
+    # The ptpython history file format is different from a standard readline
+    # history file since it must handle multi-line input, and it includes some
+    # metadata as well. Use a separate history format, even though it would be
+    # nice to share.
+    histfile = os.path.expanduser("~/.drgn_history.ptpython")
+    try:
+        sys.path.insert(0, "")
+
+        print(banner)
+        embed(
+            globals=init_globals,
+            history_filename=histfile,
+            title="drgn",
+            configure=configure,
+        )
+    finally:
+        sys.path[:] = old_path
+
+
+if __name__ == "__main__":
+    # Muck around with the internals of drgn: swap out run_interactive() with our
+    # ptpython version, and then call main as if nothing happened.
+    drgn.cli.run_interactive = run_interactive
+    drgn.cli._main()

docs/release_highlights.rst

@@ -6,5 +6,6 @@ from the full `release notes <https://github.com/osandov/drgn/releases>`_.
 
 .. toctree::
 
+   release_highlights/0.0.24.rst
    release_highlights/0.0.23.rst
    release_highlights/0.0.22.rst

docs/release_highlights/0.0.24.rst

@@ -0,0 +1,115 @@
+0.0.24 (Released September 8th, 2023)
+=====================================
+
+These are some of the highlights of drgn 0.0.24. See the `GitHub release
+<https://github.com/osandov/drgn/releases/tag/v0.0.24>`_ for the full release
+notes, including more improvements and bug fixes.
+
+.. highlight:: pycon
+
+Linked List Length Helper
+-------------------------
+
+This release added :func:`~drgn.helpers.linux.list.list_count_nodes()`, which
+returns the length of a Linux kernel linked list::
+
+    >>> list_count_nodes(prog["workqueues"].address_of_())
+    29
+
+Networking Helpers
+------------------
+
+This release added a couple of Linux kernel networking helpers requested by
+Jakub Kicinski.
+
+:func:`~drgn.helpers.linux.net.netdev_priv()` returns the private data of a
+network device::
+
+    >>> dev = netdev_get_by_name(prog, "wlp0s20f3")
+    >>> netdev_priv(dev)
+    (void *)0xffff9419c9dec9c0
+    >>> netdev_priv(dev, "struct ieee80211_sub_if_data")
+    *(struct ieee80211_sub_if_data *)0xffff9419c9dec9c0 = {
+        ...
+    }
+
+:func:`~drgn.helpers.linux.net.skb_shinfo()` returns the shared info for a
+socket buffer.
+
+C++ Lookups
+-----------
+
+This release added support for a few C++ features.
+
+Simple Type Specifiers
+^^^^^^^^^^^^^^^^^^^^^^
+
+Unlike C, C++ allows referring to ``class``, ``struct``, ``union``, and
+``enum`` types without their respective keywords. For example:
+
+.. code-block:: c++
+
+    class Foo { ... };
+    Foo foo; // Equivalent to class Foo foo;
+
+Previously, drgn always required the keyword, so ``prog.type("class Foo")``
+would succeed but ``prog.type("Foo")`` would fail with a :class:`LookupError`.
+This requirement was surprising to C++ developers, so it was removed. For C++
+programs, ``prog.type("Foo")`` will now find a ``class``, ``struct``,
+``union``, or ``enum`` type named ``Foo`` (for C programs, the keyword is still
+required).
+
+Nested Classes
+^^^^^^^^^^^^^^
+
+Again unlike C, C++ allows ``class``, ``struct``, and ``union`` types to be
+defined inside of other ``class``, ``struct``, and ``union`` types. For example:
+
+.. code-block:: c++
+
+    class Foo {
+    public:
+      class Bar { ... };
+      ...
+    };
+    Foo::Bar bar;
+
+drgn can now find such types with ``prog.type("Foo::Bar")``.
+
+Member Functions
+^^^^^^^^^^^^^^^^
+
+C++ supports member functions (a.k.a. methods). For example:
+
+.. code-block:: c++
+
+    class Foo {
+      int method() { ... }
+    };
+
+drgn can now find member functions with :meth:`drgn.Program.function()`,
+:meth:`drgn.Program.object()`, or :meth:`drgn.Program[]
+<drgn.Program.__getitem__>` (e.g., ``prog.function("Foo::method")`` or
+``prog["Foo::method"]``).
+
+Split DWARF
+-----------
+
+drgn now supports split DWARF object (.dwo) files. This is enabled by the
+``-gsplit-dwarf`` option in GCC and Clang or for the Linux kernel with
+``CONFIG_DEBUG_INFO_SPLIT=y``.
+
+Split DWARF package (.dwp) file support is still in progress.
+
+Performance Improvements
+------------------------
+
+Thierry Treyer found a bug that made us search through much more debugging
+information than necessary when getting a stack trace. Fixing this made stack
+traces almost twice as fast.
+
+The C++ lookup and split DWARF support mentioned above require processing more
+information in drgn's debugging information indexing step, which it does on
+startup and whenever debugging information is manually loaded. This could've
+been a performance regression, but instead, indexing was reworked from the
+ground up in a way that's usually *faster* despite the added features.

docs/support_matrix.rst

@@ -70,7 +70,7 @@ currently fully supported are:
 
 .. Keep this in sync with vmtest/config.py.
 
-- 6.0-6.5
+- 6.0-6.6
 - 5.10-5.19
 - 5.4
 - 4.19

drgn/__init__.py

@@ -142,7 +142,7 @@
 
 
 if sys.version_info >= (3, 8):
-    _open_code = io.open_code
+    _open_code = io.open_code  # novermin
 else:
     from typing import BinaryIO