Python 3.7 support

.travis.yml

@@ -12,6 +12,7 @@ env:
     - PYVERSION=python3.4
     - PYVERSION=python3.5
     - PYVERSION=python3.6
+    - PYVERSION=python3.7
 
 addons:
   apt:

NEWS.org

@@ -1,4 +1,7 @@
 * NEWS (user visible changes)
+** 1.6.7 (2018-06-25)
+   - Improved support for prelinked binaries, especially on ARM32 and Yocto
+     systems. (Andrew Jeffery)
 
 **  1.6.6 (2018-05-02)
 

configure.ac

@@ -1,5 +1,5 @@
 AC_PREREQ([2.68])
-AC_INIT([pyflame], [1.6.6], [[email protected]])
+AC_INIT([pyflame], [1.6.7], [[email protected]])
 AC_CONFIG_AUX_DIR([build-aux])
 AC_CONFIG_HEADERS([src/config.h])
 AC_CONFIG_MACRO_DIR([m4])
@@ -111,7 +111,12 @@ PKG_CHECK_MODULES([PY36], [python-3.6], [enable_py36="yes"], [AC_MSG_WARN([Build
 AM_CONDITIONAL([ENABLE_PY36], [test x"$enable_py36" = xyes])
 AM_COND_IF([ENABLE_PY36], [AC_DEFINE([ENABLE_PY36], [1], [Python 3.6 will be enabled])])
 
-AS_IF([test x"$enable_py26" = xyes -o x"$enable_py34" = xyes -o x"$enable_py36" = xyes],
+enable_py37=no
+PKG_CHECK_MODULES([PY37], [python-3.7], [enable_py37="yes"], [AC_MSG_WARN([Building without Python 3.7 support])])
+AM_CONDITIONAL([ENABLE_PY37], [test x"$enable_py37" = xyes])
+AM_COND_IF([ENABLE_PY37], [AC_DEFINE([ENABLE_PY37], [1], [Python 3.7 will be enabled])])
+
+AS_IF([test x"$enable_py26" = xyes -o x"$enable_py34" = xyes -o x"$enable_py36" = xyes -o x"$enable_py37" = xyes],
       [AC_MSG_NOTICE([Found at least one copy of Python.h])],
       [AC_MSG_ERROR([Failed to find a supported Python.h])]
 )
@@ -127,7 +132,8 @@ echo
 echo "  with threads        = $enable_threads"
 echo "  with Python 2.6/7   = $enable_py26"
 echo "  with Python 3.4/5   = $enable_py34"
-echo "  with Python 3.6+    = $enable_py36"
+echo "  with Python 3.6     = $enable_py36"
+echo "  with Python 3.7+    = $enable_py37"
 echo
 echo "  CXX                 = $CXX"
 echo "  CXXFLAGS            = $CXXFLAGS"

docs/faq.rst

@@ -7,7 +7,7 @@ What Python Versions Are Supported?
 Python 2 is tested with Python 2.6 and 2.7. Earlier versions of Python 2 are
 likely to work as well, but have not been tested.
 
-Python 3 is tested with Python 3.4, 3.5, and 3.6. Python 3.6 introduces a new
+Python 3 is tested with Python 3.4, 3.5, 3.6 and 3.7 Python 3.6 introduces a new
 ABI for the ``PyCodeObject`` type, so Pyflame only supports the Python 3
 versions that header files were available for when Pyflame was compiled.
 

docs/man.md

@@ -87,7 +87,7 @@ The following options are less commonly used.
     cases when profiling embedded Python builds (e.g. uWSGI), and only if
     pyflame doesn't automatically detect the correct ABI. *VERSION* should be a
     two digit integer consisting of the Python major and minor version, e.g. 27
-    for Python 2.7 or 36 for Python 3.6.
+    for Python 2.7, 36 for Python 3.6 or 37 for Python 3.7.
 
 **--flamechart**
 :   Print the timestamp for each stack. This is useful for generating "flame

docs/pyflame.man

@@ -119,7 +119,7 @@ uWSGI), and only if pyflame doesn\[aq]t automatically detect the correct
 ABI.
 \f[I]VERSION\f[] should be a two digit integer consisting of the Python
 major and minor version, e.g.
-27 for Python 2.7 or 36 for Python 3.6.
+27 for Python 2.7, 36 for Python 3.6 or 37 for Python 3.7.
 .RS
 .RE
 .TP

src/Makefile.am

@@ -2,11 +2,11 @@
 #
 # The way this code is structured is the code that know about Python interpreter
 # internals is in frob.cc. This file isn't compiled directly, instead there are
-# files frob{26,34,36}.cc that define preprocessor macros to compile frob.cc for
+# files frob{26,34,36,37}.cc that define preprocessor macros to compile frob.cc for
 # different Python ABIs.
 #
 # The libtool magic here makes it so that frob26.cc is compiled with python2
-# flags, and frob3{4,6}.cc are compiled with python3.4/3.6 flags.
+# flags, and frob3{4,6,7}.cc are compiled with python3.4/3.6/3.7 flags.
 
 bin_PROGRAMS = pyflame
 pyflame_SOURCES = aslr.cc frame.cc thread.cc namespace.cc posix.cc prober.cc ptrace.cc pyflame.cc pyfrob.cc symbol.cc
@@ -34,3 +34,11 @@ libfrob36_la_CXXFLAGS = $(PY36_CFLAGS)
 noinst_LTLIBRARIES += libfrob36.la
 pyflame_LDADD += libfrob36.la
 endif
+
+if ENABLE_PY37
+libfrob37_la_SOURCES = frob37.cc offset37.c
+libfrob37_la_CFLAGS = $(PY37_CFLAGS)
+libfrob37_la_CXXFLAGS = $(PY37_CFLAGS)
+noinst_LTLIBRARIES += libfrob37.la
+pyflame_LDADD += libfrob37.la
+endif

src/frob.cc

@@ -17,6 +17,7 @@
 // information.
 
 #include <Python.h>
+
 #include <frameobject.h>
 
 #if PYFLAME_PY_VERSION >= 34
@@ -89,6 +90,22 @@ unsigned long ByteData(unsigned long addr) {
   return addr + offsetof(PyBytesObject, ob_sval);
 }
 
+#elif PYFLAME_PY_VERSION == 37
+namespace py37 {
+std::string StringDataPython3(pid_t pid, unsigned long addr);
+
+unsigned long StringSize(unsigned long addr) {
+  return addr + offsetof(PyVarObject, ob_size);
+}
+
+std::string StringData(pid_t pid, unsigned long addr) {
+  return StringDataPython3(pid, addr);
+}
+
+unsigned long ByteData(unsigned long addr) {
+  return addr + offsetof(PyBytesObject, ob_sval);
+}
+
 #else
 static_assert(false, "uh oh, bad PYFLAME_PY_VERSION");
 #endif

src/frob37.cc

@@ -0,0 +1,18 @@
+// Copyright 2016 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// ABI for Python 3.7
+#define PYFLAME_PY_VERSION 37
+
+#include "./frob.cc"

src/offset37.c

@@ -0,0 +1,26 @@
+// Copyright 2016 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#define Py_BUILD_CORE
+#include <stddef.h>
+#include <stdatomic.h>
+#include <Python.h>
+#include <internal/pystate.h>
+
+// We needd to include pystate.h which needs to include pyatomic.h
+// which needs to include stdatomic.h which currently fails to work in C++
+// with GCC. We do this in C instead.
+//
+size_t PYFLAME_PYTHON37_OFFSET = offsetof(_PyRuntimeState, gilstate) +
+                                 offsetof(struct _gilstate_runtime_state, tstate_current);

src/prober.cc

@@ -64,7 +64,7 @@ static const char usage_str[] =
      "  -x, --exclude-idle       Exclude idle time from statistics\n"
      "\n"
      "Advanced Options:\n"
-     "  --abi                    Force a particular Python ABI (26, 34, 36)\n"
+     "  --abi                    Force a particular Python ABI (26, 34, 36, 37)\n"
      "  --flamechart             Include timestamps for generating Chrome "
      "\"flamecharts\"\n");
 
@@ -79,6 +79,9 @@ static const int build_abis[] = {
 #ifdef ENABLE_PY36
     36,
 #endif
+#ifdef ENABLE_PY37
+    37,
+#endif
 };
 
 static_assert(sizeof(build_abis) > 0, "No Python ABIs detected!");
@@ -221,6 +224,9 @@ int Prober::ParseOpts(int argc, char **argv) {
           case 36:
             abi_ = PyABI::Py36;
             break;
+          case 37:
+            abi_ = PyABI::Py37;
+            break;
           default:
             std::cerr << "Unknown or unsupported ABI version: " << abi_version
                       << "\n";

src/pyfrob.cc

@@ -14,6 +14,7 @@
 
 #include "./pyfrob.h"
 
+#include <cstddef>
 #include <fstream>
 #include <sstream>
 
@@ -25,6 +26,8 @@
 #include "./ptrace.h"
 #include "./symbol.h"
 
+extern size_t PYFLAME_PYTHON37_OFFSET;
+
 #define FROB_FUNCS                                            \
   std::vector<Thread> GetThreads(pid_t pid, PyAddresses addr, \
                                  bool enable_threads);
@@ -126,11 +129,20 @@ FROB_FUNCS
 }
 #endif
 
+#ifdef ENABLE_PY37
+namespace py37 {
+FROB_FUNCS
+extern size_t tstate_offset;
+}
+#endif
+
 // Fill the addrs_ member
 int PyFrob::set_addrs_(PyABI *abi) {
   Namespace ns(pid_);
   try {
     addrs_ = Addrs(pid_, &ns, abi);
+    if (*abi >= PyABI::Py37)
+      addrs_.tstate_addr += PYFLAME_PYTHON37_OFFSET;
   } catch (const SymbolException &exc) {
     return 1;
   }
@@ -172,6 +184,11 @@ int PyFrob::DetectABI(PyABI abi) {
     case PyABI::Py36:
       get_threads_ = py36::GetThreads;
       break;
+#endif
+#ifdef ENABLE_PY37
+    case PyABI::Py37:
+      get_threads_ = py37::GetThreads;
+      break;
 #endif
     default:
       std::ostringstream os;
@@ -198,4 +215,5 @@ std::string PyFrob::Status() const {
 std::vector<Thread> PyFrob::GetThreads(void) const {
   return get_threads_(pid_, addrs_, enable_threads_);
 }
+
 }  // namespace pyflame

src/symbol.cc

@@ -137,31 +137,43 @@ PyABI ELF::WalkTable(int sym, int str, PyAddresses *addrs) {
         reinterpret_cast<const sym_t *>(p() + s->sh_offset + i * s->sh_entsize);
     const char *name =
         reinterpret_cast<const char *>(p() + d->sh_offset + sym->st_name);
-    if (!addrs->tstate_addr && strcmp(name, "_PyThreadState_Current") == 0) {
-      addrs->tstate_addr = static_cast<unsigned long>(sym->st_value);
-    } else if (!addrs->interp_head_addr && strcmp(name, "interp_head") == 0) {
-      addrs->interp_head_addr = static_cast<unsigned long>(sym->st_value);
-    } else if (!addrs->interp_head_addr &&
-               strcmp(name, "PyInterpreterState_Head") == 0) {
-      addrs->interp_head_fn_addr = static_cast<unsigned long>(sym->st_value);
-    } else if (!have_abi) {
+    if (!addrs->tstate_addr) {
+      if (strcmp(name, "_PyThreadState_Current") == 0) {
+        addrs->tstate_addr = static_cast<unsigned long>(sym->st_value);
+      } else if (strcmp(name, "_PyRuntime") == 0) {
+        addrs->tstate_addr = static_cast<unsigned long>(sym->st_value);
+      }
+    }
+
+    if (!addrs->interp_head_addr) {
+      if (strcmp(name, "interp_head") == 0) {
+        addrs->interp_head_addr = static_cast<unsigned long>(sym->st_value);
+      } else if (strcmp(name, "PyInterpreterState_Head") == 0) {
+        addrs->interp_head_fn_addr = static_cast<unsigned long>(sym->st_value);
+      }
+    }
+
+    if (!have_abi) {
       if (strcmp(name, "PyString_Type") == 0) {
         // If we find PyString_Type, this is some kind of Python 2.
         have_abi = true;
         abi = PyABI::Py26;
-      } else if (strcmp(name, "PyBytes_Type") == 0) {
+      } else if (strcmp(name, "PyBytes_Type") == 0 && abi < PyABI::Py34) {
         // If we find PyBytes_Type, it's Python 3. Continue looping though, in
-        // case we see a Python 3.6 symbol.
+        // case we see a Python 3.6+ symbol.
         abi = PyABI::Py34;
       } else if (strcmp(name, "_PyEval_RequestCodeExtraIndex") == 0 ||
                  strcmp(name, "_PyCode_GetExtra") == 0 ||
                  strcmp(name, "_PyCode_SetExtra") == 0) {
         // Symbols added for Python 3.6, see:
         // https://www.python.org/dev/peps/pep-0523/
-        have_abi = true;
         abi = PyABI::Py36;
+      } else if (strcmp(name, "_PyRuntime") == 0) {
+        have_abi = true;
+        abi = PyABI::Py37;
       }
     }
+
   }
   return abi;
 }

src/symbol.h

@@ -53,7 +53,8 @@ enum class PyABI {
   Unknown = 0,  // Unknown Python ABI
   Py26 = 26,    // ABI for Python 2.6/2.7
   Py34 = 34,    // ABI for Python 3.4/3.5
-  Py36 = 36     // ABI for Python 3.6
+  Py36 = 36,    // ABI for Python 3.6
+  Py37 = 37     // ABI for Python 3.7
 };
 
 // Symbols