From c151d2f34737f28a1f5266a003b2b0720bbd9f96 Mon Sep 17 00:00:00 2001 From: Evan Klitzke Date: Mon, 25 Jun 2018 23:19:34 -0700 Subject: [PATCH 1/2] Tag v1.6.7 --- NEWS.org | 3 +++ configure.ac | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS.org b/NEWS.org index 9b33995..bf2a6cb 100644 --- a/NEWS.org +++ b/NEWS.org @@ -1,4 +1,7 @@ * NEWS (user visible changes) +** 1.6.7 (2018-06-25) + - Improved support for prelinked binaries, especially on ARM32 and Yocto + systems. (Andrew Jeffery) ** 1.6.6 (2018-05-02) diff --git a/configure.ac b/configure.ac index 0858fc3..81d421c 100644 --- a/configure.ac +++ b/configure.ac @@ -1,5 +1,5 @@ AC_PREREQ([2.68]) -AC_INIT([pyflame], [1.6.6], [evan@eklitzke.org]) +AC_INIT([pyflame], [1.6.7], [evan@eklitzke.org]) AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_HEADERS([src/config.h]) AC_CONFIG_MACRO_DIR([m4]) From 821f021ad36370257e0a7a453d38ea33cb6115ac Mon Sep 17 00:00:00 2001 From: Gabriel Corona Date: Thu, 28 Feb 2019 00:19:38 +0100 Subject: [PATCH 2/2] Add support for Python 3.7 --- .travis.yml | 1 + configure.ac | 10 ++++++++-- docs/faq.rst | 2 +- docs/man.md | 2 +- docs/pyflame.man | 2 +- src/Makefile.am | 12 ++++++++++-- src/frob.cc | 17 +++++++++++++++++ src/frob37.cc | 18 ++++++++++++++++++ src/offset37.c | 26 ++++++++++++++++++++++++++ src/prober.cc | 8 +++++++- src/pyfrob.cc | 18 ++++++++++++++++++ src/symbol.cc | 34 +++++++++++++++++++++++----------- src/symbol.h | 3 ++- 13 files changed, 133 insertions(+), 20 deletions(-) create mode 100644 src/frob37.cc create mode 100644 src/offset37.c diff --git a/.travis.yml b/.travis.yml index 9cc00ec..8a2d6c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ env: - PYVERSION=python3.4 - PYVERSION=python3.5 - PYVERSION=python3.6 + - PYVERSION=python3.7 addons: apt: diff --git a/configure.ac b/configure.ac index 81d421c..f6a252d 100644 --- a/configure.ac +++ b/configure.ac @@ -111,7 +111,12 @@ PKG_CHECK_MODULES([PY36], [python-3.6], [enable_py36="yes"], [AC_MSG_WARN([Build AM_CONDITIONAL([ENABLE_PY36], [test x"$enable_py36" = xyes]) AM_COND_IF([ENABLE_PY36], [AC_DEFINE([ENABLE_PY36], [1], [Python 3.6 will be enabled])]) -AS_IF([test x"$enable_py26" = xyes -o x"$enable_py34" = xyes -o x"$enable_py36" = xyes], +enable_py37=no +PKG_CHECK_MODULES([PY37], [python-3.7], [enable_py37="yes"], [AC_MSG_WARN([Building without Python 3.7 support])]) +AM_CONDITIONAL([ENABLE_PY37], [test x"$enable_py37" = xyes]) +AM_COND_IF([ENABLE_PY37], [AC_DEFINE([ENABLE_PY37], [1], [Python 3.7 will be enabled])]) + +AS_IF([test x"$enable_py26" = xyes -o x"$enable_py34" = xyes -o x"$enable_py36" = xyes -o x"$enable_py37" = xyes], [AC_MSG_NOTICE([Found at least one copy of Python.h])], [AC_MSG_ERROR([Failed to find a supported Python.h])] ) @@ -127,7 +132,8 @@ echo echo " with threads = $enable_threads" echo " with Python 2.6/7 = $enable_py26" echo " with Python 3.4/5 = $enable_py34" -echo " with Python 3.6+ = $enable_py36" +echo " with Python 3.6 = $enable_py36" +echo " with Python 3.7+ = $enable_py37" echo echo " CXX = $CXX" echo " CXXFLAGS = $CXXFLAGS" diff --git a/docs/faq.rst b/docs/faq.rst index 6fe8d1e..da6cec8 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -7,7 +7,7 @@ What Python Versions Are Supported? Python 2 is tested with Python 2.6 and 2.7. Earlier versions of Python 2 are likely to work as well, but have not been tested. -Python 3 is tested with Python 3.4, 3.5, and 3.6. Python 3.6 introduces a new +Python 3 is tested with Python 3.4, 3.5, 3.6 and 3.7 Python 3.6 introduces a new ABI for the ``PyCodeObject`` type, so Pyflame only supports the Python 3 versions that header files were available for when Pyflame was compiled. diff --git a/docs/man.md b/docs/man.md index 1aa8d25..538a764 100644 --- a/docs/man.md +++ b/docs/man.md @@ -87,7 +87,7 @@ The following options are less commonly used. cases when profiling embedded Python builds (e.g. uWSGI), and only if pyflame doesn't automatically detect the correct ABI. *VERSION* should be a two digit integer consisting of the Python major and minor version, e.g. 27 - for Python 2.7 or 36 for Python 3.6. + for Python 2.7, 36 for Python 3.6 or 37 for Python 3.7. **--flamechart** : Print the timestamp for each stack. This is useful for generating "flame diff --git a/docs/pyflame.man b/docs/pyflame.man index 971a1de..7fdeb85 100644 --- a/docs/pyflame.man +++ b/docs/pyflame.man @@ -119,7 +119,7 @@ uWSGI), and only if pyflame doesn\[aq]t automatically detect the correct ABI. \f[I]VERSION\f[] should be a two digit integer consisting of the Python major and minor version, e.g. -27 for Python 2.7 or 36 for Python 3.6. +27 for Python 2.7, 36 for Python 3.6 or 37 for Python 3.7. .RS .RE .TP diff --git a/src/Makefile.am b/src/Makefile.am index 11d8ab6..4a62bdb 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -2,11 +2,11 @@ # # The way this code is structured is the code that know about Python interpreter # internals is in frob.cc. This file isn't compiled directly, instead there are -# files frob{26,34,36}.cc that define preprocessor macros to compile frob.cc for +# files frob{26,34,36,37}.cc that define preprocessor macros to compile frob.cc for # different Python ABIs. # # The libtool magic here makes it so that frob26.cc is compiled with python2 -# flags, and frob3{4,6}.cc are compiled with python3.4/3.6 flags. +# flags, and frob3{4,6,7}.cc are compiled with python3.4/3.6/3.7 flags. bin_PROGRAMS = pyflame pyflame_SOURCES = aslr.cc frame.cc thread.cc namespace.cc posix.cc prober.cc ptrace.cc pyflame.cc pyfrob.cc symbol.cc @@ -34,3 +34,11 @@ libfrob36_la_CXXFLAGS = $(PY36_CFLAGS) noinst_LTLIBRARIES += libfrob36.la pyflame_LDADD += libfrob36.la endif + +if ENABLE_PY37 +libfrob37_la_SOURCES = frob37.cc offset37.c +libfrob37_la_CFLAGS = $(PY37_CFLAGS) +libfrob37_la_CXXFLAGS = $(PY37_CFLAGS) +noinst_LTLIBRARIES += libfrob37.la +pyflame_LDADD += libfrob37.la +endif diff --git a/src/frob.cc b/src/frob.cc index 6fcff39..c1c5b30 100644 --- a/src/frob.cc +++ b/src/frob.cc @@ -17,6 +17,7 @@ // information. #include + #include #if PYFLAME_PY_VERSION >= 34 @@ -89,6 +90,22 @@ unsigned long ByteData(unsigned long addr) { return addr + offsetof(PyBytesObject, ob_sval); } +#elif PYFLAME_PY_VERSION == 37 +namespace py37 { +std::string StringDataPython3(pid_t pid, unsigned long addr); + +unsigned long StringSize(unsigned long addr) { + return addr + offsetof(PyVarObject, ob_size); +} + +std::string StringData(pid_t pid, unsigned long addr) { + return StringDataPython3(pid, addr); +} + +unsigned long ByteData(unsigned long addr) { + return addr + offsetof(PyBytesObject, ob_sval); +} + #else static_assert(false, "uh oh, bad PYFLAME_PY_VERSION"); #endif diff --git a/src/frob37.cc b/src/frob37.cc new file mode 100644 index 0000000..1914893 --- /dev/null +++ b/src/frob37.cc @@ -0,0 +1,18 @@ +// Copyright 2016 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ABI for Python 3.7 +#define PYFLAME_PY_VERSION 37 + +#include "./frob.cc" diff --git a/src/offset37.c b/src/offset37.c new file mode 100644 index 0000000..b592766 --- /dev/null +++ b/src/offset37.c @@ -0,0 +1,26 @@ +// Copyright 2016 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#define Py_BUILD_CORE +#include +#include +#include +#include + +// We needd to include pystate.h which needs to include pyatomic.h +// which needs to include stdatomic.h which currently fails to work in C++ +// with GCC. We do this in C instead. +// +size_t PYFLAME_PYTHON37_OFFSET = offsetof(_PyRuntimeState, gilstate) + + offsetof(struct _gilstate_runtime_state, tstate_current); diff --git a/src/prober.cc b/src/prober.cc index a691c83..c653357 100644 --- a/src/prober.cc +++ b/src/prober.cc @@ -64,7 +64,7 @@ static const char usage_str[] = " -x, --exclude-idle Exclude idle time from statistics\n" "\n" "Advanced Options:\n" - " --abi Force a particular Python ABI (26, 34, 36)\n" + " --abi Force a particular Python ABI (26, 34, 36, 37)\n" " --flamechart Include timestamps for generating Chrome " "\"flamecharts\"\n"); @@ -79,6 +79,9 @@ static const int build_abis[] = { #ifdef ENABLE_PY36 36, #endif +#ifdef ENABLE_PY37 + 37, +#endif }; static_assert(sizeof(build_abis) > 0, "No Python ABIs detected!"); @@ -221,6 +224,9 @@ int Prober::ParseOpts(int argc, char **argv) { case 36: abi_ = PyABI::Py36; break; + case 37: + abi_ = PyABI::Py37; + break; default: std::cerr << "Unknown or unsupported ABI version: " << abi_version << "\n"; diff --git a/src/pyfrob.cc b/src/pyfrob.cc index 7c0588d..b280d96 100644 --- a/src/pyfrob.cc +++ b/src/pyfrob.cc @@ -14,6 +14,7 @@ #include "./pyfrob.h" +#include #include #include @@ -25,6 +26,8 @@ #include "./ptrace.h" #include "./symbol.h" +extern size_t PYFLAME_PYTHON37_OFFSET; + #define FROB_FUNCS \ std::vector GetThreads(pid_t pid, PyAddresses addr, \ bool enable_threads); @@ -126,11 +129,20 @@ FROB_FUNCS } #endif +#ifdef ENABLE_PY37 +namespace py37 { +FROB_FUNCS +extern size_t tstate_offset; +} +#endif + // Fill the addrs_ member int PyFrob::set_addrs_(PyABI *abi) { Namespace ns(pid_); try { addrs_ = Addrs(pid_, &ns, abi); + if (*abi >= PyABI::Py37) + addrs_.tstate_addr += PYFLAME_PYTHON37_OFFSET; } catch (const SymbolException &exc) { return 1; } @@ -172,6 +184,11 @@ int PyFrob::DetectABI(PyABI abi) { case PyABI::Py36: get_threads_ = py36::GetThreads; break; +#endif +#ifdef ENABLE_PY37 + case PyABI::Py37: + get_threads_ = py37::GetThreads; + break; #endif default: std::ostringstream os; @@ -198,4 +215,5 @@ std::string PyFrob::Status() const { std::vector PyFrob::GetThreads(void) const { return get_threads_(pid_, addrs_, enable_threads_); } + } // namespace pyflame diff --git a/src/symbol.cc b/src/symbol.cc index 39c3e81..3a6123c 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -137,31 +137,43 @@ PyABI ELF::WalkTable(int sym, int str, PyAddresses *addrs) { reinterpret_cast(p() + s->sh_offset + i * s->sh_entsize); const char *name = reinterpret_cast(p() + d->sh_offset + sym->st_name); - if (!addrs->tstate_addr && strcmp(name, "_PyThreadState_Current") == 0) { - addrs->tstate_addr = static_cast(sym->st_value); - } else if (!addrs->interp_head_addr && strcmp(name, "interp_head") == 0) { - addrs->interp_head_addr = static_cast(sym->st_value); - } else if (!addrs->interp_head_addr && - strcmp(name, "PyInterpreterState_Head") == 0) { - addrs->interp_head_fn_addr = static_cast(sym->st_value); - } else if (!have_abi) { + if (!addrs->tstate_addr) { + if (strcmp(name, "_PyThreadState_Current") == 0) { + addrs->tstate_addr = static_cast(sym->st_value); + } else if (strcmp(name, "_PyRuntime") == 0) { + addrs->tstate_addr = static_cast(sym->st_value); + } + } + + if (!addrs->interp_head_addr) { + if (strcmp(name, "interp_head") == 0) { + addrs->interp_head_addr = static_cast(sym->st_value); + } else if (strcmp(name, "PyInterpreterState_Head") == 0) { + addrs->interp_head_fn_addr = static_cast(sym->st_value); + } + } + + if (!have_abi) { if (strcmp(name, "PyString_Type") == 0) { // If we find PyString_Type, this is some kind of Python 2. have_abi = true; abi = PyABI::Py26; - } else if (strcmp(name, "PyBytes_Type") == 0) { + } else if (strcmp(name, "PyBytes_Type") == 0 && abi < PyABI::Py34) { // If we find PyBytes_Type, it's Python 3. Continue looping though, in - // case we see a Python 3.6 symbol. + // case we see a Python 3.6+ symbol. abi = PyABI::Py34; } else if (strcmp(name, "_PyEval_RequestCodeExtraIndex") == 0 || strcmp(name, "_PyCode_GetExtra") == 0 || strcmp(name, "_PyCode_SetExtra") == 0) { // Symbols added for Python 3.6, see: // https://www.python.org/dev/peps/pep-0523/ - have_abi = true; abi = PyABI::Py36; + } else if (strcmp(name, "_PyRuntime") == 0) { + have_abi = true; + abi = PyABI::Py37; } } + } return abi; } diff --git a/src/symbol.h b/src/symbol.h index bb92b9a..818424d 100644 --- a/src/symbol.h +++ b/src/symbol.h @@ -53,7 +53,8 @@ enum class PyABI { Unknown = 0, // Unknown Python ABI Py26 = 26, // ABI for Python 2.6/2.7 Py34 = 34, // ABI for Python 3.4/3.5 - Py36 = 36 // ABI for Python 3.6 + Py36 = 36, // ABI for Python 3.6 + Py37 = 37 // ABI for Python 3.7 }; // Symbols