From 949df563720cb88143f41cfe47957b2d59e0bf67 Mon Sep 17 00:00:00 2001 From: CjHanks Date: Tue, 28 May 2013 18:59:57 -0700 Subject: [PATCH] Epoll fix ref #6 Rather than creating a new socket on every instance of epoll.wait(), store the PyType in a std::map and return the original object. --- lib/udt4/pyudt.py | 2 +- setup.py | 18 +++++++--------- src/py-udt4-epoll.cc | 48 ++++++++++++------------------------------- src/py-udt4-epoll.hh | 8 ++++---- src/py-udt4-socket.hh | 2 ++ src/py-udt4.cc | 3 ++- test/__init__.py | 0 7 files changed, 29 insertions(+), 52 deletions(-) create mode 100644 test/__init__.py diff --git a/lib/udt4/pyudt.py b/lib/udt4/pyudt.py index f9d7f79..d80f31a 100644 --- a/lib/udt4/pyudt.py +++ b/lib/udt4/pyudt.py @@ -15,7 +15,7 @@ __author__ = 'Christopher J. Hanks ' __date__ = '12/15/2012' -__version__ = (0, 5) +__version__ = (0, 6) # assert py-udt4 version is in sync with udt4.pyudt version # please read __init__.py for explanation of versiong schema diff --git a/setup.py b/setup.py index 63ecb76..ee83d86 100644 --- a/setup.py +++ b/setup.py @@ -40,21 +40,17 @@ # ----------------------------------------------------------------------- # -VERSION = { - 'major' : 0, - 'minor' : 5, - 'patch' : 1, - } +VERSION = { 'major' : 0 + , 'minor' : 6 + , 'patch' : 0 } -from distutils.core import setup, Extension +from setuptools import setup, Extension udt4 = Extension( '_udt4', - sources = [ - 'src/py-udt4.cc', - 'src/py-udt4-epoll.cc', - 'src/py-udt4-socket.cc' - ], + sources = [ 'src/py-udt4.cc', + 'src/py-udt4-epoll.cc', + 'src/py-udt4-socket.cc' ], include_dirs = ['/usr/local/include', '/usr/include/'], libraries = ['udt', 'pthread'], library_dirs = ['/usr/local/lib', '/usr/lib64/', '/usr/lib/'], diff --git a/src/py-udt4-epoll.cc b/src/py-udt4-epoll.cc index 9595740..861051c 100644 --- a/src/py-udt4-epoll.cc +++ b/src/py-udt4-epoll.cc @@ -5,6 +5,8 @@ 12/18/2012 Initial stub 12/23/2012 Verified and cleaned for release 1.0 + 05/28/2013 Attempt to fix bad implementation of EPOLL which has + memory leak due to counting. */ #include "py-udt4-epoll.hh" @@ -52,7 +54,8 @@ pyudt4_epoll_add_usock(pyudt4_epoll_obj *self, PyObject *args) return 0x0; } - self->socks.push_back(sock->sock); + Py_INCREF(sock); + self->obj_map[sock->sock] = sock; if (UDT::ERROR == UDT::epoll_add_usock(self->eid, sock->sock, &flag)) RETURN_UDT_RUNTIME_ERROR; @@ -97,6 +100,9 @@ pyudt4_epoll_remove_usock(pyudt4_epoll_obj *self, PyObject *args) return 0x0; } + self->obj_map.erase(self->obj_map.find(sock->sock)); + Py_DECREF(sock); + if (UDT::ERROR == UDT::epoll_remove_usock(self->eid, sock->sock)) RETURN_UDT_RUNTIME_ERROR; @@ -191,24 +197,19 @@ pyudt4_epoll_wait(pyudt4_epoll_obj *self, PyObject *args) /* UDTSOCKET sets */ for (std::set::iterator i = usock.read.begin(); i != usock.read.end(); ++i) { - pyudt4_socket_obj *sock = - (pyudt4_socket_obj*)_PyObject_New(pyudt4_socket_type); - - sock->sock = *i; - sock->valid = 1; - + pyudt4_socket_obj *sock = self->obj_map[*i]; + PySet_Add(uset.read, (PyObject*) sock); + Py_INCREF(sock); } for (std::set::iterator i = usock.write.begin(); i != usock.write.end(); ++i) { - pyudt4_socket_obj *sock = - (pyudt4_socket_obj*)_PyObject_New(pyudt4_socket_type); - - sock->sock = *i; - sock->valid = 1; + pyudt4_socket_obj *sock = self->obj_map[*i]; + PySet_Add(uset.write, (PyObject*) sock); + Py_INCREF(sock); } /* SYSSOCKET sets */ @@ -227,19 +228,6 @@ pyudt4_epoll_wait(pyudt4_epoll_obj *self, PyObject *args) sset.read, sset.write); } -static PyObject* -pyudt4_epoll_garbage_collect(pyudt4_epoll_obj *self) -{ - for (std::vector::iterator i = self->socks.begin(); - i != self->socks.end(); ++i) { - if (UDT::getsockstate(*i) >= BROKEN) - UDT::epoll_remove_usock(self->eid, *i); - } - - self->socks.clear(); - - Py_RETURN_NONE; -} static PyMethodDef pyudt4_epoll_methods[] = { { @@ -327,16 +315,6 @@ static PyMethodDef pyudt4_epoll_methods[] = { " frozenset(write_sys_sockets),\n" " frozenset(write_sys_sockets))\n" }, - { - "garbage_collect", - (PyCFunction)pyudt4_epoll_garbage_collect, - METH_NOARGS, - "The UDT epoll doesn't track closed UDP sockets (afaik) \n" - "to alleviate the potential resource leak, all UDTSOCKET's\n" - "are tracked. When you execute the garbage_collect, it \n" - "will iterate the sockets and remove the UDTSOCKETS from \n" - "the epoll which are known dead." - }, { 0x0 } }; diff --git a/src/py-udt4-epoll.hh b/src/py-udt4-epoll.hh index 3e1610b..68a8f33 100644 --- a/src/py-udt4-epoll.hh +++ b/src/py-udt4-epoll.hh @@ -8,7 +8,7 @@ #define __PY_UDT_EPOLL_H_ #include -#include +#include #include "py-udt4-socket.hh" @@ -21,9 +21,9 @@ typedef struct __pyudt4_epoll_obj{ /** epoll id */ int eid; - - /** UDTSOCKET eid sets */ - std::vector socks; + + /** UDTSOCKET --> PyUDT socket map */ + std::map obj_map; } pyudt4_epoll_obj; diff --git a/src/py-udt4-socket.hh b/src/py-udt4-socket.hh index 0411fb9..00bf86c 100644 --- a/src/py-udt4-socket.hh +++ b/src/py-udt4-socket.hh @@ -2,6 +2,8 @@ @author Christopher J. Hanks @license py-udt4: GPLv3 libudt4: BSD + + 05/28/2013 Support udt4.11 new bind:: function for binding to UDP */ diff --git a/src/py-udt4.cc b/src/py-udt4.cc index 1252d89..bcff909 100644 --- a/src/py-udt4.cc +++ b/src/py-udt4.cc @@ -354,7 +354,7 @@ static PyTypeObject pyudt4_perfmon_type = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - "UDT::TRACEINFO", /* tp_doc */ + "UDT::TRACEINFO", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ @@ -484,6 +484,7 @@ pyudt4_bind(PyObject *py_self, PyObject *args) if (!PyArg_ParseTuple(args, "Osi", &sock, &address, &port)) { /* try bind_to_udp */ PyObject *tret = pyudt4_bind_to_udp(py_self, args); + if (0x0 != tret) return tret; diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29