Skip to content

Commit 0ff7672

Browse files
npe9wenduwan
authored andcommitted
Initial evthread support
This is an initial stab at changing the Open MPI libevent integration to use opal threads instead of pthreads for alternative threading libraries. Currently, although opal threads are now replaceable via an mca, opal_event_use_threads() in event.h is just a macro passthrough to libevent's pthread event locking api. This was causing event list corruption with alternative thread libraries. This commit factors out libevent threading and integrates it with opal thread api. The changes are currently untested. The initial changes are primarily to solicit feedback on my approach before I begin more extensive testing and debugging. Signed-off-by: Noah Evans <[email protected]>
1 parent 20d2335 commit 0ff7672

File tree

3 files changed

+187
-2
lines changed

3 files changed

+187
-2
lines changed

opal/util/Makefile.am

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ libopalutil_la_SOURCES = \
9191
$(headers) \
9292
ethtool.c \
9393
event.c \
94+
evthread.c \
9495
if.c \
9596
net.c \
9697
info_subscriber.c \

opal/util/event.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,8 +85,7 @@ OPAL_DECLSPEC int opal_event_finalize(void);
8585
#define opal_event_set_priority(x, n) event_priority_set((x), (n))
8686

8787
/* thread support APIs */
88-
#define opal_event_use_threads() evthread_use_pthreads()
89-
88+
int opal_event_use_threads(void);
9089
/* Basic event APIs */
9190
#define opal_event_enable_debug_mode() event_enable_debug_mode()
9291

opal/util/evthread.c

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
/*
2+
* Copyright 2009-2012 Niels Provos and Nick Mathewson
3+
*
4+
* Redistribution and use in source and binary forms, with or without
5+
* modification, are permitted provided that the following conditions
6+
* are met:
7+
* 1. Redistributions of source code must retain the above copyright
8+
* notice, this list of conditions and the following disclaimer.
9+
* 2. Redistributions in binary form must reproduce the above copyright
10+
* notice, this list of conditions and the following disclaimer in the
11+
* documentation and/or other materials provided with the distribution.
12+
* 3. The name of the author may not be used to endorse or promote products
13+
* derived from this software without specific prior written permission.
14+
*
15+
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16+
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17+
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18+
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19+
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20+
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21+
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22+
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23+
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24+
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25+
*/
26+
#include "event2/event-config.h"
27+
#include "opal/mca/threads/mutex.h"
28+
#include "opal/mca/threads/tsd.h"
29+
#include "opal/mca/threads/condition.h"
30+
#include "opal/mca/threads/threads.h"
31+
/* With glibc we need to define _GNU_SOURCE to get PTHREAD_MUTEX_RECURSIVE.
32+
* This comes from evconfig-private.h
33+
*/
34+
35+
struct event_base;
36+
#include "event2/thread.h"
37+
#include "event2/util.h"
38+
#include <stdlib.h>
39+
#include <string.h>
40+
41+
42+
static void *
43+
evthread_opal_lock_alloc(unsigned locktype)
44+
{
45+
opal_thread_internal_mutex_t *lock = malloc(sizeof(opal_thread_internal_mutex_t));
46+
if (!lock)
47+
return NULL;
48+
if (opal_thread_internal_mutex_init(lock, locktype & EVTHREAD_LOCKTYPE_RECURSIVE)) {
49+
free(lock);
50+
return NULL;
51+
}
52+
return lock;
53+
}
54+
55+
static void
56+
evthread_opal_lock_free(void *lock_, unsigned locktype)
57+
{
58+
opal_thread_internal_mutex_t *lock = lock_;
59+
opal_thread_internal_mutex_destroy(lock);
60+
free(lock);
61+
}
62+
63+
static int
64+
evthread_opal_lock(unsigned mode, void *lock_)
65+
{
66+
opal_mutex_t *lock = lock_;
67+
if (mode & EVTHREAD_TRY)
68+
return opal_mutex_trylock(lock);
69+
else {
70+
opal_mutex_lock(lock);
71+
return 1;
72+
}
73+
}
74+
75+
static int
76+
evthread_opal_unlock(unsigned mode, void *lock_)
77+
{
78+
opal_mutex_t *lock = lock_;
79+
opal_mutex_unlock(lock);
80+
return 1;
81+
}
82+
83+
static unsigned long
84+
evthread_opal_get_id(void)
85+
{
86+
union {
87+
opal_thread_t thr;
88+
#if EVENT__SIZEOF_PTHREAD_T > EVENT__SIZEOF_LONG
89+
ev_uint64_t id;
90+
#else
91+
unsigned long id;
92+
#endif
93+
} r;
94+
#if EVENT__SIZEOF_PTHREAD_T < EVENT__SIZEOF_LONG
95+
memset(&r, 0, sizeof(r));
96+
#endif
97+
r.thr = *opal_thread_get_self();
98+
return (unsigned long)r.id;
99+
}
100+
101+
static void *
102+
evthread_opal_cond_alloc(unsigned condflags)
103+
{
104+
opal_cond_t *cond = malloc(sizeof(opal_cond_t));
105+
if (!cond)
106+
return NULL;
107+
if (opal_cond_init(cond)) {
108+
free(cond);
109+
return NULL;
110+
}
111+
return cond;
112+
}
113+
114+
static void
115+
evthread_opal_cond_free(void *cond_)
116+
{
117+
opal_cond_t *cond = cond_;
118+
opal_cond_destroy(cond);
119+
free(cond);
120+
}
121+
122+
static int
123+
evthread_opal_cond_signal(void *cond_, int broadcast)
124+
{
125+
opal_cond_t *cond = cond_;
126+
int r;
127+
if (broadcast)
128+
r = opal_cond_broadcast(cond);
129+
else
130+
r = opal_cond_signal(cond);
131+
return r ? -1 : 0;
132+
}
133+
134+
static int
135+
evthread_opal_cond_wait(void *cond_, void *lock_, const struct timeval *tv)
136+
{
137+
int r;
138+
opal_condition_t *cond = cond_;
139+
opal_mutex_t *lock = lock_;
140+
141+
if (tv) {
142+
struct timeval now, abstime;
143+
struct timespec ts;
144+
evutil_gettimeofday(&now, NULL);
145+
evutil_timeradd(&now, tv, &abstime);
146+
ts.tv_sec = abstime.tv_sec;
147+
ts.tv_nsec = abstime.tv_usec*1000;
148+
r = opal_condition_timedwait(cond, lock, &ts);
149+
if (r == ETIMEDOUT)
150+
return 1;
151+
else if (r)
152+
return -1;
153+
else
154+
return 0;
155+
} else {
156+
r = opal_cond_wait(cond, lock);
157+
return r ? -1 : 0;
158+
}
159+
}
160+
161+
int
162+
opal_event_use_threads(void)
163+
{
164+
struct evthread_lock_callbacks cbs = {
165+
EVTHREAD_LOCK_API_VERSION,
166+
EVTHREAD_LOCKTYPE_RECURSIVE,
167+
evthread_opal_lock_alloc,
168+
evthread_opal_lock_free,
169+
evthread_opal_lock,
170+
evthread_opal_unlock
171+
};
172+
struct evthread_condition_callbacks cond_cbs = {
173+
EVTHREAD_CONDITION_API_VERSION,
174+
evthread_opal_cond_alloc,
175+
evthread_opal_cond_free,
176+
evthread_opal_cond_signal,
177+
evthread_opal_cond_wait
178+
};
179+
180+
181+
evthread_set_lock_callbacks(&cbs);
182+
evthread_set_condition_callbacks(&cond_cbs);
183+
evthread_set_id_callback(evthread_opal_get_id);
184+
return 0;
185+
}

0 commit comments

Comments
 (0)