diff --git a/erts/include/internal/epiphany/ethread.h b/erts/include/internal/epiphany/ethread.h index 27e0e97839eb..2a05da2fd397 100644 --- a/erts/include/internal/epiphany/ethread.h +++ b/erts/include/internal/epiphany/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2015. All Rights Reserved. + * Copyright Ericsson AB 2015. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -29,26 +29,71 @@ #define ETHR_MAX_EPIPHANY_CORECOUNT 16 +/* + * The data guarded by these primitives is never shared between cores. We can + * get some cheap performance by inlining dummy implementations of these + * primitives. + */ +#define ETHR_DISABLE_EPIPHANY_ATOMICS 1 +#define ETHR_DISABLE_EPIPHANY_BARRIER 1 +#define ETHR_DISABLE_EPIPHANY_SPINLOCK 1 + +#if !ETHR_DISABLE_EPIPHANY_ATOMICS typedef struct { unsigned magic; volatile signed char level[ETHR_MAX_EPIPHANY_CORECOUNT]; volatile signed char waiting[ETHR_MAX_EPIPHANY_CORECOUNT-1]; ethr_sint32_t val; } ethr_native_atomic32_t; +#else +typedef struct { + ethr_sint32_t val; +} ethr_native_atomic32_t; +#endif /* !ETHR_DISABLE_EPIPHANY_ATOMICS */ + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) #define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG + +#if !ETHR_DISABLE_EPIPHANY_ATOMICS void ethr_native_atomic32_init(ethr_native_atomic32_t *var, ethr_sint32_t val); +ethr_sint32_t ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var, + ethr_sint32_t val, + ethr_sint32_t old_val); +#else +static ETHR_INLINE void +ethr_native_atomic32_init(ethr_native_atomic32_t *var, ethr_sint32_t val) +{ + var->val = val; +} + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var, + ethr_sint32_t val, + ethr_sint32_t old_val) +{ + ethr_sint32_t actual = var->val; + if (actual == old_val) var->val = val; + return actual; +} +#endif #define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR -static inline ethr_sint32_t * -ethr_native_atomic32_addr(ethr_native_atomic32_t *arg) { +static ETHR_INLINE ethr_sint32_t * +ethr_native_atomic32_addr(ethr_native_atomic32_t *arg) +{ return &arg->val; } -#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG -ethr_sint32_t ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var, - ethr_sint32_t val, - ethr_sint32_t old_val); +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_read(ethr_native_atomic32_t *var) +{ + return var->val; +} + +#endif /* defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) */ /* * Epiphany provides no way to barrier, but with some assumptions, we can @@ -72,12 +117,13 @@ ethr_sint32_t ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var, * accesses by simply writing something to DRAM, and then spinning until we can * read that thing back. */ -static inline void epiphany_dram_write_barrier(void); #define ETHR_LoadLoad (1 << 0) #define ETHR_LoadStore (1 << 1) #define ETHR_StoreLoad (1 << 2) #define ETHR_StoreStore (1 << 3) +#if !ETHR_DISABLE_EPIPHANY_BARRIER +static inline void epiphany_dram_write_barrier(void); #define ETHR_MEMBAR(B) do { \ __asm__ __volatile__ ("" : : : "memory"); \ __sync_synchronize(); \ @@ -87,7 +133,7 @@ static inline void epiphany_dram_write_barrier(void); } while(0) extern volatile char epiphany_dram_write_barrier_data[16]; -static inline void epiphany_dram_write_barrier(void) { +static ETHR_INLINE void epiphany_dram_write_barrier(void) { register unsigned coreid, index; register int new; /* We could use e-lib, but we don't want to include it everywhere. */ @@ -103,11 +149,19 @@ static inline void epiphany_dram_write_barrier(void) { __asm__ __volatile__ ("" : : : "memory"); while (epiphany_dram_write_barrier_data[index] != new); } +#else +#define ETHR_MEMBAR(B) do { \ + __asm__ __volatile__ ("" : : : "memory"); \ + __sync_synchronize(); \ + } while(0) +#endif /* !ETHR_DISABLE_EPIPHANY_BARRIER */ /* Spinlocks */ #define ETHR_HAVE_NATIVE_SPINLOCKS 1 #define ETHR_NATIVE_SPINLOCKS_REQUIRE_DESTRUCTION 1 #define ETHR_NATIVE_SPINLOCK_IMPL "epiphany" + +#if !ETHR_DISABLE_EPIPHANY_SPINLOCK /* * The epiphany mutexes /are/ spinlocks, but they need to be allocated in * SRAM. The mutexes are allocated from an array. @@ -123,5 +177,52 @@ void ethr_native_spin_unlock(ethr_native_spinlock_t*); int ethr_native_spin_trylock(ethr_native_spinlock_t*); int ethr_native_spin_is_locked(ethr_native_spinlock_t*); void ethr_native_spin_lock(ethr_native_spinlock_t*); +#else +/* + * When it is disabled, we assume only one thread ever touches this lock. + */ +typedef struct { +} ethr_native_spinlock_t; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) \ + || defined(ETHR_MUTEX_IMPL__) +static ETHR_INLINE void +ethr_native_spinlock_init(ethr_native_spinlock_t *spin) +{ + /* Do nothing */ +} + +static ETHR_INLINE int +ethr_native_spinlock_destroy(ethr_native_spinlock_t *spin) +{ + /* Do nothing */ + return 0; +} + +static ETHR_INLINE void +ethr_native_spin_unlock(ethr_native_spinlock_t *spin) +{ + /* Do nothing */ +} + +static ETHR_INLINE void +ethr_native_spin_lock(ethr_native_spinlock_t *spin) +{ + /* Do nothing */ +} + +#endif /* ETHR_TRY_INLINE_FUNCS || ETHR_AUX_IMPL__ || ETHR_MUTEX_IMPL__ */ + +/* A warning is generated if we include this one on AUX */ +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__) +static ETHR_INLINE int +ethr_native_spin_trylock(ethr_native_spinlock_t *spin) +{ + /* Do nothing */ + return 0; +} +#endif + +#endif /* !ETHR_DISABLE_EPIPHANY_SPINLOCK */ #endif /* ETHREAD_EPIPHANY_ETHREAD_H */ diff --git a/erts/lib_src/epiphany/ethread.c b/erts/lib_src/epiphany/ethread.c index 8c2731597282..c08723c52fcb 100644 --- a/erts/lib_src/epiphany/ethread.c +++ b/erts/lib_src/epiphany/ethread.c @@ -222,11 +222,14 @@ ethr_abort__(void) abort(); } +#if !ETHR_DISABLE_EPIPHANY_BARRIER +volatile char epiphany_dram_write_barrier_data[16]; +#endif + /* Atomics */ +#if !ETHR_DISABLE_EPIPHANY_ATOMICS #define PETERSON_MAGIC 0xBCBD1264 -volatile char epiphany_dram_write_barrier_data[16]; - /* * The "atomics" are implemented by acquiring a Peterson's algorithm mutex. The * reason the SRAM mutexes are not used instead is that atomics are not freed, @@ -304,7 +307,10 @@ ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var, return read_val; } +#endif /* !ETHR_DISABLE_EPIPHANY_ATOMICS */ + /* Spinlocks */ +#if !ETHR_DISABLE_EPIPHANY_SPINLOCK #include "epiphany.h" #define MUTEX_COUNT 16 @@ -488,3 +494,5 @@ ethr_native_spin_lock(ethr_native_spinlock_t *lock) /* We barrier just because consumers might assume a barrier is implied in a lock. */ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore); } + +#endif /* !ETHR_DISABLE_EPIPHANY_SPINLOCK */