Optimize jl_add_*_finalizer fast path to avoid a lock

* Make `finalizer_list` thread local
* Optimize usage of `arraylist_t` in `gc.c`
* Also starts to explicitly pass `ptls` around to reduce TLS access overhead

    More consistently use `ptls` for the tls pointer of the current thread
    and `ptls2` for the tls pointer of other threads.

Author: yuyichao

base/base.jl

@@ -70,14 +70,16 @@ function finalizer(o::ANY, f::ANY)
     if isimmutable(o)
         error("objects of type ", typeof(o), " cannot be finalized")
     end
-    ccall(:jl_gc_add_finalizer, Void, (Any,Any), o, f)
+    ccall(:jl_gc_add_finalizer_th, Void, (Ptr{Void}, Any, Any),
+          Core.getptls(), o, f)
 end
 function finalizer{T}(o::T, f::Ptr{Void})
     @_inline_meta
     if isimmutable(T)
         error("objects of type ", T, " cannot be finalized")
     end
-    ccall(:jl_gc_add_ptr_finalizer, Void, (Any, Ptr{Void}), o, f)
+    ccall(:jl_gc_add_ptr_finalizer, Void, (Ptr{Void}, Any, Ptr{Void}),
+          Core.getptls(), o, f)
 end
 
 finalize(o::ANY) = ccall(:jl_finalize, Void, (Any,), o)

src/codegen.cpp

@@ -1318,6 +1318,7 @@ static uint64_t compute_obj_symsize(const object::ObjectFile *obj, uint64_t offs
 extern "C" JL_DLLEXPORT
 const jl_value_t *jl_dump_function_asm(void *f, int raw_mc)
 {
+    jl_tls_states_t *ptls = jl_get_ptls_states();
     std::string code;
     llvm::raw_string_ostream stream(code);
 #ifndef LLVM37
@@ -1362,7 +1363,7 @@ const jl_value_t *jl_dump_function_asm(void *f, int raw_mc)
         return (jl_value_t*)jl_pchar_to_array((char*)fptr, symsize);
     }
 
-    int8_t gc_state = jl_gc_safe_enter();
+    int8_t gc_state = jl_gc_safe_enter(ptls);
     jl_dump_asm_internal(fptr, symsize, slide,
 #ifndef USE_MCJIT
             context,
@@ -1378,7 +1379,7 @@ const jl_value_t *jl_dump_function_asm(void *f, int raw_mc)
 #ifndef LLVM37
     fstream.flush();
 #endif
-    jl_gc_safe_leave(gc_state);
+    jl_gc_safe_leave(ptls, gc_state);
 
     return jl_cstr_to_string(const_cast<char*>(stream.str().c_str()));
 }

src/debuginfo.cpp

@@ -251,9 +251,10 @@ class JuliaJITEventListener: public JITEventListener
     virtual void NotifyFunctionEmitted(const Function &F, void *Code,
                                        size_t Size, const EmittedFunctionDetails &Details)
     {
+        jl_tls_states_t *ptls = jl_get_ptls_states();
         // This function modify linfo->fptr in GC safe region.
         // This should be fine since the GC won't scan this field.
-        int8_t gc_state = jl_gc_safe_enter();
+        int8_t gc_state = jl_gc_safe_enter(ptls);
         uv_rwlock_wrlock(&threadsafe);
         StringRef sName = F.getName();
         StringMap<jl_lambda_info_t*>::iterator linfo_it = linfo_in_flight.find(sName);
@@ -274,7 +275,7 @@ class JuliaJITEventListener: public JITEventListener
             const_cast<Function*>(&F)->deleteBody();
 #endif
         uv_rwlock_wrunlock(&threadsafe);
-        jl_gc_safe_leave(gc_state);
+        jl_gc_safe_leave(ptls, gc_state);
     }
 
     std::map<size_t, FuncInfo, revcomp>& getMap()
@@ -309,9 +310,10 @@ class JuliaJITEventListener: public JITEventListener
     virtual void NotifyObjectEmitted(const ObjectImage &obj)
 #endif
     {
+        jl_tls_states_t *ptls = jl_get_ptls_states();
         // This function modify linfo->fptr in GC safe region.
         // This should be fine since the GC won't scan this field.
-        int8_t gc_state = jl_gc_safe_enter();
+        int8_t gc_state = jl_gc_safe_enter(ptls);
         uv_rwlock_wrlock(&threadsafe);
 #ifdef LLVM36
         object::section_iterator Section = debugObj.section_begin();
@@ -601,7 +603,7 @@ class JuliaJITEventListener: public JITEventListener
 #endif
 #endif
         uv_rwlock_wrunlock(&threadsafe);
-        jl_gc_safe_leave(gc_state);
+        jl_gc_safe_leave(ptls, gc_state);
     }
 
     // must implement if we ever start freeing code
@@ -1174,17 +1176,18 @@ int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, int64_t *se
 extern "C"
 JL_DLLEXPORT jl_value_t *jl_get_dobj_data(uint64_t fptr)
 {
+    jl_tls_states_t *ptls = jl_get_ptls_states();
     // Used by Gallium.jl
     const object::ObjectFile *object = NULL;
     DIContext *context;
     int64_t slide, section_slide;
-    int8_t gc_state = jl_gc_safe_enter();
+    int8_t gc_state = jl_gc_safe_enter(ptls);
     if (!jl_DI_for_fptr(fptr, NULL, &slide, NULL, &object, NULL))
         if (!jl_dylib_DI_for_fptr(fptr, &object, &context, &slide, &section_slide, false, NULL, NULL, NULL, NULL)) {
-            jl_gc_safe_leave(gc_state);
+            jl_gc_safe_leave(ptls, gc_state);
             return jl_nothing;
         }
-    jl_gc_safe_leave(gc_state);
+    jl_gc_safe_leave(ptls, gc_state);
     if (object == NULL)
         return jl_nothing;
     return (jl_value_t*)jl_ptr_to_array_1d((jl_value_t*)jl_array_uint8_type,
@@ -1195,8 +1198,9 @@ JL_DLLEXPORT jl_value_t *jl_get_dobj_data(uint64_t fptr)
 extern "C"
 JL_DLLEXPORT uint64_t jl_get_section_start(uint64_t fptr)
 {
+    jl_tls_states_t *ptls = jl_get_ptls_states();
     // Used by Gallium.jl
-    int8_t gc_state = jl_gc_safe_enter();
+    int8_t gc_state = jl_gc_safe_enter(ptls);
     std::map<size_t, ObjectInfo, revcomp> &objmap = jl_jit_events->getObjectMap();
     std::map<size_t, ObjectInfo, revcomp>::iterator fit = objmap.lower_bound(fptr);
 
@@ -1212,7 +1216,7 @@ JL_DLLEXPORT uint64_t jl_get_section_start(uint64_t fptr)
        }
     }
     uv_rwlock_rdunlock(&threadsafe);
-    jl_gc_safe_leave(gc_state);
+    jl_gc_safe_leave(ptls, gc_state);
     return ret;
 }
 

src/gc-debug.c

@@ -172,7 +172,10 @@ static void gc_verify_track(void)
         clear_mark(GC_CLEAN);
         pre_mark();
         gc_mark_object_list(&to_finalize, 0);
-        gc_mark_object_list(&finalizer_list, 0);
+        for (int i = 0;i < jl_n_threads;i++) {
+            jl_tls_states_t *ptls2 = jl_all_tls_states[i];
+            gc_mark_object_list(&ptls2->finalizers, 0);
+        }
         gc_mark_object_list(&finalizer_list_marked, 0);
         visit_mark_stack();
         if (lostval_parents.len == 0) {
@@ -215,7 +218,10 @@ void gc_verify(void)
     gc_verifying = 1;
     pre_mark();
     gc_mark_object_list(&to_finalize, 0);
-    gc_mark_object_list(&finalizer_list, 0);
+    for (int i = 0;i < jl_n_threads;i++) {
+        jl_tls_states_t *ptls2 = jl_all_tls_states[i];
+        gc_mark_object_list(&ptls2->finalizers, 0);
+    }
     gc_mark_object_list(&finalizer_list_marked, 0);
     visit_mark_stack();
     int clean_len = bits_save[GC_CLEAN].len;