Skip to content

Commit 0bea6e3

Browse files
yuyichaoyuyichao
committed
Separate page management into gc-pages.h
1 parent a9ab910 commit 0bea6e3

File tree

4 files changed

+172
-146
lines changed

4 files changed

+172
-146
lines changed

‎src/Makefile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ SRCS := \
2828
jltypes gf typemap ast builtins module interpreter \
2929
alloc dlload sys init task array dump toplevel jl_uv jlapi signal-handling \
3030
simplevector APInt-C runtime_intrinsics runtime_ccall \
31-
threadgroup threading stackwalk gc gc-debug safepoint
31+
threadgroup threading stackwalk gc gc-debug gc-pages safepoint
3232

3333
ifeq ($(JULIACODEGEN),LLVM)
3434
SRCS += codegen disasm debuginfo llvm-simdloop llvm-gcroot
@@ -119,6 +119,7 @@ $(BUILDDIR)/disasm.o $(BUILDDIR)/disasm.dbg.obj: $(SRCDIR)/codegen_internal.h
119119
$(BUILDDIR)/builtins.o $(BUILDDIR)/builtins.dbg.obj: $(SRCDIR)/table.c
120120
$(BUILDDIR)/gc.o $(BUILDDIR)/gc.dbg.obj: $(SRCDIR)/gc.h
121121
$(BUILDDIR)/gc-debug.o $(BUILDDIR)/gc-debug.dbg.obj: $(SRCDIR)/gc.h
122+
$(BUILDDIR)/gc-pages.o $(BUILDDIR)/gc-pages.dbg.obj: $(SRCDIR)/gc.h
122123
$(BUILDDIR)/signal-handling.o $(BUILDDIR)/signal-handling.dbg.obj: $(addprefix $(SRCDIR)/,signals-*.c)
123124
$(BUILDDIR)/dump.o $(BUILDDIR)/dump.dbg.obj: $(addprefix $(SRCDIR)/,common_symbols1.inc common_symbols2.inc)
124125
$(addprefix $(BUILDDIR)/,threading.o threading.dbg.obj gc.o gc.dbg.obj init.c init.dbg.obj task.o task.dbg.obj): $(addprefix $(SRCDIR)/,threading.h threadgroup.h)

‎src/gc-pages.c

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
// This file is a part of Julia. License is MIT: http://julialang.org/license
2+
3+
#include "gc.h"
4+
5+
#ifdef __cplusplus
6+
extern "C" {
7+
#endif
8+
9+
// A region is contiguous storage for up to REGION_PG_COUNT naturally aligned GC_PAGE_SZ pages
10+
// It uses a very naive allocator (see jl_gc_alloc_page & jl_gc_free_page)
11+
#if defined(_P64) && !defined(_COMPILER_MICROSOFT_)
12+
#define REGION_PG_COUNT 16*8*4096 // 8G because virtual memory is cheap
13+
#else
14+
#define REGION_PG_COUNT 8*4096 // 512M
15+
#endif
16+
17+
static jl_mutex_t pagealloc_lock;
18+
static size_t current_pg_count = 0;
19+
20+
NOINLINE void *jl_gc_alloc_page(void)
21+
{
22+
void *ptr = NULL;
23+
int i;
24+
region_t *region;
25+
int region_i = 0;
26+
JL_LOCK_NOGC(&pagealloc_lock);
27+
while(region_i < REGION_COUNT) {
28+
region = &regions[region_i];
29+
if (region->pages == NULL) {
30+
int pg_cnt = REGION_PG_COUNT;
31+
const size_t pages_sz = sizeof(jl_gc_page_t) * pg_cnt;
32+
const size_t freemap_sz = sizeof(uint32_t) * pg_cnt / 32;
33+
const size_t meta_sz = sizeof(jl_gc_pagemeta_t) * pg_cnt;
34+
size_t alloc_size = pages_sz + freemap_sz + meta_sz;
35+
#ifdef _OS_WINDOWS_
36+
char *mem = (char*)VirtualAlloc(NULL, alloc_size + GC_PAGE_SZ,
37+
MEM_RESERVE, PAGE_READWRITE);
38+
#else
39+
if (GC_PAGE_SZ > jl_page_size)
40+
alloc_size += GC_PAGE_SZ;
41+
char *mem = (char*)mmap(0, alloc_size, PROT_READ | PROT_WRITE, MAP_NORESERVE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
42+
mem = mem == MAP_FAILED ? NULL : mem;
43+
#endif
44+
if (mem == NULL) {
45+
jl_printf(JL_STDERR, "could not allocate pools\n");
46+
gc_debug_critical_error();
47+
abort();
48+
}
49+
if (GC_PAGE_SZ > jl_page_size) {
50+
// round data pointer up to the nearest gc_page_data-aligned
51+
// boundary if mmap didn't already do so.
52+
mem = (char*)gc_page_data(mem + GC_PAGE_SZ - 1);
53+
}
54+
region->pages = (jl_gc_page_t*)mem;
55+
region->freemap = (uint32_t*)(mem + pages_sz);
56+
region->meta = (jl_gc_pagemeta_t*)(mem + pages_sz +freemap_sz);
57+
region->lb = 0;
58+
region->ub = 0;
59+
region->pg_cnt = pg_cnt;
60+
#ifdef _OS_WINDOWS_
61+
VirtualAlloc(region->freemap, region->pg_cnt / 8,
62+
MEM_COMMIT, PAGE_READWRITE);
63+
VirtualAlloc(region->meta, region->pg_cnt * sizeof(jl_gc_pagemeta_t),
64+
MEM_COMMIT, PAGE_READWRITE);
65+
#endif
66+
memset(region->freemap, 0xff, region->pg_cnt / 8);
67+
}
68+
for (i = region->lb; i < region->pg_cnt / 32; i++) {
69+
if (region->freemap[i])
70+
break;
71+
}
72+
if (i == region->pg_cnt / 32) {
73+
// region full
74+
region_i++;
75+
continue;
76+
}
77+
break;
78+
}
79+
if (region_i >= REGION_COUNT) {
80+
jl_printf(JL_STDERR, "increase REGION_COUNT or allocate less memory\n");
81+
gc_debug_critical_error();
82+
abort();
83+
}
84+
if (region->lb < i)
85+
region->lb = i;
86+
if (region->ub < i)
87+
region->ub = i;
88+
89+
#if defined(_COMPILER_MINGW_)
90+
int j = __builtin_ffs(region->freemap[i]) - 1;
91+
#elif defined(_COMPILER_MICROSOFT_)
92+
unsigned long j;
93+
_BitScanForward(&j, region->freemap[i]);
94+
#else
95+
int j = ffs(region->freemap[i]) - 1;
96+
#endif
97+
98+
region->freemap[i] &= ~(uint32_t)(1 << j);
99+
ptr = region->pages[i*32 + j].data;
100+
#ifdef _OS_WINDOWS_
101+
VirtualAlloc(ptr, GC_PAGE_SZ, MEM_COMMIT, PAGE_READWRITE);
102+
#endif
103+
current_pg_count++;
104+
#ifdef GC_FINAL_STATS
105+
max_pg_count = max_pg_count < current_pg_count ? current_pg_count : max_pg_count;
106+
#endif
107+
JL_UNLOCK_NOGC(&pagealloc_lock);
108+
return ptr;
109+
}
110+
111+
void jl_gc_free_page(void *p)
112+
{
113+
int pg_idx = -1;
114+
int i;
115+
region_t *region = regions;
116+
for (i = 0; i < REGION_COUNT && regions[i].pages != NULL; i++) {
117+
region = &regions[i];
118+
pg_idx = page_index(region, p);
119+
if (pg_idx >= 0 && pg_idx < region->pg_cnt) {
120+
break;
121+
}
122+
}
123+
assert(i < REGION_COUNT && region->pages != NULL);
124+
uint32_t msk = (uint32_t)(1 << (pg_idx % 32));
125+
assert(!(region->freemap[pg_idx/32] & msk));
126+
region->freemap[pg_idx/32] ^= msk;
127+
free(region->meta[pg_idx].ages);
128+
// tell the OS we don't need these pages right now
129+
size_t decommit_size = GC_PAGE_SZ;
130+
if (GC_PAGE_SZ < jl_page_size) {
131+
// ensure so we don't release more memory than intended
132+
size_t n_pages = (GC_PAGE_SZ + jl_page_size - 1) / GC_PAGE_SZ;
133+
decommit_size = jl_page_size;
134+
p = (void*)((uintptr_t)region->pages[pg_idx].data & ~(jl_page_size - 1)); // round down to the nearest page
135+
pg_idx = page_index(region, p);
136+
if (pg_idx + n_pages > region->pg_cnt) goto no_decommit;
137+
for (; n_pages--; pg_idx++) {
138+
msk = (uint32_t)(1 << ((pg_idx % 32)));
139+
if (!(region->freemap[pg_idx/32] & msk)) goto no_decommit;
140+
}
141+
}
142+
#ifdef _OS_WINDOWS_
143+
VirtualFree(p, decommit_size, MEM_DECOMMIT);
144+
#else
145+
madvise(p, decommit_size, MADV_DONTNEED);
146+
#endif
147+
no_decommit:
148+
if (region->lb > pg_idx / 32)
149+
region->lb = pg_idx / 32;
150+
current_pg_count--;
151+
}
152+
153+
#ifdef __cplusplus
154+
}
155+
#endif

‎src/gc.c

Lines changed: 4 additions & 136 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,9 @@
66
extern "C" {
77
#endif
88

9-
jl_mutex_t pagealloc_lock;
109
// Protect all access to `finalizer_list`, `finalizer_list_marked` and
1110
// `to_finalize`.
12-
jl_mutex_t finalizers_lock;
11+
static jl_mutex_t finalizers_lock;
1312

1413
/**
1514
* Note about GC synchronization:
@@ -273,8 +272,6 @@ static size_t max_collect_interval = 500000000UL;
273272
#define NS2MS(t) ((double)(t/1000)/1000)
274273
static int64_t live_bytes = 0;
275274
static int64_t promoted_bytes = 0;
276-
static size_t current_pg_count = 0;
277-
static size_t max_pg_count = 0;
278275

279276
JL_DLLEXPORT size_t jl_gc_total_freed_bytes=0;
280277
#ifdef GC_FINAL_STATS
@@ -424,136 +421,6 @@ inline void gc_setmark_buf(void *o, int mark_mode)
424421
gc_setmark_big(buf, mark_mode);
425422
}
426423

427-
static NOINLINE void *malloc_page(void)
428-
{
429-
void *ptr = (void*)0;
430-
int i;
431-
region_t *region;
432-
int region_i = 0;
433-
JL_LOCK_NOGC(&pagealloc_lock);
434-
while(region_i < REGION_COUNT) {
435-
region = &regions[region_i];
436-
if (region->pages == NULL) {
437-
const size_t pages_sz = sizeof(jl_gc_page_t) * REGION_PG_COUNT;
438-
const size_t freemap_sz = sizeof(uint32_t) * REGION_PG_COUNT / 32;
439-
const size_t meta_sz = sizeof(jl_gc_pagemeta_t) * REGION_PG_COUNT;
440-
size_t alloc_size = pages_sz + freemap_sz + meta_sz;
441-
#ifdef _OS_WINDOWS_
442-
char *mem = (char*)VirtualAlloc(NULL, alloc_size + GC_PAGE_SZ,
443-
MEM_RESERVE, PAGE_READWRITE);
444-
#else
445-
if (GC_PAGE_SZ > jl_page_size)
446-
alloc_size += GC_PAGE_SZ;
447-
char *mem = (char*)mmap(0, alloc_size, PROT_READ | PROT_WRITE, MAP_NORESERVE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
448-
mem = mem == MAP_FAILED ? NULL : mem;
449-
#endif
450-
if (mem == NULL) {
451-
jl_printf(JL_STDERR, "could not allocate pools\n");
452-
gc_debug_critical_error();
453-
abort();
454-
}
455-
if (GC_PAGE_SZ > jl_page_size) {
456-
// round data pointer up to the nearest gc_page_data-aligned
457-
// boundary if mmap didn't already do so.
458-
mem = (char*)gc_page_data(mem + GC_PAGE_SZ - 1);
459-
}
460-
region->pages = (jl_gc_page_t*)mem;
461-
region->freemap = (uint32_t*)(mem + pages_sz);
462-
region->meta = (jl_gc_pagemeta_t*)(mem + pages_sz +freemap_sz);
463-
region->lb = 0;
464-
region->ub = 0;
465-
region->pg_cnt = REGION_PG_COUNT;
466-
#ifdef _OS_WINDOWS_
467-
VirtualAlloc(region->freemap, region->pg_cnt / 8,
468-
MEM_COMMIT, PAGE_READWRITE);
469-
VirtualAlloc(region->meta, region->pg_cnt * sizeof(jl_gc_pagemeta_t),
470-
MEM_COMMIT, PAGE_READWRITE);
471-
#endif
472-
memset(region->freemap, 0xff, region->pg_cnt / 8);
473-
}
474-
for (i = region->lb; i < region->pg_cnt / 32; i++) {
475-
if (region->freemap[i])
476-
break;
477-
}
478-
if (i == region->pg_cnt / 32) {
479-
// region full
480-
region_i++;
481-
continue;
482-
}
483-
break;
484-
}
485-
if (region_i >= REGION_COUNT) {
486-
jl_printf(JL_STDERR, "increase REGION_COUNT or allocate less memory\n");
487-
gc_debug_critical_error();
488-
abort();
489-
}
490-
if (region->lb < i)
491-
region->lb = i;
492-
if (region->ub < i)
493-
region->ub = i;
494-
495-
#if defined(_COMPILER_MINGW_)
496-
int j = __builtin_ffs(region->freemap[i]) - 1;
497-
#elif defined(_COMPILER_MICROSOFT_)
498-
unsigned long j;
499-
_BitScanForward(&j, region->freemap[i]);
500-
#else
501-
int j = ffs(region->freemap[i]) - 1;
502-
#endif
503-
504-
region->freemap[i] &= ~(uint32_t)(1 << j);
505-
ptr = region->pages[i*32 + j].data;
506-
#ifdef _OS_WINDOWS_
507-
VirtualAlloc(ptr, GC_PAGE_SZ, MEM_COMMIT, PAGE_READWRITE);
508-
#endif
509-
current_pg_count++;
510-
max_pg_count = max_pg_count < current_pg_count ? current_pg_count : max_pg_count;
511-
JL_UNLOCK_NOGC(&pagealloc_lock);
512-
return ptr;
513-
}
514-
515-
static void free_page(void *p)
516-
{
517-
int pg_idx = -1;
518-
int i;
519-
region_t *region = regions;
520-
for (i = 0; i < REGION_COUNT && regions[i].pages != NULL; i++) {
521-
region = &regions[i];
522-
pg_idx = page_index(region, p);
523-
if (pg_idx >= 0 && pg_idx < region->pg_cnt) {
524-
break;
525-
}
526-
}
527-
assert(i < REGION_COUNT && region->pages != NULL);
528-
uint32_t msk = (uint32_t)(1 << (pg_idx % 32));
529-
assert(!(region->freemap[pg_idx/32] & msk));
530-
region->freemap[pg_idx/32] ^= msk;
531-
free(region->meta[pg_idx].ages);
532-
// tell the OS we don't need these pages right now
533-
size_t decommit_size = GC_PAGE_SZ;
534-
if (GC_PAGE_SZ < jl_page_size) {
535-
// ensure so we don't release more memory than intended
536-
size_t n_pages = (GC_PAGE_SZ + jl_page_size - 1) / GC_PAGE_SZ;
537-
decommit_size = jl_page_size;
538-
p = (void*)((uintptr_t)region->pages[pg_idx].data & ~(jl_page_size - 1)); // round down to the nearest page
539-
pg_idx = page_index(region, p);
540-
if (pg_idx + n_pages > region->pg_cnt) goto no_decommit;
541-
for (; n_pages--; pg_idx++) {
542-
msk = (uint32_t)(1 << ((pg_idx % 32)));
543-
if (!(region->freemap[pg_idx/32] & msk)) goto no_decommit;
544-
}
545-
}
546-
#ifdef _OS_WINDOWS_
547-
VirtualFree(p, decommit_size, MEM_DECOMMIT);
548-
#else
549-
madvise(p, decommit_size, MADV_DONTNEED);
550-
#endif
551-
no_decommit:
552-
if (region->lb > pg_idx / 32)
553-
region->lb = pg_idx / 32;
554-
current_pg_count--;
555-
}
556-
557424
#define should_collect() (__unlikely(gc_num.allocd>0))
558425

559426
static inline int maybe_collect(void)
@@ -795,7 +662,7 @@ static inline gcval_t *reset_page(pool_t *p, jl_gc_pagemeta_t *pg, gcval_t *fl)
795662

796663
static NOINLINE void add_page(pool_t *p)
797664
{
798-
char *data = (char*)malloc_page();
665+
char *data = (char*)jl_gc_alloc_page();
799666
if (data == NULL)
800667
jl_throw(jl_memory_exception);
801668
jl_gc_pagemeta_t *pg = page_metadata(data + GC_PAGE_OFFSET);
@@ -1070,7 +937,7 @@ static gcval_t **sweep_page(pool_t *p, jl_gc_pagemeta_t *pg, gcval_t **pfl, int
1070937
#ifdef MEMDEBUG
1071938
memset(pg->data, 0xbb, GC_PAGE_SZ);
1072939
#endif
1073-
free_page(data);
940+
jl_gc_free_page(data);
1074941
#ifdef MEMDEBUG
1075942
memset(pg, 0xbb, sizeof(jl_gc_pagemeta_t));
1076943
#endif
@@ -2109,6 +1976,7 @@ JL_DLLEXPORT jl_value_t *jl_gc_alloc_3w(void)
21091976

21101977
#ifdef GC_FINAL_STATS
21111978
static double process_t0;
1979+
size_t max_pg_count = 0;
21121980
#include <malloc.h>
21131981
void jl_print_gc_stats(JL_STREAM *s)
21141982
{

0 commit comments

Comments
 (0)