Flip freemap (and rename to allocmap)

Remove the `memset(0xff)` when allocating a region.

Author: yuyichao

src/gc-debug.c

@@ -37,7 +37,7 @@ JL_DLLEXPORT jl_taggedvalue_t *jl_gc_find_taggedvalue_pool(char *p, size_t *osiz
     size_t ofs = p - page_begin;
     int pg_idx = page_index(r, page_begin);
     // Check if this is a free page
-    if (r->freemap[pg_idx / 32] & (uint32_t)(1 << (pg_idx % 32)))
+    if (!(r->allocmap[pg_idx / 32] & (uint32_t)(1 << (pg_idx % 32))))
         return NULL;
     jl_gc_pagemeta_t *pagemeta = &r->meta[pg_idx];
     int osize = pagemeta->osize;
@@ -134,10 +134,10 @@ static void clear_mark(int bits)
         if (!region->pages)
             break;
         for (int pg_i = 0; pg_i < region->pg_cnt / 32; pg_i++) {
-            uint32_t line = region->freemap[pg_i];
-            if (!!~line) {
+            uint32_t line = region->allocmap[pg_i];
+            if (line) {
                 for (int j = 0; j < 32; j++) {
-                    if (!((line >> j) & 1)) {
+                    if ((line >> j) & 1) {
                         jl_gc_pagemeta_t *pg = page_metadata(region->pages[pg_i*32 + j].data + GC_PAGE_OFFSET);
                         jl_tls_states_t *ptls =
                             jl_all_task_states[pg->thread_n].ptls;

src/gc-pages.c

@@ -77,7 +77,7 @@ static void jl_gc_alloc_region(region_t *region)
 {
     int pg_cnt = region_pg_cnt;
     const size_t pages_sz = sizeof(jl_gc_page_t) * pg_cnt;
-    const size_t freemap_sz = sizeof(uint32_t) * pg_cnt / 32;
+    const size_t allocmap_sz = sizeof(uint32_t) * pg_cnt / 32;
     char *mem = NULL;
     while (1) {
         if (__likely((mem = jl_gc_try_alloc_region(pg_cnt))))
@@ -95,17 +95,16 @@ static void jl_gc_alloc_region(region_t *region)
         }
     }
     region->pages = (jl_gc_page_t*)mem;
-    region->freemap = (uint32_t*)(mem + pages_sz);
-    region->meta = (jl_gc_pagemeta_t*)(mem + pages_sz +freemap_sz);
+    region->allocmap = (uint32_t*)(mem + pages_sz);
+    region->meta = (jl_gc_pagemeta_t*)(mem + pages_sz +allocmap_sz);
     region->lb = 0;
     region->ub = 0;
     region->pg_cnt = pg_cnt;
 #ifdef _OS_WINDOWS_
-    VirtualAlloc(region->freemap, pg_cnt / 8, MEM_COMMIT, PAGE_READWRITE);
+    VirtualAlloc(region->allocmap, pg_cnt / 8, MEM_COMMIT, PAGE_READWRITE);
     VirtualAlloc(region->meta, pg_cnt * sizeof(jl_gc_pagemeta_t),
                  MEM_COMMIT, PAGE_READWRITE);
 #endif
-    memset(region->freemap, 0xff, pg_cnt / 8);
 }
 
 NOINLINE void *jl_gc_alloc_page(void)
@@ -119,7 +118,7 @@ NOINLINE void *jl_gc_alloc_page(void)
         if (region->pages == NULL)
             jl_gc_alloc_region(region);
         for (i = region->lb; i < region->pg_cnt / 32; i++) {
-            if (region->freemap[i])
+            if (~region->allocmap[i])
                 break;
         }
         if (i == region->pg_cnt / 32) {
@@ -139,15 +138,15 @@ NOINLINE void *jl_gc_alloc_page(void)
         region->ub = i;
 
 #if defined(_COMPILER_MINGW_)
-    int j = __builtin_ffs(region->freemap[i]) - 1;
+    int j = __builtin_ffs(~region->allocmap[i]) - 1;
 #elif defined(_COMPILER_MICROSOFT_)
     unsigned long j;
-    _BitScanForward(&j, region->freemap[i]);
+    _BitScanForward(&j, ~region->allocmap[i]);
 #else
-    int j = ffs(region->freemap[i]) - 1;
+    int j = ffs(~region->allocmap[i]) - 1;
 #endif
 
-    region->freemap[i] &= ~(uint32_t)(1 << j);
+    region->allocmap[i] |= (uint32_t)(1 << j);
     void *ptr = region->pages[i * 32 + j].data;
 #ifdef _OS_WINDOWS_
     VirtualAlloc(ptr, GC_PAGE_SZ, MEM_COMMIT, PAGE_READWRITE);
@@ -174,8 +173,8 @@ void jl_gc_free_page(void *p)
     }
     assert(i < REGION_COUNT && region->pages != NULL);
     uint32_t msk = (uint32_t)(1 << (pg_idx % 32));
-    assert(!(region->freemap[pg_idx/32] & msk));
-    region->freemap[pg_idx/32] ^= msk;
+    assert(region->allocmap[pg_idx/32] & msk);
+    region->allocmap[pg_idx/32] ^= msk;
     free(region->meta[pg_idx].ages);
     // tell the OS we don't need these pages right now
     size_t decommit_size = GC_PAGE_SZ;
@@ -185,10 +184,13 @@ void jl_gc_free_page(void *p)
         decommit_size = jl_page_size;
         p = (void*)((uintptr_t)region->pages[pg_idx].data & ~(jl_page_size - 1)); // round down to the nearest page
         pg_idx = page_index(region, p);
-        if (pg_idx + n_pages > region->pg_cnt) goto no_decommit;
+        if (pg_idx + n_pages > region->pg_cnt)
+            goto no_decommit;
         for (; n_pages--; pg_idx++) {
             msk = (uint32_t)(1 << ((pg_idx % 32)));
-            if (!(region->freemap[pg_idx/32] & msk)) goto no_decommit;
+            if (region->allocmap[pg_idx / 32] & msk) {
+                goto no_decommit;
+            }
         }
     }
 #ifdef _OS_WINDOWS_

src/gc.c

@@ -806,11 +806,11 @@ static void sweep_pool_region(gcval_t ***pfl, int region_i, int sweep_mask)
     int ub = 0;
     int lb = region->lb;
     for (int pg_i = 0; pg_i <= region->ub; pg_i++) {
-        uint32_t line = region->freemap[pg_i];
-        if (!!~line) {
+        uint32_t line = region->allocmap[pg_i];
+        if (line) {
             ub = pg_i;
             for (int j = 0; j < 32; j++) {
-                if (!((line >> j) & 1)) {
+                if ((line >> j) & 1) {
                     jl_gc_pagemeta_t *pg = &region->meta[pg_i*32 + j];
                     int p_n = pg->pool_n;
                     int t_n = pg->thread_n;

src/gc.h

@@ -177,7 +177,7 @@ typedef struct {
     //    Tag: sizeof_jl_taggedvalue_t
     //    Data: <= osize - sizeof_jl_taggedvalue_t
     jl_gc_page_t *pages; // [pg_cnt]; must be first, to preserve page alignment
-    uint32_t *freemap; // [pg_cnt / 32]
+    uint32_t *allocmap; // [pg_cnt / 32]
     jl_gc_pagemeta_t *meta; // [pg_cnt]
     int pg_cnt;
     // store a lower bound of the first free page in each region