From 0d76c13b9ebac3a4cbfc19d9f3ca7f35e6d115db Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 31 May 2024 03:24:37 +0100 Subject: [PATCH] page: Add a very basic LRU queue Add a very basic LRU queue. Right now it does no LRU (i.e it just takes the first page in the queue), but it's a good first start and survives basic memory reclaim tests (dd if=bigfile of=/dev/null for instance). Signed-off-by: Pedro Falcato --- kernel/include/onyx/mm/page_lru.h | 37 ++++++ kernel/include/onyx/mm/page_node.h | 68 +++++++++++ kernel/include/onyx/mm/page_zone.h | 111 ++++++++++++++++++ kernel/include/onyx/mm/vm_object.h | 2 + kernel/include/onyx/page.h | 8 ++ kernel/kernel/fs/filemap.cpp | 4 +- kernel/kernel/mm/Makefile | 2 +- kernel/kernel/mm/page_lru.c | 30 +++++ kernel/kernel/mm/pagealloc.cpp | 156 ++++--------------------- kernel/kernel/mm/reclaim.c | 179 ++++++++++++++++++++++++++++- kernel/kernel/mm/vm_object.cpp | 16 +++ 11 files changed, 478 insertions(+), 135 deletions(-) create mode 100644 kernel/include/onyx/mm/page_lru.h create mode 100644 kernel/include/onyx/mm/page_node.h create mode 100644 kernel/include/onyx/mm/page_zone.h create mode 100644 kernel/kernel/mm/page_lru.c diff --git a/kernel/include/onyx/mm/page_lru.h b/kernel/include/onyx/mm/page_lru.h new file mode 100644 index 000000000..7a349bfdc --- /dev/null +++ b/kernel/include/onyx/mm/page_lru.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2024 Pedro Falcato + * This file is part of Onyx, and is released under the terms of the MIT License + * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: MIT + */ + +#ifndef _ONYX_MM_PAGE_LRU_H +#define _ONYX_MM_PAGE_LRU_H + +#include +#include + +struct page; + +struct page_lru +{ + /* Very simple single LRU list (for the CLOCK algorithm) */ + struct list_head lru_list; + struct spinlock lock; +}; + +CONSTEXPR static inline void page_lru_init(struct page_lru *lru) +{ + INIT_LIST_HEAD(&lru->lru_list); + spinlock_init(&lru->lock); +} + +__BEGIN_CDECLS + +void page_add_lru(struct page *page); +void page_remove_lru(struct page *page); + +__END_CDECLS + +#endif diff --git a/kernel/include/onyx/mm/page_node.h b/kernel/include/onyx/mm/page_node.h new file mode 100644 index 000000000..15cee4a85 --- /dev/null +++ b/kernel/include/onyx/mm/page_node.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2017 - 2024 Pedro Falcato + * This file is part of Onyx, and is released under the terms of the MIT License + * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: MIT + */ + +#ifndef _ONYX_MM_PAGE_NODE_H +#define _ONYX_MM_PAGE_NODE_H + +#include +#include +#include + +struct page_node +{ + struct spinlock node_lock; + struct list_head cpu_list_node; + unsigned long used_pages; + unsigned long total_pages; + struct page_zone zones[NR_ZONES]; + +#ifdef __cplusplus + struct page_zone *pick_zone(unsigned long page); + + constexpr page_node() : node_lock{}, cpu_list_node{}, used_pages{}, total_pages{} + { + spinlock_init(&node_lock); + page_zone_init(&zones[0], "DMA32", 0, UINT32_MAX); + page_zone_init(&zones[1], "Normal", (u64) UINT32_MAX + 1, UINT64_MAX); + } + + void init() + { + INIT_LIST_HEAD(&cpu_list_node); + } + + void add_region(unsigned long base, size_t size); + struct page *alloc_order(unsigned int order, unsigned long flags); + struct page *allocate_pages(unsigned long nr_pages, unsigned long flags); + struct page *alloc_page(unsigned long flags); + void free_page(struct page *p); + + template + bool for_every_zone(Callable c) + { + for (auto &zone : zones) + { + if (!c(&zone)) + return false; + } + + return true; + } +#endif +}; + +/* ugh */ + +__BEGIN_CDECLS +extern struct page_node main_node; + +#define for_zones_in_node(node, zone) \ + for (zone = node->zones; zone < node->zones + NR_ZONES; zone++) + +__END_CDECLS +#endif diff --git a/kernel/include/onyx/mm/page_zone.h b/kernel/include/onyx/mm/page_zone.h new file mode 100644 index 000000000..4594cac95 --- /dev/null +++ b/kernel/include/onyx/mm/page_zone.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2017 - 2024 Pedro Falcato + * This file is part of Onyx, and is released under the terms of the MIT License + * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: MIT + */ + +#ifndef _ONYX_MM_PAGE_ZONE_H +#define _ONYX_MM_PAGE_ZONE_H + +#include +#include +#include + +#ifndef CONFIG_SMP_NR_CPUS +#define CONFIG_SMP_NR_CPUS 64 +#endif + +#define PAGEALLOC_NR_ORDERS 14 + +struct page_pcpu_data +{ + struct list_head page_list; + unsigned long nr_pages; + unsigned long nr_fast_path; + unsigned long nr_slow_path; + unsigned long nr_queue_reclaims; + long pagestats[PAGE_STATS_MAX]; + +#ifdef __cplusplus + constexpr page_pcpu_data() : nr_pages{}, nr_fast_path{}, nr_slow_path{}, nr_queue_reclaims{} + { + INIT_LIST_HEAD(&page_list); + for (auto &stat : pagestats) + stat = 0; + } + + /** + * @brief Allocate from pcpu state. + * IRQs must be disabled + * @return Allocated struct page, or nullptr + */ + __attribute__((always_inline)) struct page *alloc() + { + if (nr_pages == 0) [[unlikely]] + return nullptr; + + struct page *page = container_of(list_first_element(&page_list), struct page, + page_allocator_node.list_node); + list_remove(&page->page_allocator_node.list_node); + + nr_pages--; + + return page; + } + + /** + * @brief Free to pcpu state + * IRQs must be disabled + * @param page Page to free + */ + __attribute__((always_inline)) void free(struct page *page) + { + list_add_tail(&page->page_allocator_node.list_node, &page_list); + nr_pages++; + } +#endif + +} __align_cache; + +struct page_zone +{ + const char *name; + unsigned long start; + unsigned long end; + unsigned long min_watermark; + unsigned long low_watermark; + unsigned long high_watermark; + struct list_head pages[PAGEALLOC_NR_ORDERS]; + unsigned long total_pages; + long used_pages; + unsigned long splits; + unsigned long merges; + struct page_lru zone_lru; + struct spinlock lock; + struct page_pcpu_data pcpu[CONFIG_SMP_NR_CPUS] __align_cache; +}; + +#ifdef __cplusplus +constexpr void page_zone_init(page_zone *zone, const char *name, unsigned long start, + unsigned long end) +{ + zone->name = name; + zone->start = start; + zone->end = end; + zone->high_watermark = zone->min_watermark = zone->low_watermark = 0; + spinlock_init(&zone->lock); + for (auto &order : zone->pages) + { + INIT_LIST_HEAD(&order); + } + + zone->total_pages = 0; + zone->used_pages = 0; + zone->merges = zone->splits = 0; + page_lru_init(&zone->zone_lru); +} +#endif + +#endif diff --git a/kernel/include/onyx/mm/vm_object.h b/kernel/include/onyx/mm/vm_object.h index a49d9e6bf..e011dd081 100644 --- a/kernel/include/onyx/mm/vm_object.h +++ b/kernel/include/onyx/mm/vm_object.h @@ -274,6 +274,8 @@ void vm_obj_clean_page(struct vm_object *obj, struct page *page); void vm_obj_reassign_mapping(struct vm_object *vm_obj, struct vm_area_struct *vma); +bool vm_obj_remove_page(struct vm_object *obj, struct page *page); + __END_CDECLS #endif diff --git a/kernel/include/onyx/page.h b/kernel/include/onyx/page.h index 9de265954..51cf7674b 100644 --- a/kernel/include/onyx/page.h +++ b/kernel/include/onyx/page.h @@ -71,6 +71,8 @@ __BEGIN_CDECLS #define PAGE_FLAG_UPTODATE (1 << 8) #define PAGE_FLAG_WRITEBACK (1 << 9) #define PAGE_FLAG_READAHEAD (1 << 10) +#define PAGE_FLAG_LRU (1 << 11) +#define PAGE_FLAG_REFERENCED (1 << 12) struct vm_object; @@ -100,6 +102,8 @@ struct CAPABILITY("page") page struct page *next_virtual_region; } next_un; }; + + struct list_head lru_node; }; unsigned long priv; @@ -486,6 +490,7 @@ void page_drain_pcpu(); enum page_stat { NR_FILE = 0, + NR_SHARED, NR_ANON, NR_DIRTY, NR_WRITEBACK, @@ -504,6 +509,9 @@ void dec_page_stat(struct page *page, enum page_stat stat); void page_accumulate_stats(unsigned long pages[PAGE_STATS_MAX]); +struct page_lru; +struct page_lru *page_to_page_lru(struct page *page); + __END_CDECLS #endif diff --git a/kernel/kernel/fs/filemap.cpp b/kernel/kernel/fs/filemap.cpp index 52e3dcad6..c16e4648c 100644 --- a/kernel/kernel/fs/filemap.cpp +++ b/kernel/kernel/fs/filemap.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -63,6 +64,7 @@ int filemap_find_page(struct inode *ino, size_t pgoff, unsigned int flags, struc { inc_page_stat(p, NR_FILE); page_ref(p); + page_add_lru(p); } p = p2; @@ -660,8 +662,6 @@ int filemap_private_fault(struct vm_pf_context *ctx) * 'adopts' our reference. This works because amaps are inherently region-specific, and we have * the address_space locked. */ - if (!newp) - page_unref(page); return 0; enomem: diff --git a/kernel/kernel/mm/Makefile b/kernel/kernel/mm/Makefile index 8a65082b2..29820cc41 100644 --- a/kernel/kernel/mm/Makefile +++ b/kernel/kernel/mm/Makefile @@ -1,4 +1,4 @@ -mm-y:= bootmem.o page.o pagealloc.o vm_object.o vm.o vmalloc.o reclaim.o amap.o anon.o mincore.o +mm-y:= bootmem.o page.o pagealloc.o vm_object.o vm.o vmalloc.o reclaim.o amap.o anon.o mincore.o page_lru.o mm-$(CONFIG_KUNIT)+= vm_tests.o ifeq ($(CONFIG_KASAN), y) diff --git a/kernel/kernel/mm/page_lru.c b/kernel/kernel/mm/page_lru.c new file mode 100644 index 000000000..5c792d22a --- /dev/null +++ b/kernel/kernel/mm/page_lru.c @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2024 Pedro Falcato + * This file is part of Onyx, and is released under the terms of the MIT License + * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: MIT + */ +#include +#include + +void page_add_lru(struct page *page) +{ + DCHECK(!page_flag_set(page, PAGE_FLAG_LRU)); + DCHECK(page->owner != NULL); + struct page_lru *lru = page_to_page_lru(page); + spin_lock(&lru->lock); + list_add_tail(&page->lru_node, &lru->lru_list); + page_test_set_flag(page, PAGE_FLAG_LRU); + spin_unlock(&lru->lock); +} + +void page_remove_lru(struct page *page) +{ + DCHECK(page_flag_set(page, PAGE_FLAG_LRU)); + struct page_lru *lru = page_to_page_lru(page); + spin_lock(&lru->lock); + list_remove(&page->lru_node); + __atomic_and_fetch(&page->flags, ~PAGE_FLAG_LRU, __ATOMIC_RELEASE); + spin_unlock(&lru->lock); +} diff --git a/kernel/kernel/mm/pagealloc.cpp b/kernel/kernel/mm/pagealloc.cpp index 6062e268f..1c31262b9 100644 --- a/kernel/kernel/mm/pagealloc.cpp +++ b/kernel/kernel/mm/pagealloc.cpp @@ -18,6 +18,9 @@ #include #include +#include +#include +#include #include #include #include @@ -46,77 +49,10 @@ __always_inline unsigned long pow2(unsigned int exp) return (1UL << (unsigned long) exp); } -#define PAGEALLOC_NR_ORDERS 14 - #define MAX_PCPU_PAGES 1024 #define PCPU_REFILL_PAGES 512 #define PCPU_REFILL_ORDER 9 -struct page_pcpu_data -{ - struct list_head page_list; - unsigned long nr_pages{0}; - unsigned long nr_fast_path{0}; - unsigned long nr_slow_path{0}; - unsigned long nr_queue_reclaims{0}; - long pagestats[PAGE_STATS_MAX]; - - constexpr page_pcpu_data() - { - INIT_LIST_HEAD(&page_list); - for (auto &stat : pagestats) - stat = 0; - } - - /** - * @brief Allocate from pcpu state. - * IRQs must be disabled - * @return Allocated struct page, or nullptr - */ - __attribute__((always_inline)) struct page *alloc() - { - if (nr_pages == 0) [[unlikely]] - return nullptr; - - struct page *page = container_of(list_first_element(&page_list), struct page, - page_allocator_node.list_node); - list_remove(&page->page_allocator_node.list_node); - - nr_pages--; - - return page; - } - - /** - * @brief Free to pcpu state - * IRQs must be disabled - * @param page Page to free - */ - __attribute__((always_inline)) void free(struct page *page) - { - list_add_tail(&page->page_allocator_node.list_node, &page_list); - nr_pages++; - } - -} __align_cache; - -struct page_zone -{ - const char *name; - unsigned long start; - unsigned long end; - unsigned long min_watermark; - unsigned long low_watermark; - unsigned long high_watermark; - struct list_head pages[PAGEALLOC_NR_ORDERS]; - unsigned long total_pages; - long used_pages; - unsigned long splits; - unsigned long merges; - spinlock lock; - page_pcpu_data pcpu[CONFIG_SMP_NR_CPUS] __align_cache; -}; - __always_inline bool page_is_buddy(page *page) { return page->flags == PAGE_BUDDY; @@ -214,6 +150,8 @@ static void do_direct_reclaim(int order, int attempt, unsigned int gfp_flags) data.failed_order = order; data.gfp_flags = gfp_flags; data.mode = RECLAIM_MODE_DIRECT; + pr_info("pagealloc: Doing direct reclaim: order %d, attempt %d, gfp_flags %x\n", order, attempt, + gfp_flags); page_do_reclaim(&data); } @@ -343,6 +281,9 @@ static struct page *page_zone_refill_pcpu(struct page_zone *zone, unsigned int g struct page *page_zone_alloc(struct page_zone *zone, unsigned int gfp_flags, unsigned int order) { + if (zone->total_pages == 0) [[unlikely]] + return nullptr; + if (order == 0) [[likely]] { // Let's use pcpu caching for order-0 pages @@ -528,67 +469,6 @@ void page_zone_free(page_zone *zone, struct page *page, unsigned int order) page_zone_free_core(zone, page, order); } -constexpr void page_zone_init(page_zone *zone, const char *name, unsigned long start, - unsigned long end) -{ - zone->name = name; - zone->start = start; - zone->end = end; - zone->high_watermark = zone->min_watermark = zone->low_watermark = 0; - spinlock_init(&zone->lock); - for (auto &order : zone->pages) - { - INIT_LIST_HEAD(&order); - } - - zone->total_pages = 0; - zone->used_pages = 0; - zone->merges = zone->splits = 0; -} - -class page_node -{ -private: - struct spinlock node_lock; - struct list_head cpu_list_node; - unsigned long used_pages{0}; - unsigned long total_pages{0}; - struct page_zone zones[NR_ZONES]; - -public: - struct page_zone *pick_zone(unsigned long page); - - constexpr page_node() : node_lock{}, cpu_list_node{} - { - spinlock_init(&node_lock); - page_zone_init(&zones[0], "DMA32", 0, UINT32_MAX); - page_zone_init(&zones[1], "Normal", (u64) UINT32_MAX + 1, UINT64_MAX); - } - - void init() - { - INIT_LIST_HEAD(&cpu_list_node); - } - - void add_region(unsigned long base, size_t size); - struct page *alloc_order(unsigned int order, unsigned long flags); - struct page *allocate_pages(unsigned long nr_pages, unsigned long flags); - struct page *alloc_page(unsigned long flags); - void free_page(struct page *p); - - template - bool for_every_zone(Callable c) - { - for (auto &zone : zones) - { - if (!c(&zone)) - return false; - } - - return true; - } -}; - static bool page_is_initialized = false; page_node main_node; @@ -600,6 +480,11 @@ struct page_zone *page_node::pick_zone(unsigned long page) return &zones[ZONE_NORMAL]; } +struct page_lru *page_to_page_lru(struct page *page) +{ + return &main_node.pick_zone((unsigned long) page_to_phys(page))->zone_lru; +} + void page_node::add_region(uintptr_t base, size_t size) { if (size <= PAGE_SIZE) @@ -909,6 +794,7 @@ __always_inline void prepare_pages_after_alloc(struct page *page, unsigned int o } #define PAGE_ALLOC_MAX_RECLAIM_ATTEMPT 5 +void stack_trace(); struct page *page_node::alloc_order(unsigned int order, unsigned long flags) { @@ -920,7 +806,7 @@ struct page *page_node::alloc_order(unsigned int order, unsigned long flags) if (attempt == PAGE_ALLOC_MAX_RECLAIM_ATTEMPT) { /* Avoid locking up the system by just returning NULL */ - return nullptr; + goto failure; } int zone = ZONE_NORMAL; @@ -950,18 +836,25 @@ struct page *page_node::alloc_order(unsigned int order, unsigned long flags) wait_for_event(&paged_data.paged_waiters_queue, cur_seq < paged_data.reclaim_seq); } else - return nullptr; /* Since __GFP_ATOMIC cannot wait here, we simply fail. */ + goto failure; /* Since __GFP_ATOMIC cannot wait here, we simply fail. */ } else - return nullptr; /* No reclaim, just fail */ + goto failure; /* No reclaim, just fail */ attempt++; } + if (unlikely(!page)) + goto failure; + out: prepare_pages_after_alloc(page, order, flags); return page; +failure: + pr_warn("pagealloc: Failed allocation of order %u, gfp_flags %lx, on:\n", order, flags); + stack_trace(); + return nullptr; } struct page *alloc_pages(unsigned int order, unsigned long flags) @@ -983,6 +876,7 @@ void page_node::free_page(struct page *p) #ifdef CONFIG_PAGE_OWNER page_owner_freed(p); #endif + DCHECK(!page_flag_set(p, PAGE_FLAG_LRU)); unsigned long cpu_flags = spin_lock_irqsave(&node_lock); /* Reset the page */ diff --git a/kernel/kernel/mm/reclaim.c b/kernel/kernel/mm/reclaim.c index effb65cbf..c12b540fb 100644 --- a/kernel/kernel/mm/reclaim.c +++ b/kernel/kernel/mm/reclaim.c @@ -1,14 +1,18 @@ /* - * Copyright (c) 2023 Pedro Falcato + * Copyright (c) 2023 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the MIT License * check LICENSE at the root directory for more information * * SPDX-License-Identifier: MIT */ +#include + #include +#include #include #include #include +#include #include #include @@ -81,6 +85,178 @@ static void shrink_objects(struct reclaim_data *data, unsigned long free_page_ta rw_unlock_read(&shrinker_list_lock); } +enum lru_result +{ + LRU_SHRINK, + LRU_ROTATE, + LRU_ACTIVATE +}; + +#ifdef __clang__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wstring-plus-int" +#endif + +struct page_flag +{ + unsigned long val; + const char *name; +}; + +/* 10 = strlen(PAGE_FLAG_) */ +#define X(macro) \ + { \ + .val = macro, .name = #macro + 10 \ + } + +#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) + +static const struct page_flag flags[] = { + X(PAGE_FLAG_LOCKED), X(PAGE_FLAG_DIRTY), + X(PAGE_FLAG_PINNED), {.val = PAGE_BUDDY, .name = "BUDDY"}, + X(PAGE_FLAG_BUFFER), X(PAGE_FLAG_FILESYSTEM1), + X(PAGE_FLAG_WAITERS), X(PAGE_FLAG_UPTODATE), + X(PAGE_FLAG_WRITEBACK), X(PAGE_FLAG_READAHEAD), + X(PAGE_FLAG_LRU), X(PAGE_FLAG_REFERENCED), +}; + +#ifdef __clang__ +#pragma GCC diagnostic pop +#endif + +#undef X + +static void dump_page(struct page *page) +{ + char flags_buf[128]; + char *b = flags_buf; + size_t len = 128; + bool first = true; + flags_buf[0] = 0; + + for (unsigned int i = 0; i < ARRAY_SIZE(flags); i++) + { + if (page->flags & flags[i].val) + { + size_t copied = strlcpy(b, flags[i].name, len); + len -= copied; + b += copied; + if (!first) + { + if (len > 1) + *b = '|', b++, len--; + } + first = false; + } + } + + pr_crit("Page %p (pfn %016lx) ref: %lu\n", page, page_to_pfn(page), page->ref); + pr_crit(" flags: %016lx (%s) private: %016lx\n", page->flags, flags_buf, page->priv); + pr_crit(" owner: %p pageoff %lx\n", page->owner, page->pageoff); +} + +static void bug_on_page(struct page *page, const char *expr, const char *file, unsigned int line, + const char *func) +{ + pr_crit("Assertion %s failed in %s:%u, in function %s\n", expr, file, line, func); + dump_page(page); + panic(expr); +} + +#define DCHECK_PAGE(expr, page) \ + if (unlikely(!(expr))) \ + bug_on_page(page, #expr, __FILE__, __LINE__, __func__); + +static enum lru_result shrink_page(struct page *page) +{ + DCHECK_PAGE(page->owner, page); + if (page_flag_set(page, PAGE_FLAG_REFERENCED) || page_flag_set(page, PAGE_FLAG_DIRTY)) + { + __atomic_and_fetch(&page->flags, ~PAGE_FLAG_REFERENCED, __ATOMIC_RELAXED); + return LRU_ROTATE; + } + + if (!try_lock_page(page)) + return LRU_ROTATE; + + /* This should be a stable reference. TODO: What if truncation? What if the inode goes away + * after the unlock? */ + struct vm_object *obj = page->owner; + // pr_info("removing page %p\n", page); + if (!vm_obj_remove_page(obj, page)) + { + /* If we failed to remove the page, it's busy */ + unlock_page(page); + return LRU_ROTATE; + } + + unlock_page(page); + DCHECK(page->ref == 1); + dec_page_stat(page, NR_FILE); + list_remove(&page->lru_node); + __atomic_and_fetch(&page->flags, ~PAGE_FLAG_LRU, __ATOMIC_RELAXED); + if (obj->ops->free_page) + obj->ops->free_page(obj, page); + else + free_page(page); + return LRU_SHRINK; +} + +#define DEBUG_SHRINK_ZONE 1 + +static void shrink_zone(struct reclaim_data *data, struct page_node *node, struct page_zone *zone, + unsigned long target_freep) +{ + unsigned long target = target_freep; + (void) target; + struct page_lru *lru = &zone->zone_lru; + DEFINE_LIST(rotate_list); + + spin_lock(&lru->lock); + + list_for_every_safe (&lru->lru_list) + { + struct page *page = container_of(l, struct page, lru_node); + enum lru_result res = shrink_page(page); + if (res == LRU_ROTATE) + { + list_remove(&page->lru_node); + list_add_tail(&page->lru_node, &rotate_list); + } + else if (res == LRU_SHRINK) + { + target_freep--; + if (target_freep == 0) + break; + } + } + +#ifdef DEBUG_SHRINK_ZONE + pr_info("shrink_zone: Zone %s freed %lu pages (out of %lu)\n", zone->name, + target - target_freep, target); +#endif + list_splice(&rotate_list, &lru->lru_list); + spin_unlock(&lru->lock); +} + +static void shrink_page_zones(struct reclaim_data *data, struct page_node *node) +{ + struct page_zone *zone; + for_zones_in_node(node, zone) + { + unsigned long freep = zone->total_pages - zone->used_pages; + unsigned long target = 0; + + if (freep <= zone->low_watermark) + target = zone->high_watermark - freep; + + if (target == 0) + continue; + + shrink_zone(data, node, zone, target); + } +} + /** * @brief Do (direct?) page reclamation. Called from direct reclaim or pagedaemon. * @@ -102,6 +278,7 @@ int page_do_reclaim(struct reclaim_data *data) /* Lets scale according to our desperation */ if (nr_tries > 0) free_target *= nr_tries; + shrink_page_zones(data, &main_node); shrink_objects(data, free_target); #ifdef CONFIG_KASAN /* KASAN is likely to have a lot of objects under its wing, so flush it. */ diff --git a/kernel/kernel/mm/vm_object.cpp b/kernel/kernel/mm/vm_object.cpp index 5c0b5320e..e984c3ddb 100644 --- a/kernel/kernel/mm/vm_object.cpp +++ b/kernel/kernel/mm/vm_object.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -283,6 +284,7 @@ static int vmo_purge_pages(unsigned long start, unsigned long end, * reference */ page_unref(old_p); dec_page_stat(old_p, NR_FILE); + page_remove_lru(old_p); if (!vmo->ops->free_page) free_page(old_p); else @@ -511,3 +513,17 @@ void vm_obj_clean_page(struct vm_object *obj, struct page *page) vm_wp_page(vma->vm_mm, (void *) (vma->vm_start + (offset << PAGE_SHIFT) - vma->vm_offset)); } } + +bool vm_obj_remove_page(struct vm_object *obj, struct page *page) +{ + scoped_lock g{obj->page_lock}; + DCHECK(page_locked(page)); + DCHECK(page->owner == obj); + DCHECK(page->ref != 0); + /* Under the lock, pages can't get their references incremented, so we check if refs == 1 here. + */ + if (__atomic_load_n(&page->ref, __ATOMIC_RELAXED) > 1) + return false; + obj->vm_pages.store(page->pageoff, 0); + return true; +}