Skip to content

Commit 4ffca5a

Browse files
Matthew Wilcox (Oracle)akpm00
Matthew Wilcox (Oracle)
authored and
akpm00
committed
mm: support only one page_type per page
By using a few values in the top byte, users of page_type can store up to 24 bits of additional data in page_type. It also reduces the code size as (with replacement of READ_ONCE() with data_race()), the kernel can check just a single byte. eg: ffffffff811e3a79: 8b 47 30 mov 0x30(%rdi),%eax ffffffff811e3a7c: 55 push %rbp ffffffff811e3a7d: 48 89 e5 mov %rsp,%rbp ffffffff811e3a80: 25 00 00 00 82 and $0x82000000,%eax ffffffff811e3a85: 3d 00 00 00 80 cmp $0x80000000,%eax ffffffff811e3a8a: 74 4d je ffffffff811e3ad9 <folio_mapping+0x69> becomes: ffffffff811e3a69: 80 7f 33 f5 cmpb $0xf5,0x33(%rdi) ffffffff811e3a6d: 55 push %rbp ffffffff811e3a6e: 48 89 e5 mov %rsp,%rbp ffffffff811e3a71: 74 4d je ffffffff811e3ac0 <folio_mapping+0x60> replacing three instructions with one. [[email protected]: fix ubsan warnings] Link: https://lkml.kernel.org/r/[email protected] Link: https://lkml.kernel.org/r/[email protected] Signed-off-by: Matthew Wilcox (Oracle) <[email protected]> Acked-by: David Hildenbrand <[email protected]> Cc: Hyeonggon Yoo <[email protected]> Cc: Kent Overstreet <[email protected]> Signed-off-by: Andrew Morton <[email protected]>
1 parent e880034 commit 4ffca5a

File tree

3 files changed

+56
-51
lines changed

3 files changed

+56
-51
lines changed

‎include/linux/page-flags.h

+28-40
Original file line numberDiff line numberDiff line change
@@ -923,42 +923,29 @@ PAGEFLAG_FALSE(HasHWPoisoned, has_hwpoisoned)
923923
#endif
924924

925925
/*
926-
* For pages that are never mapped to userspace,
927-
* page_type may be used. Because it is initialised to -1, we invert the
928-
* sense of the bit, so __SetPageFoo *clears* the bit used for PageFoo, and
929-
* __ClearPageFoo *sets* the bit used for PageFoo. We reserve a few high and
930-
* low bits so that an underflow or overflow of _mapcount won't be
931-
* mistaken for a page type value.
926+
* For pages that do not use mapcount, page_type may be used.
927+
* The low 24 bits of pagetype may be used for your own purposes, as long
928+
* as you are careful to not affect the top 8 bits. The low bits of
929+
* pagetype will be overwritten when you clear the page_type from the page.
932930
*/
933-
934931
enum pagetype {
935-
PG_buddy = 0x40000000,
936-
PG_offline = 0x20000000,
937-
PG_table = 0x10000000,
938-
PG_guard = 0x08000000,
939-
PG_hugetlb = 0x04000000,
940-
PG_slab = 0x02000000,
941-
PG_zsmalloc = 0x01000000,
942-
PG_unaccepted = 0x00800000,
943-
944-
PAGE_TYPE_BASE = 0x80000000,
945-
946-
/*
947-
* Reserve 0xffff0000 - 0xfffffffe to catch _mapcount underflows and
948-
* allow owners that set a type to reuse the lower 16 bit for their own
949-
* purposes.
950-
*/
951-
PAGE_MAPCOUNT_RESERVE = ~0x0000ffff,
932+
/* 0x00-0x7f are positive numbers, ie mapcount */
933+
/* Reserve 0x80-0xef for mapcount overflow. */
934+
PGTY_buddy = 0xf0,
935+
PGTY_offline = 0xf1,
936+
PGTY_table = 0xf2,
937+
PGTY_guard = 0xf3,
938+
PGTY_hugetlb = 0xf4,
939+
PGTY_slab = 0xf5,
940+
PGTY_zsmalloc = 0xf6,
941+
PGTY_unaccepted = 0xf7,
942+
943+
PGTY_mapcount_underflow = 0xff
952944
};
953945

954-
#define PageType(page, flag) \
955-
((READ_ONCE(page->page_type) & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE)
956-
#define folio_test_type(folio, flag) \
957-
((READ_ONCE(folio->page.page_type) & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE)
958-
959946
static inline bool page_type_has_type(int page_type)
960947
{
961-
return page_type < PAGE_MAPCOUNT_RESERVE;
948+
return page_type < (PGTY_mapcount_underflow << 24);
962949
}
963950

964951
/* This takes a mapcount which is one more than page->_mapcount */
@@ -969,40 +956,41 @@ static inline bool page_mapcount_is_type(unsigned int mapcount)
969956

970957
static inline bool page_has_type(const struct page *page)
971958
{
972-
return page_type_has_type(READ_ONCE(page->page_type));
959+
return page_mapcount_is_type(data_race(page->page_type));
973960
}
974961

975962
#define FOLIO_TYPE_OPS(lname, fname) \
976-
static __always_inline bool folio_test_##fname(const struct folio *folio)\
963+
static __always_inline bool folio_test_##fname(const struct folio *folio) \
977964
{ \
978-
return folio_test_type(folio, PG_##lname); \
965+
return data_race(folio->page.page_type >> 24) == PGTY_##lname; \
979966
} \
980967
static __always_inline void __folio_set_##fname(struct folio *folio) \
981968
{ \
982-
VM_BUG_ON_FOLIO(!folio_test_type(folio, 0), folio); \
983-
folio->page.page_type &= ~PG_##lname; \
969+
VM_BUG_ON_FOLIO(data_race(folio->page.page_type) != UINT_MAX, \
970+
folio); \
971+
folio->page.page_type = (unsigned int)PGTY_##lname << 24; \
984972
} \
985973
static __always_inline void __folio_clear_##fname(struct folio *folio) \
986974
{ \
987975
VM_BUG_ON_FOLIO(!folio_test_##fname(folio), folio); \
988-
folio->page.page_type |= PG_##lname; \
976+
folio->page.page_type = UINT_MAX; \
989977
}
990978

991979
#define PAGE_TYPE_OPS(uname, lname, fname) \
992980
FOLIO_TYPE_OPS(lname, fname) \
993981
static __always_inline int Page##uname(const struct page *page) \
994982
{ \
995-
return PageType(page, PG_##lname); \
983+
return data_race(page->page_type >> 24) == PGTY_##lname; \
996984
} \
997985
static __always_inline void __SetPage##uname(struct page *page) \
998986
{ \
999-
VM_BUG_ON_PAGE(!PageType(page, 0), page); \
1000-
page->page_type &= ~PG_##lname; \
987+
VM_BUG_ON_PAGE(data_race(page->page_type) != UINT_MAX, page); \
988+
page->page_type = (unsigned int)PGTY_##lname << 24; \
1001989
} \
1002990
static __always_inline void __ClearPage##uname(struct page *page) \
1003991
{ \
1004992
VM_BUG_ON_PAGE(!Page##uname(page), page); \
1005-
page->page_type |= PG_##lname; \
993+
page->page_type = UINT_MAX; \
1006994
}
1007995

1008996
/*

‎kernel/vmcore_info.c

+4-4
Original file line numberDiff line numberDiff line change
@@ -198,17 +198,17 @@ static int __init crash_save_vmcoreinfo_init(void)
198198
VMCOREINFO_NUMBER(PG_private);
199199
VMCOREINFO_NUMBER(PG_swapcache);
200200
VMCOREINFO_NUMBER(PG_swapbacked);
201-
#define PAGE_SLAB_MAPCOUNT_VALUE (~PG_slab)
201+
#define PAGE_SLAB_MAPCOUNT_VALUE (PGTY_slab << 24)
202202
VMCOREINFO_NUMBER(PAGE_SLAB_MAPCOUNT_VALUE);
203203
#ifdef CONFIG_MEMORY_FAILURE
204204
VMCOREINFO_NUMBER(PG_hwpoison);
205205
#endif
206206
VMCOREINFO_NUMBER(PG_head_mask);
207-
#define PAGE_BUDDY_MAPCOUNT_VALUE (~PG_buddy)
207+
#define PAGE_BUDDY_MAPCOUNT_VALUE (PGTY_buddy << 24)
208208
VMCOREINFO_NUMBER(PAGE_BUDDY_MAPCOUNT_VALUE);
209-
#define PAGE_HUGETLB_MAPCOUNT_VALUE (~PG_hugetlb)
209+
#define PAGE_HUGETLB_MAPCOUNT_VALUE (PGTY_hugetlb << 24)
210210
VMCOREINFO_NUMBER(PAGE_HUGETLB_MAPCOUNT_VALUE);
211-
#define PAGE_OFFLINE_MAPCOUNT_VALUE (~PG_offline)
211+
#define PAGE_OFFLINE_MAPCOUNT_VALUE (PGTY_offline << 24)
212212
VMCOREINFO_NUMBER(PAGE_OFFLINE_MAPCOUNT_VALUE);
213213

214214
#ifdef CONFIG_KALLSYMS

‎mm/debug.c

+24-7
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,6 @@ const struct trace_print_flags pageflag_names[] = {
3636
{0, NULL}
3737
};
3838

39-
const struct trace_print_flags pagetype_names[] = {
40-
__def_pagetype_names,
41-
{0, NULL}
42-
};
43-
4439
const struct trace_print_flags gfpflag_names[] = {
4540
__def_gfpflag_names,
4641
{0, NULL}
@@ -51,14 +46,35 @@ const struct trace_print_flags vmaflag_names[] = {
5146
{0, NULL}
5247
};
5348

49+
#define DEF_PAGETYPE_NAME(_name) [PGTY_##_name - 0xf0] = __stringify(_name)
50+
51+
static const char *page_type_names[] = {
52+
DEF_PAGETYPE_NAME(slab),
53+
DEF_PAGETYPE_NAME(hugetlb),
54+
DEF_PAGETYPE_NAME(offline),
55+
DEF_PAGETYPE_NAME(guard),
56+
DEF_PAGETYPE_NAME(table),
57+
DEF_PAGETYPE_NAME(buddy),
58+
DEF_PAGETYPE_NAME(unaccepted),
59+
};
60+
61+
static const char *page_type_name(unsigned int page_type)
62+
{
63+
unsigned i = (page_type >> 24) - 0xf0;
64+
65+
if (i >= ARRAY_SIZE(page_type_names))
66+
return "unknown";
67+
return page_type_names[i];
68+
}
69+
5470
static void __dump_folio(struct folio *folio, struct page *page,
5571
unsigned long pfn, unsigned long idx)
5672
{
5773
struct address_space *mapping = folio_mapping(folio);
5874
int mapcount = atomic_read(&page->_mapcount);
5975
char *type = "";
6076

61-
mapcount = page_type_has_type(mapcount) ? 0 : mapcount + 1;
77+
mapcount = page_mapcount_is_type(mapcount) ? 0 : mapcount + 1;
6278
pr_warn("page: refcount:%d mapcount:%d mapping:%p index:%#lx pfn:%#lx\n",
6379
folio_ref_count(folio), mapcount, mapping,
6480
folio->index + idx, pfn);
@@ -92,7 +108,8 @@ static void __dump_folio(struct folio *folio, struct page *page,
92108
pr_warn("%sflags: %pGp%s\n", type, &folio->flags,
93109
is_migrate_cma_folio(folio, pfn) ? " CMA" : "");
94110
if (page_has_type(&folio->page))
95-
pr_warn("page_type: %x\n", folio->page.page_type);
111+
pr_warn("page_type: %x(%s)\n", folio->page.page_type >> 24,
112+
page_type_name(folio->page.page_type));
96113

97114
print_hex_dump(KERN_WARNING, "raw: ", DUMP_PREFIX_NONE, 32,
98115
sizeof(unsigned long), page,

0 commit comments

Comments
 (0)