diff --git a/arch/x86/pagetables.c b/arch/x86/pagetables.c index 92485b7c..14d76deb 100644 --- a/arch/x86/pagetables.c +++ b/arch/x86/pagetables.c @@ -24,12 +24,16 @@ */ #include #include +#include #include #include #include #include #include +static uint8_t _tmp_mapping[PAGE_SIZE] __aligned(PAGE_SIZE); +static pgentry_t *_tmp_mapping_entry; + cr3_t __aligned(PAGE_SIZE) cr3; cr3_t user_cr3; @@ -100,17 +104,11 @@ void dump_pagetables(cr3_t cr3) { dump_page_table(paddr_to_virt_kern(cr3.paddr), 4); } -static void *init_map_mfn(mfn_t mfn) { - static uint8_t _tmp[PAGE_SIZE] __aligned(PAGE_SIZE); - pgentry_t *e; - +static inline void *tmp_map_mfn(mfn_t mfn) { BUG_ON(mfn_invalid(mfn)); - - e = (pgentry_t *) l1_table_entry(get_l1_table(_tmp), _tmp); - BUG_ON(!e); - set_pgentry(e, mfn, L1_PROT); - - return _tmp; + set_pgentry(_tmp_mapping_entry, mfn, L1_PROT); + invlpg(_tmp_mapping); + return _tmp_mapping; } static mfn_t get_cr3_mfn(cr3_t *cr3_entry) { @@ -121,7 +119,7 @@ static mfn_t get_cr3_mfn(cr3_t *cr3_entry) { BUG_ON(!frame); cr3_entry->mfn = frame->mfn; - cr3_mapped = init_map_mfn(cr3_entry->mfn); + cr3_mapped = tmp_map_mfn(cr3_entry->mfn); memset(cr3_mapped, 0, PAGE_SIZE); } @@ -154,7 +152,7 @@ static mfn_t get_pgentry_mfn(mfn_t tab_mfn, pt_index_t index, unsigned long flag BUG_ON(mfn_invalid(tab_mfn)); - tab = init_map_mfn(tab_mfn); + tab = tmp_map_mfn(tab_mfn); entry = &tab[index]; mfn = mfn_from_pgentry(*entry); @@ -162,9 +160,9 @@ static mfn_t get_pgentry_mfn(mfn_t tab_mfn, pt_index_t index, unsigned long flag frame_t *frame = get_free_frame(); BUG_ON(!frame); - set_pgentry(entry, frame->mfn, flags); - mfn = mfn_from_pgentry(*entry); - tab = init_map_mfn(mfn); + mfn = frame->mfn; + set_pgentry(entry, mfn, flags); + tab = tmp_map_mfn(mfn); memset(tab, 0, PAGE_SIZE); } else { @@ -199,26 +197,29 @@ static void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, #endif if (order == PAGE_ORDER_1G) { - tab = init_map_mfn(l3t_mfn); + tab = tmp_map_mfn(l3t_mfn); entry = &tab[l3_table_index(va)]; set_pgentry(entry, mfn, l3_flags | _PAGE_PSE); + invlpg(va); goto done; } l2t_mfn = get_pgentry_mfn(l3t_mfn, l3_table_index(va), l3_flags); if (order == PAGE_ORDER_2M) { - tab = init_map_mfn(l2t_mfn); + tab = tmp_map_mfn(l2t_mfn); entry = &tab[l2_table_index(va)]; set_pgentry(entry, mfn, l2_flags | _PAGE_PSE); + invlpg(va); goto done; } l1t_mfn = get_pgentry_mfn(l2t_mfn, l2_table_index(va), l2_flags); - tab = init_map_mfn(l1t_mfn); + tab = tmp_map_mfn(l1t_mfn); entry = &tab[l1_table_index(va)]; set_pgentry(entry, mfn, l1_flags); + invlpg(va); done: spin_unlock(&lock); @@ -246,7 +247,28 @@ void *vmap_user(void *va, mfn_t mfn, unsigned int order, l1_flags); } +static inline void init_tmp_mapping(void) { + pte_t *tab = get_l1_table(_tmp_mapping); + _tmp_mapping_entry = (pgentry_t *) l1_table_entry(tab, _tmp_mapping); + BUG_ON(!_tmp_mapping_entry); +} + +static void map_tmp_mapping_entry(void) { + pml4_t *l3e = l4_table_entry(mfn_to_virt(cr3.mfn), _tmp_mapping); + pdpe_t *l2e = l3_table_entry(mfn_to_virt(l3e->mfn), _tmp_mapping); + pde_t *l1e = l2_table_entry(mfn_to_virt(l2e->mfn), _tmp_mapping); + pte_t *entry = l1_table_entry(mfn_to_virt(l1e->mfn), _tmp_mapping); + + /* Map _tmp_mapping_entry PTE of new page tables */ + kmap_4k(l1e->mfn, L1_PROT); + + /* Point _tmp_mapping_entry at new page tables location */ + _tmp_mapping_entry = paddr_to_virt_kern(_paddr(entry)); +} + void init_pagetables(void) { + init_tmp_mapping(); + for_each_memory_range (r) { switch (r->base) { case VIRT_IDENT_BASE: @@ -270,5 +292,9 @@ void init_pagetables(void) { } } - map_used_memory(); + map_frames_array(); + map_multiboot_areas(); + map_tmp_mapping_entry(); + + write_cr3(cr3.paddr); } diff --git a/common/setup.c b/common/setup.c index 5dcbb1a7..ae0ec70e 100644 --- a/common/setup.c +++ b/common/setup.c @@ -222,17 +222,15 @@ void __noreturn __text_init kernel_start(uint32_t multiboot_magic, unsigned long /* Setup final pagetables */ init_pagetables(); - - map_multiboot_areas(); - map_bios_area(); - - write_cr3(cr3.paddr); boot_flags.virt = true; WRITE_SP(get_free_pages_top(PAGE_ORDER_2M, GFP_KERNEL_MAP)); + if (opt_debug) dump_pagetables(cr3); + map_bios_area(); + if (setup_framebuffer()) display_banner(); else diff --git a/include/arch/x86/pagetable.h b/include/arch/x86/pagetable.h index 1a4ad11a..b67ab009 100644 --- a/include/arch/x86/pagetable.h +++ b/include/arch/x86/pagetable.h @@ -272,7 +272,6 @@ static inline pdpe_t *get_pdpe(const void *va) { static inline void set_pgentry(pgentry_t *e, mfn_t mfn, unsigned long flags) { *e = pgentry_from_mfn(mfn, flags); barrier(); - flush_tlb(); } /* External declarations */ diff --git a/include/lib.h b/include/lib.h index 172f17e1..21e0a87e 100644 --- a/include/lib.h +++ b/include/lib.h @@ -324,6 +324,10 @@ static inline void str(unsigned int *selector) { asm volatile("str %0" : "=m"(*selector)); } +static inline void invlpg(void *addr) { + asm volatile("invlpg (%0)" ::"r"(addr) : "memory"); +} + static inline void flush_tlb(void) { write_cr3(read_cr3()); } diff --git a/include/mm/pmm.h b/include/mm/pmm.h index 0f30a4d6..ae62f955 100644 --- a/include/mm/pmm.h +++ b/include/mm/pmm.h @@ -77,6 +77,7 @@ extern void put_free_frames(mfn_t mfn, unsigned int order); extern void reclaim_frame(mfn_t mfn, unsigned int order); extern void map_used_memory(void); +extern void map_frames_array(void); /* Static definitions */ diff --git a/mm/pmm.c b/mm/pmm.c index 8c712a6e..26bf1eb6 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -570,3 +570,9 @@ void map_used_memory(void) { } } } + +void map_frames_array(void) { + frames_array_t *array; + + list_for_each_entry (array, &frames, list) { kmap_4k(virt_to_mfn(array), L1_PROT); } +}