Add dedicated mechanism for page tables mapping/unmapping

arch/x86/pagetables.c

@@ -121,11 +121,6 @@ static void dump_pagetable(mfn_t table, int level) {
 }
 
 void dump_pagetables(cr3_t *cr3_ptr) {
-#if defined(__x86_64__)
-    int level = 4;
-#else
-    int level = 3;
-#endif
     ASSERT(cr3_ptr);
     if (mfn_invalid(cr3_ptr->mfn)) {
         warning("CR3: 0x%lx is invalid", cr3.paddr);
@@ -134,18 +129,14 @@ void dump_pagetables(cr3_t *cr3_ptr) {
 
     printk("Page Tables: CR3 paddr: 0x%lx\n", cr3.paddr);
     spin_lock(&vmap_lock);
-    dump_pagetable(cr3_ptr->mfn, level);
+    dump_pagetable(cr3_ptr->mfn, PT_LEVELS);
     spin_unlock(&vmap_lock);
 }
 
 static void dump_pagetable_va(cr3_t *cr3_ptr, void *va) {
     paddr_t tab_paddr;
     pgentry_t *tab;
-#if defined(__x86_64__)
-    int level = 4;
-#else
-    int level = 3;
-#endif
+    int level = PT_LEVELS;
 
     ASSERT(cr3_ptr);
     if (mfn_invalid(cr3_ptr->mfn)) {
@@ -551,6 +542,244 @@ static inline void init_cr3(cr3_t *cr3_ptr) {
     cr3_ptr->mfn = MFN_INVALID;
 }
 
+static inline bool is_huge_page_level(int level) {
+    if (level == 2)
+        return true;
+#if defined(__i386__)
+    return false;
+#else
+    return level == 3;
+#endif
+}
+
+static void map_pagetable(cr3_t *cr3_ptr, mfn_t table, int level) {
+    void *va = mfn_to_virt_kern(table);
+    pte_t *pt;
+
+    pt = _vmap(cr3_ptr, va, table, PAGE_ORDER_4K, L4_PROT, L3_PROT, L2_PROT, L1_PROT);
+    BUG_ON(!pt);
+
+    for (int i = 0; i < level_to_entries(level) && level > 1; i++) {
+        if (mfn_invalid(pt[i].mfn))
+            continue;
+
+        /* Do not map 2MB/1GB large pages */
+        if (!is_huge_page_level(level) || !is_pgentry_huge(pt[i].entry))
+            map_pagetable(cr3_ptr, pt[i].mfn, level - 1);
+    }
+}
+
+void map_pagetables(cr3_t *to_cr3, cr3_t *from_cr3) {
+    ASSERT(to_cr3);
+    if (mfn_invalid(to_cr3->mfn)) {
+        warning("Target CR3: 0x%lx is invalid", to_cr3->paddr);
+        return;
+    }
+
+    if (!from_cr3)
+        from_cr3 = to_cr3;
+    else if (mfn_invalid(from_cr3->mfn)) {
+        warning("Source CR3: 0x%lx is invalid", from_cr3->paddr);
+        return;
+    }
+
+    dprintk("Mapping all page tables of CR3: 0x%lx to CR3: 0x%lx\n", from_cr3->paddr,
+            to_cr3->paddr);
+
+    spin_lock(&vmap_lock);
+    /* Assume PML4 is not mapped */
+    map_pagetable(to_cr3, from_cr3->mfn, PT_LEVELS);
+    spin_unlock(&vmap_lock);
+}
+
+static void unmap_pagetable(cr3_t *cr3_ptr, mfn_t table, int level) {
+    mfn_t tmp_entry_mfn = virt_to_mfn(_tmp_mapping_entry);
+    pte_t *pt = mfn_to_virt_kern(table);
+
+    for (int i = 0; i < level_to_entries(level) && level > 1; i++) {
+        if (mfn_invalid(pt[i].mfn))
+            continue;
+
+        /* Do not touch the tmp_mapping entry! */
+        if (pt[i].mfn == tmp_entry_mfn)
+            continue;
+
+        /* Do not touch 2MB/1GB large pages */
+        if (!is_huge_page_level(level) || !is_pgentry_huge(pt[i].entry))
+            unmap_pagetable(cr3_ptr, pt[i].mfn, level - 1);
+    }
+    _vunmap(cr3_ptr, pt, NULL, NULL);
+}
+
+void unmap_pagetables(cr3_t *from_cr3, cr3_t *of_cr3) {
+    ASSERT(from_cr3);
+    if (mfn_invalid(from_cr3->mfn)) {
+        warning("Target CR3: 0x%lx is invalid", from_cr3->paddr);
+        return;
+    }
+
+    if (!of_cr3)
+        of_cr3 = from_cr3;
+    else if (mfn_invalid(of_cr3->mfn)) {
+        warning("Source CR3: 0x%lx is invalid", of_cr3->paddr);
+        return;
+    }
+
+    dprintk("Unmapping all page tables of CR3: 0x%lx from CR3: 0x%lx\n", of_cr3->paddr,
+            from_cr3->paddr);
+
+    spin_lock(&vmap_lock);
+    /* Assume PML4 is mapped */
+    unmap_pagetable(from_cr3, of_cr3->mfn, PT_LEVELS);
+    spin_unlock(&vmap_lock);
+}
+
+int map_pagetables_va(cr3_t *cr3_ptr, void *va) {
+    pgentry_t *tab;
+    int err = -EINVAL;
+
+    ASSERT(cr3_ptr);
+    if (mfn_invalid(cr3_ptr->mfn)) {
+        warning("CR3: 0x%lx is invalid", cr3_ptr->paddr);
+        return err;
+    }
+
+    if (!is_canon_va(va)) {
+        warning("Virtual address 0x%p is not canonical", va);
+        return err;
+    }
+
+    err = -EFAULT;
+    spin_lock(&vmap_lock);
+    tab = _vmap(cr3_ptr, mfn_to_virt_kern(cr3_ptr->mfn), cr3_ptr->mfn, PAGE_ORDER_4K,
+                L4_PROT, L3_PROT, L2_PROT, L1_PROT);
+    if (!tab)
+        goto unlock;
+
+#if defined(__x86_64__)
+    pml4_t *l4e = l4_table_entry((pml4_t *) tab, va);
+    if (mfn_invalid(l4e->mfn)) {
+        err = -ENOENT;
+        goto unlock;
+    }
+
+    tab = _vmap(cr3_ptr, mfn_to_virt_kern(l4e->mfn), l4e->mfn, PAGE_ORDER_4K, L4_PROT,
+                L3_PROT, L2_PROT, L1_PROT);
+    if (!tab)
+        goto unlock;
+#endif
+
+    pdpe_t *l3e = l3_table_entry((pdpe_t *) tab, va);
+    if (mfn_invalid(l3e->mfn)) {
+        err = -ENOENT;
+        goto unlock;
+    }
+
+    if (l3e->PS)
+        goto done;
+
+    tab = _vmap(cr3_ptr, mfn_to_virt_kern(l3e->mfn), l3e->mfn, PAGE_ORDER_4K, L4_PROT,
+                L3_PROT, L2_PROT, L1_PROT);
+    if (!tab)
+        goto unlock;
+
+    pde_t *l2e = l2_table_entry((pde_t *) tab, va);
+    if (mfn_invalid(l2e->mfn)) {
+        err = -ENOENT;
+        goto unlock;
+    }
+
+    if (l2e->PS)
+        goto done;
+
+    tab = _vmap(cr3_ptr, mfn_to_virt_kern(l2e->mfn), l2e->mfn, PAGE_ORDER_4K, L4_PROT,
+                L3_PROT, L2_PROT, L1_PROT);
+    if (!tab)
+        goto unlock;
+
+done:
+    err = 0;
+unlock:
+    spin_unlock(&vmap_lock);
+    return err;
+}
+
+int unmap_pagetables_va(cr3_t *cr3_ptr, void *va) {
+    mfn_t tmp_entry_mfn = virt_to_mfn(_tmp_mapping_entry);
+    pgentry_t *tab, *tables[PT_LEVELS] = {NULL};
+    int level = 0;
+    int err = -EINVAL;
+
+    ASSERT(cr3_ptr);
+    if (mfn_invalid(cr3_ptr->mfn)) {
+        warning("CR3: 0x%lx is invalid", cr3_ptr->paddr);
+        return err;
+    }
+
+    if (!is_canon_va(va)) {
+        warning("Virtual address 0x%p is not canonical", va);
+        return err;
+    }
+
+    err = -EFAULT;
+    spin_lock(&vmap_lock);
+    tab = _vmap(cr3_ptr, mfn_to_virt_kern(cr3_ptr->mfn), cr3_ptr->mfn, PAGE_ORDER_4K,
+                L4_PROT, L3_PROT, L2_PROT, L1_PROT);
+    if (!tab)
+        goto cleanup;
+    tables[level++] = tab;
+
+#if defined(__x86_64__)
+    pml4_t *l4e = l4_table_entry((pml4_t *) tab, va);
+    if (mfn_invalid(l4e->mfn)) {
+        err = -ENOENT;
+        goto cleanup;
+    }
+
+    tab = _vmap(cr3_ptr, mfn_to_virt_kern(l4e->mfn), l4e->mfn, PAGE_ORDER_4K, L4_PROT,
+                L3_PROT, L2_PROT, L1_PROT);
+    if (!tab)
+        goto cleanup;
+    tables[level++] = tab;
+#endif
+
+    pdpe_t *l3e = l3_table_entry((pdpe_t *) tab, va);
+    if (mfn_invalid(l3e->mfn)) {
+        err = -ENOENT;
+        goto cleanup;
+    }
+
+    if (l3e->PS)
+        goto done;
+
+    tab = _vmap(cr3_ptr, mfn_to_virt_kern(l3e->mfn), l3e->mfn, PAGE_ORDER_4K, L4_PROT,
+                L3_PROT, L2_PROT, L1_PROT);
+    if (!tab)
+        goto cleanup;
+    tables[level++] = tab;
+
+    pde_t *l2e = l2_table_entry((pde_t *) tab, va);
+    if (mfn_invalid(l2e->mfn)) {
+        err = -ENOENT;
+        goto cleanup;
+    }
+
+    if (l2e->PS)
+        goto done;
+
+    /* Do not touch the tmp_mapping entry! */
+    if (l2e->mfn != tmp_entry_mfn)
+        tables[level] = mfn_to_virt_kern(l2e->mfn);
+
+done:
+    err = 0;
+cleanup:
+    for (unsigned i = 0; i < ARRAY_SIZE(tables) && tables[i]; i++)
+        _vunmap(cr3_ptr, tables[i], NULL, NULL);
+    spin_unlock(&vmap_lock);
+    return err;
+}
+
 void init_pagetables(void) {
     init_cr3(&cr3);
     init_cr3(&user_cr3);

include/arch/x86/page.h

@@ -158,6 +158,12 @@ typedef unsigned long mfn_t;
 #define VA_BITS        32
 #endif
 
+#ifdef __i386__
+#define PT_LEVELS 3 /* assumes PAE */
+#else
+#define PT_LEVELS (la57_enabled() ? 5 : 4)
+#endif
+
 #define _paddr(addr) ((paddr_t) _ul(addr))
 
 #define PADDR_INVALID (~0x0UL)

include/arch/x86/pagetable.h

@@ -284,6 +284,14 @@ static inline void set_pgentry(pgentry_t *e, mfn_t mfn, unsigned long flags) {
     barrier();
 }
 
+static inline bool is_pgentry_huge(pgentry_t e) {
+    return !!(e & _PAGE_PSE);
+}
+
+static inline bool is_pgentry_present(pgentry_t e) {
+    return !!(e & _PAGE_PRESENT);
+}
+
 /* External declarations */
 
 extern pte_t l1_pt_entries1[L1_PT_ENTRIES];
@@ -307,6 +315,11 @@ extern int get_user_va_mfn_order(void *va, mfn_t *mfn, unsigned int *order);
 extern frame_t *find_kern_va_frame(const void *va);
 extern frame_t *find_user_va_frame(const void *va);
 
+extern void map_pagetables(cr3_t *to_cr3, cr3_t *from_cr3);
+extern void unmap_pagetables(cr3_t *from_cr3, cr3_t *of_cr3);
+extern int map_pagetables_va(cr3_t *cr3_ptr, void *va);
+extern int unmap_pagetables_va(cr3_t *cr3_ptr, void *va);
+
 #endif /* __ASSEMBLY__ */
 
 #endif /* KTF_PAGETABLE_H */

include/mm/pmm.h

@@ -31,7 +31,7 @@
 #include <mm/regions.h>
 
 struct frame_flags {
-    uint16_t : 11, mapped : 1, uncacheable : 1, free : 1, pagetable : 1;
+    uint16_t : 12, uncacheable : 1, free : 1, pagetable : 1;
 };
 typedef struct frame_flags frame_flags_t;
 
@@ -83,7 +83,6 @@ extern frame_t *find_free_paddr_frame(paddr_t paddr);
 extern frame_t *find_busy_paddr_frame(paddr_t paddr);
 extern frame_t *find_paddr_frame(paddr_t paddr);
 
-extern void map_used_memory(void);
 extern void map_frames_array(void);
 
 /* Static definitions */

mm/pmm.c

@@ -638,19 +638,6 @@ void put_free_frames(mfn_t mfn, unsigned int order) {
     spin_unlock(&lock);
 }
 
-void map_used_memory(void) {
-    frame_t *frame;
-
-    for_each_order (order) {
-        list_for_each_entry (frame, &busy_frames[order], list) {
-            if (!frame->flags.mapped) {
-                kmap(frame->mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT);
-                frame->flags.mapped = true;
-            }
-        }
-    }
-}
-
 void map_frames_array(void) {
     frames_array_t *array;
 

tests/unittests.c

@@ -190,6 +190,18 @@ int unit_tests(void *_unused) {
     cpu_freq_expect("Prototyp Amazing Foo One @ 1GHz", 1000000000);
     cpu_freq_expect("Prototyp Amazing Foo Two @ 1.00GHz", 1000000000);
 
+    map_pagetables(&cr3, NULL);
+    map_pagetables(&cr3, &user_cr3);
+    pte_t *pte = get_pte(unit_tests);
+    printk("PTE: 0x%lx\n", pte->entry);
+    unmap_pagetables(&cr3, NULL);
+    unmap_pagetables(&cr3, &user_cr3);
+
+    map_pagetables_va(&cr3, unit_tests);
+    pte_t *pte2 = get_pte(unit_tests);
+    printk("PTE: 0x%lx\n", pte2->entry);
+    unmap_pagetables_va(&cr3, unit_tests);
+
     task_t *task1, *task2, *task_user1, *task_user1_se, *task_user1_int80, *task_user2,
         *task_user3, *task_user4;