diff --git a/.clang-format b/.clang-format index 0cbe2ddf1..3675a83a5 100644 --- a/.clang-format +++ b/.clang-format @@ -15,7 +15,8 @@ AllowShortLoopsOnASingleLine: false DerivePointerAlignment: true PointerAlignment: Right ColumnLimit: 100 -ForEachMacros: ['list_for_every','list_for_every_safe','list_for_every_rcu','mt_for_each'] +ForEachMacros: ['list_for_every','list_for_every_safe','list_for_every_rcu','mt_for_each','list_for_each_entry', + 'list_for_each_entry_safe', 'list_for_each_entry_rcu'] IncludeBlocks: Regroup IndentExternBlock: NoIndent IncludeCategories: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f50f5661d..a04bed73d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,7 +28,7 @@ jobs: - name: Download the x86_64-onyx-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: x86_64-onyx-linux @@ -54,19 +54,19 @@ jobs: ./scripts/create_disk_image.sh disk-image.img --bootable efi - name: Upload a Build Artifact(Onyx.iso) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx ISO path: Onyx.iso - name: Upload a Build Artifact(kernel/vmonyx) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: vmonyx path: kernel/vmonyx - name: Upload a Build Artifact(Disk image) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Disk-image.img path: disk-image.img @@ -89,7 +89,7 @@ jobs: # Runs a single command using the runners shell - name: Download the onyx-llvm-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: onyx-llvm-linux @@ -115,19 +115,19 @@ jobs: ./scripts/create_disk_image.sh disk-image-llvm.img --bootable efi - name: Upload a Build Artifact(Onyx.iso) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx-iso-llvm path: Onyx.iso - name: Upload a Build Artifact(kernel/vmonyx) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: vmonyx-llvm path: kernel/vmonyx - name: Upload a Build Artifact(disk-image-llvm) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Disk-image-llvm.img path: disk-image-llvm.img @@ -147,7 +147,7 @@ jobs: - name: Download the riscv64-onyx-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: riscv64-onyx-linux @@ -176,7 +176,7 @@ jobs: zstd -T0 -15 riscv64-onyx-image.tar - name: Upload the riscv64 boot image - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx boot image (riscv64) path: riscv64-onyx-image.tar.zst @@ -199,7 +199,7 @@ jobs: # Runs a single command using the runners shell - name: Download the onyx-llvm-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: onyx-llvm-linux @@ -229,7 +229,7 @@ jobs: zstd -T0 -15 riscv64-onyx-image-llvm.tar - name: Upload the riscv64 boot image (LLVM) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx boot image (riscv64) (llvm) path: riscv64-onyx-image-llvm.tar.zst @@ -295,7 +295,7 @@ jobs: zstd -T0 -15 minimal-sysroot-${{ matrix.target_arch }}.tar - name: Upload the minimal sysroot - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: minimal-sysroot-${{ matrix.target_arch }} path: minimal-sysroot-${{ matrix.target_arch }}.tar.zst @@ -322,7 +322,7 @@ jobs: submodules: "recursive" - name: Download the minimal sysroot - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 - name: Extract minimal sysroot run: | @@ -413,7 +413,7 @@ jobs: mv ${{ env.toolchain_id_no_os }} toolchain_binaries-${{ matrix.toolchain }} - name: Upload the toolchain - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: ${{ env.toolchain_id }} path: ${{ env.toolchain_id }}.tar.zst @@ -434,7 +434,7 @@ jobs: - name: Download the arm64-onyx-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: arm64-onyx-linux @@ -463,7 +463,7 @@ jobs: zstd -T0 -15 arm64-onyx-image.tar - name: Upload the arm64 boot image - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx boot image (arm64) path: arm64-onyx-image.tar.zst @@ -487,7 +487,7 @@ jobs: # Runs a single command using the runners shell - name: Download the onyx-llvm-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: onyx-llvm-linux @@ -517,7 +517,7 @@ jobs: zstd -T0 -15 arm64-onyx-image-llvm.tar - name: Upload the arm64 boot image (LLVM) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx boot image (arm64) (llvm) path: arm64-onyx-image-llvm.tar.zst @@ -537,7 +537,7 @@ jobs: - name: Download the x86_64-onyx-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: x86_64-onyx-linux @@ -582,7 +582,7 @@ jobs: # Runs a single command using the runners shell - name: Download the onyx-llvm-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: onyx-llvm-linux @@ -625,7 +625,7 @@ jobs: - name: Download the riscv64-onyx-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: riscv64-onyx-linux @@ -669,7 +669,7 @@ jobs: # Runs a single command using the runners shell - name: Download the onyx-llvm-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: onyx-llvm-linux diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 64d238f66..05a421121 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -24,7 +24,7 @@ jobs: python-version: '3.11.0' # Require the same version of python as Onyx builds, in order to ease the build - name: Download the x86_64-onyx-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: x86_64-onyx-linux @@ -51,19 +51,19 @@ jobs: ./scripts/create_disk_image.sh disk-image.img --bootable efi - name: Upload a Build Artifact(Onyx.iso) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx ISO path: Onyx.iso - name: Upload a Build Artifact(kernel/vmonyx) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: vmonyx path: kernel/vmonyx - name: Upload a Build Artifact(Disk image) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Disk-image.img path: disk-image.img @@ -86,7 +86,7 @@ jobs: # Runs a single command using the runners shell - name: Download the onyx-llvm-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: onyx-llvm-linux @@ -113,19 +113,19 @@ jobs: ./scripts/create_disk_image.sh disk-image-llvm.img --bootable efi - name: Upload a Build Artifact(Onyx.iso) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx-iso-llvm path: Onyx.iso - name: Upload a Build Artifact(kernel/vmonyx) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: vmonyx-llvm path: kernel/vmonyx - name: Upload a Build Artifact(disk-image-llvm) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Disk-image-llvm.img path: disk-image-llvm.img @@ -145,7 +145,7 @@ jobs: - name: Download the riscv64-onyx-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: riscv64-onyx-linux @@ -174,7 +174,7 @@ jobs: zstd -T0 -15 riscv64-onyx-image.tar - name: Upload the riscv64 boot image - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx boot image (riscv64) path: riscv64-onyx-image.tar.zst @@ -197,7 +197,7 @@ jobs: # Runs a single command using the runners shell - name: Download the onyx-llvm-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: onyx-llvm-linux @@ -227,7 +227,7 @@ jobs: zstd -T0 -15 riscv64-onyx-image-llvm.tar - name: Upload the riscv64 boot image (LLVM) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx boot image (riscv64) (llvm) path: riscv64-onyx-image-llvm.tar.zst @@ -292,7 +292,7 @@ jobs: zstd -T0 -15 minimal-sysroot-${{ matrix.target_arch }}.tar - name: Upload the minimal sysroot - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: minimal-sysroot-${{ matrix.target_arch }} path: minimal-sysroot-${{ matrix.target_arch }}.tar.zst @@ -319,7 +319,7 @@ jobs: submodules: "recursive" - name: Download the minimal sysroot - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 - name: Extract minimal sysroot run: | @@ -410,7 +410,7 @@ jobs: mv ${{ env.toolchain_id_no_os }} toolchain_binaries-${{ matrix.toolchain }} - name: Upload the toolchain - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: ${{ env.toolchain_id }} path: ${{ env.toolchain_id }}.tar.zst @@ -430,7 +430,7 @@ jobs: - name: Download the arm64-onyx-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: arm64-onyx-linux @@ -459,7 +459,7 @@ jobs: zstd -T0 -15 arm64-onyx-image.tar - name: Upload the arm64 boot image - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx boot image (arm64) path: arm64-onyx-image.tar.zst @@ -483,7 +483,7 @@ jobs: # Runs a single command using the runners shell - name: Download the onyx-llvm-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: onyx-llvm-linux @@ -513,7 +513,7 @@ jobs: zstd -T0 -15 arm64-onyx-image-llvm.tar - name: Upload the arm64 boot image (LLVM) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx boot image (arm64) (llvm) path: arm64-onyx-image-llvm.tar.zst diff --git a/kernel/Makefile b/kernel/Makefile index ac4209406..2aca9944b 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -31,7 +31,7 @@ CFLAGS:=$(CFLAGS) -MMD -ffreestanding -fbuiltin -Wall -Wextra -fstack-protector- ifneq ($(ONYX_USING_CLANG), yes) # TODO: Find suitable alternatives for clang -CFLAGS:=$(CFLAGS) -Wno-format-truncation -Wshadow-compatible-local +CFLAGS:=$(CFLAGS) -Wno-format-truncation -Wshadow-compatible-local -Wno-narrowing # Note: We need to define _GLIBCXX_INCLUDE_NEXT_C_HEADERS so the shenanigans in the # compiler (in this case, only GCC)'s stdlib.h C++ wrappers get bypassed. If we let stdlib.h include cstdlib, diff --git a/kernel/arch/riscv64/syscall_table.json b/kernel/arch/riscv64/syscall_table.json index a309eca71..0d17049a1 100644 --- a/kernel/arch/riscv64/syscall_table.json +++ b/kernel/arch/riscv64/syscall_table.json @@ -2795,5 +2795,26 @@ ] ], "return_type": "int" + }, + { + "name": "madvise", + "nr": 160, + "nr_args": 3, + "args": [ + [ + "void *", + "addr" + ], + [ + "size_t", + "length" + ], + [ + "int", + "advice" + ] + ], + "return_type": "int", + "abi": "c" } ] diff --git a/kernel/arch/x86_64/boot.S b/kernel/arch/x86_64/boot.S index 9e507bc14..80edde080 100644 --- a/kernel/arch/x86_64/boot.S +++ b/kernel/arch/x86_64/boot.S @@ -240,8 +240,6 @@ ENTRY(efi_entry_multiboot2_64) cmove 8(%rdi), %rdx cmpl $MULTIBOOT_TAG_TYPE_EFI64_IH, (%rdi) cmove 8(%rdi), %rcx - cmpl $MULTIBOOT_TAG_TYPE_LOAD_BASE_ADDR, (%rdi) - cmovel 8(%rdi), %esi /* Increment the pointer by size (rdi + 4) */ addl 4(%rdi), %edi @@ -260,22 +258,34 @@ ENTRY(efi_entry_multiboot2_64) push %rdx mov %rcx, %rdi - xchg %rsi, %rdx - sub $PHYS_BASE, %rdx + lea efi_entry_multiboot2_64(%rip), %rcx + movabs $efi_entry_multiboot2_64, %rsi + movabs $KERNEL_VIRTUAL_BASE, %rdx + sub %rdx, %rsi + sub %rsi, %rcx + mov %rcx, %rdx + lea kernel_phys_offset(%rip), %r9 + mov %rdx, (%r9) lea efi_state(%rip), %rcx mov $pml4, %r8 add %rdx, %r8 - lea kernel_phys_offset(%rip), %r9 - mov %rdx, (%r9) + mov (%rsp), %rsi + push %r8 call efi_handoff + pop %rdi + pop %r12 + lea x86_stack_top(%rip), %rsp + call efi_switch_mmu + call x86_efi_switch_tables + /* We're in pure 64-bit mode, paging enabled, with LA57 if it exists, ints off */ mov $x86_start, %rax mov $efi_entry_mb2, %rdx /* First arg = multiboot2 info */ mov %rbx, %rdi /* Second arg = EFI_SYSTEM_TABLE */ - pop %rsi + mov %r12, %rsi jmp *%rax 99: hlt @@ -295,9 +305,6 @@ END(efi_entry_multiboot2_64) ENTRY(x86_efi_switch_tables) /* We're still running in identity mode */ lea efi_gdtr(%rip), %rax - /* Fixup the gdtr's gdt address */ - lea efi_gdt_begin(%rip), %rdi - mov %rdi, 2(%rax) lgdt (%rax) mov $KERNEL_DS, %ax diff --git a/kernel/arch/x86_64/efistub/early.cpp b/kernel/arch/x86_64/efistub/early.cpp index a46565dca..2a02c4500 100644 --- a/kernel/arch/x86_64/efistub/early.cpp +++ b/kernel/arch/x86_64/efistub/early.cpp @@ -171,7 +171,7 @@ BOOT_SECTION static void setup_mmu(PML *page_tables, unsigned long phys_base, extern "C" void x86_efi_enable_57_mmu(PML *); -static void efi_switch_mmu(PML *page_tables) +extern "C" void efi_switch_mmu(PML *page_tables) { auto cr4 = x86_read_cr4(); // Note: Some people in Tianocore thought it's a brilliant idea to break backwards compatibility @@ -258,6 +258,4 @@ extern "C" BOOT_SECTION void efi_handoff(EFI_HANDLE image_handle, EFI_SYSTEM_TAB __asm__ __volatile__("int3"); __asm__ __volatile__("cli"); - x86_efi_switch_tables(); - efi_switch_mmu(page_tables); } diff --git a/kernel/arch/x86_64/isr.cpp b/kernel/arch/x86_64/isr.cpp index 8301650ba..24b8f31f5 100644 --- a/kernel/arch/x86_64/isr.cpp +++ b/kernel/arch/x86_64/isr.cpp @@ -223,6 +223,8 @@ void stack_segment_fault(struct registers *ctx) } #ifdef CONFIG_VERBOSE_SEGV +#undef REQUIRES_SHARED +#define REQUIRES_SHARED(...) vm_area_struct *vm_search(struct mm_address_space *mm, void *addr, size_t length) REQUIRES_SHARED(mm->vm_lock); @@ -236,7 +238,7 @@ static void attempt_map_pointer(unsigned long word) size_t pos = 0; struct mm_address_space *mm = get_current_address_space(); - scoped_mutex g{mm->vm_lock}; + scoped_rwlock g{mm->vm_lock}; // Lets try to "symbolize" it struct vm_area_struct *vm = vm_search(mm, (void *) word, 1); if (vm) @@ -658,7 +660,7 @@ void print_int_stacks() /* User or corrupted stack, skip */ pr_emerg(" (%s stack skipped, bad stack)\n", type); tps = next; - continue; + break; } if (is_trap) diff --git a/kernel/arch/x86_64/mmu.cpp b/kernel/arch/x86_64/mmu.cpp index 69230ea39..2f9833211 100644 --- a/kernel/arch/x86_64/mmu.cpp +++ b/kernel/arch/x86_64/mmu.cpp @@ -1092,4 +1092,132 @@ int mmu_map_kasan_shadow(void *shadow_start, size_t pages) return 0; } +static void pte_kasan_range(pte_t *pte, unsigned long start, unsigned long end) +{ + unsigned long zero_shadow = VA2PA(zero_shadow_map); + unsigned long next_start; + for (; start < end; pte++, start = next_start) + { + next_start = min(pte_addr_end(start), end); + pte_t old = *pte; + if (pte_addr(old) != zero_shadow) + { + CHECK(pte_write(old)); + continue; + } + + void *shadow = alloc_boot_page(1, 0); + memset(PHYS_TO_VIRT(shadow), 0, PAGE_SIZE); + set_pte(pte, pte_mkpte((u64) shadow, + __pgprot(_PAGE_WRITE | _PAGE_PRESENT | _PAGE_NX | _PAGE_GLOBAL))); + } +} + +#define SHADOW(type, hwtype) (VA2PA(shadow_##hwtype)) + +static void pmd_kasan_range(pmd_t *pmd, unsigned long start, unsigned long end) +{ + pte_t *pte; + unsigned long next_start; + for (; start < end; pmd++, start = next_start) + { + next_start = min(pmd_addr_end(start), end); + if (pmd_none(*pmd)) + continue; + DCHECK(!pmd_huge(*pmd)); + pte = pte_offset(pmd, start); + if (pmd_addr(*pmd) == SHADOW(pte, pt)) + { + pte_t *newpte = (pte_t *) alloc_boot_page(1, 0); + memcpy(PHYS_TO_VIRT(newpte), shadow_pt, PAGE_SIZE); + set_pmd(pmd, pmd_mkpmd((unsigned long) newpte, __pgprot(KERNEL_PGTBL))); + pte = pte_offset(pmd, start); + } + + CHECK(pmd_val(*pmd) & (_PAGE_WRITE | _PAGE_PRESENT)); + pte_kasan_range(pte, start, next_start); + } +} + +static void pud_kasan_range(pud_t *pud, unsigned long start, unsigned long end) +{ + pmd_t *pmd; + unsigned long next_start; + for (; start < end; pud++, start = next_start) + { + next_start = min(pud_addr_end(start), end); + if (pud_none(*pud)) + continue; + DCHECK(!pud_huge(*pud)); + pmd = pmd_offset(pud, start); + if (pud_addr(*pud) == SHADOW(pmd, pd)) + { + pmd_t *newpmd = (pmd_t *) alloc_boot_page(1, 0); + memcpy(PHYS_TO_VIRT(newpmd), shadow_pd, PAGE_SIZE); + set_pud(pud, pud_mkpud((unsigned long) newpmd, __pgprot(KERNEL_PGTBL))); + pmd = pmd_offset(pud, start); + } + + CHECK(pud_val(*pud) & (_PAGE_WRITE | _PAGE_PRESENT)); + pmd_kasan_range(pmd, start, next_start); + } +} + +static void p4d_kasan_range(p4d_t *p4d, unsigned long start, unsigned long end) +{ + pud_t *pud; + unsigned long next_start; + for (; start < end; p4d++, start = next_start) + { + next_start = min(p4d_addr_end(start), end); + if (WARN_ON(p4d_none(*p4d))) + continue; + DCHECK(!p4d_huge(*p4d)); + pud = pud_offset(p4d, start); + if (p4d_addr(*p4d) == SHADOW(pud, pdpt)) + { + pud_t *newpud = (pud_t *) alloc_boot_page(1, 0); + memcpy(PHYS_TO_VIRT(newpud), shadow_pdpt, PAGE_SIZE); + set_p4d(p4d, p4d_mkp4d((unsigned long) newpud, __pgprot(KERNEL_PGTBL))); + pud = pud_offset(p4d, start); + } + + CHECK(p4d_val(*p4d) & (_PAGE_WRITE | _PAGE_PRESENT)); + pud_kasan_range(pud, start, next_start); + } +} + +static void pgd_kasan_range(pgd_t *pgd, unsigned long start, unsigned long end) +{ + p4d_t *p4d; + unsigned long next_start; + for (; start < end; pgd++, start = next_start) + { + next_start = min(pgd_addr_end(start), end); + if (WARN_ON(pgd_none(*pgd))) + continue; + p4d = p4d_offset(pgd, start); + if (pml5_present() && pgd_addr(*pgd) == SHADOW(p4d, pml4)) + { + p4d_t *newp4d = (p4d_t *) alloc_boot_page(1, 0); + memcpy(PHYS_TO_VIRT(newp4d), shadow_pml4, PAGE_SIZE); + set_pgd(pgd, pgd_mkpgd((unsigned long) newp4d, __pgprot(KERNEL_PGTBL))); + p4d = p4d_offset(pgd, start); + } + + p4d_kasan_range(p4d, start, next_start); + } +} + +void init_shadow_for_phys(unsigned long addr, size_t len) +{ + unsigned long start = (unsigned long) kasan_get_ptr((unsigned long) PHYS_TO_VIRT(addr)); + unsigned long end = ALIGN_TO(start + (len >> 3), PAGE_SIZE); + pr_info("kasan: initializing shadow for [%lx, %lx] (%lx, %lx)\n", addr, addr + len - 1, start, + end - 1); + pgd_t *pgd = pgd_offset(&kernel_address_space, start); + pgd_kasan_range(pgd, start, end); + mmu_invalidate_range(start, (end - start) >> PAGE_SHIFT, &kernel_address_space); +} + #endif diff --git a/kernel/arch/x86_64/syscall_table.json b/kernel/arch/x86_64/syscall_table.json index 133deb966..4f00d9bb1 100644 --- a/kernel/arch/x86_64/syscall_table.json +++ b/kernel/arch/x86_64/syscall_table.json @@ -2839,5 +2839,26 @@ ] ], "return_type": "int" + }, + { + "name": "madvise", + "nr": 160, + "nr_args": 3, + "args": [ + [ + "void *", + "addr" + ], + [ + "size_t", + "length" + ], + [ + "int", + "advice" + ] + ], + "return_type": "int", + "abi": "c" } ] diff --git a/kernel/drivers/firmware/efi/efi.cpp b/kernel/drivers/firmware/efi/efi.cpp index 2e84db0ab..0034bd204 100644 --- a/kernel/drivers/firmware/efi/efi.cpp +++ b/kernel/drivers/firmware/efi/efi.cpp @@ -9,6 +9,7 @@ #include +#include #include #include #include @@ -89,7 +90,7 @@ unsigned long efi_memory_desc_flags_to_vm(uint64_t attributes) */ void efi_remap_efi_region(EFI_MEMORY_DESCRIPTOR &desc) { - pr_info("Remapping [%016lx, %016lx]\n", desc.PhysicalStart, + pr_warn("Remapping [%016lx, %016lx]\n", desc.PhysicalStart, desc.PhysicalStart + (desc.NumberOfPages << PAGE_SHIFT) - 1); bool mapping_over_null = desc.PhysicalStart == 0; @@ -137,6 +138,47 @@ static void efi_print_info() st->FirmwareRevision & 0xffff); } +static void efi_dump_mem_desc(const EFI_MEMORY_DESCRIPTOR *desc) +{ + pr_debug( + "Descriptor type %u physical start %lx virtual start %lx nr_pages %lx attributes %08lx\n", + desc->Type, desc->PhysicalStart, desc->VirtualStart, desc->NumberOfPages, desc->Attribute); +} + +#define EFI_RESERVED_TYPE 0 +#define EFI_LOADER_CODE 1 +#define EFI_LOADER_DATA 2 +#define EFI_BOOT_SERVICES_CODE 3 +#define EFI_BOOT_SERVICES_DATA 4 +#define EFI_RUNTIME_SERVICES_CODE 5 +#define EFI_RUNTIME_SERVICES_DATA 6 +#define EFI_CONVENTIONAL_MEMORY 7 +#define EFI_UNUSABLE_MEMORY 8 +#define EFI_ACPI_RECLAIM_MEMORY 9 +#define EFI_ACPI_MEMORY_NVS 10 +#define EFI_MEMORY_MAPPED_IO 11 +#define EFI_MEMORY_MAPPED_IO_PORT_SPACE 12 +#define EFI_PAL_CODE 13 +#define EFI_PERSISTENT_MEMORY 14 +#define EFI_UNACCEPTED_MEMORY 15 +#define EFI_MAX_MEMORY_TYPE 16 + +static bool should_map_efi(const EFI_MEMORY_DESCRIPTOR *desc) +{ + if (desc->Attribute & EFI_MEMORY_RUNTIME) + return true; + + /* Some runtime services like touching boot services memory on SVAM. Let's map it just for them, + * as a hack. */ + switch (desc->Type) + { + case EFI_BOOT_SERVICES_CODE: + case EFI_BOOT_SERVICES_DATA: + return true; + } + + return false; +} /** * @brief Initializes EFI * @@ -163,12 +205,24 @@ void efi_init(EFI_SYSTEM_TABLE *system_table, EFI_MEMORY_DESCRIPTOR *descriptors size_t nr_descriptors = mmap_size / descriptor_size; EFI_MEMORY_DESCRIPTOR *desc = descriptors; + EFI_MEMORY_DESCRIPTOR *map = NULL; + size_t nr_maps = 0; + for (size_t i = 0; i < nr_descriptors; desc = (EFI_MEMORY_DESCRIPTOR *) ((char *) desc + descriptor_size), i++) { - if (desc->Attribute & EFI_MEMORY_RUNTIME) +#ifdef CONFIG_EFI_DUMP_MEMMAP + efi_dump_mem_desc(desc); +#endif + if (should_map_efi(desc)) { + nr_maps++; + EFI_MEMORY_DESCRIPTOR *newmap = + (EFI_MEMORY_DESCRIPTOR *) kreallocarray(map, nr_maps, descriptor_size, GFP_ATOMIC); + CHECK(newmap != NULL); efi_remap_efi_region(*desc); + memcpy(&newmap[nr_maps - 1], desc, descriptor_size); + map = newmap; } } @@ -176,7 +230,7 @@ void efi_init(EFI_SYSTEM_TABLE *system_table, EFI_MEMORY_DESCRIPTOR *descriptors efi_guard g; EFI_STATUS st = g.system_table()->RuntimeServices->SetVirtualAddressMap( - mmap_size, descriptor_size, descriptor_version, descriptors); + nr_maps, descriptor_size, descriptor_version, map); assert(st == EFI_SUCCESS); } diff --git a/kernel/drivers/net/e1000/e1000.cpp b/kernel/drivers/net/e1000/e1000.cpp index 67a11d4a4..09efc913e 100644 --- a/kernel/drivers/net/e1000/e1000.cpp +++ b/kernel/drivers/net/e1000/e1000.cpp @@ -155,21 +155,20 @@ int e1000_pollrx(netif *nif) { e1000_device *dev = (e1000_device *) nif->priv; + bool found_one = false; uint16_t old_cur = 0; while ((dev->rx_descs[dev->rx_cur].status & RSTA_DD)) { auto &rxd = dev->rx_descs[dev->rx_cur]; - e1000_process_packet(nif, rxd); - dev->rx_descs[dev->rx_cur].status = 0; old_cur = dev->rx_cur; - dev->rx_cur = (dev->rx_cur + 1) % number_rx_desc; - - e1000_write(REG_RXDESCTAIL, old_cur, dev); + found_one = true; } + if (found_one) + e1000_write(REG_RXDESCTAIL, old_cur, dev); return 0; } @@ -608,7 +607,7 @@ unsigned int e1000_device::prepare_extended_descs(packetbuf *buf) if (!xmited) { buffer_start_off = (buf->data - (unsigned char *) buf->buffer_start); - length -= buffer_start_off; + length = buf->tail - buf->data; desc.popts = (buf->needs_csum ? POPTS_TXSM : 0); } diff --git a/kernel/drivers/net/rtl8168/rtl8168.cpp b/kernel/drivers/net/rtl8168/rtl8168.cpp index e66e8d5dc..46b7f6264 100644 --- a/kernel/drivers/net/rtl8168/rtl8168.cpp +++ b/kernel/drivers/net/rtl8168/rtl8168.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include diff --git a/kernel/drivers/nvme/nvme.cpp b/kernel/drivers/nvme/nvme.cpp index fcdbcc2a1..a58ea281b 100644 --- a/kernel/drivers/nvme/nvme.cpp +++ b/kernel/drivers/nvme/nvme.cpp @@ -833,6 +833,8 @@ bool nvme_device::nvme_queue::handle_cq() NVME_CQE_STATUS_DNR(cqe->dw3) ? ", do not repeat" : ""); pr_err("nvme%u: Related SQE dump: %*ph\n", dev_->device_index_, (int) sizeof(nvmesqe), &command->cmd); + if (NVME_CQE_STATUS_DNR(cqe->dw3)) + blk_request_dump(command->req, KERN_ERR); command->req->r_flags |= BIO_REQ_EIO; } @@ -972,12 +974,26 @@ int nvme_device::nvme_queue::pull_sq() if (!req) break; - if (int st = device_io_submit(req); st < 0) + nvmecmd *cmd = &request_to_pdu(req)->cmd; + if (int st = prepare_nvme_request(req->r_flags & BIO_REQ_OP_MASK, cmd, req, + (nvme_namespace *) req->r_bdev->device_info); + st < 0) { - printf("nvme: device_io_submit failed with err %d, unpulling sqe\n", st); + pr_err("nvme: prepare_nvme_request failed with err %d, unpulling sqe\n", st); unpull_seq(req); break; } + + int cid = allocate_cid(); + DCHECK(cid >= 0); + DCHECK(((sq_tail_ + 1) % sq_size_) != sq_head_); + + auto next_entry = sq_tail_; + + sq_tail_ = (sq_tail_ + 1) % sq_size_; + cmd->cmd.cdw0.cid = cid; + queued_commands_[cmd->cmd.cdw0.cid] = cmd; + memcpy(&sq_[next_entry], &cmd->cmd, sizeof(nvmesqe)); } /* Queue is full, ring the doorbell */ diff --git a/kernel/include/onyx/atomic.h b/kernel/include/onyx/atomic.h index 720d2720a..80bd01aed 100644 --- a/kernel/include/onyx/atomic.h +++ b/kernel/include/onyx/atomic.h @@ -16,6 +16,10 @@ #define atomic_and_relaxed(var, mask) (__atomic_and_fetch(&(var), mask, __ATOMIC_RELAXED)) #define atomic_or_relaxed(var, mask) (__atomic_or_fetch(&(var), mask, __ATOMIC_RELAXED)) +#ifdef __cplusplus +#define __auto_type auto +#endif + #define cmpxchg(ptr, old, new) \ ({ \ __auto_type __old = (old); \ diff --git a/kernel/include/onyx/bdev_base_types.h b/kernel/include/onyx/bdev_base_types.h index 9741e6eec..48a40eb9c 100644 --- a/kernel/include/onyx/bdev_base_types.h +++ b/kernel/include/onyx/bdev_base_types.h @@ -89,6 +89,8 @@ static inline void *b_request_to_data(struct request *req) return req + 1; } +void blk_request_dump(struct request *req, const char *log_lvl); + struct blk_plug { struct list_head request_list; diff --git a/kernel/include/onyx/block.h b/kernel/include/onyx/block.h index 5c888d325..17b874050 100644 --- a/kernel/include/onyx/block.h +++ b/kernel/include/onyx/block.h @@ -212,5 +212,6 @@ int block_set_bsize(struct blockdev *bdev, unsigned int block_size); int bdev_do_open(struct blockdev *bdev, bool exclusive); void bdev_release(struct blockdev *bdev); unsigned int bdev_sector_size(struct blockdev *bdev); +u64 bdev_get_size(struct blockdev *bdev); __END_CDECLS #endif diff --git a/kernel/include/onyx/buffer.h b/kernel/include/onyx/buffer.h index 6cbea2ca3..480773ac0 100644 --- a/kernel/include/onyx/buffer.h +++ b/kernel/include/onyx/buffer.h @@ -49,6 +49,7 @@ struct block_buf #define BLOCKBUF_FLAG_WRITEBACK (1 << 1) #define BLOCKBUF_FLAG_UPTODATE (1 << 2) #define BLOCKBUF_FLAG_AREAD (1 << 3) +#define BLOCKBUF_FLAG_HOLE (1 << 4) static inline bool bb_test_and_set(struct block_buf *buf, unsigned int flag) { diff --git a/kernel/include/onyx/dentry.h b/kernel/include/onyx/dentry.h index 3da2cd8ab..c6fbc9778 100644 --- a/kernel/include/onyx/dentry.h +++ b/kernel/include/onyx/dentry.h @@ -19,11 +19,14 @@ #include #include #include +#include #ifdef __cplusplus #include #endif +struct path; + __BEGIN_CDECLS #define INLINE_NAME_MAX 40 @@ -85,7 +88,9 @@ struct dentry *dentry_create(const char *name, struct inode *inode, struct dentr = 0 #endif ); -char *dentry_to_file_name(struct dentry *dentry); +char *d_path(const struct path *path, char *buf, unsigned int buflen); + +extern seqlock_t rename_lock; /** * @brief Finish a VFS lookup @@ -290,6 +295,9 @@ bool dentry_does_not_have_parent(dentry *dir, dentry *to_not_have); void dentry_do_unlink(dentry *entry); void dentry_rename(dentry *dent, const char *name, dentry *parent, dentry *dst); void dentry_move(dentry *target, dentry *new_parent); +dentry *__dentry_try_to_open(std::string_view name, dentry *dir, bool lock_ino); +dentry *dentry_open_from_cache(dentry *dent, std::string_view name); +dentry *dentry_wait_for_pending(dentry *dent); #endif diff --git a/kernel/include/onyx/font.h b/kernel/include/onyx/font.h index 5e6fee1ef..72cf2d89b 100644 --- a/kernel/include/onyx/font.h +++ b/kernel/include/onyx/font.h @@ -15,6 +15,7 @@ struct font unsigned int chars; unsigned int *mask; unsigned char *cursor_bitmap; + unsigned int (*utf2char)(unsigned int codepoint); }; struct font *get_font_data(void); diff --git a/kernel/include/onyx/kcsan.h b/kernel/include/onyx/kcsan.h index a20e172b2..42ef94691 100644 --- a/kernel/include/onyx/kcsan.h +++ b/kernel/include/onyx/kcsan.h @@ -46,6 +46,25 @@ void __kcsan_check_access(const volatile void *ptr, size_t size, int type); #define __KCSAN_BARRIER_TO_SIGNAL_FENCE_rmb __ATOMIC_ACQUIRE #define __KCSAN_BARRIER_TO_SIGNAL_FENCE_release __ATOMIC_RELEASE +struct kcsan_scoped_access +{ + union { + struct list_head list; /* scoped_accesses list */ + /* + * Not an entry in scoped_accesses list; stack depth from where + * the access was initialized. + */ + int stack_depth; + }; + + /* Access information. */ + const volatile void *ptr; + size_t size; + int type; + /* Location where scoped access was set up. */ + unsigned long ip; +}; + struct kcsan_ctx { int disable_count; /* disable counter */ @@ -79,7 +98,7 @@ struct kcsan_ctx /* List of scoped accesses; likely to be empty. */ struct list_head scoped_accesses; - +#define CONFIG_KCSAN_WEAK_MEMORY 1 #ifdef CONFIG_KCSAN_WEAK_MEMORY /* * Scoped access for modeling access reordering to detect missing memory diff --git a/kernel/include/onyx/list.h b/kernel/include/onyx/list.h index c96b406b3..7de253643 100644 --- a/kernel/include/onyx/list.h +++ b/kernel/include/onyx/list.h @@ -206,6 +206,12 @@ static inline void list_splice_tail(struct list_head *src, struct list_head *dst list_splice_internal(src, dst->prev, dst); } +static inline void list_splice_tail_init(struct list_head *src, struct list_head *dst) +{ + list_splice_tail(src, dst); + INIT_LIST_HEAD(src); +} + static inline int list_is_head(const struct list_head *list, const struct list_head *head) { return list == head; @@ -216,6 +222,7 @@ static inline int list_is_head(const struct list_head *list, const struct list_h #define list_prepare_entry(pos, head, member) ((pos) ?: list_entry(head, __typeof__(*pos), member)) #define list_entry_is_head(pos, head, member) list_is_head(&pos->member, (head)) #define list_first_entry(ptr, type, member) list_entry((ptr)->next, type, member) +#define list_last_entry(ptr, type, member) list_entry((ptr)->prev, type, member) #define list_for_each_entry_continue(pos, head, member) \ for (pos = list_next_entry(pos, member); !list_entry_is_head(pos, head, member); \ diff --git a/kernel/include/onyx/mm/kasan.h b/kernel/include/onyx/mm/kasan.h index e20c77f49..378524324 100644 --- a/kernel/include/onyx/mm/kasan.h +++ b/kernel/include/onyx/mm/kasan.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 - 2022 Pedro Falcato + * Copyright (c) 2019 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -18,6 +18,7 @@ __BEGIN_CDECLS void kasan_init(); +void kasan_page_alloc_init(); int kasan_alloc_shadow(unsigned long addr, size_t size, bool accessible); void kasan_set_state(unsigned long *ptr, size_t size, int state); diff --git a/kernel/include/onyx/mm/slab.h b/kernel/include/onyx/mm/slab.h index be5d16be5..5c70dead4 100644 --- a/kernel/include/onyx/mm/slab.h +++ b/kernel/include/onyx/mm/slab.h @@ -117,6 +117,10 @@ void kfree(void *ptr); void *kcalloc(size_t nr, size_t size, int flags); +void *krealloc(void *ptr, size_t size, int flags); + +void *kreallocarray(void *ptr, size_t m, size_t n, int flags); + /** * @brief Purge a cache * This function goes through every free slab and gives it back to the page allocator. diff --git a/kernel/include/onyx/mm/vm_object.h b/kernel/include/onyx/mm/vm_object.h index fd62f106e..6b25cb0cd 100644 --- a/kernel/include/onyx/mm/vm_object.h +++ b/kernel/include/onyx/mm/vm_object.h @@ -47,11 +47,17 @@ static inline int vmo_status_to_errno(vmo_status_t st) struct page; struct vm_object; +struct file; + struct vm_object_ops { void (*free_page)(struct vm_object *vmo, struct page *page); void (*truncate_partial)(struct vm_object *vmobj, struct page *page, size_t offset, size_t len); ssize_t (*writepage)(struct vm_object *vm_obj, struct page *page, size_t off); + int (*write_begin)(struct file *file, struct vm_object *vm_obj, off_t offset, size_t len, + struct page **page); + int (*write_end)(struct file *file, struct vm_object *vm_obj, off_t offset, + unsigned int written, unsigned int to_write, struct page *page); }; #define VMO_FLAG_LOCK_FUTURE_PAGES (1 << 0) diff --git a/kernel/include/onyx/mm_address_space.h b/kernel/include/onyx/mm_address_space.h index 24aed1079..27759c5ed 100644 --- a/kernel/include/onyx/mm_address_space.h +++ b/kernel/include/onyx/mm_address_space.h @@ -12,7 +12,7 @@ #include #include -#include +#include #include @@ -46,7 +46,7 @@ struct mm_address_space struct maple_tree region_tree; unsigned long start CPP_DFLINIT; unsigned long end CPP_DFLINIT; - struct mutex vm_lock CPP_DFLINIT; + struct rwlock vm_lock CPP_DFLINIT; /* mmap(2) base */ void *mmap_base CPP_DFLINIT; diff --git a/kernel/include/onyx/mount.h b/kernel/include/onyx/mount.h index 99e2a42b5..ac214952a 100644 --- a/kernel/include/onyx/mount.h +++ b/kernel/include/onyx/mount.h @@ -10,6 +10,7 @@ #include #include +#include struct dentry; struct superblock; @@ -21,6 +22,7 @@ struct mount struct dentry *mnt_root; struct superblock *mnt_sb; struct dentry *mnt_point; + struct mount *mnt_parent; unsigned int mnt_flags; /* TODO: percpu */ unsigned long mnt_count; @@ -28,6 +30,8 @@ struct mount struct rcu_head mnt_rcu; struct list_head mnt_mp_node; struct list_head mnt_node; + struct list_head mnt_submounts; + struct list_head mnt_submount_node; }; static inline void mnt_get(struct mount *mnt) @@ -47,6 +51,8 @@ int do_mount(const char *source, const char *target, const char *fstype, unsigne struct mount *mnt_traverse(struct dentry *mountpoint); +extern seqlock_t mount_lock; + __END_CDECLS #endif diff --git a/kernel/include/onyx/net/arp.h b/kernel/include/onyx/net/arp.h index b4a9b5cc3..0ca432b36 100644 --- a/kernel/include/onyx/net/arp.h +++ b/kernel/include/onyx/net/arp.h @@ -1,7 +1,9 @@ /* - * Copyright (c) 2016, 2017 Pedro Falcato + * Copyright (c) 2016 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only */ #ifndef _ONYX_NET_ARP_H @@ -38,7 +40,7 @@ typedef struct struct netif; -expected, int> arp_resolve_in(uint32_t ip, struct netif *netif); -int arp_handle_packet(netif *netif, packetbuf *buf); +struct neighbour *arp_resolve_in(uint32_t ip, struct netif *netif); +int arp_handle_packet(struct netif *netif, struct packetbuf *buf); #endif diff --git a/kernel/include/onyx/net/dll.h b/kernel/include/onyx/net/dll.h index 96bde9923..c9a210a4d 100644 --- a/kernel/include/onyx/net/dll.h +++ b/kernel/include/onyx/net/dll.h @@ -6,7 +6,8 @@ #ifndef _ONYX_NET_DLL_H #define _ONYX_NET_DLL_H -struct packetbuf; +#include + struct netif; enum class tx_type diff --git a/kernel/include/onyx/net/inet_route.h b/kernel/include/onyx/net/inet_route.h index a9ce68114..d373b7e3f 100644 --- a/kernel/include/onyx/net/inet_route.h +++ b/kernel/include/onyx/net/inet_route.h @@ -1,12 +1,15 @@ /* - * Copyright (c) 2020 Pedro Falcato + * Copyright (c) 2020 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only */ #ifndef _ONYX_NET_INET_ROUTE_H #define _ONYX_NET_INET_ROUTE_H +#include #include #include @@ -57,7 +60,55 @@ struct inet_route netif *nif; unsigned short flags; - shared_ptr dst_hw; + struct neighbour *dst_hw{}; + + inet_route() = default; + ~inet_route() + { + if (!IS_ERR_OR_NULL(dst_hw)) + neigh_put(dst_hw); + } + + inet_route &operator=(const inet_route &rhs) + { + if (&rhs == this) + return *this; + this->src_addr = rhs.src_addr; + this->dst_addr = rhs.dst_addr; + this->mask = rhs.mask; + this->gateway_addr = rhs.gateway_addr; + this->nif = rhs.nif; + this->flags = rhs.flags; + this->dst_hw = rhs.dst_hw; + if (rhs.dst_hw) + neigh_get(rhs.dst_hw); + return *this; + } + + inet_route(const inet_route &rhs) + { + *this = rhs; + } + + inet_route &operator=(inet_route &&rhs) + { + if (&rhs == this) + return *this; + this->src_addr = rhs.src_addr; + this->dst_addr = rhs.dst_addr; + this->mask = rhs.mask; + this->gateway_addr = rhs.gateway_addr; + this->nif = rhs.nif; + this->flags = rhs.flags; + this->dst_hw = rhs.dst_hw; + rhs.dst_hw = nullptr; + return *this; + } + + inet_route(inet_route &&rhs) + { + *this = cul::move(rhs); + } }; #endif diff --git a/kernel/include/onyx/net/ip.h b/kernel/include/onyx/net/ip.h index e96774258..83519d305 100644 --- a/kernel/include/onyx/net/ip.h +++ b/kernel/include/onyx/net/ip.h @@ -231,4 +231,6 @@ constexpr size_t inet_header_size(int domain) return size; } +int ip_finish_output(struct neighbour *n, struct packetbuf *pbf, struct netif *nif); + #endif diff --git a/kernel/include/onyx/net/ipv6.h b/kernel/include/onyx/net/ipv6.h index 2d2f26bdf..9f94b3cd6 100644 --- a/kernel/include/onyx/net/ipv6.h +++ b/kernel/include/onyx/net/ipv6.h @@ -120,4 +120,6 @@ int netif_addrcfg(netif *nif, const in6_addr &if_id); const struct inet_proto_family *get_v6_proto(); } // namespace ip::v6 +int ip6_finish_output(struct neighbour *neigh, struct packetbuf *pbf, struct netif *nif); + #endif diff --git a/kernel/include/onyx/net/ndp.h b/kernel/include/onyx/net/ndp.h index 0f24e18ec..8691df484 100644 --- a/kernel/include/onyx/net/ndp.h +++ b/kernel/include/onyx/net/ndp.h @@ -1,7 +1,9 @@ /* - * Copyright (c) 2020 Pedro Falcato + * Copyright (c) 2020 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only */ #ifndef _ONYX_NET_NDP_H @@ -9,10 +11,7 @@ #include -#include -#include - -expected, int> ndp_resolve(const in6_addr &ip, struct netif *netif); +struct neighbour *ndp_resolve(const in6_addr &ip, struct netif *netif); int ndp_handle_packet(netif *netif, packetbuf *buf); #endif diff --git a/kernel/include/onyx/net/neighbour.h b/kernel/include/onyx/net/neighbour.h index 5dfaa377e..a42e51a62 100644 --- a/kernel/include/onyx/net/neighbour.h +++ b/kernel/include/onyx/net/neighbour.h @@ -1,7 +1,9 @@ /* - * Copyright (c) 2020 Pedro Falcato + * Copyright (c) 2020 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only */ #ifndef _ONYX_NET_NEIGHBOUR_H @@ -9,54 +11,67 @@ #include +#include #include #include #include #include +#include +#include #include #include -#include -#include -#include -#include - #define NEIGHBOUR_VALIDITY_STATIC (~0UL) void neighbour_revalidate(clockevent* ev); -class neighbour; - #define NEIGHBOUR_FLAG_UNINITIALISED (1 << 0) #define NEIGHBOUR_FLAG_BADENTRY (1 << 1) #define NEIGHBOUR_FLAG_HAS_RESPONSE (1 << 2) #define NEIGHBOUR_FLAG_BROADCAST (1 << 3) +#define NUD_REACHABLE (1 << 4) +#define NUD_INCOMPLETE (1 << 5) +#define NUD_STALE (1 << 6) +#define NUD_PROBE (1 << 7) +#define NUD_FAILED (1 << 8) -class neighbour_table; +struct neighbour_table; union neigh_proto_addr { - in_addr in4addr; - in6_addr in6addr; + struct in_addr in4addr; + struct in6_addr in6addr; +}; + +struct neigh_ops +{ + int (*resolve)(struct neighbour* neigh, struct netif* nif); + int (*output)(struct neighbour* neigh, struct packetbuf* pbf, struct netif* nif); }; -class neighbour +struct neighbour { -protected: - unsigned char* hwaddr_; - unsigned int hwaddr_len_; + unsigned int refcount; + unsigned int hwaddr_len; + union { + unsigned char hwaddr[16]; + struct rcu_head rcu_head; + }; + int domain; struct clockevent expiry_timer; unsigned long validity_ms; - -public: unsigned int flags; - neighbour_table* table; - + struct neighbour_table* table; union neigh_proto_addr proto_addr; + const struct neigh_ops* neigh_ops; + struct list_head list_node; + seqlock_t neigh_seqlock; + struct list_head packet_queue; explicit neighbour(int _domain, const neigh_proto_addr& addr) - : hwaddr_{}, hwaddr_len_{}, domain{_domain}, flags{0} + : refcount{1}, + hwaddr_len{}, domain{_domain}, flags{NEIGHBOUR_FLAG_UNINITIALISED}, neigh_ops{} { if (_domain == AF_INET) proto_addr.in4addr.s_addr = addr.in4addr.s_addr; @@ -64,17 +79,11 @@ class neighbour memcpy(&proto_addr.in6addr, &addr.in6addr, sizeof(in6_addr)); else __builtin_unreachable(); + neigh_seqlock = {}; + INIT_LIST_HEAD(&packet_queue); } - ~neighbour() - { - delete[] hwaddr_; - } - - cul::slice hwaddr() - { - return {hwaddr_, hwaddr_len_}; - } + ~neighbour() = default; void set_validity(unsigned long validity) { @@ -118,14 +127,36 @@ class neighbour else __builtin_unreachable(); } - - void set_hwaddr(cul::slice& addr) - { - hwaddr_ = addr.data(); - hwaddr_len_ = addr.size_bytes(); - } }; +static inline bool neigh_needs_resolve(struct neighbour* neigh) +{ + /* Only bother trying to resolve neighbours if they're not yet resolved, or if there are no + * requests pending. */ + return !(READ_ONCE(neigh->flags) & (NUD_PROBE | NUD_REACHABLE | NUD_INCOMPLETE)); +} + +void neigh_start_resolve(struct neighbour* neigh, struct netif* nif); +void neigh_output_queued(struct neighbour* neigh); + +static inline void __neigh_complete_lookup(struct neighbour* neigh, const void* hwaddr, + unsigned int len) +{ + memcpy(neigh->hwaddr, hwaddr, len); + neigh->hwaddr_len = len; + neigh->flags &= ~(NUD_PROBE | NUD_INCOMPLETE | NUD_FAILED | NUD_STALE); + neigh->flags |= NUD_REACHABLE; + neigh_output_queued(neigh); +} + +static inline void neigh_complete_lookup(struct neighbour* neigh, const void* hwaddr, + unsigned int len) +{ + write_seqlock(&neigh->neigh_seqlock); + __neigh_complete_lookup(neigh, hwaddr, len); + write_sequnlock(&neigh->neigh_seqlock); +} + static inline fnv_hash_t hash_protoaddr(const neigh_proto_addr& addr, int domain) { if (domain == AF_INET) @@ -136,41 +167,61 @@ static inline fnv_hash_t hash_protoaddr(const neigh_proto_addr& addr, int domain __builtin_unreachable(); } -fnv_hash_t hash_neighbour(shared_ptr& neigh); +#define NEIGH_TAB_NR_CHAINS 32 -class neighbour_table +struct neighbour_table { -protected: - cul::hashtable, 32, fnv_hash_t, hash_neighbour> neighbour_cache; - - /* TODO: Is another lock type optimal here? Note that spinlocks have low overhead vs rwlocks - * and lookups are *usually* quick. - */ - spinlock lock; + struct list_head neigh_tab[NEIGH_TAB_NR_CHAINS]; + struct spinlock lock; const int domain; - -public: - neighbour_table(int domain) : neighbour_cache{}, lock{}, domain{domain} + neighbour_table(int domain) : lock{}, domain{domain} { spinlock_init(&lock); + for (int i = 0; i < NEIGH_TAB_NR_CHAINS; i++) + INIT_LIST_HEAD(&neigh_tab[i]); } - - /** - * @brief Add or get an existing neighbour entry - * - * @param addr the protocol address of the neighbour - * @return cul::pair, bool> a shared pointer to the neighbour + a bool - * signaling if we created it or not. - */ - cul::pair, bool> add(const neigh_proto_addr& addr, - bool only_lookup = false); - void remove(neighbour* neigh); - - /** - * @brief Clears cache entries - * - */ - void clear_cache(); }; +typedef unsigned int gfp_t; + +struct neighbour* neigh_find(struct neighbour_table* table, const union neigh_proto_addr* addr); +struct neighbour* neigh_add(struct neighbour_table* table, const union neigh_proto_addr* addr, + gfp_t gfp, const struct neigh_ops* ops, int* added); +void neigh_remove(struct neighbour_table* table, struct neighbour* neigh); +void neigh_clear(struct neighbour_table* table); +void neigh_free(struct neighbour* neigh); + +static inline void neigh_get(struct neighbour* neigh) +{ + __atomic_add_fetch(&neigh->refcount, 1, __ATOMIC_RELAXED); +} + +static inline void neigh_put(struct neighbour* neigh) +{ + if (__atomic_sub_fetch(&neigh->refcount, 1, __ATOMIC_RELAXED) == 0) + neigh_free(neigh); +} + +static inline bool neigh_get_careful(struct neighbour* neigh) +{ + unsigned int ref = READ_ONCE(neigh->refcount); + unsigned int old; + + do + { + if (unlikely(ref == 0)) + return false; + old = ref; + } while ((ref = cmpxchg(&neigh->refcount, ref, ref + 1)) != old); + + return true; +} + +int neigh_output(struct neighbour* neigh, struct packetbuf* pbf, struct netif* nif); + +static inline void neigh_set_ops(struct neighbour* neigh, struct neigh_ops* ops) +{ + WRITE_ONCE(neigh->neigh_ops, ops); +} + #endif diff --git a/kernel/include/onyx/net/netif.h b/kernel/include/onyx/net/netif.h index 1f87763de..cb2facd3d 100644 --- a/kernel/include/onyx/net/netif.h +++ b/kernel/include/onyx/net/netif.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 - 2022 Pedro Falcato + * Copyright (c) 2017 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -14,7 +14,6 @@ #include #include struct netif; -#include #include #include @@ -29,7 +28,7 @@ struct netif; #define NETIF_LOOPBACK (1 << 5) #define NETIF_HAS_RX_AVAILABLE (1 << 6) #define NETIF_DOING_RX_POLL (1 << 7) -#define NETIF_MISSED_RX (1 << 8) +#define NETIF_SCHEDULED (1 << 8) struct packetbuf; diff --git a/kernel/include/onyx/net/netkernel.h b/kernel/include/onyx/net/netkernel.h index 5d37966f8..ccd9c3f83 100644 --- a/kernel/include/onyx/net/netkernel.h +++ b/kernel/include/onyx/net/netkernel.h @@ -10,6 +10,7 @@ #define _ONYX_NET_NETKERNEL_H #include +#include #include #include diff --git a/kernel/include/onyx/packetbuf.h b/kernel/include/onyx/packetbuf.h index d74915ddc..6e5bea9c5 100644 --- a/kernel/include/onyx/packetbuf.h +++ b/kernel/include/onyx/packetbuf.h @@ -30,6 +30,7 @@ struct vm_object; struct tcp_packetbuf_info { u32 seq, seq_len; + u8 ack : 1, syn : 1, fin : 1, rst : 1; }; struct packetbuf; @@ -101,7 +102,10 @@ struct packetbuf unsigned int zero_copy : 1; int domain; - struct list_head list_node; + union { + struct list_head list_node; + struct bst_node bst_node; + }; union { struct inet_route route; @@ -121,6 +125,7 @@ struct packetbuf data{}, tail{}, end{}, buffer_start{}, csum_offset{nullptr}, csum_start{nullptr}, header_length{}, gso_size{}, gso_flags{}, needs_csum{0}, zero_copy{0}, domain{0} { + route = {}; } /** @@ -378,6 +383,23 @@ static inline void *pbf_put(struct packetbuf *pbf, unsigned int size) return ret; } +/** + * @brief Get a header, and advance head by size. + * + * @param pbf Packetbuf + * @param size The length of the data. + * + * @return void* The address of the header. + */ +static inline void *pbf_pull(struct packetbuf *pbf, unsigned int size) +{ + if (unlikely(size > (pbf->tail - pbf->data))) + return NULL; + void *ret = pbf->data; + pbf->data += size; + return ret; +} + /** * @brief Calculates the total length of the buffer. * diff --git a/kernel/include/onyx/page.h b/kernel/include/onyx/page.h index 167f5c629..79674d008 100644 --- a/kernel/include/onyx/page.h +++ b/kernel/include/onyx/page.h @@ -534,6 +534,15 @@ static inline uint64_t get_kernel_phys_offset() */ unsigned long pages_under_high_watermark(); +/** + * @brief Calculate a free page target (for reclaim) + * + * @param gfp GFP used for the failed allocation/reclaim + * @param order Order allocation that failed + * @return Free page target. If 0, probably shouldn't reclaim. + */ +unsigned long page_reclaim_target(gfp_t gfp, unsigned int order); + /** * @brief Drain pages from all zones' pcpu caches * @@ -613,10 +622,17 @@ PAGEFLAG_OPS(active, ACTIVE); PAGEFLAG_OPS(lru, LRU); PAGEFLAG_OPS(uptodate, UPTODATE); PAGEFLAG_OPS(dirty, DIRTY); +PAGEFLAG_OPS(buffer, BUFFER); struct vm_object *page_vmobj(struct page *page); unsigned long page_pgoff(struct page *page); +static inline void page_zero_range(struct page *page, unsigned int off, unsigned int len) +{ + u8 *ptr = (u8 *) PAGE_TO_VIRT(page); + memset(ptr + off, 0, len); +} + __END_CDECLS #endif diff --git a/kernel/include/onyx/path.h b/kernel/include/onyx/path.h index c6982a3c3..aea05ddcc 100644 --- a/kernel/include/onyx/path.h +++ b/kernel/include/onyx/path.h @@ -37,12 +37,12 @@ static inline void path_init(struct path *p) p->mount = NULL; } -static inline bool path_is_null(struct path *p) +static inline bool path_is_null(const struct path *p) { return !p->dentry && !p->mount; } -static inline bool path_is_equal(struct path *p1, struct path *p2) +static inline bool path_is_equal(const struct path *p1, const struct path *p2) { return p1->mount == p2->mount && p1->dentry == p2->dentry; } diff --git a/kernel/include/onyx/pgtable.h b/kernel/include/onyx/pgtable.h index 1e8e73391..2100bde56 100644 --- a/kernel/include/onyx/pgtable.h +++ b/kernel/include/onyx/pgtable.h @@ -232,6 +232,7 @@ static inline unsigned long pte_addr_end(unsigned long addr) pte_t pte_get(struct mm_address_space *mm, unsigned long addr); pte_t *ptep_get_locked(struct mm_address_space *mm, unsigned long addr, struct spinlock **lock); int pgtable_prealloc(struct mm_address_space *mm, unsigned long virt); +int zap_page_range(unsigned long start, unsigned long end, struct vm_area_struct *vma); __END_CDECLS #endif diff --git a/kernel/include/onyx/pid.h b/kernel/include/onyx/pid.h index 2d15c0870..83eb7b392 100644 --- a/kernel/include/onyx/pid.h +++ b/kernel/include/onyx/pid.h @@ -111,6 +111,11 @@ struct pid : public refcountable bool is_orphaned_and_has_stopped_jobs(process *ignore) const; int kill_pgrp(int sig, int flags, siginfo_t *info) const; + + bool is(enum pid_type type) + { + return !list_is_empty(&member_list[type]); + } }; static inline pid::auto_pid pid_create(process *leader) diff --git a/kernel/include/onyx/poll.h b/kernel/include/onyx/poll.h index 74a884766..1e5ee0b85 100644 --- a/kernel/include/onyx/poll.h +++ b/kernel/include/onyx/poll.h @@ -232,7 +232,7 @@ class poll_table void signal() { - signaled = true; + WRITE_ONCE(signaled, true); } bool may_queue() const @@ -245,13 +245,16 @@ class poll_table is_queueing = false; } - bool was_signaled() const + bool was_signaled() { - return signaled; + bool res = READ_ONCE(signaled); + if (res) + WRITE_ONCE(signaled, false); + return res; } /* timeout in ms - negative means infinite, 0 means don't sleep */ - sleep_result sleep_poll(hrtime_t timeout, bool timeout_valid) const; + sleep_result sleep_poll(hrtime_t timeout, bool timeout_valid); }; void poll_wait_helper(void *poll_file, struct wait_queue *q); diff --git a/kernel/include/onyx/process.h b/kernel/include/onyx/process.h index 81d663738..686c4243d 100644 --- a/kernel/include/onyx/process.h +++ b/kernel/include/onyx/process.h @@ -328,6 +328,18 @@ __attribute__((pure)) static inline struct process *get_current_process() return (thread == NULL) ? NULL : (struct process *) thread->owner; } +static inline mode_t get_current_umask() +{ + if (unlikely(!get_current_process())) + return 0; + return get_current_process()->ctx.umask; +} + +static inline mode_t do_umask(mode_t mode) +{ + return mode & ~get_current_umask(); +} + /** * @brief Get the number of active processes * diff --git a/kernel/include/onyx/rculist.h b/kernel/include/onyx/rculist.h index ebf44c55f..28335a5d1 100644 --- a/kernel/include/onyx/rculist.h +++ b/kernel/include/onyx/rculist.h @@ -8,6 +8,7 @@ #ifndef _ONYX_RCULIST_H #define _ONYX_RCULIST_H +#include #include #include @@ -39,4 +40,20 @@ static inline void list_remove_rcu(struct list_head *node) list_remove_bulk(node->prev, node->next); } +#define list_entry_rcu(ptr, type, member) container_of(READ_ONCE(ptr), type, member) +/** + * list_for_each_entry_rcu - iterate over rcu list of given type + * @pos: the type * to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the list_head within the struct. + * @cond: optional lockdep expression if called from non-RCU protection. + * + * This list-traversal primitive may safely run concurrently with + * the _rcu list-mutation primitives such as list_add_rcu() + * as long as the traversal is guarded by rcu_read_lock(). + */ +#define list_for_each_entry_rcu(pos, head, member, cond...) \ + for (pos = list_entry_rcu((head)->next, __typeof__(*pos), member); &pos->member != (head); \ + pos = list_entry_rcu(pos->member.next, __typeof__(*pos), member)) + #endif diff --git a/kernel/include/onyx/riscv/include/platform/pgtable.h b/kernel/include/onyx/riscv/include/platform/pgtable.h index 25a65eb64..5d749fb29 100644 --- a/kernel/include/onyx/riscv/include/platform/pgtable.h +++ b/kernel/include/onyx/riscv/include/platform/pgtable.h @@ -375,7 +375,7 @@ static inline pte_t pte_wrprotect(pte_t pte) static inline pgprot_t calc_pgprot(u64 phys, u64 prots) { - bool special_mapping = phys == (u64) page_to_phys(vm_get_zero_page()); + bool special_mapping = phys == (u64) page_to_phys(vm_get_zero_page()) || prots & VM_PFNMAP; pgprotval_t page_prots = (prots & VM_EXEC ? _PAGE_EXEC : 0) | (prots & VM_WRITE ? _PAGE_WRITE : 0) | (prots & (VM_READ | VM_WRITE) ? _PAGE_READ : 0) | diff --git a/kernel/include/onyx/scheduler.h b/kernel/include/onyx/scheduler.h index 3fda63826..e57eb9c85 100644 --- a/kernel/include/onyx/scheduler.h +++ b/kernel/include/onyx/scheduler.h @@ -87,6 +87,7 @@ typedef struct thread #endif #ifdef CONFIG_KCSAN struct kcsan_ctx kcsan_ctx; + int kcsan_stack_depth; #endif /* And arch dependent stuff in this ifdef */ #ifdef __x86_64__ @@ -107,6 +108,9 @@ typedef struct thread fs{}, gs{} #endif { +#ifdef CONFIG_KCSAN + kcsan_stack_depth = 0; +#endif } /** diff --git a/kernel/include/onyx/tty.h b/kernel/include/onyx/tty.h index d7dbdf406..434b573e1 100644 --- a/kernel/include/onyx/tty.h +++ b/kernel/include/onyx/tty.h @@ -186,6 +186,7 @@ __END_CDECLS #define ANSI_SAVE_CURSOR 's' #define ANSI_RESTORE_CURSOR 'u' #define CSI_DELETE_CHARS 'P' +#define CSI_ERASE_CHARS 'X' #define CSI_INSERT_BLANK '@' #define CSI_INSERT_LINE 'L' #define CSI_DELETE_LINE 'M' @@ -197,21 +198,23 @@ __END_CDECLS #define ESC_SAVECUR '7' #define ESC_RESTORECUR '8' -#define ANSI_SGR_RESET 0 -#define ANSI_SGR_BOLD 1 -#define ANSI_SGR_FAINT 2 -#define ANSI_SGR_ITALIC 3 -#define ANSI_SGR_UNDERLINE 4 -#define ANSI_SGR_SLOWBLINK 5 -#define ANSI_SGR_RAPIDBLINK 6 -#define ANSI_SGR_REVERSE 7 -#define ANSI_SGR_BLINKOFF 25 -#define ANSI_SGR_SETFGMIN 30 -#define ANSI_SGR_SETFGMAX 37 -#define ANSI_SGR_DEFAULTFG 39 -#define ANSI_SGR_SETBGMIN 40 -#define ANSI_SGR_SETBGMAX 47 -#define ANSI_SGR_DEFAULTBG 49 +#define ANSI_SGR_RESET 0 +#define ANSI_SGR_BOLD 1 +#define ANSI_SGR_FAINT 2 +#define ANSI_SGR_ITALIC 3 +#define ANSI_SGR_UNDERLINE 4 +#define ANSI_SGR_SLOWBLINK 5 +#define ANSI_SGR_RAPIDBLINK 6 +#define ANSI_SGR_REVERSE 7 +#define ANSI_SGR_NOUNDERLINE 24 +#define ANSI_SGR_BLINKOFF 25 +#define ANSI_SGR_NOREVERSE 27 +#define ANSI_SGR_SETFGMIN 30 +#define ANSI_SGR_SETFGMAX 37 +#define ANSI_SGR_DEFAULTFG 39 +#define ANSI_SGR_SETBGMIN 40 +#define ANSI_SGR_SETBGMAX 47 +#define ANSI_SGR_DEFAULTBG 49 #define ANSI_COLOR_RED "\x1b[31m" #define ANSI_COLOR_GREEN "\x1b[32m" diff --git a/kernel/include/onyx/vm.h b/kernel/include/onyx/vm.h index 8a3d93b78..40ec25de9 100644 --- a/kernel/include/onyx/vm.h +++ b/kernel/include/onyx/vm.h @@ -58,6 +58,7 @@ __BEGIN_CDECLS #define VM_DONT_MAP_OVER (1 << 8) #define VM_NOFLUSH (1 << 9) #define VM_SHARED (1 << 10) +#define VM_PFNMAP (1 << 11) /* Internal flags used by the mm code */ #define __VM_CACHE_TYPE_REGULAR 0 @@ -87,8 +88,6 @@ static inline unsigned long vm_prot_to_cache_type(uint64_t prot) #define PHYS_TO_VIRT(x) (void *) ((uintptr_t) (x) + PHYS_BASE) -#define VM_PFNMAP (1 << 1) - struct vm_object; struct vm_pf_context; @@ -767,7 +766,7 @@ struct page *vmalloc_to_pages(void *ptr); */ static inline bool vma_is_pfnmap(struct vm_area_struct *vma) { - return vma == NULL; + return vma == NULL || vma->vm_flags & VM_PFNMAP; } void vm_do_mmu_mprotect(struct mm_address_space *as, void *address, size_t nr_pgs, int old_prots, diff --git a/kernel/include/onyx/wait_queue.h b/kernel/include/onyx/wait_queue.h index 8424a433b..f489e9b00 100644 --- a/kernel/include/onyx/wait_queue.h +++ b/kernel/include/onyx/wait_queue.h @@ -18,7 +18,8 @@ #include #include -#define WQ_TOKEN_EXCLUSIVE (1u << 0) +#define WQ_TOKEN_EXCLUSIVE (1u << 0) +#define WQ_TOKEN_NO_DEQUEUE (1u << 1) /* Return values for wait_queue_token::wake */ #define WQ_WAKE_DO_NOT_WAKE -1 @@ -125,9 +126,9 @@ bool signal_is_pending(); goto out_final; \ init_wq_token(&token); \ \ - set_current_state(state); \ while (true) \ { \ + set_current_state(state); \ token.thread = get_current_thread(); \ wait_queue_add(wq, &token); \ if (cond) \ @@ -156,9 +157,9 @@ bool signal_is_pending(); goto out_final; \ init_wq_token(&token); \ \ - set_current_state(state); \ while (true) \ { \ + set_current_state(state); \ token.thread = get_current_thread(); \ wait_queue_add(wq, &token); \ if (cond) \ diff --git a/kernel/include/onyx/x86/include/platform/pgtable.h b/kernel/include/onyx/x86/include/platform/pgtable.h index db8f0e66b..c4731692e 100644 --- a/kernel/include/onyx/x86/include/platform/pgtable.h +++ b/kernel/include/onyx/x86/include/platform/pgtable.h @@ -422,7 +422,7 @@ static inline pgprot_t calc_pgprot(u64 phys, u64 prot) bool readable = prot & (VM_READ | VM_WRITE) || !noexec; unsigned int cache_type = vm_prot_to_cache_type(prot); uint8_t caching_bits = cache_to_paging_bits(cache_type); - bool special_mapping = phys == (u64) page_to_phys(vm_get_zero_page()); + bool special_mapping = phys == (u64) page_to_phys(vm_get_zero_page()) || prot & VM_PFNMAP; pgprotval_t page_prots = (noexec ? _PAGE_NX : 0) | (global ? _PAGE_GLOBAL : 0) | (user ? _PAGE_USER : 0) | diff --git a/kernel/kernel/Makefile b/kernel/kernel/Makefile index 5d70cae79..4913d89b8 100644 --- a/kernel/kernel/Makefile +++ b/kernel/kernel/Makefile @@ -9,7 +9,7 @@ kern-y+= arc4random.o binfmt.o compression.o copy.o cppnew.o cpprt.o crc32.o dev kern-$(CONFIG_UBSAN)+= ubsan.o -kern-y+= fonts/font.o photon/photon.o binfmt/elf.o binfmt/module_loader.o binfmt/exec.o \ +kern-y+= photon/photon.o binfmt/elf.o binfmt/module_loader.o binfmt/exec.o \ libdict/rb_tree.o libdict/tree_common.o libdict/wb_tree.o time/tickless.o binfmt/shebang.o kern-$(CONFIG_COMPAT)+= binfmt/elf_compat.o @@ -32,6 +32,7 @@ include kernel/input/Makefile include kernel/tty/Makefile include kernel/sched/Makefile include kernel/kcsan/Makefile +include kernel/fonts/Makefile kernel/syscall_thunk.cpp: generate_syscall_bits.py $(ARCHDIR)/syscall_table.json mkdir -p include/onyx/gen diff --git a/kernel/kernel/binfmt/exec.cpp b/kernel/kernel/binfmt/exec.cpp index 6c6b4bff8..816678882 100644 --- a/kernel/kernel/binfmt/exec.cpp +++ b/kernel/kernel/binfmt/exec.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -35,6 +36,9 @@ expected process_copy_envarg(const char **envarg, size_t curren const char **b = envarg; const char *ptr = nullptr; long st; + /* TODO: Take into account rlim_stack */ + unsigned long limit = (8 * DEFAULT_USER_STACK_LEN) / 4; + limit = cul::max(limit, (unsigned long) ARG_MAX); while ((st = get_user64((unsigned long *) b, (unsigned long *) &ptr)) == 0 && ptr != nullptr) { @@ -57,10 +61,10 @@ expected process_copy_envarg(const char **envarg, size_t curren size_t buffer_size = (nr_args + 1) * sizeof(void *) + string_size; // Check if we overflow the ARG_MAX - if (current_size + buffer_size > ARG_MAX) + if (current_size + buffer_size > limit) return unexpected{-E2BIG}; - char *new_ = (char *) zalloc(buffer_size); + char *new_ = (char *) kcalloc(buffer_size, 1, GFP_KERNEL); if (!new_) return unexpected{-ENOMEM}; @@ -328,7 +332,7 @@ int flush_old_exec(struct exec_state *state) vm_set_aspace(state->new_address_space.get()); curr->address_space = cul::move(state->new_address_space); - mutex_init(&curr->address_space->vm_lock); + rwlock_init(&curr->address_space->vm_lock); /* Close O_CLOEXEC files */ file_do_cloexec(&curr->ctx); @@ -501,7 +505,7 @@ int sys_execve(const char *p, const char **argv, const char **envp) current->flags &= ~PROCESS_FORKED; struct stack_info si; - si.length = DEFAULT_USER_STACK_LEN; + si.length = DEFAULT_USER_STACK_LEN * 8; if (process_alloc_stack(&si) < 0) goto error_die_signal; diff --git a/kernel/kernel/fbpriv.h b/kernel/kernel/fbpriv.h new file mode 100644 index 000000000..3c53e9829 --- /dev/null +++ b/kernel/kernel/fbpriv.h @@ -0,0 +1,280 @@ +/* + * copied from from linux kernel 2.2.4 + * removed internal stuff (#ifdef __KERNEL__) + */ + +#ifdef HAVE_XORG_CONFIG_H +#include +#endif + +#ifndef _LINUX_FB_H +#define _LINUX_FB_H + +#ifndef __onyx__ + +#include + +#else + +#include + +#endif + +/* Definitions of frame buffers */ + +#define FB_MAJOR 29 + +#define FB_MODES_SHIFT 5 /* 32 modes per framebuffer */ +#define FB_NUM_MINORS 256 /* 256 Minors */ +#define FB_MAX (FB_NUM_MINORS / (1 << FB_MODES_SHIFT)) +#define GET_FB_IDX(node) (MINOR(node) >> FB_MODES_SHIFT) + +/* ioctls + 0x46 is 'F' */ +#define FBIOGET_VSCREENINFO 0x4600 +#define FBIOPUT_VSCREENINFO 0x4601 +#define FBIOGET_FSCREENINFO 0x4602 +#define FBIOGETCMAP 0x4604 +#define FBIOPUTCMAP 0x4605 +#define FBIOPAN_DISPLAY 0x4606 +/* 0x4607-0x460B are defined below */ +/* #define FBIOGET_MONITORSPEC 0x460C */ +/* #define FBIOPUT_MONITORSPEC 0x460D */ +/* #define FBIOSWITCH_MONIBIT 0x460E */ +#define FBIOGET_CON2FBMAP 0x460F +#define FBIOPUT_CON2FBMAP 0x4610 +#define FBIOBLANK 0x4611 + +#define FB_TYPE_PACKED_PIXELS 0 /* Packed Pixels */ +#define FB_TYPE_PLANES 1 /* Non interleaved planes */ +#define FB_TYPE_INTERLEAVED_PLANES 2 /* Interleaved planes */ +#define FB_TYPE_TEXT 3 /* Text/attributes */ + +#define FB_AUX_TEXT_MDA 0 /* Monochrome text */ +#define FB_AUX_TEXT_CGA 1 /* CGA/EGA/VGA Color text */ +#define FB_AUX_TEXT_S3_MMIO 2 /* S3 MMIO fasttext */ +#define FB_AUX_TEXT_MGA_STEP16 3 /* MGA Millenium I: text, attr, 14 reserved bytes */ +#define FB_AUX_TEXT_MGA_STEP8 4 /* other MGAs: text, attr, 6 reserved bytes */ + +#define FB_VISUAL_MONO01 0 /* Monochr. 1=Black 0=White */ +#define FB_VISUAL_MONO10 1 /* Monochr. 1=White 0=Black */ +#define FB_VISUAL_TRUECOLOR 2 /* True color */ +#define FB_VISUAL_PSEUDOCOLOR 3 /* Pseudo color (like atari) */ +#define FB_VISUAL_DIRECTCOLOR 4 /* Direct color */ +#define FB_VISUAL_STATIC_PSEUDOCOLOR 5 /* Pseudo color readonly */ + +#define FB_ACCEL_NONE 0 /* no hardware accelerator */ +#define FB_ACCEL_ATARIBLITT 1 /* Atari Blitter */ +#define FB_ACCEL_AMIGABLITT 2 /* Amiga Blitter */ +#define FB_ACCEL_S3_TRIO64 3 /* Cybervision64 (S3 Trio64) */ +#define FB_ACCEL_NCR_77C32BLT 4 /* RetinaZ3 (NCR 77C32BLT) */ +#define FB_ACCEL_S3_VIRGE 5 /* Cybervision64/3D (S3 ViRGE) */ +#define FB_ACCEL_ATI_MACH64GX 6 /* ATI Mach 64GX family */ +#define FB_ACCEL_DEC_TGA 7 /* DEC 21030 TGA */ +#define FB_ACCEL_ATI_MACH64CT 8 /* ATI Mach 64CT family */ +#define FB_ACCEL_ATI_MACH64VT 9 /* ATI Mach 64CT family VT class */ +#define FB_ACCEL_ATI_MACH64GT 10 /* ATI Mach 64CT family GT class */ +#define FB_ACCEL_SUN_CREATOR 11 /* Sun Creator/Creator3D */ +#define FB_ACCEL_SUN_CGSIX 12 /* Sun cg6 */ +#define FB_ACCEL_SUN_LEO 13 /* Sun leo/zx */ +#define FB_ACCEL_IMS_TWINTURBO 14 /* IMS Twin Turbo */ +#define FB_ACCEL_3DLABS_PERMEDIA2 15 /* 3Dlabs Permedia 2 */ +#define FB_ACCEL_MATROX_MGA2064W 16 /* Matrox MGA2064W (Millenium) */ +#define FB_ACCEL_MATROX_MGA1064SG 17 /* Matrox MGA1064SG (Mystique) */ +#define FB_ACCEL_MATROX_MGA2164W 18 /* Matrox MGA2164W (Millenium II) */ +#define FB_ACCEL_MATROX_MGA2164W_AGP 19 /* Matrox MGA2164W (Millenium II) */ +#define FB_ACCEL_MATROX_MGAG100 20 /* Matrox G100 (Productiva G100) */ +#define FB_ACCEL_MATROX_MGAG200 21 /* Matrox G200 (Myst, Mill, ...) */ +#define FB_ACCEL_SUN_CG14 22 /* Sun cgfourteen */ +#define FB_ACCEL_SUN_BWTWO 23 /* Sun bwtwo */ +#define FB_ACCEL_SUN_CGTHREE 24 /* Sun cgthree */ +#define FB_ACCEL_SUN_TCX 25 /* Sun tcx */ +#define FB_ACCEL_MATROX_MGAG400 26 /* Matrox G400 */ +#define FB_ACCEL_NV3 27 /* nVidia RIVA 128 */ +#define FB_ACCEL_NV4 28 /* nVidia RIVA TNT */ +#define FB_ACCEL_NV5 29 /* nVidia RIVA TNT2 */ +#define FB_ACCEL_CT_6555x 30 /* C&T 6555x */ +#define FB_ACCEL_3DFX_BANSHEE 31 /* 3Dfx Banshee */ +#define FB_ACCEL_ATI_RAGE128 32 /* ATI Rage128 family */ + +struct fb_fix_screeninfo +{ + char id[16]; /* identification string eg "TT Builtin" */ + char *smem_start; /* Start of frame buffer mem */ + /* (physical address) */ + uint32_t smem_len; /* Length of frame buffer mem */ + uint32_t type; /* see FB_TYPE_* */ + uint32_t type_aux; /* Interleave for interleaved Planes */ + uint32_t visual; /* see FB_VISUAL_* */ + uint16_t xpanstep; /* zero if no hardware panning */ + uint16_t ypanstep; /* zero if no hardware panning */ + uint16_t ywrapstep; /* zero if no hardware ywrap */ + uint32_t line_length; /* length of a line in bytes */ + char *mmio_start; /* Start of Memory Mapped I/O */ + /* (physical address) */ + uint32_t mmio_len; /* Length of Memory Mapped I/O */ + uint32_t accel; /* Type of acceleration available */ + uint16_t reserved[3]; /* Reserved for future compatibility */ +}; + +/* Interpretation of offset for color fields: All offsets are from the right, + * inside a "pixel" value, which is exactly 'bits_per_pixel' wide (means: you + * can use the offset as right argument to <<). A pixel afterwards is a bit + * stream and is written to video memory as that unmodified. This implies + * big-endian byte order if bits_per_pixel is greater than 8. + */ +struct fb_bitfield +{ + uint32_t offset; /* beginning of bitfield */ + uint32_t length; /* length of bitfield */ + uint32_t msb_right; /* != 0 : Most significant bit is */ + /* right */ +}; + +#define FB_NONSTD_HAM 1 /* Hold-And-Modify (HAM) */ + +#define FB_ACTIVATE_NOW 0 /* set values immediately (or vbl) */ +#define FB_ACTIVATE_NXTOPEN 1 /* activate on next open */ +#define FB_ACTIVATE_TEST 2 /* don't set, round up impossible */ +#define FB_ACTIVATE_MASK 15 +/* values */ +#define FB_ACTIVATE_VBL 16 /* activate values on next vbl */ +#define FB_CHANGE_CMAP_VBL 32 /* change colormap on vbl */ +#define FB_ACTIVATE_ALL 64 /* change all VCs on this fb */ + +#define FB_ACCELF_TEXT 1 /* text mode acceleration */ + +#define FB_SYNC_HOR_HIGH_ACT 1 /* horizontal sync high active */ +#define FB_SYNC_VERT_HIGH_ACT 2 /* vertical sync high active */ +#define FB_SYNC_EXT 4 /* external sync */ +#define FB_SYNC_COMP_HIGH_ACT 8 /* composite sync high active */ +#define FB_SYNC_BROADCAST 16 /* broadcast video timings */ + /* vtotal = 144d/288n/576i => PAL */ + /* vtotal = 121d/242n/484i => NTSC */ +#define FB_SYNC_ON_GREEN 32 /* sync on green */ + +#define FB_VMODE_NONINTERLACED 0 /* non interlaced */ +#define FB_VMODE_INTERLACED 1 /* interlaced */ +#define FB_VMODE_DOUBLE 2 /* double scan */ +#define FB_VMODE_MASK 255 + +#define FB_VMODE_YWRAP 256 /* ywrap instead of panning */ +#define FB_VMODE_SMOOTH_XPAN 512 /* smooth xpan possible (internally used) */ +#define FB_VMODE_CONUPDATE 512 /* don't update x/yoffset */ + +struct fb_var_screeninfo +{ + uint32_t xres; /* visible resolution */ + uint32_t yres; + uint32_t xres_virtual; /* virtual resolution */ + uint32_t yres_virtual; + uint32_t xoffset; /* offset from virtual to visible */ + uint32_t yoffset; /* resolution */ + + uint32_t bits_per_pixel; /* guess what */ + uint32_t grayscale; /* != 0 Graylevels instead of colors */ + + struct fb_bitfield red; /* bitfield in fb mem if true color, */ + struct fb_bitfield green; /* else only length is significant */ + struct fb_bitfield blue; + struct fb_bitfield transp; /* transparency */ + + uint32_t nonstd; /* != 0 Non standard pixel format */ + + uint32_t activate; /* see FB_ACTIVATE_* */ + + uint32_t height; /* height of picture in mm */ + uint32_t width; /* width of picture in mm */ + + uint32_t accel_flags; /* acceleration flags (hints) */ + + /* Timing: All values in pixclocks, except pixclock (of course) */ + uint32_t pixclock; /* pixel clock in ps (pico seconds) */ + uint32_t left_margin; /* time from sync to picture */ + uint32_t right_margin; /* time from picture to sync */ + uint32_t upper_margin; /* time from sync to picture */ + uint32_t lower_margin; + uint32_t hsync_len; /* length of horizontal sync */ + uint32_t vsync_len; /* length of vertical sync */ + uint32_t sync; /* see FB_SYNC_* */ + uint32_t vmode; /* see FB_VMODE_* */ + uint32_t reserved[6]; /* Reserved for future compatibility */ +}; + +struct fb_cmap +{ + uint32_t start; /* First entry */ + uint32_t len; /* Number of entries */ + uint16_t *red; /* Red values */ + uint16_t *green; + uint16_t *blue; + uint16_t *transp; /* transparency, can be NULL */ +}; + +struct fb_con2fbmap +{ + uint32_t console; + uint32_t framebuffer; +}; + +struct fb_monspecs +{ + uint32_t hfmin; /* hfreq lower limit (Hz) */ + uint32_t hfmax; /* hfreq upper limit (Hz) */ + uint16_t vfmin; /* vfreq lower limit (Hz) */ + uint16_t vfmax; /* vfreq upper limit (Hz) */ + unsigned dpms : 1; /* supports DPMS */ +}; + +#if 1 + +#define FBCMD_GET_CURRENTPAR 0xDEAD0005 +#define FBCMD_SET_CURRENTPAR 0xDEAD8005 + +#endif + +#if 1 /* Preliminary */ + +/* + * Hardware Cursor + */ + +#define FBIOGET_FCURSORINFO 0x4607 +#define FBIOGET_VCURSORINFO 0x4608 +#define FBIOPUT_VCURSORINFO 0x4609 +#define FBIOGET_CURSORSTATE 0x460A +#define FBIOPUT_CURSORSTATE 0x460B + +struct fb_fix_cursorinfo +{ + uint16_t crsr_width; /* width and height of the cursor in */ + uint16_t crsr_height; /* pixels (zero if no cursor) */ + uint16_t crsr_xsize; /* cursor size in display pixels */ + uint16_t crsr_ysize; + uint16_t crsr_color1; /* colormap entry for cursor color1 */ + uint16_t crsr_color2; /* colormap entry for cursor color2 */ +}; + +struct fb_var_cursorinfo +{ + uint16_t width; + uint16_t height; + uint16_t xspot; + uint16_t yspot; + uint8_t data[1]; /* field with [height][width] */ +}; + +struct fb_cursorstate +{ + int16_t xoffset; + int16_t yoffset; + uint16_t mode; +}; + +#define FB_CURSOR_OFF 0 +#define FB_CURSOR_ON 1 +#define FB_CURSOR_FLASH 2 + +#endif /* Preliminary */ + +#endif /* _LINUX_FB_H */ diff --git a/kernel/kernel/fonts/.gitignore b/kernel/kernel/fonts/.gitignore new file mode 100644 index 000000000..064a8d8ef --- /dev/null +++ b/kernel/kernel/fonts/.gitignore @@ -0,0 +1 @@ +*.c diff --git a/kernel/kernel/fonts/Makefile b/kernel/kernel/fonts/Makefile new file mode 100644 index 000000000..fb81f82a4 --- /dev/null +++ b/kernel/kernel/fonts/Makefile @@ -0,0 +1,4 @@ +kernel/fonts/font.c: kernel/fonts/u_vga16.bdf.gz scripts/bdf2c.py + scripts/bdf2c.py kernel/fonts/u_vga16.bdf.gz $@ + +obj-y+= kernel/fonts/font.o diff --git a/kernel/kernel/fonts/font.c b/kernel/kernel/fonts/font.c deleted file mode 100644 index dde161557..000000000 --- a/kernel/kernel/fonts/font.c +++ /dev/null @@ -1,4661 +0,0 @@ -// Created from bdf2c Version 3, (c) 2009, 2010 by Lutz Sammer -// License AGPLv3: GNU Affero General Public License version 3 - -#include - -#include - -const unsigned char __cursor__bitmap[] = { - XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, - XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, ________, ________, -}; - -static const unsigned char __font_bitmap__[] = { - /* 0 $00 'C0000' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 1 $01 'C0001' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXXX_, - X______X, - X_X__X_X, - X______X, - X______X, - X_X__X_X, - X__XX__X, - X______X, - X______X, - _XXXXXX_, - ________, - ________, - ________, - ________, - /* 2 $02 'C0002' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXXX_, - XXXXXXXX, - XX_XX_XX, - XXXXXXXX, - XXXXXXXX, - XX_XX_XX, - XXX__XXX, - XXXXXXXX, - XXXXXXXX, - _XXXXXX_, - ________, - ________, - ________, - ________, - /* 3 $03 'C0003' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - _XX_XX__, - XXXXXXX_, - XXXXXXX_, - XXXXXXX_, - XXXXXXX_, - _XXXXX__, - __XXX___, - ___X____, - ________, - ________, - ________, - ________, - /* 4 $04 'C0004' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ___X____, - __XXX___, - _XXXXX__, - XXXXXXX_, - _XXXXX__, - __XXX___, - ___X____, - ________, - ________, - ________, - ________, - ________, - /* 5 $05 'C0005' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ___XX___, - __XXXX__, - __XXXX__, - XXX__XXX, - XXX__XXX, - XXX__XXX, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 6 $06 'C0006' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ___XX___, - __XXXX__, - _XXXXXX_, - XXXXXXXX, - XXXXXXXX, - _XXXXXX_, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 7 $07 'C0007' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ___XX___, - __XXXX__, - __XXXX__, - ___XX___, - ________, - ________, - ________, - ________, - ________, - ________, - /* 8 $08 'C0008' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXX__XXX, - XX____XX, - XX____XX, - XXX__XXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - /* 9 $09 'C0009' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - __XXXX__, - _XX__XX_, - _X____X_, - _X____X_, - _XX__XX_, - __XXXX__, - ________, - ________, - ________, - ________, - ________, - /* 10 $0a 'C000a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XX____XX, - X__XX__X, - X_XXXX_X, - X_XXXX_X, - X__XX__X, - XX____XX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - /* 11 $0b 'C000b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XXXX_, - _____XX_, - ____XXX_, - ___XX_X_, - _XXXX___, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXXX___, - ________, - ________, - ________, - ________, - /* 12 $0c 'C000c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXXX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - __XXXX__, - ___XX___, - _XXXXXX_, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 13 $0d 'C000d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXXXXX, - __XX__XX, - __XXXXXX, - __XX____, - __XX____, - __XX____, - __XX____, - _XXX____, - XXXX____, - XXX_____, - ________, - ________, - ________, - ________, - /* 14 $0e 'C000e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXXXX, - _XX___XX, - _XXXXXXX, - _XX___XX, - _XX___XX, - _XX___XX, - _XX___XX, - _XX__XXX, - XXX__XXX, - XXX__XX_, - XX______, - ________, - ________, - ________, - /* 15 $0f 'C000f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ___XX___, - ___XX___, - XX_XX_XX, - __XXXX__, - XXX__XXX, - __XXXX__, - XX_XX_XX, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 16 $10 'C0010' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - X_______, - XX______, - XXX_____, - XXXX____, - XXXXX___, - XXXXXXX_, - XXXXX___, - XXXX____, - XXX_____, - XX______, - X_______, - ________, - ________, - ________, - ________, - /* 17 $11 'C0011' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ______X_, - _____XX_, - ____XXX_, - ___XXXX_, - __XXXXX_, - XXXXXXX_, - __XXXXX_, - ___XXXX_, - ____XXX_, - _____XX_, - ______X_, - ________, - ________, - ________, - ________, - /* 18 $12 'C0012' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XX___, - __XXXX__, - _XXXXXX_, - ___XX___, - ___XX___, - ___XX___, - _XXXXXX_, - __XXXX__, - ___XX___, - ________, - ________, - ________, - ________, - ________, - /* 19 $13 'C0013' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - ________, - _XX__XX_, - _XX__XX_, - ________, - ________, - ________, - ________, - /* 20 $14 'C0014' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXXXX, - XX_XX_XX, - XX_XX_XX, - XX_XX_XX, - _XXXX_XX, - ___XX_XX, - ___XX_XX, - ___XX_XX, - ___XX_XX, - ___XX_XX, - ________, - ________, - ________, - ________, - /* 21 $15 'C0015' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - _XXXXX__, - XX___XX_, - _XX_____, - __XXX___, - _XX_XX__, - XX___XX_, - XX___XX_, - _XX_XX__, - __XXX___, - ____XX__, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - /* 22 $16 'C0016' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXX_, - XXXXXXX_, - XXXXXXX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - /* 23 $17 'C0017' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XX___, - __XXXX__, - _XXXXXX_, - ___XX___, - ___XX___, - ___XX___, - _XXXXXX_, - __XXXX__, - ___XX___, - _XXXXXX_, - ________, - ________, - ________, - ________, - /* 24 $18 'C0018' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XX___, - __XXXX__, - _XXXXXX_, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 25 $19 'C0019' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - _XXXXXX_, - __XXXX__, - ___XX___, - ________, - ________, - ________, - ________, - /* 26 $1a 'C001a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ___XX___, - ____XX__, - XXXXXXX_, - ____XX__, - ___XX___, - ________, - ________, - ________, - ________, - ________, - ________, - /* 27 $1b 'C001b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - __XX____, - _XX_____, - XXXXXXX_, - _XX_____, - __XX____, - ________, - ________, - ________, - ________, - ________, - ________, - /* 28 $1c 'C001c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - XX______, - XX______, - XX______, - XXXXXXX_, - ________, - ________, - ________, - ________, - ________, - ________, - /* 29 $1d 'C001d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - __X_X___, - _XX_XX__, - XXXXXXX_, - _XX_XX__, - __X_X___, - ________, - ________, - ________, - ________, - ________, - ________, - /* 30 $1e 'C001e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ___X____, - __XXX___, - __XXX___, - _XXXXX__, - _XXXXX__, - XXXXXXX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - ________, - /* 31 $1f 'C001f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - XXXXXXX_, - XXXXXXX_, - _XXXXX__, - _XXXXX__, - __XXX___, - __XXX___, - ___X____, - ________, - ________, - ________, - ________, - ________, - /* 32 $20 'C0020' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 33 $21 'C0021' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XX___, - __XXXX__, - __XXXX__, - __XXXX__, - ___XX___, - ___XX___, - ___XX___, - ________, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 34 $22 'C0022' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - _XX__XX_, - _XX__XX_, - _XX__XX_, - __X__X__, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 35 $23 'C0023' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - _XX_XX__, - _XX_XX__, - XXXXXXX_, - _XX_XX__, - _XX_XX__, - _XX_XX__, - XXXXXXX_, - _XX_XX__, - _XX_XX__, - ________, - ________, - ________, - ________, - /* 36 $24 'C0024' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - _XXXXX__, - XX___XX_, - XX____X_, - XX______, - _XXXXX__, - _____XX_, - _____XX_, - X____XX_, - XX___XX_, - _XXXXX__, - ___XX___, - ___XX___, - ________, - ________, - /* 37 $25 'C0025' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - XX____X_, - XX___XX_, - ____XX__, - ___XX___, - __XX____, - _XX_____, - XX___XX_, - X____XX_, - ________, - ________, - ________, - ________, - /* 38 $26 'C0026' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXX___, - _XX_XX__, - _XX_XX__, - __XXX___, - _XXX_XX_, - XX_XXX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 39 $27 'C0027' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - __XX____, - __XX____, - __XX____, - _XX_____, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 40 $28 'C0028' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ____XX__, - ___XX___, - __XX____, - __XX____, - __XX____, - __XX____, - __XX____, - __XX____, - ___XX___, - ____XX__, - ________, - ________, - ________, - ________, - /* 41 $29 'C0029' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XX____, - ___XX___, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - ___XX___, - __XX____, - ________, - ________, - ________, - ________, - /* 42 $2a 'C002a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XX__XX_, - __XXXX__, - XXXXXXXX, - __XXXX__, - _XX__XX_, - ________, - ________, - ________, - ________, - ________, - ________, - /* 43 $2b 'C002b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ___XX___, - ___XX___, - _XXXXXX_, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - ________, - ________, - /* 44 $2c 'C002c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ___XX___, - ___XX___, - ___XX___, - __XX____, - ________, - ________, - ________, - /* 45 $2d 'C002d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXX_, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 46 $2e 'C002e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 47 $2f 'C002f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ______X_, - _____XX_, - ____XX__, - ___XX___, - __XX____, - _XX_____, - XX______, - X_______, - ________, - ________, - ________, - ________, - /* 48 $30 'C0030' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXX___, - _XX_XX__, - XX___XX_, - XX___XX_, - XX_X_XX_, - XX_X_XX_, - XX___XX_, - XX___XX_, - _XX_XX__, - __XXX___, - ________, - ________, - ________, - ________, - /* 49 $31 'C0031' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XX___, - __XXX___, - _XXXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - _XXXXXX_, - ________, - ________, - ________, - ________, - /* 50 $32 'C0032' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXX__, - XX___XX_, - _____XX_, - ____XX__, - ___XX___, - __XX____, - _XX_____, - XX______, - XX___XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - /* 51 $33 'C0033' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXX__, - XX___XX_, - _____XX_, - _____XX_, - __XXXX__, - _____XX_, - _____XX_, - _____XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 52 $34 'C0034' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ____XX__, - ___XXX__, - __XXXX__, - _XX_XX__, - XX__XX__, - XXXXXXX_, - ____XX__, - ____XX__, - ____XX__, - ___XXXX_, - ________, - ________, - ________, - ________, - /* 53 $35 'C0035' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXXXX_, - XX______, - XX______, - XX______, - XXXXXX__, - _____XX_, - _____XX_, - _____XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 54 $36 'C0036' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXX___, - _XX_____, - XX______, - XX______, - XXXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 55 $37 'C0037' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXXXX_, - XX___XX_, - _____XX_, - _____XX_, - ____XX__, - ___XX___, - __XX____, - __XX____, - __XX____, - __XX____, - ________, - ________, - ________, - ________, - /* 56 $38 'C0038' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 57 $39 'C0039' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXXX_, - _____XX_, - _____XX_, - _____XX_, - ____XX__, - _XXXX___, - ________, - ________, - ________, - ________, - /* 58 $3a 'C003a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ___XX___, - ___XX___, - ________, - ________, - ________, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - ________, - /* 59 $3b 'C003b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ___XX___, - ___XX___, - ________, - ________, - ________, - ___XX___, - ___XX___, - __XX____, - ________, - ________, - ________, - ________, - /* 60 $3c 'C003c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - _____XX_, - ____XX__, - ___XX___, - __XX____, - _XX_____, - __XX____, - ___XX___, - ____XX__, - _____XX_, - ________, - ________, - ________, - ________, - /* 61 $3d 'C003d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXXXXX_, - ________, - ________, - _XXXXXX_, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 62 $3e 'C003e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - _XX_____, - __XX____, - ___XX___, - ____XX__, - _____XX_, - ____XX__, - ___XX___, - __XX____, - _XX_____, - ________, - ________, - ________, - ________, - /* 63 $3f 'C003f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - ____XX__, - ___XX___, - ___XX___, - ___XX___, - ________, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 64 $40 'C0040' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX_XXXX_, - XX_XXXX_, - XX_XXXX_, - XX_XXX__, - XX______, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 65 $41 'C0041' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___X____, - __XXX___, - _XX_XX__, - XX___XX_, - XX___XX_, - XXXXXXX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 66 $42 'C0042' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXXX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XXXXX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - XXXXXX__, - ________, - ________, - ________, - ________, - /* 67 $43 'C0043' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXXX__, - _XX__XX_, - XX____X_, - XX______, - XX______, - XX______, - XX______, - XX____X_, - _XX__XX_, - __XXXX__, - ________, - ________, - ________, - ________, - /* 68 $44 'C0044' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXX___, - _XX_XX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX_XX__, - XXXXX___, - ________, - ________, - ________, - ________, - /* 69 $45 'C0045' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXXXX_, - _XX__XX_, - _XX___X_, - _XX_X___, - _XXXX___, - _XX_X___, - _XX_____, - _XX___X_, - _XX__XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - /* 70 $46 'C0046' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXXXX_, - _XX__XX_, - _XX___X_, - _XX_X___, - _XXXX___, - _XX_X___, - _XX_____, - _XX_____, - _XX_____, - XXXX____, - ________, - ________, - ________, - ________, - /* 71 $47 'C0047' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXXX__, - _XX__XX_, - XX____X_, - XX______, - XX______, - XX_XXXX_, - XX___XX_, - XX___XX_, - _XX__XX_, - __XXX_X_, - ________, - ________, - ________, - ________, - /* 72 $48 'C0048' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XXXXXXX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 73 $49 'C0049' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXXX__, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 74 $4a 'C004a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XXXX_, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXXX___, - ________, - ________, - ________, - ________, - /* 75 $4b 'C004b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXX__XX_, - _XX__XX_, - _XX__XX_, - _XX_XX__, - _XXXX___, - _XXXX___, - _XX_XX__, - _XX__XX_, - _XX__XX_, - XXX__XX_, - ________, - ________, - ________, - ________, - /* 76 $4c 'C004c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXX____, - _XX_____, - _XX_____, - _XX_____, - _XX_____, - _XX_____, - _XX_____, - _XX___X_, - _XX__XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - /* 77 $4d 'C004d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - XXX_XXX_, - XXXXXXX_, - XXXXXXX_, - XX_X_XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 78 $4e 'C004e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - XXX__XX_, - XXXX_XX_, - XXXXXXX_, - XX_XXXX_, - XX__XXX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 79 $4f 'C004f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 80 $50 'C0050' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXXX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XXXXX__, - _XX_____, - _XX_____, - _XX_____, - _XX_____, - XXXX____, - ________, - ________, - ________, - ________, - /* 81 $51 'C0051' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX_X_XX_, - XX_XXXX_, - _XXXXX__, - ____XX__, - ____XXX_, - ________, - ________, - /* 82 $52 'C0052' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXXX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XXXXX__, - _XX_XX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - XXX__XX_, - ________, - ________, - ________, - ________, - /* 83 $53 'C0053' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - _XX_____, - __XXX___, - ____XX__, - _____XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 84 $54 'C0054' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXXX_, - _XXXXXX_, - _X_XX_X_, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 85 $55 'C0055' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 86 $56 'C0056' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XX_XX__, - __XXX___, - ___X____, - ________, - ________, - ________, - ________, - /* 87 $57 'C0057' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX_X_XX_, - XX_X_XX_, - XX_X_XX_, - XXXXXXX_, - XXX_XXX_, - _XX_XX__, - ________, - ________, - ________, - ________, - /* 88 $58 'C0058' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - XX___XX_, - _XX_XX__, - _XXXXX__, - __XXX___, - __XXX___, - _XXXXX__, - _XX_XX__, - XX___XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 89 $59 'C0059' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - __XXXX__, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 90 $5a 'C005a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXXXX_, - XX___XX_, - X____XX_, - ____XX__, - ___XX___, - __XX____, - _XX_____, - XX____X_, - XX___XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - /* 91 $5b 'C005b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXXX__, - __XX____, - __XX____, - __XX____, - __XX____, - __XX____, - __XX____, - __XX____, - __XX____, - __XXXX__, - ________, - ________, - ________, - ________, - /* 92 $5c 'C005c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - X_______, - XX______, - XXX_____, - _XXX____, - __XXX___, - ___XXX__, - ____XXX_, - _____XX_, - ______X_, - ________, - ________, - ________, - ________, - /* 93 $5d 'C005d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXXX__, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - __XXXX__, - ________, - ________, - ________, - ________, - /* 94 $5e 'C005e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___X____, - __XXX___, - _XX_XX__, - XX___XX_, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 95 $5f 'C005f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXXX, - ________, - ________, - /* 96 $60 'C0060' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX____, - __XX____, - ___XX___, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 97 $61 'C0061' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXXX___, - ____XX__, - _XXXXX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 98 $62 'C0062' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXX_____, - _XX_____, - _XX_____, - _XXXX___, - _XX_XX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 99 $63 'C0063' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXXXX__, - XX___XX_, - XX______, - XX______, - XX______, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 100 $64 'C0064' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XXX__, - ____XX__, - ____XX__, - __XXXX__, - _XX_XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 101 $65 'C0065' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXXXX__, - XX___XX_, - XXXXXXX_, - XX______, - XX______, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 102 $66 'C0066' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXX___, - _XX_XX__, - _XX__X__, - _XX_____, - XXXX____, - _XX_____, - _XX_____, - _XX_____, - _XX_____, - XXXX____, - ________, - ________, - ________, - ________, - /* 103 $67 'C0067' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXX_XX_, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXXXX__, - ____XX__, - XX__XX__, - _XXXX___, - ________, - /* 104 $68 'C0068' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXX_____, - _XX_____, - _XX_____, - _XX_XX__, - _XXX_XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - XXX__XX_, - ________, - ________, - ________, - ________, - /* 105 $69 'C0069' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XX___, - ___XX___, - ________, - __XXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 106 $6a 'C006a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _____XX_, - _____XX_, - ________, - ____XXX_, - _____XX_, - _____XX_, - _____XX_, - _____XX_, - _____XX_, - _____XX_, - _XX__XX_, - _XX__XX_, - __XXXX__, - ________, - /* 107 $6b 'C006b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXX_____, - _XX_____, - _XX_____, - _XX__XX_, - _XX_XX__, - _XXXX___, - _XXXX___, - _XX_XX__, - _XX__XX_, - XXX__XX_, - ________, - ________, - ________, - ________, - /* 108 $6c 'C006c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 109 $6d 'C006d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XXX_XX__, - XXXXXXX_, - XX_X_XX_, - XX_X_XX_, - XX_X_XX_, - XX_X_XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 110 $6e 'C006e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XX_XXX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - ________, - ________, - ________, - ________, - /* 111 $6f 'C006f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 112 $70 'C0070' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XX_XXX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XXXXX__, - _XX_____, - _XX_____, - XXXX____, - ________, - /* 113 $71 'C0071' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXX_XX_, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXXXX__, - ____XX__, - ____XX__, - ___XXXX_, - ________, - /* 114 $72 'C0072' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XX_XXX__, - _XXX_XX_, - _XX__XX_, - _XX_____, - _XX_____, - _XX_____, - XXXX____, - ________, - ________, - ________, - ________, - /* 115 $73 'C0073' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXXXX__, - XX___XX_, - _XX_____, - __XXX___, - ____XX__, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 116 $74 'C0074' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___X____, - __XX____, - __XX____, - XXXXXX__, - __XX____, - __XX____, - __XX____, - __XX____, - __XX_XX_, - ___XXX__, - ________, - ________, - ________, - ________, - /* 117 $75 'C0075' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 118 $76 'C0076' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - __XXXX__, - ___XX___, - ________, - ________, - ________, - ________, - /* 119 $77 'C0077' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XX___XX_, - XX___XX_, - XX_X_XX_, - XX_X_XX_, - XX_X_XX_, - XXXXXXX_, - _XX_XX__, - ________, - ________, - ________, - ________, - /* 120 $78 'C0078' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XX___XX_, - _XX_XX__, - __XXX___, - __XXX___, - __XXX___, - _XX_XX__, - XX___XX_, - ________, - ________, - ________, - ________, - /* 121 $79 'C0079' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXXX_, - _____XX_, - ____XX__, - XXXXX___, - ________, - /* 122 $7a 'C007a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XXXXXXX_, - XX__XX__, - ___XX___, - __XX____, - _XX_____, - XX___XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - /* 123 $7b 'C007b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ____XXX_, - ___XX___, - ___XX___, - ___XX___, - _XXX____, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ____XXX_, - ________, - ________, - ________, - ________, - /* 124 $7c 'C007c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ________, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 125 $7d 'C007d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXX____, - ___XX___, - ___XX___, - ___XX___, - ____XXX_, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - _XXX____, - ________, - ________, - ________, - ________, - /* 126 $7e 'C007e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXX_XX_, - XX_XXX__, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 127 $7f 'C007f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ___X____, - __XXX___, - _XX_XX__, - XX___XX_, - XX___XX_, - XX___XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - ________, - /* 128 $80 'C0080' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXXX__, - _XX__XX_, - XX____X_, - XX______, - XX______, - XX______, - XX____X_, - _XX__XX_, - __XXXX__, - ____XX__, - _____XX_, - _XXXXX__, - ________, - ________, - /* 129 $81 'C0081' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX__XX__, - ________, - ________, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 130 $82 'C0082' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ____XX__, - ___XX___, - __XX____, - ________, - _XXXXX__, - XX___XX_, - XXXXXXX_, - XX______, - XX______, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 131 $83 'C0083' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ___X____, - __XXX___, - _XX_XX__, - ________, - _XXXX___, - ____XX__, - _XXXXX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 132 $84 'C0084' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX__XX__, - ________, - ________, - _XXXX___, - ____XX__, - _XXXXX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 133 $85 'C0085' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - _XX_____, - __XX____, - ___XX___, - ________, - _XXXX___, - ____XX__, - _XXXXX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 134 $86 'C0086' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - __XXX___, - _XX_XX__, - __XXX___, - ________, - _XXXX___, - ____XX__, - _XXXXX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 135 $87 'C0087' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - __XXXX__, - _XX__XX_, - _XX_____, - _XX_____, - _XX__XX_, - __XXXX__, - ____XX__, - _____XX_, - __XXXX__, - ________, - ________, - ________, - /* 136 $88 'C0088' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ___X____, - __XXX___, - _XX_XX__, - ________, - _XXXXX__, - XX___XX_, - XXXXXXX_, - XX______, - XX______, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 137 $89 'C0089' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - ________, - ________, - _XXXXX__, - XX___XX_, - XXXXXXX_, - XX______, - XX______, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 138 $8a 'C008a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - _XX_____, - __XX____, - ___XX___, - ________, - _XXXXX__, - XX___XX_, - XXXXXXX_, - XX______, - XX______, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 139 $8b 'C008b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XX__XX_, - ________, - ________, - __XXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 140 $8c 'C008c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ___XX___, - __XXXX__, - _XX__XX_, - ________, - __XXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 141 $8d 'C008d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - _XX_____, - __XX____, - ___XX___, - ________, - __XXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 142 $8e 'C008e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - XX___XX_, - ________, - ___X____, - __XXX___, - _XX_XX__, - XX___XX_, - XX___XX_, - XXXXXXX_, - XX___XX_, - XX___XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 143 $8f 'C008f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XXX___, - _XX_XX__, - __XXX___, - ________, - __XXX___, - _XX_XX__, - XX___XX_, - XX___XX_, - XXXXXXX_, - XX___XX_, - XX___XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 144 $90 'C0090' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - __XX____, - _XX_____, - ________, - XXXXXXX_, - _XX__XX_, - _XX_____, - _XXXXX__, - _XX_____, - _XX_____, - _XX__XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - /* 145 $91 'C0091' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XX__XX__, - _XXX_XX_, - __XX_XX_, - _XXXXXX_, - XX_XX___, - XX_XX___, - _XX_XXX_, - ________, - ________, - ________, - ________, - /* 146 $92 'C0092' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXXXX_, - _XX_XX__, - XX__XX__, - XX__XX__, - XXXXXXX_, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XXX_, - ________, - ________, - ________, - ________, - /* 147 $93 'C0093' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ___X____, - __XXX___, - _XX_XX__, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 148 $94 'C0094' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 149 $95 'C0095' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - _XX_____, - __XX____, - ___XX___, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 150 $96 'C0096' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - __XX____, - _XXXX___, - XX__XX__, - ________, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 151 $97 'C0097' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - _XX_____, - __XX____, - ___XX___, - ________, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 152 $98 'C0098' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - ________, - ________, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXXX_, - _____XX_, - ____XX__, - _XXXX___, - ________, - /* 153 $99 'C0099' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - XX___XX_, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 154 $9a 'C009a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - XX___XX_, - ________, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 155 $9b 'C009b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ___XX___, - ___XX___, - __XXXX__, - _XX__XX_, - _XX_____, - _XX_____, - _XX_____, - _XX__XX_, - __XXXX__, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 156 $9c 'C009c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - __XXX___, - _XX_XX__, - _XX__X__, - _XX_____, - XXXX____, - _XX_____, - _XX_____, - _XX_____, - _XX_____, - XXX__XX_, - XXXXXX__, - ________, - ________, - ________, - ________, - /* 157 $9d 'C009d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XX__XX_, - _XX__XX_, - __XXXX__, - ___XX___, - _XXXXXX_, - ___XX___, - _XXXXXX_, - ___XX___, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 158 $9e 'C009e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - XXXXX___, - XX__XX__, - XX__XX__, - XXXXX___, - XX___X__, - XX__XX__, - XX_XXXX_, - XX__XX__, - XX__XX__, - XX__XX__, - XX___XX_, - ________, - ________, - ________, - ________, - /* 159 $9f 'C009f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ____XXX_, - ___XX_XX, - ___XX___, - ___XX___, - ___XX___, - _XXXXXX_, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XX_XX___, - _XXX____, - ________, - ________, - /* 160 $a0 'C00a0' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ___XX___, - __XX____, - _XX_____, - ________, - _XXXX___, - ____XX__, - _XXXXX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 161 $a1 'C00a1' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ____XX__, - ___XX___, - __XX____, - ________, - __XXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 162 $a2 'C00a2' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ___XX___, - __XX____, - _XX_____, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 163 $a3 'C00a3' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ___XX___, - __XX____, - _XX_____, - ________, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 164 $a4 'C00a4' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXX_XX_, - XX_XXX__, - ________, - XX_XXX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - ________, - ________, - ________, - ________, - /* 165 $a5 'C00a5' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - _XXX_XX_, - XX_XXX__, - ________, - XX___XX_, - XXX__XX_, - XXXX_XX_, - XXXXXXX_, - XX_XXXX_, - XX__XXX_, - XX___XX_, - XX___XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 166 $a6 'C00a6' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - __XXXX__, - _XX_XX__, - _XX_XX__, - __XXXXX_, - ________, - _XXXXXX_, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 167 $a7 'C00a7' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - __XXX___, - _XX_XX__, - _XX_XX__, - __XXX___, - ________, - _XXXXX__, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 168 $a8 'C00a8' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XX____, - __XX____, - ________, - __XX____, - __XX____, - _XX_____, - XX______, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 169 $a9 'C00a9' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXX_, - XX______, - XX______, - XX______, - XX______, - ________, - ________, - ________, - ________, - ________, - /* 170 $aa 'C00aa' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXX_, - _____XX_, - _____XX_, - _____XX_, - _____XX_, - ________, - ________, - ________, - ________, - ________, - /* 171 $ab 'C00ab' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - XX______, - XX______, - XX____X_, - XX___XX_, - XX__XX__, - ___XX___, - __XX____, - _XX_____, - XX_XXX__, - X____XX_, - ____XX__, - ___XX___, - __XXXXX_, - ________, - ________, - /* 172 $ac 'C00ac' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - XX______, - XX______, - XX____X_, - XX___XX_, - XX__XX__, - ___XX___, - __XX____, - _XX__XX_, - XX__XXX_, - X__XXXX_, - __XXXXX_, - _____XX_, - _____XX_, - ________, - ________, - /* 173 $ad 'C00ad' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XX___, - ___XX___, - ________, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - __XXXX__, - __XXXX__, - ___XX___, - ________, - ________, - ________, - ________, - /* 174 $ae 'C00ae' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - __XX_XX_, - _XX_XX__, - XX_XX___, - _XX_XX__, - __XX_XX_, - ________, - ________, - ________, - ________, - ________, - ________, - /* 175 $af 'C00af' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XX_XX___, - _XX_XX__, - __XX_XX_, - _XX_XX__, - XX_XX___, - ________, - ________, - ________, - ________, - ________, - ________, - /* 176 $b0 'C00b0' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___X___X, - _X___X__, - ___X___X, - _X___X__, - ___X___X, - _X___X__, - ___X___X, - _X___X__, - ___X___X, - _X___X__, - ___X___X, - _X___X__, - ___X___X, - _X___X__, - ___X___X, - _X___X__, - /* 177 $b1 'C00b1' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - _X_X_X_X, - X_X_X_X_, - _X_X_X_X, - X_X_X_X_, - _X_X_X_X, - X_X_X_X_, - _X_X_X_X, - X_X_X_X_, - _X_X_X_X, - X_X_X_X_, - _X_X_X_X, - X_X_X_X_, - _X_X_X_X, - X_X_X_X_, - _X_X_X_X, - X_X_X_X_, - /* 178 $b2 'C00b2' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - XX_XXX_X, - _XXX_XXX, - XX_XXX_X, - _XXX_XXX, - XX_XXX_X, - _XXX_XXX, - XX_XXX_X, - _XXX_XXX, - XX_XXX_X, - _XXX_XXX, - XX_XXX_X, - _XXX_XXX, - XX_XXX_X, - _XXX_XXX, - XX_XXX_X, - _XXX_XXX, - /* 179 $b3 'C00b3' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 180 $b4 'C00b4' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XXXXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 181 $b5 'C00b5' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XXXXX___, - ___XX___, - XXXXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 182 $b6 'C00b6' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - XXXX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 183 $b7 'C00b7' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 184 $b8 'C00b8' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XXXXX___, - ___XX___, - XXXXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 185 $b9 'C00b9' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - XXXX_XX_, - _____XX_, - XXXX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 186 $ba 'C00ba' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 187 $bb 'C00bb' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XXXXXXX_, - _____XX_, - XXXX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 188 $bc 'C00bc' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - XXXX_XX_, - _____XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 189 $bd 'C00bd' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 190 $be 'C00be' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XXXXX___, - ___XX___, - XXXXX___, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 191 $bf 'C00bf' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - XXXXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 192 $c0 'C00c0' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 193 $c1 'C00c1' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XXXXXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 194 $c2 'C00c2' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXXX, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 195 $c3 'C00c3' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XXXXX, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 196 $c4 'C00c4' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 197 $c5 'C00c5' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XXXXXXXX, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 198 $c6 'C00c6' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XXXXX, - ___XX___, - ___XXXXX, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 199 $c7 'C00c7' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XXX, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 200 $c8 'C00c8' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XXX, - __XX____, - __XXXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 201 $c9 'C00c9' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - __XXXXXX, - __XX____, - __XX_XXX, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 202 $ca 'C00ca' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - XXXX_XXX, - ________, - XXXXXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 203 $cb 'C00cb' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XXXXXXXX, - ________, - XXXX_XXX, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 204 $cc 'C00cc' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XXX, - __XX____, - __XX_XXX, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 205 $cd 'C00cd' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XXXXXXXX, - ________, - XXXXXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 206 $ce 'C00ce' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - XXXX_XXX, - ________, - XXXX_XXX, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 207 $cf 'C00cf' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XXXXXXXX, - ________, - XXXXXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 208 $d0 'C00d0' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - XXXXXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 209 $d1 'C00d1' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XXXXXXXX, - ________, - XXXXXXXX, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 210 $d2 'C00d2' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXXX, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 211 $d3 'C00d3' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XXXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 212 $d4 'C00d4' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XXXXX, - ___XX___, - ___XXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 213 $d5 'C00d5' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ___XXXXX, - ___XX___, - ___XXXXX, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 214 $d6 'C00d6' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - __XXXXXX, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 215 $d7 'C00d7' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - XXXXXXXX, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 216 $d8 'C00d8' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XXXXXXXX, - ___XX___, - XXXXXXXX, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 217 $d9 'C00d9' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XXXXX___, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 218 $da 'C00da' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ___XXXXX, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 219 $db 'C00db' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - /* 220 $dc 'C00dc' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - /* 221 $dd 'C00dd' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - /* 222 $de 'C00de' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - /* 223 $df 'C00df' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 224 $e0 'C00e0' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXX_XX_, - XX_XXX__, - XX_XX___, - XX_XX___, - XX_XX___, - XX_XXX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 225 $e1 'C00e1' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXX___, - XX__XX__, - XX__XX__, - XX__XX__, - XX_XX___, - XX__XX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX__XX__, - ________, - ________, - ________, - ________, - /* 226 $e2 'C00e2' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXXXX_, - XX___XX_, - XX___XX_, - XX______, - XX______, - XX______, - XX______, - XX______, - XX______, - XX______, - ________, - ________, - ________, - ________, - /* 227 $e3 'C00e3' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - XXXXXXX_, - _XX_XX__, - _XX_XX__, - _XX_XX__, - _XX_XX__, - _XX_XX__, - _XX_XX__, - _XX_XX__, - ________, - ________, - ________, - ________, - /* 228 $e4 'C00e4' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - XXXXXXX_, - XX___XX_, - _XX_____, - __XX____, - ___XX___, - __XX____, - _XX_____, - XX___XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - /* 229 $e5 'C00e5' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXXXXX_, - XX_XX___, - XX_XX___, - XX_XX___, - XX_XX___, - XX_XX___, - _XXX____, - ________, - ________, - ________, - ________, - /* 230 $e6 'C00e6' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XXXXX__, - _XX_____, - _XX_____, - XX______, - ________, - ________, - ________, - /* 231 $e7 'C00e7' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - _XXX_XX_, - XX_XXX__, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 232 $e8 'C00e8' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - _XXXXXX_, - ___XX___, - __XXXX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - __XXXX__, - ___XX___, - _XXXXXX_, - ________, - ________, - ________, - ________, - /* 233 $e9 'C00e9' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - __XXX___, - _XX_XX__, - XX___XX_, - XX___XX_, - XXXXXXX_, - XX___XX_, - XX___XX_, - _XX_XX__, - __XXX___, - ________, - ________, - ________, - ________, - /* 234 $ea 'C00ea' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXX___, - _XX_XX__, - XX___XX_, - XX___XX_, - XX___XX_, - _XX_XX__, - _XX_XX__, - _XX_XX__, - _XX_XX__, - XXX_XXX_, - ________, - ________, - ________, - ________, - /* 235 $eb 'C00eb' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XXXX_, - __XX____, - ___XX___, - ____XX__, - __XXXXX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - __XXXX__, - ________, - ________, - ________, - ________, - /* 236 $ec 'C00ec' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXXXXX_, - XX_XX_XX, - XX_XX_XX, - XX_XX_XX, - _XXXXXX_, - ________, - ________, - ________, - ________, - ________, - ________, - /* 237 $ed 'C00ed' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ______XX, - _____XX_, - _XXXXXX_, - XX_XX_XX, - XX_XX_XX, - XXXX__XX, - _XXXXXX_, - _XX_____, - XX______, - ________, - ________, - ________, - ________, - /* 238 $ee 'C00ee' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XXX__, - __XX____, - _XX_____, - _XX_____, - _XXXXX__, - _XX_____, - _XX_____, - _XX_____, - __XX____, - ___XXX__, - ________, - ________, - ________, - ________, - /* 239 $ef 'C00ef' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 240 $f0 'C00f0' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - XXXXXXX_, - ________, - ________, - XXXXXXX_, - ________, - ________, - XXXXXXX_, - ________, - ________, - ________, - ________, - ________, - /* 241 $f1 'C00f1' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ___XX___, - ___XX___, - _XXXXXX_, - ___XX___, - ___XX___, - ________, - ________, - XXXXXXXX, - ________, - ________, - ________, - ________, - /* 242 $f2 'C00f2' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - __XX____, - ___XX___, - ____XX__, - _____XX_, - ____XX__, - ___XX___, - __XX____, - ________, - _XXXXXX_, - ________, - ________, - ________, - ________, - /* 243 $f3 'C00f3' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ____XX__, - ___XX___, - __XX____, - _XX_____, - __XX____, - ___XX___, - ____XX__, - ________, - _XXXXXX_, - ________, - ________, - ________, - ________, - /* 244 $f4 'C00f4' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ____XXX_, - ___XX_XX, - ___XX_XX, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 245 $f5 'C00f5' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XX_XX___, - XX_XX___, - XX_XX___, - _XXX____, - ________, - ________, - ________, - ________, - /* 246 $f6 'C00f6' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ___XX___, - ___XX___, - ________, - _XXXXXX_, - ________, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - ________, - /* 247 $f7 'C00f7' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXX_XX_, - XX_XXX__, - ________, - _XXX_XX_, - XX_XXX__, - ________, - ________, - ________, - ________, - ________, - ________, - /* 248 $f8 'C00f8' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - __XXX___, - _XX_XX__, - _XX_XX__, - __XXX___, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 249 $f9 'C00f9' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 250 $fa 'C00fa' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ___XX___, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 251 $fb 'C00fb' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ____XXXX, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - XXX_XX__, - _XX_XX__, - _XX_XX__, - __XXXX__, - ___XXX__, - ________, - ________, - ________, - ________, - /* 252 $fc 'C00fc' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - XX_XX___, - _XX_XX__, - _XX_XX__, - _XX_XX__, - _XX_XX__, - _XX_XX__, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 253 $fd 'C00fd' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - _XXX____, - XX_XX___, - __XX____, - _XX_____, - XX__X___, - XXXXX___, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 254 $fe 'C00fe' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - _XXXXX__, - _XXXXX__, - _XXXXX__, - _XXXXX__, - _XXXXX__, - _XXXXX__, - _XXXXX__, - ________, - ________, - ________, - ________, - ________, - /* 255 $ff 'C00ff' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, -}; - -/*/ character width for each encoding */ -const unsigned char __font_widths__[] = { - 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, -}; - -/*/ character encoding for each index entry */ -const unsigned short __font_index__[] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, - 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, - 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, - 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, - 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, - 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, - 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, - 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, - 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, - 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, -}; - -static unsigned int font_mask[8] = {128, 64, 32, 16, 8, 4, 2, 1}; -/* bitmap font structure */ -struct font boot_font = {.width = 8, - .height = 16, - .chars = 256, - .font_bitmap = (void *) __font_bitmap__, - .mask = font_mask, - .cursor_bitmap = (void *) __cursor__bitmap}; diff --git a/kernel/kernel/fonts/u_vga16.bdf.gz b/kernel/kernel/fonts/u_vga16.bdf.gz new file mode 100644 index 000000000..59b9c3f28 Binary files /dev/null and b/kernel/kernel/fonts/u_vga16.bdf.gz differ diff --git a/kernel/kernel/framebuffer.cpp b/kernel/kernel/framebuffer.cpp index a5e1a5ba2..bbe273049 100644 --- a/kernel/kernel/framebuffer.cpp +++ b/kernel/kernel/framebuffer.cpp @@ -5,11 +5,17 @@ */ #include +#include #include +#include +#include +#include -struct framebuffer *primary_fb = NULL; +#include "fbpriv.h" -struct framebuffer *get_primary_framebuffer(void) +struct framebuffer *primary_fb = nullptr; + +struct framebuffer *get_primary_framebuffer() { return primary_fb; } @@ -18,3 +24,108 @@ void set_framebuffer(struct framebuffer *fb) { primary_fb = fb; } + +int fbdev_do_fscreeninfo(void *argp) +{ + fb_fix_screeninfo info; + strcpy(info.id, "bootfb"); + info.accel = FB_ACCEL_NONE; + info.line_length = primary_fb->pitch; + info.xpanstep = 0; + info.ypanstep = 0; + info.ywrapstep = 0; + info.visual = FB_VISUAL_TRUECOLOR; + info.type = FB_TYPE_PACKED_PIXELS; + info.type_aux = 0; + info.smem_start = (char *) primary_fb->framebuffer_phys; + info.smem_len = primary_fb->framebuffer_size; + info.mmio_start = info.smem_start; + info.mmio_len = info.smem_len; + info.reserved[0] = 0; + + return copy_to_user(argp, &info, sizeof(info)); +} + +int fbdev_do_vscreeninfo(void *argp) +{ + fb_var_screeninfo info; + info.xres = primary_fb->width; + info.yres = primary_fb->height; + info.bits_per_pixel = primary_fb->bpp; + info.xres_virtual = info.xres; + info.yres_virtual = info.yres; + info.yoffset = 0; + info.xoffset = 0; + info.pixclock = 25000000 / info.xres * 2000 / info.yres; + info.left_margin = (info.xres / 8) & 0xf8; + info.hsync_len = info.left_margin; + info.red.offset = primary_fb->color.red_shift; + info.red.length = 8; + info.green.offset = primary_fb->color.green_shift; + info.green.length = 8; + info.blue.offset = primary_fb->color.blue_shift; + info.blue.length = 8; + info.transp.offset = primary_fb->color.resv_shift; + info.transp.length = 8; + info.red.msb_right = 0; + info.green.msb_right = 0; + info.blue.msb_right = 0; + info.transp.msb_right = 0; + info.grayscale = 0; + info.nonstd = 0; + info.activate = FB_ACTIVATE_NOW; + info.vsync_len = 10; + info.upper_margin = 32; + info.lower_margin = 16; + info.right_margin = 0; + info.sync = 0; + info.accel_flags = 0; + info.vmode = FB_VMODE_NONINTERLACED; + + return copy_to_user(argp, &info, sizeof(info)); +} + +unsigned int fbdev_ioctl(int request, void *argp, struct file *file) +{ + switch (request) + { + case FBIOGET_FSCREENINFO: + return fbdev_do_fscreeninfo(argp); + case FBIOGET_VSCREENINFO: + return fbdev_do_vscreeninfo(argp); + case FBIOPUT_VSCREENINFO: + case FBIOPUTCMAP: + return 0; // noop + } + + return -ENOTTY; +} + +void *fbdev_mmap(struct vm_area_struct *area, struct file *node) +{ + area->vm_flags |= VM_PFNMAP; + area->vm_obj = vmo_create(0x1000, nullptr); + if (!area->vm_obj) + return NULL; + vmo_assign_mapping(area->vm_obj, area); + return __map_pages_to_vaddr(area->vm_mm, (void *) area->vm_start, + (void *) primary_fb->framebuffer_phys, + area->vm_end - area->vm_start, area->vm_flags); +} + +const file_ops fbdev_fops = {.read = nullptr, // TODO + .ioctl = fbdev_ioctl, + .mmap = fbdev_mmap}; + +/** + * @brief Initialize fb0 + * + */ +void fbdev_init() +{ + auto ex = dev_register_chardevs(0, 1, 0, &fbdev_fops, "fb0"); + + ex.unwrap()->show(0644); +} + +INIT_LEVEL_CORE_KERNEL_ENTRY(fbdev_init); diff --git a/kernel/kernel/fs/Makefile b/kernel/kernel/fs/Makefile index 95213c47a..627f202a5 100644 --- a/kernel/kernel/fs/Makefile +++ b/kernel/kernel/fs/Makefile @@ -1,6 +1,6 @@ fs-y:= anon_inode.o block.o dentry.o dev.o file.o null.o partition.o pipe.o poll.o pseudo.o \ superblock.o sysfs.o tmpfs.o vfs.o zero.o buffer.o inode.o namei.o filemap.o writeback.o readahead.o \ - flock.o mount.o + flock.o mount.o d_path.o include kernel/fs/ext2/Makefile include kernel/fs/block/Makefile diff --git a/kernel/kernel/fs/block.cpp b/kernel/kernel/fs/block.cpp index 6d6649b47..8c0e35bc5 100644 --- a/kernel/kernel/fs/block.cpp +++ b/kernel/kernel/fs/block.cpp @@ -40,6 +40,11 @@ struct hd_geometry static int block_reread_parts(struct blockdev *bdev); +u64 bdev_get_size(struct blockdev *bdev) +{ + return bdev->nr_sectors * bdev->sector_size; +} + unsigned int blkdev_ioctl(int request, void *argp, struct file *f) { auto d = (blockdev *) f->f_ino->i_helper; @@ -47,7 +52,7 @@ unsigned int blkdev_ioctl(int request, void *argp, struct file *f) switch (request) { case BLKGETSIZE64: { - u64 len = d->nr_sectors * d->sector_size; + u64 len = bdev_get_size(d); return copy_to_user(argp, &len, sizeof(u64)); } @@ -66,6 +71,18 @@ unsigned int blkdev_ioctl(int request, void *argp, struct file *f) return copy_to_user(argp, &d->sector_size, sizeof(unsigned int)); } + case BLKFLSBUF: { + if (!is_root_user()) + return -EACCES; + /* Synchronize the block device's page cache and truncate all the pages out! */ + if (int st = filemap_fdatasync(d->b_ino, 0, -1UL); st < 0) + return st; + + if (int st = vmo_punch_range(d->b_ino->i_pages, 0, -1UL); st < 0) + return st; + return 0; + } + default: return -EINVAL; } @@ -91,8 +108,207 @@ struct block_inode static unique_ptr create(const struct blockdev *dev, flush::writeback_dev *wbdev); }; +static void buffer_write_readpages_endio(struct bio_req *bio) NO_THREAD_SAFETY_ANALYSIS +{ + for (size_t i = 0; i < bio->nr_vecs; i++) + { + struct page_iov *iov = &bio->vec[i]; + DCHECK(page_locked(iov->page)); + struct block_buf *head = (struct block_buf *) iov->page->priv; + + spin_lock(&head->pagestate_lock); + bool uptodate = true; + + for (struct block_buf *b = head; b != nullptr; b = b->next) + { + if (b->page_off == iov->page_off) + { + bb_clear_flag(b, BLOCKBUF_FLAG_AREAD); + CHECK(bb_test_and_set(b, BLOCKBUF_FLAG_UPTODATE)); + wake_address(b); + continue; + } + + if (!bb_test_flag(b, BLOCKBUF_FLAG_UPTODATE)) + uptodate = false; + } + + spin_unlock(&head->pagestate_lock); + + if (uptodate) + { + if ((bio->flags & BIO_STATUS_MASK) == BIO_REQ_DONE) + page_set_uptodate(iov->page); + } + } +} + +static int block_readpage_write(struct vm_object *vm_obj, off_t offset, size_t len, + struct page *page) +{ + struct blockdev *bdev = (struct blockdev *) vm_obj->ino->i_helper; + unsigned int page_off = offset - (page->pageoff << PAGE_SHIFT); + unsigned int page_len = min(len, PAGE_SIZE - page_off); + int st; + + auto nr_blocks = PAGE_SIZE / bdev->block_size; + size_t starting_block_nr = (page->pageoff << PAGE_SHIFT) / bdev->block_size; + size_t curr_off = 0; + auto block_size = bdev->block_size; + u64 nblocks = bdev->nr_sectors / (block_size / bdev->sector_size); + + if (!page_test_set_flag(page, PAGE_FLAG_BUFFER)) + goto skip_setup; + + for (size_t i = 0; i < nr_blocks; i++) + { + struct block_buf *b; + if (!(b = page_add_blockbuf(page, curr_off))) + { + page_destroy_block_bufs(page); + st = -ENOMEM; + goto out_err; + } + + b->block_nr = starting_block_nr + i; + if (b->block_nr >= nblocks) + { + bb_test_and_set(b, BLOCKBUF_FLAG_HOLE); + bb_test_and_set(b, BLOCKBUF_FLAG_UPTODATE); + } + + b->block_size = bdev->block_size; + b->dev = bdev; + curr_off += bdev->block_size; + } +skip_setup: + + for (struct block_buf *buf = (struct block_buf *) page->priv; buf; buf = buf->next) + { + /* Go through the page, read-in block buffers. If we _fully_ overwrite a block, don't bring + * that one in. */ + if (buf->page_off >= page_off && buf->page_off + buf->block_size <= page_len) + continue; + sector_t block = buf->block_nr; + if (bb_test_flag(buf, BLOCKBUF_FLAG_UPTODATE)) + continue; + if (bb_test_flag(buf, BLOCKBUF_FLAG_HOLE)) + continue; + if (!bb_test_and_set(buf, BLOCKBUF_FLAG_AREAD)) + continue; + CHECK(!bb_test_flag(buf, BLOCKBUF_FLAG_UPTODATE)); + + struct bio_req *bio = bio_alloc(GFP_NOFS, 1); + if (!bio) + { + bb_clear_flag(buf, BLOCKBUF_FLAG_AREAD); + st = -ENOMEM; + goto out_err; + } + + /* Note: We do not need to ref, we hold the lock, no one can throw this page away + * while locked (almost like an implicit reference). */ + bio->sector_number = block * (block_size / bdev->sector_size); + bio->flags = BIO_REQ_READ_OP; + bio->b_end_io = buffer_write_readpages_endio; + bio_push_pages(bio, page, buf->page_off, buf->block_size); + st = bio_submit_request(bdev, bio); + bio_put(bio); + + if (st < 0) + { + bb_clear_flag(buf, BLOCKBUF_FLAG_AREAD); + goto out_err; + } + } + + for (struct block_buf *buf = (struct block_buf *) page->priv; buf; buf = buf->next) + { + if (bb_test_flag(buf, BLOCKBUF_FLAG_AREAD)) + wait_for( + buf, + [](void *ptr) -> bool { + struct block_buf *b = (struct block_buf *) ptr; + return !bb_test_flag(b, BLOCKBUF_FLAG_AREAD); + }, + 0, 0); + } + + return 0; + +out_err: + return st; +} + +static int block_write_begin(struct file *filp, struct vm_object *vm_obj, off_t offset, size_t len, + struct page **ppage) NO_THREAD_SAFETY_ANALYSIS +{ + struct page *page; + int st = filemap_find_page(filp->f_ino, offset >> PAGE_SHIFT, + FIND_PAGE_ACTIVATE | FIND_PAGE_NO_READPAGE | FIND_PAGE_LOCK, &page, + &filp->f_ra_state); + if (st < 0) + return st; + + if (!page_test_uptodate(page)) + { + st = block_readpage_write(vm_obj, offset, len, page); + if (st < 0) + goto err_read_page; + } + + *ppage = page; + return 0; + +err_read_page: + unlock_page(page); + page_unref(page); + return st; +} + +static int __block_write_end(struct file *file, struct vm_object *vm_obj, off_t offset, + unsigned int written, unsigned int to_write, struct page *page) +{ + unsigned int page_off = offset - (page->pageoff << PAGE_SHIFT); + bool uptodate = true; + CHECK(page_test_buffer(page)); + struct block_buf *head = (struct block_buf *) page->priv; + spin_lock(&head->pagestate_lock); + + for (struct block_buf *buf = (struct block_buf *) page->priv; buf; buf = buf->next) + { + if (buf->page_off <= page_off && buf->page_off + buf->block_size >= offset + written) + { + /* Fully contained in the write, mark uptodate */ + bb_test_and_set(buf, BLOCKBUF_FLAG_UPTODATE); + wake_address(buf); + } + + if (!bb_test_flag(buf, BLOCKBUF_FLAG_UPTODATE)) + uptodate = false; + } + + if (uptodate && !page_test_uptodate(page)) + page_set_uptodate(page); + spin_unlock(&head->pagestate_lock); + return 0; +} + +static int block_write_end(struct file *file, struct vm_object *vm_obj, off_t offset, + unsigned int written, unsigned int to_write, + struct page *page) NO_THREAD_SAFETY_ANALYSIS +{ + __block_write_end(file, vm_obj, offset, written, to_write, page); + unlock_page(page); + page_unref(page); + /* Block devices don't need to set i_size */ + return 0; +} + static const struct vm_object_ops block_vm_obj_ops = { .free_page = buffer_free_page, + .write_begin = block_write_begin, + .write_end = block_write_end, }; /** diff --git a/kernel/kernel/fs/block/io-queue.cpp b/kernel/kernel/fs/block/io-queue.cpp index 71d7db2aa..d744b0016 100644 --- a/kernel/kernel/fs/block/io-queue.cpp +++ b/kernel/kernel/fs/block/io-queue.cpp @@ -115,7 +115,7 @@ void io_queue::__restart_queue() void io_queue::submit_batch(struct list_head *req_list, u32 nr_reqs) { scoped_lock g{lock_}; - list_splice_tail(req_list, &req_list_); + list_splice_tail_init(req_list, &req_list_); if (nr_entries_ - used_entries_ > 0) __restart_queue(); } diff --git a/kernel/kernel/fs/block/request.cpp b/kernel/kernel/fs/block/request.cpp index 0f2d7ee2f..4b3dee905 100644 --- a/kernel/kernel/fs/block/request.cpp +++ b/kernel/kernel/fs/block/request.cpp @@ -280,6 +280,24 @@ bool blk_merge_plug(struct blk_plug *plug, struct bio_req *bio) return false; } +void blk_request_dump(struct request *req, const char *log_lvl) +{ + printk("%srequest %p: flags %x sectors %llu nsectors %llu nr sgls %zu\n", log_lvl, req, + req->r_flags, (unsigned long long) req->r_sector, (unsigned long long) req->r_nsectors, + req->r_nr_sgls); + printk("%s bdev %p queue %p\n", log_lvl, req->r_bdev, req->r_queue); + + for_every_bio(req, [log_lvl](struct bio_req *bio) -> bool { + printk("bio flags %x sector %llu b_end_io %ps\n", bio->flags, + (unsigned long long) bio->sector_number, bio->b_end_io); + for (size_t i = 0; i < bio->nr_vecs; i++) + printk("%spage_iov %zu: page %p (pfn %lu), length %x, page_off %u\n", log_lvl, i, + bio->vec[i].page, page_to_pfn(bio->vec[i].page), bio->vec[i].length, + bio->vec[i].page_off); + return true; + }); +} + #ifdef CONFIG_KUNIT static blockdev test_bdev; @@ -421,6 +439,7 @@ static struct queue_properties nvme_queue_properties() qp.inter_sgl_boundary_mask = PAGE_SIZE - 1; qp.max_sectors_per_request = 0xffff; qp.dma_address_mask = 3; + qp.max_sgl_desc_length = PAGE_SIZE; return qp; } diff --git a/kernel/kernel/fs/buffer.cpp b/kernel/kernel/fs/buffer.cpp index 31026126d..c5dbb0505 100644 --- a/kernel/kernel/fs/buffer.cpp +++ b/kernel/kernel/fs/buffer.cpp @@ -151,7 +151,7 @@ bool page_has_writeback_bufs(struct page *p) struct block_buf *page_add_blockbuf(struct page *page, unsigned int page_off) { assert(page->flags & PAGE_FLAG_BUFFER); - DCHECK(page_locked(page)); + CHECK_PAGE(page_locked(page), page); auto buf = (struct block_buf *) kmem_cache_alloc(buffer_cache, GFP_KERNEL); if (!buf) @@ -255,9 +255,6 @@ void page_destroy_block_bufs(struct page *page) ssize_t bbuffer_readpage(struct page *p, size_t off, struct inode *ino) { - p->flags |= PAGE_FLAG_BUFFER; - p->priv = 0; - auto blkdev = reinterpret_cast(ino->i_helper); DCHECK(blkdev != nullptr); @@ -291,6 +288,9 @@ ssize_t bbuffer_readpage(struct page *p, size_t off, struct inode *ino) if (iost < 0) return iost; + if (!page_test_set_buffer(p)) + goto skip_setup; + for (size_t i = 0; i < nr_blocks; i++) { struct block_buf *b; @@ -307,7 +307,10 @@ ssize_t bbuffer_readpage(struct page *p, size_t off, struct inode *ino) curr_off += block_size; } - p->flags |= PAGE_FLAG_UPTODATE; +skip_setup: + page_set_uptodate(p); + for (struct block_buf *b = (struct block_buf *) p->priv; b; b = b->next) + bb_test_and_set(b, BLOCKBUF_FLAG_UPTODATE); return PAGE_SIZE; } @@ -411,6 +414,168 @@ static ssize_t buffer_directio(struct file *filp, size_t off, iovec_iter *iter, return to_read; } +static void buffer_readpages_endio(struct bio_req *bio) NO_THREAD_SAFETY_ANALYSIS +{ + for (size_t i = 0; i < bio->nr_vecs; i++) + { + struct page_iov *iov = &bio->vec[i]; + DCHECK(page_locked(iov->page)); + struct block_buf *head = (struct block_buf *) iov->page->priv; + + spin_lock(&head->pagestate_lock); + bool uptodate = true; + + for (struct block_buf *b = head; b != nullptr; b = b->next) + { + if (b->page_off >= iov->page_off && + b->page_off + b->block_size <= iov->page_off + iov->length) + { + bb_clear_flag(b, BLOCKBUF_FLAG_AREAD); + CHECK(bb_test_and_set(b, BLOCKBUF_FLAG_UPTODATE)); + continue; + } + + if (!bb_test_flag(b, BLOCKBUF_FLAG_UPTODATE)) + uptodate = false; + } + + spin_unlock(&head->pagestate_lock); + + if (uptodate) + { + if ((bio->flags & BIO_STATUS_MASK) == BIO_REQ_DONE) + page_test_set_flag(iov->page, PAGE_FLAG_UPTODATE); + unlock_page(iov->page); + } + } +} + +static int buffer_readpages(struct readpages_state *state, + struct inode *ino) NO_THREAD_SAFETY_ANALYSIS +{ + blockdev *blkdev = reinterpret_cast(ino->i_helper); + int st; + struct page *page; + unsigned int nr_ios = 0; + auto block_size = blkdev->block_size; + u64 nblocks = blkdev->nr_sectors / (block_size / blkdev->sector_size); + + while ((page = readpages_next_page(state))) + { + const unsigned long pgoff = page->pageoff; + nr_ios = 0; + auto nr_blocks = PAGE_SIZE / block_size; + size_t starting_block_nr = (pgoff << PAGE_SHIFT) / block_size; + size_t curr_off = 0; + + if (!page_test_set_flag(page, PAGE_FLAG_BUFFER)) + goto skip_setup; + + for (size_t i = 0; i < nr_blocks; i++) + { + struct block_buf *b; + if (!(b = page_add_blockbuf(page, curr_off))) + { + page_destroy_block_bufs(page); + st = -ENOMEM; + goto out_err; + } + + b->block_nr = starting_block_nr + i; + if (b->block_nr >= nblocks) + bb_test_and_set(b, BLOCKBUF_FLAG_HOLE | BLOCKBUF_FLAG_UPTODATE); + b->block_size = block_size; + b->dev = blkdev; + curr_off += block_size; + } + + if (starting_block_nr + nr_blocks <= nblocks) + { + /* Fast, simple case. Fire off a single BIO for this whole contiguous page. This makes + * it so we can fire off larger BIOs for, e.g, NVMe, which then increases the chance of + * it getting merged with other bios, etc. + */ + struct block_buf *b = (struct block_buf *) page->priv; + struct bio_req *bio = bio_alloc(GFP_NOFS, 1); + if (!bio) + { + st = -ENOMEM; + goto out_err; + } + + bb_test_and_set(b, BLOCKBUF_FLAG_AREAD); + + bio->sector_number = b->block_nr * (block_size / blkdev->sector_size); + bio->flags = BIO_REQ_READ_OP; + bio->b_end_io = buffer_readpages_endio; + bio_push_pages(bio, page, 0, PAGE_SIZE); + st = bio_submit_request(blkdev, bio); + bio_put(bio); + + if (st < 0) + { + bb_clear_flag(b, BLOCKBUF_FLAG_AREAD); + goto out_err; + } + + nr_ios++; + goto end_read; + } + + skip_setup: + for (struct block_buf *b = (struct block_buf *) page->priv; b != nullptr; b = b->next) + { + sector_t block = b->block_nr; + if (bb_test_flag(b, BLOCKBUF_FLAG_UPTODATE)) + continue; + if (bb_test_flag(b, BLOCKBUF_FLAG_HOLE)) + continue; + if (!bb_test_and_set(b, BLOCKBUF_FLAG_AREAD)) + continue; + CHECK(!bb_test_flag(b, BLOCKBUF_FLAG_UPTODATE)); + + struct bio_req *bio = bio_alloc(GFP_NOFS, 1); + if (!bio) + { + bb_clear_flag(b, BLOCKBUF_FLAG_AREAD); + st = -ENOMEM; + goto out_err; + } + + /* Note: We do not need to ref, we hold the lock, no one can throw this page away + * while locked (almost like an implicit reference). */ + bio->sector_number = block * (block_size / blkdev->sector_size); + bio->flags = BIO_REQ_READ_OP; + bio->b_end_io = buffer_readpages_endio; + bio_push_pages(bio, page, b->page_off, b->block_size); + st = bio_submit_request(blkdev, bio); + bio_put(bio); + + if (st < 0) + { + bb_clear_flag(b, BLOCKBUF_FLAG_AREAD); + goto out_err; + } + + nr_ios++; + } + + end_read: + if (nr_ios == 0) + unlock_page(page); + page_unref(page); + } + + return 0; +out_err: + /* On error, release the page we're holding. We do not unlock it if we submitted any IOs for the + * page, the endio page will do it for us. */ + if (nr_ios == 0) + unlock_page(page); + page_unref(page); + return st; +} + static int block_prepare_write(struct inode *ino, struct page *page, size_t page_off, size_t offset, size_t len) { @@ -437,6 +602,7 @@ struct file_ops buffer_ops = { .writepages = filemap_writepages, .fsyncdata = filemap_writepages, .directio = buffer_directio, + .readpages = buffer_readpages, }; struct block_buf *sb_read_block(const struct superblock *sb, unsigned long block) @@ -634,6 +800,25 @@ void block_buf_tear_down_assoc(struct vm_object *object) } } +static void bforget(struct block_buf *buf) +{ + /* De-dirty the buffer (and page) if possible */ + struct page *page = buf->this_page; + spin_lock(&buf->pagestate_lock); + bb_clear_flag(buf, BLOCKBUF_FLAG_DIRTY); + bool isdirty = false; + for (struct block_buf *b = (struct block_buf *) page->priv; b; b = b->next) + { + if (bb_test_flag(b, BLOCKBUF_FLAG_DIRTY)) + isdirty = true; + } + + if (!isdirty) + page_clear_dirty(page); + spin_unlock(&buf->pagestate_lock); + block_buf_put(buf); +} + /** * @brief Forget a block_buf's inode * This will remove it from the assoc list @@ -657,6 +842,8 @@ void block_buf_forget_inode(struct block_buf *buf) buf->assoc_buffers_obj = nullptr; break; } + + bforget(buf); } void buffer_free_page(struct vm_object *vmo, struct page *page) diff --git a/kernel/kernel/fs/d_path.c b/kernel/kernel/fs/d_path.c new file mode 100644 index 000000000..069a02d6b --- /dev/null +++ b/kernel/kernel/fs/d_path.c @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2024 Pedro Falcato + * This file is part of Onyx, and is released under the terms of the GPLv2 License + * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct rbuf +{ + char *buf; + int len; +}; + +static char *rbuf_path(struct rbuf *rbuf) +{ + if (rbuf->len < 0) + return ERR_PTR(-ENAMETOOLONG); + return rbuf->buf; +} + +static void prepend_char(struct rbuf *rbuf, char c) +{ + rbuf->len--; + if (rbuf->len >= 0) + { + rbuf->buf--; + *rbuf->buf = c; + } +} + +static void prepend_str(struct rbuf *rbuf, const char *str, size_t len) +{ + rbuf->len -= len; + if (rbuf->len >= 0) + { + rbuf->buf -= len; + memcpy(rbuf->buf, str, len); + } +} + +static void prepend_dentry(struct rbuf *rbuf, struct dentry *dentry) +{ + /* TODO: we should RCU-protect d_name */ + spin_lock(&dentry->d_lock); + prepend_str(rbuf, dentry->d_name, dentry->d_name_length); + spin_unlock(&dentry->d_lock); +} + +static bool follow_mount_up(struct mount *mnt, struct path *out) +{ + struct dentry *dentry = mnt->mnt_root, *mountpoint; + + while (mnt->mnt_parent) + { + mountpoint = mnt->mnt_point; + mnt = mnt->mnt_parent; + if (mnt->mnt_root != dentry) + { + out->mount = mnt; + out->dentry = mountpoint; + return true; + } + } + + return false; +} + +static void walk_path(const struct path *path, const struct path *root, struct rbuf *rbuf) +{ + /* TODO: While d_parent on .. Just Works, we don't need to keep track of the struct mnt. This + * will need to be changed once that changes (mnt should keep track of mnt_parent). + **/ + struct dentry *dentry = path->dentry; + struct mount *mnt = path->mount; + + prepend_char(rbuf, '\0'); + while (dentry != NULL && dentry != root->dentry) + { + if (dentry == mnt->mnt_root) + { + struct path p; + if (!follow_mount_up(mnt, &p)) + break; + dentry = p.dentry; + mnt = p.mount; + } + + prepend_dentry(rbuf, dentry); + prepend_char(rbuf, '/'); + dentry = dentry->d_parent; + } + + if (*rbuf->buf != '/') + prepend_char(rbuf, '/'); +} + +static char *__d_path(const struct path *path, const struct path *root, char *buf, + unsigned int buflen) +{ + struct rbuf rbuf0 = {buf + buflen, buflen}, rbuf1; + unsigned int seq = 0, m_seq = 0; + rcu_read_lock(); + +retry_mnt: + read_seqbegin_or_lock(&mount_lock, &m_seq); + +retry: + read_seqbegin_or_lock(&rename_lock, &seq); + rbuf1 = rbuf0; + walk_path(path, root, &rbuf1); + + if (read_seqretry(&rename_lock, seq)) + { + seq = 1; + goto retry; + } + + done_seqretry(&rename_lock, seq); + + if (read_seqretry(&mount_lock, m_seq)) + { + m_seq = 1; + goto retry_mnt; + } + + done_seqretry(&mount_lock, m_seq); + rcu_read_unlock(); + return rbuf_path(&rbuf1); +} + +char *d_path(const struct path *path, char *buf, unsigned int buflen) +{ + struct path root = get_filesystem_root(); + char *ret = __d_path(path, &root, buf, buflen); + path_put(&root); + return ret; +} diff --git a/kernel/kernel/fs/dentry.cpp b/kernel/kernel/fs/dentry.cpp index 9d4f42739..5166f56e0 100644 --- a/kernel/kernel/fs/dentry.cpp +++ b/kernel/kernel/fs/dentry.cpp @@ -36,7 +36,7 @@ static struct slab_cache *dentry_cache; /* rename_lock is held (write!) throughout a *dcache-level* rename. This protects against hashtable * entries going bad, and against ->d_parent being changed. It's held in read-mode when traversing * the dcache hashtable. */ -static seqlock_t rename_lock; +seqlock_t rename_lock; fnv_hash_t hash_dentry(dentry *&d) { @@ -101,7 +101,7 @@ static dentry *d_lookup_internal(dentry *dent, std::string_view name) return nullptr; } -static dentry *dentry_open_from_cache(dentry *dent, std::string_view name) +dentry *dentry_open_from_cache(dentry *dent, std::string_view name) { unsigned int old; struct dentry *found; @@ -561,7 +561,7 @@ static expected dentry_create_pending_lookup(const char *name, in return dentry_add_to_cache_careful(dent, parent); } -static dentry *__dentry_try_to_open(std::string_view name, dentry *dir, bool lock_ino) +dentry *__dentry_try_to_open(std::string_view name, dentry *dir, bool lock_ino) { DCHECK(dentry_is_dir(dir)); if (auto d = dentry_open_from_cache(dir, name); d) @@ -692,89 +692,6 @@ struct path_element struct list_head node; }; -char *dentry_to_file_name(struct dentry *dentry) -{ - /* Calculate the initial length as / + the null terminator */ - size_t buf_len = 2; - char *buf = nullptr; - char *s = nullptr; - struct path p = get_filesystem_root(); - auto fs_root = p.dentry; - - if (fs_root == dentry) - { - path_put(&p); - return strdup("/"); - } - - dget(fs_root); - path_put(&p); - - auto d = dentry; - struct list_head element_list; - INIT_LIST_HEAD(&element_list); - - /* Get another ref here to have prettier code */ - dget(d); - - /* TODO: Is this logic safe from race conditions? */ - while (d != fs_root && d != nullptr) - { - path_element *p = new path_element; - if (!p) - goto error; - p->d = d; - /* Add 1 to the len because of the separator */ - buf_len += d->d_name_length + 1; - list_add(&p->node, &element_list); - - if (d->d_flags & DENTRY_FLAG_MOUNT_ROOT) - { - // HACK! - d = dentry_parent(d); - while (d && d->d_flags & DENTRY_FLAG_MOUNTPOINT) - d = dentry_parent(d); - } - else - d = dentry_parent(d); - } - - /* Remove one from the end to avoid trailing slashes */ - buf_len--; - - buf = (char *) malloc(buf_len); - if (!buf) - goto error; - buf[0] = '/'; - s = &buf[1]; - - list_for_every_safe (&element_list) - { - auto elem = container_of(l, struct path_element, node); - auto dent = elem->d; - memcpy(s, dent->d_name, dent->d_name_length); - s += dent->d_name_length; - *s++ = '/'; - dput(dent); - delete elem; - } - - buf[buf_len - 1] = '\0'; - dput(fs_root); - return buf; - -error: - dput(fs_root); - list_for_every_safe (&element_list) - { - auto elem = container_of(l, struct path_element, node); - dput(elem->d); - delete elem; - } - - return nullptr; -} - void dentry_shrink_subtree(struct dentry *dentry); void dentry_do_unlink(dentry *entry) diff --git a/kernel/kernel/fs/dev.cpp b/kernel/kernel/fs/dev.cpp index 1afa45845..53f1d4173 100644 --- a/kernel/kernel/fs/dev.cpp +++ b/kernel/kernel/fs/dev.cpp @@ -611,8 +611,13 @@ static off_t devfs_getdirent(struct dirent *buf, off_t off, struct file *file) { /* .. */ auto parent = dentry_parent(dent); - put_dentry_to_dirent(buf, parent, ".."); - dput(parent); + if (parent) + { + put_dentry_to_dirent(buf, parent, ".."); + dput(parent); + } + else + put_dentry_to_dirent(buf, dent, ".."); } else { diff --git a/kernel/kernel/fs/ext2/ext2.cpp b/kernel/kernel/fs/ext2/ext2.cpp index fe5b76a8c..40ecb829b 100644 --- a/kernel/kernel/fs/ext2/ext2.cpp +++ b/kernel/kernel/fs/ext2/ext2.cpp @@ -164,11 +164,10 @@ static ssize_t ext2_writepage(struct vm_object *obj, page *page, size_t off) REQ buf = buf->next; } - unlock_page(page); - /* For this to have been a valid dirty page, we must've been able to submit more than 0 ios (a * page full of zero blocks cannot be dirty, as prepare_write must be called). */ - CHECK(nr_ios > 0); + CHECK_PAGE(nr_ios > 0, page); + unlock_page(page); return PAGE_SIZE; } @@ -207,7 +206,7 @@ int ext2_map_page(struct page *page, size_t off, struct inode *ino) if (block == EXT2_ERR_INV_BLOCK) { // Zero the block, since it's a hole - memset((char *) PAGE_TO_VIRT(page) + curr_off, 0, sb->block_size); + page_zero_range(page, b->page_off, sb->block_size); bb_test_and_set(b, BLOCKBUF_FLAG_UPTODATE); } else @@ -237,6 +236,9 @@ ssize_t ext2_readpage(struct page *page, size_t off, struct inode *ino) for (struct block_buf *b = (struct block_buf *) page->priv; b != nullptr; b = b->next) { sector_t block = b->block_nr; + if (bb_test_flag(b, BLOCKBUF_FLAG_UPTODATE)) + continue; + if (block != EXT2_ERR_INV_BLOCK) { /* TODO: Coalesce reads */ @@ -580,8 +582,7 @@ struct inode *ext2_create_file(const char *name, mode_t mode, dev_t dev, struct if (int st = ext2_add_direntry(name, inumber, inode, vfs_ino, fs); st < 0) { thread_change_addr_limit(old); - printk("ext2 error %d\n", st); - errno = EINVAL; + errno = -st; goto free_ino_error; } @@ -904,7 +905,7 @@ struct inode *ext2_mkdir(struct dentry *dentry, mode_t mode, struct dentry *dir) */ void ext2_superblock::error(const char *str) const { - printk("ext2_error: %s\n", str); + pr_err("ext2_error: %s\n", str); sb->s_state = EXT2_ERROR_FS; block_buf_dirty(sb_bb); diff --git a/kernel/kernel/fs/ext2/ext2.h b/kernel/kernel/fs/ext2/ext2.h index 9f513cd5c..badd22a2c 100644 --- a/kernel/kernel/fs/ext2/ext2.h +++ b/kernel/kernel/fs/ext2/ext2.h @@ -662,13 +662,4 @@ inode *ext2_get_inode(ext2_superblock *sb, uint32_t inode_num); inode *ext2_create_file(const char *name, mode_t mode, dev_t dev, dentry *dir); int ext2_unlink(const char *name, int flags, dentry *dir); -/** - * @brief Detects if a symlink is a fast symlink - * - * @param inode Pointer to ext2_inode struct - * @param fs Pointer to ext2_superblock struct - * @return True if a fast symlink, else false. - */ -bool ext2_is_fast_symlink(struct ext2_inode *inode, struct ext2_superblock *fs); - #endif diff --git a/kernel/kernel/fs/ext2/ext2_ll.cpp b/kernel/kernel/fs/ext2/ext2_ll.cpp index 8a3d71f69..0c33ef7a6 100644 --- a/kernel/kernel/fs/ext2/ext2_ll.cpp +++ b/kernel/kernel/fs/ext2/ext2_ll.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -85,14 +86,11 @@ ext2_inode *ext2_superblock::get_inode(ext2_inode_no inode) const } ext2_inode *ino = (ext2_inode *) malloc(inode_size); - if (!ino) return nullptr; ext2_inode *on_disk = (ext2_inode *) ((char *) block_buf_data(buf) + off); - - memcpy(ino, on_disk, inode_size); - + memcpy(ino, on_disk, min(inode_size, (u16) sizeof(struct ext2_inode))); return ino; } @@ -126,8 +124,7 @@ void ext2_superblock::update_inode(const ext2_inode *ino, ext2_inode_no inode_no } ext2_inode *on_disk = (ext2_inode *) ((char *) block_buf_data(buf) + off); - - memcpy(on_disk, ino, inode_size); + memcpy(on_disk, ino, min(inode_size, (u16) sizeof(struct ext2_inode))); block_buf_dirty(buf); @@ -208,9 +205,9 @@ int ext2_add_direntry(const char *name, uint32_t inum, struct ext2_inode *ino, i ext2_superblock *fs) { uint8_t *buffer; - uint8_t *buf = buffer = (uint8_t *) zalloc(fs->block_size); + uint8_t *buf = buffer = (uint8_t *) kcalloc(fs->block_size, 1, GFP_NOFS); if (!buf) - return errno = ENOMEM, -1; + return -ENOMEM; if (inum == 0) panic("Bad inode number passed to ext2_add_direntry"); @@ -347,7 +344,7 @@ int ext2_remove_direntry(uint32_t inum, struct inode *dir, struct ext2_superbloc { int st = -ENOENT; uint8_t *buf_start; - uint8_t *buf = buf_start = (uint8_t *) zalloc(fs->block_size); + uint8_t *buf = buf_start = (uint8_t *) kcalloc(fs->block_size, 1, GFP_NOFS); if (!buf) return errno = ENOMEM, -1; @@ -415,7 +412,7 @@ int ext2_retrieve_dirent(inode *inode, const char *name, ext2_superblock *fs, ext2_dirent_result *res) { int st = -ENOENT; - char *buf = static_cast(zalloc(fs->block_size)); + char *buf = static_cast(kcalloc(fs->block_size, 1, GFP_NOFS)); if (!buf) return -ENOMEM; @@ -553,7 +550,7 @@ int ext2_dir_empty(struct inode *ino) struct ext2_superblock *fs = ext2_superblock_from_inode(ino); int st = 1; - char *buf = (char *) zalloc(fs->block_size); + char *buf = (char *) kcalloc(fs->block_size, 1, GFP_NOFS); if (!buf) return -ENOMEM; diff --git a/kernel/kernel/fs/ext2/inode.cpp b/kernel/kernel/fs/ext2/inode.cpp index ff613a42a..1a9176349 100644 --- a/kernel/kernel/fs/ext2/inode.cpp +++ b/kernel/kernel/fs/ext2/inode.cpp @@ -257,7 +257,7 @@ int ext2_free_space(size_t new_len, inode *ino); void ext2_free_inode_space(inode *inode_, ext2_superblock *fs) { ext2_free_space(0, inode_); - assert(inode_->i_blocks == 0); + WARN_ON(inode_->i_blocks != 0); } struct ext2_block_coords @@ -351,7 +351,7 @@ int ext2_truncate_branch(ext2_block_no block, ext2_block_coords &curr_coords, st return EXT2_TRUNCATED_PARTIALLY; /* Truncated fully, we can free this block */ /* Note: we must "forget" the inode block buf */ - block_buf_forget_inode(buf); + block_buf_forget_inode(buf.release()); sb->free_block(block); ino->i_blocks -= sb->block_size >> 9; return EXT2_TRUNCATED_FULLY; diff --git a/kernel/kernel/fs/ext2/symlink.cpp b/kernel/kernel/fs/ext2/symlink.cpp index 839432764..37174d4f5 100644 --- a/kernel/kernel/fs/ext2/symlink.cpp +++ b/kernel/kernel/fs/ext2/symlink.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 - 2021 Pedro Falcato + * Copyright (c) 2017 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -25,15 +26,16 @@ * @param fs Pointer to ext2_superblock struct * @return True if a fast symlink, else false. */ -bool ext2_is_fast_symlink(struct ext2_inode *inode, struct ext2_superblock *fs) +bool ext2_is_fast_symlink(struct inode *inode, struct ext2_inode *e2inode, + struct ext2_superblock *fs) { /* Essentially, we're comparing the extended attribute blocks * with the inode's i_blocks, and if it's zero we know the inode isn't storing * the link in filesystem blocks, so we look to the ext2_inode->i_data. */ - int ea_blocks = inode->i_file_acl ? (fs->block_size >> 9) : 0; - return (inode->i_blocks - ea_blocks == 0 && EXT2_CALCULATE_SIZE64(inode) <= 60); + int ea_blocks = e2inode->i_file_acl ? (fs->block_size >> 9) : 0; + return (inode->i_blocks - ea_blocks == 0 && inode->i_size <= 60); } #define EXT2_FAST_SYMLINK_SIZE 60 @@ -78,14 +80,10 @@ char *ext2_read_symlink(struct inode *ino, struct ext2_superblock *fs) { auto raw = ext2_get_inode_from_node(ino); - if (ext2_is_fast_symlink(raw, fs)) - { + if (ext2_is_fast_symlink(ino, raw, fs)) return ext2_do_fast_symlink(raw); - } else - { return ext2_do_slow_symlink(ino); - } } char *ext2_readlink(struct file *f) @@ -113,11 +111,11 @@ int ext2_set_symlink(inode *ino, const char *dest) unsigned long old = thread_change_addr_limit(VM_KERNEL_ADDR_LIMIT); // TODO: Kind of dumb that it's not a const void *, fix? - ssize_t read = file_write_cache((void *) dest, length, ino, 0); + ssize_t read = file_write_cache((void *) dest, length - 1, ino, 0); thread_change_addr_limit(old); - if (read != (ssize_t) length) + if (read != (ssize_t) length - 1) return -errno; } @@ -128,18 +126,75 @@ int ext2_set_symlink(inode *ino, const char *dest) inode *ext2_symlink(struct dentry *dentry, const char *dest, struct dentry *dir) { - auto inode = ext2_create_file(dentry->d_name, S_IFLNK | S_IRWXG | S_IRWXO | S_IRWXU, 0, dir); + struct inode *vfs_ino = dir->d_inode; + struct ext2_superblock *fs = ext2_superblock_from_inode(vfs_ino); + uint32_t inumber = 0; + struct inode *ino = nullptr; + unsigned long old = 0; + struct creds *c = nullptr; + + if (WARN_ON(dentry->d_name_length == 0)) + return errno = EIO, nullptr; + + auto res = fs->allocate_inode(); + if (res.has_error()) + { + errno = -res.error(); + return nullptr; + } + + auto p = res.value(); + inumber = p.first; + + struct ext2_inode *inode = p.second; + struct ext2_inode *dir_inode = ext2_get_inode_from_node(vfs_ino); + if (!inode) return nullptr; - if (auto st = ext2_set_symlink(inode, dest); st < 0) + memset(inode, 0, sizeof(struct ext2_inode)); + inode->i_ctime = inode->i_atime = inode->i_mtime = (uint32_t) clock_get_posix_time(); + + c = creds_get(); + + inode->i_uid = c->euid; + inode->i_gid = c->egid; + + creds_put(c); + inode->i_mode = EXT2_INO_TYPE_SYMLINK | (S_IRWXG | S_IRWXO | S_IRWXU); + + ino = ext2_fs_ino_to_vfs_ino(inode, inumber, fs); + if (!ino) + { + errno = ENOMEM; + goto free_ino_error; + } + + fs->update_inode(inode, inumber); + fs->update_inode(dir_inode, vfs_ino->i_inode); + + old = thread_change_addr_limit(VM_KERNEL_ADDR_LIMIT); + + if (auto st = ext2_set_symlink(ino, dest); st < 0) { - ext2_unlink(dentry->d_name, 0, dir); - inode_dec_nlink(inode); - inode_unref(inode); errno = -st; - return nullptr; + goto free_ino_error; + } + + if (int st = ext2_add_direntry(dentry->d_name, inumber, inode, vfs_ino, fs); st < 0) + { + thread_change_addr_limit(old); + errno = -st; + goto free_ino_error; } - return inode; + inode_inc_nlink(ino); + + thread_change_addr_limit(old); + superblock_add_inode(vfs_ino->i_sb, ino); + return ino; + +free_ino_error: + inode_unref(ino); + return nullptr; } diff --git a/kernel/kernel/fs/file.cpp b/kernel/kernel/fs/file.cpp index 55e1b164a..21e8cfeb7 100644 --- a/kernel/kernel/fs/file.cpp +++ b/kernel/kernel/fs/file.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 - 2023 Pedro Falcato + * Copyright (c) 2017 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -39,7 +39,7 @@ * * @return Pointer to struct fd_table, or nullptr */ -fd_table *fdtable_alloc(); +static fd_table *fdtable_alloc(); /** * @brief Free a struct fd_table @@ -364,38 +364,65 @@ struct file *get_file_description(int fd) return __get_file_description(fd, get_current_process()); } -int copy_file_descriptors(struct process *process, struct ioctx *ctx) +static int dup_fdtable(struct ioctx *ctx, struct fd_table *table) { - fd_table *table = fdtable_alloc(); - if (!table) - return -ENOMEM; + struct fd_table *oldt = ctx->table; + unsigned int nr_fds = oldt->file_desc_entries; - scoped_lock g{ctx->fdlock}; - - fd_table *oldt = ctx->table; + /* Release the ioctx lock, required for the next few allocations */ + spin_unlock(&ctx->fdlock); - table->file_desc = (file **) malloc(oldt->file_desc_entries * sizeof(void *)); - table->file_desc_entries = oldt->file_desc_entries; + table->file_desc = (file **) kmalloc(nr_fds * sizeof(void *), GFP_KERNEL); + table->file_desc_entries = nr_fds; if (!table->file_desc) - { return -ENOMEM; - } - table->cloexec_fds = (unsigned long *) malloc(table->file_desc_entries / 8); + table->cloexec_fds = (unsigned long *) kmalloc(nr_fds / 8, GFP_KERNEL); if (!table->cloexec_fds) { - free(table->file_desc); + kfree(table->file_desc); return -ENOMEM; } - table->open_fds = (unsigned long *) malloc(table->file_desc_entries / 8); + table->open_fds = (unsigned long *) kmalloc(nr_fds / 8, GFP_KERNEL); if (!table->open_fds) { - free(table->file_desc); - free(table->cloexec_fds); + kfree(table->file_desc); + kfree(table->cloexec_fds); return -ENOMEM; } + spin_lock(&ctx->fdlock); + return 0; +} + +int copy_file_descriptors(struct process *process, struct ioctx *ctx) +{ + int err; + struct fd_table *oldt; + struct fd_table *table = fdtable_alloc(); + if (!table) + return -ENOMEM; + + spin_lock(&ctx->fdlock); + + for (;;) + { + err = dup_fdtable(ctx, table); + /* dup_fdtable drops the lock and doesn't re-lock on error */ + if (err) + return err; + if (table->file_desc_entries >= ctx->table->file_desc_entries) + break; + + /* Need to re-alloc everything, so free the old data */ + kfree(table->cloexec_fds); + kfree(table->file_desc); + kfree(table->open_fds); + table->file_desc_entries = 0; + } + + oldt = ctx->table; memcpy(table->cloexec_fds, oldt->cloexec_fds, table->file_desc_entries / 8); memcpy(table->open_fds, oldt->open_fds, table->file_desc_entries / 8); @@ -406,8 +433,8 @@ int copy_file_descriptors(struct process *process, struct ioctx *ctx) fd_get(table->file_desc[i]); } + spin_unlock(&ctx->fdlock); rcu_assign_pointer(process->ctx.table, table); - return 0; } @@ -417,20 +444,20 @@ int allocate_file_descriptor_table(struct process *process) if (!table) return -ENOMEM; - table->file_desc = (file **) zalloc(FILE_DESCRIPTOR_GROW_NR * sizeof(void *)); + table->file_desc = (file **) kcalloc(FILE_DESCRIPTOR_GROW_NR, sizeof(void *), GFP_KERNEL); if (!table->file_desc) return -ENOMEM; table->file_desc_entries = FILE_DESCRIPTOR_GROW_NR; - table->cloexec_fds = (unsigned long *) zalloc(FILE_DESCRIPTOR_GROW_NR / 8); + table->cloexec_fds = (unsigned long *) kcalloc(FILE_DESCRIPTOR_GROW_NR / 8, 1, GFP_KERNEL); if (!table->cloexec_fds) { free(table->file_desc); return -ENOMEM; } - table->open_fds = (unsigned long *) zalloc(FILE_DESCRIPTOR_GROW_NR / 8); + table->open_fds = (unsigned long *) kcalloc(FILE_DESCRIPTOR_GROW_NR / 8, 1, GFP_KERNEL); if (!table->open_fds) { free(table->file_desc); @@ -457,32 +484,43 @@ static void defer_free_fd_table_rcu(fd_table *table_) #define FD_ENTRIES_TO_FDSET_SIZE(x) ((x) / 8) /* Enlarges the file descriptor table by FILE_DESCRIPTOR_GROW_NR(64) entries */ -int enlarge_file_descriptor_table(struct process *process, unsigned int new_size) +static int enlarge_fdtable(struct process *process, unsigned int new_size) { + int err = -ENOMEM; struct fd_table *oldt = process->ctx.table; - fd_table *table = fdtable_alloc(); - if (!table) - return -ENOMEM; - unsigned int old_nr_fds = oldt->file_desc_entries; + struct fd_table *table = nullptr; + struct file **ftable = nullptr; + unsigned long *cloexec_fds = nullptr; + unsigned long *open_fds = nullptr; new_size = ALIGN_TO(new_size, FILE_DESCRIPTOR_GROW_NR); - if (new_size > INT_MAX || new_size >= process->get_rlimit(RLIMIT_NOFILE).rlim_cur) - { - fdtable_free(table); return -EMFILE; - } - unsigned int new_nr_fds = new_size; + /* Can't allocate with the fdlock held... */ + spin_unlock(&process->ctx.fdlock); + + table = fdtable_alloc(); + if (!table) + goto error; - struct file **ftable = (file **) zalloc(new_nr_fds * sizeof(void *)); - unsigned long *cloexec_fds = (unsigned long *) zalloc(FD_ENTRIES_TO_FDSET_SIZE(new_nr_fds)); - /* We use zalloc here to implicitly zero free fds */ - unsigned long *open_fds = (unsigned long *) zalloc(FD_ENTRIES_TO_FDSET_SIZE(new_nr_fds)); + ftable = (struct file **) kcalloc(new_size, sizeof(void *), GFP_KERNEL); + cloexec_fds = (unsigned long *) kcalloc(FD_ENTRIES_TO_FDSET_SIZE(new_size), 1, GFP_KERNEL); + /* We use kcalloc here to implicitly zero free fds */ + open_fds = (unsigned long *) kcalloc(FD_ENTRIES_TO_FDSET_SIZE(new_size), 1, GFP_KERNEL); if (!ftable || !cloexec_fds || !open_fds) goto error; + spin_lock(&process->ctx.fdlock); + if (process->ctx.table != oldt) + { + /* Someone changed the fd table while we were gone, retry */ + err = -EAGAIN; + goto error_nolock; + } + + DCHECK(process->ctx.table->file_desc_entries == old_nr_fds); /* Note that we use old_nr_fds for these copies specifically as to not go * out of bounds. */ @@ -490,7 +528,7 @@ int enlarge_file_descriptor_table(struct process *process, unsigned int new_size memcpy(cloexec_fds, oldt->cloexec_fds, FD_ENTRIES_TO_FDSET_SIZE(old_nr_fds)); memcpy(open_fds, oldt->open_fds, FD_ENTRIES_TO_FDSET_SIZE(old_nr_fds)); - table->file_desc_entries = new_nr_fds; + table->file_desc_entries = new_size; rcu_assign_pointer(table->file_desc, ftable); rcu_assign_pointer(table->cloexec_fds, cloexec_fds); rcu_assign_pointer(table->open_fds, open_fds); @@ -501,13 +539,15 @@ int enlarge_file_descriptor_table(struct process *process, unsigned int new_size return 0; error: + spin_lock(&process->ctx.fdlock); +error_nolock: if (table) fdtable_free(table); free(ftable); free(cloexec_fds); free(open_fds); - return -ENOMEM; + return err; } void process_destroy_file_descriptors(process *process) @@ -580,8 +620,10 @@ int alloc_fd(int fdbase) /* TODO: Make it so we can enlarge it directly to the size we want */ int new_entries = table->file_desc_entries + FILE_DESCRIPTOR_GROW_NR; - if (int st = enlarge_file_descriptor_table(current, new_entries); st < 0) + if (int st = enlarge_fdtable(current, new_entries); st < 0) { + if (st == -EAGAIN) + continue; return st; } } @@ -701,11 +743,6 @@ void handle_open_flags(struct file *fd, int flags) fd->f_seek = fd->f_ino->i_size; } -static inline mode_t get_current_umask() -{ - return get_current_process()->ctx.umask; -} - bool may_noatime(file *f) { creds_guard g; @@ -870,6 +907,7 @@ int sys_dup23_internal(int oldfd, int newfd, int dupflags, unsigned int flags) // printk("pid %d oldfd %d newfd %d\n", get_current_process()->pid, oldfd, newfd); struct process *current = get_current_process(); struct ioctx *ioctx = ¤t->ctx; + struct fd_table *table; if (newfd < 0 || oldfd < 0) return -EBADF; @@ -884,19 +922,28 @@ int sys_dup23_internal(int oldfd, int newfd, int dupflags, unsigned int flags) scoped_lock g{ioctx->fdlock}; +retry: + table = rcu_dereference(ioctx->table); if ((unsigned int) newfd >= ioctx->table->file_desc_entries) { - int st = enlarge_file_descriptor_table(current, (unsigned int) newfd + 1); + int st = enlarge_fdtable(current, (unsigned int) newfd + 1); if (st < 0) { + if (st == -EAGAIN) + { + /* EAGAIN = someone touched the fd table while allocating, retry */ + goto retry; + } + // open() expects EMFILE, dup2/3 expects EBADF if (st == -EMFILE) st = -EBADF; return st; } + + table = rcu_dereference(ioctx->table); } - fd_table *table = rcu_dereference(ioctx->table); if (oldfd == newfd) return flags & DUP23_DUP3 ? -EINVAL : oldfd; @@ -1537,28 +1584,25 @@ int sys_fchdir(int fildes) int sys_getcwd(char *path, size_t size) { + char pathbuf[PATH_MAX]; + size_t pathlen = 0; if (size == 0 && path != nullptr) return -EINVAL; struct path cwd = get_current_directory(); - char *name = dentry_to_file_name(cwd.dentry); + char *name = d_path(&cwd, pathbuf, PATH_MAX); path_put(&cwd); - if (!name) - return -errno; + if (IS_ERR(name)) + return PTR_ERR(name); - if (strlen(name) + 1 > size) - { - free(name); + pathlen = pathbuf + PATH_MAX - name; + if (pathlen > size) return -ERANGE; - } - if (copy_to_user(path, name, strlen(name) + 1) < 0) - { - free(name); + if (copy_to_user(path, name, pathlen) < 0) return -errno; - } - return strlen(name); + return pathlen - 1; } int get_dirfd(int dirfd, struct path *cwd) @@ -2110,9 +2154,9 @@ void file_free(struct file *file) * * @return Pointer to struct fd_table, or nullptr */ -fd_table *fdtable_alloc() +static fd_table *fdtable_alloc() { - auto table = (fd_table *) kmem_cache_alloc(fdtable_cache, 0); + auto table = (fd_table *) kmem_cache_alloc(fdtable_cache, GFP_KERNEL); if (table) memset(table, 0, sizeof(*table)); return table; diff --git a/kernel/kernel/fs/filemap.cpp b/kernel/kernel/fs/filemap.cpp index db7cbebfe..d75fb87f4 100644 --- a/kernel/kernel/fs/filemap.cpp +++ b/kernel/kernel/fs/filemap.cpp @@ -26,7 +26,9 @@ int filemap_find_page(struct inode *ino, size_t pgoff, unsigned int flags, struc { struct page *p = nullptr; int st = 0; - vmo_status_t vst = vmo_get(ino->i_pages, pgoff << PAGE_SHIFT, 0, &p); + vmo_status_t vst = VMO_STATUS_OK; +retry: + vst = vmo_get(ino->i_pages, pgoff << PAGE_SHIFT, 0, &p); if (vst != VMO_STATUS_OK) { if (vst == VMO_STATUS_BUS_ERROR) @@ -48,7 +50,7 @@ int filemap_find_page(struct inode *ino, size_t pgoff, unsigned int flags, struc } /* Let's allocate a new page */ - p = alloc_page(GFP_KERNEL); + p = alloc_page(PAGE_ALLOC_NO_ZERO | GFP_KERNEL); if (!p) return -ENOMEM; p->owner = ino->i_pages; @@ -80,7 +82,7 @@ int filemap_find_page(struct inode *ino, size_t pgoff, unsigned int flags, struc page_promote_referenced(p); } - if (!(flags & (FIND_PAGE_NO_READPAGE | FIND_PAGE_NO_RA)) && ra_state && !S_ISBLK(ino->i_mode)) + if (!(flags & (FIND_PAGE_NO_READPAGE | FIND_PAGE_NO_RA)) && ra_state) { rw_lock_read(&ino->i_pages->truncate_lock); /* If we found PAGE_FLAG_READAHEAD, kick off more IO */ @@ -108,6 +110,14 @@ int filemap_find_page(struct inode *ino, size_t pgoff, unsigned int flags, struc rw_lock_read(&ino->i_pages->truncate_lock); lock_page(p); + if (p->owner != ino->i_pages) + { + unlock_page(p); + page_unref(p); + rw_unlock_read(&ino->i_pages->truncate_lock); + goto retry; + } + if (!page_flag_set(p, PAGE_FLAG_UPTODATE)) { ssize_t st2 = ino->i_fops->readpage(p, pgoff << PAGE_SHIFT, ino); @@ -126,7 +136,15 @@ int filemap_find_page(struct inode *ino, size_t pgoff, unsigned int flags, struc } if (flags & FIND_PAGE_LOCK) + { lock_page(p); + if (p->owner != ino->i_pages) + { + unlock_page(p); + page_unref(p); + goto retry; + } + } out: if (st == 0) @@ -376,6 +394,50 @@ ssize_t file_write_cache(void *buffer, size_t len, struct inode *ino, size_t off return file_write_cache_unlocked(buffer, len, ino, offset); } +static int default_write_begin(struct file *filp, struct vm_object *vm_obj, off_t off, size_t len, + struct page **ppage) NO_THREAD_SAFETY_ANALYSIS +{ + struct page *page = nullptr; + struct inode *ino = vm_obj->ino; + int st = filemap_find_page(filp->f_ino, off >> PAGE_SHIFT, FIND_PAGE_ACTIVATE, &page, + &filp->f_ra_state); + + if (st < 0) + return st; + + auto cache_off = off % PAGE_SIZE; + size_t aligned_off = off & -PAGE_SIZE; + auto rest = PAGE_SIZE - cache_off; + + if (rest > len) + rest = len; + + lock_page(page); + + if (st = ino->i_fops->prepare_write(ino, page, aligned_off, cache_off, rest); st < 0) + { + unlock_page(page); + page_unpin(page); + return st; + } + + *ppage = page; + return 0; +} + +static int default_write_end(struct file *file, struct vm_object *vm_obj, off_t offset, + unsigned int written, unsigned int to_write, + struct page *page) NO_THREAD_SAFETY_ANALYSIS +{ + struct inode *ino = vm_obj->ino; + unlock_page(page); + page_unref(page); + + if (written > 0 && (size_t) offset + written > ino->i_size && !S_ISBLK(ino->i_mode)) + inode_set_size(ino, offset + written); + return 0; +} + /** * @brief Write to a generic file (using the page cache) using iovec_iter * @@ -385,9 +447,11 @@ ssize_t file_write_cache(void *buffer, size_t len, struct inode *ino, size_t off * @param flags Flags * @return Written bytes, or negative error code */ -ssize_t filemap_write_iter(struct file *filp, size_t off, iovec_iter *iter, unsigned int flags) +ssize_t filemap_write_iter(struct file *filp, size_t off, iovec_iter *iter, + unsigned int flags) NO_THREAD_SAFETY_ANALYSIS { struct inode *ino = filp->f_ino; + struct vm_object *vm_obj = ino->i_pages; if (filp->f_flags & O_DIRECT) return filemap_do_direct(filp, off, iter, DIRECT_IO_WRITE); @@ -398,48 +462,33 @@ ssize_t filemap_write_iter(struct file *filp, size_t off, iovec_iter *iter, unsi while (!iter->empty()) { - struct page *page = nullptr; - int st2 = filemap_find_page(filp->f_ino, off >> PAGE_SHIFT, FIND_PAGE_ACTIVATE, &page, - &filp->f_ra_state); + int st2; + struct page *page; + st2 = (vm_obj->ops->write_begin ?: default_write_begin)(filp, vm_obj, off, iter->bytes, + &page); if (st2 < 0) return st ?: st2; - void *buffer = PAGE_TO_VIRT(page); - - auto cache_off = off % PAGE_SIZE; - size_t aligned_off = off & -PAGE_SIZE; - auto rest = PAGE_SIZE - cache_off; - - if (rest > iter->bytes) - rest = iter->bytes; - - lock_page(page); - - if (st2 = ino->i_fops->prepare_write(ino, page, aligned_off, cache_off, rest); st2 < 0) - { - unlock_page(page); - page_unpin(page); - return st ?: st2; - } + void *buffer = PAGE_TO_VIRT(page); + unsigned int page_off = off - (page->pageoff << PAGE_SHIFT); + unsigned int len = min(iter->bytes, PAGE_SIZE - page_off); /* copy_from_iter advances the iter automatically */ - ssize_t copied = copy_from_iter(iter, (u8 *) buffer + cache_off, rest); + ssize_t copied = copy_from_iter(iter, (u8 *) buffer + page_off, len); if (copied > 0) filemap_mark_dirty(page, off >> PAGE_SHIFT); - unlock_page(page); - page_unpin(page); - + st2 = (vm_obj->ops->write_end ?: default_write_end)(filp, vm_obj, off, + copied > 0 ? copied : 0, len, page); if (copied <= 0) return st ?: copied; + if (st2 < 0) + return st ?: st2; /* note: if copied < rest, we either faulted or ran out of len. in any case, it's handled */ off += copied; st += copied; - - if (off > ino->i_size && !S_ISBLK(ino->i_mode)) - inode_set_size(ino, off); } return st; @@ -498,7 +547,8 @@ void filemap_clear_dirty(struct page *page) REQUIRES(page) { /* Clear the dirty flag for IO */ struct vm_object *obj = page_vmobj(page); - __atomic_and_fetch(&page->flags, ~PAGE_FLAG_DIRTY, __ATOMIC_RELEASE); + if (!page_test_clear_dirty(page)) + return; { scoped_lock g{obj->page_lock}; @@ -649,7 +699,7 @@ static int filemap_mkwrite_private(struct vm_pf_context *ctx, static int vm_prepare_write(struct inode *inode, struct page *p) REQUIRES(p) { - DCHECK(page_locked(p)); + DCHECK_PAGE(page_locked(p), p); /* Correctness: We set the i_size before truncating pages from the page cache, so this should * not race... I think? */ @@ -675,30 +725,32 @@ static int filemap_mkwrite_shared(struct vm_pf_context *ctx, static int filemap_fault(struct vm_pf_context *ctx) NO_THREAD_SAFETY_ANALYSIS { - struct vm_area_struct *region = ctx->entry; + struct vm_area_struct *vma = ctx->entry; struct fault_info *info = ctx->info; struct page *page = nullptr; - struct inode *ino = region->vm_file->f_ino; + struct inode *ino = vma->vm_file->f_ino; int st = 0; - unsigned long pgoff = (ctx->vpage - region->vm_start) >> PAGE_SHIFT; - bool needs_invalidate = false; + unsigned long pgoff = (ctx->vpage - vma->vm_start) >> PAGE_SHIFT; + pte_t *ptep; + pte_t oldpte = ctx->oldpte; + struct spinlock *lock; /* We need to lock the page in case we're mapping it (that is, it's either a read-fault on * a private region, or any fault on a MAP_SHARED). */ - bool locked = (vma_private(region) && !ctx->info->write) || vma_shared(region); + bool locked = (vma_private(vma) && !ctx->info->write) || vma_shared(vma); /* Permission checks have already been handled before .fault() */ /* If a page was present, use that as the CoW source */ - if (vma_private(region) && ctx->mapping_info & PAGE_PRESENT) + if (vma_private(vma) && pte_present(oldpte)) { - page = phys_to_page(MAPPING_INFO_PADDR(ctx->mapping_info)); - DCHECK(info->write && !(ctx->mapping_info & PAGE_WRITABLE)); + page = phys_to_page(pte_addr(oldpte)); + DCHECK(info->write && !pte_write(oldpte)); } if (!page) { - unsigned long fileoff = (region->vm_offset >> PAGE_SHIFT) + pgoff; + unsigned long fileoff = (vma->vm_offset >> PAGE_SHIFT) + pgoff; if (ino->i_size <= (fileoff << PAGE_SHIFT)) { info->signal = VM_SIGBUS; @@ -706,24 +758,13 @@ static int filemap_fault(struct vm_pf_context *ctx) NO_THREAD_SAFETY_ANALYSIS } unsigned ffp_flags = FIND_PAGE_ACTIVATE | (locked ? FIND_PAGE_LOCK : 0); - st = filemap_find_page(region->vm_file->f_ino, fileoff, ffp_flags, &page, - ®ion->vm_file->f_ra_state); + st = filemap_find_page(vma->vm_file->f_ino, fileoff, ffp_flags, &page, + &vma->vm_file->f_ra_state); if (st < 0) goto err; } -#ifdef FILEMAP_PARANOID - if (ctx->mapping_info & PAGE_PRESENT) - { - unsigned long mapped = MAPPING_INFO_PADDR(ctx->mapping_info); - unsigned long fetched = (unsigned long) page_to_phys(page); - if (mapped != fetched) - panic("%s[%d]: filemap: Mapped page %lx != fetched %lx %s\n", - get_current_process()->name.data(), get_current_process()->pid_, mapped, fetched, - amap ? "from amap" : "from filemap"); - } -#endif if (!info->write) { /* Write-protect the page */ @@ -731,7 +772,7 @@ static int filemap_fault(struct vm_pf_context *ctx) NO_THREAD_SAFETY_ANALYSIS } else { - if (vma_private(region)) + if (vma_private(vma)) { DCHECK(!locked); st = filemap_mkwrite_private(ctx, page); @@ -743,22 +784,63 @@ static int filemap_fault(struct vm_pf_context *ctx) NO_THREAD_SAFETY_ANALYSIS /* We should invalidate the TLB if we had a mapping before. Note: I don't like that * we're mapping *over* the page, again. But it is what it is, and currently the code is * a little cleaner. */ - needs_invalidate = ctx->mapping_info & PAGE_PRESENT; page = ctx->page; DCHECK(page != nullptr); } - if (!vm_map_page(region->vm_mm, ctx->vpage, (u64) page_to_phys(page), ctx->page_rwx, - ctx->entry)) + if (pgtable_prealloc(vma->vm_mm, ctx->vpage) < 0) goto enomem; - if (needs_invalidate) - vm_invalidate_range(ctx->vpage, 1); + ptep = ptep_get_locked(vma->vm_mm, ctx->vpage, &lock); + if (ptep->pte != oldpte.pte) + { + /* Have to retry. Either this page is going away, or someone else nicely handled it for us. + */ + goto out_unlock_pte; + } + + if (ctx->page_rwx & VM_WRITE && !pte_none(oldpte) && + pte_addr(oldpte) == (unsigned long) page_to_phys(page)) + { + /* Okay, logic is simple in case we're just toggling the W bit. This can happen for various + * reasons, including mkwrite_private deciding we don't need to CoW, or a shared fault. In + * this case, we can avoid doing a TLB shootdown. Doing a local TLB invalidation is okay. It + * might result in spurious faults for other threads, but it's just way faster than + * purposefully doing IPIs. + */ + set_pte(ptep, pte_mkwrite(oldpte)); + tlbi_upgrade_pte_prots(vma->vm_mm, ctx->vpage); + } + else + { + /* New page. Just Map It. Sucks that we're copying this around... */ + struct page *oldp = NULL; + if (!pte_present(oldpte)) + increment_vm_stat(vma->vm_mm, resident_set_size, PAGE_SIZE); + + page_add_mapcount(page); + set_pte(ptep, pte_mkpte((u64) page_to_phys(page), + calc_pgprot((u64) page_to_phys(page), ctx->page_rwx))); + + if (unlikely(pte_present(oldpte) && !pte_special(oldpte))) + oldp = phys_to_page(pte_addr(oldpte)); + + /* We did our page table thing, now release the lock. We're going to need to IPI and it's + * best we do it with no spinlock held. + */ + spin_unlock(lock); + + if (pte_present(oldpte)) + vm_invalidate_range(ctx->vpage, 1); + /* After the IPI we can sub the mapcount - which may involve some freeing here... */ + if (oldp) + page_sub_mapcount(oldp); + goto out; + } - /* Only unref if this page is not new. When we allocate a new page - because of CoW, - * amap_add 'adopts' our reference. This works because amaps are inherently region-specific, - * and we have the address_space locked. - */ +out_unlock_pte: + spin_unlock(lock); +out: if (locked) unlock_page(page); page_unref(page); diff --git a/kernel/kernel/fs/mount.c b/kernel/kernel/fs/mount.c index 43462e930..173e8f418 100644 --- a/kernel/kernel/fs/mount.c +++ b/kernel/kernel/fs/mount.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -38,7 +39,7 @@ static inline struct blockdev *blkdev_get_dev(struct file *f) static struct list_head mount_hashtable[MT_HASH_SIZE]; static struct list_head mp_hashtable[MT_HASH_SIZE]; -static seqlock_t mount_lock; +seqlock_t mount_lock; static void mnt_init(struct mount *mnt, unsigned long flags) { @@ -46,6 +47,8 @@ static void mnt_init(struct mount *mnt, unsigned long flags) mnt->mnt_flags = flags; mnt->mnt_point = mnt->mnt_root = NULL; mnt->mnt_sb = NULL; + mnt->mnt_parent = NULL; + INIT_LIST_HEAD(&mnt->mnt_submounts); } static unsigned int mnt_hashbucket(struct mount *mnt) @@ -169,21 +172,16 @@ static int mnt_commit(struct mount *mnt, const char *target) * flags on a dentry, etc. */ if (strcmp(target, "/")) { - struct file *filp = open_vfs(AT_FDCWD, target); - if (!filp) - return -errno; - if (!dentry_is_dir(filp->f_dentry)) - { - fd_put(filp); - return -ENOTDIR; - } + struct path mountpoint; + int err; + + err = path_openat(AT_FDCWD, target, LOOKUP_MUST_BE_DIR, &mountpoint); + if (err < 0) + return err; - mnt->mnt_point = filp->f_dentry; - dget(mnt->mnt_point); - /* Another hack... */ - mnt->mnt_root->d_parent = mnt->mnt_point; - /* TODO: This isn't quite safe when we get proper mnt putting and umount */ - fd_put(filp); + /* Path reference gets dilluted into these two members */ + mnt->mnt_point = mountpoint.dentry; + mnt->mnt_parent = mountpoint.mount; } else { @@ -201,6 +199,9 @@ static int mnt_commit(struct mount *mnt, const char *target) list_add_tail(&mnt->mnt_mp_node, &mp_hashtable[mnt_mp_hashbucket(mnt)]); list_add_tail(&mnt->mnt_node, &mount_hashtable[mnt_hashbucket(mnt)]); + if (mnt->mnt_parent) + list_add_tail(&mnt->mnt_submount_node, &mnt->mnt_parent->mnt_submounts); + /* Ref up for the mount root */ dget(mnt->mnt_root); @@ -337,6 +338,12 @@ static bool attempt_disconnect(struct mount *mount) struct dentry *mp = mount->mnt_point; list_remove(&mount->mnt_mp_node); list_remove(&mount->mnt_node); + if (mount->mnt_parent) + { + list_remove(&mount->mnt_submount_node); + mnt_put(mount->mnt_parent); + } + ok = true; /* Check if we have nothing mounted at mp anymore. If so, unset DENTRY_FLAG_MOUNTPOINT. @@ -375,8 +382,6 @@ static int do_umount_path(struct path *path, int flags) WARN_ON(mount->mnt_root->d_ref != 1); - /* Undo our fake d_parent... */ - mount->mnt_root->d_parent = NULL; /* Finally, put our root */ dput(mount->mnt_root); @@ -391,18 +396,21 @@ static int do_umount_path(struct path *path, int flags) int sys_umount2(const char *utarget, int flags) { + int err = -EINVAL; + const char *target; + if (!is_root_user()) return -EPERM; - const char *target = strcpy_from_user(utarget); + + target = strcpy_from_user(utarget); if (!target) return -errno; if (flags & ~UMOUNT_NOFOLLOW) - return -EINVAL; + goto out; struct path path; - int err = - path_openat(AT_FDCWD, target, - LOOKUP_MUST_BE_DIR | (flags & UMOUNT_NOFOLLOW ? LOOKUP_NOFOLLOW : 0), &path); + err = path_openat(AT_FDCWD, target, + LOOKUP_MUST_BE_DIR | (flags & UMOUNT_NOFOLLOW ? LOOKUP_NOFOLLOW : 0), &path); if (err < 0) goto out; diff --git a/kernel/kernel/fs/namei.cpp b/kernel/kernel/fs/namei.cpp index a4d523268..b10f188a4 100644 --- a/kernel/kernel/fs/namei.cpp +++ b/kernel/kernel/fs/namei.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -159,9 +160,7 @@ static int dentry_follow_symlink(nameidata &data, dentry *symlink, unsigned int auto target_str = readlink_vfs(&f); if (!target_str) - { return -errno; - } /* Empty symlinks = -ENOENT. See nameitests for more info. */ if (target_str[0] == '\0') @@ -202,6 +201,122 @@ static int dentry_follow_symlink(nameidata &data, dentry *symlink, unsigned int #define NAMEI_NO_FOLLOW_SYM (1U << 1) #define NAMEI_ALLOW_NEGATIVE (1U << 2) +static void follow_mount_up(struct mount *mnt, struct path *out) +{ + struct dentry *dentry = mnt->mnt_root, *mountpoint; + + while (mnt->mnt_parent) + { + mountpoint = mnt->mnt_point; + mnt = mnt->mnt_parent; + if (mnt->mnt_root != dentry) + { + out->mount = mnt; + out->dentry = mountpoint; + return; + } + } + + /* Should not be hittable, I think... */ + CHECK(0); +} + +static bool finish_mount_up(struct path *path, unsigned int seq) +{ + mnt_get(path->mount); + smp_mb(); + + if (path->mount->mnt_flags & MNT_DOOMED) + { + mnt_put(path->mount); + goto retry; + } + + /* I don't think there's a way this can fail, if we grabbed the mount itself */ + dget(path->dentry); + + if (read_seqretry(&mount_lock, seq)) + goto retry; + + return true; +retry: + return false; +} + +static int mount_dotdot(struct mount *mnt, struct path *path) +{ + rcu_read_lock(); + + for (;;) + { + unsigned int seq = read_seqbegin(&mount_lock); + follow_mount_up(mnt, path); + + /* Commit this follow_up by grabbing a reference to mount and mountpoint */ + if (finish_mount_up(path, seq)) + break; + } + + rcu_read_unlock(); + return 0; +} + +static int do_dotdot(nameidata &data, struct path *out) +{ + struct path *curr = &data.cur; + if (curr->dentry == curr->mount->mnt_root) + { + /* We're the mount's root? Gotta take things in a different way. */ + return mount_dotdot(curr->mount, out); + } + + struct dentry *dentry = dentry_parent(curr->dentry); + if (!dentry) + { + /* /.. = right where we are */ + return 0; + } + + struct path p = {dentry, curr->mount}; + mnt_get(curr->mount); + *out = p; + return 0; +} + +static int __namei_walk_component(std::string_view v, nameidata &data, struct path *out, + unsigned int flags) +{ + if (!v.compare(".")) + { + *out = data.cur; + path_get(out); + return 0; + } + + if (!v.compare("..")) + return do_dotdot(data, out); + + struct dentry *dent = dentry_open_from_cache(data.cur.dentry, v); + + if (dent) + { + if (dent->d_flags & DENTRY_FLAG_PENDING) + dent = dentry_wait_for_pending(dent); + } + + if (!dent) + { + dent = __dentry_try_to_open(v, data.cur.dentry, !(flags & DENTRY_LOOKUP_UNLOCKED)); + if (!dent) + return -errno; + } + + struct path p = {.dentry = dent, .mount = data.cur.mount}; + mnt_get(data.cur.mount); + *out = p; + return 0; +} + static int namei_walk_component(std::string_view v, nameidata &data, unsigned int flags = 0) { const bool is_last_name = @@ -209,9 +324,7 @@ static int namei_walk_component(std::string_view v, nameidata &data, unsigned in const bool dont_follow_last = data.lookup_flags & LOOKUP_NOFOLLOW; const bool unlocked_lookup = flags & NAMEI_UNLOCKED; - auto_dentry dwrapper; - dentry *new_found; - + struct path path; file f; f.f_ino = data.cur.dentry->d_inode; @@ -226,42 +339,33 @@ static int namei_walk_component(std::string_view v, nameidata &data, unsigned in /* Stop from escaping the chroot */ return 0; } - else - { - dwrapper = dentry_lookup_internal(v, data.cur.dentry, - unlocked_lookup ? DENTRY_LOOKUP_UNLOCKED : 0); - if (!dwrapper) - { - DCHECK(errno != 0); - return -errno; - } + + path_init(&path); + int err = __namei_walk_component(v, data, &path, unlocked_lookup ? DENTRY_LOOKUP_UNLOCKED : 0); + if (err < 0) + return err; #if 0 - printk("Lookup %s found %p%s\n", v.data(), new_found, - d_is_negative(new_found) ? " (negative)" : ""); + pr_warn("Lookup %s found %p%s\n", v.data(), path.dentry, + d_is_negative(path.dentry) ? " (negative)" : ""); #endif - } - new_found = dwrapper.get_dentry(); - struct mount *mnt = data.cur.mount; + struct mount *mnt = path.mount; - if (d_is_negative(new_found)) + if (d_is_negative(path.dentry)) { /* Check if the caller tolerates negative dentries as the lookup result. This only applies * for the last name. For !last_name, negative is always ENOENT */ if (!is_last_name || !(flags & NAMEI_ALLOW_NEGATIVE)) - return -ENOENT; + { + err = -ENOENT; + goto err_out; + } } - else if (dentry_is_symlink(new_found)) + else if (dentry_is_symlink(path.dentry)) { if (flags & NAMEI_NO_FOLLOW_SYM) - { - /* Save parent and location for the caller */ - struct path p = path{dwrapper.release(), mnt}; - mnt_get(mnt); - data.setcur(p); - return 0; - } + goto out; /* POSIX states that paths that end in a trailing slash are required to be the same as * /. For example: open("/usr/bin/") == open("/usr/bin/."). Therefore, we have to * special case that. @@ -272,7 +376,10 @@ static int namei_walk_component(std::string_view v, nameidata &data, unsigned in // printk("Following symlink for path elem %s\n", v.data()); if (is_last_name && (data.lookup_flags & LOOKUP_FAIL_IF_LINK)) - return -ELOOP; + { + err = -ELOOP; + goto err_out; + } else if (is_last_name && !should_follow_symlink) { // printk("Cannot follow symlink. Trailing slash: %s\n", must_be_dir ? "yes" : @@ -280,26 +387,31 @@ static int namei_walk_component(std::string_view v, nameidata &data, unsigned in } else [[likely]] { - return dentry_follow_symlink(data, new_found); + err = dentry_follow_symlink(data, path.dentry); + path_put(&path); + return err; } } - else if (dentry_is_mountpoint(new_found)) + else if (dentry_is_mountpoint(path.dentry)) { - struct mount *new_mount = mnt_traverse(new_found); + struct mount *new_mount = mnt_traverse(path.dentry); if (new_mount) { - dwrapper = new_mount->mnt_root; - dget(dwrapper.get_dentry()); + struct dentry *d = new_mount->mnt_root; + dget(d); mnt = new_mount; - new_found = dwrapper.get_dentry(); + data.setcur((struct path){d, mnt}); + path_put(&path); + return 0; } } - if (mnt == data.cur.mount) - mnt_get(mnt); - data.setcur(path{dwrapper.release(), mnt}); - +out: + data.setcur(path); return 0; +err_out: + path_put(&path); + return err; } /** @@ -324,9 +436,9 @@ static int namei_resolve_path(nameidata &data) { #define NAMEI_DEBUG 0 #if NAMEI_DEBUG - printk("pdepth %d %s %s\n", data.pdepth, data.paths[data.pdepth].view.data(), - data.paths[data.pdepth].token_type == fs_token_type::LAST_NAME_IN_PATH ? "last" - : "regular"); + pr_info("pdepth %d %s %s\n", data.pdepth, data.paths[data.pdepth].view.data(), + data.paths[data.pdepth].token_type == fs_token_type::LAST_NAME_IN_PATH ? "last" + : "regular"); #endif auto &path = data.paths[data.pdepth]; if (path.token_type == fs_token_type::LAST_NAME_IN_PATH) @@ -487,7 +599,8 @@ static int do_creat(dentry *dir, struct inode *inode, struct dentry *dentry, mod DCHECK(d_is_negative(dentry)); - struct inode *new_inode = inode->i_fops->creat(dentry, (int) mode | S_IFREG, dir); + struct inode *new_inode = + inode->i_fops->creat(dentry, do_umask((int) (mode & ~S_IFMT) | S_IFREG), dir); if (!new_inode) return -errno; @@ -725,6 +838,13 @@ static int do_lookup_parent_last(nameidata &data) if (st == 0) { + bool finished_path = true; + for (int i = 0; i < data.pdepth; i++) + if (data.paths[i].token_type != fs_token_type::LAST_NAME_IN_PATH) + finished_path = false; + if (finished_path) + return 0; + if (data.pdepth > 0) { data.pdepth--; @@ -790,6 +910,7 @@ static int namei_lookup_parentat(int dirfd, const char *name, unsigned int flags return st; DCHECK(!path_is_null(&namedata.cur)); + DCHECK(namedata.paths[namedata.pdepth].token_type != fs_token_type::LAST_NAME_IN_PATH); *outn = namedata.paths[namedata.pdepth]; *parent = namedata.getcur(); return 0; @@ -855,6 +976,7 @@ static expected namei_create_generic(int dirfd, const char goto put_unlock_err; } + mode = do_umask(mode); switch (mode & S_IFMT) { case S_IFREG: diff --git a/kernel/kernel/fs/pipe.cpp b/kernel/kernel/fs/pipe.cpp index a032192da..5e37c21ff 100644 --- a/kernel/kernel/fs/pipe.cpp +++ b/kernel/kernel/fs/pipe.cpp @@ -926,7 +926,7 @@ int named_pipe_open(struct file *f) int pipe::open_named(struct file *filp) { scoped_mutex g{pipe_lock}; - ssize_t st; + ssize_t st = 0; // As per standard named pipe behavior, block until a peer shows up if ((filp->f_flags & O_RDWRMASK) == O_RDONLY) @@ -934,7 +934,8 @@ int pipe::open_named(struct file *filp) reader_count++; wake_all(&write_queue); COMPILER_BARRIER(); - st = wait_for_event_mutex_interruptible(&read_queue, writer_count != 0, &pipe_lock); + if (!(filp->f_flags & O_NONBLOCK)) + st = wait_for_event_mutex_interruptible(&read_queue, writer_count != 0, &pipe_lock); } else if ((filp->f_flags & O_RDWRMASK) == O_WRONLY) { @@ -942,9 +943,11 @@ int pipe::open_named(struct file *filp) wake_all(&read_queue); COMPILER_BARRIER(); // Use a lambda to go around the multiple wait_for_event problem - st = [&]() REQUIRES(pipe_lock) -> ssize_t { - return wait_for_event_mutex_interruptible(&write_queue, reader_count != 0, &pipe_lock); - }(); + if (!(filp->f_flags & O_NONBLOCK)) + st = [&]() REQUIRES(pipe_lock) -> ssize_t { + return wait_for_event_mutex_interruptible(&write_queue, reader_count != 0, + &pipe_lock); + }(); } else if ((filp->f_flags & O_RDWRMASK) == O_RDWR) { diff --git a/kernel/kernel/fs/poll.cpp b/kernel/kernel/fs/poll.cpp index f4fdc7791..f11c00f8b 100644 --- a/kernel/kernel/fs/poll.cpp +++ b/kernel/kernel/fs/poll.cpp @@ -23,6 +23,7 @@ void poll_file_entry::wait_on() wait_token.thread = get_current_thread(); wait_token.context = f; wait_token.callback = wake_callback; + wait_token.flags = WQ_TOKEN_NO_DEQUEUE; wait_queue_add(queue, &wait_token); } @@ -49,7 +50,7 @@ void poll_file::wait(wait_queue *queue) file->wait_on(); } -sleep_result poll_table::sleep_poll(hrtime_t timeout, bool timeout_valid) const +sleep_result poll_table::sleep_poll(hrtime_t timeout, bool timeout_valid) { if (timeout == 0 && timeout_valid) return sleep_result::timeout; @@ -75,7 +76,7 @@ sleep_result poll_table::sleep_poll(hrtime_t timeout, bool timeout_valid) const else sched_sleep(timeout); - if (signaled) + if (was_signaled()) return sleep_result::woken_up; else if (signal_is_pending()) return sleep_result::signal; diff --git a/kernel/kernel/fs/readahead.c b/kernel/kernel/fs/readahead.c index 2f458c463..be85aea25 100644 --- a/kernel/kernel/fs/readahead.c +++ b/kernel/kernel/fs/readahead.c @@ -68,6 +68,8 @@ static void readpages_finish(struct readpages_state *state) NO_THREAD_SAFETY_ANA } } +u64 bdev_get_size(struct blockdev *bdev); + static int filemap_do_readahead(struct inode *inode, struct readahead_state *ra_state, unsigned long pgoff) NO_THREAD_SAFETY_ANALYSIS { @@ -76,6 +78,9 @@ static int filemap_do_readahead(struct inode *inode, struct readahead_state *ra_ size_t endpg; struct blk_plug plug; + if (S_ISBLK(inode->i_mode)) + size = bdev_get_size(inode->i_helper); + /* Do basic bounds checks on our readahead window */ if (!size) return 0; diff --git a/kernel/kernel/fs/tmpfs.cpp b/kernel/kernel/fs/tmpfs.cpp index cc2486ed5..5070c9d21 100644 --- a/kernel/kernel/fs/tmpfs.cpp +++ b/kernel/kernel/fs/tmpfs.cpp @@ -112,8 +112,10 @@ int tmpfs_unlink(const char *name, int flags, struct dentry *dir) return -ENOTEMPTY; } + /* One ref for its tmpfs existence, one ref for dentry_lookup_internal */ + DCHECK(READ_ONCE(child->d_ref) >= 2); + dput(child); dput(child); - return 0; } @@ -190,10 +192,13 @@ off_t tmpfs_getdirent(struct dirent *buf, off_t off, struct file *file) { /* .. */ auto parent = dentry_parent(dent); - if (!parent) // We're root, so use ourselves - parent = dent; - put_dentry_to_dirent(buf, parent, ".."); - dput(parent); + if (parent) + { + put_dentry_to_dirent(buf, parent, ".."); + dput(parent); + } + else + put_dentry_to_dirent(buf, dent, ".."); } else { diff --git a/kernel/kernel/fs/vfs.cpp b/kernel/kernel/fs/vfs.cpp index 293d7cc61..42b295a93 100644 --- a/kernel/kernel/fs/vfs.cpp +++ b/kernel/kernel/fs/vfs.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2023 Pedro Falcato + * Copyright (c) 2016 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -32,6 +32,8 @@ #include #include +#include + struct file *fs_root = nullptr; struct file *mount_list = nullptr; @@ -357,10 +359,27 @@ ssize_t read_vfs(size_t offset, size_t len, void *buffer, struct file *file) return read_iter_vfs(file, offset, &iter, 0); } -int ioctl_vfs(int request, char *argp, struct file *this_) +int ioctl_vfs(int request, char *argp, struct file *file) { - if (this_->f_ino->i_fops->ioctl != nullptr) - return this_->f_ino->i_fops->ioctl(request, (void *) argp, this_); + switch (request) + { + case FIONBIO: { + int on; + + if (copy_from_user(&on, argp, sizeof(on)) < 0) + return -EFAULT; + + if (on) + file->f_flags |= O_NONBLOCK; + else + file->f_flags &= ~O_NONBLOCK; + + return 0; + } + } + + if (file->f_ino->i_fops->ioctl != nullptr) + return file->f_ino->i_fops->ioctl(request, (void *) argp, file); return -ENOTTY; } diff --git a/kernel/kernel/kcsan/core.c b/kernel/kernel/kcsan/core.c index f113cc09e..e6c926eda 100644 --- a/kernel/kernel/kcsan/core.c +++ b/kernel/kernel/kcsan/core.c @@ -225,24 +225,6 @@ __always_inline struct kcsan_ctx *get_ctx(void) __always_inline void check_access(const volatile void *ptr, size_t size, int type, unsigned long ip); -struct kcsan_scoped_access { - union { - struct list_head list; /* scoped_accesses list */ - /* - * Not an entry in scoped_accesses list; stack depth from where - * the access was initialized. - */ - int stack_depth; - }; - - /* Access information. */ - const volatile void *ptr; - size_t size; - int type; - /* Location where scoped access was set up. */ - unsigned long ip; -}; - /* Check scoped accesses; never inline because this is a slow-path! */ static noinline void kcsan_check_scoped_accesses(void) { @@ -417,7 +399,7 @@ void kcsan_restore_irqtrace(struct task_struct *task) __always_inline int get_kcsan_stack_depth(void) { #if CONFIG_KCSAN_WEAK_MEMORY - return current->kcsan_stack_depth; + return get_current_thread() ? get_current_thread()->kcsan_stack_depth : 0; #else BUILD_BUG(); return 0; @@ -427,7 +409,8 @@ __always_inline int get_kcsan_stack_depth(void) __always_inline void add_kcsan_stack_depth(int val) { #if CONFIG_KCSAN_WEAK_MEMORY - current->kcsan_stack_depth += val; + if (get_current_thread()) + get_current_thread()->kcsan_stack_depth += val; #else BUILD_BUG(); #endif diff --git a/kernel/kernel/kcsan/kcsan.h b/kernel/kernel/kcsan/kcsan.h index 930db8f79..3628cfd83 100644 --- a/kernel/kernel/kcsan/kcsan.h +++ b/kernel/kernel/kcsan/kcsan.h @@ -16,9 +16,9 @@ #define CONFIG_KCSAN_NUM_WATCHPOINTS 64 #define CONFIG_KCSAN_UDELAY_TASK 80 #define CONFIG_KCSAN_UDELAY_INTERRUPT 20 -#define CONFIG_KCSAN_SKIP_WATCH 4000 +#define CONFIG_KCSAN_SKIP_WATCH 800 #define CONFIG_KCSAN_IGNORE_ATOMICS 0 -#define CONFIG_KCSAN_WEAK_MEMORY 0 +#define CONFIG_KCSAN_WEAK_MEMORY 1 #define UL(a) ((a) + 0UL) #define ULL(a) ((a) + 0ULL) diff --git a/kernel/kernel/kernlog.cpp b/kernel/kernel/kernlog.cpp index a10fec6c1..47315946f 100644 --- a/kernel/kernel/kernlog.cpp +++ b/kernel/kernel/kernlog.cpp @@ -72,12 +72,16 @@ struct printk_header struct printk_buf { - char _log_buf[LOG_BUF_SIZE]; - size_t log_tail = 0; - size_t log_head = 0; - unsigned int msg_seq = 0; + char _log_buf[LOG_BUF_SIZE]{}; + size_t log_tail; + size_t log_head; + unsigned int msg_seq; static constexpr size_t logmask = LOG_BUF_SIZE - 1; + constexpr printk_buf() : log_tail{0}, log_head{0}, msg_seq{0} + { + } + /* Note: The functions below all require the printk_lock (or some other kind of mutual * exclusion) */ @@ -243,7 +247,7 @@ u32 printk_buf::find_and_print(char *buf, size_t *psize, u32 initial_seq, u32 fl return seen; } -static struct printk_buf printk_buf; +static constinit struct printk_buf printk_buf; static struct spinlock printk_lock; #define MAX_LINE 1024 static char flush_buf[MAX_LINE]; diff --git a/kernel/kernel/mm/Makefile b/kernel/kernel/mm/Makefile index 3301a421d..aeb810808 100644 --- a/kernel/kernel/mm/Makefile +++ b/kernel/kernel/mm/Makefile @@ -1,4 +1,4 @@ -mm-y:= bootmem.o page.o pagealloc.o vm_object.o vm.o vmalloc.o reclaim.o anon.o mincore.o page_lru.o swap.o rmap.o slab_cache_pool.o +mm-y:= bootmem.o page.o pagealloc.o vm_object.o vm.o vmalloc.o reclaim.o anon.o mincore.o page_lru.o swap.o rmap.o slab_cache_pool.o madvise.o mm-$(CONFIG_KUNIT)+= vm_tests.o mm-$(CONFIG_X86)+= memory.o mm-$(CONFIG_RISCV)+= memory.o diff --git a/kernel/kernel/mm/anon.cpp b/kernel/kernel/mm/anon.cpp index 874f8579d..b3d16a7db 100644 --- a/kernel/kernel/mm/anon.cpp +++ b/kernel/kernel/mm/anon.cpp @@ -20,11 +20,14 @@ const struct vm_operations anon_vmops = {.fault = vm_anon_fault}; int vm_anon_fault(struct vm_pf_context *ctx) { - struct vm_area_struct *region = ctx->entry; + struct vm_area_struct *vma = ctx->entry; struct fault_info *info = ctx->info; - struct page *page = nullptr, *oldp = nullptr; - bool needs_invd = false; + struct page *page = nullptr; + pte_t *ptep; + struct spinlock *lock; + /* pte_present is done in do_wp_page, not here. */ + CHECK(!pte_present(ctx->oldpte)); /* Permission checks have already been handled before .fault() */ if (!info->write) { @@ -32,42 +35,15 @@ int vm_anon_fault(struct vm_pf_context *ctx) page = vm_get_zero_page(); /* Write protect the page and don't bother flushing the TLB */ ctx->page_rwx &= ~VM_WRITE; - ctx->page_rwx |= VM_NOFLUSH; - goto map; } else { - bool copy_old = false; - if (pte_present(ctx->oldpte)) - { - oldp = phys_to_page(pte_addr(ctx->oldpte)); - DCHECK(info->write && !pte_write(ctx->oldpte)); - if (oldp != vm_get_zero_page()) - copy_old = true; - needs_invd = true; - - if (copy_old && 0 && page_flag_set(oldp, PAGE_FLAG_ANON) && page_mapcount(oldp) == 1) - { - /* If this is an anon page *and* mapcount = 1, avoid allocating a new page. Since - * mapcount = 1 (AND *ANON*), no one else can grab a ref. */ - /* TODO: We might be able to explore this - we may avoid the TLB shootdown and just - * change prots, but it would require significant code refactoring as-is. */ - /* TODO: checking mapcount = 1 probably isn't this easy once we get swapping, - * because refs may come and go. Will we need the page lock? */ - page = oldp; - page_ref(page); - goto map; - } - - /* oldp's mapcount will be decremented in vm_map_page */ - } - struct anon_vma *anon = anon_vma_prepare(ctx->entry); if (!anon) return -ENOMEM; - /* Allocate a brand-new (possibly zero-filled) page */ - page = alloc_page((copy_old ? PAGE_ALLOC_NO_ZERO : 0) | GFP_KERNEL); + /* Allocate a brand-new, zero-filled page */ + page = alloc_page(GFP_KERNEL); if (!page) goto enomem; page_set_anon(page); @@ -75,19 +51,22 @@ int vm_anon_fault(struct vm_pf_context *ctx) page->pageoff = ctx->vpage; page_add_lru(page); page_set_dirty(page); - - if (copy_old) - copy_page_to_page(page_to_phys(page), page_to_phys(oldp)); - goto map; } -map: - if (!vm_map_page(region->vm_mm, ctx->vpage, (u64) page_to_phys(page), ctx->page_rwx, - ctx->entry)) + if (pgtable_prealloc(vma->vm_mm, ctx->vpage) < 0) goto enomem; - if (needs_invd) - vm_invalidate_range(ctx->vpage, 1); + ptep = ptep_get_locked(vma->vm_mm, ctx->vpage, &lock); + if (ptep->pte != ctx->oldpte.pte) + goto out; + + increment_vm_stat(vma->vm_mm, resident_set_size, PAGE_SIZE); + page_add_mapcount(page); + set_pte(ptep, pte_mkpte((u64) page_to_phys(page), + calc_pgprot((u64) page_to_phys(page), ctx->page_rwx))); + +out: + spin_unlock(lock); /* The mapcount holds the only reference we need for anon pages... */ if (info->write) page_unref(page); diff --git a/kernel/kernel/mm/asan/asan.cpp b/kernel/kernel/mm/asan/asan.cpp index 4d479c5a0..5e6a5fad1 100644 --- a/kernel/kernel/mm/asan/asan.cpp +++ b/kernel/kernel/mm/asan/asan.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 - 2023 Pedro Falcato + * Copyright (c) 2019 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -705,3 +706,10 @@ size_t kasan_get_redzone_size(size_t objsize) else return 1024; } + +void init_shadow_for_phys(unsigned long addr, size_t len); + +void kasan_page_alloc_init() +{ + for_every_phys_region(init_shadow_for_phys); +} diff --git a/kernel/kernel/mm/bootmem.cpp b/kernel/kernel/mm/bootmem.cpp index 9df4eb7cc..2a6e55434 100644 --- a/kernel/kernel/mm/bootmem.cpp +++ b/kernel/kernel/mm/bootmem.cpp @@ -24,6 +24,7 @@ struct memory_range size_t size; }; +static unsigned long phys_range_seq = 0; memory_range phys_ranges[DEFAULT_NR_MEMORY_RANGES]; unsigned int nr_phys_ranges = 0; @@ -32,8 +33,14 @@ unsigned int nr_resv_ranges = 0; void for_every_phys_region(void (*callback)(unsigned long start, size_t size)) { +again: for (unsigned int i = 0; i < nr_phys_ranges; i++) + { + unsigned long seq = phys_range_seq; callback(phys_ranges[i].start, phys_ranges[i].size); + if (phys_range_seq != seq) + goto again; + } } static void __bootmem_add_range(unsigned long start, size_t size) @@ -66,6 +73,7 @@ void bootmem_add_range(unsigned long start, size_t size) // We need to run this because we might already have memory reservations registered bootmem_re_reserve_memory(); + phys_range_seq++; } static void bootmem_remove_range(unsigned int index) @@ -73,6 +81,7 @@ static void bootmem_remove_range(unsigned int index) auto tail_ranges = nr_phys_ranges - index - 1; memmove(&phys_ranges[index], &phys_ranges[index + 1], tail_ranges * sizeof(memory_range)); nr_phys_ranges--; + phys_range_seq++; } static void bootmem_add_reserve(unsigned long start, size_t size) @@ -160,6 +169,7 @@ void bootmem_reserve(unsigned long start, size_t size) bootmem_add_reserve(start, size); bootmem_reserve_memory_ranges(start, size); + phys_range_seq++; } void *alloc_boot_page(size_t nr_pages, long flags) diff --git a/kernel/kernel/mm/madvise.c b/kernel/kernel/mm/madvise.c new file mode 100644 index 000000000..b61f837d3 --- /dev/null +++ b/kernel/kernel/mm/madvise.c @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2024 Pedro Falcato + * This file is part of Onyx, and is released under the terms of the GPLv2 License + * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only + */ + +#include + +#include +#include +#include +#include + +#include + +static bool madvise_needs_write(int advice) +{ + return true; +} + +static bool madvise_valid_advice(int advice) +{ + switch (advice) + { + case MADV_DONTNEED: + return true; + } + + return false; +} + +static int do_madvise_vma(struct vm_area_struct *vma, unsigned long start, unsigned long end, + int advice) +{ + switch (advice) + { + case MADV_DONTNEED: + return zap_page_range(start, end, vma); + } + + WARN_ON(1); + return -ENOSYS; +} + +static int do_madvise_walk(struct mm_address_space *mm, unsigned long start, size_t len, int advice) +{ + unsigned long limit = start + len; + unsigned long last_vma_end = start; + int ret = -ENOMEM; + struct vm_area_struct *vma; + + MA_STATE(mas, &mm->region_tree, start, limit - 1); + + mas_for_each(&mas, vma, limit - 1) + { + /* Break if we see a gap between VMAs, or if this vma is beyond limit */ + if (vma->vm_start >= limit) + break; + + if (vma->vm_start != last_vma_end) + { + ret = -ENOMEM; + break; + } + + ret = do_madvise_vma(vma, max(vma->vm_start, start), min(limit, vma->vm_end), advice); + if (ret) + break; + last_vma_end = vma->vm_end; + } + + return ret; +} + +int sys_madvise(void *addr, size_t len, int advice) +{ + int ret; + unsigned long start = (unsigned long) addr; + /* TODO: Remove this open coding */ + struct mm_address_space *mm = get_current_thread()->aspace; + + if (!madvise_valid_advice(advice)) + return -EINVAL; + if (start & (PAGE_SIZE - 1)) + return -EINVAL; + + len = ALIGN_TO(len, PAGE_SIZE); + + if (start + len <= start) + return -EINVAL; + + rw_lock_read(&mm->vm_lock); + ret = do_madvise_walk(mm, start, len, advice); + rw_unlock_read(&mm->vm_lock); + return ret; +} diff --git a/kernel/kernel/mm/memory.c b/kernel/kernel/mm/memory.c index 27bd22519..c2ac0c8fd 100644 --- a/kernel/kernel/mm/memory.c +++ b/kernel/kernel/mm/memory.c @@ -760,6 +760,26 @@ int vm_mmu_unmap(struct mm_address_space *mm, void *addr, size_t pages, struct v return 0; } +int zap_page_range(unsigned long start, unsigned long end, struct vm_area_struct *vma) +{ + struct mm_address_space *mm = vma->vm_mm; + struct unmap_info unmap_info; + unmap_info.vma = vma; + unmap_info.mm = mm; + unmap_info.kernel = 0; + unmap_info.full = 0; + unmap_info.freepgtables = 0; + tlbi_tracker_init(&unmap_info.tlbi); + + spin_lock(&mm->page_table_lock); + pgd_unmap_range(&unmap_info, pgd_offset(mm, start), start, end); + spin_unlock(&mm->page_table_lock); + + if (tlbi_active(&unmap_info.tlbi)) + tlbi_end_batch(&unmap_info.tlbi); + return 0; +} + bool paging_write_protect(void *addr, struct mm_address_space *mm) { spin_lock(&mm->page_table_lock); diff --git a/kernel/kernel/mm/mincore.cpp b/kernel/kernel/mm/mincore.cpp index 8e2b81392..110aab9de 100644 --- a/kernel/kernel/mm/mincore.cpp +++ b/kernel/kernel/mm/mincore.cpp @@ -9,6 +9,9 @@ #include #include +#undef REQUIRES_SHARED +#define REQUIRES_SHARED(...) + // TODO: Export this stuff in some header, and avoid sticking everything into vm.cpp vm_area_struct *vm_search(struct mm_address_space *mm, void *addr, size_t length) REQUIRES_SHARED(mm->vm_lock); @@ -17,13 +20,13 @@ static long do_pagemap(struct mm_address_space *as, unsigned long start, unsigne u64 *pagemap) { - scoped_mutex g{as->vm_lock}; + scoped_rwlock g{as->vm_lock}; long pfns_processed = 0; vm_area_struct *vma; unsigned long index = start; void *entry_; - mt_for_each(&as->region_tree, entry_, index, end) + mt_for_each (&as->region_tree, entry_, index, end) { vma = (vm_area_struct *) entry_; if (vma->vm_start > end) diff --git a/kernel/kernel/mm/pagealloc.cpp b/kernel/kernel/mm/pagealloc.cpp index 84959a25d..976a2b365 100644 --- a/kernel/kernel/mm/pagealloc.cpp +++ b/kernel/kernel/mm/pagealloc.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -315,6 +316,40 @@ struct page *page_zone_alloc(struct page_zone *zone, unsigned int gfp_flags, uns return page_zone_alloc_core(zone, gfp_flags, order); } +static bool page_zone_may_alloc(struct page_zone *zone, gfp_t gfp, unsigned int order) +{ + bool may = false; + unsigned long flags = spin_lock_irqsave(&zone->lock); + bool may_use_reserves = gfp & __GFP_ATOMIC; + unsigned long free_pages = zone->total_pages - zone->used_pages; + + if (free_pages < (1UL << order)) + goto out; + + if (!may_use_reserves && zone->min_watermark > free_pages - (1UL << order)) + goto out; + + if (!list_is_empty(&zone->pages[order])) [[likely]] + { + may = true; + goto out; + } + + /* Ok, this order has no pages, see if we could split other higher order ones */ + for (int i = order + 1; i < PAGEALLOC_NR_ORDERS; i++) + { + if (!list_is_empty(&zone->pages[i])) + { + may = true; + goto out; + } + } + +out: + spin_unlock_irqrestore(&zone->lock, flags); + return may; +} + static void page_zone_add(unsigned long start, unsigned int order, struct page_zone *zone) { scoped_lock g{zone->lock}; @@ -373,18 +408,19 @@ __always_inline struct page *get_buddy(struct page *page, unsigned int order, pa { unsigned long pfn = page_to_pfn(page); unsigned long pfn2 = pfn ^ (1UL << order); + unsigned long addr2 = pfn2 << PAGE_SHIFT; // Check if we can indeed merge with a buddy. if so // 1) the buddy is not past maxpfn (phys_to_page_mayfail) // 2) the buddy is free and the same order as us // 3) the buddy is in the same zone - struct page *p = phys_to_page_mayfail(pfn2 << PAGE_SHIFT); + struct page *p = phys_to_page_mayfail(addr2); if (!p) [[unlikely]] return nullptr; if (!(p->flags & PAGE_BUDDY) || p->priv != order) return nullptr; - if (pfn2 < zone->start || pfn2 > zone->end) + if (addr2 < zone->start || addr2 > zone->end) return nullptr; return p; } @@ -506,6 +542,9 @@ void page_node::add_region(uintptr_t base, size_t size) unsigned long end = cul::clamp(start + size, zone->end) + 1; unsigned long nr_pages = (end - start) >> PAGE_SHIFT; printf("pagealloc: Adding [%016lx, %016lx] to zone %s\n", start, end - 1, zone->name); +#ifdef CONFIG_KASAN + kasan_set_state((unsigned long *) PHYS_TO_VIRT(base), size, 1); +#endif page_zone_add_region(start, nr_pages, zone); nr_global_pages.add_fetch(nr_pages, mem_order::release); start = end; @@ -590,6 +629,9 @@ void page_init(size_t memory_size, unsigned long maxpfn) __kbrk(PHYS_TO_VIRT(ptr), (void *) ((unsigned long) PHYS_TO_VIRT(ptr) + needed_memory)); page_allocate_pagemap(maxpfn); +#ifdef CONFIG_KASAN + kasan_page_alloc_init(); +#endif for_every_phys_region([](unsigned long start, size_t size) { /* page_add_region can't return an error value since it halts @@ -772,6 +814,10 @@ __always_inline void prepare_pages_after_alloc(struct page *page, unsigned int o auto pages = pow2(order); +#ifdef CONFIG_KASAN + kasan_set_state((unsigned long *) PAGE_TO_VIRT(page), (1UL << (order + PAGE_SHIFT)), 0); +#endif + if (page_should_zero(flags)) { memset(PAGE_TO_VIRT(page), 0, 1UL << (order + PAGE_SHIFT)); @@ -893,6 +939,10 @@ void page_node::free_page(struct page *p) if (page_flag_set(p, PAGE_FLAG_ANON)) dec_page_stat(p, NR_ANON); +#ifdef CONFIG_KASAN + kasan_set_state((unsigned long *) PAGE_TO_VIRT(p), PAGE_SIZE, 1); +#endif + /* Reset the page */ p->flags = 0; p->owner = nullptr; @@ -1001,3 +1051,44 @@ void page_accumulate_stats(unsigned long pages[PAGE_STATS_MAX]) return true; }); } + +/** + * @brief Calculate a free page target (for reclaim) + * + * @param gfp GFP used for the failed allocation/reclaim + * @param order Order allocation that failed + * @return Free page target. If 0, probably shouldn't reclaim. + */ +unsigned long page_reclaim_target(gfp_t gfp, unsigned int order) +{ + bool may = false; + unsigned long free_target = pages_under_high_watermark(); + if (free_target > 0) + return free_target; + + /* Everything is over the high watermark. Check if we indeed can accomplish this allocation. + * This does a slight emulation of alloc_page logic paths. + */ + int zone = ZONE_NORMAL; + + if (gfp & PAGE_ALLOC_4GB_LIMIT) + zone = ZONE_DMA32; + + while (zone >= 0) + { + may = page_zone_may_alloc(&main_node.zones[zone], gfp, order); + if (may) + break; + zone--; + } + + if (may) + return 0; + + /* We are above the high watermark, however we can't allocate this order. Start freeing pages, + * as a fixed % of total pages, scaled by order (capped to 3). We heuristically pick 1.5% of + * total pages. + */ + free_target = (nr_global_pages / 66) * cul::max(order, 3U); + return free_target; +} diff --git a/kernel/kernel/mm/reclaim.c b/kernel/kernel/mm/reclaim.c index 199b9f549..5eb9fceca 100644 --- a/kernel/kernel/mm/reclaim.c +++ b/kernel/kernel/mm/reclaim.c @@ -437,12 +437,35 @@ static void isolate_pages(struct page_lru *lru, enum lru_state list, struct list list_splice(&rotate_list, &lru->lru_lists[list]); } +struct pagebatch +{ + struct page *batch[32]; + int nr; +}; + +static bool page_batch_add(struct pagebatch *batch, struct page *page) +{ + batch->batch[batch->nr++] = page; + return batch->nr == 32; +} + +static void page_unref_batch(struct pagebatch *batch) +{ + /* LRU lock *is not held* */ + for (int i = 0; i < batch->nr; i++) + page_unref(batch->batch[i]); + batch->nr = 0; +} + static unsigned long shrink_page_list(struct reclaim_data *data, struct page_lru *lru, struct list_head *page_list) { DEFINE_LIST(rotate_list); DEFINE_LIST(activate_list); + struct pagebatch free_batch; unsigned long freedp = 0; + + free_batch.nr = 0; list_for_every_safe (page_list) { struct page *page = container_of(l, struct page, lru_node); @@ -473,7 +496,12 @@ static unsigned long shrink_page_list(struct reclaim_data *data, struct page_lru list_add_tail(&page->lru_node, &lru->lru_lists[LRU_INACTIVE_FILE + page_to_state(page)]); page_set_lru(page); inc_page_stat(page, NR_INACTIVE_FILE + page_to_state(page)); - page_unref(page); + if (page_batch_add(&free_batch, page)) + { + spin_unlock(&lru->lock); + page_unref_batch(&free_batch); + spin_lock(&lru->lock); + } } list_for_every_safe (&activate_list) @@ -484,11 +512,16 @@ static unsigned long shrink_page_list(struct reclaim_data *data, struct page_lru inc_page_stat(page, NR_ACTIVE_FILE + page_to_state(page)); page_clear_referenced(page); list_add_tail(&page->lru_node, &lru->lru_lists[LRU_ACTIVE_FILE + page_to_state(page)]); - page_unref(page); + if (page_batch_add(&free_batch, page)) + { + spin_unlock(&lru->lock); + page_unref_batch(&free_batch); + spin_lock(&lru->lock); + } } spin_unlock(&lru->lock); - + page_unref_batch(&free_batch); out: return freedp; } @@ -573,6 +606,13 @@ static void shrink_page_zones(struct reclaim_data *data, struct page_node *node) if (freep <= zone->low_watermark) target = zone->high_watermark - freep; + /* This logic is weird and leaky (we don't get nearly as many details from + * page_reclaim_target as we'd wish), but it should do the job. Get 1.5 * max(order, 3)% of + * the zone free. + */ + if (target == 0 && data->failed_order > 0) + target = (zone->total_pages / 66) * max(data->failed_order, 3); + if (target == 0) continue; @@ -594,7 +634,7 @@ int page_do_reclaim(struct reclaim_data *data) int max_tries = data->attempt > 0 ? 5 : 3; int nr_tries = 0; - while ((free_target = pages_under_high_watermark()) > 0) + while ((free_target = page_reclaim_target(data->gfp_flags, data->failed_order)) > 0) { if (nr_tries == max_tries) return -1; diff --git a/kernel/kernel/mm/slab.c b/kernel/kernel/mm/slab.c index 753de0bab..1456e39ef 100644 --- a/kernel/kernel/mm/slab.c +++ b/kernel/kernel/mm/slab.c @@ -19,7 +19,7 @@ #include #include -static struct mutex cache_list_lock; +static DECLARE_MUTEX(cache_list_lock); static struct list_head cache_list GUARDED_BY(cache_list_lock) = LIST_HEAD_INIT(cache_list); #define KMEM_CACHE_KEEP_THRESHOLD 131072 @@ -124,7 +124,7 @@ struct slab_cache *kmem_cache_create(const char *name, size_t size, size_t align #else c->redzone = 0; #endif - c->flags = flags | KMEM_CACHE_VMALLOC; + c->flags = flags; c->bufctl_off = 0; // Minimum object alignment is 16 @@ -496,7 +496,7 @@ NO_ASAN static struct slab *kmem_cache_create_slab(struct slab_cache *cache, uns { unsigned int order = pages2order(slab_size >> PAGE_SHIFT); slab_size = 1UL << (order + PAGE_SHIFT); - pages = alloc_pages(order, PAGE_ALLOC_NO_ZERO | PAGE_ALLOC_CONTIGUOUS); + pages = alloc_pages(order, flags | PAGE_ALLOC_NO_ZERO | PAGE_ALLOC_CONTIGUOUS); if (!pages) return NULL; start = (char *) PAGE_TO_VIRT(pages); @@ -529,6 +529,7 @@ NO_ASAN static struct slab *kmem_cache_create_slab(struct slab_cache *cache, uns asan_poison_shadow((unsigned long) ptr, redzone, KASAN_LEFT_REDZONE); #endif ptr += redzone; + CHECK(((unsigned long) ptr % cache->alignment) == 0); if (cache->ctor) cache->ctor(ptr); struct bufctl *ctl = (struct bufctl *) (ptr + cache->bufctl_off); @@ -802,7 +803,7 @@ static int kmem_cache_alloc_refill_mag(struct slab_cache *cache, * use __GFP_ATOMIC for the first slab, and anything else is very much extra, thus hardly * "__GFP_ATOMIC". */ flags &= ~__GFP_ATOMIC; - flags |= __GFP_NOWAIT; + flags |= __GFP_NOWAIT | __GFP_NOWARN; list_add_tail(&s->slab_list_node, &allocated_slabs); } @@ -1540,8 +1541,8 @@ void kmalloc_init() { // We start at 16 bytes size_t size = 1UL << (4 + i); - unsigned int flags = KMEM_CACHE_VMALLOC; -#if 0 + unsigned int flags = 0; +#if 1 // TODO: Toggling VMALLOC only for larger sizes is not working well... // at least for will-it-scale/page_fault1, it results in major performance regressions. // Is this a TLB issue? Maybe? @@ -1622,10 +1623,10 @@ void *calloc(size_t nr, size_t size) return kcalloc(nr, size, GFP_ATOMIC); } -void *realloc(void *ptr, size_t size) +void *krealloc(void *ptr, size_t size, int flags) { if (!ptr) - return malloc(size); + return kmalloc(size, flags); struct slab *old_slab = kmem_pointer_to_slab(ptr); @@ -1641,7 +1642,7 @@ void *realloc(void *ptr, size_t size) return ptr; } - void *newbuf = malloc(size); + void *newbuf = kmalloc(size, flags); if (!newbuf) return NULL; __memcpy(newbuf, ptr, old_slab->cache->objsize); @@ -1649,17 +1650,22 @@ void *realloc(void *ptr, size_t size) return newbuf; } +void *realloc(void *ptr, size_t size) +{ + return krealloc(ptr, size, GFP_ATOMIC); +} + int posix_memalign(void **pptr, size_t align, size_t len) { *pptr = NULL; return -1; } -void *reallocarray(void *ptr, size_t m, size_t n) +void *kreallocarray(void *ptr, size_t m, size_t n, int flags) { if (array_overflows(m, n)) return NULL; - return realloc(ptr, n * m); + return krealloc(ptr, n * m, flags); } #ifdef CONFIG_KASAN @@ -1668,12 +1674,13 @@ void kmem_free_kasan(void *ptr) { struct slab_cache *cache; struct slab *slab = kmem_pointer_to_slab(ptr); + struct bufctl *buf = ptr; assert(slab != NULL); cache = slab->cache; - kmem_bufctl_from_ptr(cache, ptr)->flags = 0; + buf->flags = 0; spin_lock(&cache->lock); - kmem_free_to_slab(cache, slab, ptr); + kmem_free_to_slab(cache, slab, kmem_bufctl_to_ptr(cache, buf)); spin_unlock(&cache->lock); } @@ -1765,7 +1772,6 @@ void slab_shrink_caches(unsigned long target_freep) struct slab_rendezvous rndvz; sched_disable_preempt(); kmem_slab_freeze_start(&rndvz); - sched_enable_preempt(); list_for_every (&cache_list) { @@ -1781,6 +1787,7 @@ void slab_shrink_caches(unsigned long target_freep) } kmem_slab_freeze_end(&rndvz); + sched_enable_preempt(); list_for_every (&cache_list) { diff --git a/kernel/kernel/mm/swap.c b/kernel/kernel/mm/swap.c index ddcc488ab..6800b9373 100644 --- a/kernel/kernel/mm/swap.c +++ b/kernel/kernel/mm/swap.c @@ -522,11 +522,27 @@ static void swap_final_put(swp_entry_t swp) return; } - __swap_add_counter(-1); *map = 0; spin_unlock(&bg->lock); } +static u8 swap_get_map(swp_entry_t swp) +{ + struct swap_area *sa = swap_areas[SWP_TYPE(swp)]; + unsigned long eff_off = SWP_OFFSET(swp) - sa->swap_off; + struct swap_block_group *bg = &sa->block_groups[eff_off / MAX_BLOCK_GROUP_SIZE]; + u8 *map; + u8 count; + + spin_lock(&bg->lock); + + map = bg->start + (eff_off % MAX_BLOCK_GROUP_SIZE); + count = *map; + spin_unlock(&bg->lock); + + return count; +} + void swap_unset_swapcache(swp_entry_t swp) { struct swap_area *sa = swap_areas[SWP_TYPE(swp)]; @@ -565,6 +581,8 @@ void swap_inc_map(struct page *page) __swap_inc_map(swpval_to_swp_entry(page->priv)); } +void dump_page(struct page *page); + static int swap_add_to_swapcache(struct page *page) { struct page *result; @@ -583,7 +601,13 @@ static int swap_add_to_swapcache(struct page *page) /* WARN if a page was stale in the swap cache */ if (WARN_ON(result != page)) + { + pr_warn("swap: swap space %lu had a stale swapcache entry at %lx (%p vs %p)\n", + SWP_TYPE(entry), SWP_OFFSET(entry), result, page); + dump_page(result); + pr_warn("swap: swap map entry %hhx\n", swap_get_map(entry)); return -EINVAL; + } return 0; } @@ -611,6 +635,7 @@ int swap_add(struct page *page) if (err) { swap_put_page(page); + page_clear_swap(page); return err; } diff --git a/kernel/kernel/mm/vm.cpp b/kernel/kernel/mm/vm.cpp index ca038d772..a7938a51a 100644 --- a/kernel/kernel/mm/vm.cpp +++ b/kernel/kernel/mm/vm.cpp @@ -71,6 +71,14 @@ static bool limits_are_contained(struct vm_area_struct *reg, unsigned long start unsigned long limit); static bool vm_mapping_is_cow(struct vm_area_struct *entry); +/* TODO */ +#undef REQUIRES_SHARED +#undef REQUIRES +#undef EXCLUDES +#define REQUIRES_SHARED(...) +#define REQUIRES(...) +#define EXCLUDES(...) + vm_area_struct *vm_search(struct mm_address_space *mm, void *addr, size_t length) REQUIRES_SHARED(mm->vm_lock); static void vma_pre_adjust(struct vm_area_struct *vma); @@ -295,8 +303,6 @@ void do_vm_unmap(struct mm_address_space *as, void *range, size_t pages) struct vm_area_struct *entry = vm_find_region(as, range); assert(entry != nullptr); - MUST_HOLD_MUTEX(&entry->vm_mm->vm_lock); - vm_mmu_unmap(entry->vm_mm, range, pages, entry); } @@ -342,8 +348,6 @@ bool vm_mapping_requires_write_protect(struct vm_area_struct *reg) static void vma_destroy(struct vm_area_struct *region) { - MUST_HOLD_MUTEX(®ion->vm_mm->vm_lock); - /* First, unref things */ if (region->vm_file) { @@ -416,7 +420,7 @@ int vm_clone_as(mm_address_space *addr_space, mm_address_space *original) original = get_current_address_space(); if (!original) original = &kernel_address_space; - scoped_mutex g{original->vm_lock}; + scoped_rwlock g{original->vm_lock}; return paging_clone_as(addr_space, original); } @@ -512,7 +516,7 @@ int vm_fork_address_space(struct mm_address_space *addr_space) EXCLUDES(addr_spa EXCLUDES(get_current_address_space()->vm_lock) { struct mm_address_space *current_mm = get_current_address_space(); - scoped_mutex g{current_mm->vm_lock}; + scoped_rwlock g{current_mm->vm_lock}; #ifdef CONFIG_DEBUG_ADDRESS_SPACE_ACCT mmu_verify_address_space_accounting(get_current_address_space()); @@ -553,7 +557,7 @@ int vm_fork_address_space(struct mm_address_space *addr_space) EXCLUDES(addr_spa assert(addr_space->active_mask.is_empty()); - mutex_init(&addr_space->vm_lock); + rwlock_init(&addr_space->vm_lock); validate_mm_tree(addr_space); return 0; } @@ -569,17 +573,10 @@ void vm_change_perms(void *range, size_t pages, int perms) NO_THREAD_SAFETY_ANAL { struct mm_address_space *as; bool kernel = is_higher_half(range); - bool needs_release = false; - if (kernel) - as = &kernel_address_space; - else - as = get_current_process()->get_aspace(); + DCHECK(!kernel); - if (mutex_owner(&as->vm_lock) != get_current_thread()) - { - needs_release = true; - mutex_lock(&as->vm_lock); - } + as = &kernel_address_space; + rw_lock_write(&as->vm_lock); for (size_t i = 0; i < pages; i++) { @@ -589,9 +586,7 @@ void vm_change_perms(void *range, size_t pages, int perms) NO_THREAD_SAFETY_ANAL } vm_invalidate_range((unsigned long) range, pages); - - if (needs_release) - mutex_unlock(&as->vm_lock); + rw_unlock_write(&as->vm_lock); } static bool vma_can_merge_into(struct vm_area_struct *vma, size_t size, int vm_flags, @@ -829,7 +824,7 @@ void *vm_mmap(void *addr, size_t length, int prot, int flags, struct file *file, if (off & (PAGE_SIZE - 1)) return ERR_PTR(-EINVAL); - scoped_mutex g{mm->vm_lock}; + scoped_rwlock g{mm->vm_lock}; /* Calculate the pages needed for the overall size */ size_t pages = vm_size_to_pages(length); @@ -1136,7 +1131,7 @@ int vm_mprotect(struct mm_address_space *as, void *__addr, size_t size, int prot unsigned long limit = addr + size; VMA_ITERATOR(vmi, as, addr, limit); - scoped_mutex g{as->vm_lock}; + scoped_rwlock g{as->vm_lock}; /* Note: vm_munmap has some vma detaching logic for the simple fact that POSIX does not * allow for a partial unmap in case of an error. Whereas this is not the case for mprotect. @@ -1239,7 +1234,7 @@ uint64_t sys_brk(void *newbrk) { mm_address_space *as = get_current_address_space(); - scoped_mutex g{as->vm_lock}; + scoped_rwlock g{as->vm_lock}; if (newbrk == nullptr) { @@ -1458,14 +1453,14 @@ int vm_handle_page_fault(struct fault_info *info) if (irq_is_disabled()) panic("Page fault while IRQs were disabled\n"); - /* Surrender immediately if there's no user address space or the fault was inside vm code */ - if (!as || mutex_holds_lock(&as->vm_lock)) + /* Surrender immediately if there's no user address space */ + if (!as) { info->signal = VM_SIGSEGV; return -1; } - scoped_mutex g{as->vm_lock}; + scoped_rwlock g{as->vm_lock}; struct vm_area_struct *entry = vm_find_region(as, (void *) info->fault_address); if (!entry) @@ -1554,7 +1549,7 @@ void vm_destroy_addr_space(struct mm_address_space *mm) bool free_pgd = true; /* First, iterate through the maple tree and free/unmap stuff */ - scoped_mutex g{mm->vm_lock}; + scoped_rwlock g{mm->vm_lock}; vm_area_struct *entry; void *entry_; @@ -1972,7 +1967,7 @@ int vm_create_address_space(struct mm_address_space *mm) assert(mm->active_mask.is_empty() == true); - mutex_init(&mm->vm_lock); + rwlock_init(&mm->vm_lock); return 0; } @@ -2008,8 +2003,6 @@ void *vm_get_fallback_pgd() void vm_remove_region(struct mm_address_space *as, struct vm_area_struct *region) REQUIRES(as->vm_lock) { - MUST_HOLD_MUTEX(&as->vm_lock); - void *ret = mtree_erase(&as->region_tree, region->vm_start); CHECK(ret == region); } @@ -2020,8 +2013,6 @@ int __vm_munmap(struct mm_address_space *as, void *__addr, size_t size) REQUIRES unsigned long limit = ALIGN_TO(((unsigned long) __addr) + size, PAGE_SIZE); struct list_head list = LIST_HEAD_INIT(list); - MUST_HOLD_MUTEX(&as->vm_lock); - struct vm_area_struct *vma = vm_search(as, (void *) addr, PAGE_SIZE); if (!vma) return -EINVAL; @@ -2093,7 +2084,7 @@ int __vm_munmap(struct mm_address_space *as, void *__addr, size_t size) REQUIRES */ int vm_munmap(struct mm_address_space *as, void *__addr, size_t size) { - scoped_mutex g{as->vm_lock}; + scoped_rwlock g{as->vm_lock}; auto addr = (unsigned long) __addr; if (addr < as->start || addr > as->end) @@ -2165,8 +2156,6 @@ static int __vm_expand_mapping(struct vm_area_struct *region, size_t new_size) static int vm_expand_mapping(struct mm_address_space *as, struct vm_area_struct *region, size_t new_size) REQUIRES(as->vm_lock) { - MUST_HOLD_MUTEX(&as->vm_lock); - if (!vm_can_expand(as, region, new_size)) return -1; @@ -2241,7 +2230,7 @@ void *sys_mremap(void *old_address, size_t old_size, size_t new_size, int flags, bool fixed = flags & MREMAP_FIXED; bool wants_create_new_mapping_of_pages = old_size == 0 && may_move; void *ret = MAP_FAILED; - scoped_mutex g{current->address_space->vm_lock}; + scoped_rwlock g{current->address_space->vm_lock}; auto as = current->get_aspace(); /* TODO: Unsure on what to do if new_size > old_size */ @@ -2419,7 +2408,7 @@ int get_phys_pages(void *_addr, unsigned int flags, struct page **pages, size_t return ret; } - scoped_mutex g{as->vm_lock}; + scoped_rwlock g{as->vm_lock}; size_t pages_gotten = 0; @@ -2512,7 +2501,7 @@ int sys_msync(void *ptr, size_t length, int flags) return -EINVAL; /* Hogging the vm_lock is bad mkay, todo... */ - scoped_mutex g{as->vm_lock}; + scoped_rwlock g{as->vm_lock}; struct vm_area_struct *vma = vm_search(as, (void *) addr, length); diff --git a/kernel/kernel/mm/vm_object.cpp b/kernel/kernel/mm/vm_object.cpp index 23d66cb33..e89413283 100644 --- a/kernel/kernel/mm/vm_object.cpp +++ b/kernel/kernel/mm/vm_object.cpp @@ -240,6 +240,11 @@ static void vm_obj_truncate_out(struct vm_object *obj, struct page *const *batch CHECK(pg->owner == obj); int st = obj->vm_pages.store(pg->pageoff, 0); CHECK(st == 0); + /* Once we release this lock, other people trying to get a stable reference to this page + * will re-check owner under the page lock. If it observes a pg->owner == original_vmobj, + * it'll keep going without realizing this page is being/was truncated. So we need to + * WRITE_ONCE pg->owner here. */ + WRITE_ONCE(pg->owner, nullptr); } spin_unlock(&obj->page_lock); @@ -570,6 +575,7 @@ bool vm_obj_remove_page(struct vm_object *obj, struct page *page) obj->vm_pages.store(page_pgoff(page), 0); if (page_test_swap(page)) swap_unset_swapcache(swpval_to_swp_entry(page->priv)); + /* We do not need to reset owner here, because we're the only reference */ ret = true; out: spin_unlock(&obj->page_lock); diff --git a/kernel/kernel/net/ipv4/arp.cpp b/kernel/kernel/net/ipv4/arp.cpp index 4db8b7dcd..d9fc9d865 100644 --- a/kernel/kernel/net/ipv4/arp.cpp +++ b/kernel/kernel/net/ipv4/arp.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2022 Pedro Falcato + * Copyright (c) 2016 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -9,9 +9,9 @@ #include #include -/* Don't change the include order! Maybe TOFIX? */ #include #include +#include #include #include #include @@ -22,21 +22,21 @@ #include -/* TODO: Maybe the neighbour_table could replace some code below, if we add a few virtual functions - */ static neighbour_table arp_table{AF_INET}; static constexpr hrtime_t arp_response_timeout = 250 * NS_PER_MS; /* 20 minutes in milis */ static constexpr unsigned long arp_validity_time_ms = 1200000; -int arp_do_request(netif *netif, packetbuf *packet) +static int arp_do_request(netif *netif, packetbuf *packet, arp_request_t *arp_hdr) { - auto arp_hdr = (arp_request_t *) packet->data; - auto target_addr = arp_hdr->target_proto_address; uint8_t hw_address[6]; + /* TODO */ + (void) arp_validity_time_ms; + (void) arp_response_timeout; + if (netif->local_ip.sin_addr.s_addr == target_addr) { memcpy(hw_address, netif->mac_address, 6); @@ -73,60 +73,51 @@ int arp_do_request(netif *netif, packetbuf *packet) return netif_send_packet(netif, buf.get()); } +static int arp_resolve(struct neighbour *neigh, struct netif *netif); + +static const struct neigh_ops arp_ops = { + .resolve = arp_resolve, + .output = ip_finish_output, +}; + int arp_handle_packet(netif *netif, packetbuf *buf) { - auto arp = (arp_request_t *) buf->data; + struct neighbour *neigh; + int added; + arp_request_t *arp = (arp_request_t *) pbf_pull(buf, sizeof(arp_request_t)); + if (!arp) + return -1; - if (buf->length() < sizeof(arp_request_t)) - return -EINVAL; auto op = htons(arp->operation); if (op == ARP_OP_REQUEST) - return arp_do_request(netif, buf); + return arp_do_request(netif, buf, arp); if (op != ARP_OP_REPLY) return 0; in_addr_t req_ip = arp->sender_proto_address; - neigh_proto_addr addr; + union neigh_proto_addr addr; addr.in4addr.s_addr = req_ip; - auto [ptr, __created] = arp_table.add(addr, true); - if (!ptr) - return 0; - - unsigned char *mac = new unsigned char[ETH_ALEN]; - if (!mac) + neigh = neigh_add(&arp_table, &addr, GFP_ATOMIC, &arp_ops, &added); + if (!neigh) return 0; - memcpy(mac, arp->sender_hw_address, ETH_ALEN); - - cul::slice sl{mac, ETH_ALEN}; - ptr->set_hwaddr(sl); - ptr->flags |= NEIGHBOUR_FLAG_HAS_RESPONSE; - + neigh_complete_lookup(neigh, arp->sender_hw_address, ETH_ALEN); return 0; } -int arp_submit_request(shared_ptr &ptr, uint32_t target_addr, struct netif *netif) +static int arp_resolve(struct neighbour *neigh, struct netif *netif) { + in_addr_t target_addr = neigh->proto_addr.in4addr.s_addr; + if (target_addr == INADDR_BROADCAST || target_addr == INADDR_LOOPBACK || netif->flags & NETIF_LOOPBACK) { - auto _ptr = new unsigned char[ETH_ALEN]; - - if (_ptr) - { - bool is_bcast = target_addr == INADDR_BROADCAST; - memset(_ptr, is_bcast ? 0xff : 0, ETH_ALEN); - auto sl = cul::slice{_ptr, ETH_ALEN}; - ptr->set_hwaddr(sl); - ptr->flags |= NEIGHBOUR_FLAG_HAS_RESPONSE | (is_bcast ? NEIGHBOUR_FLAG_BROADCAST : 0); - } - else - { - return -ENOMEM; - } - + const unsigned char bcast_eth[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; + const unsigned char loopback_eth[ETH_ALEN] = {}; + __neigh_complete_lookup(neigh, target_addr == INADDR_BROADCAST ? bcast_eth : loopback_eth, + ETH_ALEN); return 0; } @@ -155,7 +146,7 @@ int arp_submit_request(shared_ptr &ptr, uint32_t target_addr, struct arp->target_hw_address[4] = 0xFF; arp->target_hw_address[5] = 0xFF; arp->sender_proto_address = netif->local_ip.sin_addr.s_addr; - arp->target_proto_address = target_addr; + arp->target_proto_address = neigh->proto_addr.in4addr.s_addr; if (int st = netif->dll_ops->setup_header(buf.get(), tx_type::broadcast, tx_protocol::arp, netif, nullptr); st < 0) @@ -164,48 +155,18 @@ int arp_submit_request(shared_ptr &ptr, uint32_t target_addr, struct return netif_send_packet(netif, buf.get()); } -expected, int> arp_resolve_in(uint32_t ip, struct netif *netif) +struct neighbour *arp_resolve_in(uint32_t ip, struct netif *netif) { - neigh_proto_addr addr; + struct neighbour *neigh; + int added; + union neigh_proto_addr addr; addr.in4addr.s_addr = ip; - auto [ptr, created] = arp_table.add(addr); - if (!ptr) - return unexpected{-ENOMEM}; - - if (ptr->flags & NEIGHBOUR_FLAG_UNINITIALISED) - { - if (created) - { - if (arp_submit_request(ptr, ip, netif) < 0) - { - arp_table.remove(ptr.get_data()); - return unexpected{-ENOMEM}; - } - } - - auto t0 = clocksource_get_time(); - - /* TODO: Add a wait_for_bit that can let us wait for random things - * without taking up permanent space in the structure - */ - while (!(ptr->flags & NEIGHBOUR_FLAG_HAS_RESPONSE) && - clocksource_get_time() - t0 <= arp_response_timeout) - sched_sleep_ms(15); - - if (!(ptr->flags & NEIGHBOUR_FLAG_HAS_RESPONSE)) - { - if (created) - arp_table.remove(ptr.get_data()); - return unexpected{-ENETUNREACH}; - } - - if (created) - { - ptr->set_validity(arp_validity_time_ms); - ptr->set_initialised(); - } - } + neigh = neigh_add(&arp_table, &addr, GFP_ATOMIC, &arp_ops, &added); + if (!neigh) + return (struct neighbour *) ERR_PTR(-ENOMEM); - return ptr; + if (neigh_needs_resolve(neigh)) + neigh_start_resolve(neigh, netif); + return neigh; } diff --git a/kernel/kernel/net/ipv4/icmp.cpp b/kernel/kernel/net/ipv4/icmp.cpp index 5ef15d6f4..7a28ef2b9 100644 --- a/kernel/kernel/net/ipv4/icmp.cpp +++ b/kernel/kernel/net/ipv4/icmp.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2022 Pedro Falcato + * Copyright (c) 2016 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -72,6 +72,7 @@ void send_echo_reply(ip_header *iphdr, icmp_header *icmphdr, uint16_t length, ne response_icmp->code = 0; response_icmp->rest = icmphdr->rest; memcpy(buf->put(data_length), &icmphdr->echo.data, data_length); + response_icmp->checksum = 0; response_icmp->checksum = ipsum(response_icmp, length); inet_sock_address from{src, 0}; diff --git a/kernel/kernel/net/ipv4/ipv4.cpp b/kernel/kernel/net/ipv4/ipv4.cpp index 3b0f0070c..2c8716b92 100644 --- a/kernel/kernel/net/ipv4/ipv4.cpp +++ b/kernel/kernel/net/ipv4/ipv4.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2022 Pedro Falcato + * Copyright (c) 2016 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -253,26 +254,27 @@ static tx_type detect_tx_type(const inet_route &route) return tx_type::unicast; } -int send_fragment(const inet_route &route, fragment *frag, netif *nif) +int send_frag_multicast(tx_type type, const inet_route &route, fragment *frag, netif *nif) { - auto buf = frag->this_buf; - auto type = detect_tx_type(route); - int st = 0; - - const void *hwaddr = nullptr; - - if (type != tx_type::broadcast) [[likely]] - { - if (route.dst_hw->flags & NEIGHBOUR_FLAG_BADENTRY) - return -EHOSTUNREACH; - hwaddr = route.dst_hw->hwaddr().data(); - } - - if ((st = nif->dll_ops->setup_header(buf, type, tx_protocol::ipv4, nif, hwaddr)) < 0) - [[unlikely]] + /* Sending an IPv4 fragment deviates from the neighbour paths because we truly don't communicate + * with a neighbour - at least not with a single one. */ + int st; + if ((st = nif->dll_ops->setup_header(frag->this_buf, type, tx_protocol::ipv4, nif, nullptr)) < + 0) [[unlikely]] return st; - return netif_send_packet(nif, buf); + return netif_send_packet(nif, frag->this_buf); +} + +int send_fragment(const inet_route &route, fragment *frag, netif *nif) +{ + tx_type type = detect_tx_type(route); + if (type != tx_type::unicast) + return send_frag_multicast(type, route, frag, nif); + struct neighbour *neigh = route.dst_hw; + CHECK(neigh != NULL); + frag->this_buf->route = route; + return neigh_output(neigh, frag->this_buf, nif); } int do_fragmentation(struct send_info *sinfo, size_t payload_size, packetbuf *buf, @@ -290,9 +292,7 @@ int do_fragmentation(struct send_info *sinfo, size_t payload_size, packetbuf *bu list_for_every (&frags) { struct fragment *frag = container_of(l, struct fragment, list_node); - st = send_fragment(sinfo->route, frag, netif); - if (st < 0) goto out; } @@ -324,7 +324,7 @@ int send_packet(const iflow &flow, packetbuf *buf, const cul::slice & sinfo.type = flow.protocol; sinfo.frags_following = false; - if (needs_fragmentation(buf->length(), netif)) + if (needs_fragmentation(payload_size, netif)) { /* TODO: Support ISO(IP segmentation offloading) */ sinfo.identification = allocate_id(); @@ -451,14 +451,14 @@ int handle_packet(netif *nif, packetbuf *buf) route.flags |= INET4_ROUTE_FLAG_BROADCAST; } - new (&buf->route) inet_route{route}; + buf->route = cul::move(route); if (header->proto == IPPROTO_UDP) - return udp_handle_packet(route, buf); + return udp_handle_packet(buf->route, buf); else if (header->proto == IPPROTO_TCP) - return tcp_handle_packet(route, buf); + return tcp_handle_packet(buf->route, buf); else if (header->proto == IPPROTO_ICMP) - return icmp::handle_packet(route, buf); + return icmp::handle_packet(buf->route, buf); else { /* Oh, no, an unhandled protocol! Send an ICMP error message */ @@ -478,7 +478,7 @@ int handle_packet(netif *nif, packetbuf *buf) dst_un.dgram = dgram; dst_un.next_hop_mtu = 0; - icmp::send_dst_unreachable(dst_un, nif); + // icmp::send_dst_unreachable(dst_un, nif); } return 0; @@ -615,6 +615,7 @@ expected route(const inet_sock_address &from, const inet_sock_a */ shared_ptr best_route; int highest_metric = 0; + int longest_prefix = -1; auto dest = to.in4.s_addr; // TODO: Multicast @@ -641,8 +642,9 @@ expected route(const inet_sock_address &from, const inet_sock_a /* Do a bitwise and between the destination address and the mask * If the result = r.dest, we can use this interface. */ + int mask_bits; #if 0 - printk("dest %x, mask %x, supposed dest %x\n", dest, r->mask, r->dest); + pr_info("dest %pI4, mask %pI4, supposed dest %pI4\n", &dest, &r->mask, &r->dest); #endif if ((dest & r->mask) != r->dest) continue; @@ -650,27 +652,27 @@ expected route(const inet_sock_address &from, const inet_sock_a if (required_netif && r->nif != required_netif) continue; #if 0 - printk("%s is good\n", r->nif->name); - printk("is loopback set %u\n", r->nif->flags & NETIF_LOOPBACK); -#endif - - int mods = 0; + pr_info("%s is good\n", r->nif->name); + pr_info("is loopback set %u\n", r->nif->flags & NETIF_LOOPBACK); if (r->flags & INET4_ROUTE_FLAG_GATEWAY) - mods--; + pr_info("gateway %pI4\n", &r->gateway); +#endif + /* TODO: This can and should be computed beforehand */ + mask_bits = count_bits(r->mask); - if (r->metric + mods > highest_metric) + if (mask_bits > longest_prefix || + (longest_prefix == mask_bits && r->metric > highest_metric)) { best_route = r; highest_metric = r->metric; + longest_prefix = mask_bits; } } routing_table_lock.unlock_read(); if (!best_route) - { return unexpected{-ENETUNREACH}; - } inet_route r; r.dst_addr.in4 = to.in4; @@ -681,30 +683,18 @@ expected route(const inet_sock_address &from, const inet_sock_a r.gateway_addr.in4.s_addr = best_route->gateway; if (addr_is_broadcast(to.in4.s_addr, r)) - { r.flags |= INET4_ROUTE_FLAG_BROADCAST; - } else if (addr_is_multicast(to.in4.s_addr)) - { r.flags |= INET4_ROUTE_FLAG_MULTICAST; - } auto to_resolve = r.dst_addr.in4.s_addr; if (r.flags & INET4_ROUTE_FLAG_GATEWAY) - { to_resolve = r.gateway_addr.in4.s_addr; - } - - auto res = arp_resolve_in(to_resolve, r.nif); - - if (res.has_error()) [[unlikely]] - { - return unexpected(-ENETUNREACH); - } - - r.dst_hw = res.value(); + r.dst_hw = arp_resolve_in(to_resolve, r.nif); + if (IS_ERR(r.dst_hw)) + return unexpected{PTR_ERR(r.dst_hw)}; return r; } @@ -901,3 +891,20 @@ bool inet_proto_family::add_socket(inet_socket *sock) auto sock_table = proto_info->get_socket_table(); return sock_table->add_socket(sock, ADD_SOCKET_UNLOCKED); } + +int ip_finish_output(struct neighbour *neigh, struct packetbuf *pbf, struct netif *nif) +{ + auto type = ip::v4::detect_tx_type(pbf->route); + int st = 0; + const void *hwaddr = nullptr; + + if (neigh->flags & NUD_FAILED) + return -EHOSTUNREACH; + hwaddr = neigh->hwaddr; + + if ((st = nif->dll_ops->setup_header(pbf, type, tx_protocol::ipv4, nif, hwaddr)) < 0) + [[unlikely]] + return st; + + return netif_send_packet(nif, pbf); +} diff --git a/kernel/kernel/net/ipv6/ipv6.cpp b/kernel/kernel/net/ipv6/ipv6.cpp index 79fc0876e..12eb1f075 100644 --- a/kernel/kernel/net/ipv6/ipv6.cpp +++ b/kernel/kernel/net/ipv6/ipv6.cpp @@ -1,11 +1,12 @@ /* - * Copyright (c) 2020 - 2022 Pedro Falcato + * Copyright (c) 2020 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * * SPDX-License-Identifier: GPL-2.0-only */ +#include #include #include #include @@ -38,6 +39,16 @@ static constexpr tx_type ipv6_addr_to_tx_type(const in6_addr &dst) return dst.s6_addr[0] == 0xff ? tx_type::multicast : tx_type::unicast; } +static int output_mcast(struct packetbuf *pbf, struct netif *nif, struct ip6hdr *hdr) +{ + int st; + if ((st = nif->dll_ops->setup_header(pbf, tx_type::multicast, tx_protocol::ipv6, nif, + &hdr->dst_addr)) < 0) + return st; + + return netif_send_packet(nif, pbf); +} + int send_packet(const iflow &flow, packetbuf *buf) { const auto &route = flow.route; @@ -72,25 +83,12 @@ int send_packet(const iflow &flow, packetbuf *buf) hdr->next_header = next_header; hdr->hop_limit = flow.ttl; - int st = 0; - const auto ttype = ipv6_addr_to_tx_type(route.dst_addr.in6); - const void *dst_hw = nullptr; + if (ttype == tx_type::multicast) + return output_mcast(buf, netif, hdr); - if (ttype == tx_type::unicast) - { - dst_hw = route.dst_hw->hwaddr().data(); - } - else if (ttype == tx_type::multicast) - { - /* Let the lower layer figure out the multicast address */ - dst_hw = &hdr->dst_addr; - } - - if ((st = netif->dll_ops->setup_header(buf, ttype, tx_protocol::ipv6, netif, dst_hw)) < 0) - return st; - - return netif_send_packet(netif, buf); + buf->route = route; + return neigh_output(buf->route.dst_hw, buf, netif); } int bind_internal(sockaddr_in6 *in, inet_socket *sock) @@ -223,6 +221,7 @@ expected route_from_routing_table(const inet_sock_address &to, shared_ptr best_route; int highest_metric = 0; auto dest = to.in6; + int longest_prefix = -1; { @@ -233,12 +232,14 @@ expected route_from_routing_table(const inet_sock_address &to, /* Do a bitwise and between the destination address and the mask * If the result = r.dest, we can use this interface. */ + int mask_bits; + const auto masked = dest & r->mask; #if 0 print_v6_addr(dest); print_v6_addr(r->mask); print_v6_addr(r->dest); #endif - if ((dest & r->mask) != r->dest) + if (masked != r->dest) continue; if (required_netif && r->nif != required_netif) @@ -247,15 +248,22 @@ expected route_from_routing_table(const inet_sock_address &to, printk("%s is good\n", r->nif->name); printk("is loopback set %u\n", r->nif->flags & NETIF_LOOPBACK); #endif + /* TODO: This can and should be computed beforehand */ + mask_bits = count_bits(r->mask.__in6_union.__s6_addr32[0]) + + count_bits(r->mask.__in6_union.__s6_addr32[1]) + + count_bits(r->mask.__in6_union.__s6_addr32[2]) + + count_bits(r->mask.__in6_union.__s6_addr32[3]); - int mods = 0; - if (r->flags & INET4_ROUTE_FLAG_GATEWAY) - mods--; - - if (r->metric + mods > highest_metric) +#if 0 + pr_warn("dest %pI6 mask %pI6 route dest %pI6 mask_bits %d\n", &dest, &r->mask, &r->dest, + mask_bits); +#endif + if (mask_bits > longest_prefix || + (longest_prefix == mask_bits && r->metric > highest_metric)) { best_route = r; highest_metric = r->metric; + longest_prefix = mask_bits; } } } @@ -351,14 +359,10 @@ expected route(const inet_sock_address &from, const inet_sock_a // printk("Gateway %x\n", ntohs(r.gateway_addr.in6.s6_addr16[0])); } - auto res = ndp_resolve(to_resolve, rt.nif); - - if (res.has_error()) [[unlikely]] - { - return unexpected{-ENETUNREACH}; - } + rt.dst_hw = ndp_resolve(to_resolve, rt.nif); - rt.dst_hw = res.value(); + if (IS_ERR(rt.dst_hw)) [[unlikely]] + return unexpected{PTR_ERR(rt.dst_hw)}; return rt; } @@ -516,6 +520,16 @@ int inet_socket::setsockopt_inet6(int level, int opt, const void *optval, sockle this->ttl = ttl; return 0; } + + case IPV6_V6ONLY: { + auto ex = get_socket_option(optval, len); + + if (ex.has_error()) + return ex.error(); + + ipv6_only = ex.value() ? 1 : 0; + return 0; + } } return -ENOPROTOOPT; @@ -528,7 +542,27 @@ int inet_socket::getsockopt_inet6(int level, int opt, void *optval, socklen_t *l case IPV6_MULTICAST_HOPS: case IPV6_UNICAST_HOPS: return put_option(ttl, optval, len); + case IPV6_V6ONLY: { + int val = ipv6_only; + return put_option(val, optval, len); + } } return -ENOPROTOOPT; } + +int ip6_finish_output(struct neighbour *neigh, struct packetbuf *pbf, struct netif *nif) +{ + int st = 0; + const void *hwaddr = nullptr; + + if (neigh->flags & NUD_FAILED) + return -EHOSTUNREACH; + hwaddr = neigh->hwaddr; + + if ((st = nif->dll_ops->setup_header(pbf, tx_type::unicast, tx_protocol::ipv6, nif, hwaddr)) < + 0) [[unlikely]] + return st; + + return netif_send_packet(nif, pbf); +} diff --git a/kernel/kernel/net/ipv6/ndp.cpp b/kernel/kernel/net/ipv6/ndp.cpp index 058e5827a..f19712735 100644 --- a/kernel/kernel/net/ipv6/ndp.cpp +++ b/kernel/kernel/net/ipv6/ndp.cpp @@ -1,7 +1,9 @@ /* - * Copyright (c) 2020 Pedro Falcato + * Copyright (c) 2020 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only */ #include #include @@ -12,6 +14,7 @@ #include #include +#include #include #include #include @@ -36,12 +39,17 @@ struct icmp6_source_link_layer_opt unsigned char hwaddr[]; }; -/* TODO: Maybe the neighbour_table could replace some code below, if we add a few virtual functions - */ - /* FIXME: The ndp table should be per interface */ static neighbour_table ndp_table{AF_INET6}; + +int ndp_submit_request(struct neighbour *neigh, struct netif *netif); + +const struct neigh_ops ndp_ops = { + .resolve = ndp_submit_request, + .output = ip6_finish_output, +}; + static constexpr hrtime_t ndp_response_timeout = 250 * NS_PER_MS; /* 20 minutes in milis */ @@ -49,17 +57,19 @@ static constexpr unsigned long ndp_validity_time_ms = 1200000; int ndp_handle_na(netif *netif, packetbuf *buf) { + (void) ndp_response_timeout; + (void) ndp_validity_time_ms; if (buf->length() < sizeof(nd_neighbor_advert)) return -EINVAL; auto ndp = (struct nd_neighbor_advert *) buf->data; - + int added; neigh_proto_addr addr; addr.in6addr = ndp->nd_na_target; - auto [ptr, __created] = ndp_table.add(addr, true); - if (!ptr) - return 0; + struct neighbour *neigh = neigh_add(&ndp_table, &addr, GFP_ATOMIC, &ndp_ops, &added); + if (!neigh) + return -ENOMEM; const char *optptr = (const char *) (ndp + 1); ssize_t options_len = buf->length() - sizeof(nd_neighbor_solicit); @@ -91,15 +101,7 @@ int ndp_handle_na(netif *netif, packetbuf *buf) if (!target) return 0; - unsigned char *mac = new unsigned char[ETH_ALEN]; - if (!mac) - return 0; - memcpy(mac, target, ETH_ALEN); - - cul::slice sl{mac, ETH_ALEN}; - ptr->set_hwaddr(sl); - ptr->flags |= NEIGHBOUR_FLAG_HAS_RESPONSE; - + neigh_complete_lookup(neigh, target, ETH_ALEN); return 0; } @@ -183,24 +185,13 @@ in6_addr solicited_node_address(const in6_addr &our_address) return ret; } -int ndp_submit_request(shared_ptr &ptr, const in6_addr &target_addr, struct netif *netif) +int ndp_submit_request(struct neighbour *neigh, struct netif *netif) { + const in6_addr &target_addr = neigh->proto_addr.in6addr; if (target_addr == in6addr_loopback || netif->flags & NETIF_LOOPBACK) { - auto _ptr = new unsigned char[ETH_ALEN]; - - if (_ptr) - { - memset(_ptr, 0, ETH_ALEN); - auto sl = cul::slice{_ptr, ETH_ALEN}; - ptr->set_hwaddr(sl); - ptr->flags |= NEIGHBOUR_FLAG_HAS_RESPONSE; - } - else - { - return -ENOMEM; - } - + const unsigned char loopback_eth[ETH_ALEN] = {}; + __neigh_complete_lookup(neigh, loopback_eth, ETH_ALEN); return 0; } @@ -229,48 +220,18 @@ int ndp_submit_request(shared_ptr &ptr, const in6_addr &target_addr, sizeof(buf_) - sizeof(icmp6_hdr)}); } -expected, int> ndp_resolve(const in6_addr &ip, struct netif *netif) +struct neighbour *ndp_resolve(const in6_addr &ip, struct netif *netif) { - neigh_proto_addr addr; + struct neighbour *neigh; + int added; + union neigh_proto_addr addr; addr.in6addr = ip; - auto [ptr, created] = ndp_table.add(addr); - if (!ptr) - return unexpected{-ENOMEM}; - - if (ptr->flags & NEIGHBOUR_FLAG_UNINITIALISED) - { - if (created) - { - if (ndp_submit_request(ptr, ip, netif) < 0) - { - ndp_table.remove(ptr.get_data()); - return unexpected{-ENOMEM}; - } - } - - auto t0 = clocksource_get_time(); - - /* TODO: Add a wait_for_bit that can let us wait for random things - * without taking up permanent space in the structure - */ - while (!(ptr->flags & NEIGHBOUR_FLAG_HAS_RESPONSE) && - clocksource_get_time() - t0 <= ndp_response_timeout) - sched_sleep_ms(15); - - if (!(ptr->flags & NEIGHBOUR_FLAG_HAS_RESPONSE)) - { - if (created) - ndp_table.remove(ptr.get_data()); - return unexpected{-ENETUNREACH}; - } - - if (created) - { - ptr->set_validity(ndp_validity_time_ms); - ptr->set_initialised(); - } - } + neigh = neigh_add(&ndp_table, &addr, GFP_ATOMIC, &ndp_ops, &added); + if (!neigh) + return (struct neighbour *) ERR_PTR(-ENOMEM); - return ptr; + if (neigh_needs_resolve(neigh)) + neigh_start_resolve(neigh, netif); + return neigh; } diff --git a/kernel/kernel/net/neighbour.cpp b/kernel/kernel/net/neighbour.cpp index 11e6f1b41..8887139cf 100644 --- a/kernel/kernel/net/neighbour.cpp +++ b/kernel/kernel/net/neighbour.cpp @@ -1,86 +1,187 @@ /* - * Copyright (c) 2020 Pedro Falcato + * Copyright (c) 2020 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only */ +#include #include +#include +#include #include +#include #include -#include -#include - void neighbour_revalidate(clockevent* ev) { // TODO: Implement ev->deadline = clocksource_get_time() + NS_PER_SEC * 60 * 20; } -cul::pair, bool> neighbour_table::add(const neigh_proto_addr& addr, - bool only_lookup) +struct neighbour* neigh_find(struct neighbour_table* table, const union neigh_proto_addr* addr) { - scoped_lock g{lock}; + struct neighbour* neigh; + u32 hash = hash_protoaddr(*addr, table->domain); + u32 index = hash & (NEIGH_TAB_NR_CHAINS - 1); - auto hash = hash_protoaddr(addr, domain); + rcu_read_lock(); - auto it = neighbour_cache.get_hash_list_begin(hash); - auto end = neighbour_cache.get_hash_list_end(hash); + list_for_each_entry_rcu (neigh, &table->neigh_tab[index], list_node) + { + if (neigh->addr_equals(*addr)) + { + if (neigh_get_careful(neigh)) + goto out; + } + } - while (it != end) + neigh = NULL; +out: + rcu_read_unlock(); + return neigh; +} + +void neigh_free(struct neighbour* neigh) +{ + kfree_rcu(neigh, rcu_head); +} + +struct neighbour* neigh_add(struct neighbour_table* table, const union neigh_proto_addr* addr, + gfp_t gfp, const struct neigh_ops* ops, int* added) +{ + struct neighbour *neigh, *n2; + u32 hash, index; + *added = 1; + + neigh = neigh_find(table, addr); + if (neigh) { - auto neigh = *it; - auto n_hwaddr = neigh->hwaddr(); - if (neigh->addr_equals(addr)) - return {neigh, false}; - it++; + *added = 0; + return neigh; } - if (only_lookup) - return {nullptr, false}; + neigh = (struct neighbour*) kmalloc(sizeof(*neigh), gfp); + if (!neigh) + return NULL; - auto ptr = make_shared(domain, addr); + hash = hash_protoaddr(*addr, table->domain); + index = hash & (NEIGH_TAB_NR_CHAINS - 1); - if (!ptr) - return {nullptr, false}; + new (neigh) neighbour(table->domain, *addr); + neigh->neigh_ops = ops; + spin_lock(&table->lock); - ptr->flags |= NEIGHBOUR_FLAG_UNINITIALISED; + /* No need for _rcu since we hold the lock */ + list_for_each_entry (n2, &table->neigh_tab[index], list_node) + { + if (n2->addr_equals(*addr)) + { + /* We can skip neigh_get_careful here since we hold the spinlock */ + neigh_get(n2); + spin_unlock(&table->lock); + kfree(neigh); + *added = 0; + return n2; + } + } - if (!neighbour_cache.add_element(ptr)) - return {nullptr, false}; + /* Not found, add */ + neigh_get(neigh); + list_add_tail_rcu(&neigh->list_node, &table->neigh_tab[index]); - return {ptr, true}; + spin_unlock(&table->lock); + return neigh; } -void neighbour_table::remove(neighbour* ptr) +void neigh_remove(struct neighbour_table* table, struct neighbour* neigh) { - scoped_lock g{lock}; - auto& hw = ptr->proto_addr; - auto hash = hash_protoaddr(hw, domain); - - auto it = neighbour_cache.get_hash_list_begin(hash); - auto end = neighbour_cache.get_hash_list_end(hash); + spin_lock(&table->lock); + list_remove_rcu(&neigh->list_node); + spin_unlock(&table->lock); + neigh_put(neigh); +} - while (it != end) +static void neigh_clear_chain(struct neighbour_table* table, u32 i) +{ + struct neighbour* n; + list_for_each_entry (n, &table->neigh_tab[i], list_node) { - auto neigh = *it; - if (neigh == ptr) - { - neighbour_cache.remove_element(neigh, hash, it); - return; - } + list_remove(&n->list_node); + neigh_put(n); + } +} + +void neigh_clear(struct neighbour_table* table) +{ + spin_lock(&table->lock); + + for (u32 i = 0; i < NEIGH_TAB_NR_CHAINS; i++) + neigh_clear_chain(table, i); + + spin_unlock(&table->lock); +} + +void neigh_start_resolve(struct neighbour* neigh, struct netif* nif) +{ + write_seqlock(&neigh->neigh_seqlock); + + if (neigh_needs_resolve(neigh)) + neigh->neigh_ops->resolve(neigh, nif); - it++; + if (neigh->flags & NUD_STALE) + { + neigh->flags &= ~NUD_STALE; + neigh->flags |= NUD_PROBE; + } + else + { + neigh->flags |= NUD_INCOMPLETE; } + + write_sequnlock(&neigh->neigh_seqlock); } -void neighbour_table::clear_cache() +int neigh_output(struct neighbour* neigh, struct packetbuf* pbf, struct netif* nif) { - scoped_lock g{lock}; + CHECK(pbf->route.nif); + unsigned int flags = READ_ONCE(neigh->flags); + if (likely(flags & NUD_REACHABLE)) + return neigh->neigh_ops->output(neigh, pbf, nif); + /* Slow path - check the neighbour's state, try to resolve it and queue our own packet. Per + * RFC1122: The link layer SHOULD save (rather than discard) at least one (the latest) + * packet of each set of packets destined to the same unresolved IP address, and transmit + * the saved packet when the address has been resolved. + */ + + if (flags & NUD_REACHABLE) + { + /* Just send it */ + return neigh->neigh_ops->output(neigh, pbf, nif); + } + + /* Probe pending (or will be). Append our packet and leave. This requires the lock. */ + spin_lock(&neigh->neigh_seqlock.lock); + list_add_tail(&pbf->list_node, &neigh->packet_queue); + pbf_get(pbf); + spin_unlock(&neigh->neigh_seqlock.lock); + + if (flags & (NUD_PROBE | NUD_INCOMPLETE)) + return 0; - neighbour_cache.empty(); + /* Not reachable nor probe nor incomplete - we don't have a probe. Start a resolve. */ + neigh_start_resolve(neigh, nif); + return 0; } -fnv_hash_t hash_neighbour(shared_ptr& neigh) +void neigh_output_queued(struct neighbour* neigh) { - return hash_protoaddr(neigh->proto_addr, neigh->get_domain()); + struct packetbuf *pbf, *next; + list_for_each_entry_safe (pbf, next, &neigh->packet_queue, list_node) + { + CHECK(pbf->route.nif != NULL); + list_remove(&pbf->list_node); + neigh->neigh_ops->output(neigh, pbf, pbf->route.nif); + pbf_put_ref(pbf); + } } diff --git a/kernel/kernel/net/netif.cpp b/kernel/kernel/net/netif.cpp index 7d9c4b929..68faa8260 100644 --- a/kernel/kernel/net/netif.cpp +++ b/kernel/kernel/net/netif.cpp @@ -13,10 +13,9 @@ #include #include #include +#include #include #include -#include -#include #include #include #include @@ -111,7 +110,8 @@ void netif_register_loopback_route6(struct netif *netif) { struct inet6_route route; - route.mask = IN6ADDR_LOOPBACK_INIT; + route.mask.s6_addr32[0] = route.mask.s6_addr32[1] = route.mask.s6_addr32[2] = + route.mask.s6_addr32[3] = ~0; route.dest = IN6ADDR_LOOPBACK_INIT; route.gateway = IN6ADDR_ANY_INIT; @@ -126,10 +126,6 @@ void netif_register_if(struct netif *netif) { INIT_LIST_HEAD(&netif->inet6_addr_list); - assert(udp_init_netif(netif) == 0); - - assert(tcp_init_netif(netif) == 0); - auto ex = dev_register_chardevs(0, 1, 0, &netif_fops, netif->name); if (ex.has_error()) panic("netif_register_if failed"); @@ -281,79 +277,52 @@ INIT_LEVEL_CORE_PERCPU_CTOR(init_rx_queues); void netif_signal_rx(netif *nif) { - unsigned int flags, og_flags; - - do - { - flags = nif->flags; - og_flags = flags; - - flags |= NETIF_HAS_RX_AVAILABLE; - - if (og_flags & NETIF_DOING_RX_POLL) - flags |= NETIF_MISSED_RX; - - } while (!__atomic_compare_exchange_n(&nif->flags, &og_flags, flags, false, __ATOMIC_ACQUIRE, - __ATOMIC_RELAXED)); - - if (og_flags & NETIF_HAS_RX_AVAILABLE) + unsigned int flags = + __atomic_fetch_or(&nif->flags, NETIF_HAS_RX_AVAILABLE | NETIF_SCHEDULED, __ATOMIC_SEQ_CST); + if (flags & NETIF_SCHEDULED) return; auto queue = get_per_cpu_ptr(rx_queue); unsigned long cpu_flags = spin_lock_irqsave(&queue->lock); - list_add_tail(&nif->rx_queue_node, &queue->to_rx_list); - spin_unlock_irqrestore(&queue->lock, cpu_flags); - softirq_raise(softirq_vector::SOFTIRQ_VECTOR_NETRX); } void netif_do_rxpoll(netif *nif) { - __atomic_or_fetch(&nif->flags, NETIF_DOING_RX_POLL, __ATOMIC_RELAXED); + unsigned int desired; + atomic_or_relaxed(nif->flags, NETIF_DOING_RX_POLL); - while (true) + do { + retry: + atomic_and_relaxed(nif->flags, ~NETIF_HAS_RX_AVAILABLE); nif->poll_rx(nif); - - unsigned int flags, og_flags; - - do - { - og_flags = flags = nif->flags; - - if (!(og_flags & NETIF_MISSED_RX)) - { - nif->rx_end(nif); - flags &= ~(NETIF_HAS_RX_AVAILABLE | NETIF_DOING_RX_POLL); - } - - flags &= ~NETIF_MISSED_RX; - - } while (!__atomic_compare_exchange_n(&nif->flags, &og_flags, flags, false, - __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)); - - if (!(flags & NETIF_DOING_RX_POLL)) - break; - } + desired = READ_ONCE(nif->flags); + if (desired & NETIF_HAS_RX_AVAILABLE) + goto retry; + nif->rx_end(nif); + } while (cmpxchg(&nif->flags, desired, desired & ~NETIF_DOING_RX_POLL) != desired); } int netif_do_rx() { auto queue = get_per_cpu_ptr(rx_queue); - scoped_lock g{queue->lock}; - - list_for_every (&queue->to_rx_list) + unsigned long flags = spin_lock_irqsave(&queue->lock); + while (!list_is_empty(&queue->to_rx_list)) { - netif *n = container_of(l, netif, rx_queue_node); - - netif_do_rxpoll(n); + struct netif *nif = list_first_entry(&queue->to_rx_list, struct netif, rx_queue_node); + list_remove(&nif->rx_queue_node); + spin_unlock_irqrestore(&queue->lock, flags); + netif_do_rxpoll(nif); + flags = spin_lock_irqsave(&queue->lock); + atomic_and_relaxed(nif->flags, ~NETIF_SCHEDULED); } - list_reset(&queue->to_rx_list); + spin_unlock_irqrestore(&queue->lock, flags); return 0; } diff --git a/kernel/kernel/net/socket.cpp b/kernel/kernel/net/socket.cpp index b659b57cb..0ea909d02 100644 --- a/kernel/kernel/net/socket.cpp +++ b/kernel/kernel/net/socket.cpp @@ -340,20 +340,6 @@ unsigned int socket_ioctl(int request, void *argp, struct file *file) { switch (request) { - case FIONBIO: { - int on; - - if (copy_from_user(&on, argp, sizeof(on)) < 0) - return -EFAULT; - - if (on) - file->f_flags |= O_NONBLOCK; - else - file->f_flags &= ~O_NONBLOCK; - - return 0; - } - #ifdef CONFIG_NET case SIOCGIFNAME: { return do_siocgifname((struct ifreq *) argp); diff --git a/kernel/kernel/net/unix.cpp b/kernel/kernel/net/unix.cpp index 9669e4a4c..977e94031 100644 --- a/kernel/kernel/net/unix.cpp +++ b/kernel/kernel/net/unix.cpp @@ -1204,6 +1204,7 @@ ssize_t un_socket::recvmsg(struct msghdr *msg, int flags) int un_get_name(sockaddr_un *addr, socklen_t *addrlen, const un_name &name) { + char pathbuf[PATH_MAX]; addr->sun_family = AF_UNIX; if (name.is_anon()) @@ -1213,15 +1214,19 @@ int un_get_name(sockaddr_un *addr, socklen_t *addrlen, const un_name &name) } else if (name.is_fs_sock_) { - auto filename = dentry_to_file_name(name.dentry_); - if (!filename) - return -errno; - size_t copied = strlcpy(addr->sun_path, filename, sizeof(addr->sun_path)); + /* TODO: Fix this path garbage. It will work for now, but unix sockets need to keep struct + * path's, not dentries. + **/ + struct path path; + path_init(&path); + path.dentry = name.dentry_; + char *p = d_path(&path, pathbuf, PATH_MAX); + if (IS_ERR(p)) + return PTR_ERR(p); + size_t copied = strlcpy(addr->sun_path, p, sizeof(addr->sun_path)); auto len = cul::clamp(copied, sizeof(addr->sun_path)); - *addrlen = sizeof(sa_family_t) + len; - free(filename); } else { diff --git a/kernel/kernel/process.cpp b/kernel/kernel/process.cpp index e8e31d5d7..dfba72cf5 100644 --- a/kernel/kernel/process.cpp +++ b/kernel/kernel/process.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -172,10 +173,11 @@ process *process_create(const std::string_view &cmd_line, ioctx *ctx, process *p assert(process_ids != nullptr); } - auto p = make_unique(); + unique_ptr p{(struct process *) kmalloc(sizeof(struct process), GFP_KERNEL)}; if (!p) return errno = ENOMEM, nullptr; + new (p.get()) process; auto proc = p.get(); /* TODO: idm_get_id doesn't wrap? POSIX COMPLIANCE */ @@ -206,6 +208,7 @@ process *process_create(const std::string_view &cmd_line, ioctx *ctx, process *p if (copy_file_descriptors(proc, ctx) < 0) return nullptr; + proc->ctx.umask = READ_ONCE(ctx->umask); } else { @@ -326,8 +329,6 @@ void process_remove_from_list(process *process); template static void for_every_child(process *proc, Callable cb) { - scoped_lock g{proc->children_lock}; - for (process *p = proc->children; p != nullptr; p = p->next_sibbling) { if (cb(p) == false) @@ -595,9 +596,12 @@ pid_t sys_wait4(pid_t pid, int *wstatus, int options, rusage *usage) return -EINVAL; wait_info w{pid, (unsigned int) options}; + spin_lock(¤t->children_lock); - int st = - wait_for_event_interruptible(¤t->wait_child_event, wait_handle_processes(current, w)); + int st = wait_for_event_locked_interruptible( + ¤t->wait_child_event, wait_handle_processes(current, w), ¤t->children_lock); + + spin_unlock(¤t->children_lock); #if 0 printk("st %d w.status %d\n", st, w.status); @@ -968,12 +972,13 @@ void process_kill_other_threads(void) { current->remove_thread(current_thread); current_thread->owner = nullptr; - scoped_lock g{current->signal_lock}; + spin_lock(¤t->parent->children_lock); current->exit_code = exit_code; /* Finally, wake up any possible concerned parents */ wait_queue_wake_all(¤t->parent->wait_child_event); current->signal_group_flags |= SIGNAL_GROUP_EXIT; + spin_unlock(¤t->parent->children_lock); } kernel_raise_signal(SIGCHLD, parent, 0, &info); @@ -1264,18 +1269,21 @@ ssize_t process::query(void *ubuf, ssize_t len, unsigned long what, size_t *howm ssize_t process::query_vm_regions(void *ubuf, ssize_t len, unsigned long what, size_t *howmany, void *arg) { - scoped_mutex g{address_space->vm_lock}; + scoped_rwlock g{address_space->vm_lock}; size_t needed_len = 0; + char pathbuf[PATH_MAX]; vm_for_every_region(*address_space, [&](vm_area_struct *region) -> bool { needed_len += sizeof(onx_process_vm_region); if (is_file_backed(region)) { - auto path = dentry_to_file_name(region->vm_file->f_dentry); - - needed_len += strlen(path) + 1; - free(path); + char *path = d_path(®ion->vm_file->f_path, pathbuf, PATH_MAX); + /* Path too long? Ignore */ + if (IS_ERR(path)) + needed_len = 1; + else + needed_len = pathbuf + PATH_MAX - path; } if (needed_len % alignof(onx_process_vm_region)) @@ -1332,9 +1340,20 @@ ssize_t process::query_vm_regions(void *ubuf, ssize_t len, unsigned long what, s if (is_file_backed(region)) { - auto path = dentry_to_file_name(region->vm_file->f_dentry); - strcpy(reg->name, path); - reg->size += strlen(path) + 1; + char *path = d_path(®ion->vm_file->f_path, pathbuf, PATH_MAX); + /* Path too long? Ignore */ + if (IS_ERR(path)) + reg->name[0] = '\0'; + else + { + /* FIXME: This is a vulnerability. d_path can change between the initial scan and + * writing. Please fix this!! + **/ + strcpy(reg->name, path); + reg->size += strlen(path); + } + + reg->size++; } if (reg->size % alignof(onx_process_vm_region)) diff --git a/kernel/kernel/radix.cpp b/kernel/kernel/radix.cpp index 0fce3f054..891c3ae25 100644 --- a/kernel/kernel/radix.cpp +++ b/kernel/kernel/radix.cpp @@ -51,7 +51,7 @@ static slab_cache *node_cache; __init static void radix_init_slab() { node_cache = kmem_cache_create("radix_tree_node", sizeof(radix_tree_node), 0, - KMEM_CACHE_HWALIGN, nullptr); + KMEM_CACHE_HWALIGN | KMEM_CACHE_VMALLOC, nullptr); CHECK(node_cache != nullptr); } diff --git a/kernel/kernel/sched/scheduler.cpp b/kernel/kernel/sched/scheduler.cpp index 23c3739a8..a9d08f97a 100644 --- a/kernel/kernel/sched/scheduler.cpp +++ b/kernel/kernel/sched/scheduler.cpp @@ -212,6 +212,48 @@ void sched_unlock(thread *thread, unsigned long cpu_flags) PER_CPU_VAR(long runnable_delta) = 0; +extern void sched_idle(void *); + +static thread_t *sched_steal_job(unsigned int cpu) +{ + for (unsigned int i = 0; i < get_nr_cpus(); i++) + { + if (i == cpu) + continue; + if (other_cpu_get(tasks_in_queues, i) <= 1) + continue; + struct spinlock *sched_lock = get_per_cpu_ptr_any(scheduler_lock, i); + thread **thread_queues = (thread **) get_per_cpu_ptr_any(thread_queues_head, i); + if (spin_try_lock(sched_lock)) + continue; + + for (int j = NUM_PRIO - 1; j >= 0; j--) + { + /* If this queue has a thread, we found a runnable thread! */ + if (thread_queues[j]) + { + thread_t *ret = thread_queues[j]; + if (ret->entry == sched_idle) + continue; + /* Advance the queue by one */ + thread_queues[j] = ret->next_prio; + if (thread_queues[j]) + ret->prev_prio = nullptr; + ret->next_prio = nullptr; + other_cpu_add(tasks_in_queues, -1, i); + add_per_cpu(tasks_in_queues, 1); + ret->cpu = cpu; + spin_unlock(sched_lock); + return ret; + } + } + + spin_unlock(sched_lock); + } + + return nullptr; +} + thread_t *__sched_find_next(unsigned int cpu) { thread_t *current_thread = get_current_thread(); @@ -252,6 +294,13 @@ thread_t *__sched_find_next(unsigned int cpu) { thread_t *ret = thread_queues[i]; + if (ret->entry == sched_idle) + { + thread_t *stolen = sched_steal_job(cpu); + if (stolen) + return stolen; + } + /* Advance the queue by one */ thread_queues[i] = ret->next_prio; if (thread_queues[i]) diff --git a/kernel/kernel/signal.cpp b/kernel/kernel/signal.cpp index 3c92995f4..c6840b44d 100644 --- a/kernel/kernel/signal.cpp +++ b/kernel/kernel/signal.cpp @@ -725,11 +725,9 @@ int signal_send_all(int signal, int flags, siginfo_t *info) pid::auto_pid process_get_pgrp(process *p) { scoped_lock g{p->pgrp_lock}; - auto pg = p->process_group; - + CHECK(pg.get() != nullptr); pg->ref(); - return pg; } diff --git a/kernel/kernel/smp.cpp b/kernel/kernel/smp.cpp index 1464970aa..ba9bdacbf 100644 --- a/kernel/kernel/smp.cpp +++ b/kernel/kernel/smp.cpp @@ -99,13 +99,15 @@ void sync_call_cntrlblk::complete(unsigned int cpu) #ifdef DEBUG_SMP_SYNC_CALL mask.remove_cpu_atomic(cpu); #endif - waiting_for_completion--; - if (flags & SYNC_CALL_NOWAIT && waiting_for_completion.load(mem_order::acquire) == 0) + if (flags & SYNC_CALL_NOWAIT && waiting_for_completion.sub_fetch(1, mem_order::seq_cst) == 0) { // Free the control block, no one is waiting for us ctlblk_pool.free(this); + return; } + + waiting_for_completion--; } void sync_call_cntrlblk::wait(sync_call_func local, void *context) diff --git a/kernel/kernel/tty/tty.cpp b/kernel/kernel/tty/tty.cpp index 6ee531e72..044d9e92b 100644 --- a/kernel/kernel/tty/tty.cpp +++ b/kernel/kernel/tty/tty.cpp @@ -568,6 +568,21 @@ unsigned int do_tty_cnotty(tty *tty) return 0; } +static unsigned int do_tiocspgrp(struct tty *tty, pid_t pid) +{ + struct process *current = get_current_process(); + if (current->ctty != tty || tty->session != current->session) + return -ENOTTY; + + pid::auto_pid p = pid::lookup(pid); + if (!p || !p->is(PIDTYPE_PGRP)) + return -ESRCH; + if (!p->is_in_session(tty->session)) + return -EPERM; + tty->foreground_pgrp = pid; + return 0; +} + unsigned int tty_ioctl(int request, void *argp, struct file *dev) { struct tty *slave_tty; @@ -671,9 +686,10 @@ unsigned int tty_ioctl(int request, void *argp, struct file *dev) case TIOCSPGRP: { scoped_mutex g{tty->lock}; - auto pgrp = get_current_process()->process_group; - tty->foreground_pgrp = pgrp->get_pid(); - return 0; + pid_t pid; + if (copy_from_user(&pid, argp, sizeof(int))) + return -EFAULT; + return do_tiocspgrp(tty, pid); } case TIOCGPGRP: { diff --git a/kernel/kernel/tty/vt/vterm.cpp b/kernel/kernel/tty/vt/vterm.cpp index 7abd7359c..266cd6498 100644 --- a/kernel/kernel/tty/vt/vterm.cpp +++ b/kernel/kernel/tty/vt/vterm.cpp @@ -41,24 +41,41 @@ struct tty; struct color { - uint8_t r; - uint8_t g; - uint8_t b; - uint8_t a; + union { + struct + { + uint8_t r; + uint8_t g; + uint8_t b; + uint8_t a; + }; + u32 rgba; + }; }; -static struct color default_fg = {.r = 0xaa, .g = 0xaa, .b = 0xaa}; +static struct color default_fg = {204, 204, 204}; static struct color default_bg = {}; const struct color color_table[] = { - {.r = 0, .g = 0, .b = 0}, /* Black */ - {.r = 0xff}, /* Red */ - {.g = 0xff}, /* Green */ - {.r = 0xff, .g = 0xff}, /* Yellow */ - {.b = 0xff}, /* Blue */ - {.r = 0xff, .b = 0xff}, /* Magenta */ - {.g = 0xff, .b = 0xff}, /* Cyan */ - {.r = 0xaa, .g = 0xaa, .b = 0xaa} /* White */ + {.r = 0, .g = 0, .b = 0}, /* Black */ + {.r = 205, .g = 49, .b = 49}, /* Red */ + {.r = 19, .g = 161, .b = 14}, /* Green */ + {229, 229, 16}, /* Yellow */ + {36, 114, 200}, /* Blue */ + {188, 63, 188}, /* Magenta */ + {17, 168, 205}, /* Cyan */ + {204, 204, 204} /* White */ +}; + +const struct color bright_color_table[] = { + {102, 102, 102}, /* Black */ + {241, 76, 76}, /* Red */ + {22, 198, 12}, /* Green */ + {245, 245, 67}, /* Yellow */ + {59, 142, 234}, /* Blue */ + {214, 112, 214}, /* Magenta */ + {41, 184, 219}, /* Cyan */ + {242, 242, 242}, /* White */ }; #define VTERM_CONSOLE_CELL_DIRTY (1 << 0) @@ -66,25 +83,25 @@ const struct color color_table[] = { struct console_cell { - uint32_t codepoint; + uint32_t codepoint : 31; + u32 dirty : 1; struct color bg; struct color fg; - uint32_t flags; }; static inline void vterm_set_dirty(struct console_cell *c) { - c->flags |= VTERM_CONSOLE_CELL_DIRTY; + c->dirty = VTERM_CONSOLE_CELL_DIRTY; } static inline void vterm_clear_dirty(struct console_cell *c) { - c->flags &= ~VTERM_CONSOLE_CELL_DIRTY; + c->dirty &= ~VTERM_CONSOLE_CELL_DIRTY; } static inline bool vterm_is_dirty(struct console_cell *c) { - return c->flags & VTERM_CONSOLE_CELL_DIRTY; + return c->dirty; } #define VTERM_MESSAGE_FLUSH 1 @@ -98,6 +115,70 @@ struct vterm_message struct vterm_message *next; }; +enum Gx_type +{ + Gx_LATIN1, + Gx_GRAPH, +}; + +/* Translations from Linux */ +static const unsigned short translations[][256] = { + /* 8-bit Latin-1 mapped to Unicode -- trivial mapping */ + [Gx_LATIN1] = {0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, + 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, + 0x001e, 0x001f, 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, 0x0030, 0x0031, + 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, + 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, + 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, + 0x006e, 0x006f, 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, 0x0080, 0x0081, + 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, + 0x008c, 0x008d, 0x008e, 0x008f, 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, + 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, + 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, 0x00b0, 0x00b1, 0x00b2, 0x00b3, + 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, + 0x00be, 0x00bf, 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, 0x00d0, 0x00d1, + 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, + 0x00dc, 0x00dd, 0x00de, 0x00df, 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, + 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, + 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff}, + /* VT100 graphics mapped to Unicode */ + [Gx_GRAPH] = {0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, + 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, + 0x001e, 0x001f, 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x2192, 0x2190, 0x2191, 0x2193, 0x002f, 0x2588, 0x0031, + 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, + 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, + 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x00a0, 0x25c6, 0x2592, 0x2409, 0x240c, + 0x240d, 0x240a, 0x00b0, 0x00b1, 0x2591, 0x240b, 0x2518, 0x2510, 0x250c, 0x2514, + 0x253c, 0x23ba, 0x23bb, 0x2500, 0x23bc, 0x23bd, 0x251c, 0x2524, 0x2534, 0x252c, + 0x2502, 0x2264, 0x2265, 0x03c0, 0x2260, 0x00a3, 0x00b7, 0x007f, 0x0080, 0x0081, + 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, + 0x008c, 0x008d, 0x008e, 0x008f, 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, + 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, + 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, 0x00b0, 0x00b1, 0x00b2, 0x00b3, + 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, + 0x00be, 0x00bf, 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, 0x00d0, 0x00d1, + 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, + 0x00dc, 0x00dd, 0x00de, 0x00df, 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, + 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, + 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff}, +}; + #define MAX_ARGS 4 struct vterm { @@ -120,6 +201,14 @@ struct vterm struct cond condvar; struct mutex condvar_mutex; struct vterm_message *msgs; + bool reversed; + bool flush_all; + bool numlck; + unsigned long *dirty_row_bitmap; + unsigned int bitmap_size; + enum Gx_type gx[2]; + u8 charset; + bool bold; // Buffer used for any multibyte buffering for utf8 char multibyte_buffer[10]; @@ -157,6 +246,7 @@ struct vterm void do_ri(); void do_nl(); void do_cr(); + void delete_lines(unsigned long nr); }; void vterm_append_msg(struct vterm *vterm, struct vterm_message *msg) ACQUIRE(vterm->condvar_mutex) @@ -191,6 +281,16 @@ void vterm_send_message(struct vterm *vterm, unsigned long message, void *ctx) mutex_unlock(&vterm->condvar_mutex); } +#define LONG_SIZE_BITS __LONG_WIDTH__ + +static inline void vterm_dirty_cell(unsigned int x, unsigned int y, struct vterm *vt) +{ + struct console_cell *cell = &vt->cells[y * vt->columns + x]; + CHECK(y < vt->rows); + vterm_set_dirty(cell); + vt->dirty_row_bitmap[y / LONG_SIZE_BITS] |= (1UL << (y % LONG_SIZE_BITS)); +} + static inline uint32_t unpack_rgba(struct color color, struct framebuffer *fb) { uint32_t c = ((color.r << fb->color.red_shift) & fb->color.red_mask) | @@ -205,6 +305,7 @@ static void draw_char(uint32_t c, unsigned int x, unsigned int y, struct framebu { struct font *font = get_font_data(); + c = font->utf2char(c); if (c >= font->chars) c = '?'; @@ -270,21 +371,77 @@ void vterm_flush_all(struct vterm *vterm) do_vterm_flush_all(vterm); } +static inline bool same_colour(const struct color *c1, const struct color *c2) +{ + return c1->rgba == c2->rgba; +} + +static void vterm_clear_range(struct vterm *vt, unsigned int start_x, unsigned int start_y, + unsigned int end_x, unsigned int end_y) +{ + unsigned int x = start_x, y = start_y; + unsigned int len; + + if (start_y == end_y) + len = end_x - start_x; + else + len = (vt->columns - start_x) + end_x + (end_y - start_y - 1) * vt->columns; + + struct console_cell *cell = vt->cells + (y * vt->columns) + x; + + for (unsigned int i = 0; i < len; i++) + { + CHECK(cell < vt->cells + (vt->columns * vt->rows)); + if (cell->codepoint != ' ' || !same_colour(&cell->bg, &vt->bg) || + !same_colour(&cell->fg, &vt->fg)) + { + cell->codepoint = ' '; + cell->bg = vt->bg; + cell->fg = vt->fg; + vterm_dirty_cell(x, y, vt); + } + + cell++; + if (unlikely(++x == vt->columns)) + { + x = 0; + y++; + } + } +} + static void __vterm_scroll(struct framebuffer *fb, struct vterm *vt, unsigned int nr_lines, unsigned int top, unsigned int bottom) { + if (top + nr_lines >= bottom) + nr_lines = (bottom - top) - 1; + unsigned int start = vt->columns * top; unsigned int dest = vt->columns * (top + nr_lines); unsigned int end = vt->columns * (bottom - nr_lines); - memcpy(vt->cells + start, vt->cells + dest, sizeof(struct console_cell) * (end - start)); + unsigned int x = 0, y = top; + + if (bottom > vt->rows || top >= bottom || nr_lines < 1) + return; - for (unsigned int i = 0; i < vt->columns * nr_lines; i++) + for (unsigned int i = 0; i < end - start; i++) { - struct console_cell *c = &vt->cells[(bottom - nr_lines) * vt->columns + i]; - c->codepoint = ' '; - c->bg = vt->bg; - c->fg = vt->fg; + struct console_cell *dst = vt->cells + start + i; + struct console_cell *src = vt->cells + dest + i; + if (dst->codepoint != src->codepoint || !same_colour(&dst->bg, &src->bg) || + !same_colour(&dst->fg, &src->fg)) + vterm_dirty_cell(x, y, vt); + dst->bg = src->bg; + dst->fg = src->fg; + dst->codepoint = src->codepoint; + if (++x == vt->columns) + { + x = 0; + y++; + } } + + vterm_clear_range(vt, 0, bottom - nr_lines, 0, bottom); } static void vterm_scroll(struct framebuffer *fb, struct vterm *vt) @@ -295,19 +452,35 @@ static void vterm_scroll(struct framebuffer *fb, struct vterm *vt) static void __vterm_scroll_down(struct framebuffer *fb, struct vterm *vt, unsigned int nr, unsigned int top, unsigned int bottom) { - unsigned int start = vt->columns * (top + nr); - unsigned int dest = vt->columns * top; - unsigned int end = vt->columns * (bottom - nr); - DCHECK(end > start); - memmove(vt->cells + start, vt->cells + dest, sizeof(struct console_cell) * (end - start)); + if (top + nr >= bottom) + nr = (bottom - top) - 1; + + unsigned int dest, end; + dest = (top + nr) * vt->columns; + end = bottom * vt->columns; + unsigned int x = vt->columns - 1, y = bottom - 1; - for (unsigned int i = 0; i < vt->columns * nr; i++) + if (bottom > vt->rows || top >= bottom || nr < 1) + return; + + for (unsigned int i = 0; i < end - dest; i++) { - struct console_cell *c = &vt->cells[dest + i]; - c->codepoint = ' '; - c->bg = vt->bg; - c->fg = vt->fg; + struct console_cell *src = vt->cells + ((bottom - nr) * vt->columns) - i - 1; + struct console_cell *dst = vt->cells + end - i - 1; + if (dst->codepoint != src->codepoint || !same_colour(&dst->bg, &src->bg) || + !same_colour(&dst->fg, &src->fg)) + vterm_dirty_cell(x, y, vt); + dst->bg = src->bg; + dst->fg = src->fg; + dst->codepoint = src->codepoint; + if (x-- == 0) + { + x = vt->columns - 1; + y--; + } } + + vterm_clear_range(vt, 0, top, 0, top + nr); } static void vterm_scroll_down(struct framebuffer *fb, struct vterm *vt) @@ -315,20 +488,22 @@ static void vterm_scroll_down(struct framebuffer *fb, struct vterm *vt) __vterm_scroll_down(fb, vt, 1, vt->top, vt->bottom); } +static inline bool vterm_needs_dirty(struct console_cell *cell, utf32_t c, struct color fg, + struct color bg) +{ + return c != cell->codepoint || !same_colour(&fg, &cell->fg) || !same_colour(&bg, &cell->bg); +} + void vterm_set_char(utf32_t c, unsigned int x, unsigned int y, struct color fg, struct color bg, struct vterm *vterm) { struct console_cell *cell = &vterm->cells[y * vterm->columns + x]; + vterm_dirty_cell(x, y, vterm); + if (c < 256) + c = translations[vterm->gx[vterm->charset]][c]; cell->codepoint = c; cell->fg = fg; cell->bg = bg; - vterm_set_dirty(cell); -} - -void vterm_dirty_cell(unsigned int x, unsigned int y, struct vterm *vt) -{ - struct console_cell *cell = &vt->cells[y * vt->columns + x]; - vterm_set_dirty(cell); } bool vterm_putc(utf32_t c, struct vterm *vt) @@ -341,9 +516,17 @@ bool vterm_putc(utf32_t c, struct vterm *vt) if (c == '\a') return false; - if (c == '\016' || c == '\017') + if (c == '\016') + { + /* Shift-out */ + vt->charset = 1; + return false; + } + + if (c == '\017') { - // Shift in and shift out do nothing right now + /* Shift-in */ + vt->charset = 0; return false; } @@ -483,17 +666,25 @@ void vterm_flush(struct vterm *vterm); void do_vterm_flush(struct vterm *vterm) { struct font *f = get_font_data(); - for (unsigned int i = 0; i < vterm->columns; i++) + int base_row = 0; + for (unsigned int i = 0; i < vterm->bitmap_size; i++, base_row += LONG_SIZE_BITS) { - for (unsigned int j = 0; j < vterm->rows; j++) + while (vterm->dirty_row_bitmap[i] != 0) { - struct console_cell *cell = &vterm->cells[j * vterm->columns + i]; + int bit = __builtin_ffsl(vterm->dirty_row_bitmap[i]) - 1; + int row = bit + base_row; + vterm->dirty_row_bitmap[i] &= ~(1UL << bit); - if (vterm_is_dirty(cell)) + for (unsigned int j = 0; j < vterm->columns; j++) { - draw_char(cell->codepoint, i * f->width, j * f->height, vterm->fb, cell->fg, - cell->bg); - vterm_clear_dirty(cell); + struct console_cell *cell = &vterm->cells[row * vterm->columns + j]; + + if (vterm_is_dirty(cell)) + { + draw_char(cell->codepoint, j * f->width, row * f->height, vterm->fb, cell->fg, + cell->bg); + vterm_clear_dirty(cell); + } } } } @@ -503,6 +694,13 @@ void platform_serial_write(const char *s, size_t size); void vterm_flush(struct vterm *vterm) { + if (vterm->flush_all) + { + do_vterm_flush_all(vterm); + vterm->flush_all = false; + return; + } + if (vterm->multithread_enabled) vterm_send_message(vterm, VTERM_MESSAGE_FLUSH, NULL); else @@ -516,10 +714,10 @@ void vterm_fill_screen(struct vterm *vterm, uint32_t character, struct color fg, for (unsigned int j = 0; j < vterm->rows; j++) { struct console_cell *cell = &vterm->cells[i * vterm->rows + j]; + vterm_dirty_cell(i, j, vterm); cell->codepoint = character; cell->bg = bg; cell->fg = fg; - vterm_set_dirty(cell); } } } @@ -665,6 +863,32 @@ void vterm_blink_thread(void *ctx) } } +static void vterm_make_bold(struct vterm *vt) +{ + if (!vt->bold) + return; + for (int i = 0; i < 8; i++) + { + if (same_colour(&vt->fg, &color_table[i])) + { + vt->fg = bright_color_table[i]; + break; + } + } +} + +static void vterm_undo_bold(struct vterm *vt) +{ + for (int i = 0; i < 8; i++) + { + if (same_colour(&vt->fg, &bright_color_table[i])) + { + vt->fg = color_table[i]; + break; + } + } +} + void vterm_ansi_do_sgr(unsigned long n, struct vterm *vt) { switch (n) @@ -672,13 +896,33 @@ void vterm_ansi_do_sgr(unsigned long n, struct vterm *vt) case ANSI_SGR_RESET: { vt->bg = default_bg; vt->fg = default_fg; + vt->reversed = false; + vt->bold = false; + vterm_undo_bold(vt); break; } case ANSI_SGR_REVERSE: { + if (vt->reversed) + return; + vterm_undo_bold(vt); + struct color temp = vt->bg; + vt->bg = vt->fg; + vt->fg = temp; + vterm_make_bold(vt); + vt->reversed = true; + break; + } + + case ANSI_SGR_NOREVERSE: { + if (!vt->reversed) + return; + vterm_undo_bold(vt); struct color temp = vt->bg; vt->bg = vt->fg; vt->fg = temp; + vterm_make_bold(vt); + vt->reversed = false; break; } @@ -689,6 +933,7 @@ void vterm_ansi_do_sgr(unsigned long n, struct vterm *vt) case ANSI_SGR_DEFAULTFG: { vt->fg = default_fg; + vterm_make_bold(vt); break; } @@ -711,8 +956,14 @@ void vterm_ansi_do_sgr(unsigned long n, struct vterm *vt) } case ANSI_SGR_BLINKOFF: { - if (vt->blink_thread) - thread_destroy(vt->blink_thread); + // if (vt->blink_thread) + // thread_destroy(vt->blink_thread); + break; + } + + case ANSI_SGR_BOLD: { + vt->bold = true; + vterm_make_bold(vt); break; } @@ -725,8 +976,12 @@ void vterm_ansi_do_sgr(unsigned long n, struct vterm *vt) else if (n >= ANSI_SGR_SETFGMIN && n <= ANSI_SGR_SETFGMAX) { int index = n - ANSI_SGR_SETFGMIN; - vt->fg = color_table[index]; + vt->fg = vt->bold ? bright_color_table[index] : color_table[index]; } +#if 0 + else + pr_info("vterm: unimplemented CSI %lu m\n", n); +#endif } } } @@ -737,42 +992,19 @@ void vterm_ansi_erase_in_line(unsigned long n, struct vterm *vt) { /* Clear from cursor to end */ case 0: { - for (unsigned int i = vt->cursor_x; i < vt->columns; i++) - { - struct console_cell *c = &vt->cells[vt->cursor_y * vt->columns + i]; - c->codepoint = ' '; - c->fg = vt->fg; - c->bg = vt->bg; - vterm_set_dirty(c); - } + vterm_clear_range(vt, vt->cursor_x, vt->cursor_y, vt->columns, vt->cursor_y); break; } /* Clear from cursor to beginning */ case 1: { - unsigned int x = vt->cursor_x; - - for (unsigned int i = 0; i <= x; i++) - { - struct console_cell *c = &vt->cells[vt->cursor_y * vt->columns + i]; - c->codepoint = ' '; - c->fg = vt->fg; - c->bg = vt->bg; - vterm_set_dirty(c); - } + vterm_clear_range(vt, 0, vt->cursor_y, vt->cursor_x + 1, vt->cursor_y); break; } /* Clear entire line */ case 2: { - for (unsigned int i = 0; i < vt->columns; i++) - { - struct console_cell *c = &vt->cells[vt->cursor_y * vt->columns + i]; - c->codepoint = ' '; - c->fg = vt->fg; - c->bg = vt->bg; - vterm_set_dirty(c); - } + vterm_clear_range(vt, 0, vt->cursor_y, 0, vt->cursor_y + 1); break; } } @@ -784,39 +1016,13 @@ void vterm_ansi_erase_in_display(unsigned long n, struct vterm *vt) { /* Cursor to end of display */ case 0: { - /* Calculate the cidx, then loop through until the end of the array */ - unsigned int cidx = vt->cursor_y * vt->columns + vt->cursor_x; - unsigned int max = vt->rows * vt->columns; - if (cidx + 1 < cidx) - break; - - unsigned int iters = max - cidx; - - for (unsigned int i = 0; i < iters; i++) - { - struct console_cell *c = &vt->cells[cidx + i]; - c->codepoint = ' '; - c->fg = vt->fg; - c->bg = vt->bg; - vterm_set_dirty(c); - } - + vterm_clear_range(vt, vt->cursor_x, vt->cursor_y, 0, vt->rows); break; } /* Cursor to start of display */ case 1: { - unsigned int cidx = vt->cursor_y * vt->columns + vt->cursor_x; - - for (unsigned int i = 0; i <= cidx; i++) - { - struct console_cell *c = &vt->cells[i]; - c->codepoint = ' '; - c->fg = vt->fg; - c->bg = vt->bg; - vterm_set_dirty(c); - } - + vterm_clear_range(vt, 0, 0, vt->cursor_x + 1, vt->cursor_y); break; } @@ -853,7 +1059,7 @@ void vterm_csi_delete_chars(unsigned long chars, struct vterm *vt) c->bg = vt->bg; } - vterm_set_dirty(c); + vterm_dirty_cell(i, vt->cursor_y, vt); } } @@ -874,7 +1080,7 @@ void vterm::insert_blank(unsigned long nr) cell.bg = bg; } - vterm_set_dirty(&cell); + vterm_dirty_cell(i, cursor_y, this); } } @@ -891,10 +1097,7 @@ void vterm::do_ri() { vterm_dirty_cell(cursor_x, cursor_y, this); if (cursor_y == top) - { vterm_scroll_down(fb, this); - vterm_flush_all(this); - } else if (cursor_y) cursor_y--; } @@ -906,10 +1109,7 @@ void vterm::do_nl() if (cursor_y == bottom) { if (cursor_y <= bottom && cursor_y >= top) - { vterm_scroll(fb, this); - vterm_flush_all(this); - } cursor_y--; } } @@ -952,6 +1152,18 @@ void vterm::do_generic_escape(char escape) cursor_y = saved_y; break; } + + case ')': { + /* TODO: Do properly. This is quite awkward to do because parsing ESC ( 0 isn't + * trivial since all other ESC's don't take any sort of arguments. */ + gx[1] = Gx_GRAPH; + break; + } + + default: { + // pr_info("vterm: Unhandled ESC %c\n", escape); + break; + } } } @@ -1114,6 +1326,12 @@ void vterm::process_escape_char(char c) } } +template +static inline T clamp(T val, T min, T max) +{ + return cul::min(cul::max(val, min), max); +} + void vterm::do_csi_command(char escape) { if (csi_data.dec_private) @@ -1148,7 +1366,7 @@ void vterm::do_csi_command(char escape) case ANSI_CURSOR_HORIZONTAL_ABS: { if (args[0] > columns - 1) args[0] = columns - 1; - cursor_x = args[0]; + cursor_x = args[0] - 1; break; } @@ -1166,14 +1384,12 @@ void vterm::do_csi_command(char escape) case ANSI_SCROLL_UP: { for (unsigned long i = 0; i < args[0]; i++) vterm_scroll(fb, this); - vterm_flush_all(this); break; } case ANSI_SCROLL_DOWN: { for (unsigned long i = 0; i < args[0]; i++) vterm_scroll_down(fb, this); - vterm_flush_all(this); break; } @@ -1256,7 +1472,7 @@ void vterm::do_csi_command(char escape) if (args[0] == 0) args[0] = 1; - // delete_lines(args[0]); + delete_lines(args[0]); break; } @@ -1277,12 +1493,21 @@ void vterm::do_csi_command(char escape) { top = args[0] - 1; bottom = args[1]; + vterm_dirty_cell(cursor_x, cursor_y, this); + cursor_x = 0; + cursor_y = 0; } break; } + case CSI_ERASE_CHARS: { + unsigned long nr = clamp(args[0], 1, columns - cursor_x); + vterm_clear_range(this, cursor_x, cursor_y, cursor_x + nr, cursor_y); + break; + } + default: { - // printf("Unimplemented escape %c\n", escape); + // pr_info("vt: Unimplemented escape %c\n", escape); break; } } @@ -1306,7 +1531,12 @@ void vterm::insert_lines(unsigned long nr) nr = possible_lines_to_insert; __vterm_scroll_down(fb, this, nr, cursor_y, bottom); - vterm_flush_all(this); +} + +void vterm::delete_lines(unsigned long nr) +{ + nr = clamp(nr, 1, rows - cursor_y); + __vterm_scroll(fb, this, nr, cursor_y, bottom); } size_t vterm::do_escape(const char *buffer, size_t len) @@ -1327,32 +1557,23 @@ size_t vterm::do_escape(const char *buffer, size_t len) char escape = csi_data.escape_character; #if 0 - char buf[50]; if (in_csi) - snprintf(buf, 50, "Seq: %c nargs %lu args {%lu, %lu}\n", escape, csi_data.nr_args, - csi_data.args[0], csi_data.args[1]); - if (in_csi && !csi_data.dec_private) - platform_serial_write(buf, strlen(buf)); - // platform_serial_write("Seq: ", strlen("Seq: ")); - // platform_serial_write(&escape, 1); - // platform_serial_write("\n", 1); + pr_info("Seq: %c nargs %lu args {%lu, %lu}\n", escape, csi_data.nr_args, csi_data.args[0], + csi_data.args[1]); + else if (in_dec) + pr_info("doing DEC escape %c\n", escape); + else + pr_info("doing generic escape %c\n", escape); #endif if (in_dec) - { do_dec_command(escape); - } else if (in_csi) - { do_csi_command(escape); - } else - { do_generic_escape(escape); - } reset_escape_status(); - return processed; } @@ -1364,7 +1585,6 @@ ssize_t vterm_write_tty(const void *buffer, size_t size, struct tty *tty) mutex_lock(&vt->vt_lock); size_t i = 0; const char *data = (const char *) buffer; - bool did_scroll = false; for (; i < size; i++) { @@ -1393,18 +1613,14 @@ ssize_t vterm_write_tty(const void *buffer, size_t size, struct tty *tty) platform_serial_write(x, strlen(x)); #endif // platform_serial_write(data + i, 1); - if (vterm_putc(codepoint, vt)) - did_scroll = true; + vterm_putc(codepoint, vt); /* We sub a 1 because we're incrementing on the for loop */ i += codepoint_length - 1; } } - if (!did_scroll) - vterm_flush(vt); - else - vterm_flush_all(vt); + vterm_flush(vt); update_cursor(vt); mutex_unlock(&vt->vt_lock); @@ -1456,7 +1672,12 @@ void vterm_init(struct tty *tty) vt->fg = default_fg; vt->bg = default_bg; - assert(vt->cells != NULL); + int bitmap_size = + vt->rows / (sizeof(unsigned long) * 8) + ((vt->rows % (sizeof(unsigned long) * 8)) != 0); + vt->dirty_row_bitmap = + (unsigned long *) kcalloc(bitmap_size, sizeof(unsigned long), GFP_KERNEL); + CHECK(vt->dirty_row_bitmap); + vt->bitmap_size = bitmap_size; vterm_fill_screen(vt, ' ', vt->fg, vt->bg); @@ -1494,7 +1715,6 @@ static int vterm_write_con(const char *buffer, size_t size, unsigned int flags, size_t i = 0; const char *data = (const char *) buffer; - bool did_scroll = false; for (; i < size; i++) { @@ -1524,18 +1744,13 @@ static int vterm_write_con(const char *buffer, size_t size, unsigned int flags, vterm_putc('\r', vt); } - if (vterm_putc(codepoint, vt)) - did_scroll = true; - + vterm_putc(codepoint, vt); /* We sub a 1 because we're incrementing on the for loop */ i += codepoint_length - 1; } } - if (!did_scroll) - vterm_flush(vt); - else - vterm_flush_all(vt); + vterm_flush(vt); update_cursor(vt); if (has_lock) @@ -1590,42 +1805,78 @@ struct key_action }; struct key_action key_actions[] = { - {KEYMAP_KEY_A, "a", "A", "\01"}, {KEYMAP_KEY_B, "b", "B", "\02"}, - {KEYMAP_KEY_C, "c", "C", "\03"}, {KEYMAP_KEY_D, "d", "D", "\04"}, - {KEYMAP_KEY_E, "e", "E", "\05"}, {KEYMAP_KEY_F, "f", "F", "\06"}, - {KEYMAP_KEY_G, "g", "G", "\07"}, {KEYMAP_KEY_H, "h", "H", "\010"}, - {KEYMAP_KEY_I, "i", "I", "\011"}, {KEYMAP_KEY_J, "j", "J", "\012"}, - {KEYMAP_KEY_K, "k", "K", "\013"}, {KEYMAP_KEY_L, "l", "L", "\014"}, - {KEYMAP_KEY_M, "m", "M", "\015"}, {KEYMAP_KEY_N, "n", "N", "\016"}, - {KEYMAP_KEY_O, "o", "O", "\017"}, {KEYMAP_KEY_P, "p", "P", "\020"}, - {KEYMAP_KEY_Q, "q", "Q", "\021"}, {KEYMAP_KEY_R, "r", "R", "\022"}, - {KEYMAP_KEY_S, "s", "S", "\023"}, {KEYMAP_KEY_T, "t", "T", "\024"}, - {KEYMAP_KEY_U, "u", "U", "\025"}, {KEYMAP_KEY_V, "v", "V", "\026"}, - {KEYMAP_KEY_W, "w", "W", "\027"}, {KEYMAP_KEY_X, "x", "X", "\030"}, - {KEYMAP_KEY_Y, "y", "Y", "\031"}, {KEYMAP_KEY_Z, "z", "Z", "\032"}, - {KEYMAP_KEY_0, "0", ")"}, {KEYMAP_KEY_1, "1", "!"}, - {KEYMAP_KEY_2, "2", "@"}, {KEYMAP_KEY_3, "3", "#"}, - {KEYMAP_KEY_4, "4", "$"}, {KEYMAP_KEY_5, "5", "%"}, - {KEYMAP_KEY_6, "6", "^"}, {KEYMAP_KEY_7, "7", "&"}, - {KEYMAP_KEY_8, "8", "*"}, {KEYMAP_KEY_9, "9", "("}, - {KEYMAP_KEY_COMMA, ",", "<"}, {KEYMAP_KEY_DOT, ".", ">"}, - {KEYMAP_KEY_KEYPAD_0, "0"}, {KEYMAP_KEY_KEYPAD_1, "1"}, - {KEYMAP_KEY_KEYPAD_2, "2"}, {KEYMAP_KEY_KEYPAD_3, "3"}, - {KEYMAP_KEY_KEYPAD_4, "4"}, {KEYMAP_KEY_KEYPAD_5, "5"}, - {KEYMAP_KEY_KEYPAD_6, "6"}, {KEYMAP_KEY_KEYPAD_7, "7"}, - {KEYMAP_KEY_KEYPAD_8, "8"}, {KEYMAP_KEY_KEYPAD_9, "9"}, - {KEYMAP_KEY_MINUS, "-", "_"}, {KEYMAP_KEY_EQUALS, "=", "+"}, - {KEYMAP_KEY_LEFTBRACE, "[", "{"}, {KEYMAP_KEY_RIGHTBRACE, "]", "}"}, - {KEYMAP_KEY_ENTER, "\r"}, {KEYMAP_KEY_SEMICOLON, ";", ":"}, - {KEYMAP_KEY_GRAVE, "`", "~"}, {KEYMAP_KEY_TAB, "\t"}, - {KEYMAP_KEY_APOSTROPHE, "'", "\""}, {KEYMAP_KEY_SLASH, "/", "?"}, - {KEYMAP_KEY_BACKSLASH, "|"}, {KEYMAP_KEY_BACKSPACE, "\x7f"}, - {KEYMAP_KEY_KEYPAD_DOT, "."}, {KEYMAP_KEY_KEYPAD_SLASH, "/"}, - {KEYMAP_KEY_KEYPAD_ASTERISK, "*"}, {KEYMAP_KEY_KEYPAD_MINUS, "-"}, - {KEYMAP_KEY_KEYPAD_PLUS, "+"}, {KEYMAP_KEY_KEYPAD_ENTER, "\n"}, - {KEYMAP_KEY_SPACE, " ", " "}, {KEYMAP_KEY_ARROW_LEFT, "\033[D"}, - {KEYMAP_KEY_ARROW_UP, "\033[A"}, {KEYMAP_KEY_ARROW_DOWN, "\033[B"}, - {KEYMAP_KEY_ARROW_RIGHT, "\033[C"}, {KEYMAP_KEY_ESC, "\033"}, + {KEYMAP_KEY_A, "a", "A", "\01"}, + {KEYMAP_KEY_B, "b", "B", "\02"}, + {KEYMAP_KEY_C, "c", "C", "\03"}, + {KEYMAP_KEY_D, "d", "D", "\04"}, + {KEYMAP_KEY_E, "e", "E", "\05"}, + {KEYMAP_KEY_F, "f", "F", "\06"}, + {KEYMAP_KEY_G, "g", "G", "\07"}, + {KEYMAP_KEY_H, "h", "H", "\010"}, + {KEYMAP_KEY_I, "i", "I", "\011"}, + {KEYMAP_KEY_J, "j", "J", "\012"}, + {KEYMAP_KEY_K, "k", "K", "\013"}, + {KEYMAP_KEY_L, "l", "L", "\014"}, + {KEYMAP_KEY_M, "m", "M", "\015"}, + {KEYMAP_KEY_N, "n", "N", "\016"}, + {KEYMAP_KEY_O, "o", "O", "\017"}, + {KEYMAP_KEY_P, "p", "P", "\020"}, + {KEYMAP_KEY_Q, "q", "Q", "\021"}, + {KEYMAP_KEY_R, "r", "R", "\022"}, + {KEYMAP_KEY_S, "s", "S", "\023"}, + {KEYMAP_KEY_T, "t", "T", "\024"}, + {KEYMAP_KEY_U, "u", "U", "\025"}, + {KEYMAP_KEY_V, "v", "V", "\026"}, + {KEYMAP_KEY_W, "w", "W", "\027"}, + {KEYMAP_KEY_X, "x", "X", "\030"}, + {KEYMAP_KEY_Y, "y", "Y", "\031"}, + {KEYMAP_KEY_Z, "z", "Z", "\032"}, + {KEYMAP_KEY_0, "0", ")"}, + {KEYMAP_KEY_1, "1", "!"}, + {KEYMAP_KEY_2, "2", "@"}, + {KEYMAP_KEY_3, "3", "#"}, + {KEYMAP_KEY_4, "4", "$"}, + {KEYMAP_KEY_5, "5", "%"}, + {KEYMAP_KEY_6, "6", "^"}, + {KEYMAP_KEY_7, "7", "&"}, + {KEYMAP_KEY_8, "8", "*"}, + {KEYMAP_KEY_9, "9", "("}, + {KEYMAP_KEY_COMMA, ",", "<"}, + {KEYMAP_KEY_DOT, ".", ">"}, + {KEYMAP_KEY_KEYPAD_0, "0"}, + {KEYMAP_KEY_KEYPAD_1, "1"}, + {KEYMAP_KEY_KEYPAD_2, "2"}, + {KEYMAP_KEY_KEYPAD_3, "3"}, + {KEYMAP_KEY_KEYPAD_4, "4"}, + {KEYMAP_KEY_KEYPAD_5, "5"}, + {KEYMAP_KEY_KEYPAD_6, "6"}, + {KEYMAP_KEY_KEYPAD_7, "7"}, + {KEYMAP_KEY_KEYPAD_8, "8"}, + {KEYMAP_KEY_KEYPAD_9, "9"}, + {KEYMAP_KEY_MINUS, "-", "_"}, + {KEYMAP_KEY_EQUALS, "=", "+"}, + {KEYMAP_KEY_LEFTBRACE, "[", "{"}, + {KEYMAP_KEY_RIGHTBRACE, "]", "}"}, + {KEYMAP_KEY_ENTER, "\r"}, + {KEYMAP_KEY_SEMICOLON, ";", ":"}, + {KEYMAP_KEY_GRAVE, "`", "~"}, + {KEYMAP_KEY_TAB, "\t"}, + {KEYMAP_KEY_APOSTROPHE, "'", "\""}, + {KEYMAP_KEY_SLASH, "/", "?"}, + {KEYMAP_KEY_BACKSLASH, "|"}, + {KEYMAP_KEY_BACKSPACE, "\x7f"}, + {KEYMAP_KEY_KEYPAD_DOT, "."}, + {KEYMAP_KEY_KEYPAD_SLASH, "/"}, + {KEYMAP_KEY_KEYPAD_ASTERISK, "*"}, + {KEYMAP_KEY_KEYPAD_MINUS, "-"}, + {KEYMAP_KEY_KEYPAD_PLUS, "+"}, + {KEYMAP_KEY_KEYPAD_ENTER, "\n"}, + {KEYMAP_KEY_SPACE, " ", " "}, + {KEYMAP_KEY_ARROW_LEFT, "\033[D", NULL, "\033[1;5D"}, + {KEYMAP_KEY_ARROW_UP, "\033[A", NULL, "\033[1;5A"}, + {KEYMAP_KEY_ARROW_DOWN, "\033[B", NULL, "\033[1;5B"}, + {KEYMAP_KEY_ARROW_RIGHT, "\033[C", NULL, "\033[1;5C"}, + {KEYMAP_KEY_ESC, "\033"}, }; struct key_action pt_pt_key_actions[] = { @@ -1697,10 +1948,10 @@ struct key_action pt_pt_key_actions[] = { {KEYMAP_KEY_KEYPAD_ENTER, "\n"}, {KEYMAP_KEY_SPACE, " ", " "}, {KEYMAP_102ND, "<", ">"}, - {KEYMAP_KEY_ARROW_LEFT, "\033[D"}, - {KEYMAP_KEY_ARROW_UP, "\033[A"}, - {KEYMAP_KEY_ARROW_DOWN, "\033[B"}, - {KEYMAP_KEY_ARROW_RIGHT, "\033[C"}, + {KEYMAP_KEY_ARROW_LEFT, "\033[D", NULL, "\033[1;5D"}, + {KEYMAP_KEY_ARROW_UP, "\033[A", NULL, "\033[1;5A"}, + {KEYMAP_KEY_ARROW_DOWN, "\033[B", NULL, "\033[1;5B"}, + {KEYMAP_KEY_ARROW_RIGHT, "\033[C", NULL, "\033[1;5C"}, {KEYMAP_KEY_ESC, "\033"}, }; @@ -1714,6 +1965,45 @@ void __vterm_receive_input(void *p) void sched_dump_threads(void); +static bool is_numpad_code(keycode_t code) +{ + switch (code) + { + /* Ew... Depends on the KEYPAD's enum layout */ + case KEYMAP_KEY_KEYPAD_7 ... KEYMAP_KEY_KEYPAD_PLUS: + return true; + default: + return false; + } +} + +static const char *numpad_replacement(keycode_t code, const char *str) +{ + switch (code) + { + case KEYMAP_KEY_KEYPAD_7: + return "\033[H"; + case KEYMAP_KEY_KEYPAD_8: + return "\033[A"; + case KEYMAP_KEY_KEYPAD_9: + return "\033[5~"; + case KEYMAP_KEY_KEYPAD_4: + return "\033[D"; + case KEYMAP_KEY_KEYPAD_5: + return "\033[E"; + case KEYMAP_KEY_KEYPAD_6: + return "\033[C"; + case KEYMAP_KEY_KEYPAD_1: + return "\033[F"; + case KEYMAP_KEY_KEYPAD_2: + return "\033[B"; + case KEYMAP_KEY_KEYPAD_3: + return "\033[6~"; + default: + return str; + } +} + int vterm_handle_key(struct vterm *vt, struct input_device *dev, struct input_event *ev) { /* We have no interest in release events */ @@ -1725,6 +2015,9 @@ int vterm_handle_key(struct vterm *vt, struct input_device *dev, struct input_ev sched_dump_threads(); #endif + if (ev->code == KEYMAP_KEY_KEYPAD_NUMLCK) + vt->numlck = !vt->numlck; + struct key_action *acts = pt_pt_key_actions; struct key_action *desired_action = NULL; @@ -1762,6 +2055,9 @@ int vterm_handle_key(struct vterm *vt, struct input_device *dev, struct input_ev action_string = desired_action->action; } + if (!vt->numlck && is_numpad_code(ev->code)) + action_string = numpad_replacement(ev->code, action_string); + if (likely(action_string)) { struct dpc_work w; diff --git a/kernel/kernel/ubsan.cpp b/kernel/kernel/ubsan.cpp index f1d23b12f..2d5bbb572 100644 --- a/kernel/kernel/ubsan.cpp +++ b/kernel/kernel/ubsan.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 - 2023 Pedro Falcato + * Copyright (c) 2022 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -181,6 +182,8 @@ void ubsan_exit_kunit() static void ubsan_report_end() { + if (!die_on_every_ubsan) + WARN_ON(1); printk("=================================================================================\n"); if (die_on_every_ubsan) [[unlikely]] ubsan_abort(); diff --git a/kernel/kernel/wait_queue.cpp b/kernel/kernel/wait_queue.cpp index 9df15b598..4459f67b4 100644 --- a/kernel/kernel/wait_queue.cpp +++ b/kernel/kernel/wait_queue.cpp @@ -29,23 +29,11 @@ void wait_queue_wait(struct wait_queue *queue) sched_yield(); } -struct wait_queue_token *wait_queue_wake_unlocked(struct wait_queue *queue) +struct wait_queue_token *wait_queue_wake_unlocked(struct wait_queue_token *token) { - MUST_HOLD_LOCK(&queue->lock); - assert(list_is_empty(&queue->token_list) == false); - - struct list_head *token_lh = list_first_element(&queue->token_list); - - assert(token_lh != NULL); - - struct wait_queue_token *token = container_of(token_lh, struct wait_queue_token, token_node); - - list_remove(token_lh); - - list_assert_correct(&queue->token_list); - + if (!(token->flags & WQ_TOKEN_NO_DEQUEUE)) + list_remove(&token->token_node); token->signaled = true; - return token; } @@ -59,7 +47,8 @@ void wait_queue_wake(struct wait_queue *queue) return; } - struct wait_queue_token *t = wait_queue_wake_unlocked(queue); + struct wait_queue_token *t = wait_queue_wake_unlocked( + list_first_entry(&queue->token_list, struct wait_queue_token, token_node)); if (t->callback) t->callback(t->context, t); @@ -71,11 +60,12 @@ void wait_queue_wake(struct wait_queue *queue) void wait_queue_wake_all(struct wait_queue *queue) { + struct wait_queue_token *waiter, *next; unsigned long cpu_flags = spin_lock_irqsave(&queue->lock); - while (!list_is_empty(&queue->token_list)) + list_for_each_entry_safe (waiter, next, &queue->token_list, token_node) { - struct wait_queue_token *t = wait_queue_wake_unlocked(queue); + struct wait_queue_token *t = wait_queue_wake_unlocked(waiter); if (t->callback) t->callback(t->context, t); diff --git a/kernel/lib/libk/stdio/sprintf.c b/kernel/lib/libk/stdio/sprintf.c index 2aef5b004..727689e33 100644 --- a/kernel/lib/libk/stdio/sprintf.c +++ b/kernel/lib/libk/stdio/sprintf.c @@ -12,9 +12,12 @@ #include #include +#include #include #include +#include + #ifdef DO_STREAMS struct stream { @@ -399,6 +402,144 @@ static int dump_buffer(struct stream *stream, void *ptr, struct printf_specifier return written; } +static void find_v6_zeroes_range(struct in6_addr *addr, int *start, int *end) +{ + int best_start = -1, best_end = -1; + int curr_start = -1, curr_end = -1; + bool in_zeroes = false; + for (int i = 0; i < 8; i++) + { + if (addr->s6_addr16[i] == 0) + { + if (!in_zeroes) + curr_start = i; + curr_end = i; + in_zeroes = true; + } + else + { + /* The use of the symbol "::" MUST be used to its maximum capability (longest sequence + * wins). It also MUST NOT be used to compress a single field. [...] the first sequence + * of zero bits MUST be shortened (when the length is equal) -- rfc5952, non-verbatim */ + if (in_zeroes && curr_end - curr_start > 1 && + curr_end - curr_start > best_end - best_start) + { + best_start = curr_start; + best_end = curr_end; + } + + in_zeroes = false; + } + } + + if (in_zeroes && curr_end - curr_start > 1 && curr_end - curr_start > best_end - best_start) + { + best_start = curr_start; + best_end = curr_end; + } + + *start = best_start; + *end = best_end; +} + +static int print_ipaddr(struct stream *stream, void *ptr, struct printf_specifier *spec, + const char **pfmt) +{ + const char *fmt = *pfmt; + if (*fmt == '4') + { + u32 *inp = ptr; + u32 in = *inp; + char buf[sizeof("255.255.255.255")]; + int j = 0; + for (int i = 0; i < 4; i++, in >>= 8) + { + int k = 3; + u8 byte = in & 0xff; + char tmp[4]; + + if (i > 0) + buf[j++] = '.'; + + if (byte > 0) + { + while (byte) + { + tmp[k--] = '0' + (byte % 10); + byte /= 10; + } + } + else + { + tmp[k--] = '0'; + } + + while (k < 3) + buf[j++] = tmp[++k]; + } + + buf[j] = '\0'; + *pfmt = ++fmt; + return printf_do_string(stream, buf, spec->fwidth, spec->precision, spec->flags); + } + else if (*fmt == '6') + { + /* IPv6 address printing as per rfc5952 */ + int compr_start, compr_end; + struct in6_addr *addr = ptr; + *pfmt = ++fmt; + char buf[sizeof("0000:0000:0000:0000:0000:0000:0000:0000")]; + int j = 0; + find_v6_zeroes_range(addr, &compr_start, &compr_end); + for (int i = 0; i < 8; i++) + { + int k = 4; + u16 val = ntohs(addr->s6_addr16[i]); + char tmp[5]; + + if (i >= compr_start && i <= compr_end) + { + /* Add the extra ':' to indicate a compressed ipv6 addr range */ + if (i == compr_end) + { + buf[j++] = ':'; + /* Add another ':' if nothing comes after us */ + if (compr_end == 7) + buf[j++] = ':'; + } + + continue; + } + + if (i > 0) + buf[j++] = ':'; + + /* Leading zeros MUST be suppressed */ + if (val > 0) + { + while (val) + { + /* rfc5952 demands lowercase hex digits */ + tmp[k--] = digits[val % 16]; + val /= 16; + } + } + else + { + tmp[k--] = '0'; + } + + while (k < 4) + buf[j++] = tmp[++k]; + } + + buf[j] = '\0'; + return printf_do_string(stream, buf, spec->fwidth, spec->precision, spec->flags); + } + + return 0; +} + static int print_pointer(struct stream *stream, void *ptr, struct printf_specifier *spec, const char **pfmt) { @@ -415,6 +556,9 @@ static int print_pointer(struct stream *stream, void *ptr, struct printf_specifi case 'h': *pfmt = ++fmt; return dump_buffer(stream, ptr, spec); + case 'I': + *pfmt = ++fmt; + return print_ipaddr(stream, ptr, spec, pfmt); case 'x': *pfmt = ++fmt; /* fallthrough */ diff --git a/kernel/scripts/bdf2c.py b/kernel/scripts/bdf2c.py new file mode 100755 index 000000000..6b74e3251 --- /dev/null +++ b/kernel/scripts/bdf2c.py @@ -0,0 +1,353 @@ +#!/bin/python3 +# Copyright (c) 2024 Pedro Falcato +# This file is part of Onyx, and is released under the terms of the GPLv2 License +# check LICENSE at the root directory for more information +# +# SPDX-License-Identifier: GPL-2.0-only +# +import io +import sys +import gzip + +defines = """#define ________ 0x00 +#define _______X 0x01 +#define ______X_ 0x02 +#define ______XX 0x03 +#define _____X__ 0x04 +#define _____X_X 0x05 +#define _____XX_ 0x06 +#define _____XXX 0x07 +#define ____X___ 0x08 +#define ____X__X 0x09 +#define ____X_X_ 0x0A +#define ____X_XX 0x0B +#define ____XX__ 0x0C +#define ____XX_X 0x0D +#define ____XXX_ 0x0E +#define ____XXXX 0x0F +#define ___X____ 0x10 +#define ___X___X 0x11 +#define ___X__X_ 0x12 +#define ___X__XX 0x13 +#define ___X_X__ 0x14 +#define ___X_X_X 0x15 +#define ___X_XX_ 0x16 +#define ___X_XXX 0x17 +#define ___XX___ 0x18 +#define ___XX__X 0x19 +#define ___XX_X_ 0x1A +#define ___XX_XX 0x1B +#define ___XXX__ 0x1C +#define ___XXX_X 0x1D +#define ___XXXX_ 0x1E +#define ___XXXXX 0x1F +#define __X_____ 0x20 +#define __X____X 0x21 +#define __X___X_ 0x22 +#define __X___XX 0x23 +#define __X__X__ 0x24 +#define __X__X_X 0x25 +#define __X__XX_ 0x26 +#define __X__XXX 0x27 +#define __X_X___ 0x28 +#define __X_X__X 0x29 +#define __X_X_X_ 0x2A +#define __X_X_XX 0x2B +#define __X_XX__ 0x2C +#define __X_XX_X 0x2D +#define __X_XXX_ 0x2E +#define __X_XXXX 0x2F +#define __XX____ 0x30 +#define __XX___X 0x31 +#define __XX__X_ 0x32 +#define __XX__XX 0x33 +#define __XX_X__ 0x34 +#define __XX_X_X 0x35 +#define __XX_XX_ 0x36 +#define __XX_XXX 0x37 +#define __XXX___ 0x38 +#define __XXX__X 0x39 +#define __XXX_X_ 0x3A +#define __XXX_XX 0x3B +#define __XXXX__ 0x3C +#define __XXXX_X 0x3D +#define __XXXXX_ 0x3E +#define __XXXXXX 0x3F +#define _X______ 0x40 +#define _X_____X 0x41 +#define _X____X_ 0x42 +#define _X____XX 0x43 +#define _X___X__ 0x44 +#define _X___X_X 0x45 +#define _X___XX_ 0x46 +#define _X___XXX 0x47 +#define _X__X___ 0x48 +#define _X__X__X 0x49 +#define _X__X_X_ 0x4A +#define _X__X_XX 0x4B +#define _X__XX__ 0x4C +#define _X__XX_X 0x4D +#define _X__XXX_ 0x4E +#define _X__XXXX 0x4F +#define _X_X____ 0x50 +#define _X_X___X 0x51 +#define _X_X__X_ 0x52 +#define _X_X__XX 0x53 +#define _X_X_X__ 0x54 +#define _X_X_X_X 0x55 +#define _X_X_XX_ 0x56 +#define _X_X_XXX 0x57 +#define _X_XX___ 0x58 +#define _X_XX__X 0x59 +#define _X_XX_X_ 0x5A +#define _X_XX_XX 0x5B +#define _X_XXX__ 0x5C +#define _X_XXX_X 0x5D +#define _X_XXXX_ 0x5E +#define _X_XXXXX 0x5F +#define _XX_____ 0x60 +#define _XX____X 0x61 +#define _XX___X_ 0x62 +#define _XX___XX 0x63 +#define _XX__X__ 0x64 +#define _XX__X_X 0x65 +#define _XX__XX_ 0x66 +#define _XX__XXX 0x67 +#define _XX_X___ 0x68 +#define _XX_X__X 0x69 +#define _XX_X_X_ 0x6A +#define _XX_X_XX 0x6B +#define _XX_XX__ 0x6C +#define _XX_XX_X 0x6D +#define _XX_XXX_ 0x6E +#define _XX_XXXX 0x6F +#define _XXX____ 0x70 +#define _XXX___X 0x71 +#define _XXX__X_ 0x72 +#define _XXX__XX 0x73 +#define _XXX_X__ 0x74 +#define _XXX_X_X 0x75 +#define _XXX_XX_ 0x76 +#define _XXX_XXX 0x77 +#define _XXXX___ 0x78 +#define _XXXX__X 0x79 +#define _XXXX_X_ 0x7A +#define _XXXX_XX 0x7B +#define _XXXXX__ 0x7C +#define _XXXXX_X 0x7D +#define _XXXXXX_ 0x7E +#define _XXXXXXX 0x7F +#define X_______ 0x80 +#define X______X 0x81 +#define X_____X_ 0x82 +#define X_____XX 0x83 +#define X____X__ 0x84 +#define X____X_X 0x85 +#define X____XX_ 0x86 +#define X____XXX 0x87 +#define X___X___ 0x88 +#define X___X__X 0x89 +#define X___X_X_ 0x8A +#define X___X_XX 0x8B +#define X___XX__ 0x8C +#define X___XX_X 0x8D +#define X___XXX_ 0x8E +#define X___XXXX 0x8F +#define X__X____ 0x90 +#define X__X___X 0x91 +#define X__X__X_ 0x92 +#define X__X__XX 0x93 +#define X__X_X__ 0x94 +#define X__X_X_X 0x95 +#define X__X_XX_ 0x96 +#define X__X_XXX 0x97 +#define X__XX___ 0x98 +#define X__XX__X 0x99 +#define X__XX_X_ 0x9A +#define X__XX_XX 0x9B +#define X__XXX__ 0x9C +#define X__XXX_X 0x9D +#define X__XXXX_ 0x9E +#define X__XXXXX 0x9F +#define X_X_____ 0xA0 +#define X_X____X 0xA1 +#define X_X___X_ 0xA2 +#define X_X___XX 0xA3 +#define X_X__X__ 0xA4 +#define X_X__X_X 0xA5 +#define X_X__XX_ 0xA6 +#define X_X__XXX 0xA7 +#define X_X_X___ 0xA8 +#define X_X_X__X 0xA9 +#define X_X_X_X_ 0xAA +#define X_X_X_XX 0xAB +#define X_X_XX__ 0xAC +#define X_X_XX_X 0xAD +#define X_X_XXX_ 0xAE +#define X_X_XXXX 0xAF +#define X_XX____ 0xB0 +#define X_XX___X 0xB1 +#define X_XX__X_ 0xB2 +#define X_XX__XX 0xB3 +#define X_XX_X__ 0xB4 +#define X_XX_X_X 0xB5 +#define X_XX_XX_ 0xB6 +#define X_XX_XXX 0xB7 +#define X_XXX___ 0xB8 +#define X_XXX__X 0xB9 +#define X_XXX_X_ 0xBA +#define X_XXX_XX 0xBB +#define X_XXXX__ 0xBC +#define X_XXXX_X 0xBD +#define X_XXXXX_ 0xBE +#define X_XXXXXX 0xBF +#define XX______ 0xC0 +#define XX_____X 0xC1 +#define XX____X_ 0xC2 +#define XX____XX 0xC3 +#define XX___X__ 0xC4 +#define XX___X_X 0xC5 +#define XX___XX_ 0xC6 +#define XX___XXX 0xC7 +#define XX__X___ 0xC8 +#define XX__X__X 0xC9 +#define XX__X_X_ 0xCA +#define XX__X_XX 0xCB +#define XX__XX__ 0xCC +#define XX__XX_X 0xCD +#define XX__XXX_ 0xCE +#define XX__XXXX 0xCF +#define XX_X____ 0xD0 +#define XX_X___X 0xD1 +#define XX_X__X_ 0xD2 +#define XX_X__XX 0xD3 +#define XX_X_X__ 0xD4 +#define XX_X_X_X 0xD5 +#define XX_X_XX_ 0xD6 +#define XX_X_XXX 0xD7 +#define XX_XX___ 0xD8 +#define XX_XX__X 0xD9 +#define XX_XX_X_ 0xDA +#define XX_XX_XX 0xDB +#define XX_XXX__ 0xDC +#define XX_XXX_X 0xDD +#define XX_XXXX_ 0xDE +#define XX_XXXXX 0xDF +#define XXX_____ 0xE0 +#define XXX____X 0xE1 +#define XXX___X_ 0xE2 +#define XXX___XX 0xE3 +#define XXX__X__ 0xE4 +#define XXX__X_X 0xE5 +#define XXX__XX_ 0xE6 +#define XXX__XXX 0xE7 +#define XXX_X___ 0xE8 +#define XXX_X__X 0xE9 +#define XXX_X_X_ 0xEA +#define XXX_X_XX 0xEB +#define XXX_XX__ 0xEC +#define XXX_XX_X 0xED +#define XXX_XXX_ 0xEE +#define XXX_XXXX 0xEF +#define XXXX____ 0xF0 +#define XXXX___X 0xF1 +#define XXXX__X_ 0xF2 +#define XXXX__XX 0xF3 +#define XXXX_X__ 0xF4 +#define XXXX_X_X 0xF5 +#define XXXX_XX_ 0xF6 +#define XXXX_XXX 0xF7 +#define XXXXX___ 0xF8 +#define XXXXX__X 0xF9 +#define XXXXX_X_ 0xFA +#define XXXXX_XX 0xFB +#define XXXXXX__ 0xFC +#define XXXXXX_X 0xFD +#define XXXXXXX_ 0xFE +#define XXXXXXXX 0xFF"""; + +def handle_startchar(font_file, line: str): + fontname = line.split(' ')[1] + font_file.write('\t/* ' + fontname.strip() + ' */\n') + +def write_bitmap(file, line: str): + to_write = bin(int(line,base=16)).removeprefix('0b').zfill(8).replace('0', '_').replace('1', 'X') + ',\n' + file.write('\t' + to_write) + +def main(): + charmap = {} + opener = open + if sys.argv[1].endswith(".gz"): + opener = gzip.open + font = opener(sys.argv[1], 'rt') + with open(sys.argv[2], 'w') as file: + file.write('#include \n\n') + file.write(defines) + + file.write('\n' + 'static const unsigned char font_bitmap[] = {\n') + yres = 0 + inbitmap = False + bitmap_row = 0 + chars = 0 + top_encoding = 0 + + for line in font: + if line.startswith('STARTCHAR'): + handle_startchar(file, line) + chars += 1 + if line.startswith('ENCODING'): + top_encoding = int(line.split(' ')[1]) + file.write('\t/* U+' + hex(top_encoding).removeprefix('0x') + ' */\n') + charmap[top_encoding] = chars + if line.startswith('BBX'): + yres = int(line.split(' ')[2]) + if line.startswith('BITMAP'): + inbitmap = True + bitmap_row = 0 + elif inbitmap: + write_bitmap(file, line) + bitmap_row += 1 + if bitmap_row == yres: + inbitmap = False + + file.write('\n};\n\n') + + file.write("static const unsigned short char_indices[] = {\n"); + + for (key, val) in charmap.items(): + file.write(f" [{key}] = {val},\n"); + + file.write("};\n\n"); + + file.write(f""" +static unsigned int utf_to_char(unsigned int c) +{{ + if (c >= {top_encoding + 1}) + return 0; + unsigned short index = char_indices[c] - 1; + + if (index == (unsigned short) -1) + return 0; + return index; +}}\n +"""); + + file.write(f""" +static const unsigned char __cursor__bitmap[] = {{ + XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, + XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, ________, ________, +}}; +static const unsigned int font_mask[8] = {{128, 64, 32, 16, 8, 4, 2, 1}}; +/* bitmap font structure */ +struct font boot_font = {{.width = 8, + .height = {yres}, + .chars = {chars}, + .font_bitmap = (void *) font_bitmap, + .mask = (void *) font_mask, + .cursor_bitmap = (void *) __cursor__bitmap, + .utf2char = utf_to_char}};\n +"""); + + + +main() diff --git a/musl b/musl index b1f7fbcfa..35aac5715 160000 --- a/musl +++ b/musl @@ -1 +1 @@ -Subproject commit b1f7fbcfa806ead2af7ef209203bb94d7ed4c8b8 +Subproject commit 35aac5715b84bbe506923552247ea84406b2f24c diff --git a/usystem/network/netctld/include/netctl.hpp b/usystem/network/netctld/include/netctl.hpp index 5b7fe77b4..d0ef5ed39 100644 --- a/usystem/network/netctld/include/netctl.hpp +++ b/usystem/network/netctld/include/netctl.hpp @@ -1,7 +1,9 @@ /* - * Copyright (c) 2020 Pedro Falcato + * Copyright (c) 2020 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the MIT License * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: MIT */ #pragma once @@ -12,6 +14,7 @@ #include #include +#include #include #include @@ -111,7 +114,14 @@ class instance dhcpcd::create_instance(new_name); - netctl::v6::configure_if(*this); + try + { + netctl::v6::configure_if(*this); + } + catch (std::exception& e) + { + std::cout << "netctl ipv6 address configuration failed: " << e.what() << "\n"; + } } ~instance() diff --git a/usystem/network/netctld/src/dhcpcd.cpp b/usystem/network/netctld/src/dhcpcd.cpp index 9089b8a49..5af2f7989 100644 --- a/usystem/network/netctld/src/dhcpcd.cpp +++ b/usystem/network/netctld/src/dhcpcd.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 - 2022 Pedro Falcato + * Copyright (c) 2017 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the MIT License * check LICENSE at the root directory for more information * @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -87,8 +88,8 @@ off_t dhcp_add_option(dhcp_packet_t *pkt, off_t off, unsigned char len, const vo off_t dhcp_close_options(dhcp_packet_t *pkt, off_t off) { /* Add the needed padding */ - memset(&pkt->options[off], 0, 3); - off += 3; + // memset(&pkt->options[off], 0, 3); + // off += 3; pkt->options[off] = DHO_END; return off + 1; @@ -101,11 +102,14 @@ bool packet::decode() unsigned char *opt = (unsigned char *) &packet_->options; if (length <= DHCP_FIXED_NON_UDP) + { + fprintf(stderr, "dhcpcd: Bad packet length %zu, ignoring!\n", length); return false; + } if (memcmp(opt, DHCP_OPTIONS_COOKIE, 4) == 1) { - printf("dhcpcd: Bad cookie\n"); + fprintf(stderr, "dhcpcd: Bad cookie, ignoring!\n"); return false; } @@ -114,9 +118,12 @@ bool packet::decode() opt += 4; while (*opt != DHO_END) { - /* Check of OOB */ + /* Check for OOB */ if (opt >= limit) + { + fprintf(stderr, "dhcpcd: Went out of bounds processing options, ignoring!\n"); return false; + } unsigned char type = *opt; opt++; @@ -133,22 +140,36 @@ bool packet::decode() } if (!has_message_type) + { + fprintf(stderr, "dhcpcd: Does not have message type, ignoring!\n"); return false; + } return true; } void instance::send_discover() { + const char *vendor_class_identifier = "Onyx dhcpcd (netctld)"; + char hostname[512]; + uint16_t max_msg_size = htons(576); auto boot_packet = buf; memset(boot_packet, 0, sizeof(dhcp_packet_t)); + if (gethostname(hostname, sizeof(hostname) - 1) < 0) + err(1, "gethostname"); + + for (size_t i = 0; i < strlen(hostname); i++) + hostname[i] = tolower(hostname[i]); + + hostname[sizeof(hostname) - 1] = 0; + memcpy(&boot_packet->chaddr, &mac, 6); boot_packet->xid = xid; boot_packet->hlen = 6; boot_packet->htype = HTYPE_ETHER; boot_packet->op = BOOTREQUEST; - boot_packet->flags = 0; + boot_packet->flags = htons(BOOTP_BROADCAST); off_t off = DHCP_MIN_OPT_OFFSET; memcpy(&boot_packet->options, DHCP_OPTIONS_COOKIE, 4); @@ -156,9 +177,17 @@ void instance::send_discover() unsigned char message_type = DHCPDISCOVER; off = dhcp_add_option(boot_packet, off, 1, &message_type, sizeof(message_type), DHO_DHCP_MESSAGE_TYPE); - unsigned char opts[3] = {DHO_SUBNET_MASK, DHO_ROUTERS, DHO_DOMAIN_NAME_SERVERS}; + unsigned char opts[] = {DHO_SUBNET_MASK, DHO_ROUTERS, DHO_DOMAIN_NAME_SERVERS, + DHO_HOST_NAME, DHO_DOMAIN_NAME, DHO_BROADCAST_ADDRESS, + DHO_NTP_SERVERS}; + off = dhcp_add_option(boot_packet, off, sizeof(opts), &opts, sizeof(opts), + DHO_DHCP_PARAMETER_REQUEST_LIST); + off = dhcp_add_option(boot_packet, off, 2, &max_msg_size, 2, DHO_DHCP_MAX_MESSAGE_SIZE); + off = dhcp_add_option(boot_packet, off, strlen(hostname), hostname, strlen(hostname), + DHO_HOST_NAME); off = - dhcp_add_option(boot_packet, off, 3, &opts, sizeof(opts), DHO_DHCP_PARAMETER_REQUEST_LIST); + dhcp_add_option(boot_packet, off, strlen(vendor_class_identifier), vendor_class_identifier, + strlen(vendor_class_identifier), DHO_VENDOR_CLASS_IDENTIFIER); off = dhcp_close_options(boot_packet, off); if (send(sockfd, boot_packet, DHCP_FIXED_NON_UDP + off, 0) < 0) @@ -237,6 +266,7 @@ std::unique_ptr instance::get_packets(std::function pred auto pdata = message_type->option.data(); + printf("dhcpcd: Got message type %x\n", *pdata); if (*pdata == DHCPOFFER) { if (got_dhcp_offer) @@ -254,6 +284,28 @@ std::unique_ptr instance::get_packets(std::function pred return p; } +static bool check_for_dhcpoffer(dhcpcd::packet *data) +{ + auto message_type = data->get_option(DHO_DHCP_MESSAGE_TYPE, 1); + assert(message_type != nullptr); + + auto pdata = message_type->option.data(); + + if (*pdata != DHCPOFFER) + { + fprintf(stderr, "dhcpcd: Expecting DHCPOFFER, got %x, ignoring\n", *pdata); + return false; + } + + if (!data->get_option(DHO_ROUTERS, 4)) + { + fprintf(stderr, "dhcpcd: DHCPOFFER does not supply DHO_ROUTERS, ignoring\n"); + return false; + } + + return true; +} + int instance::setup_netif() { /* DHCP essentially works like this: @@ -269,19 +321,7 @@ int instance::setup_netif() std::unique_ptr packet; /* If for some reason we can't retrieve a packet, the get_packets will throw an exception */ - while (!(packet = get_packets([](dhcpcd::packet *data) -> bool { - if (!data->get_option(DHO_ROUTERS, 4)) - return false; - - auto message_type = data->get_option(DHO_DHCP_MESSAGE_TYPE, 1); - - assert(message_type != nullptr); - - /* Sanitise the option parameters */ - - auto pdata = message_type->option.data(); - return *pdata == DHCPOFFER; - }))) + while (!(packet = get_packets(check_for_dhcpoffer))) { }