WIP: trying to fix memory conversion error

Signed-off-by: Jake Correnti <[email protected]>

Author: jakecorrenti

src/libkrun/src/lib.rs

@@ -1193,7 +1193,10 @@ pub extern "C" fn krun_start_enter(ctx_id: u32) -> i32 {
             match receiver.recv() {
                 Err(e) => error!("Error in receiver: {:?}", e),
                 Ok((gpa, size, private)) => {
-                    let _ = convert_memory(&mapper_vmm, gpa, size, private);
+                    let ret = convert_memory(&mapper_vmm, gpa, size, private);
+                    if ret < 0 {
+                        panic!("{}", format!("ERROR CONVERTING MEMORY: {:x} {:x} {}", gpa, size, private));
+                    }
                 },
             }
         }).unwrap();
@@ -1227,74 +1230,139 @@ pub extern "C" fn krun_start_enter(ctx_id: u32) -> i32 {
     }
 }
 
+use vm_memory::Address;
+fn has_guest_memory(gpa: u64, size: u64, memfd_regions: &Vec<(vm_memory::GuestAddress, u64, u64)>) -> (bool, i64) {
+    for (a, s, i) in memfd_regions.iter() {
+        if gpa >= a.0 && gpa + size <= a.0 + *s {
+            return (true, *i as i64);
+        }
+    }
+    (false, -1)
+}
+
 use std::sync::Arc;
 use vm_memory::GuestAddress;
+use vm_memory::GuestMemory;
+use libc::fallocate;
+use libc::madvise;
+use libc::FALLOC_FL_KEEP_SIZE;
+use libc::FALLOC_FL_PUNCH_HOLE;
+use libc::MADV_DONTNEED;
+use libc::MADV_REMOVE;
 fn convert_memory(vmm: &Arc<Mutex<vmm::Vmm>>, gpa: u64, size: u64, to_private: bool) -> i32 {
+
+    // check to make sure the starting address is aligned with the page size
+    // check to make sure the size of the memory block is also aligned with the page size
+    // if size is non-positive then return -1
+    // retrive the memory region associated with the starting address and the size
+    // set the memory attributes to the desired value
+    // if converting memory from shared to private, then discard the ram range
+    // if converting from private to shared, then discard the guest memfd range
+
     let mut ret = -1;
-    let is_aligned = |n: u64, m: u64| -> bool {
-        ((n) % (m)) == 0
-    };
-    
-    if !is_aligned(gpa, 4096) || !is_aligned(size, 4096) {
+    let vmm = vmm.as_ref().lock().unwrap();
+    let guest_memfd_regions = vmm.guest_memfd_regions();
+
+    let page_size = 4096;
+
+    // check to make sure the starting address is aligned with the page size
+    // check to make sure the size of the memory block is also aligned with the page size
+    if (gpa % page_size) != 0 || (size % page_size) != 0 {
+        println!("ALIGNMENT ERROR");
         return -1;
     }
 
+    // if size is non-positive then return -1
     if size < 1 {
+        println!("BAD SIZE");
         return -1;
     }
 
-    use vm_memory::guest_memory::GuestMemory;
-    let binding = vmm.as_ref().lock().unwrap();
-    let region = binding.guest_memory().find_region(GuestAddress(gpa));
-
+    let region = vmm.guest_memory().find_region(GuestAddress(gpa));
     if let None = region {
-        // Ignore converting non-assigned region to shared
+        // ignore converting non-assigned region to shared
         //
         // TDX requires vMMIO region to be shared to inject #VE to guest. OVMF issues
-        // conservatively MapGPA(shared) on 32bit PCI MMIO region, and V vIO-APIC 0xFEC00000 4K
-        // page. OVMF assigns 32bit PCI MMIO region to [top of low memory: typically 2GB=0xC000000,
-        // 0xFC00000)
+        // conservatively MapGPA(shared) on 32bit PCI MMIO region, and vIO-APIC 0xFEC 4k page. OVMF
+        // assignes 32bit PCI MMIO region to [top of low memory: typically 2GB=0xC0000000,
+        // 0xFC000000)
         if !to_private {
+            println!("IGNORE NON-ASSIGNED");
             return 0;
         }
+
+        println!("ERROR FINDING REGION");
         return -1;
     }
-
     let region = region.unwrap();
 
-    use vm_memory::Address;
-    let mut region_has_guest_memfd = false;
-    for region in binding.guest_memfd_regions().iter() {
-       if gpa >= region.0.raw_value() && gpa + size <= region.0.raw_value() + region.1 {
-           region_has_guest_memfd = true;
-       }
-    }
+    // retrive the memory region associated with the starting address and the size
+    let (region_has_guest_memfd, memfd_id) = has_guest_memory(gpa, size, guest_memfd_regions);
 
+    // check to make sure there is a guest memfd backing
     if !region_has_guest_memfd {
-        println!("DOESNT HAVE MEMFD");
-
-        // Because vMMIO region must be shared, guest TD may convert vMMIO region to shared
-        // explicitly. Don't complain such case. See memory_region_type() for checking if the
-        // region is MMIO region.
-      // if !to_private && !is_ram && !is_ram_device && !is_rom && !is_romd {
-      //     ret = 0;
-      // } else {
-      //     println!("Convert non guest_memfd backed memory region (0x{:x}, 0x{:x}, {}", gpa, size, if to_private { "private" } else { "shared" });
-      // }
-
-        return ret;
+        panic!("cannot convert non guest_memfd backed memory region");
+        return -1;
     }
-    
+
+    // set the memory attributes to the desired value
     let attr = kvm_bindings::kvm_memory_attributes {
-        address: gpa,
-        size,
-        attributes: if to_private {
-        // KVM_MEMORY_ATTRIBUTE_PRIVATE,
-        1 << 3 } else { 0 },
-        flags: 0,
+       address: gpa,
+       size,
+       attributes: if to_private {
+       // KVM_MEMORY_ATTRIBUTE_PRIVATE,
+       1 << 3 } else { 0 },
+       flags: 0,
     };
 
-    binding.vm_fd().set_memory_attributes(attr).unwrap();
+    vmm.vm_fd().set_memory_attributes(attr).unwrap();
+
+    use vm_memory::GuestMemoryRegion;
+
+    let offset = gpa - region.start_addr().raw_value();
+    if to_private {
+        // ram_block_discard_range
+        println!("DISCARDING MEMORY RANGE WHEN CONVERTING SHARED TO PRIVATE");
+
+       let host_addr = region.get_host_address(vm_memory::MemoryRegionAddress(gpa)).unwrap();
+
+       // let host_addr = gpa + offset;
+       unsafe {
+           let _ret = madvise(
+               host_addr as *mut libc::c_void,
+               size.try_into().unwrap(),
+               // madv_remove,
+               MADV_DONTNEED,
+           );
+           println!("gpa: {:x} size: {:x}", gpa, size);
+           
+           if _ret < 0 {
+              println!("error discarding memory range: gpa: 0x{:x} size: 0x{:x} res: {}", gpa, size, std::io::Error::last_os_error()); 
+              return _ret;
+           }
+       }
+    } else {
+        // ram_block_discard_guest_memfd_range
+        println!("DISCARDING GUEST MEMFD MEMORY REGION WHEN CONVERTING PRIVATE TO SHARED");
+        unsafe {
+            let _ret = fallocate(
+                memfd_id as i32,
+                FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
+                offset as i64,
+                size as i64,
+            );
+
+            if _ret < 0 {
+               println!("error discarding guest memfd memory range: {}", std::io::Error::last_os_error()); 
+               return _ret;
+            }
+        }
+    }
 
+    0
+}
+
+fn ram_block_discard_range<T>(rb: &vm_memory::GuestRegionMmap<T>, start: u64, length: u64) -> i64 {
+    let mut ret = -1;
     ret
 }

src/rutabaga_gfx/src/generated/generate

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+# Copyright 2018 The ChromiumOS Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Generates bindings that are used gpu_renderer.
+
+A sysroot and virglrenderer source checkout is required. The defaults to the
+root directory.
+"""
+
+from __future__ import print_function
+import argparse
+import multiprocessing.pool
+import os
+import subprocess
+import sys
+import tempfile
+
+# Bright green.
+PASS_COLOR = "\033[1;32m"
+# Bright red.
+FAIL_COLOR = "\033[1;31m"
+# Default color.
+END_COLOR = "\033[0m"
+
+verbose = False
+
+
+def generate_module(
+    module_name, allowlist, blocklist, header, clang_args, lib_name, derive_default
+):
+    args = [
+        "bindgen",
+        "--no-layout-tests",
+        "--allowlist-function",
+        allowlist,
+        "--allowlist-var",
+        allowlist,
+        "--allowlist-type",
+        allowlist,
+        "--blocklist-function",
+        blocklist,
+        "--blocklist-item",
+        blocklist,
+        "--blocklist-type",
+        blocklist,
+        "--no-prepend-enum-name",
+        "-o",
+        module_name + "_bindings.rs",
+    ]
+
+    if lib_name:
+        args.extend(["--raw-line", '#[cfg(feature = "{}")]'.format(module_name)])
+        args.extend(["--raw-line", '#[link(name = "{}")] extern {{}}'.format(lib_name)])
+
+    if derive_default:
+        args.append("--with-derive-default")
+
+    args.extend([header, "--"])
+    args.extend(clang_args)
+
+    if verbose:
+        print(" ".join(args))
+
+    if subprocess.Popen(args).wait() == 0:
+        return "pass"
+    else:
+        return "bindgen failed"
+
+
+def download_virgl(src, dst, branch):
+    virgl_src = tempfile.TemporaryDirectory(prefix="virglrenderer-src")
+
+    args = ["git", "clone"]
+
+    if branch:
+        args.extend(["-b", branch])
+
+    args.extend([src, dst])
+
+    if verbose:
+        print(" ".join(args))
+
+    if subprocess.Popen(args).wait() == 0:
+        return True
+    else:
+        return False
+
+
+def get_parser():
+    """Gets the argument parser"""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--sysroot", default="/", help="sysroot directory (default=%(default)s)")
+    parser.add_argument("--virglrenderer", help="virglrenderer src dir/repo (default=%(default)s)")
+    parser.add_argument(
+        "--virgl_branch",
+        default="master",
+        help="virglrenderer branch name (default=%(default)s)",
+    )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="enable verbose output (default=%(default)s)",
+    )
+    return parser
+
+
+def main(argv):
+    global verbose
+    os.chdir(os.path.dirname(sys.argv[0]))
+    opts = get_parser().parse_args(argv)
+    if opts.verbose:
+        verbose = True
+
+    if opts.virglrenderer:
+        if "://" in opts.virglrenderer:
+            virgl_src_dir_temp = tempfile.TemporaryDirectory(prefix="virglrenderer-src")
+            virgl_src_dir = virgl_src_dir_temp.name
+            if not download_virgl(opts.virglrenderer, virgl_src_dir, opts.virgl_branch):
+                print("failed to clone '{}' to '{}'".format(virgl_src_dir, opts.virgl_branch))
+                sys.exit(1)
+        else:
+            virgl_src_dir = opts.virglrenderer
+
+        header = os.path.join(virgl_src_dir, "src/virglrenderer.h")
+    else:
+        header = os.path.join(opts.sysroot, "usr/include/virgl/virglrenderer.h")
+
+    clang_args = [
+        "-I",
+        os.path.join(opts.sysroot, "usr/include"),
+        "-D",
+        "VIRGL_RENDERER_UNSTABLE_APIS",
+    ]
+
+    modules = (
+        (
+            "virgl_renderer",
+            "(virgl|VIRGL)_.+",  # allowlist
+            ".*(va_list|debug_callback).*",  # blocklist
+            header,
+            clang_args,
+            "virglrenderer",
+            True,
+        ),
+    )
+
+    pool = multiprocessing.pool.Pool(len(modules))
+    results = pool.starmap(generate_module, modules, 1)
+
+    return_fail = False
+    print("---")
+    print("generate module summary:")
+    for module, result in zip(modules, results):
+        result_color = FAIL_COLOR
+        if result == "pass":
+            result_color = PASS_COLOR
+        else:
+            return_fail = True
+
+        print("%15s: %s%s%s" % (module[0], result_color, result, END_COLOR))
+
+    if return_fail:
+        sys.exit(1)
+
+    with open("mod.rs", "w") as f:
+        print("/* generated by generate.py */", file=f)
+        print("#![allow(dead_code)]", file=f)
+        print("#![allow(non_camel_case_types)]", file=f)
+        print("#![allow(non_snake_case)]", file=f)
+        print("#![allow(non_upper_case_globals)]", file=f)
+        print("pub mod virgl_debug_callback_bindings;", file=f)
+        for module in modules:
+            print("pub mod", module[0] + "_bindings;", file=f)
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

src/rutabaga_gfx/src/generated/generate

@@ -358,7 +358,7 @@ pub fn build_microvm(
     #[cfg(target_os = "macos")] _map_sender: Sender<MemoryMapping>,
 ) -> std::result::Result<Arc<Mutex<Vmm>>, StartMicrovmError> {
 
-    let mut guest_memfd_regions: Vec<(GuestAddress, u64)> = vec![];
+    let mut guest_memfd_regions: Vec<(vm_memory::GuestAddress, u64, u64)> = vec![];
 
     #[cfg(not(feature = "efi"))]
     let kernel_bundle = vm_resources
@@ -993,7 +993,7 @@ pub(crate) fn setup_vm(
     kvm: &KvmContext,
     guest_memory: &GuestMemoryMmap,
     tee_config: &TeeConfig,
-    guest_memfd_regions: &mut Vec<(GuestAddress, u64)>,
+    guest_memfd_regions: &mut Vec<(vm_memory::GuestAddress, u64, u64)>,
 ) -> std::result::Result<Vm, StartMicrovmError> {
     let mut vm = Vm::new(kvm.fd(), tee_config)
         .map_err(Error::Vm)