diff --git a/Bender.yml b/Bender.yml index 139ca8c321..836b4862b0 100644 --- a/Bender.yml +++ b/Bender.yml @@ -50,6 +50,26 @@ sources: - core/mmu_sv39/ptw.sv - core/cva6_accel_first_pass_decoder_stub.sv + - target: cv64a6_imafdch_sv39 + files: + - core/include/cv64a6_imafdch_sv39_config_pkg.sv + - core/include/riscv_pkg.sv + - core/include/ariane_pkg.sv + - core/mmu_sv39x4/cva6_tlb_sv39x4.sv + - core/mmu_sv39x4/cva6_mmu_sv39x4.sv + - core/mmu_sv39x4/cva6_ptw_sv39x4.sv + - core/cva6_accel_first_pass_decoder_stub.sv + + - target: cv64a6_imafdch_sv39_wb + files: + - core/include/cv64a6_imafdch_sv39_wb_config_pkg.sv + - core/include/riscv_pkg.sv + - core/include/ariane_pkg.sv + - core/mmu_sv39x4/cva6_tlb_sv39x4.sv + - core/mmu_sv39x4/cva6_mmu_sv39x4.sv + - core/mmu_sv39x4/cva6_ptw_sv39x4.sv + - core/cva6_accel_first_pass_decoder_stub.sv + - target: cv32a6_imac_sv0 files: - core/include/cv32a6_imac_sv0_config_pkg.sv diff --git a/Flist.ariane b/Flist.ariane index 11b4659b31..8046385dff 100644 --- a/Flist.ariane +++ b/Flist.ariane @@ -84,11 +84,13 @@ core/load_unit.sv core/load_store_unit.sv core/lsu_bypass.sv core/mmu_sv39/mmu.sv +core/mmu_sv39x4/cva6_mmu_sv39x4.sv core/mult.sv core/multiplier.sv core/serdiv.sv core/perf_counters.sv core/mmu_sv39/ptw.sv +core/mmu_sv39x4/ptw_sv39x4.sv core/ariane_regfile_ff.sv core/re_name.sv core/scoreboard.sv @@ -96,6 +98,7 @@ core/store_buffer.sv core/amo_buffer.sv core/store_unit.sv core/mmu_sv39/tlb.sv +core/mmu_sv39x4/tlb_sv39x4.sv core/commit_stage.sv core/cache_subsystem/wt_dcache_ctrl.sv core/cache_subsystem/wt_dcache_mem.sv diff --git a/ci/build-hyp-riscv-tests.sh b/ci/build-hyp-riscv-tests.sh new file mode 100755 index 0000000000..bbabd3a98d --- /dev/null +++ b/ci/build-hyp-riscv-tests.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +VERSION="920c1379cf6ca2374c6c5207dca425a933d8c7fd" + +cd $ROOT/tmp + +if [ -z ${NUM_JOBS} ]; then + NUM_JOBS=1 +fi + +[ -d $ROOT/tmp/riscv-hyp-tests ] || git clone https://github.com/ninolomata/riscv-hyp-tests +cd riscv-hyp-tests +git checkout $VERSION +git submodule update --init --recursive +make PLAT=cva6 \ No newline at end of file diff --git a/core/Flist.cva6 b/core/Flist.cva6 index 83d2479309..e7c61c338d 100644 --- a/core/Flist.cva6 +++ b/core/Flist.cva6 @@ -189,6 +189,11 @@ ${CVA6_REPO_DIR}/core/mmu_sv39/mmu.sv ${CVA6_REPO_DIR}/core/mmu_sv39/ptw.sv ${CVA6_REPO_DIR}/core/mmu_sv39/tlb.sv +// MMU Sv39x4 +${CVA6_REPO_DIR}/core/mmu_sv39x4/cva6_mmu_sv39x4.sv +${CVA6_REPO_DIR}/core/mmu_sv39x4/cva6_ptw_sv39x4.sv +${CVA6_REPO_DIR}/core/mmu_sv39x4/cva6_tlb_sv39x4.sv + // MMU Sv32 ${CVA6_REPO_DIR}/core/mmu_sv32/cva6_mmu_sv32.sv ${CVA6_REPO_DIR}/core/mmu_sv32/cva6_ptw_sv32.sv diff --git a/core/acc_dispatcher.sv b/core/acc_dispatcher.sv index ac479c4b08..d645c25219 100644 --- a/core/acc_dispatcher.sv +++ b/core/acc_dispatcher.sv @@ -297,22 +297,29 @@ module acc_dispatcher logic acc_st_disp; // Unpack the accelerator response - assign acc_trans_id_o = acc_resp_i.trans_id; - assign acc_result_o = acc_resp_i.result; - assign acc_valid_o = acc_resp_i.resp_valid; - assign acc_exception_o = '{cause: riscv::ILLEGAL_INSTR, tval : '0, valid: acc_resp_i.error}; - assign acc_fflags_valid_o = acc_resp_i.fflags_valid; - assign acc_fflags_o = acc_resp_i.fflags; + assign acc_trans_id_o = acc_resp_i.trans_id; + assign acc_result_o = acc_resp_i.result; + assign acc_valid_o = acc_resp_i.resp_valid; + assign acc_exception_o = '{ + cause: riscv::ILLEGAL_INSTR, + tval : '0, + tval2 : '0, + tinst : '0, + gva : '0, + valid: acc_resp_i.error + }; + assign acc_fflags_valid_o = acc_resp_i.fflags_valid; + assign acc_fflags_o = acc_resp_i.fflags; // Always ready to receive responses assign acc_req_o.resp_ready = 1'b1; // Signal dispatched load/store to issue stage - assign acc_ld_disp = acc_req_valid && (acc_insn_queue_o.operation == ACCEL_OP_LOAD); - assign acc_st_disp = acc_req_valid && (acc_insn_queue_o.operation == ACCEL_OP_STORE); + assign acc_ld_disp = acc_req_valid && (acc_insn_queue_o.operation == ACCEL_OP_LOAD); + assign acc_st_disp = acc_req_valid && (acc_insn_queue_o.operation == ACCEL_OP_STORE); // Cache invalidation - assign inval_valid_o = acc_resp_i.inval_valid; - assign inval_addr_o = acc_resp_i.inval_addr; + assign inval_valid_o = acc_resp_i.inval_valid; + assign inval_addr_o = acc_resp_i.inval_addr; /************************** * Accelerator commit * diff --git a/core/branch_unit.sv b/core/branch_unit.sv index 2b764d62a8..43eabdb1e3 100644 --- a/core/branch_unit.sv +++ b/core/branch_unit.sv @@ -23,6 +23,8 @@ module branch_unit #( input logic clk_i, // Asynchronous reset active low - SUBSYSTEM input logic rst_ni, + // Virtualization mode state - CSR_REGFILE + input logic v_i, // Debug mode state - CSR_REGFILE input logic debug_mode_i, // FU data needed to execute instruction - ISSUE_STAGE @@ -115,6 +117,9 @@ module branch_unit #( if (CVA6Cfg.TvalEn) branch_exception_o.tval = {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{pc_i[CVA6Cfg.VLEN-1]}}, pc_i}; else branch_exception_o.tval = '0; + branch_exception_o.tval2 = {CVA6Cfg.GPLEN{1'b0}}; + branch_exception_o.tinst = '0; + branch_exception_o.gva = CVA6Cfg.RVH ? v_i : 1'b0; // Only throw instruction address misaligned exception if this is indeed a `taken` conditional branch or // an unconditional jump if (branch_valid_i && (target_address[0] || (!CVA6Cfg.RVC && target_address[1])) && jump_taken) begin diff --git a/core/commit_stage.sv b/core/commit_stage.sv index a69e4aff82..7f2f9e914e 100644 --- a/core/commit_stage.sv +++ b/core/commit_stage.sv @@ -81,7 +81,9 @@ module commit_stage // Request a pipeline flush - CONTROLLER output logic flush_commit_o, // Flush TLBs and pipeline - CONTROLLER - output logic sfence_vma_o + output logic sfence_vma_o, + output logic hfence_vvma_o, + output logic hfence_gvma_o ); // ila_0 i_ila_commit ( @@ -142,6 +144,8 @@ module commit_stage fence_i_o = 1'b0; fence_o = 1'b0; sfence_vma_o = 1'b0; + hfence_vvma_o = 1'b0; + hfence_gvma_o = 1'b0; csr_write_fflags_o = 1'b0; flush_commit_o = 1'b0; @@ -212,6 +216,30 @@ module commit_stage commit_ack_o[0] = no_st_pending_i; end // ------------------ + // HFENCE.VVMA Logic + // ------------------ + // hfence.vvma is idempotent so we can safely re-execute it after returning + // from interrupt service routine + // check if this instruction was a HFENCE_VVMA + if (CVA6Cfg.RVH && commit_instr_i[0].op == HFENCE_VVMA) begin + // no store pending so we can flush the TLBs and pipeline + hfence_vvma_o = no_st_pending_i; + // wait for the store buffer to drain until flushing the pipeline + commit_ack_o[0] = no_st_pending_i; + end + // ------------------ + // HFENCE.GVMA Logic + // ------------------ + // hfence.gvma is idempotent so we can safely re-execute it after returning + // from interrupt service routine + // check if this instruction was a HFENCE_GVMA + if (CVA6Cfg.RVH && commit_instr_i[0].op == HFENCE_GVMA) begin + // no store pending so we can flush the TLBs and pipeline + hfence_gvma_o = no_st_pending_i; + // wait for the store buffer to drain until flushing the pipeline + commit_ack_o[0] = no_st_pending_i; + end + // ------------------ // FENCE.I Logic // ------------------ // fence.i is idempotent so we can safely re-execute it after returning @@ -308,6 +336,10 @@ module commit_stage exception_o.valid = 1'b0; exception_o.cause = '0; exception_o.tval = '0; + exception_o.tval2 = '0; + exception_o.tinst = '0; + exception_o.gva = 1'b0; + // we need a valid instruction in the commit stage if (commit_instr_i[0].valid) begin // ------------------------ @@ -319,6 +351,11 @@ module commit_stage // the instruction bits from the ID stage. If a earlier exception happened we don't care // as we will overwrite it anyway in the next IF bl exception_o.tval = commit_instr_i[0].ex.tval; + if (CVA6Cfg.RVH) begin + exception_o.tinst = commit_instr_i[0].ex.tinst; + exception_o.tval2 = commit_instr_i[0].ex.tval2; + exception_o.gva = commit_instr_i[0].ex.gva; + end end // ------------------------ // Earlier Exceptions diff --git a/core/controller.sv b/core/controller.sv index b835bec1cd..2ce2a50d53 100644 --- a/core/controller.sv +++ b/core/controller.sv @@ -23,6 +23,8 @@ module controller input logic clk_i, // Asynchronous reset active low - SUBSYSTEM input logic rst_ni, + // Virtualization mode - CSR_REGFILE + input logic v_i, // Set PC om PC Gen - FRONTEND output logic set_pc_commit_o, // Flush the IF stage - FRONTEND @@ -43,6 +45,8 @@ module controller input logic flush_dcache_ack_i, // Flush TLBs - EX_STAGE output logic flush_tlb_o, + output logic flush_tlb_vvma_o, + output logic flush_tlb_gvma_o, // Halt request from CSR (WFI instruction) - CSR_REGFILE input logic halt_csr_i, // Halt request from accelerator dispatcher - ACC_DISPATCHER @@ -65,6 +69,8 @@ module controller input logic fence_i, // We got an instruction to flush the TLBs and pipeline - COMMIT_STAGE input logic sfence_vma_i, + input logic hfence_vvma_i, + input logic hfence_gvma_i, // Flush request from commit stage - COMMIT_STAGE input logic flush_commit_i, // Flush request from accelerator - ACC_DISPATCHER @@ -88,6 +94,8 @@ module controller flush_dcache = 1'b0; flush_icache_o = 1'b0; flush_tlb_o = 1'b0; + flush_tlb_vvma_o = 1'b0; + flush_tlb_gvma_o = 1'b0; flush_bp_o = 1'b0; // ------------ // Mis-predict @@ -157,7 +165,34 @@ module controller flush_id_o = 1'b1; flush_ex_o = 1'b1; - flush_tlb_o = 1'b1; + if (CVA6Cfg.RVH && v_i) flush_tlb_vvma_o = 1'b1; + else flush_tlb_o = 1'b1; + end + + // --------------------------------- + // HFENCE.VVMA + // --------------------------------- + if (CVA6Cfg.RVH && hfence_vvma_i) begin + set_pc_commit_o = 1'b1; + flush_if_o = 1'b1; + flush_unissued_instr_o = 1'b1; + flush_id_o = 1'b1; + flush_ex_o = 1'b1; + + flush_tlb_vvma_o = 1'b1; + end + + // --------------------------------- + // HFENCE.GVMA + // --------------------------------- + if (CVA6Cfg.RVH && hfence_gvma_i) begin + set_pc_commit_o = 1'b1; + flush_if_o = 1'b1; + flush_unissued_instr_o = 1'b1; + flush_id_o = 1'b1; + flush_ex_o = 1'b1; + + flush_tlb_gvma_o = 1'b1; end // --------------------------------- diff --git a/core/csr_regfile.sv b/core/csr_regfile.sv index cbbc3073b4..464e18fc09 100644 --- a/core/csr_regfile.sv +++ b/core/csr_regfile.sv @@ -22,6 +22,7 @@ module csr_regfile parameter type scoreboard_entry_t = logic, parameter type rvfi_probes_csr_t = logic, parameter int AsidWidth = 1, + parameter int VmidWidth = 1, parameter int unsigned MHPMCounterNum = 6 ) ( // Subsystem Clock - SUBSYSTEM @@ -71,12 +72,16 @@ module csr_regfile output logic [CVA6Cfg.VLEN-1:0] trap_vector_base_o, // Current privilege level the CPU is in - EX_STAGE output riscv::priv_lvl_t priv_lvl_o, + // Current virtualization mode state the CPU is in - EX_STAGE + output logic v_o, // Imprecise FP exception from the accelerator (fcsr.fflags format) - ACC_DISPATCHER input logic [4:0] acc_fflags_ex_i, // An FP exception from the accelerator occurred - ACC_DISPATCHER input logic acc_fflags_ex_valid_i, // Floating point extension status - ID_STAGE output riscv::xs_t fs_o, + // Floating point extension virtual status - ID_STAGE + output riscv::xs_t vfs_o, // Floating-Point Accured Exceptions - COMMIT_STAGE output logic [4:0] fflags_o, // Floating-Point Dynamic Rounding Mode - EX_STAGE @@ -89,18 +94,38 @@ module csr_regfile output irq_ctrl_t irq_ctrl_o, // Enable virtual address translation - EX_STAGE output logic en_translation_o, + // Enable G-Stage address translation - EX_STAGE + output logic en_g_translation_o, // Enable virtual address translation for load and stores - EX_STAGE output logic en_ld_st_translation_o, + // Enable G-Stage address translation for load and stores - EX_STAGE + output logic en_ld_st_g_translation_o, // Privilege level at which load and stores should happen - EX_STAGE output riscv::priv_lvl_t ld_st_priv_lvl_o, + // Virtualization mode at which load and stores should happen - EX_STAGE + output logic ld_st_v_o, + // Current instruction is a Hypervisor Load/Store Instruction - EX_STAGE + input logic csr_hs_ld_st_inst_i, // Supervisor User Memory - EX_STAGE output logic sum_o, + // Virtual Supervisor User Memory - EX_STAGE + output logic vs_sum_o, // Make Executable Readable - EX_STAGE output logic mxr_o, + // Make Executable Readable for VS-mode - EX_STAGE + output logic vmxr_o, // TO_BE_COMPLETED - EX_STAGE output logic [CVA6Cfg.PPNW-1:0] satp_ppn_o, // TO_BE_COMPLETED - EX_STAGE output logic [AsidWidth-1:0] asid_o, + // TO_BE_COMPLETED - EX_STAGE + output logic [CVA6Cfg.PPNW-1:0] vsatp_ppn_o, + // TO_BE_COMPLETED - EX_STAGE + output logic [CVA6Cfg.ASID_WIDTH-1:0] vs_asid_o, + // TO_BE_COMPLETED - EX_STAGE + output logic [CVA6Cfg.PPNW-1:0] hgatp_ppn_o, + // TO_BE_COMPLETED - EX_STAGE + output logic [CVA6Cfg.VMID_WIDTH-1:0] vmid_o, // external interrupt in - SUBSYSTEM input logic [1:0] irq_i, // inter processor interrupt -> connected to machine mode sw - SUBSYSTEM @@ -113,8 +138,12 @@ module csr_regfile output logic tvm_o, // timeout wait - ID_STAGE output logic tw_o, + // virtual timeout wait - ID_STAGE + output logic vtw_o, // trap sret - ID_STAGE output logic tsr_o, + // hypervisor user mode - ID_STAGE + output logic hu_o, // we are in debug mode -> that will change some decoding - EX_STAGE output logic debug_mode_o, // we are in single-step mode - COMMIT_STAGE @@ -145,6 +174,13 @@ module csr_regfile ); localparam logic [63:0] SMODE_STATUS_READ_MASK = ariane_pkg::smode_status_read_mask(CVA6Cfg); + localparam logic [63:0] HS_DELEG_INTERRUPTS = { + {32{1'b0}}, ariane_pkg::hs_deleg_interrupts(CVA6Cfg) + }; + localparam logic [63:0] VS_DELEG_INTERRUPTS = { + {32{1'b0}}, ariane_pkg::vs_deleg_interrupts(CVA6Cfg) + }; + localparam int SELECT_COUNTER_WIDTH = CVA6Cfg.IS_XLEN64 ? 6 : 5; typedef struct packed { logic [CVA6Cfg.ModeW-1:0] mode; @@ -152,13 +188,23 @@ module csr_regfile logic [CVA6Cfg.PPNW-1:0] ppn; } satp_t; + typedef struct packed { + logic [CVA6Cfg.ModeW-1:0] mode; + logic [1:0] warl0; + logic [CVA6Cfg.VMIDW-1:0] vmid; + logic [CVA6Cfg.PPNW-1:0] ppn; + } hgatp_t; + // internal signal to keep track of access exceptions logic read_access_exception, update_access_exception, privilege_violation; + logic virtual_read_access_exception, virtual_update_access_exception, virtual_privilege_violation; logic csr_we, csr_read; logic [CVA6Cfg.XLEN-1:0] csr_wdata, csr_rdata; riscv::priv_lvl_t trap_to_priv_lvl; + logic trap_to_v; // register for enabling load store address translation, this is critical, hence the register logic en_ld_st_translation_d, en_ld_st_translation_q; + logic en_ld_st_g_translation_d, en_ld_st_g_translation_q; logic mprv; logic mret; // return from M-mode exception logic sret; // return from S-mode exception @@ -166,12 +212,19 @@ module csr_regfile // CSR write causes us to mark the FPU state as dirty logic dirty_fp_state_csr; riscv::mstatus_rv_t mstatus_q, mstatus_d; + riscv::hstatus_rv_t hstatus_q, hstatus_d; + riscv::mstatus_rv_t vsstatus_q, vsstatus_d; logic [CVA6Cfg.XLEN-1:0] mstatus_extended; + logic [CVA6Cfg.XLEN-1:0] vsstatus_extended; satp_t satp_q, satp_d; + satp_t vsatp_q, vsatp_d; + hgatp_t hgatp_q, hgatp_d; riscv::dcsr_t dcsr_q, dcsr_d; riscv::csr_t csr_addr; + riscv::csr_t conv_csr_addr; // privilege level register riscv::priv_lvl_t priv_lvl_d, priv_lvl_q; + logic v_q, v_d; // virtualization mode // we are in debug logic debug_mode_q, debug_mode_d; logic mtvec_rst_load_q; // used to determine whether we came out of reset @@ -189,6 +242,8 @@ module csr_regfile logic [CVA6Cfg.XLEN-1:0] mepc_q, mepc_d; logic [CVA6Cfg.XLEN-1:0] mcause_q, mcause_d; logic [CVA6Cfg.XLEN-1:0] mtval_q, mtval_d; + logic [CVA6Cfg.XLEN-1:0] mtinst_q, mtinst_d; + logic [CVA6Cfg.XLEN-1:0] mtval2_q, mtval2_d; logic fiom_d, fiom_q; logic [CVA6Cfg.XLEN-1:0] stvec_q, stvec_d; @@ -197,6 +252,20 @@ module csr_regfile logic [CVA6Cfg.XLEN-1:0] sepc_q, sepc_d; logic [CVA6Cfg.XLEN-1:0] scause_q, scause_d; logic [CVA6Cfg.XLEN-1:0] stval_q, stval_d; + + logic [CVA6Cfg.XLEN-1:0] hedeleg_q, hedeleg_d; + logic [CVA6Cfg.XLEN-1:0] hideleg_q, hideleg_d; + logic [CVA6Cfg.XLEN-1:0] hcounteren_q, hcounteren_d; + logic [CVA6Cfg.XLEN-1:0] hgeie_q, hgeie_d; + logic [CVA6Cfg.XLEN-1:0] htinst_q, htinst_d; + logic [CVA6Cfg.XLEN-1:0] htval_q, htval_d; + + logic [CVA6Cfg.XLEN-1:0] vstvec_q, vstvec_d; + logic [CVA6Cfg.XLEN-1:0] vsscratch_q, vsscratch_d; + logic [CVA6Cfg.XLEN-1:0] vsepc_q, vsepc_d; + logic [CVA6Cfg.XLEN-1:0] vscause_q, vscause_d; + logic [CVA6Cfg.XLEN-1:0] vstval_q, vstval_d; + logic [CVA6Cfg.XLEN-1:0] dcache_q, dcache_d; logic [CVA6Cfg.XLEN-1:0] icache_q, icache_d; logic [CVA6Cfg.XLEN-1:0] acc_cons_q, acc_cons_d; @@ -216,6 +285,7 @@ module csr_regfile | (CVA6Cfg.XLEN'(CVA6Cfg.RVC) << 2) // C - Compressed extension | (CVA6Cfg.XLEN'(CVA6Cfg.RVD) << 3) // D - Double precision floating-point extension | (CVA6Cfg.XLEN'(CVA6Cfg.RVF) << 5) // F - Single precision floating-point extension + | (CVA6Cfg.XLEN'(CVA6Cfg.RVH) << 7) // H - Hypervisor extension | (CVA6Cfg.XLEN'(1) << 8) // I - RV32I/64I/128I base ISA | (CVA6Cfg.XLEN'(1) << 12) // M - Integer Multiply/Divide extension | (CVA6Cfg.XLEN'(0) << 13) // N - User level interrupts supported @@ -233,40 +303,53 @@ module csr_regfile // Assignments // ---------------- assign csr_addr = riscv::csr_t'(csr_addr_i); + assign conv_csr_addr = (CVA6Cfg.RVH) ? riscv::convert_vs_access_csr( + (riscv::csr_t'(csr_addr_i)), v_q + ) : csr_addr; assign fs_o = mstatus_q.fs; + assign vfs_o = (CVA6Cfg.RVH) ? vsstatus_q.fs : riscv::Off; assign vs_o = mstatus_q.vs; // ---------------- // CSR Read logic // ---------------- assign mstatus_extended = CVA6Cfg.IS_XLEN64 ? mstatus_q[CVA6Cfg.XLEN-1:0] : {mstatus_q.sd, mstatus_q.wpri3[7:0], mstatus_q[22:0]}; - + if (CVA6Cfg.RVH) begin + if (CVA6Cfg.IS_XLEN64) begin : gen_vsstatus_64read + assign vsstatus_extended = vsstatus_q[CVA6Cfg.XLEN-1:0]; + end else begin : gen_vsstatus_32read + assign vsstatus_extended = {vsstatus_q.sd, vsstatus_q.wpri3[7:0], vsstatus_q[22:0]}; + end + end else begin + assign vsstatus_extended = '0; + end always_comb begin : csr_read_process // a read access exception can only occur if we attempt to read a CSR which does not exist read_access_exception = 1'b0; + virtual_read_access_exception = 1'b0; csr_rdata = '0; perf_addr_o = csr_addr.address[11:0]; index = '0; if (csr_read) begin - unique case (csr_addr.address) + unique case (conv_csr_addr.address) riscv::CSR_FFLAGS: begin - if (CVA6Cfg.FpPresent) begin + if (CVA6Cfg.FpPresent && !(mstatus_q.fs == riscv::Off || (CVA6Cfg.RVH && v_q && vsstatus_q.fs == riscv::Off))) begin csr_rdata = {{CVA6Cfg.XLEN - 5{1'b0}}, fcsr_q.fflags}; end else begin read_access_exception = 1'b1; end end riscv::CSR_FRM: begin - if (CVA6Cfg.FpPresent) begin + if (CVA6Cfg.FpPresent && !(mstatus_q.fs == riscv::Off || (CVA6Cfg.RVH && v_q && vsstatus_q.fs == riscv::Off))) begin csr_rdata = {{CVA6Cfg.XLEN - 3{1'b0}}, fcsr_q.frm}; end else begin read_access_exception = 1'b1; end end riscv::CSR_FCSR: begin - if (CVA6Cfg.FpPresent) begin + if (CVA6Cfg.FpPresent && !(mstatus_q.fs == riscv::Off || (CVA6Cfg.RVH && v_q && vsstatus_q.fs == riscv::Off))) begin csr_rdata = {{CVA6Cfg.XLEN - 8{1'b0}}, fcsr_q.frm, fcsr_q.fflags}; end else begin read_access_exception = 1'b1; @@ -274,7 +357,7 @@ module csr_regfile end // non-standard extension riscv::CSR_FTRAN: begin - if (CVA6Cfg.FpPresent) begin + if (CVA6Cfg.FpPresent && !(mstatus_q.fs == riscv::Off || (CVA6Cfg.RVH && v_q && vsstatus_q.fs == riscv::Off))) begin csr_rdata = {{CVA6Cfg.XLEN - 7{1'b0}}, fcsr_q.fprec}; end else begin read_access_exception = 1'b1; @@ -298,16 +381,53 @@ module csr_regfile riscv::CSR_TDATA1: read_access_exception = 1'b1; // not implemented riscv::CSR_TDATA2: read_access_exception = 1'b1; // not implemented riscv::CSR_TDATA3: read_access_exception = 1'b1; // not implemented + riscv::CSR_VSSTATUS: + if (CVA6Cfg.RVH) csr_rdata = vsstatus_extended; + else read_access_exception = 1'b1; + riscv::CSR_VSIE: + if (CVA6Cfg.RVH) + csr_rdata = (mie_q & VS_DELEG_INTERRUPTS[CVA6Cfg.XLEN-1:0] & hideleg_q) >> 1; + else read_access_exception = 1'b1; + riscv::CSR_VSIP: + if (CVA6Cfg.RVH) + csr_rdata = (mip_q & VS_DELEG_INTERRUPTS[CVA6Cfg.XLEN-1:0] & hideleg_q) >> 1; + else read_access_exception = 1'b1; + riscv::CSR_VSTVEC: + if (CVA6Cfg.RVH) csr_rdata = vstvec_q; + else read_access_exception = 1'b1; + riscv::CSR_VSSCRATCH: + if (CVA6Cfg.RVH) csr_rdata = vsscratch_q; + else read_access_exception = 1'b1; + riscv::CSR_VSEPC: + if (CVA6Cfg.RVH) csr_rdata = vsepc_q; + else read_access_exception = 1'b1; + riscv::CSR_VSCAUSE: + if (CVA6Cfg.RVH) csr_rdata = vscause_q; + else read_access_exception = 1'b1; + riscv::CSR_VSTVAL: + if (CVA6Cfg.RVH) csr_rdata = vstval_q; + else read_access_exception = 1'b1; + riscv::CSR_VSATP: + // intercept reads to VSATP if in VS-Mode and VTVM is enabled + if (CVA6Cfg.RVH) begin + if (priv_lvl_o == riscv::PRIV_LVL_S && hstatus_q.vtvm && v_q) + virtual_read_access_exception = 1'b1; + else csr_rdata = vsatp_q; + end else begin + read_access_exception = 1'b1; + end // supervisor registers riscv::CSR_SSTATUS: begin if (CVA6Cfg.RVS) csr_rdata = mstatus_extended & SMODE_STATUS_READ_MASK[CVA6Cfg.XLEN-1:0]; else read_access_exception = 1'b1; end riscv::CSR_SIE: - if (CVA6Cfg.RVS) csr_rdata = mie_q & mideleg_q; + if (CVA6Cfg.RVS) + csr_rdata = (CVA6Cfg.RVH) ? mie_q & mideleg_q & ~HS_DELEG_INTERRUPTS[CVA6Cfg.XLEN-1:0] : mie_q & mideleg_q; else read_access_exception = 1'b1; riscv::CSR_SIP: - if (CVA6Cfg.RVS) csr_rdata = mip_q & mideleg_q; + if (CVA6Cfg.RVS) + csr_rdata = (CVA6Cfg.RVH) ? mip_q & mideleg_q & ~HS_DELEG_INTERRUPTS[CVA6Cfg.XLEN-1:0] : mip_q & mideleg_q; else read_access_exception = 1'b1; riscv::CSR_STVEC: if (CVA6Cfg.RVS) csr_rdata = stvec_q; @@ -339,6 +459,59 @@ module csr_regfile read_access_exception = 1'b1; end end + riscv::CSR_SENVCFG: + if (CVA6Cfg.RVS) csr_rdata = '0 | fiom_q; + else read_access_exception = 1'b1; + // hypervisor mode registers + riscv::CSR_HSTATUS: + if (CVA6Cfg.RVH) csr_rdata = hstatus_q[CVA6Cfg.XLEN-1:0]; + else read_access_exception = 1'b1; + riscv::CSR_HEDELEG: + if (CVA6Cfg.RVH) csr_rdata = hedeleg_q; + else read_access_exception = 1'b1; + riscv::CSR_HIDELEG: + if (CVA6Cfg.RVH) csr_rdata = hideleg_q; + else read_access_exception = 1'b1; + riscv::CSR_HIE: + if (CVA6Cfg.RVH) csr_rdata = mie_q & HS_DELEG_INTERRUPTS[CVA6Cfg.XLEN-1:0]; + else read_access_exception = 1'b1; + riscv::CSR_HIP: + if (CVA6Cfg.RVH) csr_rdata = mip_q & HS_DELEG_INTERRUPTS[CVA6Cfg.XLEN-1:0]; + else read_access_exception = 1'b1; + riscv::CSR_HVIP: + if (CVA6Cfg.RVH) csr_rdata = mip_q & VS_DELEG_INTERRUPTS[CVA6Cfg.XLEN-1:0]; + else read_access_exception = 1'b1; + riscv::CSR_HCOUNTEREN: + if (CVA6Cfg.RVH) csr_rdata = hcounteren_q; + else read_access_exception = 1'b1; + riscv::CSR_HTVAL: + if (CVA6Cfg.RVH) csr_rdata = htval_q; + else read_access_exception = 1'b1; + riscv::CSR_HTINST: + if (CVA6Cfg.RVH) csr_rdata = htinst_q; + else read_access_exception = 1'b1; + riscv::CSR_HGEIE: + if (CVA6Cfg.RVH) csr_rdata = '0; + else read_access_exception = 1'b1; + riscv::CSR_HGEIP: + if (CVA6Cfg.RVH) csr_rdata = '0; + else read_access_exception = 1'b1; + riscv::CSR_HENVCFG: + if (CVA6Cfg.RVH) csr_rdata = '0 | {{CVA6Cfg.XLEN - 1{1'b0}}, fiom_q}; + else read_access_exception = 1'b1; + riscv::CSR_HGATP: begin + if (CVA6Cfg.RVH) begin + // intercept reads to HGATP if in HS-Mode and TVM is enabled + if (priv_lvl_o == riscv::PRIV_LVL_S && !v_q && mstatus_q.tvm) begin + read_access_exception = 1'b1; + end else begin + csr_rdata = hgatp_q; + end + end else begin + read_access_exception = 1'b1; + end + end + // machine mode registers riscv::CSR_MSTATUS: csr_rdata = mstatus_extended; riscv::CSR_MSTATUSH: @@ -358,6 +531,12 @@ module csr_regfile riscv::CSR_MEPC: csr_rdata = mepc_q; riscv::CSR_MCAUSE: csr_rdata = mcause_q; riscv::CSR_MTVAL: csr_rdata = mtval_q; + riscv::CSR_MTINST: + if (CVA6Cfg.RVH) csr_rdata = mtinst_q; + else read_access_exception = 1'b1; + riscv::CSR_MTVAL2: + if (CVA6Cfg.RVH) csr_rdata = mtval2_q; + else read_access_exception = 1'b1; riscv::CSR_MIP: csr_rdata = mip_q; riscv::CSR_MENVCFG: begin if (CVA6Cfg.RVU) csr_rdata = '0 | fiom_q; @@ -608,10 +787,14 @@ module csr_regfile logic [CVA6Cfg.XLEN-1:0] mask; always_comb begin : csr_update automatic satp_t satp; + automatic satp_t vsatp; + automatic hgatp_t hgatp; automatic logic [63:0] instret; satp = satp_q; + hgatp = hgatp_q; + vsatp = vsatp_q; instret = instret_q; mcountinhibit_d = mcountinhibit_q; @@ -633,24 +816,28 @@ module csr_regfile else cycle_d = cycle_q; end - eret_o = 1'b0; - flush_o = 1'b0; - update_access_exception = 1'b0; + eret_o = 1'b0; + flush_o = 1'b0; + update_access_exception = 1'b0; + virtual_update_access_exception = 1'b0; - set_debug_pc_o = 1'b0; + set_debug_pc_o = 1'b0; - perf_we_o = 1'b0; - perf_data_o = 'b0; + perf_we_o = 1'b0; + perf_data_o = 'b0; - fcsr_d = fcsr_q; + fcsr_d = fcsr_q; - priv_lvl_d = priv_lvl_q; - debug_mode_d = debug_mode_q; - dcsr_d = dcsr_q; - dpc_d = dpc_q; - dscratch0_d = dscratch0_q; - dscratch1_d = dscratch1_q; - mstatus_d = mstatus_q; + priv_lvl_d = priv_lvl_q; + v_d = v_q; + debug_mode_d = debug_mode_q; + dcsr_d = dcsr_q; + dpc_d = dpc_q; + dscratch0_d = dscratch0_q; + dscratch1_d = dscratch1_q; + mstatus_d = mstatus_q; + hstatus_d = hstatus_q; + vsstatus_d = vsstatus_q; // check whether we come out of reset // this is a workaround. some tools have issues @@ -664,40 +851,58 @@ module csr_regfile mtvec_d = mtvec_q; end - medeleg_d = medeleg_q; - mideleg_d = mideleg_q; - mip_d = mip_q; - mie_d = mie_q; - mepc_d = mepc_q; - mcause_d = mcause_q; - mcounteren_d = mcounteren_q; - mscratch_d = mscratch_q; - mtval_d = mtval_q; - fiom_d = fiom_q; - dcache_d = dcache_q; - icache_d = icache_q; - acc_cons_d = acc_cons_q; - - sepc_d = sepc_q; - scause_d = scause_q; - stvec_d = stvec_q; - scounteren_d = scounteren_q; - sscratch_d = sscratch_q; - stval_d = stval_q; - satp_d = satp_q; - - en_ld_st_translation_d = en_ld_st_translation_q; - dirty_fp_state_csr = 1'b0; - - pmpcfg_d = pmpcfg_q; - pmpaddr_d = pmpaddr_q; + medeleg_d = medeleg_q; + mideleg_d = mideleg_q; + mip_d = mip_q; + mie_d = mie_q; + mepc_d = mepc_q; + mcause_d = mcause_q; + mcounteren_d = mcounteren_q; + mscratch_d = mscratch_q; + mtval_d = mtval_q; + mtinst_d = mtinst_q; + mtval2_d = mtval2_q; + fiom_d = fiom_q; + dcache_d = dcache_q; + icache_d = icache_q; + acc_cons_d = acc_cons_q; + + vsstatus_d = vsstatus_q; + vstvec_d = vstvec_q; + vsscratch_d = vsscratch_q; + vsepc_d = vsepc_q; + vscause_d = vscause_q; + vstval_d = vstval_q; + vsatp_d = vsatp_q; + + sepc_d = sepc_q; + scause_d = scause_q; + stvec_d = stvec_q; + scounteren_d = scounteren_q; + sscratch_d = sscratch_q; + stval_d = stval_q; + satp_d = satp_q; + hedeleg_d = hedeleg_q; + hideleg_d = hideleg_q; + hgeie_d = hgeie_q; + hgatp_d = hgatp_q; + hcounteren_d = hcounteren_q; + htinst_d = htinst_q; + htval_d = htval_q; + + en_ld_st_translation_d = en_ld_st_translation_q; + en_ld_st_g_translation_d = en_ld_st_g_translation_q; + dirty_fp_state_csr = 1'b0; + + pmpcfg_d = pmpcfg_q; + pmpaddr_d = pmpaddr_q; // check for correct access rights and that we are writing if (csr_we) begin - unique case (csr_addr.address) + unique case (conv_csr_addr.address) // Floating-Point riscv::CSR_FFLAGS: begin - if (CVA6Cfg.FpPresent) begin + if (CVA6Cfg.FpPresent && !(mstatus_q.fs == riscv::Off || (CVA6Cfg.RVH && v_q && vsstatus_q.fs == riscv::Off))) begin dirty_fp_state_csr = 1'b1; fcsr_d.fflags = csr_wdata[4:0]; // this instruction has side-effects @@ -707,7 +912,7 @@ module csr_regfile end end riscv::CSR_FRM: begin - if (CVA6Cfg.FpPresent) begin + if (CVA6Cfg.FpPresent && !(mstatus_q.fs == riscv::Off || (CVA6Cfg.RVH && v_q && vsstatus_q.fs == riscv::Off))) begin dirty_fp_state_csr = 1'b1; fcsr_d.frm = csr_wdata[2:0]; // this instruction has side-effects @@ -717,7 +922,7 @@ module csr_regfile end end riscv::CSR_FCSR: begin - if (CVA6Cfg.FpPresent) begin + if (CVA6Cfg.FpPresent && !(mstatus_q.fs == riscv::Off || (CVA6Cfg.RVH && v_q && vsstatus_q.fs == riscv::Off))) begin dirty_fp_state_csr = 1'b1; fcsr_d[7:0] = csr_wdata[7:0]; // ignore writes to reserved space // this instruction has side-effects @@ -727,7 +932,7 @@ module csr_regfile end end riscv::CSR_FTRAN: begin - if (CVA6Cfg.FpPresent) begin + if (CVA6Cfg.FpPresent && !(mstatus_q.fs == riscv::Off || (CVA6Cfg.RVH && v_q && vsstatus_q.fs == riscv::Off))) begin dirty_fp_state_csr = 1'b1; fcsr_d.fprec = csr_wdata[6:0]; // ignore writes to reserved space // this instruction has side-effects @@ -764,6 +969,74 @@ module csr_regfile riscv::CSR_TDATA1: update_access_exception = 1'b1; // not implemented riscv::CSR_TDATA2: update_access_exception = 1'b1; // not implemented riscv::CSR_TDATA3: update_access_exception = 1'b1; // not implemented + // virtual supervisor registers + riscv::CSR_VSSTATUS: begin + if (CVA6Cfg.RVH) begin + mask = ariane_pkg::SMODE_STATUS_WRITE_MASK[CVA6Cfg.XLEN-1:0]; + vsstatus_d = (vsstatus_q & ~{{64-CVA6Cfg.XLEN{1'b0}}, mask}) | {{64-CVA6Cfg.XLEN{1'b0}}, (csr_wdata & mask)}; + // hardwire to zero if floating point extension is not present + vsstatus_d.xs = riscv::Off; + if (!CVA6Cfg.FpPresent) begin + vsstatus_d.fs = riscv::Off; + end + // this instruction has side-effects + flush_o = 1'b1; + end else begin + update_access_exception = 1'b1; + end + end + riscv::CSR_VSIE: + if (CVA6Cfg.RVH) mie_d = (mie_q & ~hideleg_q) | ((csr_wdata << 1) & hideleg_q); + else update_access_exception = 1'b1; + riscv::CSR_VSIP: begin + if (CVA6Cfg.RVH) begin + // only the virtual supervisor software interrupt is write-able, iff delegated + mask = CVA6Cfg.XLEN'(riscv::MIP_VSSIP) & hideleg_q; + mip_d = (mip_q & ~mask) | ((csr_wdata << 1) & mask); + end else begin + update_access_exception = 1'b1; + end + end + riscv::CSR_VSTVEC: begin + if (CVA6Cfg.RVH) begin + vstvec_d = {csr_wdata[CVA6Cfg.XLEN-1:2], 1'b0, csr_wdata[0]}; + end else begin + update_access_exception = 1'b1; + end + end + riscv::CSR_VSSCRATCH: + if (CVA6Cfg.RVH) vsscratch_d = csr_wdata; + else update_access_exception = 1'b1; + riscv::CSR_VSEPC: + if (CVA6Cfg.RVH) vsepc_d = {csr_wdata[CVA6Cfg.XLEN-1:1], 1'b0}; + else update_access_exception = 1'b1; + riscv::CSR_VSCAUSE: + if (CVA6Cfg.RVH) vscause_d = csr_wdata; + else update_access_exception = 1'b1; + riscv::CSR_VSTVAL: + if (CVA6Cfg.RVH) vstval_d = csr_wdata; + else update_access_exception = 1'b1; + // virtual supervisor address translation and protection + riscv::CSR_VSATP: begin + if (CVA6Cfg.RVH) begin + if (priv_lvl_o == riscv::PRIV_LVL_S && hstatus_q.vtvm && v_q) begin + virtual_update_access_exception = 1'b1; + end else begin + vsatp = satp_t'(csr_wdata); + // only make ASID_LEN - 1 bit stick, that way software can figure out how many ASID bits are supported + vsatp.asid = vsatp.asid & {{(CVA6Cfg.ASIDW - CVA6Cfg.ASID_WIDTH) {1'b0}}, {CVA6Cfg.ASID_WIDTH{1'b1}}}; + // only update if we actually support this mode + if (config_pkg::vm_mode_t'(vsatp.mode) == config_pkg::ModeOff || + config_pkg::vm_mode_t'(vsatp.mode) == CVA6Cfg.MODE_SV) + vsatp_d = vsatp; + end + // changing the mode can have side-effects on address translation (e.g.: other instructions), re-fetch + // the next instruction by executing a flush + flush_o = 1'b1; + end else begin + update_access_exception = 1'b1; + end + end // sstatus is a subset of mstatus - mask it accordingly riscv::CSR_SSTATUS: begin if (CVA6Cfg.RVS) begin @@ -787,8 +1060,9 @@ module csr_regfile // if the corresponding bit in mideleg is set riscv::CSR_SIE: begin if (CVA6Cfg.RVS) begin + mask = (CVA6Cfg.RVH) ? mideleg_q & ~HS_DELEG_INTERRUPTS[CVA6Cfg.XLEN-1:0] : mideleg_q; // the mideleg makes sure only delegate-able register (and therefore also only implemented registers) are written - mie_d = (mie_q & ~mideleg_q) | (csr_wdata & mideleg_q); + mie_d = (mie_q & ~mask) | (csr_wdata & mask); end else begin update_access_exception = 1'b1; end @@ -797,7 +1071,7 @@ module csr_regfile riscv::CSR_SIP: begin if (CVA6Cfg.RVS) begin // only the supervisor software interrupt is write-able, iff delegated - mask = riscv::MIP_SSIP & mideleg_q; + mask = CVA6Cfg.XLEN'(riscv::MIP_SSIP) & mideleg_q; mip_d = (mip_q & ~mask) | (csr_wdata & mask); end else begin update_access_exception = 1'b1; @@ -843,7 +1117,123 @@ module csr_regfile update_access_exception = 1'b1; end end - + riscv::CSR_SENVCFG: + if (CVA6Cfg.RVU) fiom_d = csr_wdata[0]; + else update_access_exception = 1'b1; + //hypervisor mode registers + riscv::CSR_HSTATUS: begin + if (CVA6Cfg.RVH) begin + mask = ariane_pkg::HSTATUS_WRITE_MASK[CVA6Cfg.XLEN-1:0]; + hstatus_d = (hstatus_q & ~{{64-CVA6Cfg.XLEN{1'b0}}, mask}) | {{64-CVA6Cfg.XLEN{1'b0}}, (csr_wdata & mask)}; + // this instruction has side-effects + flush_o = 1'b1; + end else begin + update_access_exception = 1'b1; + end + end + riscv::CSR_HEDELEG: begin + if (CVA6Cfg.RVH) begin + mask = (1 << riscv::INSTR_ADDR_MISALIGNED) | + (1 << riscv::INSTR_ACCESS_FAULT) | + (1 << riscv::ILLEGAL_INSTR) | + (1 << riscv::BREAKPOINT) | + (1 << riscv::LD_ADDR_MISALIGNED) | + (1 << riscv::LD_ACCESS_FAULT) | + (1 << riscv::ST_ADDR_MISALIGNED) | + (1 << riscv::ST_ACCESS_FAULT) | + (1 << riscv::ENV_CALL_UMODE) | + (1 << riscv::INSTR_PAGE_FAULT) | + (1 << riscv::LOAD_PAGE_FAULT) | + (1 << riscv::STORE_PAGE_FAULT); + hedeleg_d = (hedeleg_q & ~mask) | (csr_wdata & mask); + end else begin + update_access_exception = 1'b1; + end + end + riscv::CSR_HIDELEG: begin + if (CVA6Cfg.RVH) begin + hideleg_d = (hideleg_q & ~VS_DELEG_INTERRUPTS[CVA6Cfg.XLEN-1:0]) | (csr_wdata & VS_DELEG_INTERRUPTS[CVA6Cfg.XLEN-1:0]); + end else begin + update_access_exception = 1'b1; + end + end + riscv::CSR_HIE: begin + if (CVA6Cfg.RVH) begin + mask = HS_DELEG_INTERRUPTS[CVA6Cfg.XLEN-1:0]; + mie_d = (mie_q & ~mask) | (csr_wdata & mask); + end else begin + update_access_exception = 1'b1; + end + end + riscv::CSR_HIP: begin + if (CVA6Cfg.RVH) begin + mask = CVA6Cfg.XLEN'(riscv::MIP_VSSIP); + mip_d = (mip_q & ~mask) | (csr_wdata & mask); + end else begin + update_access_exception = 1'b1; + end + end + riscv::CSR_HVIP: begin + if (CVA6Cfg.RVH) begin + mask = VS_DELEG_INTERRUPTS[CVA6Cfg.XLEN-1:0]; + mip_d = (mip_q & ~mask) | (csr_wdata & mask); + end else begin + update_access_exception = 1'b1; + end + end + riscv::CSR_HCOUNTEREN: begin + if (CVA6Cfg.RVH) begin + hcounteren_d = {{CVA6Cfg.XLEN - 32{1'b0}}, csr_wdata[31:0]}; + end else begin + update_access_exception = 1'b1; + end + end + riscv::CSR_HTVAL: begin + if (CVA6Cfg.RVH) begin + htval_d = csr_wdata; + end else begin + update_access_exception = 1'b1; + end + end + riscv::CSR_HTINST: begin + if (CVA6Cfg.RVH) begin + htinst_d = {{CVA6Cfg.XLEN - 32{1'b0}}, csr_wdata[31:0]}; + end else begin + update_access_exception = 1'b1; + end + end + //TODO Hyp: implement hgeie write + riscv::CSR_HGEIE: begin + if (!CVA6Cfg.RVH) begin + update_access_exception = 1'b1; + end + end + riscv::CSR_HGATP: begin + if (CVA6Cfg.RVH) begin + // intercept HGATP writes if in HS-Mode and TVM is enabled + if (priv_lvl_o == riscv::PRIV_LVL_S && !v_q && mstatus_q.tvm) + update_access_exception = 1'b1; + else begin + hgatp = hgatp_t'(csr_wdata); + //hardwire PPN[1:0] to zero + hgatp[1:0] = 2'b0; + // only make VMID_LEN - 1 bit stick, that way software can figure out how many VMID bits are supported + hgatp.vmid = hgatp.vmid & {{(CVA6Cfg.VMIDW - CVA6Cfg.VMID_WIDTH) {1'b0}}, {CVA6Cfg.VMID_WIDTH{1'b1}}}; + // only update if we actually support this mode + if (config_pkg::vm_mode_t'(hgatp.mode) == config_pkg::ModeOff || + config_pkg::vm_mode_t'(hgatp.mode) == CVA6Cfg.MODE_SV) + hgatp_d = hgatp; + end + // changing the mode can have side-effects on address translation (e.g.: other instructions), re-fetch + // the next instruction by executing a flush + flush_o = 1'b1; + end else begin + update_access_exception = 1'b1; + end + end + riscv::CSR_HENVCFG: + if (CVA6Cfg.RVH) fiom_d = csr_wdata[0]; + else update_access_exception = 1'b1; riscv::CSR_MSTATUS: begin mstatus_d = {{64 - CVA6Cfg.XLEN{1'b0}}, csr_wdata}; mstatus_d.xs = riscv::Off; @@ -869,11 +1259,22 @@ module csr_regfile riscv::CSR_MEDELEG: begin if (CVA6Cfg.RVS) begin mask = (1 << riscv::INSTR_ADDR_MISALIGNED) | + (1 << riscv::INSTR_ACCESS_FAULT) | + (1 << riscv::ILLEGAL_INSTR) | (1 << riscv::BREAKPOINT) | + (1 << riscv::LD_ADDR_MISALIGNED) | + (1 << riscv::LD_ACCESS_FAULT) | + (1 << riscv::ST_ADDR_MISALIGNED) | + (1 << riscv::ST_ACCESS_FAULT) | (1 << riscv::ENV_CALL_UMODE) | + ((CVA6Cfg.RVH ? 1 : 0) << riscv::ENV_CALL_VSMODE) | (1 << riscv::INSTR_PAGE_FAULT) | (1 << riscv::LOAD_PAGE_FAULT) | - (1 << riscv::STORE_PAGE_FAULT); + (1 << riscv::STORE_PAGE_FAULT) | + ((CVA6Cfg.RVH ? 1 : 0) << riscv::INSTR_GUEST_PAGE_FAULT) | + ((CVA6Cfg.RVH ? 1 : 0) << riscv::LOAD_GUEST_PAGE_FAULT) | + ((CVA6Cfg.RVH ? 1 : 0) << riscv::VIRTUAL_INSTRUCTION) | + ((CVA6Cfg.RVH ? 1 : 0) << riscv::STORE_GUEST_PAGE_FAULT); medeleg_d = (medeleg_q & ~mask) | (csr_wdata & mask); end else begin update_access_exception = 1'b1; @@ -883,15 +1284,36 @@ module csr_regfile // we do not support user interrupt delegation riscv::CSR_MIDELEG: begin if (CVA6Cfg.RVS) begin - mask = riscv::MIP_SSIP | riscv::MIP_STIP | riscv::MIP_SEIP; - mideleg_d = (mideleg_q & ~mask) | (csr_wdata & mask); + mask = CVA6Cfg.XLEN'(riscv::MIP_SSIP) + | CVA6Cfg.XLEN'(riscv::MIP_STIP) + | CVA6Cfg.XLEN'(riscv::MIP_SEIP); + if (CVA6Cfg.RVH) begin + mideleg_d = (mideleg_q & ~mask) | (csr_wdata & mask) | HS_DELEG_INTERRUPTS[CVA6Cfg.XLEN-1:0]; + end else begin + mideleg_d = (mideleg_q & ~mask) | (csr_wdata & mask); + end end else begin update_access_exception = 1'b1; end end // mask the register so that unsupported interrupts can never be set riscv::CSR_MIE: begin - mask = riscv::MIP_SSIP | riscv::MIP_STIP | riscv::MIP_SEIP | riscv::MIP_MSIP | riscv::MIP_MTIP | riscv::MIP_MEIP; + if (CVA6Cfg.RVH) begin + mask = HS_DELEG_INTERRUPTS[CVA6Cfg.XLEN-1:0] + | CVA6Cfg.XLEN'(riscv::MIP_SSIP) + | CVA6Cfg.XLEN'(riscv::MIP_STIP) + | CVA6Cfg.XLEN'(riscv::MIP_SEIP) + | CVA6Cfg.XLEN'(riscv::MIP_MSIP) + | CVA6Cfg.XLEN'(riscv::MIP_MTIP) + | CVA6Cfg.XLEN'(riscv::MIP_MEIP); + end else begin + mask = CVA6Cfg.XLEN'(riscv::MIP_SSIP) + | CVA6Cfg.XLEN'(riscv::MIP_STIP) + | CVA6Cfg.XLEN'(riscv::MIP_SEIP) + | CVA6Cfg.XLEN'(riscv::MIP_MSIP) + | CVA6Cfg.XLEN'(riscv::MIP_MTIP) + | CVA6Cfg.XLEN'(riscv::MIP_MEIP); + end mie_d = (mie_q & ~mask) | (csr_wdata & mask); // we only support supervisor and M-mode interrupts end @@ -913,8 +1335,23 @@ module csr_regfile if (CVA6Cfg.TvalEn) mtval_d = csr_wdata; else update_access_exception = 1'b1; end + riscv::CSR_MTINST: + if (CVA6Cfg.RVH) mtinst_d = {{CVA6Cfg.XLEN - 32{1'b0}}, csr_wdata[31:0]}; + else update_access_exception = 1'b1; + riscv::CSR_MTVAL2: + if (CVA6Cfg.RVH) mtval2_d = csr_wdata; + else update_access_exception = 1'b1; riscv::CSR_MIP: begin - mask = riscv::MIP_SSIP | riscv::MIP_STIP | riscv::MIP_SEIP; + if (CVA6Cfg.RVH) begin + mask = CVA6Cfg.XLEN'(riscv::MIP_SSIP) + | CVA6Cfg.XLEN'(riscv::MIP_STIP) + | CVA6Cfg.XLEN'(riscv::MIP_SEIP) + | CVA6Cfg.XLEN'(riscv::MIP_VSSIP); + end else begin + mask = CVA6Cfg.XLEN'(riscv::MIP_SSIP) + | CVA6Cfg.XLEN'(riscv::MIP_STIP) + | CVA6Cfg.XLEN'(riscv::MIP_SEIP); + end mip_d = (mip_q & ~mask) | (csr_wdata & mask); end riscv::CSR_MENVCFG: if (CVA6Cfg.RVU) fiom_d = csr_wdata[0]; @@ -1101,9 +1538,17 @@ module csr_regfile if (!CVA6Cfg.RVU) begin mstatus_d.mpp = riscv::PRIV_LVL_M; end + + if (CVA6Cfg.RVH) begin + hstatus_d.vsxl = riscv::XLEN_64; + vsstatus_d.uxl = riscv::XLEN_64; + end // mark the floating point extension register as dirty if (CVA6Cfg.FpPresent && (dirty_fp_state_csr || dirty_fp_state_i)) begin mstatus_d.fs = riscv::Dirty; + if (CVA6Cfg.RVH && v_q) begin + vsstatus_d.fs = riscv::Dirty; + end end // mark the vector extension register as dirty if (CVA6Cfg.RVV && dirty_v_state_i) begin @@ -1111,6 +1556,9 @@ module csr_regfile end // hardwired extension registers mstatus_d.sd = (mstatus_q.xs == riscv::Dirty) | (mstatus_q.fs == riscv::Dirty); + if (CVA6Cfg.RVH) begin + vsstatus_d.sd = (vsstatus_q.xs == riscv::Dirty) | (vsstatus_q.fs == riscv::Dirty); + end // reserve PMPCFG bits 5 and 6 (hardwire to 0) for (int i = 0; i < CVA6Cfg.NrPMPEntries; i++) pmpcfg_d[i].reserved = 2'b0; @@ -1146,6 +1594,7 @@ module csr_regfile // update exception CSRs // we got an exception update cause, pc and stval register trap_to_priv_lvl = riscv::PRIV_LVL_M; + trap_to_v = 1'b0; // Exception is taken and we are not in debug mode // exceptions in debug mode don't update any fields if ((CVA6Cfg.DebugEn && !debug_mode_q && ex_i.cause != riscv::DEBUG_REQUEST && ex_i.valid) || (!CVA6Cfg.DebugEn && ex_i.valid)) begin @@ -1153,32 +1602,73 @@ module csr_regfile flush_o = 1'b0; // figure out where to trap to // a m-mode trap might be delegated if we are taking it in S mode - // first figure out if this was an exception or an interrupt e.g.: look at bit (CVA6Cfg.XLEN-1) - // the cause register can only be $clog2(CVA6Cfg.XLEN) bits long (as we only support CVA6Cfg.XLEN exceptions) - if (CVA6Cfg.RVS && ((ex_i.cause[CVA6Cfg.XLEN-1] && mideleg_q[ex_i.cause[$clog2( - CVA6Cfg.XLEN - )-1:0]]) || (~ex_i.cause[CVA6Cfg.XLEN-1] && medeleg_q[ex_i.cause[$clog2( - CVA6Cfg.XLEN - )-1:0]]))) begin - // traps never transition from a more-privileged mode to a less privileged mode - // so if we are already in M mode, stay there - if (priv_lvl_o == riscv::PRIV_LVL_M) trap_to_priv_lvl = riscv::PRIV_LVL_M; - else trap_to_priv_lvl = riscv::PRIV_LVL_S; + // first figure out if this was an exception or an interrupt e.g.: look at bit (XLEN-1) + // the cause register can only be $clog2(CVA6Cfg.XLEN) bits long (as we only support XLEN exceptions) + if (CVA6Cfg.RVH) begin + if ((ex_i.cause[CVA6Cfg.XLEN-1] && mideleg_q[ex_i.cause[$clog2( + CVA6Cfg.XLEN + )-1:0]] && ~hideleg_q[ex_i.cause[$clog2( + CVA6Cfg.XLEN + )-1:0]]) || (~ex_i.cause[CVA6Cfg.XLEN-1] && medeleg_q[ex_i.cause[$clog2( + CVA6Cfg.XLEN + )-1:0]] && ~hedeleg_q[ex_i.cause[$clog2( + CVA6Cfg.XLEN + )-1:0]])) begin + // traps never transition from a more-privileged mode to a less privileged mode + // so if we are already in M mode, stay there + trap_to_priv_lvl = (priv_lvl_o == riscv::PRIV_LVL_M) ? riscv::PRIV_LVL_M : riscv::PRIV_LVL_S; + end else if ((ex_i.cause[CVA6Cfg.XLEN-1] && hideleg_q[ex_i.cause[$clog2( + CVA6Cfg.XLEN + )-1:0]]) || (~ex_i.cause[CVA6Cfg.XLEN-1] && hedeleg_q[ex_i.cause[$clog2( + CVA6Cfg.XLEN + )-1:0]])) begin + trap_to_priv_lvl = (priv_lvl_o == riscv::PRIV_LVL_M) ? riscv::PRIV_LVL_M : riscv::PRIV_LVL_S; + // trap to VS only if it is the currently active mode + trap_to_v = v_q; + end + end else begin + if (CVA6Cfg.RVS && (ex_i.cause[CVA6Cfg.XLEN-1] && mideleg_q[ex_i.cause[$clog2( + CVA6Cfg.XLEN + )-1:0]]) || (~ex_i.cause[CVA6Cfg.XLEN-1] && medeleg_q[ex_i.cause[$clog2( + CVA6Cfg.XLEN + )-1:0]])) begin + // traps never transition from a more-privileged mode to a less privileged mode + // so if we are already in M mode, stay there + trap_to_priv_lvl = (priv_lvl_o == riscv::PRIV_LVL_M) ? riscv::PRIV_LVL_M : riscv::PRIV_LVL_S; + end end // trap to supervisor mode if (CVA6Cfg.RVS && trap_to_priv_lvl == riscv::PRIV_LVL_S) begin - // update sstatus - mstatus_d.sie = 1'b0; - mstatus_d.spie = mstatus_q.sie; - // this can either be user or supervisor mode - mstatus_d.spp = priv_lvl_q[0]; - // set cause - scause_d = ex_i.cause; - // set epc - sepc_d = {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{pc_i[CVA6Cfg.VLEN-1]}}, pc_i}; - // set mtval or stval - stval_d = (ariane_pkg::ZERO_TVAL + if (CVA6Cfg.RVH && trap_to_v) begin + // update sstatus + vsstatus_d.sie = 1'b0; + vsstatus_d.spie = vsstatus_q.sie; + // this can either be user or supervisor mode + vsstatus_d.spp = priv_lvl_q[0]; + // set cause + vscause_d = ex_i.cause[CVA6Cfg.XLEN-1] ? {ex_i.cause[CVA6Cfg.XLEN-1:2], 2'b01} : ex_i.cause; + // set epc + vsepc_d = {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{pc_i[CVA6Cfg.VLEN-1]}}, pc_i}; + // set vstval + vstval_d = (ariane_pkg::ZERO_TVAL + && (ex_i.cause inside { + riscv::ILLEGAL_INSTR, + riscv::BREAKPOINT, + riscv::ENV_CALL_UMODE + } || ex_i.cause[CVA6Cfg.XLEN-1])) ? '0 : ex_i.tval; + end else begin + // update sstatus + mstatus_d.sie = 1'b0; + mstatus_d.spie = mstatus_q.sie; + // this can either be user or supervisor mode + mstatus_d.spp = priv_lvl_q[0]; + // set cause + scause_d = ex_i.cause; + // set epc + sepc_d = {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{pc_i[CVA6Cfg.VLEN-1]}}, pc_i}; + // set mtval or stval + stval_d = (ariane_pkg::ZERO_TVAL && (ex_i.cause inside { riscv::ILLEGAL_INSTR, riscv::BREAKPOINT, @@ -1186,6 +1676,25 @@ module csr_regfile riscv::ENV_CALL_SMODE, riscv::ENV_CALL_MMODE } || ex_i.cause[CVA6Cfg.XLEN-1])) ? '0 : ex_i.tval; + if (CVA6Cfg.RVH) begin + htinst_d = (ariane_pkg::ZERO_TVAL + && (ex_i.cause inside { + riscv::INSTR_ACCESS_FAULT, + riscv::ILLEGAL_INSTR, + riscv::BREAKPOINT, + riscv::ENV_CALL_UMODE, + riscv::ENV_CALL_SMODE, + riscv::ENV_CALL_MMODE, + riscv::INSTR_PAGE_FAULT, + riscv::INSTR_GUEST_PAGE_FAULT, + riscv::VIRTUAL_INSTRUCTION + } || ex_i.cause[CVA6Cfg.XLEN-1])) ? '0 : {{CVA6Cfg.XLEN - 32 {1'b0}}, ex_i.tinst}; + hstatus_d.spvp = v_q ? priv_lvl_q[0] : hstatus_d.spvp; + htval_d = {{CVA6Cfg.XLEN - CVA6Cfg.GPLEN + 2{1'b0}}, ex_i.tval2[CVA6Cfg.GPLEN-1:2]}; + hstatus_d.gva = ex_i.gva; + hstatus_d.spv = v_q; + end + end // trap to machine mode end else begin // update mstatus @@ -1205,13 +1714,36 @@ module csr_regfile riscv::ENV_CALL_UMODE, riscv::ENV_CALL_SMODE, riscv::ENV_CALL_MMODE - } || ex_i.cause[CVA6Cfg.XLEN-1])) ? '0 : ex_i.tval; + } || ex_i.cause[CVA6Cfg.GPLEN-1])) ? '0 : ex_i.tval; end else begin mtval_d = '0; end + + if (CVA6Cfg.RVH) begin + // save previous virtualization mode + mstatus_d.mpv = v_q; + mtinst_d = (ariane_pkg::ZERO_TVAL + && (ex_i.cause inside { + riscv::INSTR_ADDR_MISALIGNED, + riscv::INSTR_ACCESS_FAULT, + riscv::ILLEGAL_INSTR, + riscv::BREAKPOINT, + riscv::ENV_CALL_UMODE, + riscv::ENV_CALL_SMODE, + riscv::ENV_CALL_MMODE, + riscv::INSTR_PAGE_FAULT, + riscv::INSTR_GUEST_PAGE_FAULT, + riscv::VIRTUAL_INSTRUCTION + } || ex_i.cause[CVA6Cfg.XLEN-1])) ? '0 : {{CVA6Cfg.XLEN - 32 {1'b0}}, ex_i.tinst}; + mtval2_d = {{CVA6Cfg.XLEN - CVA6Cfg.GPLEN + 2{1'b0}}, ex_i.tval2[CVA6Cfg.GPLEN-1:2]}; + mstatus_d.gva = ex_i.gva; + end end priv_lvl_d = trap_to_priv_lvl; + if (CVA6Cfg.RVH) begin + v_d = trap_to_v; + end end // ------------------------------ @@ -1226,11 +1758,15 @@ module csr_regfile // we are currently not in debug mode and could potentially enter if (!debug_mode_q) begin dcsr_d.prv = priv_lvl_o; + // save virtualization mode bit + dcsr_d.v = (!CVA6Cfg.RVH) ? 1'b0 : v_q; // trigger module fired // caused by a breakpoint if (CVA6Cfg.DebugEn && ex_i.valid && ex_i.cause == riscv::BREAKPOINT) begin dcsr_d.prv = priv_lvl_o; + // save virtualization mode bit + dcsr_d.v = (!CVA6Cfg.RVH) ? 1'b0 : v_q; // check that we actually want to enter debug depending on the privilege level we are currently in unique case (priv_lvl_o) riscv::PRIV_LVL_M: begin @@ -1239,14 +1775,14 @@ module csr_regfile end riscv::PRIV_LVL_S: begin if (CVA6Cfg.RVS) begin - debug_mode_d = dcsr_q.ebreaks; - set_debug_pc_o = dcsr_q.ebreaks; + debug_mode_d = (CVA6Cfg.RVH && v_q) ? dcsr_q.ebreakvs : dcsr_q.ebreaks; + set_debug_pc_o = (CVA6Cfg.RVH && v_q) ? dcsr_q.ebreakvs : dcsr_q.ebreaks; end end riscv::PRIV_LVL_U: begin if (CVA6Cfg.RVU) begin - debug_mode_d = dcsr_q.ebreaku; - set_debug_pc_o = dcsr_q.ebreaku; + debug_mode_d = (CVA6Cfg.RVH && v_q) ? dcsr_q.ebreakvu : dcsr_q.ebreaku; + set_debug_pc_o = (CVA6Cfg.RVH && v_q) ? dcsr_q.ebreakvu : dcsr_q.ebreaku; end end default: ; @@ -1259,6 +1795,7 @@ module csr_regfile // we've got a debug request if (CVA6Cfg.DebugEn && ex_i.valid && ex_i.cause == riscv::DEBUG_REQUEST) begin dcsr_d.prv = priv_lvl_o; + dcsr_d.v = (!CVA6Cfg.RVH) ? 1'b0 : v_q; // save the PC dpc_d = {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{pc_i[CVA6Cfg.VLEN-1]}}, pc_i}; // enter debug mode @@ -1272,6 +1809,7 @@ module csr_regfile // single step enable and we just retired an instruction if (CVA6Cfg.DebugEn && dcsr_q.step && commit_ack_i[0]) begin dcsr_d.prv = priv_lvl_o; + dcsr_d.v = (!CVA6Cfg.RVH) ? 1'b0 : v_q; // valid CTRL flow change if (commit_instr_i[0].fu == CTRL_FLOW) begin // we saved the correct target address during execute @@ -1307,13 +1845,48 @@ module csr_regfile // ------------------------------ // Set the address translation at which the load and stores should occur // we can use the previous values since changing the address translation will always involve a pipeline flush - if (ariane_pkg::MMU_PRESENT && mprv && CVA6Cfg.RVS && config_pkg::vm_mode_t'(satp_q.mode) == CVA6Cfg.MODE_SV && (mstatus_q.mpp != riscv::PRIV_LVL_M)) - en_ld_st_translation_d = 1'b1; - else // otherwise we go with the regular settings - en_ld_st_translation_d = en_translation_o; + if (CVA6Cfg.RVH) begin + if (mprv && (mstatus_q.mpv == 1'b0) && (config_pkg::vm_mode_t'(satp_q.mode) == CVA6Cfg.MODE_SV) && (mstatus_q.mpp != riscv::PRIV_LVL_M)) begin + en_ld_st_translation_d = 1'b1; + end else if (mprv && (mstatus_q.mpv == 1'b1)) begin + if (config_pkg::vm_mode_t'(vsatp_q.mode) == CVA6Cfg.MODE_SV) begin + en_ld_st_translation_d = 1'b1; + end else begin + en_ld_st_translation_d = 1'b0; + end + end else begin // otherwise we go with the regular settings + en_ld_st_translation_d = en_translation_o; + end + + if (mprv && (mstatus_q.mpv == 1'b1)) begin + if (config_pkg::vm_mode_t'(hgatp_q.mode) == CVA6Cfg.MODE_SV) begin + en_ld_st_g_translation_d = 1'b1; + end else begin + en_ld_st_g_translation_d = 1'b0; + end + end else begin + en_ld_st_g_translation_d = en_g_translation_o; + end - ld_st_priv_lvl_o = (mprv) ? mstatus_q.mpp : priv_lvl_o; - en_ld_st_translation_o = en_ld_st_translation_q; + if (csr_hs_ld_st_inst_i) ld_st_priv_lvl_o = riscv::priv_lvl_t'(hstatus_q.spvp); + else ld_st_priv_lvl_o = (mprv) ? mstatus_q.mpp : priv_lvl_o; + + ld_st_v_o = ((mprv ? mstatus_q.mpv : v_q) || (csr_hs_ld_st_inst_i)); + + en_ld_st_translation_o = (en_ld_st_translation_q && !csr_hs_ld_st_inst_i) || (config_pkg::vm_mode_t'(vsatp_q.mode) == CVA6Cfg.MODE_SV && csr_hs_ld_st_inst_i); + + en_ld_st_g_translation_o = (en_ld_st_g_translation_q && !csr_hs_ld_st_inst_i) || (csr_hs_ld_st_inst_i && config_pkg::vm_mode_t'(hgatp_q.mode) == CVA6Cfg.MODE_SV && csr_hs_ld_st_inst_i); + end else begin + if (ariane_pkg::MMU_PRESENT && mprv && CVA6Cfg.RVS && config_pkg::vm_mode_t'(satp_q.mode) == CVA6Cfg.MODE_SV && (mstatus_q.mpp != riscv::PRIV_LVL_M)) + en_ld_st_translation_d = 1'b1; + else // otherwise we go with the regular settings + en_ld_st_translation_d = en_translation_o; + + ld_st_priv_lvl_o = (mprv) ? mstatus_q.mpp : priv_lvl_o; + en_ld_st_translation_o = en_ld_st_translation_q; + ld_st_v_o = 1'b0; + en_ld_st_g_translation_o = 1'b0; + end // ------------------------------ // Return from Environment // ------------------------------ @@ -1334,9 +1907,16 @@ module csr_regfile end // set mpie to 1 mstatus_d.mpie = 1'b1; + if (CVA6Cfg.RVH) begin + // set virtualization mode + v_d = mstatus_q.mpv; + //set mstatus mpv to false + mstatus_d.mpv = 1'b0; + if (mstatus_q.mpp != riscv::PRIV_LVL_M) mstatus_d.mprv = 1'b0; + end end - if (CVA6Cfg.RVS && sret) begin + if (CVA6Cfg.RVS && sret && ((CVA6Cfg.RVH && !v_q) || !CVA6Cfg.RVH)) begin // return from exception, IF doesn't care from where we are returning eret_o = 1'b1; // return the previous supervisor interrupt enable flag @@ -1347,14 +1927,38 @@ module csr_regfile mstatus_d.spp = 1'b0; // set spie to 1 mstatus_d.spie = 1'b1; + if (CVA6Cfg.RVH) begin + // set virtualization mode + v_d = hstatus_q.spv; + //set hstatus spv to false + hstatus_d.spv = 1'b0; + mstatus_d.mprv = 1'b0; + end + end + + if (CVA6Cfg.RVH && sret && v_q) begin + // return from exception, IF doesn't care from where we are returning + eret_o = 1'b1; + // return the previous supervisor interrupt enable flag + vsstatus_d.sie = vsstatus_q.spie; + // restore the previous privilege level + priv_lvl_d = riscv::priv_lvl_t'({1'b0, vsstatus_q.spp}); + // set spp to user mode + vsstatus_d.spp = 1'b0; + // set spie to 1 + vsstatus_d.spie = 1'b1; end // return from debug mode if (CVA6Cfg.DebugEn && dret) begin // return from exception, IF doesn't care from where we are returning - eret_o = 1'b1; + eret_o = 1'b1; // restore the previous privilege level - priv_lvl_d = riscv::priv_lvl_t'(dcsr_q.prv); + priv_lvl_d = riscv::priv_lvl_t'(dcsr_q.prv); + if (CVA6Cfg.RVH) begin + // restore the previous virtualization mode + v_d = dcsr_q.v; + end // actually return from debug mode debug_mode_d = 1'b0; end @@ -1408,8 +2012,9 @@ module csr_regfile assign irq_ctrl_o.mie = mie_q; assign irq_ctrl_o.mip = mip_q; - assign irq_ctrl_o.sie = mstatus_q.sie; + assign irq_ctrl_o.sie = (CVA6Cfg.RVH && v_q) ? vsstatus_q.sie : mstatus_q.sie; assign irq_ctrl_o.mideleg = mideleg_q; + assign irq_ctrl_o.hideleg = (CVA6Cfg.RVH) ? hideleg_q : '0; assign irq_ctrl_o.global_enable = (~debug_mode_q) // interrupts are enabled during single step or we are not stepping // No need to check interrupts during single step if we don't support DEBUG mode @@ -1418,29 +2023,74 @@ module csr_regfile | (priv_lvl_o != riscv::PRIV_LVL_M)); always_comb begin : privilege_check - // ----------------- - // Privilege Check - // ----------------- - privilege_violation = 1'b0; - // if we are reading or writing, check for the correct privilege level this has - // precedence over interrupts - if (csr_op_i inside {CSR_WRITE, CSR_SET, CSR_CLEAR, CSR_READ}) begin - if ((riscv::priv_lvl_t'(priv_lvl_o & csr_addr.csr_decode.priv_lvl) != csr_addr.csr_decode.priv_lvl)) begin - privilege_violation = 1'b1; - end - // check access to debug mode only CSRs - if ((!CVA6Cfg.DebugEn && csr_addr_i[11:4] == 8'h7b) || (CVA6Cfg.DebugEn && csr_addr_i[11:4] == 8'h7b && !debug_mode_q)) begin - privilege_violation = 1'b1; + if (CVA6Cfg.RVH) begin + automatic riscv::priv_lvl_t access_priv; + automatic riscv::priv_lvl_t curr_priv; + automatic logic [SELECT_COUNTER_WIDTH-1:0] sel_cnt_en; + // transforms S mode accesses into HS mode + access_priv = (priv_lvl_o == riscv::PRIV_LVL_S && !v_q) ? riscv::PRIV_LVL_HS : priv_lvl_o; + curr_priv = priv_lvl_o; + sel_cnt_en = {{SELECT_COUNTER_WIDTH - 5{1'b0}}, csr_addr_i[4:0]}; + // ----------------- + // Privilege Check + // ----------------- + privilege_violation = 1'b0; + virtual_privilege_violation = 1'b0; + // if we are reading or writing, check for the correct privilege level this has + // precedence over interrupts + if (csr_op_i inside {CSR_WRITE, CSR_SET, CSR_CLEAR, CSR_READ}) begin + if (access_priv < csr_addr.csr_decode.priv_lvl) begin + if (v_q && csr_addr.csr_decode.priv_lvl == riscv::PRIV_LVL_HS) + virtual_privilege_violation = 1'b1; + else privilege_violation = 1'b1; + end + // check access to debug mode only CSRs + if ((!CVA6Cfg.DebugEn && csr_addr_i[11:4] == 8'h7b) || (CVA6Cfg.DebugEn && csr_addr_i[11:4] == 8'h7b && !debug_mode_q)) begin + privilege_violation = 1'b1; + end + // check counter-enabled counter CSR accesses + // counter address range is C00 to C1F + if (csr_addr_i inside {[riscv::CSR_CYCLE : riscv::CSR_HPM_COUNTER_31]}) begin + if (curr_priv == riscv::PRIV_LVL_S && CVA6Cfg.RVS) begin + virtual_privilege_violation = v_q & mcounteren_q[sel_cnt_en] & ~hcounteren_q[sel_cnt_en]; + privilege_violation = ~mcounteren_q[sel_cnt_en]; + end else if (priv_lvl_o == riscv::PRIV_LVL_U && CVA6Cfg.RVU) begin + virtual_privilege_violation = v_q & mcounteren_q[sel_cnt_en] & ~hcounteren_q[sel_cnt_en]; + if (v_q) begin + privilege_violation = ~mcounteren_q[sel_cnt_en] & ~scounteren_q[sel_cnt_en] & hcounteren_q[sel_cnt_en]; + end else begin + privilege_violation = ~mcounteren_q[sel_cnt_en] & ~scounteren_q[sel_cnt_en]; + end + end else if (priv_lvl_o == riscv::PRIV_LVL_M) begin + privilege_violation = 1'b0; + end + end end - // check counter-enabled counter CSR accesses - // counter address range is C00 to C1F - if (csr_addr_i inside {[riscv::CSR_CYCLE : riscv::CSR_HPM_COUNTER_31]}) begin - if (priv_lvl_o == riscv::PRIV_LVL_S && CVA6Cfg.RVS) begin - privilege_violation = ~mcounteren_q[csr_addr_i[4:0]]; - end else if (priv_lvl_o == riscv::PRIV_LVL_U && CVA6Cfg.RVU) begin - privilege_violation = ~mcounteren_q[csr_addr_i[4:0]] | ~scounteren_q[csr_addr_i[4:0]]; - end else if (priv_lvl_o == riscv::PRIV_LVL_M) begin - privilege_violation = 1'b0; + end else begin + // ----------------- + // Privilege Check + // ----------------- + privilege_violation = 1'b0; + // if we are reading or writing, check for the correct privilege level this has + // precedence over interrupts + if (csr_op_i inside {CSR_WRITE, CSR_SET, CSR_CLEAR, CSR_READ}) begin + if ((riscv::priv_lvl_t'(priv_lvl_o & csr_addr.csr_decode.priv_lvl) != csr_addr.csr_decode.priv_lvl)) begin + privilege_violation = 1'b1; + end + // check access to debug mode only CSRs + if ((!CVA6Cfg.DebugEn && csr_addr_i[11:4] == 8'h7b) || (CVA6Cfg.DebugEn && csr_addr_i[11:4] == 8'h7b && !debug_mode_q)) begin + privilege_violation = 1'b1; + end + // check counter-enabled counter CSR accesses + // counter address range is C00 to C1F + if (csr_addr_i inside {[riscv::CSR_CYCLE : riscv::CSR_HPM_COUNTER_31]}) begin + if (priv_lvl_o == riscv::PRIV_LVL_S && CVA6Cfg.RVS) begin + privilege_violation = ~mcounteren_q[csr_addr_i[4:0]]; + end else if (priv_lvl_o == riscv::PRIV_LVL_U && CVA6Cfg.RVU) begin + privilege_violation = ~mcounteren_q[csr_addr_i[4:0]] | ~scounteren_q[csr_addr_i[4:0]]; + end else if (priv_lvl_o == riscv::PRIV_LVL_M) begin + privilege_violation = 1'b0; + end end end end @@ -1449,7 +2099,9 @@ module csr_regfile // CSR Exception Control // ---------------------- always_comb begin : exception_ctrl - csr_exception_o = {{CVA6Cfg.XLEN{1'b0}}, {CVA6Cfg.XLEN{1'b0}}, 1'b0}; + csr_exception_o = { + {CVA6Cfg.XLEN{1'b0}}, {CVA6Cfg.XLEN{1'b0}}, {CVA6Cfg.GPLEN{1'b0}}, {32{1'b0}}, 1'b0, 1'b0 + }; // ---------------------------------- // Illegal Access (decode exception) // ---------------------------------- @@ -1466,6 +2118,11 @@ module csr_regfile csr_exception_o.cause = riscv::ILLEGAL_INSTR; csr_exception_o.valid = 1'b1; end + + if (CVA6Cfg.RVH && (virtual_update_access_exception || virtual_read_access_exception || virtual_privilege_violation)) begin + csr_exception_o.cause = riscv::VIRTUAL_INSTRUCTION; + csr_exception_o.valid = 1'b1; + end end // ------------------- @@ -1490,7 +2147,7 @@ module csr_regfile trap_vector_base_o = {mtvec_q[CVA6Cfg.VLEN-1:2], 2'b0}; // output user mode stvec if (CVA6Cfg.RVS && trap_to_priv_lvl == riscv::PRIV_LVL_S) begin - trap_vector_base_o = {stvec_q[CVA6Cfg.VLEN-1:2], 2'b0}; + trap_vector_base_o = (CVA6Cfg.RVH && trap_to_v) ? {vstvec_q[CVA6Cfg.VLEN-1:2], 2'b0} : {stvec_q[CVA6Cfg.VLEN-1:2], 2'b0}; end // if we are in debug mode jump to a specific address @@ -1505,14 +2162,18 @@ module csr_regfile // activated for _that_ privilege level. if (ex_i.cause[CVA6Cfg.XLEN-1] && ((((CVA6Cfg.RVS || CVA6Cfg.RVU) && trap_to_priv_lvl == riscv::PRIV_LVL_M && mtvec_q[0]) || (!CVA6Cfg.RVS && !CVA6Cfg.RVU && mtvec_q[0])) - || (CVA6Cfg.RVS && trap_to_priv_lvl == riscv::PRIV_LVL_S && stvec_q[0]))) begin + || (CVA6Cfg.RVS && trap_to_priv_lvl == riscv::PRIV_LVL_S && !trap_to_v && stvec_q[0]))) begin trap_vector_base_o[7:2] = ex_i.cause[5:0]; end + if (ex_i.cause[CVA6Cfg.XLEN-1] && + (CVA6Cfg.RVH && trap_to_priv_lvl == riscv::PRIV_LVL_S && trap_to_v && vstvec_q[0])) begin + trap_vector_base_o[7:2] = {ex_i.cause[5:2], 2'b01}; + end epc_o = mepc_q[CVA6Cfg.VLEN-1:0]; - // we are returning from supervisor mode, so take the sepc register + // we are returning from supervisor or virtual supervisor mode, so take the sepc register if (CVA6Cfg.RVS && sret) begin - epc_o = sepc_q[CVA6Cfg.VLEN-1:0]; + epc_o = (CVA6Cfg.RVH && v_q) ? vsepc_q[CVA6Cfg.VLEN-1:0] : sepc_q[CVA6Cfg.VLEN-1:0]; end // we are returning from debug mode, to take the dpc register if (CVA6Cfg.DebugEn && dret) begin @@ -1529,7 +2190,7 @@ module csr_regfile // bit and the interrupt signal from the interrupt controller. csr_rdata_o = csr_rdata; - unique case (csr_addr.address) + unique case (conv_csr_addr.address) riscv::CSR_MIP: csr_rdata_o = csr_rdata | ({{CVA6Cfg.XLEN - 1{1'b0}}, irq_i[1]} << riscv::IRQ_S_EXT); // in supervisor mode we also need to check whether we delegated this bit @@ -1545,23 +2206,44 @@ module csr_regfile // in debug mode we execute with privilege level M assign priv_lvl_o = (CVA6Cfg.DebugEn && debug_mode_q) ? riscv::PRIV_LVL_M : priv_lvl_q; + assign v_o = CVA6Cfg.RVH ? v_q : 1'b0; // FPU outputs assign fflags_o = fcsr_q.fflags; assign frm_o = fcsr_q.frm; assign fprec_o = fcsr_q.fprec; // MMU outputs assign satp_ppn_o = satp_q.ppn; + assign vsatp_ppn_o = CVA6Cfg.RVH ? vsatp_q.ppn : '0; + assign hgatp_ppn_o = CVA6Cfg.RVH ? hgatp_q.ppn : '0; assign asid_o = satp_q.asid[AsidWidth-1:0]; + assign vs_asid_o = CVA6Cfg.RVH ? vsatp_q.asid[CVA6Cfg.ASID_WIDTH-1:0] : '0; + assign vmid_o = CVA6Cfg.RVH ? hgatp_q.vmid[CVA6Cfg.VMID_WIDTH-1:0] : '0; assign sum_o = mstatus_q.sum; + assign vs_sum_o = CVA6Cfg.RVH ? vsstatus_q.sum : '0; + assign hu_o = CVA6Cfg.RVH ? hstatus_q.hu : '0; // we support bare memory addressing and SV39 - assign en_translation_o = ((CVA6Cfg.RVS && config_pkg::vm_mode_t'(satp_q.mode) == CVA6Cfg.MODE_SV) && + if (CVA6Cfg.RVH) begin + assign en_translation_o = (((config_pkg::vm_mode_t'(satp_q.mode) == CVA6Cfg.MODE_SV && !v_q) || (config_pkg::vm_mode_t'(vsatp_q.mode) == CVA6Cfg.MODE_SV && v_q)) && priv_lvl_o != riscv::PRIV_LVL_M) ? 1'b1 : 1'b0; + assign en_g_translation_o = (config_pkg::vm_mode_t'(hgatp_q.mode) == CVA6Cfg.MODE_SV && + priv_lvl_o != riscv::PRIV_LVL_M && v_q) + ? 1'b1 + : 1'b0; + end else begin + assign en_translation_o = (CVA6Cfg.RVS && config_pkg::vm_mode_t'(satp_q.mode) == CVA6Cfg.MODE_SV && + priv_lvl_o != riscv::PRIV_LVL_M) + ? 1'b1 + : 1'b0; + assign en_g_translation_o = 1'b0; + end assign mxr_o = mstatus_q.mxr; - assign tvm_o = mstatus_q.tvm; + assign vmxr_o = CVA6Cfg.RVH ? vsstatus_q.mxr : '0; + assign tvm_o = (CVA6Cfg.RVH && v_q) ? hstatus_q.vtvm : mstatus_q.tvm; assign tw_o = mstatus_q.tw; - assign tsr_o = mstatus_q.tsr; + assign vtw_o = CVA6Cfg.RVH ? hstatus_q.vtw : '0; + assign tsr_o = (CVA6Cfg.RVH && v_q) ? hstatus_q.vtsr : mstatus_q.tsr; assign halt_csr_o = wfi_q; `ifdef PITON_ARIANE assign icache_en_o = icache_q[0]; @@ -1622,6 +2304,29 @@ module csr_regfile stval_q <= {CVA6Cfg.XLEN{1'b0}}; satp_q <= {CVA6Cfg.XLEN{1'b0}}; end + + if (CVA6Cfg.RVH) begin + v_q <= '0; + mtval2_q <= {CVA6Cfg.XLEN{1'b0}}; + mtinst_q <= {CVA6Cfg.XLEN{1'b0}}; + hstatus_q <= 64'b0; + hedeleg_q <= {CVA6Cfg.XLEN{1'b0}}; + hideleg_q <= {CVA6Cfg.XLEN{1'b0}}; + hgeie_q <= {CVA6Cfg.XLEN{1'b0}}; + hgatp_q <= {CVA6Cfg.XLEN{1'b0}}; + hcounteren_q <= {CVA6Cfg.XLEN{1'b0}}; + htval_q <= {CVA6Cfg.XLEN{1'b0}}; + htinst_q <= {CVA6Cfg.XLEN{1'b0}}; + // virtual supervisor mode registers + vsstatus_q <= 64'b0; + vsepc_q <= {CVA6Cfg.XLEN{1'b0}}; + vscause_q <= {CVA6Cfg.XLEN{1'b0}}; + vstvec_q <= {CVA6Cfg.XLEN{1'b0}}; + vsscratch_q <= {CVA6Cfg.XLEN{1'b0}}; + vstval_q <= {CVA6Cfg.XLEN{1'b0}}; + vsatp_q <= {CVA6Cfg.XLEN{1'b0}}; + en_ld_st_g_translation_q <= 1'b0; + end // timer and counters cycle_q <= 64'b0; instret_q <= 64'b0; @@ -1679,6 +2384,29 @@ module csr_regfile if (CVA6Cfg.TvalEn) stval_q <= stval_d; satp_q <= satp_d; end + if (CVA6Cfg.RVH) begin + v_q <= v_d; + mtval2_q <= mtval2_d; + mtinst_q <= mtinst_d; + // hypervisor mode registers + hstatus_q <= hstatus_d; + hedeleg_q <= hedeleg_d; + hideleg_q <= hideleg_d; + hgeie_q <= hgeie_d; + hgatp_q <= hgatp_d; + hcounteren_q <= hcounteren_d; + htval_q <= htval_d; + htinst_q <= htinst_d; + // virtual supervisor mode registers + vsstatus_q <= vsstatus_d; + vsepc_q <= vsepc_d; + vscause_q <= vscause_d; + vstvec_q <= vstvec_d; + vsscratch_q <= vsscratch_d; + vstval_q <= vstval_d; + vsatp_q <= vsatp_d; + en_ld_st_g_translation_q <= en_ld_st_g_translation_d; + end // timer and counters cycle_q <= cycle_d; instret_q <= instret_d; diff --git a/core/cva6.sv b/core/cva6.sv index cf445b853c..9d8a953651 100644 --- a/core/cva6.sv +++ b/core/cva6.sv @@ -42,6 +42,9 @@ module cva6 logic [CVA6Cfg.XLEN-1:0] cause; // cause of exception logic [CVA6Cfg.XLEN-1:0] tval; // additional information of causing exception (e.g.: instruction causing it), // address of LD/ST fault + logic [CVA6Cfg.GPLEN-1:0] tval2; // additional information when the causing exception in a guest exception + logic [31:0] tinst; // transformed instruction information + logic gva; // signals when a guest virtual address is written to tval logic valid; }, @@ -130,6 +133,7 @@ module cva6 logic [CVA6Cfg.XLEN-1:0] mie; logic [CVA6Cfg.XLEN-1:0] mip; logic [CVA6Cfg.XLEN-1:0] mideleg; + logic [CVA6Cfg.XLEN-1:0] hideleg; logic sie; logic global_enable; }, @@ -137,7 +141,11 @@ module cva6 localparam type lsu_ctrl_t = struct packed { logic valid; logic [CVA6Cfg.VLEN-1:0] vaddr; + logic [31:0] tinst; + logic hs_ld_st_inst; + logic hlvx_inst; logic overflow; + logic g_overflow; logic [CVA6Cfg.XLEN-1:0] data; logic [(CVA6Cfg.XLEN/8)-1:0] be; fu_t fu; @@ -297,20 +305,28 @@ module cva6 localparam type interrupts_t = struct packed { logic [CVA6Cfg.XLEN-1:0] S_SW; + logic [CVA6Cfg.XLEN-1:0] VS_SW; logic [CVA6Cfg.XLEN-1:0] M_SW; logic [CVA6Cfg.XLEN-1:0] S_TIMER; + logic [CVA6Cfg.XLEN-1:0] VS_TIMER; logic [CVA6Cfg.XLEN-1:0] M_TIMER; logic [CVA6Cfg.XLEN-1:0] S_EXT; + logic [CVA6Cfg.XLEN-1:0] VS_EXT; logic [CVA6Cfg.XLEN-1:0] M_EXT; + logic [CVA6Cfg.XLEN-1:0] HS_EXT; }; localparam interrupts_t INTERRUPTS = '{ S_SW: (1 << (CVA6Cfg.XLEN - 1)) | CVA6Cfg.XLEN'(riscv::IRQ_S_SOFT), + VS_SW: (1 << (CVA6Cfg.XLEN - 1)) | CVA6Cfg.XLEN'(riscv::IRQ_VS_SOFT), M_SW: (1 << (CVA6Cfg.XLEN - 1)) | CVA6Cfg.XLEN'(riscv::IRQ_M_SOFT), S_TIMER: (1 << (CVA6Cfg.XLEN - 1)) | CVA6Cfg.XLEN'(riscv::IRQ_S_TIMER), + VS_TIMER: (1 << (CVA6Cfg.XLEN - 1)) | CVA6Cfg.XLEN'(riscv::IRQ_VS_TIMER), M_TIMER: (1 << (CVA6Cfg.XLEN - 1)) | CVA6Cfg.XLEN'(riscv::IRQ_M_TIMER), S_EXT: (1 << (CVA6Cfg.XLEN - 1)) | CVA6Cfg.XLEN'(riscv::IRQ_S_EXT), - M_EXT: (1 << (CVA6Cfg.XLEN - 1)) | CVA6Cfg.XLEN'(riscv::IRQ_M_EXT) + VS_EXT: (1 << (CVA6Cfg.XLEN - 1)) | CVA6Cfg.XLEN'(riscv::IRQ_VS_EXT), + M_EXT: (1 << (CVA6Cfg.XLEN - 1)) | CVA6Cfg.XLEN'(riscv::IRQ_M_EXT), + HS_EXT: (1 << (CVA6Cfg.XLEN - 1)) | CVA6Cfg.XLEN'(riscv::IRQ_HS_EXT) }; // ------------------------------------------ @@ -318,6 +334,7 @@ module cva6 // Signals connecting more than one module // ------------------------------------------ riscv::priv_lvl_t priv_lvl; + logic v; exception_t ex_commit; // exception from commit stage bp_resolve_t resolved_branch; logic [ CVA6Cfg.VLEN-1:0] pc_commit; @@ -359,6 +376,7 @@ module cva6 fu_data_t fu_data_id_ex; logic [CVA6Cfg.VLEN-1:0] pc_id_ex; logic is_compressed_instr_id_ex; + logic [31:0] tinst_ex; // fixed latency units logic flu_ready_ex_id; logic [CVA6Cfg.TRANS_ID_BITS-1:0] flu_trans_id_ex_id; @@ -410,6 +428,7 @@ module cva6 logic single_step_acc_commit; // CSR logic csr_valid_id_ex; + logic csr_hs_ld_st_inst_ex; // CVXIF logic [CVA6Cfg.TRANS_ID_BITS-1:0] x_trans_id_ex_id; logic [CVA6Cfg.XLEN-1:0] x_result_ex_id; @@ -457,16 +476,26 @@ module cva6 // -------------- logic [4:0] fflags_csr_commit; riscv::xs_t fs; + riscv::xs_t vfs; logic [2:0] frm_csr_id_issue_ex; logic [6:0] fprec_csr_ex; riscv::xs_t vs; logic enable_translation_csr_ex; + logic enable_g_translation_csr_ex; logic en_ld_st_translation_csr_ex; + logic en_ld_st_g_translation_csr_ex; riscv::priv_lvl_t ld_st_priv_lvl_csr_ex; + logic ld_st_v_csr_ex; logic sum_csr_ex; + logic vs_sum_csr_ex; logic mxr_csr_ex; + logic vmxr_csr_ex; logic [CVA6Cfg.PPNW-1:0] satp_ppn_csr_ex; logic [CVA6Cfg.ASID_WIDTH-1:0] asid_csr_ex; + logic [CVA6Cfg.PPNW-1:0] vsatp_ppn_csr_ex; + logic [CVA6Cfg.ASID_WIDTH-1:0] vs_asid_csr_ex; + logic [CVA6Cfg.PPNW-1:0] hgatp_ppn_csr_ex; + logic [CVA6Cfg.VMID_WIDTH-1:0] vmid_csr_ex; logic [11:0] csr_addr_ex_csr; fu_op csr_op_commit_csr; logic [CVA6Cfg.XLEN-1:0] csr_wdata_commit_csr; @@ -474,7 +503,9 @@ module cva6 exception_t csr_exception_csr_commit; logic tvm_csr_id; logic tw_csr_id; + logic vtw_csr_id; logic tsr_csr_id; + logic hu; irq_ctrl_t irq_ctrl_csr_id; logic dcache_en_csr_nbdcache; logic csr_write_fflags_commit_cs; @@ -510,9 +541,13 @@ module cva6 logic flush_ctrl_ex; logic flush_ctrl_bp; logic flush_tlb_ctrl_ex; + logic flush_tlb_vvma_ctrl_ex; + logic flush_tlb_gvma_ctrl_ex; logic fence_i_commit_controller; logic fence_commit_controller; logic sfence_vma_commit_controller; + logic hfence_vvma_commit_controller; + logic hfence_gvma_commit_controller; logic halt_ctrl; logic halt_csr_ctrl; logic dcache_flush_ctrl_cache; @@ -613,7 +648,9 @@ module cva6 .rvfi_is_compressed_o(rvfi_is_compressed), .priv_lvl_i (priv_lvl), + .v_i (v), .fs_i (fs), + .vfs_i (vfs), .frm_i (frm_csr_id_issue_ex), .vs_i (vs), .irq_i (irq_i), @@ -621,7 +658,9 @@ module cva6 .debug_mode_i(debug_mode), .tvm_i (tvm_csr_id), .tw_i (tw_csr_id), - .tsr_i (tsr_csr_id) + .vtw_i (vtw_csr_id), + .tsr_i (tsr_csr_id), + .hu_i (hu) ); logic [CVA6Cfg.NrWbPorts-1:0][CVA6Cfg.TRANS_ID_BITS-1:0] trans_id_ex_id; @@ -717,6 +756,7 @@ module cva6 .fu_data_o (fu_data_id_ex), .pc_o (pc_id_ex), .is_compressed_instr_o (is_compressed_instr_id_ex), + .tinst_o (tinst_ex), // fixed latency unit ready .flu_ready_i (flu_ready_ex_id), // ALU @@ -783,37 +823,39 @@ module cva6 .icache_drsp_t(icache_drsp_t), .lsu_ctrl_t(lsu_ctrl_t) ) ex_stage_i ( - .clk_i (clk_i), - .rst_ni (rst_ni), - .debug_mode_i (debug_mode), - .flush_i (flush_ctrl_ex), - .rs1_forwarding_i (rs1_forwarding_id_ex), - .rs2_forwarding_i (rs2_forwarding_id_ex), - .fu_data_i (fu_data_id_ex), - .pc_i (pc_id_ex), + .clk_i(clk_i), + .rst_ni(rst_ni), + .debug_mode_i(debug_mode), + .flush_i(flush_ctrl_ex), + .rs1_forwarding_i(rs1_forwarding_id_ex), + .rs2_forwarding_i(rs2_forwarding_id_ex), + .fu_data_i(fu_data_id_ex), + .pc_i(pc_id_ex), .is_compressed_instr_i(is_compressed_instr_id_ex), + .tinst_i(tinst_ex), // fixed latency units - .flu_result_o (flu_result_ex_id), - .flu_trans_id_o (flu_trans_id_ex_id), - .flu_valid_o (flu_valid_ex_id), - .flu_exception_o (flu_exception_ex_id), - .flu_ready_o (flu_ready_ex_id), + .flu_result_o(flu_result_ex_id), + .flu_trans_id_o(flu_trans_id_ex_id), + .flu_valid_o(flu_valid_ex_id), + .flu_exception_o(flu_exception_ex_id), + .flu_ready_o(flu_ready_ex_id), // ALU - .alu_valid_i (alu_valid_id_ex), + .alu_valid_i(alu_valid_id_ex), // Branches and Jumps - .branch_valid_i (branch_valid_id_ex), - .branch_predict_i (branch_predict_id_ex), // branch predict to ex - .resolved_branch_o (resolved_branch), - .resolve_branch_o (resolve_branch_ex_id), + .branch_valid_i(branch_valid_id_ex), + .branch_predict_i(branch_predict_id_ex), // branch predict to ex + .resolved_branch_o(resolved_branch), + .resolve_branch_o(resolve_branch_ex_id), // CSR - .csr_valid_i (csr_valid_id_ex), - .csr_addr_o (csr_addr_ex_csr), - .csr_commit_i (csr_commit_commit_ex), // from commit + .csr_valid_i(csr_valid_id_ex), + .csr_addr_o(csr_addr_ex_csr), + .csr_commit_i(csr_commit_commit_ex), // from commit + .csr_hs_ld_st_inst_o(csr_hs_ld_st_inst_ex), // signals a Hypervisor Load/Store Instruction // MULT - .mult_valid_i (mult_valid_id_ex), + .mult_valid_i(mult_valid_id_ex), // LSU - .lsu_ready_o (lsu_ready_ex_id), - .lsu_valid_i (lsu_valid_id_ex), + .lsu_ready_o(lsu_ready_ex_id), + .lsu_valid_i(lsu_valid_id_ex), .load_result_o (load_result_ex_id), .load_trans_id_o (load_trans_id_ex_id), @@ -825,64 +867,76 @@ module cva6 .store_valid_o (store_valid_ex_id), .store_exception_o(store_exception_ex_id), - .lsu_commit_i (lsu_commit_commit_ex), // from commit - .lsu_commit_ready_o (lsu_commit_ready_ex_commit), // to commit - .commit_tran_id_i (lsu_commit_trans_id), // from commit - .stall_st_pending_i (stall_st_pending_ex), - .no_st_pending_o (no_st_pending_ex), + .lsu_commit_i (lsu_commit_commit_ex), // from commit + .lsu_commit_ready_o (lsu_commit_ready_ex_commit), // to commit + .commit_tran_id_i (lsu_commit_trans_id), // from commit + .stall_st_pending_i (stall_st_pending_ex), + .no_st_pending_o (no_st_pending_ex), // FPU - .fpu_ready_o (fpu_ready_ex_id), - .fpu_valid_i (fpu_valid_id_ex), - .fpu_fmt_i (fpu_fmt_id_ex), - .fpu_rm_i (fpu_rm_id_ex), - .fpu_frm_i (frm_csr_id_issue_ex), - .fpu_prec_i (fprec_csr_ex), - .fpu_trans_id_o (fpu_trans_id_ex_id), - .fpu_result_o (fpu_result_ex_id), - .fpu_valid_o (fpu_valid_ex_id), - .fpu_exception_o (fpu_exception_ex_id), - .amo_valid_commit_i (amo_valid_commit), - .amo_req_o (amo_req), - .amo_resp_i (amo_resp), + .fpu_ready_o (fpu_ready_ex_id), + .fpu_valid_i (fpu_valid_id_ex), + .fpu_fmt_i (fpu_fmt_id_ex), + .fpu_rm_i (fpu_rm_id_ex), + .fpu_frm_i (frm_csr_id_issue_ex), + .fpu_prec_i (fprec_csr_ex), + .fpu_trans_id_o (fpu_trans_id_ex_id), + .fpu_result_o (fpu_result_ex_id), + .fpu_valid_o (fpu_valid_ex_id), + .fpu_exception_o (fpu_exception_ex_id), + .amo_valid_commit_i (amo_valid_commit), + .amo_req_o (amo_req), + .amo_resp_i (amo_resp), // CoreV-X-Interface - .x_valid_i (x_issue_valid_id_ex), - .x_ready_o (x_issue_ready_ex_id), - .x_off_instr_i (x_off_instr_id_ex), - .x_trans_id_o (x_trans_id_ex_id), - .x_exception_o (x_exception_ex_id), - .x_result_o (x_result_ex_id), - .x_valid_o (x_valid_ex_id), - .x_we_o (x_we_ex_id), - .cvxif_req_o (cvxif_req), - .cvxif_resp_i (cvxif_resp), + .x_valid_i (x_issue_valid_id_ex), + .x_ready_o (x_issue_ready_ex_id), + .x_off_instr_i (x_off_instr_id_ex), + .x_trans_id_o (x_trans_id_ex_id), + .x_exception_o (x_exception_ex_id), + .x_result_o (x_result_ex_id), + .x_valid_o (x_valid_ex_id), + .x_we_o (x_we_ex_id), + .cvxif_req_o (cvxif_req), + .cvxif_resp_i (cvxif_resp), // Accelerator - .acc_valid_i (acc_valid_acc_ex), + .acc_valid_i (acc_valid_acc_ex), // Performance counters - .itlb_miss_o (itlb_miss_ex_perf), - .dtlb_miss_o (dtlb_miss_ex_perf), + .itlb_miss_o (itlb_miss_ex_perf), + .dtlb_miss_o (dtlb_miss_ex_perf), // Memory Management - .enable_translation_i (enable_translation_csr_ex), // from CSR - .en_ld_st_translation_i (en_ld_st_translation_csr_ex), - .flush_tlb_i (flush_tlb_ctrl_ex), - .priv_lvl_i (priv_lvl), // from CSR - .ld_st_priv_lvl_i (ld_st_priv_lvl_csr_ex), // from CSR - .sum_i (sum_csr_ex), // from CSR - .mxr_i (mxr_csr_ex), // from CSR - .satp_ppn_i (satp_ppn_csr_ex), // from CSR - .asid_i (asid_csr_ex), // from CSR - .icache_areq_i (icache_areq_cache_ex), - .icache_areq_o (icache_areq_ex_cache), + .enable_translation_i (enable_translation_csr_ex), // from CSR + .enable_g_translation_i (enable_g_translation_csr_ex), // from CSR + .en_ld_st_translation_i (en_ld_st_translation_csr_ex), + .en_ld_st_g_translation_i(en_ld_st_g_translation_csr_ex), + .flush_tlb_i (flush_tlb_ctrl_ex), + .flush_tlb_vvma_i (flush_tlb_vvma_ctrl_ex), + .flush_tlb_gvma_i (flush_tlb_gvma_ctrl_ex), + .priv_lvl_i (priv_lvl), // from CSR + .v_i (v), // from CSR + .ld_st_priv_lvl_i (ld_st_priv_lvl_csr_ex), // from CSR + .ld_st_v_i (ld_st_v_csr_ex), // from CSR + .sum_i (sum_csr_ex), // from CSR + .vs_sum_i (vs_sum_csr_ex), // from CSR + .mxr_i (mxr_csr_ex), // from CSR + .vmxr_i (vmxr_csr_ex), // from CSR + .satp_ppn_i (satp_ppn_csr_ex), // from CSR + .asid_i (asid_csr_ex), // from CSR + .vsatp_ppn_i (vsatp_ppn_csr_ex), // from CSR + .vs_asid_i (vs_asid_csr_ex), // from CSR + .hgatp_ppn_i (hgatp_ppn_csr_ex), // from CSR + .vmid_i (vmid_csr_ex), // from CSR + .icache_areq_i (icache_areq_cache_ex), + .icache_areq_o (icache_areq_ex_cache), // DCACHE interfaces - .dcache_req_ports_i (dcache_req_ports_cache_ex), - .dcache_req_ports_o (dcache_req_ports_ex_cache), - .dcache_wbuffer_empty_i (dcache_commit_wbuffer_empty), - .dcache_wbuffer_not_ni_i(dcache_commit_wbuffer_not_ni), + .dcache_req_ports_i (dcache_req_ports_cache_ex), + .dcache_req_ports_o (dcache_req_ports_ex_cache), + .dcache_wbuffer_empty_i (dcache_commit_wbuffer_empty), + .dcache_wbuffer_not_ni_i (dcache_commit_wbuffer_not_ni), // PMP - .pmpcfg_i (pmpcfg), - .pmpaddr_i (pmpaddr), + .pmpcfg_i (pmpcfg), + .pmpaddr_i (pmpaddr), //RVFI - .rvfi_lsu_ctrl_o (rvfi_lsu_ctrl), - .rvfi_mem_paddr_o (rvfi_mem_paddr) + .rvfi_lsu_ctrl_o (rvfi_lsu_ctrl), + .rvfi_mem_paddr_o (rvfi_mem_paddr) ); // --------- @@ -928,6 +982,8 @@ module cva6 .fence_i_o (fence_i_commit_controller), .fence_o (fence_commit_controller), .sfence_vma_o (sfence_vma_commit_controller), + .hfence_vvma_o (hfence_vvma_commit_controller), + .hfence_gvma_o (hfence_gvma_commit_controller), .flush_commit_o (flush_commit), .* ); @@ -943,59 +999,73 @@ module cva6 .rvfi_probes_csr_t (rvfi_probes_csr_t), .MHPMCounterNum (MHPMCounterNum) ) csr_regfile_i ( - .flush_o (flush_csr_ctrl), - .halt_csr_o (halt_csr_ctrl), - .commit_instr_i (commit_instr_id_commit), - .commit_ack_i (commit_macro_ack), - .boot_addr_i (boot_addr_i[CVA6Cfg.VLEN-1:0]), - .hart_id_i (hart_id_i[CVA6Cfg.XLEN-1:0]), - .ex_i (ex_commit), - .csr_op_i (csr_op_commit_csr), - .csr_write_fflags_i (csr_write_fflags_commit_cs), - .dirty_fp_state_i (dirty_fp_state), - .dirty_v_state_i (dirty_v_state), - .csr_addr_i (csr_addr_ex_csr), - .csr_wdata_i (csr_wdata_commit_csr), - .csr_rdata_o (csr_rdata_csr_commit), - .pc_i (pc_commit), - .csr_exception_o (csr_exception_csr_commit), - .epc_o (epc_commit_pcgen), - .eret_o (eret), - .set_debug_pc_o (set_debug_pc), - .trap_vector_base_o (trap_vector_base_commit_pcgen), - .priv_lvl_o (priv_lvl), - .acc_fflags_ex_i (acc_resp_fflags), - .acc_fflags_ex_valid_i (acc_resp_fflags_valid), - .fs_o (fs), - .fflags_o (fflags_csr_commit), - .frm_o (frm_csr_id_issue_ex), - .fprec_o (fprec_csr_ex), - .vs_o (vs), - .irq_ctrl_o (irq_ctrl_csr_id), - .ld_st_priv_lvl_o (ld_st_priv_lvl_csr_ex), - .en_translation_o (enable_translation_csr_ex), - .en_ld_st_translation_o(en_ld_st_translation_csr_ex), - .sum_o (sum_csr_ex), - .mxr_o (mxr_csr_ex), - .satp_ppn_o (satp_ppn_csr_ex), - .asid_o (asid_csr_ex), - .tvm_o (tvm_csr_id), - .tw_o (tw_csr_id), - .tsr_o (tsr_csr_id), - .debug_mode_o (debug_mode), - .single_step_o (single_step_csr_commit), - .dcache_en_o (dcache_en_csr_nbdcache), - .icache_en_o (icache_en_csr), - .acc_cons_en_o (acc_cons_en_csr), - .perf_addr_o (addr_csr_perf), - .perf_data_o (data_csr_perf), - .perf_data_i (data_perf_csr), - .perf_we_o (we_csr_perf), - .pmpcfg_o (pmpcfg), - .pmpaddr_o (pmpaddr), - .mcountinhibit_o (mcountinhibit_csr_perf), + .flush_o (flush_csr_ctrl), + .halt_csr_o (halt_csr_ctrl), + .commit_instr_i (commit_instr_id_commit), + .commit_ack_i (commit_macro_ack), + .boot_addr_i (boot_addr_i[CVA6Cfg.VLEN-1:0]), + .hart_id_i (hart_id_i[CVA6Cfg.XLEN-1:0]), + .ex_i (ex_commit), + .csr_op_i (csr_op_commit_csr), + .csr_write_fflags_i (csr_write_fflags_commit_cs), + .dirty_fp_state_i (dirty_fp_state), + .dirty_v_state_i (dirty_v_state), + .csr_addr_i (csr_addr_ex_csr), + .csr_wdata_i (csr_wdata_commit_csr), + .csr_rdata_o (csr_rdata_csr_commit), + .pc_i (pc_commit), + .csr_exception_o (csr_exception_csr_commit), + .epc_o (epc_commit_pcgen), + .eret_o (eret), + .set_debug_pc_o (set_debug_pc), + .trap_vector_base_o (trap_vector_base_commit_pcgen), + .priv_lvl_o (priv_lvl), + .v_o (v), + .acc_fflags_ex_i (acc_resp_fflags), + .acc_fflags_ex_valid_i (acc_resp_fflags_valid), + .fs_o (fs), + .vfs_o (vfs), + .fflags_o (fflags_csr_commit), + .frm_o (frm_csr_id_issue_ex), + .fprec_o (fprec_csr_ex), + .vs_o (vs), + .irq_ctrl_o (irq_ctrl_csr_id), + .ld_st_priv_lvl_o (ld_st_priv_lvl_csr_ex), + .ld_st_v_o (ld_st_v_csr_ex), + .csr_hs_ld_st_inst_i (csr_hs_ld_st_inst_ex), + .en_translation_o (enable_translation_csr_ex), + .en_g_translation_o (enable_g_translation_csr_ex), + .en_ld_st_translation_o (en_ld_st_translation_csr_ex), + .en_ld_st_g_translation_o(en_ld_st_g_translation_csr_ex), + .sum_o (sum_csr_ex), + .vs_sum_o (vs_sum_csr_ex), + .mxr_o (mxr_csr_ex), + .vmxr_o (vmxr_csr_ex), + .satp_ppn_o (satp_ppn_csr_ex), + .asid_o (asid_csr_ex), + .vsatp_ppn_o (vsatp_ppn_csr_ex), + .vs_asid_o (vs_asid_csr_ex), + .hgatp_ppn_o (hgatp_ppn_csr_ex), + .vmid_o (vmid_csr_ex), + .tvm_o (tvm_csr_id), + .tw_o (tw_csr_id), + .vtw_o (vtw_csr_id), + .tsr_o (tsr_csr_id), + .hu_o (hu), + .debug_mode_o (debug_mode), + .single_step_o (single_step_csr_commit), + .dcache_en_o (dcache_en_csr_nbdcache), + .icache_en_o (icache_en_csr), + .acc_cons_en_o (acc_cons_en_csr), + .perf_addr_o (addr_csr_perf), + .perf_data_o (data_csr_perf), + .perf_data_i (data_perf_csr), + .perf_we_o (we_csr_perf), + .pmpcfg_o (pmpcfg), + .pmpaddr_o (pmpaddr), + .mcountinhibit_o (mcountinhibit_csr_perf), //RVFI - .rvfi_csr_o (rvfi_csr), + .rvfi_csr_o (rvfi_csr), .debug_req_i, .ipi_i, .irq_i, @@ -1056,6 +1126,8 @@ module cva6 .CVA6Cfg(CVA6Cfg), .bp_resolve_t(bp_resolve_t) ) controller_i ( + // virtualization mode + .v_i (v), // flush ports .set_pc_commit_o (set_pc_ctrl_pcgen), .flush_unissued_instr_o(flush_unissued_instr_ctrl_id), @@ -1064,6 +1136,8 @@ module cva6 .flush_ex_o (flush_ctrl_ex), .flush_bp_o (flush_ctrl_bp), .flush_tlb_o (flush_tlb_ctrl_ex), + .flush_tlb_vvma_o (flush_tlb_vvma_ctrl_ex), + .flush_tlb_gvma_o (flush_tlb_gvma_ctrl_ex), .flush_dcache_o (dcache_flush_ctrl_cache), .flush_dcache_ack_i (dcache_flush_ack_cache_ctrl), @@ -1079,6 +1153,8 @@ module cva6 .fence_i_i (fence_i_commit_controller), .fence_i (fence_commit_controller), .sfence_vma_i (sfence_vma_commit_controller), + .hfence_vvma_i (hfence_vvma_commit_controller), + .hfence_gvma_i (hfence_gvma_commit_controller), .flush_commit_i (flush_commit), .flush_acc_i (flush_acc), diff --git a/core/cva6_rvfi.sv b/core/cva6_rvfi.sv index e268974e93..55bea9fbf9 100644 --- a/core/cva6_rvfi.sv +++ b/core/cva6_rvfi.sv @@ -37,6 +37,7 @@ module cva6_rvfi | (CVA6Cfg.XLEN'(CVA6Cfg.RVC) << 2) // C - Compressed extension | (CVA6Cfg.XLEN'(CVA6Cfg.RVD) << 3) // D - Double precision floating-point extension | (CVA6Cfg.XLEN'(CVA6Cfg.RVF) << 5) // F - Single precision floating-point extension + | (CVA6Cfg.XLEN'(CVA6Cfg.RVH) << 7) // H - Hypervisor extension | (CVA6Cfg.XLEN'(1) << 8) // I - RV32I/64I/128I base ISA | (CVA6Cfg.XLEN'(1) << 12) // M - Integer Multiply/Divide extension | (CVA6Cfg.XLEN'(0) << 13) // N - User level interrupts supported diff --git a/core/cvxif_fu.sv b/core/cvxif_fu.sv index 47233b17ec..4630be9e1a 100644 --- a/core/cvxif_fu.sv +++ b/core/cvxif_fu.sv @@ -95,6 +95,9 @@ module cvxif_fu x_exception_o.cause = x_valid_o ? {{(CVA6Cfg.XLEN-6){1'b0}}, cvxif_resp_i.x_result.exccode} : '0; x_exception_o.valid = x_valid_o ? cvxif_resp_i.x_result.exc : '0; x_exception_o.tval = '0; + x_exception_o.tinst = '0; + x_exception_o.tval2 = '0; + x_exception_o.gva = '0; x_we_o = x_valid_o ? cvxif_resp_i.x_result.we : '0; if (illegal_n) begin if (~x_valid_o) begin @@ -104,6 +107,9 @@ module cvxif_fu x_exception_o.cause = riscv::ILLEGAL_INSTR; x_exception_o.valid = 1'b1; if (CVA6Cfg.TvalEn) x_exception_o.tval = illegal_instr_n; + x_exception_o.tinst = '0; + x_exception_o.tval2 = '0; + x_exception_o.gva = '0; x_we_o = '0; illegal_n = '0; // Reset flag for illegal instr. illegal_id and illegal instr values are a don't care, no need to reset it. end diff --git a/core/decoder.sv b/core/decoder.sv index e8b9b2af0e..8bd9312ef6 100644 --- a/core/decoder.sv +++ b/core/decoder.sv @@ -58,10 +58,14 @@ module decoder input irq_ctrl_t irq_ctrl_i, // Current privilege level - CSR_REGFILE input riscv::priv_lvl_t priv_lvl_i, + // Current virtualization mode - CSR_REGFILE + input logic v_i, // Is debug mode - CSR_REGFILE input logic debug_mode_i, // Floating point extension status - CSR_REGFILE input riscv::xs_t fs_i, + // Virtual floating point extension status - CSR_REGFILE + input riscv::xs_t vfs_i, // Floating-point dynamic rounding mode - CSR_REGFILE input logic [2:0] frm_i, // Vector extension status - CSR_REGFILE @@ -70,8 +74,12 @@ module decoder input logic tvm_i, // Timeout wait - CSR_REGFILE input logic tw_i, + // Virtual timeout wait - CSR_REGFILE + input logic vtw_i, // Trap sret - CSR_REGFILE input logic tsr_i, + // Hypervisor user mode - CSR_REGFILE + input logic hu_i, // Instruction to be added to scoreboard entry - ISSUE_STAGE output scoreboard_entry_t instruction_o, // Instruction - ISSUE_STAGE @@ -83,6 +91,7 @@ module decoder logic illegal_instr_bm; logic illegal_instr_zic; logic illegal_instr_non_bm; + logic virtual_illegal_instr; // this instruction is an environment call (ecall), it is handled like an exception logic ecall; // this instruction is a software break-point @@ -91,6 +100,8 @@ module decoder logic check_fprm; riscv::instruction_t instr; assign instr = riscv::instruction_t'(instruction_i); + // transformed instruction + logic [31:0] tinst; // -------------------- // Immediate select // -------------------- @@ -150,6 +161,7 @@ module decoder illegal_instr_non_bm = 1'b0; illegal_instr_bm = 1'b0; illegal_instr_zic = 1'b0; + virtual_illegal_instr = 1'b0; instruction_o.pc = pc_i; instruction_o.trans_id = '0; instruction_o.fu = NONE; @@ -165,6 +177,7 @@ module decoder instruction_o.use_zimm = 1'b0; instruction_o.bp = branch_predict_i; instruction_o.vfp = 1'b0; + tinst = '0; ecall = 1'b0; ebreak = 1'b0; check_fprm = 1'b0; @@ -180,7 +193,13 @@ module decoder unique case (instr.itype.funct3) 3'b000: begin // check if the RD and and RS1 fields are zero, this may be reset for the SENCE.VMA instruction - if (instr.itype.rs1 != '0 || instr.itype.rd != '0) illegal_instr = 1'b1; + if (instr.itype.rs1 != '0 || instr.itype.rd != '0) begin + if (CVA6Cfg.RVH && v_i) begin + virtual_illegal_instr = 1'b1; + end else begin + illegal_instr = 1'b1; + end + end // decode the immiediate field case (instr.itype.imm) // ECALL -> inject exception @@ -194,13 +213,21 @@ module decoder // check privilege level, SRET can only be executed in S and M mode // we'll just decode an illegal instruction if we are in the wrong privilege level if (CVA6Cfg.RVU && priv_lvl_i == riscv::PRIV_LVL_U) begin - illegal_instr = 1'b1; + if (CVA6Cfg.RVH && v_i) begin + virtual_illegal_instr = 1'b1; + end else begin + illegal_instr = 1'b1; + end // do not change privilege level if this is an illegal instruction instruction_o.op = ariane_pkg::ADD; end // if we are in S-Mode and Trap SRET (tsr) is set -> trap on illegal instruction if (priv_lvl_i == riscv::PRIV_LVL_S && tsr_i) begin - illegal_instr = 1'b1; + if (CVA6Cfg.RVH && v_i) begin + virtual_illegal_instr = 1'b1; + end else begin + illegal_instr = 1'b1; + end // do not change privilege level if this is an illegal instruction instruction_o.op = ariane_pkg::ADD; end @@ -236,9 +263,14 @@ module decoder illegal_instr = 1'b1; instruction_o.op = ariane_pkg::ADD; end + if (CVA6Cfg.RVH && priv_lvl_i == riscv::PRIV_LVL_S && v_i && vtw_i && !tw_i) begin + virtual_illegal_instr = 1'b1; + instruction_o.op = ariane_pkg::ADD; + end // we don't support U mode interrupts so WFI is illegal in this context if (CVA6Cfg.RVU && priv_lvl_i == riscv::PRIV_LVL_U) begin - illegal_instr = 1'b1; + if (CVA6Cfg.RVH && v_i) virtual_illegal_instr = 1'b1; + else illegal_instr = 1'b1; instruction_o.op = ariane_pkg::ADD; end end @@ -248,17 +280,110 @@ module decoder // check privilege level, SFENCE.VMA can only be executed in M/S mode // only if S mode is supported // otherwise decode an illegal instruction - illegal_instr = (CVA6Cfg.RVS && (priv_lvl_i inside {riscv::PRIV_LVL_M, riscv::PRIV_LVL_S}) && instr.itype.rd == '0) ? 1'b0 : 1'b1; + if (CVA6Cfg.RVH && v_i) begin + virtual_illegal_instr = (priv_lvl_i == riscv::PRIV_LVL_S) ? 1'b0 : 1'b1; + end else begin + illegal_instr = (CVA6Cfg.RVS && (priv_lvl_i inside {riscv::PRIV_LVL_M, riscv::PRIV_LVL_S}) && instr.itype.rd == '0) ? 1'b0 : 1'b1; + end instruction_o.op = ariane_pkg::SFENCE_VMA; // check TVM flag and intercept SFENCE.VMA call if necessary - if (CVA6Cfg.RVS && priv_lvl_i == riscv::PRIV_LVL_S && tvm_i) - illegal_instr = 1'b1; - end else begin - illegal_instr = 1'b1; + if (CVA6Cfg.RVS && priv_lvl_i == riscv::PRIV_LVL_S && tvm_i) begin + if (CVA6Cfg.RVH && v_i) virtual_illegal_instr = 1'b1; + else illegal_instr = 1'b1; + end + end + if (CVA6Cfg.RVH) begin + if (instr.instr[31:25] == 7'b10001) begin + // check privilege level, HFENCE.VVMA can only be executed in M/S mode + // otherwise decode an illegal instruction or virtual illegal instruction + if (v_i) begin + virtual_illegal_instr = 1'b1; + end else begin + illegal_instr = (priv_lvl_i inside {riscv::PRIV_LVL_M, riscv::PRIV_LVL_S}) ? 1'b0 : 1'b1; + end + instruction_o.op = ariane_pkg::HFENCE_VVMA; + end + if (instr.instr[31:25] == 7'b110001) begin + // check privilege level, HFENCE.GVMA can only be executed in M/S mode + // otherwise decode an illegal instruction or virtual illegal instruction + if (v_i) begin + virtual_illegal_instr = 1'b1; + end else begin + illegal_instr = (priv_lvl_i inside {riscv::PRIV_LVL_M, riscv::PRIV_LVL_S}) ? 1'b0 : 1'b1; + end + instruction_o.op = ariane_pkg::HFENCE_GVMA; + // check TVM flag and intercept HFENCE.GVMA call if necessary + if (priv_lvl_i == riscv::PRIV_LVL_S && !v_i && tvm_i) illegal_instr = 1'b1; + end end end endcase end + 3'b100: begin + if (instr.instr[25] != 1'b0) begin + instruction_o.fu = STORE; + imm_select = NOIMM; + instruction_o.rs1[4:0] = instr.stype.rs1; + instruction_o.rs2[4:0] = instr.stype.rs2; + end else begin + instruction_o.fu = LOAD; + imm_select = NOIMM; + instruction_o.rs1[4:0] = instr.itype.rs1; + instruction_o.rd[4:0] = instr.itype.rd; + end + // Hypervisor load/store instructions when V=1 cause virtual instruction + if (CVA6Cfg.RVH) begin + if (v_i) virtual_illegal_instr = 1'b1; + // Hypervisor load/store instructions in U-mode when hstatus.HU=0 cause an illegal instruction trap. + else if (!hu_i && priv_lvl_i == riscv::PRIV_LVL_U) illegal_instr = 1'b1; + unique case (instr.rtype.funct7) + 7'b011_0000: begin + if (instr.rtype.rs2 == 5'b0) begin + instruction_o.op = ariane_pkg::HLV_B; + end + if (instr.rtype.rs2 == 5'b1) begin + instruction_o.op = ariane_pkg::HLV_BU; + end + end + 7'b011_0010: begin + if (instr.rtype.rs2 == 5'b0) begin + instruction_o.op = ariane_pkg::HLV_H; + end + if (instr.rtype.rs2 == 5'b1) begin + instruction_o.op = ariane_pkg::HLV_HU; + end + if (instr.rtype.rs2 == 5'b11) begin + instruction_o.op = ariane_pkg::HLVX_HU; + end + end + 7'b011_0100: begin + if (instr.rtype.rs2 == 5'b0) begin + instruction_o.op = ariane_pkg::HLV_W; + end + if (instr.rtype.rs2 == 5'b1) begin + instruction_o.op = ariane_pkg::HLV_WU; + end + if (instr.rtype.rs2 == 5'b11) begin + instruction_o.op = ariane_pkg::HLVX_WU; + end + end + 7'b011_0001: instruction_o.op = ariane_pkg::HSV_B; + 7'b011_0011: instruction_o.op = ariane_pkg::HSV_H; + 7'b011_0101: instruction_o.op = ariane_pkg::HSV_W; + 7'b011_0110: instruction_o.op = ariane_pkg::HLV_D; + 7'b011_0111: instruction_o.op = ariane_pkg::HSV_D; + + endcase + tinst = { + instr.rtype.funct7, + instr.rtype.rs2, + 5'b0, + instr.rtype.funct3, + instr.rtype.rd, + instr.rtype.opcode + }; + end + end // atomically swaps values in the CSR and integer register 3'b001: begin // CSRRW imm_select = IIMM; @@ -331,7 +456,7 @@ module decoder // -------------------------------------------- if (instr.rvftype.funct2 == 2'b10) begin // Prefix 10 for all Xfvec ops // only generate decoder if FP extensions are enabled (static) - if (CVA6Cfg.FpPresent && CVA6Cfg.XFVec && fs_i != riscv::Off) begin + if (CVA6Cfg.FpPresent && CVA6Cfg.XFVec && fs_i != riscv::Off && ((CVA6Cfg.RVH && (!v_i || vfs_i != riscv::Off)) || !CVA6Cfg.RVH)) begin automatic logic allow_replication; // control honoring of replication flag instruction_o.fu = FPU_VEC; // Same unit, but sets 'vectorial' signal @@ -852,6 +977,10 @@ module decoder else illegal_instr = 1'b1; default: illegal_instr = 1'b1; endcase + if (CVA6Cfg.RVH) begin + tinst = {7'b0, instr.stype.rs2, 5'b0, instr.stype.funct3, 5'b0, instr.stype.opcode}; + tinst[1] = is_compressed_i ? 1'b0 : 'b1; + end end riscv::OpcodeLoad: begin @@ -874,13 +1003,17 @@ module decoder else illegal_instr = 1'b1; default: illegal_instr = 1'b1; endcase + if (CVA6Cfg.RVH) begin + tinst = {17'b0, instr.itype.funct3, instr.itype.rd, instr.itype.opcode}; + tinst[1] = is_compressed_i ? 1'b0 : 'b1; + end end // -------------------------------- // Floating-Point Load/store // -------------------------------- riscv::OpcodeStoreFp: begin - if (CVA6Cfg.FpPresent && fs_i != riscv::Off) begin // only generate decoder if FP extensions are enabled (static) + if (CVA6Cfg.FpPresent && fs_i != riscv::Off && ((CVA6Cfg.RVH && (!v_i || vfs_i != riscv::Off)) || !CVA6Cfg.RVH)) begin // only generate decoder if FP extensions are enabled (static) instruction_o.fu = STORE; imm_select = SIMM; instruction_o.rs1[4:0] = instr.stype.rs1; @@ -902,11 +1035,15 @@ module decoder else illegal_instr = 1'b1; default: illegal_instr = 1'b1; endcase + if (CVA6Cfg.RVH) begin + tinst = {7'b0, instr.stype.rs2, 5'b0, instr.stype.funct3, 5'b0, instr.stype.opcode}; + tinst[1] = is_compressed_i ? 1'b0 : 'b1; + end end else illegal_instr = 1'b1; end riscv::OpcodeLoadFp: begin - if (CVA6Cfg.FpPresent && fs_i != riscv::Off) begin // only generate decoder if FP extensions are enabled (static) + if (CVA6Cfg.FpPresent && fs_i != riscv::Off && ((CVA6Cfg.RVH && (!v_i || vfs_i != riscv::Off)) || !CVA6Cfg.RVH)) begin // only generate decoder if FP extensions are enabled (static) instruction_o.fu = LOAD; imm_select = IIMM; instruction_o.rs1[4:0] = instr.itype.rs1; @@ -928,6 +1065,10 @@ module decoder else illegal_instr = 1'b1; default: illegal_instr = 1'b1; endcase + if (CVA6Cfg.RVH) begin + tinst = {17'b0, instr.itype.funct3, instr.itype.rd, instr.itype.opcode}; + tinst[1] = is_compressed_i ? 1'b0 : 'b1; + end end else illegal_instr = 1'b1; end @@ -935,7 +1076,7 @@ module decoder // Floating-Point Reg-Reg Operations // ---------------------------------- riscv::OpcodeMadd, riscv::OpcodeMsub, riscv::OpcodeNmsub, riscv::OpcodeNmadd: begin - if (CVA6Cfg.FpPresent && fs_i != riscv::Off) begin // only generate decoder if FP extensions are enabled (static) + if (CVA6Cfg.FpPresent && fs_i != riscv::Off && ((CVA6Cfg.RVH && (!v_i || vfs_i != riscv::Off)) || !CVA6Cfg.RVH)) begin // only generate decoder if FP extensions are enabled (static) instruction_o.fu = FPU; instruction_o.rs1[4:0] = instr.r4type.rs1; instruction_o.rs2[4:0] = instr.r4type.rs2; @@ -990,7 +1131,7 @@ module decoder end riscv::OpcodeOpFp: begin - if (CVA6Cfg.FpPresent && fs_i != riscv::Off) begin // only generate decoder if FP extensions are enabled (static) + if (CVA6Cfg.FpPresent && fs_i != riscv::Off && ((CVA6Cfg.RVH && (!v_i || vfs_i != riscv::Off)) || !CVA6Cfg.RVH)) begin // only generate decoder if FP extensions are enabled (static) instruction_o.fu = FPU; instruction_o.rs1[4:0] = instr.rftype.rs1; instruction_o.rs2[4:0] = instr.rftype.rs2; @@ -1186,6 +1327,16 @@ module decoder end else begin illegal_instr = 1'b1; end + tinst = { + instr.atype.funct5, + instr.atype.aq, + instr.atype.rl, + instr.atype.rs2, + 5'b0, + instr.atype.funct3, + instr.atype.rd, + instr.atype.opcode + }; end // -------------------------------- @@ -1366,6 +1517,8 @@ module decoder if (CVA6Cfg.TvalEn) instruction_o.ex.tval = (is_compressed_i) ? {{CVA6Cfg.XLEN-16{1'b0}}, compressed_instr_i} : {{CVA6Cfg.XLEN-32{1'b0}}, instruction_i}; else instruction_o.ex.tval = '0; + if (CVA6Cfg.RVH) instruction_o.ex.tinst = tinst; + else instruction_o.ex.tinst = '0; // instructions which will throw an exception are marked as valid // e.g.: they can be committed anytime and do not need to wait for any functional unit // check here if we decoded an invalid instruction or if the compressed decoder already decoded @@ -1374,13 +1527,17 @@ module decoder if (!CVA6Cfg.CvxifEn) instruction_o.ex.valid = 1'b1; // we decoded an illegal exception here instruction_o.ex.cause = riscv::ILLEGAL_INSTR; + end else if (CVA6Cfg.RVH && virtual_illegal_instr) begin + instruction_o.ex.valid = 1'b1; + // we decoded an virtual illegal exception here + instruction_o.ex.cause = riscv::VIRTUAL_INSTRUCTION; // we got an ecall, set the correct cause depending on the current privilege level end else if (ecall) begin // this exception is valid instruction_o.ex.valid = 1'b1; // depending on the privilege mode, set the appropriate cause if (priv_lvl_i == riscv::PRIV_LVL_S && CVA6Cfg.RVS) begin - instruction_o.ex.cause = riscv::ENV_CALL_SMODE; + instruction_o.ex.cause = (CVA6Cfg.RVH && v_i) ? riscv::ENV_CALL_VSMODE : riscv::ENV_CALL_SMODE; end else if (priv_lvl_i == riscv::PRIV_LVL_U && CVA6Cfg.RVU) begin instruction_o.ex.cause = riscv::ENV_CALL_UMODE; end else if (priv_lvl_i == riscv::PRIV_LVL_M) begin @@ -1391,6 +1548,9 @@ module decoder instruction_o.ex.valid = 1'b1; // set breakpoint cause instruction_o.ex.cause = riscv::BREAKPOINT; + // set gva bit + if (CVA6Cfg.RVH) instruction_o.ex.gva = v_i; + else instruction_o.ex.gva = 1'b0; end // ----------------- // Interrupt Control @@ -1399,6 +1559,24 @@ module decoder // throw any previous exception. // we have three interrupt sources: external interrupts, software interrupts, timer interrupts (order of precedence) // for two privilege levels: Supervisor and Machine Mode + // Virtual Supervisor Timer Interrupt + if (CVA6Cfg.RVH) begin + if (irq_ctrl_i.mie[riscv::IRQ_VS_TIMER] && irq_ctrl_i.mip[riscv::IRQ_VS_TIMER]) begin + interrupt_cause = INTERRUPTS.VS_TIMER; + end + // Virtual Supervisor Software Interrupt + if (irq_ctrl_i.mie[riscv::IRQ_VS_SOFT] && irq_ctrl_i.mip[riscv::IRQ_VS_SOFT]) begin + interrupt_cause = INTERRUPTS.VS_SW; + end + // Virtual Supervisor External Interrupt + if (irq_ctrl_i.mie[riscv::IRQ_VS_EXT] && (irq_ctrl_i.mip[riscv::IRQ_VS_EXT])) begin + interrupt_cause = INTERRUPTS.VS_EXT; + end + // Hypervisor Guest External Interrupts + if (irq_ctrl_i.mie[riscv::IRQ_HS_EXT] && irq_ctrl_i.mip[riscv::IRQ_HS_EXT]) begin + interrupt_cause = INTERRUPTS.HS_EXT; + end + end // Supervisor Timer Interrupt if (irq_ctrl_i.mie[riscv::IRQ_S_TIMER] && irq_ctrl_i.mip[riscv::IRQ_S_TIMER]) begin interrupt_cause = INTERRUPTS.S_TIMER; @@ -1431,9 +1609,28 @@ module decoder // mode equals the delegated privilege mode (S or U) and that mode’s interrupt enable bit // (SIE or UIE in mstatus) is set, or if the current privilege mode is less than the delegated privilege mode. if (irq_ctrl_i.mideleg[interrupt_cause[$clog2(CVA6Cfg.XLEN)-1:0]]) begin - if ((CVA6Cfg.RVS && irq_ctrl_i.sie && priv_lvl_i == riscv::PRIV_LVL_S) || (CVA6Cfg.RVU && priv_lvl_i == riscv::PRIV_LVL_U)) begin - instruction_o.ex.valid = 1'b1; - instruction_o.ex.cause = interrupt_cause; + if (CVA6Cfg.RVH) begin : hyp_int_gen + if (v_i && irq_ctrl_i.hideleg[interrupt_cause[$clog2(CVA6Cfg.XLEN)-1:0]]) begin + if ((irq_ctrl_i.sie && priv_lvl_i == riscv::PRIV_LVL_S) || priv_lvl_i == riscv::PRIV_LVL_U) begin + instruction_o.ex.valid = 1'b1; + instruction_o.ex.cause = interrupt_cause; + end + end else if (v_i && ~irq_ctrl_i.hideleg[interrupt_cause[$clog2( + CVA6Cfg.XLEN + )-1:0]]) begin + instruction_o.ex.valid = 1'b1; + instruction_o.ex.cause = interrupt_cause; + end else if (!v_i && ((irq_ctrl_i.sie && priv_lvl_i == riscv::PRIV_LVL_S) || priv_lvl_i == riscv::PRIV_LVL_U) && ~irq_ctrl_i.hideleg[interrupt_cause[$clog2( + CVA6Cfg.XLEN + )-1:0]]) begin + instruction_o.ex.valid = 1'b1; + instruction_o.ex.cause = interrupt_cause; + end + end else begin + if ((CVA6Cfg.RVS && irq_ctrl_i.sie && priv_lvl_i == riscv::PRIV_LVL_S) || (CVA6Cfg.RVU && priv_lvl_i == riscv::PRIV_LVL_U)) begin + instruction_o.ex.valid = 1'b1; + instruction_o.ex.cause = interrupt_cause; + end end end else begin instruction_o.ex.valid = 1'b1; diff --git a/core/ex_stage.sv b/core/ex_stage.sv index 760b5a5f8f..a07a8ec7e3 100644 --- a/core/ex_stage.sv +++ b/core/ex_stage.sv @@ -48,6 +48,8 @@ module ex_stage input logic [CVA6Cfg.VLEN-1:0] pc_i, // Report whether isntruction is compressed - ISSUE_STAGE input logic is_compressed_instr_i, + // Report instruction encoding - ISSUE_STAGE + input logic [31:0] tinst_i, // Fixed Latency Unit result - ISSUE_STAGE output logic [CVA6Cfg.XLEN-1:0] flu_result_o, // ID of the scoreboard entry at which a=to write back - ISSUE_STAGE @@ -152,22 +154,46 @@ module ex_stage input logic acc_valid_i, // Enable virtual memory translation - CSR_REGFILE input logic enable_translation_i, + // Enable G-Stage memory translation - CSR_REGFILE + input logic enable_g_translation_i, // Enable virtual memory translation for load/stores - CSR_REGFILE input logic en_ld_st_translation_i, + // Enable G-Stage memory translation for load/stores - CSR_REGFILE + input logic en_ld_st_g_translation_i, // Flush TLB - CONTROLLER input logic flush_tlb_i, + input logic flush_tlb_vvma_i, + input logic flush_tlb_gvma_i, // Privilege mode - CSR_REGFILE input riscv::priv_lvl_t priv_lvl_i, + // Virtualization mode - CSR_REGFILE + input logic v_i, // Privilege level at which load and stores should happen - CSR_REGFILE input riscv::priv_lvl_t ld_st_priv_lvl_i, + // Virtualization mode at which load and stores should happen - CSR_REGFILE + input logic ld_st_v_i, + // Instruction is hypervisor load/store - CSR_REGFILE + output logic csr_hs_ld_st_inst_o, // Supervisor user memory - CSR_REGFILE input logic sum_i, + // Virtual Supervisor user memory - CSR_REGFILE + input logic vs_sum_i, // Make executable readable - CSR_REGFILE input logic mxr_i, + // Make executable readable Virtual Supervisor - CSR_REGFILE + input logic vmxr_i, // TO_BE_COMPLETED - CSR_REGFILE input logic [CVA6Cfg.PPNW-1:0] satp_ppn_i, // TO_BE_COMPLETED - CSR_REGFILE input logic [CVA6Cfg.ASID_WIDTH-1:0] asid_i, + // TO_BE_COMPLETED - CSR_REGFILE + input logic [CVA6Cfg.PPNW-1:0] vsatp_ppn_i, + // TO_BE_COMPLETED - CSR_REGFILE + input logic [CVA6Cfg.ASID_WIDTH-1:0] vs_asid_i, + // TO_BE_COMPLETED - CSR_REGFILE + input logic [CVA6Cfg.PPNW-1:0] hgatp_ppn_i, + // TO_BE_COMPLETED - CSR_REGFILE + input logic [CVA6Cfg.VMID_WIDTH-1:0] vmid_i, // icache translation response - CACHE input icache_arsp_t icache_areq_i, // icache translation request - CACHE @@ -220,10 +246,14 @@ module ex_stage logic current_instruction_is_sfence_vma; + logic current_instruction_is_hfence_vvma; + logic current_instruction_is_hfence_gvma; // These two register store the rs1 and rs2 parameters in case of `SFENCE_VMA` // instruction to be used for TLB flush in the next clock cycle. + logic [CVA6Cfg.VMID_WIDTH-1:0] vmid_to_be_flushed; logic [CVA6Cfg.ASID_WIDTH-1:0] asid_to_be_flushed; logic [CVA6Cfg.VLEN-1:0] vaddr_to_be_flushed; + logic [CVA6Cfg.GPLEN-1:0] gpaddr_to_be_flushed; // from ALU to branch unit logic alu_branch_res; // branch comparison result @@ -261,6 +291,7 @@ module ex_stage ) branch_unit_i ( .clk_i, .rst_ni, + .v_i, .debug_mode_i, .fu_data_i, .pc_i, @@ -397,7 +428,7 @@ module ex_stage .flush_i, .stall_st_pending_i, .no_st_pending_o, - .fu_data_i (lsu_data), + .fu_data_i (lsu_data), .lsu_ready_o, .lsu_valid_i, .load_trans_id_o, @@ -408,22 +439,37 @@ module ex_stage .store_result_o, .store_valid_o, .store_exception_o, - .commit_i (lsu_commit_i), - .commit_ready_o (lsu_commit_ready_o), + .commit_i (lsu_commit_i), + .commit_ready_o (lsu_commit_ready_o), .commit_tran_id_i, .enable_translation_i, + .enable_g_translation_i, .en_ld_st_translation_i, + .en_ld_st_g_translation_i, .icache_areq_i, .icache_areq_o, .priv_lvl_i, + .v_i, .ld_st_priv_lvl_i, + .ld_st_v_i, + .csr_hs_ld_st_inst_o, .sum_i, + .vs_sum_i, .mxr_i, + .vmxr_i, .satp_ppn_i, + .vsatp_ppn_i, + .hgatp_ppn_i, .asid_i, - .asid_to_be_flushed_i (asid_to_be_flushed), - .vaddr_to_be_flushed_i(vaddr_to_be_flushed), + .vs_asid_i, + .asid_to_be_flushed_i (asid_to_be_flushed), + .vmid_i, + .vmid_to_be_flushed_i (vmid_to_be_flushed), + .vaddr_to_be_flushed_i (vaddr_to_be_flushed), + .gpaddr_to_be_flushed_i(gpaddr_to_be_flushed), .flush_tlb_i, + .flush_tlb_vvma_i, + .flush_tlb_gvma_i, .itlb_miss_o, .dtlb_miss_o, .dcache_req_ports_i, @@ -433,6 +479,7 @@ module ex_stage .amo_valid_commit_i, .amo_req_o, .amo_resp_i, + .tinst_i, .pmpcfg_i, .pmpaddr_i, .rvfi_lsu_ctrl_o, @@ -471,33 +518,80 @@ module ex_stage end if (CVA6Cfg.RVS) begin - always_ff @(posedge clk_i or negedge rst_ni) begin - if (~rst_ni) begin - current_instruction_is_sfence_vma <= 1'b0; - end else begin - if (flush_i) begin + if (CVA6Cfg.RVH) begin + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + current_instruction_is_sfence_vma <= 1'b0; + current_instruction_is_hfence_vvma <= 1'b0; + current_instruction_is_hfence_gvma <= 1'b0; + end else begin + if (flush_i) begin + current_instruction_is_sfence_vma <= 1'b0; + current_instruction_is_hfence_vvma <= 1'b0; + current_instruction_is_hfence_gvma <= 1'b0; + end else if ((fu_data_i.operation == SFENCE_VMA && !v_i) && csr_valid_i) begin + current_instruction_is_sfence_vma <= 1'b1; + end else if (((fu_data_i.operation == SFENCE_VMA && v_i) || fu_data_i.operation == HFENCE_VVMA) && csr_valid_i) begin + current_instruction_is_hfence_vvma <= 1'b1; + end else if ((fu_data_i.operation == HFENCE_GVMA) && csr_valid_i) begin + current_instruction_is_hfence_gvma <= 1'b1; + end + end + end + end else begin + assign current_instruction_is_hfence_vvma = 1'b0; + assign current_instruction_is_hfence_gvma = 1'b0; + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin current_instruction_is_sfence_vma <= 1'b0; - end else if ((fu_data_i.operation == SFENCE_VMA) && csr_valid_i) begin - current_instruction_is_sfence_vma <= 1'b1; + end else begin + if (flush_i) begin + current_instruction_is_sfence_vma <= 1'b0; + end else if (fu_data_i.operation == SFENCE_VMA && csr_valid_i) begin + current_instruction_is_sfence_vma <= 1'b1; + end end end end - - // This process stores the rs1 and rs2 parameters of a SFENCE_VMA instruction. - always_ff @(posedge clk_i or negedge rst_ni) begin - if (~rst_ni) begin - asid_to_be_flushed <= '0; - vaddr_to_be_flushed <= '0; - // if the current instruction in EX_STAGE is a sfence.vma, in the next cycle no writes will happen - end else if ((~current_instruction_is_sfence_vma) && (~((fu_data_i.operation == SFENCE_VMA) && csr_valid_i))) begin - vaddr_to_be_flushed <= rs1_forwarding_i; - asid_to_be_flushed <= rs2_forwarding_i[CVA6Cfg.ASID_WIDTH-1:0]; + if (CVA6Cfg.RVH) begin + // This process stores the rs1 and rs2 parameters of a SFENCE_VMA instruction. + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + vmid_to_be_flushed <= '0; + asid_to_be_flushed <= '0; + vaddr_to_be_flushed <= '0; + gpaddr_to_be_flushed <= '0; + // if the current instruction in EX_STAGE is a sfence.vma, in the next cycle no writes will happen + end else if ((~(current_instruction_is_sfence_vma || current_instruction_is_hfence_vvma || current_instruction_is_hfence_gvma)) && (~((fu_data_i.operation == SFENCE_VMA || fu_data_i.operation == HFENCE_VVMA || fu_data_i.operation == HFENCE_GVMA ) && csr_valid_i))) begin + vaddr_to_be_flushed <= rs1_forwarding_i; + gpaddr_to_be_flushed <= {2'b00, rs1_forwarding_i[CVA6Cfg.GPLEN-1:2]}; + asid_to_be_flushed <= rs2_forwarding_i[CVA6Cfg.ASID_WIDTH-1:0]; + vmid_to_be_flushed <= rs2_forwarding_i[CVA6Cfg.VMID_WIDTH-1:0]; + end + end + end else begin + assign vmid_to_be_flushed = '0; + assign gpaddr_to_be_flushed = '0; + // This process stores the rs1 and rs2 parameters of a SFENCE_VMA instruction. + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + asid_to_be_flushed <= '0; + vaddr_to_be_flushed <= '0; + // if the current instruction in EX_STAGE is a sfence.vma, in the next cycle no writes will happen + end else if ((~current_instruction_is_sfence_vma) && (~((fu_data_i.operation == SFENCE_VMA) && csr_valid_i))) begin + vaddr_to_be_flushed <= rs1_forwarding_i; + asid_to_be_flushed <= rs2_forwarding_i[CVA6Cfg.ASID_WIDTH-1:0]; + end end end end else begin - assign current_instruction_is_sfence_vma = 1'b0; - assign asid_to_be_flushed = '0; - assign vaddr_to_be_flushed = '0; + assign current_instruction_is_sfence_vma = 1'b0; + assign current_instruction_is_hfence_vvma = 1'b0; + assign current_instruction_is_hfence_gvma = 1'b0; + assign asid_to_be_flushed = '0; + assign vaddr_to_be_flushed = '0; + assign vmid_to_be_flushed = '0; + assign gpaddr_to_be_flushed = '0; end endmodule diff --git a/core/frontend/frontend.sv b/core/frontend/frontend.sv index 2c64d630da..bc3dea9de0 100644 --- a/core/frontend/frontend.sv +++ b/core/frontend/frontend.sv @@ -93,6 +93,9 @@ module frontend logic icache_valid_q; ariane_pkg::frontend_exception_t icache_ex_valid_q; logic [ CVA6Cfg.VLEN-1:0] icache_vaddr_q; + logic [ CVA6Cfg.GPLEN-1:0] icache_gpaddr_q; + logic [ 31:0] icache_tinst_q; + logic icache_gva_q; logic instr_queue_ready; logic [CVA6Cfg.INSTR_PER_FETCH-1:0] instr_queue_consumed; // upper-most branch-prediction from last cycle @@ -165,7 +168,6 @@ module frontend .addr_o (addr), .instr_o (instr) ); - // -------------------- // Branch Prediction // -------------------- @@ -418,6 +420,9 @@ module frontend icache_data_q <= '0; icache_valid_q <= 1'b0; icache_vaddr_q <= 'b0; + icache_gpaddr_q <= 'b0; + icache_tinst_q <= 'b0; + icache_gva_q <= 1'b0; icache_ex_valid_q <= ariane_pkg::FE_NONE; btb_q <= '0; bht_q <= '0; @@ -429,8 +434,20 @@ module frontend if (icache_dreq_i.valid) begin icache_data_q <= icache_data; icache_vaddr_q <= icache_dreq_i.vaddr; + if (CVA6Cfg.RVH) begin + icache_gpaddr_q <= icache_dreq_i.ex.tval2[CVA6Cfg.GPLEN-1:0]; + icache_tinst_q <= icache_dreq_i.ex.tinst; + icache_gva_q <= icache_dreq_i.ex.gva; + end else begin + icache_gpaddr_q <= 'b0; + icache_tinst_q <= 'b0; + icache_gva_q <= 1'b0; + end + // Map the only three exceptions which can occur in the frontend to a two bit enum - if (ariane_pkg::MMU_PRESENT && icache_dreq_i.ex.cause == riscv::INSTR_PAGE_FAULT) begin + if (ariane_pkg::MMU_PRESENT && icache_dreq_i.ex.cause == riscv::INSTR_GUEST_PAGE_FAULT) begin + icache_ex_valid_q <= ariane_pkg::FE_INSTR_GUEST_PAGE_FAULT; + end else if (ariane_pkg::MMU_PRESENT && icache_dreq_i.ex.cause == riscv::INSTR_PAGE_FAULT) begin icache_ex_valid_q <= ariane_pkg::FE_INSTR_PAGE_FAULT; end else if (icache_dreq_i.ex.cause == riscv::INSTR_ACCESS_FAULT) begin icache_ex_valid_q <= ariane_pkg::FE_INSTR_ACCESS_FAULT; @@ -538,6 +555,9 @@ module frontend .addr_i (addr), // from re-aligner .exception_i (icache_ex_valid_q), // from I$ .exception_addr_i (icache_vaddr_q), + .exception_gpaddr_i (icache_gpaddr_q), + .exception_tinst_i (icache_tinst_q), + .exception_gva_i (icache_gva_q), .predict_address_i (predict_address), .cf_type_i (cf_type), .valid_i (instruction_valid), // from re-aligner diff --git a/core/frontend/instr_queue.sv b/core/frontend/instr_queue.sv index 8df597aff5..13b22abb51 100644 --- a/core/frontend/instr_queue.sv +++ b/core/frontend/instr_queue.sv @@ -69,6 +69,9 @@ module instr_queue input ariane_pkg::frontend_exception_t exception_i, // Exception address - CACHE input logic [CVA6Cfg.VLEN-1:0] exception_addr_i, + input logic [CVA6Cfg.GPLEN-1:0] exception_gpaddr_i, + input logic [31:0] exception_tinst_i, + input logic exception_gva_i, // Branch predict - FRONTEND input logic [CVA6Cfg.VLEN-1:0] predict_address_i, // Instruction predict address - FRONTEND @@ -86,10 +89,13 @@ module instr_queue ); typedef struct packed { - logic [31:0] instr; // instruction word - ariane_pkg::cf_t cf; // branch was taken - ariane_pkg::frontend_exception_t ex; // exception happened - logic [CVA6Cfg.VLEN-1:0] ex_vaddr; // lower CVA6Cfg.VLEN bits of tval for exception + logic [31:0] instr; // instruction word + ariane_pkg::cf_t cf; // branch was taken + ariane_pkg::frontend_exception_t ex; // exception happened + logic [CVA6Cfg.VLEN-1:0] ex_vaddr; // lower VLEN bits of tval for exception + logic [CVA6Cfg.GPLEN-1:0] ex_gpaddr; // lower GPLEN bits of tval2 for exception + logic [31:0] ex_tinst; // tinst of exception + logic ex_gva; } instr_data_t; logic [CVA6Cfg.LOG2_INSTR_PER_FETCH-1:0] branch_index; @@ -207,6 +213,15 @@ ariane_pkg::FETCH_FIFO_DEPTH assign instr_data_in[i].cf = cf[i+idx_is_q]; assign instr_data_in[i].ex = exception_i; // exceptions hold for the whole fetch packet assign instr_data_in[i].ex_vaddr = exception_addr_i; + if (CVA6Cfg.RVH) begin : gen_hyp_ex_with_C + assign instr_data_in[i].ex_gpaddr = exception_gpaddr_i; + assign instr_data_in[i].ex_tinst = exception_tinst_i; + assign instr_data_in[i].ex_gva = exception_gva_i; + end else begin : gen_no_hyp_ex_with_C + assign instr_data_in[i].ex_gpaddr = '0; + assign instr_data_in[i].ex_tinst = '0; + assign instr_data_in[i].ex_gva = 1'b0; + end /* verilator lint_on WIDTH */ end end else begin : gen_multiple_instr_per_fetch_without_C @@ -236,6 +251,15 @@ ariane_pkg::FETCH_FIFO_DEPTH assign instr_data_in[0].cf = cf_type_i[0]; assign instr_data_in[0].ex = exception_i; // exceptions hold for the whole fetch packet assign instr_data_in[0].ex_vaddr = exception_addr_i; + if (CVA6Cfg.RVH) begin : gen_hyp_ex_without_C + assign instr_data_in[0].ex_gpaddr = exception_gpaddr_i; + assign instr_data_in[0].ex_tinst = exception_tinst_i; + assign instr_data_in[0].ex_gva = exception_gva_i; + end else begin : gen_no_hyp_ex_without_C + assign instr_data_in[0].ex_gpaddr = '0; + assign instr_data_in[0].ex_tinst = '0; + assign instr_data_in[0].ex_gva = 1'b0; + end /* verilator lint_on WIDTH */ end @@ -284,6 +308,9 @@ ariane_pkg::FETCH_FIFO_DEPTH fetch_entry_o.ex.cause = '0; fetch_entry_o.ex.tval = '0; + fetch_entry_o.ex.tval2 = '0; + fetch_entry_o.ex.gva = 1'b0; + fetch_entry_o.ex.tinst = '0; fetch_entry_o.branch_predict.predict_address = address_out; fetch_entry_o.branch_predict.cf = ariane_pkg::NoCF; // output mux select @@ -291,6 +318,8 @@ ariane_pkg::FETCH_FIFO_DEPTH if (idx_ds_q[i]) begin if (instr_data_out[i].ex == ariane_pkg::FE_INSTR_ACCESS_FAULT) begin fetch_entry_o.ex.cause = riscv::INSTR_ACCESS_FAULT; + end else if (CVA6Cfg.RVH && instr_data_out[i].ex == ariane_pkg::FE_INSTR_GUEST_PAGE_FAULT) begin + fetch_entry_o.ex.cause = riscv::INSTR_GUEST_PAGE_FAULT; end else begin fetch_entry_o.ex.cause = riscv::INSTR_PAGE_FAULT; end @@ -300,6 +329,11 @@ ariane_pkg::FETCH_FIFO_DEPTH fetch_entry_o.ex.tval = { {(CVA6Cfg.XLEN - CVA6Cfg.VLEN) {1'b0}}, instr_data_out[i].ex_vaddr }; + if (CVA6Cfg.RVH) begin + fetch_entry_o.ex.tval2 = instr_data_out[i].ex_gpaddr; + fetch_entry_o.ex.tinst = instr_data_out[i].ex_tinst; + fetch_entry_o.ex.gva = instr_data_out[i].ex_gva; + end fetch_entry_o.branch_predict.cf = instr_data_out[i].cf; pop_instr[i] = fetch_entry_valid_o & fetch_entry_ready_i; end @@ -325,6 +359,16 @@ ariane_pkg::FETCH_FIFO_DEPTH if (CVA6Cfg.TvalEn) fetch_entry_o.ex.tval = {{64 - CVA6Cfg.VLEN{1'b0}}, instr_data_out[0].ex_vaddr}; else fetch_entry_o.ex.tval = '0; + if (CVA6Cfg.RVH) begin + fetch_entry_o.ex.tval2 = instr_data_out[0].ex_gpaddr; + fetch_entry_o.ex.tinst = instr_data_out[0].ex_tinst; + fetch_entry_o.ex.gva = instr_data_out[0].ex_gva; + end else begin + fetch_entry_o.ex.tval2 = '0; + fetch_entry_o.ex.tinst = '0; + fetch_entry_o.ex.gva = 1'b0; + end + fetch_entry_o.branch_predict.predict_address = address_out; fetch_entry_o.branch_predict.cf = instr_data_out[0].cf; diff --git a/core/id_stage.sv b/core/id_stage.sv index 36c5a65dd5..5c4d51658b 100644 --- a/core/id_stage.sv +++ b/core/id_stage.sv @@ -51,8 +51,12 @@ module id_stage #( output logic rvfi_is_compressed_o, // Current privilege level - CSR_REGFILE input riscv::priv_lvl_t priv_lvl_i, + // Current virtualization mode - CSR_REGFILE + input logic v_i, // Floating point extension status - CSR_REGFILE input riscv::xs_t fs_i, + // Floating point extension virtual status - CSR_REGFILE + input riscv::xs_t vfs_i, // Floating point dynamic rounding mode - CSR_REGFILE input logic [2:0] frm_i, // Vector extension status - CSR_REGFILE @@ -67,8 +71,12 @@ module id_stage #( input logic tvm_i, // Timeout wait - CSR_REGFILE input logic tw_i, + // Virtual timeout wait - CSR_REGFILE + input logic vtw_i, // Trap sret - CSR_REGFILE - input logic tsr_i + input logic tsr_i, + // Hypervisor user mode - CSR_REGFILE + input logic hu_i ); // ID/ISSUE register stage typedef struct packed { @@ -169,13 +177,17 @@ module id_stage #( .branch_predict_i (fetch_entry_i.branch_predict), .ex_i (fetch_entry_i.ex), .priv_lvl_i (priv_lvl_i), + .v_i (v_i), .debug_mode_i (debug_mode_i), .fs_i, + .vfs_i, .frm_i, .vs_i, .tvm_i, .tw_i, + .vtw_i, .tsr_i, + .hu_i, .instruction_o (decoded_instruction), .orig_instr_o (orig_instr), .is_control_flow_instr_o (is_control_flow_instr) diff --git a/core/include/ariane_pkg.sv b/core/include/ariane_pkg.sv index 792c117cbd..8c666868bf 100644 --- a/core/include/ariane_pkg.sv +++ b/core/include/ariane_pkg.sv @@ -135,6 +135,26 @@ package ariane_pkg; | riscv::SSTATUS_FS | riscv::SSTATUS_SUM | riscv::SSTATUS_MXR; + + localparam logic [63:0] HSTATUS_WRITE_MASK = riscv::HSTATUS_VSBE + | riscv::HSTATUS_GVA + | riscv::HSTATUS_SPV + | riscv::HSTATUS_SPVP + | riscv::HSTATUS_HU + | riscv::HSTATUS_VTVM + | riscv::HSTATUS_VTW + | riscv::HSTATUS_VTSR; + + // hypervisor delegable interrupts + function automatic logic [31:0] hs_deleg_interrupts(config_pkg::cva6_cfg_t Cfg); + return riscv::MIP_VSSIP | riscv::MIP_VSTIP | riscv::MIP_VSEIP; + endfunction + + // virtual supervisor delegable interrupts + function automatic logic [31:0] vs_deleg_interrupts(config_pkg::cva6_cfg_t Cfg); + return riscv::MIP_VSSIP | riscv::MIP_VSTIP | riscv::MIP_VSEIP; + endfunction + // --------------- // AXI // --------------- @@ -286,6 +306,8 @@ package ariane_pkg; FENCE, FENCE_I, SFENCE_VMA, + HFENCE_VVMA, + HFENCE_GVMA, CSR_WRITE, CSR_READ, CSR_SET, @@ -302,6 +324,20 @@ package ariane_pkg; LB, SB, LBU, + // Hypervisor Virtual-Machine Load and Store Instructions + HLV_B, + HLV_BU, + HLV_H, + HLV_HU, + HLVX_HU, + HLV_W, + HLVX_WU, + HSV_B, + HSV_H, + HSV_W, + HLV_WU, + HLV_D, + HSV_D, // Atomic Memory Operations AMO_LRW, AMO_LRD, @@ -587,7 +623,8 @@ package ariane_pkg; typedef enum logic [1:0] { FE_NONE, FE_INSTR_ACCESS_FAULT, - FE_INSTR_PAGE_FAULT + FE_INSTR_PAGE_FAULT, + FE_INSTR_GUEST_PAGE_FAULT } frontend_exception_t; // AMO request going to cache. this request is unconditionally valid as soon @@ -695,7 +732,7 @@ package ariane_pkg; // ---------------------- function automatic logic [1:0] extract_transfer_size(fu_op op); case (op) - LD, SD, FLD, FSD, + LD, HLV_D, SD, HSV_D, FLD, FSD, AMO_LRD, AMO_SCD, AMO_SWAPD, AMO_ADDD, AMO_ANDD, AMO_ORD, @@ -704,7 +741,8 @@ package ariane_pkg; AMO_MINDU: begin return 2'b11; end - LW, LWU, SW, FLW, FSW, + LW, LWU, HLV_W, HLV_WU, HLVX_WU, + SW, HSV_W, FLW, FSW, AMO_LRW, AMO_SCW, AMO_SWAPW, AMO_ADDW, AMO_ANDW, AMO_ORW, @@ -713,9 +751,60 @@ package ariane_pkg; AMO_MINWU: begin return 2'b10; end - LH, LHU, SH, FLH, FSH: return 2'b01; - LB, LBU, SB, FLB, FSB: return 2'b00; - default: return 2'b11; + LH, LHU, HLV_H, HLV_HU, HLVX_HU, SH, HSV_H, FLH, FSH: return 2'b01; + LB, LBU, HLV_B, HLV_BU, SB, HSV_B, FLB, FSB: return 2'b00; + default: return 2'b11; endcase endfunction + // ---------------------- + // MMU Functions + // ---------------------- + + // checks if final translation page size is 1G when H-extension is enabled + function automatic logic is_trans_1G(input logic s_st_enbl, input logic g_st_enbl, + input logic is_s_1G, input logic is_g_1G); + return (((is_s_1G && s_st_enbl) || !s_st_enbl) && ((is_g_1G && g_st_enbl) || !g_st_enbl)); + endfunction : is_trans_1G + + // checks if final translation page size is 2M when H-extension is enabled + function automatic logic is_trans_2M(input logic s_st_enbl, input logic g_st_enbl, + input logic is_s_1G, input logic is_s_2M, + input logic is_g_1G, input logic is_g_2M); + return (s_st_enbl && g_st_enbl) ? + ((is_s_2M && (is_g_1G || is_g_2M)) || (is_g_2M && (is_s_1G || is_s_2M))) : + ((is_s_2M && s_st_enbl) || (is_g_2M && g_st_enbl)); + endfunction : is_trans_2M + + // computes the paddr based on the page size, ppn and offset + function automatic logic [40:0] make_gpaddr(input logic s_st_enbl, input logic is_1G, + input logic is_2M, input logic [63:0] vaddr, + input riscv::pte_t pte); + logic [40:0] gpaddr; + if (s_st_enbl) begin + gpaddr = {pte.ppn[28:0], vaddr[11:0]}; + // Giga page + if (is_1G) gpaddr[29:12] = vaddr[29:12]; + // Mega page + if (is_2M) gpaddr[20:12] = vaddr[20:12]; + end else begin + gpaddr = vaddr[40:0]; + end + return gpaddr; + endfunction : make_gpaddr + + // computes the final gppn based on the guest physical address + function automatic logic [28:0] make_gppn(input logic s_st_enbl, input logic is_1G, + input logic is_2M, input logic [28:0] vpn, + input riscv::pte_t pte); + logic [28:0] gppn; + if (s_st_enbl) begin + gppn = pte.ppn[28:0]; + if (is_2M) gppn[8:0] = vpn[8:0]; + if (is_1G) gppn[17:0] = vpn[17:0]; + end else begin + gppn = vpn; + end + return gppn; + endfunction : make_gppn + endpackage diff --git a/core/include/build_config_pkg.sv b/core/include/build_config_pkg.sv index 5328debf39..dbf51c85b8 100644 --- a/core/include/build_config_pkg.sv +++ b/core/include/build_config_pkg.sv @@ -32,10 +32,12 @@ package build_config_pkg; cfg.XLEN = CVA6Cfg.XLEN; cfg.VLEN = (CVA6Cfg.XLEN == 32) ? 32 : 64; cfg.PLEN = (CVA6Cfg.XLEN == 32) ? 34 : 56; + cfg.GPLEN = (CVA6Cfg.XLEN == 32) ? 34 : 41; cfg.IS_XLEN32 = IS_XLEN32; cfg.IS_XLEN64 = IS_XLEN64; cfg.XLEN_ALIGN_BYTES = $clog2(CVA6Cfg.XLEN / 8); cfg.ASID_WIDTH = (CVA6Cfg.XLEN == 64) ? 16 : 1; + cfg.VMID_WIDTH = (CVA6Cfg.XLEN == 64) ? 14 : 1; cfg.FPGA_EN = CVA6Cfg.FPGA_EN; cfg.NrCommitPorts = CVA6Cfg.NrCommitPorts; @@ -53,6 +55,7 @@ package build_config_pkg; cfg.RVB = CVA6Cfg.RVB; cfg.RVV = CVA6Cfg.RVV; cfg.RVC = CVA6Cfg.RVC; + cfg.RVH = CVA6Cfg.RVH; cfg.RVZCB = CVA6Cfg.RVZCB; cfg.RVZCMP = CVA6Cfg.RVZCMP; cfg.XFVec = CVA6Cfg.XFVec; @@ -131,9 +134,12 @@ package build_config_pkg; cfg.ModeW = (CVA6Cfg.XLEN == 32) ? 1 : 4; cfg.ASIDW = (CVA6Cfg.XLEN == 32) ? 9 : 16; + cfg.VMIDW = (CVA6Cfg.XLEN == 32) ? 7 : 14; cfg.PPNW = (CVA6Cfg.XLEN == 32) ? 22 : 44; + cfg.GPPNW = (CVA6Cfg.XLEN == 32) ? 22 : 29; cfg.MODE_SV = (CVA6Cfg.XLEN == 32) ? config_pkg::ModeSv32 : config_pkg::ModeSv39; cfg.SV = (cfg.MODE_SV == config_pkg::ModeSv32) ? 32 : 39; + cfg.SVX = (cfg.MODE_SV == config_pkg::ModeSv32) ? 34 : 41; return cfg; endfunction diff --git a/core/include/config_pkg.sv b/core/include/config_pkg.sv index 9ed4dc809f..27cd6b21d6 100644 --- a/core/include/config_pkg.sv +++ b/core/include/config_pkg.sv @@ -80,6 +80,8 @@ package config_pkg; bit RVV; // Compress RISC-V extension bit RVC; + // Hypervisor RISC-V extension + bit RVH; // Zcb RISC-V extension bit RVZCB; // Zcmp RISC-V extension @@ -168,10 +170,12 @@ package config_pkg; int unsigned XLEN; int unsigned VLEN; int unsigned PLEN; + int unsigned GPLEN; bit IS_XLEN32; bit IS_XLEN64; int unsigned XLEN_ALIGN_BYTES; int unsigned ASID_WIDTH; + int unsigned VMID_WIDTH; bit FPGA_EN; /// Number of commit ports, i.e., maximum number of instructions that the @@ -194,6 +198,7 @@ package config_pkg; bit RVB; bit RVV; bit RVC; + bit RVH; bit RVZCB; bit RVZCMP; bit XFVec; @@ -273,9 +278,12 @@ package config_pkg; int unsigned ModeW; int unsigned ASIDW; + int unsigned VMIDW; int unsigned PPNW; + int unsigned GPPNW; vm_mode_t MODE_SV; int unsigned SV; + int unsigned SVX; } cva6_cfg_t; /// Empty configuration to sanity check proper parameter passing. Whenever diff --git a/core/include/cv32a60x_config_pkg.sv b/core/include/cv32a60x_config_pkg.sv index edad870779..469f4f37a2 100644 --- a/core/include/cv32a60x_config_pkg.sv +++ b/core/include/cv32a60x_config_pkg.sv @@ -23,6 +23,7 @@ package cva6_config_pkg; localparam CVA6ConfigZcbExtEn = 1; localparam CVA6ConfigZcmpExtEn = 1; localparam CVA6ConfigAExtEn = 1; + localparam CVA6ConfigHExtEn = 0; // always disabled localparam CVA6ConfigBExtEn = 1; localparam CVA6ConfigVExtEn = 0; localparam CVA6ConfigZiCondExtEn = 1; @@ -93,6 +94,7 @@ package cva6_config_pkg; RVB: bit'(CVA6ConfigBExtEn), RVV: bit'(CVA6ConfigVExtEn), RVC: bit'(CVA6ConfigCExtEn), + RVH: bit'(CVA6ConfigHExtEn), RVZCB: bit'(CVA6ConfigZcbExtEn), RVZCMP: bit'(CVA6ConfigZcmpExtEn), XFVec: bit'(CVA6ConfigFVecEn), diff --git a/core/include/cv32a65x_config_pkg.sv b/core/include/cv32a65x_config_pkg.sv index c419cadc08..6631764fc7 100644 --- a/core/include/cv32a65x_config_pkg.sv +++ b/core/include/cv32a65x_config_pkg.sv @@ -22,6 +22,7 @@ package cva6_config_pkg; localparam CVA6ConfigZcbExtEn = 1; localparam CVA6ConfigZcmpExtEn = 0; localparam CVA6ConfigAExtEn = 0; + localparam CVA6ConfigHExtEn = 0; // always disabled localparam CVA6ConfigBExtEn = 1; localparam CVA6ConfigVExtEn = 0; localparam CVA6ConfigZiCondExtEn = 0; @@ -92,6 +93,7 @@ package cva6_config_pkg; RVB: bit'(CVA6ConfigBExtEn), RVV: bit'(CVA6ConfigVExtEn), RVC: bit'(CVA6ConfigCExtEn), + RVH: bit'(CVA6ConfigHExtEn), RVZCB: bit'(CVA6ConfigZcbExtEn), RVZCMP: bit'(CVA6ConfigZcmpExtEn), XFVec: bit'(CVA6ConfigFVecEn), diff --git a/core/include/cv32a6_embedded_config_pkg.sv b/core/include/cv32a6_embedded_config_pkg.sv index 7f4a6e932d..10a3d4e85a 100644 --- a/core/include/cv32a6_embedded_config_pkg.sv +++ b/core/include/cv32a6_embedded_config_pkg.sv @@ -22,6 +22,7 @@ package cva6_config_pkg; localparam CVA6ConfigZcbExtEn = 1; localparam CVA6ConfigZcmpExtEn = 0; localparam CVA6ConfigAExtEn = 0; + localparam CVA6ConfigHExtEn = 0; // always disabled localparam CVA6ConfigBExtEn = 1; localparam CVA6ConfigVExtEn = 0; localparam CVA6ConfigZiCondExtEn = 0; @@ -92,6 +93,7 @@ package cva6_config_pkg; RVB: bit'(CVA6ConfigBExtEn), RVV: bit'(CVA6ConfigVExtEn), RVC: bit'(CVA6ConfigCExtEn), + RVH: bit'(CVA6ConfigHExtEn), RVZCB: bit'(CVA6ConfigZcbExtEn), RVZCMP: bit'(CVA6ConfigZcmpExtEn), XFVec: bit'(CVA6ConfigFVecEn), diff --git a/core/include/cv32a6_ima_sv32_fpga_config_pkg.sv b/core/include/cv32a6_ima_sv32_fpga_config_pkg.sv index c02b3504b4..8b13bf00b8 100644 --- a/core/include/cv32a6_ima_sv32_fpga_config_pkg.sv +++ b/core/include/cv32a6_ima_sv32_fpga_config_pkg.sv @@ -23,6 +23,7 @@ package cva6_config_pkg; localparam CVA6ConfigZcbExtEn = 0; localparam CVA6ConfigZcmpExtEn = 0; localparam CVA6ConfigAExtEn = 1; + localparam CVA6ConfigHExtEn = 0; // always disabled localparam CVA6ConfigBExtEn = 0; localparam CVA6ConfigVExtEn = 0; localparam CVA6ConfigZiCondExtEn = 0; @@ -93,6 +94,7 @@ package cva6_config_pkg; RVB: bit'(CVA6ConfigBExtEn), RVV: bit'(CVA6ConfigVExtEn), RVC: bit'(CVA6ConfigCExtEn), + RVH: bit'(CVA6ConfigHExtEn), RVZCB: bit'(CVA6ConfigZcbExtEn), RVZCMP: bit'(CVA6ConfigZcmpExtEn), XFVec: bit'(CVA6ConfigFVecEn), diff --git a/core/include/cv32a6_imac_sv0_config_pkg.sv b/core/include/cv32a6_imac_sv0_config_pkg.sv index 6b8b3b50b2..7cbe984049 100644 --- a/core/include/cv32a6_imac_sv0_config_pkg.sv +++ b/core/include/cv32a6_imac_sv0_config_pkg.sv @@ -23,6 +23,7 @@ package cva6_config_pkg; localparam CVA6ConfigZcbExtEn = 0; localparam CVA6ConfigZcmpExtEn = 0; localparam CVA6ConfigAExtEn = 1; + localparam CVA6ConfigHExtEn = 0; // always disabled localparam CVA6ConfigBExtEn = 0; localparam CVA6ConfigVExtEn = 0; localparam CVA6ConfigZiCondExtEn = 0; @@ -93,6 +94,7 @@ package cva6_config_pkg; RVB: bit'(CVA6ConfigBExtEn), RVV: bit'(CVA6ConfigVExtEn), RVC: bit'(CVA6ConfigCExtEn), + RVH: bit'(CVA6ConfigHExtEn), RVZCB: bit'(CVA6ConfigZcbExtEn), RVZCMP: bit'(CVA6ConfigZcmpExtEn), XFVec: bit'(CVA6ConfigFVecEn), diff --git a/core/include/cv32a6_imac_sv32_config_pkg.sv b/core/include/cv32a6_imac_sv32_config_pkg.sv index daf87fd16b..d9b3f69bc8 100644 --- a/core/include/cv32a6_imac_sv32_config_pkg.sv +++ b/core/include/cv32a6_imac_sv32_config_pkg.sv @@ -23,6 +23,7 @@ package cva6_config_pkg; localparam CVA6ConfigZcbExtEn = 0; localparam CVA6ConfigZcmpExtEn = 0; localparam CVA6ConfigAExtEn = 1; + localparam CVA6ConfigHExtEn = 0; // always disabled localparam CVA6ConfigBExtEn = 0; localparam CVA6ConfigVExtEn = 0; localparam CVA6ConfigZiCondExtEn = 0; @@ -93,6 +94,7 @@ package cva6_config_pkg; RVB: bit'(CVA6ConfigBExtEn), RVV: bit'(CVA6ConfigVExtEn), RVC: bit'(CVA6ConfigCExtEn), + RVH: bit'(CVA6ConfigHExtEn), RVZCB: bit'(CVA6ConfigZcbExtEn), RVZCMP: bit'(CVA6ConfigZcmpExtEn), XFVec: bit'(CVA6ConfigFVecEn), diff --git a/core/include/cv32a6_imafc_sv32_config_pkg.sv b/core/include/cv32a6_imafc_sv32_config_pkg.sv index 30c2e1b361..e0492b1327 100644 --- a/core/include/cv32a6_imafc_sv32_config_pkg.sv +++ b/core/include/cv32a6_imafc_sv32_config_pkg.sv @@ -23,6 +23,7 @@ package cva6_config_pkg; localparam CVA6ConfigZcbExtEn = 0; localparam CVA6ConfigZcmpExtEn = 0; localparam CVA6ConfigAExtEn = 1; + localparam CVA6ConfigHExtEn = 0; // always disabled localparam CVA6ConfigBExtEn = 0; localparam CVA6ConfigVExtEn = 0; localparam CVA6ConfigZiCondExtEn = 0; @@ -93,6 +94,7 @@ package cva6_config_pkg; RVB: bit'(CVA6ConfigBExtEn), RVV: bit'(CVA6ConfigVExtEn), RVC: bit'(CVA6ConfigCExtEn), + RVH: bit'(CVA6ConfigHExtEn), RVZCB: bit'(CVA6ConfigZcbExtEn), RVZCMP: bit'(CVA6ConfigZcmpExtEn), XFVec: bit'(CVA6ConfigFVecEn), diff --git a/core/include/cv64a6_imadfcv_sv39_polara_config_pkg.sv b/core/include/cv64a6_imadfcv_sv39_polara_config_pkg.sv index 7c878e5cc5..5be047ae57 100644 --- a/core/include/cv64a6_imadfcv_sv39_polara_config_pkg.sv +++ b/core/include/cv64a6_imadfcv_sv39_polara_config_pkg.sv @@ -24,6 +24,7 @@ package cva6_config_pkg; localparam CVA6ConfigZcmpExtEn = 0; localparam CVA6ConfigAExtEn = 1; localparam CVA6ConfigBExtEn = 0; + localparam CVA6ConfigHExtEn = 0; localparam CVA6ConfigVExtEn = 1; localparam CVA6ConfigZiCondExtEn = 0; @@ -93,6 +94,7 @@ package cva6_config_pkg; RVB: bit'(CVA6ConfigBExtEn), RVV: bit'(CVA6ConfigVExtEn), RVC: bit'(CVA6ConfigCExtEn), + RVH: bit'(CVA6ConfigHExtEn), RVZCB: bit'(CVA6ConfigZcbExtEn), RVZCMP: bit'(CVA6ConfigZcmpExtEn), XFVec: bit'(CVA6ConfigFVecEn), diff --git a/core/include/cv64a6_imafdc_sv39_config_pkg.sv b/core/include/cv64a6_imafdc_sv39_config_pkg.sv index 4ca6d622f5..7b1f53d1ce 100644 --- a/core/include/cv64a6_imafdc_sv39_config_pkg.sv +++ b/core/include/cv64a6_imafdc_sv39_config_pkg.sv @@ -23,6 +23,7 @@ package cva6_config_pkg; localparam CVA6ConfigZcbExtEn = 1; localparam CVA6ConfigZcmpExtEn = 0; localparam CVA6ConfigAExtEn = 1; + localparam CVA6ConfigHExtEn = 0; // always disabled localparam CVA6ConfigBExtEn = 1; localparam CVA6ConfigVExtEn = 0; localparam CVA6ConfigZiCondExtEn = 1; @@ -93,6 +94,7 @@ package cva6_config_pkg; RVB: bit'(CVA6ConfigBExtEn), RVV: bit'(CVA6ConfigVExtEn), RVC: bit'(CVA6ConfigCExtEn), + RVH: bit'(CVA6ConfigHExtEn), RVZCB: bit'(CVA6ConfigZcbExtEn), RVZCMP: bit'(CVA6ConfigZcmpExtEn), XFVec: bit'(CVA6ConfigFVecEn), diff --git a/core/include/cv64a6_imafdc_sv39_hpdcache_config_pkg.sv b/core/include/cv64a6_imafdc_sv39_hpdcache_config_pkg.sv index 072c213db7..143766cde0 100644 --- a/core/include/cv64a6_imafdc_sv39_hpdcache_config_pkg.sv +++ b/core/include/cv64a6_imafdc_sv39_hpdcache_config_pkg.sv @@ -32,6 +32,7 @@ package cva6_config_pkg; localparam CVA6ConfigAExtEn = 1; localparam CVA6ConfigBExtEn = 1; localparam CVA6ConfigVExtEn = 0; + localparam CVA6ConfigHExtEn = 0; localparam CVA6ConfigZiCondExtEn = 1; localparam CVA6ConfigAxiIdWidth = 4; @@ -100,6 +101,7 @@ package cva6_config_pkg; RVB: bit'(CVA6ConfigBExtEn), RVV: bit'(CVA6ConfigVExtEn), RVC: bit'(CVA6ConfigCExtEn), + RVH: bit'(CVA6ConfigHExtEn), RVZCB: bit'(CVA6ConfigZcbExtEn), RVZCMP: bit'(CVA6ConfigZcmpExtEn), XFVec: bit'(CVA6ConfigFVecEn), diff --git a/core/include/cv64a6_imafdc_sv39_openpiton_config_pkg.sv b/core/include/cv64a6_imafdc_sv39_openpiton_config_pkg.sv index c5f695985d..e59e717550 100644 --- a/core/include/cv64a6_imafdc_sv39_openpiton_config_pkg.sv +++ b/core/include/cv64a6_imafdc_sv39_openpiton_config_pkg.sv @@ -23,6 +23,7 @@ package cva6_config_pkg; localparam CVA6ConfigZcbExtEn = 0; localparam CVA6ConfigZcmpExtEn = 0; localparam CVA6ConfigAExtEn = 1; + localparam CVA6ConfigHExtEn = 0; // always disabled localparam CVA6ConfigBExtEn = 0; localparam CVA6ConfigVExtEn = 0; localparam CVA6ConfigZiCondExtEn = 0; @@ -93,6 +94,7 @@ package cva6_config_pkg; RVB: bit'(CVA6ConfigBExtEn), RVV: bit'(CVA6ConfigVExtEn), RVC: bit'(CVA6ConfigCExtEn), + RVH: bit'(CVA6ConfigHExtEn), RVZCB: bit'(CVA6ConfigZcbExtEn), RVZCMP: bit'(CVA6ConfigZcmpExtEn), XFVec: bit'(CVA6ConfigFVecEn), diff --git a/core/include/cv64a6_imafdc_sv39_wb_config_pkg.sv b/core/include/cv64a6_imafdc_sv39_wb_config_pkg.sv index 2dbfd59e5a..1e9012eca5 100644 --- a/core/include/cv64a6_imafdc_sv39_wb_config_pkg.sv +++ b/core/include/cv64a6_imafdc_sv39_wb_config_pkg.sv @@ -25,6 +25,7 @@ package cva6_config_pkg; localparam CVA6ConfigAExtEn = 1; localparam CVA6ConfigBExtEn = 1; localparam CVA6ConfigVExtEn = 0; + localparam CVA6ConfigHExtEn = 0; localparam CVA6ConfigZiCondExtEn = 1; localparam CVA6ConfigAxiIdWidth = 4; @@ -93,6 +94,7 @@ package cva6_config_pkg; RVB: bit'(CVA6ConfigBExtEn), RVV: bit'(CVA6ConfigVExtEn), RVC: bit'(CVA6ConfigCExtEn), + RVH: bit'(CVA6ConfigHExtEn), RVZCB: bit'(CVA6ConfigZcbExtEn), RVZCMP: bit'(CVA6ConfigZcmpExtEn), XFVec: bit'(CVA6ConfigFVecEn), diff --git a/core/include/cv64a6_imafdch_sv39_config_pkg.sv b/core/include/cv64a6_imafdch_sv39_config_pkg.sv new file mode 100644 index 0000000000..db6044fbfc --- /dev/null +++ b/core/include/cv64a6_imafdch_sv39_config_pkg.sv @@ -0,0 +1,141 @@ +// Copyright 2021 Thales DIS design services SAS +// +// Licensed under the Solderpad Hardware Licence, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// SPDX-License-Identifier: Apache-2.0 WITH SHL-2.0 +// You may obtain a copy of the License at https://solderpad.org/licenses/ +// +// Original Author: Jean-Roch COULON - Thales + + +package cva6_config_pkg; + + localparam CVA6ConfigXlen = 64; + + localparam CVA6ConfigFpuEn = 1; + localparam CVA6ConfigF16En = 0; + localparam CVA6ConfigF16AltEn = 0; + localparam CVA6ConfigF8En = 0; + localparam CVA6ConfigFVecEn = 0; + + localparam CVA6ConfigCvxifEn = 1; + localparam CVA6ConfigCExtEn = 1; + localparam CVA6ConfigZcbExtEn = 1; + localparam CVA6ConfigZcmpExtEn = 0; + localparam CVA6ConfigAExtEn = 1; + localparam CVA6ConfigHExtEn = 1; + localparam CVA6ConfigBExtEn = 1; + localparam CVA6ConfigVExtEn = 0; + localparam CVA6ConfigZiCondExtEn = 1; + + localparam CVA6ConfigAxiIdWidth = 4; + localparam CVA6ConfigAxiAddrWidth = 64; + localparam CVA6ConfigAxiDataWidth = 64; + localparam CVA6ConfigFetchUserEn = 0; + localparam CVA6ConfigFetchUserWidth = CVA6ConfigXlen; + localparam CVA6ConfigDataUserEn = 0; + localparam CVA6ConfigDataUserWidth = CVA6ConfigXlen; + + localparam CVA6ConfigIcacheByteSize = 16384; + localparam CVA6ConfigIcacheSetAssoc = 4; + localparam CVA6ConfigIcacheLineWidth = 128; + localparam CVA6ConfigDcacheByteSize = 32768; + localparam CVA6ConfigDcacheSetAssoc = 8; + localparam CVA6ConfigDcacheLineWidth = 128; + + localparam CVA6ConfigDcacheIdWidth = 1; + localparam CVA6ConfigMemTidWidth = 2; + + localparam CVA6ConfigWtDcacheWbufDepth = 8; + + localparam CVA6ConfigNrCommitPorts = 2; + localparam CVA6ConfigNrScoreboardEntries = 8; + + localparam CVA6ConfigFPGAEn = 0; + + localparam CVA6ConfigNrLoadPipeRegs = 1; + localparam CVA6ConfigNrStorePipeRegs = 0; + localparam CVA6ConfigNrLoadBufEntries = 2; + + localparam CVA6ConfigInstrTlbEntries = 16; + localparam CVA6ConfigDataTlbEntries = 16; + + localparam CVA6ConfigRASDepth = 2; + localparam CVA6ConfigBTBEntries = 32; + localparam CVA6ConfigBHTEntries = 128; + + localparam CVA6ConfigTvalEn = 1; + + localparam CVA6ConfigNrPMPEntries = 8; + + localparam CVA6ConfigPerfCounterEn = 1; + + localparam config_pkg::cache_type_t CVA6ConfigDcacheType = config_pkg::WT; + + localparam CVA6ConfigMmuPresent = 1; + + localparam CVA6ConfigRvfiTrace = 1; + + localparam config_pkg::cva6_user_cfg_t cva6_cfg = '{ + NrCommitPorts: unsigned'(CVA6ConfigNrCommitPorts), + AxiAddrWidth: unsigned'(CVA6ConfigAxiAddrWidth), + AxiDataWidth: unsigned'(CVA6ConfigAxiDataWidth), + AxiIdWidth: unsigned'(CVA6ConfigAxiIdWidth), + AxiUserWidth: unsigned'(CVA6ConfigDataUserWidth), + MemTidWidth: unsigned'(CVA6ConfigMemTidWidth), + NrLoadBufEntries: unsigned'(CVA6ConfigNrLoadBufEntries), + FpuEn: bit'(CVA6ConfigFpuEn), + XF16: bit'(CVA6ConfigF16En), + XF16ALT: bit'(CVA6ConfigF16AltEn), + XF8: bit'(CVA6ConfigF8En), + RVA: bit'(CVA6ConfigAExtEn), + RVB: bit'(CVA6ConfigBExtEn), + RVV: bit'(CVA6ConfigVExtEn), + RVC: bit'(CVA6ConfigCExtEn), + RVH: bit'(CVA6ConfigHExtEn), + RVZCB: bit'(CVA6ConfigZcbExtEn), + RVZCMP: bit'(CVA6ConfigZcmpExtEn), + XFVec: bit'(CVA6ConfigFVecEn), + CvxifEn: bit'(CVA6ConfigCvxifEn), + ZiCondExtEn: bit'(CVA6ConfigZiCondExtEn), + RVS: bit'(1), + RVU: bit'(1), + HaltAddress: 64'h800, + ExceptionAddress: 64'h808, + RASDepth: unsigned'(CVA6ConfigRASDepth), + BTBEntries: unsigned'(CVA6ConfigBTBEntries), + BHTEntries: unsigned'(CVA6ConfigBHTEntries), + DmBaseAddress: 64'h0, + TvalEn: bit'(CVA6ConfigTvalEn), + NrPMPEntries: unsigned'(CVA6ConfigNrPMPEntries), + PMPCfgRstVal: {16{64'h0}}, + PMPAddrRstVal: {16{64'h0}}, + PMPEntryReadOnly: 16'd0, + NOCType: config_pkg::NOC_TYPE_AXI4_ATOP, + // idempotent region + NrNonIdempotentRules: + unsigned'( + 2 + ), + NonIdempotentAddrBase: 1024'({64'b0, 64'b0}), + NonIdempotentLength: 1024'({64'b0, 64'b0}), + NrExecuteRegionRules: unsigned'(3), + // DRAM, Boot ROM, Debug Module + ExecuteRegionAddrBase: + 1024'( + {64'h8000_0000, 64'h1_0000, 64'h0} + ), + ExecuteRegionLength: 1024'({64'h40000000, 64'h10000, 64'h1000}), + // cached region + NrCachedRegionRules: + unsigned'( + 1 + ), + CachedRegionAddrBase: 1024'({64'h8000_0000}), + CachedRegionLength: 1024'({64'h40000000}), + MaxOutstandingStores: unsigned'(7), + DebugEn: bit'(1), + AxiBurstWriteEn: bit'(0) + }; + +endpackage diff --git a/core/include/cv64a6_imafdch_sv39_wb_config_pkg.sv b/core/include/cv64a6_imafdch_sv39_wb_config_pkg.sv new file mode 100644 index 0000000000..0b3d48baac --- /dev/null +++ b/core/include/cv64a6_imafdch_sv39_wb_config_pkg.sv @@ -0,0 +1,141 @@ +// Copyright 2021 Thales DIS design services SAS +// +// Licensed under the Solderpad Hardware Licence, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// SPDX-License-Identifier: Apache-2.0 WITH SHL-2.0 +// You may obtain a copy of the License at https://solderpad.org/licenses/ +// +// Original Author: Jean-Roch COULON - Thales + + +package cva6_config_pkg; + + localparam CVA6ConfigXlen = 64; + + localparam CVA6ConfigFpuEn = 1; + localparam CVA6ConfigF16En = 0; + localparam CVA6ConfigF16AltEn = 0; + localparam CVA6ConfigF8En = 0; + localparam CVA6ConfigFVecEn = 0; + + localparam CVA6ConfigCvxifEn = 1; + localparam CVA6ConfigCExtEn = 1; + localparam CVA6ConfigZcbExtEn = 1; + localparam CVA6ConfigZcmpExtEn = 0; + localparam CVA6ConfigAExtEn = 1; + localparam CVA6ConfigBExtEn = 1; + localparam CVA6ConfigVExtEn = 0; + localparam CVA6ConfigHExtEn = 1; + localparam CVA6ConfigZiCondExtEn = 1; + + localparam CVA6ConfigAxiIdWidth = 4; + localparam CVA6ConfigAxiAddrWidth = 64; + localparam CVA6ConfigAxiDataWidth = 64; + localparam CVA6ConfigFetchUserEn = 0; + localparam CVA6ConfigFetchUserWidth = CVA6ConfigXlen; + localparam CVA6ConfigDataUserEn = 0; + localparam CVA6ConfigDataUserWidth = CVA6ConfigXlen; + + localparam CVA6ConfigIcacheByteSize = 16384; + localparam CVA6ConfigIcacheSetAssoc = 4; + localparam CVA6ConfigIcacheLineWidth = 128; + localparam CVA6ConfigDcacheByteSize = 32768; + localparam CVA6ConfigDcacheSetAssoc = 8; + localparam CVA6ConfigDcacheLineWidth = 128; + + localparam CVA6ConfigDcacheIdWidth = 1; + localparam CVA6ConfigMemTidWidth = 2; + + localparam CVA6ConfigWtDcacheWbufDepth = 8; + + localparam CVA6ConfigNrCommitPorts = 2; + localparam CVA6ConfigNrScoreboardEntries = 8; + + localparam CVA6ConfigFPGAEn = 0; + + localparam CVA6ConfigNrLoadPipeRegs = 1; + localparam CVA6ConfigNrStorePipeRegs = 0; + localparam CVA6ConfigNrLoadBufEntries = 2; + + localparam CVA6ConfigInstrTlbEntries = 16; + localparam CVA6ConfigDataTlbEntries = 16; + + localparam CVA6ConfigRASDepth = 2; + localparam CVA6ConfigBTBEntries = 32; + localparam CVA6ConfigBHTEntries = 128; + + localparam CVA6ConfigTvalEn = 1; + + localparam CVA6ConfigNrPMPEntries = 8; + + localparam CVA6ConfigPerfCounterEn = 1; + + localparam config_pkg::cache_type_t CVA6ConfigDcacheType = config_pkg::WB; + + localparam CVA6ConfigMmuPresent = 1; + + localparam CVA6ConfigRvfiTrace = 1; + + localparam config_pkg::cva6_user_cfg_t cva6_cfg = '{ + NrCommitPorts: unsigned'(CVA6ConfigNrCommitPorts), + AxiAddrWidth: unsigned'(CVA6ConfigAxiAddrWidth), + AxiDataWidth: unsigned'(CVA6ConfigAxiDataWidth), + AxiIdWidth: unsigned'(CVA6ConfigAxiIdWidth), + AxiUserWidth: unsigned'(CVA6ConfigDataUserWidth), + MemTidWidth: unsigned'(CVA6ConfigMemTidWidth), + NrLoadBufEntries: unsigned'(CVA6ConfigNrLoadBufEntries), + FpuEn: bit'(CVA6ConfigFpuEn), + XF16: bit'(CVA6ConfigF16En), + XF16ALT: bit'(CVA6ConfigF16AltEn), + XF8: bit'(CVA6ConfigF8En), + RVA: bit'(CVA6ConfigAExtEn), + RVB: bit'(CVA6ConfigBExtEn), + RVV: bit'(CVA6ConfigVExtEn), + RVC: bit'(CVA6ConfigCExtEn), + RVH: bit'(CVA6ConfigHExtEn), + RVZCB: bit'(CVA6ConfigZcbExtEn), + RVZCMP: bit'(CVA6ConfigZcmpExtEn), + XFVec: bit'(CVA6ConfigFVecEn), + CvxifEn: bit'(CVA6ConfigCvxifEn), + ZiCondExtEn: bit'(CVA6ConfigZiCondExtEn), + RVS: bit'(1), + RVU: bit'(1), + HaltAddress: 64'h800, + ExceptionAddress: 64'h808, + RASDepth: unsigned'(CVA6ConfigRASDepth), + BTBEntries: unsigned'(CVA6ConfigBTBEntries), + BHTEntries: unsigned'(CVA6ConfigBHTEntries), + DmBaseAddress: 64'h0, + TvalEn: bit'(CVA6ConfigTvalEn), + NrPMPEntries: unsigned'(CVA6ConfigNrPMPEntries), + PMPCfgRstVal: {16{64'h0}}, + PMPAddrRstVal: {16{64'h0}}, + PMPEntryReadOnly: 16'd0, + NOCType: config_pkg::NOC_TYPE_AXI4_ATOP, + // idempotent region + NrNonIdempotentRules: + unsigned'( + 2 + ), + NonIdempotentAddrBase: 1024'({64'b0, 64'b0}), + NonIdempotentLength: 1024'({64'b0, 64'b0}), + NrExecuteRegionRules: unsigned'(3), + // DRAM, Boot ROM, Debug Module + ExecuteRegionAddrBase: + 1024'( + {64'h8000_0000, 64'h1_0000, 64'h0} + ), + ExecuteRegionLength: 1024'({64'h40000000, 64'h10000, 64'h1000}), + // cached region + NrCachedRegionRules: + unsigned'( + 1 + ), + CachedRegionAddrBase: 1024'({64'h8000_0000}), + CachedRegionLength: 1024'({64'h40000000}), + MaxOutstandingStores: unsigned'(7), + DebugEn: bit'(1), + AxiBurstWriteEn: bit'(0) + }; + +endpackage diff --git a/core/include/cv64a6_imafdcv_sv39_config_pkg.sv b/core/include/cv64a6_imafdcv_sv39_config_pkg.sv index 808e046cbb..92ce5da279 100644 --- a/core/include/cv64a6_imafdcv_sv39_config_pkg.sv +++ b/core/include/cv64a6_imafdcv_sv39_config_pkg.sv @@ -24,6 +24,7 @@ package cva6_config_pkg; localparam CVA6ConfigZcmpExtEn = 0; localparam CVA6ConfigAExtEn = 1; localparam CVA6ConfigBExtEn = 0; + localparam CVA6ConfigHExtEn = 0; localparam CVA6ConfigVExtEn = 1; localparam CVA6ConfigZiCondExtEn = 0; @@ -93,6 +94,7 @@ package cva6_config_pkg; RVB: bit'(CVA6ConfigBExtEn), RVV: bit'(CVA6ConfigVExtEn), RVC: bit'(CVA6ConfigCExtEn), + RVH: bit'(CVA6ConfigHExtEn), RVZCB: bit'(CVA6ConfigZcbExtEn), RVZCMP: bit'(CVA6ConfigZcmpExtEn), XFVec: bit'(CVA6ConfigFVecEn), diff --git a/core/include/riscv_pkg.sv b/core/include/riscv_pkg.sv index 969248286b..83c2998118 100644 --- a/core/include/riscv_pkg.sv +++ b/core/include/riscv_pkg.sv @@ -30,9 +30,10 @@ package riscv; // Privilege Spec // -------------------- typedef enum logic [1:0] { - PRIV_LVL_M = 2'b11, - PRIV_LVL_S = 2'b01, - PRIV_LVL_U = 2'b00 + PRIV_LVL_M = 2'b11, + PRIV_LVL_HS = 2'b10, + PRIV_LVL_S = 2'b01, + PRIV_LVL_U = 2'b00 } priv_lvl_t; // type which holds xlen @@ -53,7 +54,7 @@ package riscv; logic sd; // signal dirty state - read-only logic [62:34] wpri6; // writes preserved reads ignored xlen_e uxl; // variable user mode xlen - hardwired to zero - logic [12:0] wpri5; // writes preserved reads ignored + logic [11:0] wpri5; // writes preserved reads ignored logic mxr; // make executable readable logic sum; // permit supervisor user memory access logic wpri4; // writes preserved reads ignored @@ -63,16 +64,39 @@ package riscv; xs_t vs; // vector extension register logic spp; // holds the previous privilege mode up to supervisor logic wpri2; // writes preserved reads ignored + logic mpie; // machine interrupts enable bit active prior to trap logic ube; // UBE controls whether explicit load and store memory accesses made from U-mode are little-endian (UBE=0) or big-endian (UBE=1) logic spie; // supervisor interrupts enable bit active prior to trap - logic [1:0] wpri1; // writes preserved reads ignored + logic [2:0] wpri1; // writes preserved reads ignored logic sie; // supervisor interrupts enable logic wpri0; // writes preserved reads ignored } sstatus_rv_t; + typedef struct packed { + logic [63:34] wpri4; // writes preserved reads ignored + xlen_e vsxl; // variable virtual supervisor mode xlen - hardwired to zero + logic [8:0] wpri3; // floating point extension register + logic vtsr; // virtual trap sret + logic vtw; // virtual time wait + logic vtvm; // virtual trap virtual memory + logic [1:0] wpri2; // writes preserved reads ignored + logic [5:0] vgein; // virtual guest external interrupt number + logic [1:0] wpri1; // writes preserved reads ignored + logic hu; // virtual-machine load/store instructions enable in U-mode + logic spvp; // supervisor previous virtual privilege + logic spv; // supervisor previous virtualization mode + logic gva; // variable set when trap writes to stval + logic vsbe; // endianness of explicit memory accesses made from VS-mode + logic [4:0] wpri0; // writes preserved reads ignored + } hstatus_rv_t; + typedef struct packed { logic sd; // signal dirty state - read-only - logic [62:36] wpri4; // writes preserved reads ignored + logic [62:40] wpri4; // writes preserved reads ignored + logic mpv; // machine previous virtualization mode + logic gva; // variable set when trap writes to stval + logic mbe; // endianness memory accesses made from M-mode + logic sbe; // endianness memory accesses made from S-mode xlen_e sxl; // variable supervisor mode xlen - hardwired to zero xlen_e uxl; // variable user mode xlen - hardwired to zero logic [8:0] wpri3; // writes preserved reads ignored @@ -97,6 +121,17 @@ package riscv; logic wpri0; // writes preserved reads ignored } mstatus_rv_t; + typedef struct packed { + logic stce; // not implemented - requires Sctc extension + logic pbmte; // not implemented - requires Svpbmt extension + logic [61:8] wpri1; // writes preserved reads ignored + logic cbze; // not implemented - requires Zicboz extension + logic cbcfe; // not implemented - requires Zicbom extension + logic [1:0] cbie; // not implemented - requires Zicbom extension + logic [2:0] wpri0; // writes preserved reads ignored + logic fiom; // fence of I/O implies memory + } envcfg_rv_t; + // -------------------- // Instruction Types // -------------------- @@ -300,27 +335,48 @@ package riscv; localparam logic [XLEN-1:0] LD_ACCESS_FAULT = 5; // Illegal access as governed by PMPs and PMAs localparam logic [XLEN-1:0] ST_ADDR_MISALIGNED = 6; localparam logic [XLEN-1:0] ST_ACCESS_FAULT = 7; // Illegal access as governed by PMPs and PMAs - localparam logic [XLEN-1:0] ENV_CALL_UMODE = 8; // environment call from user mode - localparam logic [XLEN-1:0] ENV_CALL_SMODE = 9; // environment call from supervisor mode + localparam logic [XLEN-1:0] ENV_CALL_UMODE = 8; // environment call from user mode or virtual user mode + localparam logic [XLEN-1:0] ENV_CALL_SMODE = 9; // environment call from hypervisor-extended supervisor mode + localparam logic [XLEN-1:0] ENV_CALL_VSMODE = 10; // environment call from virtual supervisor mode localparam logic [XLEN-1:0] ENV_CALL_MMODE = 11; // environment call from machine mode localparam logic [XLEN-1:0] INSTR_PAGE_FAULT = 12; // Instruction page fault localparam logic [XLEN-1:0] LOAD_PAGE_FAULT = 13; // Load page fault localparam logic [XLEN-1:0] STORE_PAGE_FAULT = 15; // Store page fault + localparam logic [XLEN-1:0] INSTR_GUEST_PAGE_FAULT = 20; // Instruction guest-page fault + localparam logic [XLEN-1:0] LOAD_GUEST_PAGE_FAULT = 21; // Load guest-page fault + localparam logic [XLEN-1:0] VIRTUAL_INSTRUCTION = 22; // virtual instruction + localparam logic [XLEN-1:0] STORE_GUEST_PAGE_FAULT = 23; // Store guest-page fault localparam logic [XLEN-1:0] DEBUG_REQUEST = 24; // Debug request localparam int unsigned IRQ_S_SOFT = 1; + localparam int unsigned IRQ_VS_SOFT = 2; localparam int unsigned IRQ_M_SOFT = 3; localparam int unsigned IRQ_S_TIMER = 5; + localparam int unsigned IRQ_VS_TIMER = 6; localparam int unsigned IRQ_M_TIMER = 7; localparam int unsigned IRQ_S_EXT = 9; + localparam int unsigned IRQ_VS_EXT = 10; localparam int unsigned IRQ_M_EXT = 11; + localparam int unsigned IRQ_HS_EXT = 12; localparam logic [31:0] MIP_SSIP = 1 << IRQ_S_SOFT; + localparam logic [31:0] MIP_VSSIP = 1 << IRQ_VS_SOFT; localparam logic [31:0] MIP_MSIP = 1 << IRQ_M_SOFT; localparam logic [31:0] MIP_STIP = 1 << IRQ_S_TIMER; + localparam logic [31:0] MIP_VSTIP = 1 << IRQ_VS_TIMER; localparam logic [31:0] MIP_MTIP = 1 << IRQ_M_TIMER; localparam logic [31:0] MIP_SEIP = 1 << IRQ_S_EXT; + localparam logic [31:0] MIP_VSEIP = 1 << IRQ_VS_EXT; localparam logic [31:0] MIP_MEIP = 1 << IRQ_M_EXT; + localparam logic [31:0] MIP_SGEIP = 1 << IRQ_HS_EXT; + + // ---------------------- + // PseudoInstructions Codes + // ---------------------- + localparam logic [31:0] READ_32_PSEUDOINSTRUCTION = 32'h00002000; + localparam logic [31:0] WRITE_32_PSEUDOINSTRUCTION = 32'h00002020; + localparam logic [31:0] READ_64_PSEUDOINSTRUCTION = 32'h00003000; + localparam logic [31:0] WRITE_64_PSEUDOINSTRUCTION = 32'h00003020; // ----- // CSRs @@ -339,17 +395,46 @@ package riscv; CSR_VL = 12'hC20, CSR_VTYPE = 12'hC21, CSR_VLENB = 12'hC22, + // Virtual Supervisor Mode CSRs + CSR_VSSTATUS = 12'h200, + CSR_VSIE = 12'h204, + CSR_VSTVEC = 12'h205, + CSR_VSSCRATCH = 12'h240, + CSR_VSEPC = 12'h241, + CSR_VSCAUSE = 12'h242, + CSR_VSTVAL = 12'h243, + CSR_VSIP = 12'h244, + CSR_VSATP = 12'h280, // Supervisor Mode CSRs CSR_SSTATUS = 12'h100, CSR_SIE = 12'h104, CSR_STVEC = 12'h105, CSR_SCOUNTEREN = 12'h106, + CSR_SENVCFG = 12'h10A, CSR_SSCRATCH = 12'h140, CSR_SEPC = 12'h141, CSR_SCAUSE = 12'h142, CSR_STVAL = 12'h143, CSR_SIP = 12'h144, CSR_SATP = 12'h180, + // Hypervisor-extended Supervisor Mode CSRs + CSR_HSTATUS = 12'h600, + CSR_HEDELEG = 12'h602, + CSR_HIDELEG = 12'h603, + CSR_HIE = 12'h604, + CSR_HCOUNTEREN = 12'h606, + CSR_HGEIE = 12'h607, + CSR_HTVAL = 12'h643, + CSR_HIP = 12'h644, + CSR_HVIP = 12'h645, + CSR_HTINST = 12'h64A, + CSR_HGEIP = 12'hE12, + CSR_HENVCFG = 12'h60A, + CSR_HENVCFGH = 12'h61A, + CSR_HGATP = 12'h680, + CSR_HCONTEXT = 12'h6A8, + CSR_HTIMEDELTA = 12'h605, + CSR_HTIMEDELTAH = 12'h615, // Machine Mode CSRs CSR_MSTATUS = 12'h300, CSR_MISA = 12'h301, @@ -394,6 +479,8 @@ package riscv; CSR_MCAUSE = 12'h342, CSR_MTVAL = 12'h343, CSR_MIP = 12'h344, + CSR_MTINST = 12'h34A, + CSR_MTVAL2 = 12'h34B, CSR_MENVCFG = 12'h30A, CSR_MENVCFGH = 12'h31A, CSR_PMPCFG0 = 12'h3A0, @@ -582,6 +669,17 @@ package riscv; return {IS_XLEN64, 31'h00000000, ~IS_XLEN64, 31'h00000000}; endfunction + localparam logic [63:0] HSTATUS_VSBE = 'h00000020; + localparam logic [63:0] HSTATUS_GVA = 'h00000040; + localparam logic [63:0] HSTATUS_SPV = 'h00000080; + localparam logic [63:0] HSTATUS_SPVP = 'h00000100; + localparam logic [63:0] HSTATUS_HU = 'h00000200; + localparam logic [63:0] HSTATUS_VGEIN = 'h0003F000; + localparam logic [63:0] HSTATUS_VTVM = 'h00100000; + localparam logic [63:0] HSTATUS_VTW = 'h00200000; + localparam logic [63:0] HSTATUS_VTSR = 'h00400000; + localparam logic [63:0] HSTATUS_VSXL = 64'h0000000300000000; + localparam logic [63:0] MSTATUS_UIE = 'h00000001; localparam logic [63:0] MSTATUS_SIE = 'h00000002; localparam logic [63:0] MSTATUS_HIE = 'h00000004; @@ -611,6 +709,15 @@ package riscv; return {IS_XLEN64, 31'h00000000, ~IS_XLEN64, 31'h00000000}; endfunction + localparam logic [63:0] MENVCFG_FIOM = 'h00000001; + localparam logic [63:0] MENVCFG_CBIE = 'h00000030; + localparam logic [63:0] MENVCFG_CBFE = 'h00000040; + localparam logic [63:0] MENVCFG_CBZE = 'h00000080; + localparam logic [63:0] MENVCFG_PBMTE = 64'h4000000000000000; + localparam logic [63:0] MENVCFG_STCE = 64'h8000000000000000; + + + typedef enum logic [2:0] { CSRRW = 3'h1, CSRRS = 3'h2, @@ -675,7 +782,9 @@ package riscv; // ----- typedef struct packed { logic [31:28] xdebugver; - logic [27:16] zero2; + logic [27:18] zero2; + logic ebreakvs; + logic ebreakvu; logic ebreakm; logic zero1; logic ebreaks; @@ -684,7 +793,7 @@ package riscv; logic stopcount; logic stoptime; logic [8:6] cause; - logic zero0; + logic v; logic mprven; logic nmip; logic step; @@ -778,6 +887,22 @@ package riscv; return 32'h00000000; endfunction + // This functions converts S-mode CSR addresses into VS-mode CSR addresses + // when V=1 (i.e., running in VS-mode). + function automatic csr_t convert_vs_access_csr(csr_t csr_addr, logic v); + csr_t ret; + ret = csr_addr; + unique case (csr_addr.address) inside + [CSR_SSTATUS : CSR_STVEC], [CSR_SSCRATCH : CSR_SATP]: begin + if (v) begin + ret.csr_decode.priv_lvl = PRIV_LVL_HS; + end + return ret; + end + default: return ret; + endcase + endfunction + // trace log compatible to spikes commit log feature // pragma translate_off diff --git a/core/issue_read_operands.sv b/core/issue_read_operands.sv index 2c24e15139..ec95ac532a 100644 --- a/core/issue_read_operands.sv +++ b/core/issue_read_operands.sv @@ -78,6 +78,8 @@ module issue_read_operands output logic alu_valid_o, // Branch instruction is valid - TO_BE_COMPLETED output logic branch_valid_o, + // Transformed instruction - TO_BE_COMPLETED + output logic [31:0] tinst_o, // TO_BE_COMPLETED - TO_BE_COMPLETED output branchpredict_sbe_t branch_predict_o, // Load Store Unit is ready - TO_BE_COMPLETED @@ -139,6 +141,7 @@ module issue_read_operands logic [CVA6Cfg.TRANS_ID_BITS-1:0] trans_id_n, trans_id_q; fu_op operator_n, operator_q; // operation to perform fu_t fu_n, fu_q; // functional unit to use + logic [31:0] tinst_n, tinst_q; // transformed instruction // forwarding signals logic forward_rs1, forward_rs2, forward_rs3; @@ -169,6 +172,7 @@ module issue_read_operands assign cvxif_valid_o = CVA6Cfg.CvxifEn ? cvxif_valid_q : '0; assign cvxif_off_instr_o = CVA6Cfg.CvxifEn ? cvxif_off_instr_q : '0; assign stall_issue_o = stall; + assign tinst_o = CVA6Cfg.RVH ? tinst_q : '0; // --------------- // Issue Stage // --------------- @@ -283,6 +287,9 @@ module issue_read_operands trans_id_n = issue_instr_i.trans_id; fu_n = issue_instr_i.fu; operator_n = issue_instr_i.op; + if (CVA6Cfg.RVH) begin + tinst_n = issue_instr_i.ex.tinst; + end // or should we forward if (forward_rs1) begin operand_a_n = rs1_i; @@ -591,22 +598,28 @@ module issue_read_operands // ---------------------- always_ff @(posedge clk_i or negedge rst_ni) begin if (!rst_ni) begin - operand_a_q <= '{default: 0}; - operand_b_q <= '{default: 0}; - imm_q <= '0; - fu_q <= NONE; - operator_q <= ADD; - trans_id_q <= '0; + operand_a_q <= '{default: 0}; + operand_b_q <= '{default: 0}; + imm_q <= '0; + fu_q <= NONE; + operator_q <= ADD; + trans_id_q <= '0; + if (CVA6Cfg.RVH) begin + tinst_q <= '0; + end pc_o <= '0; is_compressed_instr_o <= 1'b0; branch_predict_o <= {cf_t'(0), {CVA6Cfg.VLEN{1'b0}}}; end else begin - operand_a_q <= operand_a_n; - operand_b_q <= operand_b_n; - imm_q <= imm_n; - fu_q <= fu_n; - operator_q <= operator_n; - trans_id_q <= trans_id_n; + operand_a_q <= operand_a_n; + operand_b_q <= operand_b_n; + imm_q <= imm_n; + fu_q <= fu_n; + operator_q <= operator_n; + trans_id_q <= trans_id_n; + if (CVA6Cfg.RVH) begin + tinst_q <= tinst_n; + end pc_o <= issue_instr_i.pc; is_compressed_instr_o <= issue_instr_i.is_compressed; branch_predict_o <= issue_instr_i.bp; diff --git a/core/issue_stage.sv b/core/issue_stage.sv index 9bf5767fb8..fda5594297 100644 --- a/core/issue_stage.sv +++ b/core/issue_stage.sv @@ -56,6 +56,8 @@ module issue_stage output logic [CVA6Cfg.VLEN-1:0] pc_o, // Is compressed instruction - EX_STAGE output logic is_compressed_instr_o, + // Transformed trap instruction - EX_STAGE + output logic [31:0] tinst_o, // Fixed Latency Unit is ready - EX_STAGE input logic flu_ready_i, // ALU FU is valid - EX_STAGE @@ -235,6 +237,7 @@ module issue_stage .rs1_forwarding_o (rs1_forwarding_xlen), .rs2_forwarding_o (rs2_forwarding_xlen), .stall_issue_o (stall_issue_o), + .tinst_o (tinst_o), .* ); diff --git a/core/load_store_unit.sv b/core/load_store_unit.sv index c5051f94e2..7eda7f971f 100644 --- a/core/load_store_unit.sv +++ b/core/load_store_unit.sv @@ -39,6 +39,8 @@ module load_store_unit output logic no_st_pending_o, // TO_BE_COMPLETED - TO_BE_COMPLETED input logic amo_valid_commit_i, + // TO_BE_COMPLETED - TO_BE_COMPLETED + input logic [31:0] tinst_i, // FU data needed to execute instruction - ISSUE_STAGE input fu_data_t fu_data_i, // Load Store Unit is ready - ISSUE_STAGE @@ -73,8 +75,12 @@ module load_store_unit // Enable virtual memory translation - TO_BE_COMPLETED input logic enable_translation_i, + // Enable G-Stage memory translation - TO_BE_COMPLETED + input logic enable_g_translation_i, // Enable virtual memory translation for load/stores - TO_BE_COMPLETED input logic en_ld_st_translation_i, + // Enable G-Stage memory translation for load/stores - TO_BE_COMPLETED + input logic en_ld_st_g_translation_i, // Instruction cache input request - CACHES input icache_arsp_t icache_areq_i, @@ -83,22 +89,46 @@ module load_store_unit // Current privilege mode - CSR_REGFILE input riscv::priv_lvl_t priv_lvl_i, + // Current virtualization mode - CSR_REGFILE + input logic v_i, // Privilege level at which load and stores should happen - CSR_REGFILE input riscv::priv_lvl_t ld_st_priv_lvl_i, + // Virtualization mode at which load and stores should happen - CSR_REGFILE + input logic ld_st_v_i, + // Instruction is a hyp load/store - CSR_REGFILE + output logic csr_hs_ld_st_inst_o, // Supervisor User Memory - CSR_REGFILE input logic sum_i, + // Virtual Supervisor User Memory - CSR_REGFILE + input logic vs_sum_i, // Make Executable Readable - CSR_REGFILE input logic mxr_i, + // Make Executable Readable Virtual Supervisor - CSR_REGFILE + input logic vmxr_i, // TO_BE_COMPLETED - TO_BE_COMPLETED input logic [ CVA6Cfg.PPNW-1:0] satp_ppn_i, // TO_BE_COMPLETED - TO_BE_COMPLETED input logic [CVA6Cfg.ASID_WIDTH-1:0] asid_i, // TO_BE_COMPLETED - TO_BE_COMPLETED + input logic [ CVA6Cfg.PPNW-1:0] vsatp_ppn_i, + // TO_BE_COMPLETED - TO_BE_COMPLETED + input logic [CVA6Cfg.ASID_WIDTH-1:0] vs_asid_i, + // TO_BE_COMPLETED - TO_BE_COMPLETED + input logic [ CVA6Cfg.PPNW-1:0] hgatp_ppn_i, + // TO_BE_COMPLETED - TO_BE_COMPLETED + input logic [CVA6Cfg.VMID_WIDTH-1:0] vmid_i, + // TO_BE_COMPLETED - TO_BE_COMPLETED input logic [CVA6Cfg.ASID_WIDTH-1:0] asid_to_be_flushed_i, // TO_BE_COMPLETED - TO_BE_COMPLETED + input logic [CVA6Cfg.VMID_WIDTH-1:0] vmid_to_be_flushed_i, + // TO_BE_COMPLETED - TO_BE_COMPLETED input logic [ CVA6Cfg.VLEN-1:0] vaddr_to_be_flushed_i, + // TO_BE_COMPLETED - TO_BE_COMPLETED + input logic [ CVA6Cfg.GPLEN-1:0] gpaddr_to_be_flushed_i, // TLB flush - CONTROLLER input logic flush_tlb_i, + input logic flush_tlb_vvma_i, + input logic flush_tlb_gvma_i, // Instruction TLB miss - PERF_COUNTERS output logic itlb_miss_o, // Data TLB miss - PERF_COUNTERS @@ -146,23 +176,38 @@ module load_store_unit logic [ CVA6Cfg.VLEN-1:0] vaddr_i; logic [ CVA6Cfg.XLEN-1:0] vaddr_xlen; logic overflow; + logic g_overflow; logic [(CVA6Cfg.XLEN/8)-1:0] be_i; assign vaddr_xlen = $unsigned($signed(fu_data_i.imm) + $signed(fu_data_i.operand_a)); assign vaddr_i = vaddr_xlen[CVA6Cfg.VLEN-1:0]; - // we work with SV39 or SV32, so if VM is enabled, check that all bits [CVA6Cfg.XLEN-1:38] or [CVA6Cfg.XLEN-1:31] are equal + // we work with SV39 or SV32, so if VM is enabled, check that all bits [XLEN-1:38] or [XLEN-1:31] are equal assign overflow = (CVA6Cfg.IS_XLEN64 && (!((&vaddr_xlen[CVA6Cfg.XLEN-1:CVA6Cfg.SV-1]) == 1'b1 || (|vaddr_xlen[CVA6Cfg.XLEN-1:CVA6Cfg.SV-1]) == 1'b0))); + if (CVA6Cfg.RVH) begin : gen_g_overflow_hyp + assign g_overflow = (CVA6Cfg.IS_XLEN64 && (!((|vaddr_xlen[CVA6Cfg.XLEN-1:CVA6Cfg.SVX]) == 1'b0))); + end else begin : gen_g_overflow_no_hyp + assign g_overflow = 1'b0; + end logic st_valid_i; logic ld_valid_i; logic ld_translation_req; logic st_translation_req; logic [CVA6Cfg.VLEN-1:0] ld_vaddr; + logic [ 31:0] ld_tinst; + logic ld_hs_ld_st_inst; + logic ld_hlvx_inst; logic [CVA6Cfg.VLEN-1:0] st_vaddr; + logic [ 31:0] st_tinst; + logic st_hs_ld_st_inst; + logic st_hlvx_inst; logic translation_req; logic translation_valid; logic [CVA6Cfg.VLEN-1:0] mmu_vaddr; logic [CVA6Cfg.PLEN-1:0] mmu_paddr, mmu_vaddr_plen, fetch_vaddr_plen; + logic [ 31:0] mmu_tinst; + logic mmu_hs_ld_st_inst; + logic mmu_hlvx_inst; exception_t mmu_exception; logic dtlb_hit; logic [ CVA6Cfg.PPNW-1:0] dtlb_ppn; @@ -181,10 +226,53 @@ module load_store_unit exception_t ld_ex; exception_t st_ex; + logic hs_ld_st_inst; + logic hlvx_inst; // ------------------- // MMU e.g.: TLBs/PTW // ------------------- - if (MMU_PRESENT && (CVA6Cfg.XLEN == 64)) begin : gen_mmu_sv39 + if (MMU_PRESENT && CVA6Cfg.RVH && (CVA6Cfg.XLEN == 64)) begin : gen_mmu_sv39x4 + cva6_mmu_sv39x4 #( + .CVA6Cfg (CVA6Cfg), + .exception_t (exception_t), + .icache_areq_t (icache_areq_t), + .icache_arsp_t (icache_arsp_t), + .icache_dreq_t (icache_dreq_t), + .icache_drsp_t (icache_drsp_t), + .dcache_req_i_t (dcache_req_i_t), + .dcache_req_o_t (dcache_req_o_t), + .INSTR_TLB_ENTRIES(ariane_pkg::INSTR_TLB_ENTRIES), + .DATA_TLB_ENTRIES (ariane_pkg::DATA_TLB_ENTRIES) + ) i_cva6_mmu ( + // misaligned bypass + .misaligned_ex_i(misaligned_exception), + .lsu_is_store_i (st_translation_req), + .lsu_req_i (translation_req), + .lsu_vaddr_i (mmu_vaddr), + .lsu_tinst_i (mmu_tinst), + .lsu_valid_o (translation_valid), + .lsu_paddr_o (mmu_paddr), + .lsu_exception_o(mmu_exception), + .lsu_dtlb_hit_o (dtlb_hit), // send in the same cycle as the request + .lsu_dtlb_ppn_o (dtlb_ppn), // send in the same cycle as the request + // connecting PTW to D$ IF + .req_port_i (dcache_req_ports_i[0]), + .req_port_o (dcache_req_ports_o[0]), + // icache address translation requests + .icache_areq_i (icache_areq_i), + .asid_to_be_flushed_i, + .vmid_to_be_flushed_i, + .vaddr_to_be_flushed_i, + .gpaddr_to_be_flushed_i, + .icache_areq_o (icache_areq_o), + .pmpcfg_i, + .pmpaddr_i, + // Hypervisor load/store signals + .hlvx_inst_i (mmu_hlvx_inst), + .hs_ld_st_inst_i(mmu_hs_ld_st_inst), + .* + ); + end else if (MMU_PRESENT && (CVA6Cfg.XLEN == 64)) begin : gen_mmu_sv39 mmu #( .CVA6Cfg (CVA6Cfg), .exception_t (exception_t), @@ -330,6 +418,9 @@ module load_store_unit .translation_req_o (st_translation_req), .vaddr_o (st_vaddr), .rvfi_mem_paddr_o (rvfi_mem_paddr_o), + .tinst_o (st_tinst), + .hs_ld_st_inst_o (st_hs_ld_st_inst), + .hlvx_inst_o (st_hlvx_inst), .paddr_i (mmu_paddr), .ex_i (mmu_exception), .dtlb_hit_i (dtlb_hit), @@ -365,6 +456,9 @@ module load_store_unit // MMU port .translation_req_o (ld_translation_req), .vaddr_o (ld_vaddr), + .tinst_o (ld_tinst), + .hs_ld_st_inst_o (ld_hs_ld_st_inst), + .hlvx_inst_o (ld_hlvx_inst), .paddr_i (mmu_paddr), .ex_i (mmu_exception), .dtlb_hit_i (dtlb_hit), @@ -411,11 +505,14 @@ module load_store_unit // determine whether this is a load or store always_comb begin : which_op - ld_valid_i = 1'b0; - st_valid_i = 1'b0; + ld_valid_i = 1'b0; + st_valid_i = 1'b0; - translation_req = 1'b0; - mmu_vaddr = {CVA6Cfg.VLEN{1'b0}}; + translation_req = 1'b0; + mmu_vaddr = {CVA6Cfg.VLEN{1'b0}}; + mmu_tinst = {32{1'b0}}; + mmu_hs_ld_st_inst = 1'b0; + mmu_hlvx_inst = 1'b0; // check the operation to activate the right functional unit accordingly unique case (lsu_ctrl.fu) @@ -424,18 +521,53 @@ module load_store_unit ld_valid_i = lsu_ctrl.valid; translation_req = ld_translation_req; mmu_vaddr = ld_vaddr; + if (CVA6Cfg.RVH) begin + mmu_tinst = ld_tinst; + mmu_hs_ld_st_inst = ld_hs_ld_st_inst; + mmu_hlvx_inst = ld_hlvx_inst; + end end // all stores go here STORE: begin st_valid_i = lsu_ctrl.valid; translation_req = st_translation_req; mmu_vaddr = st_vaddr; + if (CVA6Cfg.RVH) begin + mmu_tinst = st_tinst; + mmu_hs_ld_st_inst = st_hs_ld_st_inst; + mmu_hlvx_inst = st_hlvx_inst; + end end // not relevant for the LSU default: ; endcase end + // ------------------------ + // Hypervisor Load/Store + // ------------------------ + // determine whether this is a hypervisor load or store + if (CVA6Cfg.RVH) begin + always_comb begin : hyp_ld_st + // check the operator to activate the right functional unit accordingly + hs_ld_st_inst = 1'b0; + hlvx_inst = 1'b0; + case (lsu_ctrl.operation) + // all loads go here + HLV_B, HLV_BU, HLV_H, HLV_HU, HLV_W, HSV_B, HSV_H, HSV_W, HLV_WU, HLV_D, HSV_D: begin + hs_ld_st_inst = 1'b1; + end + HLVX_WU, HLVX_HU: begin + hs_ld_st_inst = 1'b1; + hlvx_inst = 1'b1; + end + default: ; + endcase + end + end else begin + assign hs_ld_st_inst = 1'b0; + assign hlvx_inst = 1'b0; + end // --------------- // Byte Enable @@ -457,9 +589,9 @@ module load_store_unit // the misaligned exception is passed to the functional unit via the MMU, which in case // can augment the exception if other memory related exceptions like a page fault or access errors always_comb begin : data_misaligned_detection - - misaligned_exception = {{CVA6Cfg.XLEN{1'b0}}, {CVA6Cfg.XLEN{1'b0}}, 1'b0}; - + misaligned_exception = { + {CVA6Cfg.XLEN{1'b0}}, {CVA6Cfg.XLEN{1'b0}}, {CVA6Cfg.GPLEN{1'b0}}, {32{1'b0}}, 1'b0, 1'b0 + }; data_misaligned = 1'b0; if (lsu_ctrl.valid) begin @@ -469,7 +601,7 @@ module load_store_unit AMO_LRD, AMO_SCD, AMO_SWAPD, AMO_ADDD, AMO_ANDD, AMO_ORD, AMO_XORD, AMO_MAXD, AMO_MAXDU, AMO_MIND, - AMO_MINDU: begin + AMO_MINDU, HLV_D, HSV_D: begin if (CVA6Cfg.IS_XLEN64 && lsu_ctrl.vaddr[2:0] != 3'b000) begin data_misaligned = 1'b1; end @@ -479,13 +611,13 @@ module load_store_unit AMO_LRW, AMO_SCW, AMO_SWAPW, AMO_ADDW, AMO_ANDW, AMO_ORW, AMO_XORW, AMO_MAXW, AMO_MAXWU, AMO_MINW, - AMO_MINWU: begin + AMO_MINWU, HLV_W, HLV_WU, HLVX_WU, HSV_W: begin if (lsu_ctrl.vaddr[1:0] != 2'b00) begin data_misaligned = 1'b1; end end // half word - LH, LHU, SH, FLH, FSH: begin + LH, LHU, SH, FLH, FSH, HLV_H, HLV_HU, HLVX_HU, HSV_H: begin if (lsu_ctrl.vaddr[0] != 1'b0) begin data_misaligned = 1'b1; end @@ -502,12 +634,22 @@ module load_store_unit misaligned_exception.valid = 1'b1; if (CVA6Cfg.TvalEn) misaligned_exception.tval = {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{1'b0}}, lsu_ctrl.vaddr}; + if (CVA6Cfg.RVH) begin + misaligned_exception.tval2 = '0; + misaligned_exception.tinst = lsu_ctrl.tinst; + misaligned_exception.gva = ld_st_v_i; + end end else if (lsu_ctrl.fu == STORE) begin misaligned_exception.cause = riscv::ST_ADDR_MISALIGNED; misaligned_exception.valid = 1'b1; if (CVA6Cfg.TvalEn) misaligned_exception.tval = {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{1'b0}}, lsu_ctrl.vaddr}; + if (CVA6Cfg.RVH) begin + misaligned_exception.tval2 = '0; + misaligned_exception.tinst = lsu_ctrl.tinst; + misaligned_exception.gva = ld_st_v_i; + end end end @@ -518,12 +660,47 @@ module load_store_unit misaligned_exception.valid = 1'b1; if (CVA6Cfg.TvalEn) misaligned_exception.tval = {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{1'b0}}, lsu_ctrl.vaddr}; + if (CVA6Cfg.RVH) begin + misaligned_exception.tval2 = '0; + misaligned_exception.tinst = lsu_ctrl.tinst; + misaligned_exception.gva = ld_st_v_i; + end end else if (lsu_ctrl.fu == STORE) begin misaligned_exception.cause = riscv::ST_ACCESS_FAULT; misaligned_exception.valid = 1'b1; if (CVA6Cfg.TvalEn) misaligned_exception.tval = {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{1'b0}}, lsu_ctrl.vaddr}; + if (CVA6Cfg.RVH) begin + misaligned_exception.tval2 = '0; + misaligned_exception.tinst = lsu_ctrl.tinst; + misaligned_exception.gva = ld_st_v_i; + end + end + end + + if (ariane_pkg::MMU_PRESENT && CVA6Cfg.RVH && en_ld_st_g_translation_i && !en_ld_st_translation_i && lsu_ctrl.g_overflow) begin + + if (lsu_ctrl.fu == LOAD) begin + misaligned_exception.cause = riscv::LOAD_GUEST_PAGE_FAULT; + misaligned_exception.valid = 1'b1; + if (CVA6Cfg.TvalEn) + misaligned_exception.tval = {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{1'b0}}, lsu_ctrl.vaddr}; + if (CVA6Cfg.RVH) begin + misaligned_exception.tval2 = '0; + misaligned_exception.tinst = lsu_ctrl.tinst; + misaligned_exception.gva = ld_st_v_i; + end + end else if (lsu_ctrl.fu == STORE) begin + misaligned_exception.cause = riscv::STORE_GUEST_PAGE_FAULT; + misaligned_exception.valid = 1'b1; + if (CVA6Cfg.TvalEn) + misaligned_exception.tval = {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{1'b0}}, lsu_ctrl.vaddr}; + if (CVA6Cfg.RVH) begin + misaligned_exception.tval2 = '0; + misaligned_exception.tinst = lsu_ctrl.tinst; + misaligned_exception.gva = ld_st_v_i; + end end end end @@ -537,7 +714,11 @@ module load_store_unit assign lsu_req_i = { lsu_valid_i, vaddr_i, + tinst_i, + hs_ld_st_inst, + hlvx_inst, overflow, + g_overflow, fu_data_i.operand_b, be_i, fu_data_i.fu, diff --git a/core/load_unit.sv b/core/load_unit.sv index 0db03f3241..1be140ea05 100644 --- a/core/load_unit.sv +++ b/core/load_unit.sv @@ -51,6 +51,12 @@ module load_unit output logic translation_req_o, // Virtual address - TO_BE_COMPLETED output logic [CVA6Cfg.VLEN-1:0] vaddr_o, + // Transformed trap instruction out - TO_BE_COMPLETED + output logic [31:0] tinst_o, + // Instruction is a hyp load store instruction - TO_BE_COMPLETED + output logic hs_ld_st_inst_o, + // Hyp load store with execute permissions - TO_BE_COMPLETED + output logic hlvx_inst_o, // Physical address - TO_BE_COMPLETED input logic [CVA6Cfg.PLEN-1:0] paddr_i, // Excepted which appears before load - TO_BE_COMPLETED @@ -183,6 +189,10 @@ module load_unit assign page_offset_o = lsu_ctrl_i.vaddr[11:0]; // feed-through the virtual address for VA translation assign vaddr_o = lsu_ctrl_i.vaddr; + assign hs_ld_st_inst_o = CVA6Cfg.RVH ? lsu_ctrl_i.hs_ld_st_inst : 1'b0; + assign hlvx_inst_o = CVA6Cfg.RVH ? lsu_ctrl_i.hlvx_inst : 1'b0; + // feed-through the transformed instruction for mmu + assign tinst_o = CVA6Cfg.RVH ? lsu_ctrl_i.tinst : '0; // this is a read-only interface so set the write enable to 0 assign req_port_o.data_we = 1'b0; assign req_port_o.data_wdata = '0; @@ -202,6 +212,9 @@ module load_unit // directly forward exception fields (valid bit is set below) assign ex_o.cause = ex_i.cause; assign ex_o.tval = ex_i.tval; + assign ex_o.tval2 = CVA6Cfg.RVH ? ex_i.tval2 : '0; + assign ex_o.tinst = CVA6Cfg.RVH ? ex_i.tinst : '0; + assign ex_o.gva = CVA6Cfg.RVH ? ex_i.gva : 1'b0; // Check that NI operations follow the necessary conditions logic paddr_ni; @@ -479,10 +492,10 @@ module load_unit // prepare these signals for faster selection in the next cycle - assign rdata_is_signed = ldbuf_rdata.operation inside {ariane_pkg::LW, ariane_pkg::LH, ariane_pkg::LB}; + assign rdata_is_signed = ldbuf_rdata.operation inside {ariane_pkg::LW, ariane_pkg::LH, ariane_pkg::LB, ariane_pkg::HLV_W, ariane_pkg::HLV_H, ariane_pkg::HLV_B}; assign rdata_is_fp_signed = ldbuf_rdata.operation inside {ariane_pkg::FLW, ariane_pkg::FLH, ariane_pkg::FLB}; - assign rdata_offset = ((ldbuf_rdata.operation inside {ariane_pkg::LW, ariane_pkg::FLW}) & CVA6Cfg.IS_XLEN64) ? ldbuf_rdata.address_offset + 3 : - ( ldbuf_rdata.operation inside {ariane_pkg::LH, ariane_pkg::FLH}) ? ldbuf_rdata.address_offset + 1 : + assign rdata_offset = ((ldbuf_rdata.operation inside {ariane_pkg::LW, ariane_pkg::FLW, ariane_pkg::HLV_W}) & CVA6Cfg.IS_XLEN64) ? ldbuf_rdata.address_offset + 3 : + ( ldbuf_rdata.operation inside {ariane_pkg::LH, ariane_pkg::FLH, ariane_pkg::HLV_H}) ? ldbuf_rdata.address_offset + 1 : ldbuf_rdata.address_offset; for (genvar i = 0; i < (CVA6Cfg.XLEN / 8); i++) begin : gen_sign_bits @@ -497,11 +510,11 @@ module load_unit // result mux always_comb begin unique case (ldbuf_rdata.operation) - ariane_pkg::LW, ariane_pkg::LWU: + ariane_pkg::LW, ariane_pkg::LWU, ariane_pkg::HLV_W, ariane_pkg::HLV_WU, ariane_pkg::HLVX_WU: result_o = {{CVA6Cfg.XLEN - 32{rdata_sign_bit}}, shifted_data[31:0]}; - ariane_pkg::LH, ariane_pkg::LHU: + ariane_pkg::LH, ariane_pkg::LHU, ariane_pkg::HLV_H, ariane_pkg::HLV_HU, ariane_pkg::HLVX_HU: result_o = {{CVA6Cfg.XLEN - 32 + 16{rdata_sign_bit}}, shifted_data[15:0]}; - ariane_pkg::LB, ariane_pkg::LBU: + ariane_pkg::LB, ariane_pkg::LBU, ariane_pkg::HLV_B, ariane_pkg::HLV_BU: result_o = {{CVA6Cfg.XLEN - 32 + 24{rdata_sign_bit}}, shifted_data[7:0]}; default: begin // FLW, FLH and FLB have been defined here in default case to improve Code Coverage diff --git a/core/mmu_sv39x4/cva6_mmu_sv39x4.sv b/core/mmu_sv39x4/cva6_mmu_sv39x4.sv new file mode 100644 index 0000000000..07138a59cd --- /dev/null +++ b/core/mmu_sv39x4/cva6_mmu_sv39x4.sv @@ -0,0 +1,731 @@ +// Copyright (c) 2022 Bruno Sá and Zero-Day Labs. +// Copyright and related rights are licensed under the Solderpad Hardware +// License, Version 0.51 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law +// or agreed to in writing, software, hardware and materials distributed under +// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// Author: Bruno Sá +// Date: 14/08/2022 +// Acknowledges: Technology Innovation Institute (TII) +// +// Description: Memory Management Unit for CV32A6, contains TLB and +// address translation unit. Sv39x4 as defined in RISC-V +// privilege specification 1.12. +// This module is an adaptation of the MMU Sv39x4 developed +// by Florian Zaruba to the Sv39x4 standard. + + +module cva6_mmu_sv39x4 + import ariane_pkg::*; +#( + parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, + parameter type icache_areq_t = logic, + parameter type icache_arsp_t = logic, + parameter type icache_dreq_t = logic, + parameter type icache_drsp_t = logic, + parameter type dcache_req_i_t = logic, + parameter type dcache_req_o_t = logic, + parameter type exception_t = logic, + parameter int unsigned INSTR_TLB_ENTRIES = 4, + parameter int unsigned DATA_TLB_ENTRIES = 4 +) ( + input logic clk_i, + input logic rst_ni, + input logic flush_i, + input logic enable_translation_i, + input logic enable_g_translation_i, + input logic en_ld_st_translation_i, // enable virtual memory translation for load/stores + input logic en_ld_st_g_translation_i, // enable G-Stage translation for load/stores + // IF interface + input icache_arsp_t icache_areq_i, + output icache_areq_t icache_areq_o, + // LSU interface + // this is a more minimalistic interface because the actual addressing logic is handled + // in the LSU as we distinguish load and stores, what we do here is simple address translation + input exception_t misaligned_ex_i, + input logic lsu_req_i, // request address translation + input logic [CVA6Cfg.VLEN-1:0] lsu_vaddr_i, // virtual address in + input logic [31:0] lsu_tinst_i, // transformed instruction in + input logic lsu_is_store_i, // the translation is requested by a store + output logic csr_hs_ld_st_inst_o, // hyp load store instruction + // if we need to walk the page table we can't grant in the same cycle + // Cycle 0 + output logic lsu_dtlb_hit_o, // sent in the same cycle as the request if translation hits in the DTLB + output logic [CVA6Cfg.PPNW-1:0] lsu_dtlb_ppn_o, // ppn (send same cycle as hit) + // Cycle 1 + output logic lsu_valid_o, // translation is valid + output logic [riscv::PLEN-1:0] lsu_paddr_o, // translated address + output exception_t lsu_exception_o, // address translation threw an exception + // General control signals + input riscv::priv_lvl_t priv_lvl_i, + input logic v_i, + input riscv::priv_lvl_t ld_st_priv_lvl_i, + input logic ld_st_v_i, + input logic sum_i, + input logic vs_sum_i, + input logic mxr_i, + input logic vmxr_i, + input logic hlvx_inst_i, + input logic hs_ld_st_inst_i, + // input logic flag_mprv_i, + input logic [CVA6Cfg.PPNW-1:0] satp_ppn_i, + input logic [CVA6Cfg.PPNW-1:0] vsatp_ppn_i, + input logic [CVA6Cfg.PPNW-1:0] hgatp_ppn_i, + input logic [CVA6Cfg.ASID_WIDTH-1:0] asid_i, + input logic [CVA6Cfg.ASID_WIDTH-1:0] vs_asid_i, + input logic [CVA6Cfg.ASID_WIDTH-1:0] asid_to_be_flushed_i, + input logic [CVA6Cfg.VMID_WIDTH-1:0] vmid_i, + input logic [CVA6Cfg.VMID_WIDTH-1:0] vmid_to_be_flushed_i, + input logic [CVA6Cfg.VLEN-1:0] vaddr_to_be_flushed_i, + input logic [CVA6Cfg.GPLEN-1:0] gpaddr_to_be_flushed_i, + input logic flush_tlb_i, + input logic flush_tlb_vvma_i, + input logic flush_tlb_gvma_i, + // Performance counters + output logic itlb_miss_o, + output logic dtlb_miss_o, + // PTW memory interface + input dcache_req_o_t req_port_i, + output dcache_req_i_t req_port_o, + // PMP + input riscv::pmpcfg_t [15:0] pmpcfg_i, + input logic [15:0][riscv::PLEN-3:0] pmpaddr_i +); + localparam type tlb_update_t = struct packed { + logic valid; // valid flag + logic is_s_2M; + logic is_s_1G; + logic is_g_2M; + logic is_g_1G; + logic [28:0] vpn; + logic [CVA6Cfg.ASID_WIDTH-1:0] asid; + logic [CVA6Cfg.VMID_WIDTH-1:0] vmid; + riscv::pte_t content; + riscv::pte_t g_content; + }; + + logic iaccess_err; // insufficient privilege to access this instruction page + logic i_g_st_access_err; // insufficient privilege at g stage to access this instruction page + logic daccess_err; // insufficient privilege to access this data page + logic d_g_st_access_err; // insufficient privilege to access this data page + logic ptw_active; // PTW is currently walking a page table + logic walking_instr; // PTW is walking because of an ITLB miss + logic ptw_error; // PTW threw an exception + logic ptw_error_at_g_st; // PTW threw an exception at the G-Stage + logic ptw_err_at_g_int_st; // PTW threw an exception at the G-Stage during S-Stage translation + logic ptw_access_exception; // PTW threw an access exception (PMPs) + logic [CVA6Cfg.GPLEN-1:0] ptw_bad_gpaddr; // PTW guest page fault bad guest physical addr + + logic [CVA6Cfg.VLEN-1:0] update_vaddr; + tlb_update_t update_ptw_itlb, update_ptw_dtlb; + + logic itlb_lu_access; + riscv::pte_t itlb_content; + logic itlb_is_2M; + logic itlb_is_1G; + // data from G-stage translation + riscv::pte_t itlb_g_content; + logic itlb_lu_hit; + logic [ CVA6Cfg.GPLEN-1:0] itlb_gpaddr; + logic [CVA6Cfg.ASID_WIDTH-1:0] itlb_lu_asid; + + logic dtlb_lu_access; + riscv::pte_t dtlb_content; + logic dtlb_is_2M; + logic dtlb_is_1G; + logic [CVA6Cfg.ASID_WIDTH-1:0] dtlb_lu_asid; + // data from G-stage translation + riscv::pte_t dtlb_g_content; + logic dtlb_lu_hit; + logic [ CVA6Cfg.GPLEN-1:0] dtlb_gpaddr; + + + // Assignments + assign itlb_lu_access = icache_areq_i.fetch_req; + assign dtlb_lu_access = lsu_req_i; + assign itlb_lu_asid = v_i ? vs_asid_i : asid_i; + assign dtlb_lu_asid = (ld_st_v_i || flush_tlb_vvma_i) ? vs_asid_i : asid_i; + + + cva6_tlb_sv39x4 #( + .CVA6Cfg (CVA6Cfg), + .tlb_update_t(tlb_update_t), + .TLB_ENTRIES (INSTR_TLB_ENTRIES) + ) i_itlb ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .flush_i (flush_tlb_i), + .flush_vvma_i(flush_tlb_vvma_i), + .flush_gvma_i(flush_tlb_gvma_i), + .s_st_enbl_i (enable_translation_i), + .g_st_enbl_i (enable_g_translation_i), + .v_i (v_i), + + .update_i(update_ptw_itlb), + + .lu_access_i (itlb_lu_access), + .lu_asid_i (itlb_lu_asid), + .lu_vmid_i (vmid_i), + .asid_to_be_flushed_i (asid_to_be_flushed_i), + .vmid_to_be_flushed_i (vmid_to_be_flushed_i), + .vaddr_to_be_flushed_i (vaddr_to_be_flushed_i), + .gpaddr_to_be_flushed_i(gpaddr_to_be_flushed_i), + .lu_vaddr_i (icache_areq_i.fetch_vaddr), + .lu_content_o (itlb_content), + .lu_g_content_o (itlb_g_content), + .lu_gpaddr_o (itlb_gpaddr), + + .lu_is_2M_o(itlb_is_2M), + .lu_is_1G_o(itlb_is_1G), + .lu_hit_o (itlb_lu_hit) + ); + + cva6_tlb_sv39x4 #( + .CVA6Cfg (CVA6Cfg), + .tlb_update_t(tlb_update_t), + .TLB_ENTRIES (DATA_TLB_ENTRIES) + ) i_dtlb ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .flush_i (flush_tlb_i), + .flush_vvma_i(flush_tlb_vvma_i), + .flush_gvma_i(flush_tlb_gvma_i), + .s_st_enbl_i (en_ld_st_translation_i), + .g_st_enbl_i (en_ld_st_g_translation_i), + .v_i (ld_st_v_i), + + .update_i(update_ptw_dtlb), + + .lu_access_i (dtlb_lu_access), + .lu_asid_i (dtlb_lu_asid), + .lu_vmid_i (vmid_i), + .asid_to_be_flushed_i (asid_to_be_flushed_i), + .vmid_to_be_flushed_i (vmid_to_be_flushed_i), + .vaddr_to_be_flushed_i (vaddr_to_be_flushed_i), + .gpaddr_to_be_flushed_i(gpaddr_to_be_flushed_i), + .lu_vaddr_i (lsu_vaddr_i), + .lu_content_o (dtlb_content), + .lu_g_content_o (dtlb_g_content), + .lu_gpaddr_o (dtlb_gpaddr), + + .lu_is_2M_o(dtlb_is_2M), + .lu_is_1G_o(dtlb_is_1G), + .lu_hit_o (dtlb_lu_hit) + ); + + + cva6_ptw_sv39x4 #( + .CVA6Cfg (CVA6Cfg), + .dcache_req_i_t(dcache_req_i_t), + .dcache_req_o_t(dcache_req_o_t), + .tlb_update_t(tlb_update_t) + ) i_ptw ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .ptw_active_o (ptw_active), + .walking_instr_o (walking_instr), + .ptw_error_o (ptw_error), + .ptw_error_at_g_st_o (ptw_error_at_g_st), + .ptw_err_at_g_int_st_o (ptw_err_at_g_int_st), + .ptw_access_exception_o(ptw_access_exception), + .enable_translation_i (enable_translation_i), + .enable_g_translation_i(enable_g_translation_i), + + .update_vaddr_o(update_vaddr), + .itlb_update_o (update_ptw_itlb), + .dtlb_update_o (update_ptw_dtlb), + + .itlb_access_i(itlb_lu_access), + .itlb_hit_i (itlb_lu_hit), + .itlb_vaddr_i (icache_areq_i.fetch_vaddr), + + .dtlb_access_i(dtlb_lu_access), + .dtlb_hit_i (dtlb_lu_hit), + .dtlb_vaddr_i (lsu_vaddr_i), + .hlvx_inst_i (hlvx_inst_i), + + .req_port_i (req_port_i), + .req_port_o (req_port_o), + .pmpcfg_i, + .pmpaddr_i, + .bad_gpaddr_o(ptw_bad_gpaddr), + .* + ); + + // ila_1 i_ila_1 ( + // .clk(clk_i), // input wire clk + // .probe0({req_port_o.address_tag, req_port_o.address_index}), + // .probe1(req_port_o.data_req), // input wire [63:0] probe1 + // .probe2(req_port_i.data_gnt), // input wire [0:0] probe2 + // .probe3(req_port_i.data_rdata), // input wire [0:0] probe3 + // .probe4(req_port_i.data_rvalid), // input wire [0:0] probe4 + // .probe5(ptw_error), // input wire [1:0] probe5 + // .probe6(update_vaddr), // input wire [0:0] probe6 + // .probe7(update_ptw_itlb.valid), // input wire [0:0] probe7 + // .probe8(update_ptw_dtlb.valid), // input wire [0:0] probe8 + // .probe9(dtlb_lu_access), // input wire [0:0] probe9 + // .probe10(lsu_vaddr_i), // input wire [0:0] probe10 + // .probe11(dtlb_lu_hit), // input wire [0:0] probe11 + // .probe12(itlb_lu_access), // input wire [0:0] probe12 + // .probe13(icache_areq_i.fetch_vaddr), // input wire [0:0] probe13 + // .probe14(itlb_lu_hit) // input wire [0:0] probe13 + // ); + + //----------------------- + // Instruction Interface + //----------------------- + logic match_any_execute_region; + logic pmp_instr_allow; + // The instruction interface is a simple request response interface + always_comb begin : instr_interface + // MMU disabled: just pass through + icache_areq_o.fetch_valid = icache_areq_i.fetch_req; + icache_areq_o.fetch_paddr = icache_areq_i.fetch_vaddr[CVA6Cfg.PLEN-1:0]; // play through in case we disabled address translation + // two potential exception sources: + // 1. HPTW threw an exception -> signal with a page fault exception + // 2. We got an access error because of insufficient permissions -> throw an access exception + icache_areq_o.fetch_exception = '0; + // Check whether we are allowed to access this memory region from a fetch perspective + iaccess_err = icache_areq_i.fetch_req && enable_translation_i && (((priv_lvl_i == riscv::PRIV_LVL_U) && ~itlb_content.u) + || ((priv_lvl_i == riscv::PRIV_LVL_S) && itlb_content.u)); + + i_g_st_access_err = icache_areq_i.fetch_req && enable_g_translation_i && !itlb_g_content.u; + // MMU enabled: address from TLB, request delayed until hit. Error when TLB + // hit and no access right or TLB hit and translated address not valid (e.g. + // AXI decode error), or when PTW performs walk due to ITLB miss and raises + // an error. + if ((enable_translation_i || enable_g_translation_i)) begin + // we work with SV39 or SV32, so if VM is enabled, check that all bits [CVA6Cfg.VLEN-1:CVA6Cfg.SV-1] are equal + if (icache_areq_i.fetch_req && !((&icache_areq_i.fetch_vaddr[CVA6Cfg.VLEN-1:CVA6Cfg.SV-1]) == 1'b1 || (|icache_areq_i.fetch_vaddr[CVA6Cfg.VLEN-1:CVA6Cfg.SV-1]) == 1'b0)) begin + icache_areq_o.fetch_exception = { + riscv::INSTR_ACCESS_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{1'b0}}, icache_areq_i.fetch_vaddr}, + {CVA6Cfg.GPLEN{1'b0}}, + {{32{1'b0}}}, + v_i, + 1'b1 + }; + end + + icache_areq_o.fetch_valid = 1'b0; + + // 4K page + icache_areq_o.fetch_paddr = { + enable_g_translation_i ? itlb_g_content.ppn : itlb_content.ppn, + icache_areq_i.fetch_vaddr[11:0] + }; + // Mega page + if (itlb_is_2M) begin + icache_areq_o.fetch_paddr[20:12] = icache_areq_i.fetch_vaddr[20:12]; + end + // Giga page + if (itlb_is_1G) begin + icache_areq_o.fetch_paddr[29:12] = icache_areq_i.fetch_vaddr[29:12]; + end + // --------- + // ITLB Hit + // -------- + // if we hit the ITLB output the request signal immediately + if (itlb_lu_hit) begin + icache_areq_o.fetch_valid = icache_areq_i.fetch_req; + if (i_g_st_access_err) begin + icache_areq_o.fetch_exception = { + riscv::INSTR_GUEST_PAGE_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{1'b0}}, icache_areq_i.fetch_vaddr}, + itlb_gpaddr[CVA6Cfg.GPLEN-1:0], + {{32{1'b0}}}, + v_i, + 1'b1 + }; + // we got an access error + end else if (iaccess_err) begin + // throw a page fault + icache_areq_o.fetch_exception = { + riscv::INSTR_PAGE_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{1'b0}}, icache_areq_i.fetch_vaddr}, + {CVA6Cfg.GPLEN{1'b0}}, + {{32{1'b0}}}, + v_i, + 1'b1 + }; + end else if (!pmp_instr_allow) begin + icache_areq_o.fetch_exception = { + riscv::INSTR_ACCESS_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{1'b0}}, icache_areq_i.fetch_vaddr}, + {CVA6Cfg.GPLEN{1'b0}}, + {{32{1'b0}}}, + v_i, + 1'b1 + }; + end + end else + // --------- + // ITLB Miss + // --------- + // watch out for exceptions happening during walking the page table + if (ptw_active && walking_instr) begin + icache_areq_o.fetch_valid = ptw_error | ptw_access_exception; + if (ptw_error) begin + if (ptw_error_at_g_st) begin + icache_areq_o.fetch_exception = { + riscv::INSTR_GUEST_PAGE_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{1'b0}}, update_vaddr}, + ptw_bad_gpaddr, + (ptw_err_at_g_int_st ? (CVA6Cfg.IS_XLEN64 ? riscv::READ_64_PSEUDOINSTRUCTION : riscv::READ_32_PSEUDOINSTRUCTION) : {{32{1'b0}}}), + v_i, + 1'b1 + }; + end else begin + icache_areq_o.fetch_exception = { + riscv::INSTR_PAGE_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{1'b0}}, update_vaddr}, + {CVA6Cfg.GPLEN{1'b0}}, + {{32{1'b0}}}, + v_i, + 1'b1 + }; + end + end // TODO(moschn,zarubaf): What should the value of tval be in this case? + else + icache_areq_o.fetch_exception = { + riscv::INSTR_ACCESS_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{1'b0}}, update_vaddr}, + {CVA6Cfg.GPLEN{1'b0}}, + {32{1'b0}}, + v_i, + 1'b1 + }; + end + end + // if it didn't match any execute region throw an `Instruction Access Fault` + // or: if we are not translating, check PMPs immediately on the paddr + if ((!match_any_execute_region && !ptw_error) || (!(enable_translation_i || enable_g_translation_i) && !pmp_instr_allow)) begin + icache_areq_o.fetch_exception = { + riscv::INSTR_ACCESS_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.PLEN{1'b0}}, icache_areq_o.fetch_paddr}, + {CVA6Cfg.GPLEN{1'b0}}, + {{32{1'b0}}}, + v_i, + 1'b1 + }; + end + end + + // check for execute flag on memory + assign match_any_execute_region = config_pkg::is_inside_execute_regions( + CVA6Cfg, {{64 - CVA6Cfg.PLEN{1'b0}}, icache_areq_o.fetch_paddr} + ); + + // Instruction fetch + pmp #( + .CVA6Cfg (CVA6Cfg), + .PLEN (CVA6Cfg.PLEN), + .PMP_LEN (CVA6Cfg.PLEN - 2), + .NR_ENTRIES(CVA6Cfg.NrPMPEntries) + ) i_pmp_if ( + .addr_i (icache_areq_o.fetch_paddr), + .priv_lvl_i, + // we will always execute on the instruction fetch port + .access_type_i(riscv::ACCESS_EXEC), + // Configuration + .conf_addr_i (pmpaddr_i), + .conf_i (pmpcfg_i), + .allow_o (pmp_instr_allow) + ); + + //----------------------- + // Data Interface + //----------------------- + logic [CVA6Cfg.VLEN-1:0] lsu_vaddr_n, lsu_vaddr_q; + logic [CVA6Cfg.GPLEN-1:0] lsu_gpaddr_n, lsu_gpaddr_q; + logic [31:0] lsu_tinst_n, lsu_tinst_q; + logic hs_ld_st_inst_n, hs_ld_st_inst_q; + riscv::pte_t dtlb_pte_n, dtlb_pte_q; + riscv::pte_t dtlb_gpte_n, dtlb_gpte_q; + exception_t misaligned_ex_n, misaligned_ex_q; + logic lsu_req_n, lsu_req_q; + logic lsu_is_store_n, lsu_is_store_q; + logic dtlb_hit_n, dtlb_hit_q; + logic dtlb_is_2M_n, dtlb_is_2M_q; + logic dtlb_is_1G_n, dtlb_is_1G_q; + + // check if we need to do translation or if we are always ready (e.g.: we are not translating anything) + assign lsu_dtlb_hit_o = (en_ld_st_translation_i || en_ld_st_g_translation_i) ? dtlb_lu_hit : 1'b1; + + // Wires to PMP checks + riscv::pmp_access_t pmp_access_type; + logic pmp_data_allow; + localparam PPNWMin = (CVA6Cfg.PPNW - 1 > 29) ? 29 : CVA6Cfg.PPNW - 1; + // The data interface is simpler and only consists of a request/response interface + always_comb begin : data_interface + // save request and DTLB response + lsu_vaddr_n = lsu_vaddr_i; + lsu_tinst_n = lsu_tinst_i; + lsu_gpaddr_n = dtlb_gpaddr; + lsu_req_n = lsu_req_i; + hs_ld_st_inst_n = hs_ld_st_inst_i; + misaligned_ex_n = misaligned_ex_i; + dtlb_pte_n = dtlb_content; + dtlb_gpte_n = dtlb_g_content; + dtlb_hit_n = dtlb_lu_hit; + lsu_is_store_n = lsu_is_store_i; + dtlb_is_2M_n = dtlb_is_2M; + dtlb_is_1G_n = dtlb_is_1G; + + lsu_paddr_o = lsu_vaddr_q[CVA6Cfg.PLEN-1:0]; + lsu_dtlb_ppn_o = lsu_vaddr_n[CVA6Cfg.PLEN-1:12]; + lsu_valid_o = lsu_req_q; + lsu_exception_o = misaligned_ex_q; + csr_hs_ld_st_inst_o = hs_ld_st_inst_i || hs_ld_st_inst_q; + pmp_access_type = lsu_is_store_q ? riscv::ACCESS_WRITE : riscv::ACCESS_READ; + + // mute misaligned exceptions if there is no request otherwise they will throw accidental exceptions + misaligned_ex_n.valid = misaligned_ex_i.valid & lsu_req_i; + + // Check if the User flag is set, then we may only access it in supervisor mode + // if SUM is enabled + daccess_err = en_ld_st_translation_i && + ((ld_st_priv_lvl_i == riscv::PRIV_LVL_S && (ld_st_v_i ? !vs_sum_i : !sum_i ) && dtlb_pte_q.u) || // SUM is not set and we are trying to access a user page in supervisor mode + (ld_st_priv_lvl_i == riscv::PRIV_LVL_U && !dtlb_pte_q.u)); + d_g_st_access_err = en_ld_st_g_translation_i && !dtlb_gpte_q.u; + // translation is enabled and no misaligned exception occurred + if ((en_ld_st_translation_i || en_ld_st_g_translation_i) && !misaligned_ex_q.valid) begin + lsu_valid_o = 1'b0; + // 4K page + lsu_paddr_o = { + (en_ld_st_g_translation_i) ? dtlb_gpte_q.ppn : dtlb_pte_q.ppn, lsu_vaddr_q[11:0] + }; + lsu_dtlb_ppn_o = (en_ld_st_g_translation_i) ? dtlb_g_content.ppn : dtlb_content.ppn; + // Mega page + if (dtlb_is_2M_q) begin + lsu_paddr_o[20:12] = lsu_vaddr_q[20:12]; + lsu_dtlb_ppn_o[20:12] = lsu_vaddr_n[20:12]; + end + // Giga page + if (dtlb_is_1G_q) begin + lsu_paddr_o[PPNWMin:12] = lsu_vaddr_q[PPNWMin:12]; + lsu_dtlb_ppn_o[PPNWMin:12] = lsu_vaddr_n[PPNWMin:12]; + end + // --------- + // DTLB Hit + // -------- + if (dtlb_hit_q && lsu_req_q) begin + lsu_valid_o = 1'b1; + // exception priority: + // PAGE_FAULTS have higher priority than ACCESS_FAULTS + // virtual memory based exceptions are PAGE_FAULTS + // physical memory based exceptions are ACCESS_FAULTS (PMA/PMP) + + // this is a store + if (lsu_is_store_q) begin + // check if the page is write-able and we are not violating privileges + // also check if the dirty flag is set + if(en_ld_st_g_translation_i && (!dtlb_gpte_q.w || d_g_st_access_err || !dtlb_gpte_q.d)) begin + lsu_exception_o = { + riscv::STORE_GUEST_PAGE_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{lsu_vaddr_q[CVA6Cfg.VLEN-1]}}, lsu_vaddr_q}, + lsu_gpaddr_q, + {32{1'b0}}, + ld_st_v_i, + 1'b1 + }; + end else if (en_ld_st_translation_i && (!dtlb_pte_q.w || daccess_err || !dtlb_pte_q.d)) begin + lsu_exception_o = { + riscv::STORE_PAGE_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{lsu_vaddr_q[CVA6Cfg.VLEN-1]}}, lsu_vaddr_q}, + {CVA6Cfg.GPLEN{1'b0}}, + lsu_tinst_q, + ld_st_v_i, + 1'b1 + }; + // Check if any PMPs are violated + end else if (!pmp_data_allow) begin + lsu_exception_o = { + riscv::ST_ACCESS_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.PLEN{1'b0}}, lsu_paddr_o}, + {CVA6Cfg.GPLEN{1'b0}}, + lsu_tinst_q, + ld_st_v_i, + 1'b1 + }; + end + + // this is a load + end else begin + if (d_g_st_access_err) begin + lsu_exception_o = { + riscv::LOAD_GUEST_PAGE_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{lsu_vaddr_q[CVA6Cfg.VLEN-1]}}, lsu_vaddr_q}, + lsu_gpaddr_q, + {{32{1'b0}}}, + ld_st_v_i, + 1'b1 + }; + // check for sufficient access privileges - throw a page fault if necessary + end else if (daccess_err) begin + lsu_exception_o = { + riscv::LOAD_PAGE_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{lsu_vaddr_q[CVA6Cfg.VLEN-1]}}, lsu_vaddr_q}, + {CVA6Cfg.GPLEN{1'b0}}, + lsu_tinst_q, + ld_st_v_i, + 1'b1 + }; + // Check if any PMPs are violated + end else if (!pmp_data_allow) begin + lsu_exception_o = { + riscv::LD_ACCESS_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{lsu_vaddr_q[CVA6Cfg.VLEN-1]}}, lsu_vaddr_q}, + {CVA6Cfg.GPLEN{1'b0}}, + lsu_tinst_q, + ld_st_v_i, + 1'b1 + }; + end + end + end else + + // --------- + // DTLB Miss + // --------- + // watch out for exceptions + if (ptw_active && !walking_instr) begin + // page table walker threw an exception + if (ptw_error) begin + // an error makes the translation valid + lsu_valid_o = 1'b1; + // the page table walker can only throw page faults + if (lsu_is_store_q) begin + if (ptw_error_at_g_st) begin + lsu_exception_o = { + riscv::STORE_GUEST_PAGE_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{lsu_vaddr_q[CVA6Cfg.VLEN-1]}}, update_vaddr}, + ptw_bad_gpaddr, + (ptw_err_at_g_int_st ? (CVA6Cfg.IS_XLEN64 ? riscv::READ_64_PSEUDOINSTRUCTION : riscv::READ_32_PSEUDOINSTRUCTION) : {{32{1'b0}}}), + ld_st_v_i, + 1'b1 + }; + end else begin + lsu_exception_o = { + riscv::STORE_PAGE_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{lsu_vaddr_q[CVA6Cfg.VLEN-1]}}, update_vaddr}, + {CVA6Cfg.GPLEN{1'b0}}, + lsu_tinst_q, + ld_st_v_i, + 1'b1 + }; + end + end else begin + if (ptw_error_at_g_st) begin + lsu_exception_o = { + riscv::LOAD_GUEST_PAGE_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{lsu_vaddr_q[CVA6Cfg.VLEN-1]}}, update_vaddr}, + ptw_bad_gpaddr, + (ptw_err_at_g_int_st ? (CVA6Cfg.IS_XLEN64 ? riscv::READ_64_PSEUDOINSTRUCTION : riscv::READ_32_PSEUDOINSTRUCTION) : {{32{1'b0}}}), + ld_st_v_i, + 1'b1 + }; + end else begin + lsu_exception_o = { + riscv::LOAD_PAGE_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{lsu_vaddr_q[CVA6Cfg.VLEN-1]}}, update_vaddr}, + {CVA6Cfg.GPLEN{1'b0}}, + lsu_tinst_q, + ld_st_v_i, + 1'b1 + }; + end + end + end + + if (ptw_access_exception) begin + // an error makes the translation valid + lsu_valid_o = 1'b1; + // the page table walker can only throw page faults + lsu_exception_o = { + riscv::LD_ACCESS_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{lsu_vaddr_q[CVA6Cfg.VLEN-1]}}, update_vaddr}, + {CVA6Cfg.GPLEN{1'b0}}, + lsu_tinst_q, + ld_st_v_i, + 1'b1 + }; + end + end + end // If translation is not enabled, check the paddr immediately against PMPs + else if (lsu_req_q && !misaligned_ex_q.valid && !pmp_data_allow) begin + if (lsu_is_store_q) begin + lsu_exception_o = { + riscv::ST_ACCESS_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{lsu_vaddr_q[CVA6Cfg.VLEN-1]}}, update_vaddr}, + {CVA6Cfg.GPLEN{1'b0}}, + lsu_tinst_q, + ld_st_v_i, + 1'b1 + }; + end else begin + lsu_exception_o = { + riscv::LD_ACCESS_FAULT, + {{CVA6Cfg.XLEN - CVA6Cfg.VLEN{lsu_vaddr_q[CVA6Cfg.VLEN-1]}}, update_vaddr}, + {CVA6Cfg.GPLEN{1'b0}}, + lsu_tinst_q, + ld_st_v_i, + 1'b1 + }; + end + end + end + + // Load/store PMP check + pmp #( + .CVA6Cfg (CVA6Cfg), + .PLEN (CVA6Cfg.PLEN), + .PMP_LEN (CVA6Cfg.PLEN - 2), + .NR_ENTRIES(CVA6Cfg.NrPMPEntries) + ) i_pmp_data ( + .addr_i (lsu_paddr_o), + .priv_lvl_i (ld_st_priv_lvl_i), + .access_type_i(pmp_access_type), + // Configuration + .conf_addr_i (pmpaddr_i), + .conf_i (pmpcfg_i), + .allow_o (pmp_data_allow) + ); + + // ---------- + // Registers + // ---------- + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + lsu_vaddr_q <= '0; + lsu_gpaddr_q <= '0; + lsu_tinst_q <= '0; + hs_ld_st_inst_q <= '0; + lsu_req_q <= '0; + misaligned_ex_q <= '0; + dtlb_pte_q <= '0; + dtlb_gpte_q <= '0; + dtlb_hit_q <= '0; + lsu_is_store_q <= '0; + dtlb_is_2M_q <= '0; + dtlb_is_1G_q <= '0; + end else begin + lsu_vaddr_q <= lsu_vaddr_n; + lsu_gpaddr_q <= lsu_gpaddr_n; + lsu_tinst_q <= lsu_tinst_n; + hs_ld_st_inst_q <= hs_ld_st_inst_n; + lsu_req_q <= lsu_req_n; + misaligned_ex_q <= misaligned_ex_n; + dtlb_pte_q <= dtlb_pte_n; + dtlb_gpte_q <= dtlb_gpte_n; + dtlb_hit_q <= dtlb_hit_n; + lsu_is_store_q <= lsu_is_store_n; + dtlb_is_2M_q <= dtlb_is_2M_n; + dtlb_is_1G_q <= dtlb_is_1G_n; + end + end +endmodule diff --git a/core/mmu_sv39x4/cva6_ptw_sv39x4.sv b/core/mmu_sv39x4/cva6_ptw_sv39x4.sv new file mode 100644 index 0000000000..d73fb8cf7c --- /dev/null +++ b/core/mmu_sv39x4/cva6_ptw_sv39x4.sv @@ -0,0 +1,641 @@ +// Copyright (c) 2022 Bruno Sá and Zero-Day Labs. +// Copyright and related rights are licensed under the Solderpad Hardware +// License, Version 0.51 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law +// or agreed to in writing, software, hardware and materials distributed under +// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// Author: Bruno Sá +// Date: 14/08/2022 +// Acknowledges: Technology Innovation Institute (TII) +// +// Description: Hardware-PTW (Page-Table-Walker) for MMU Sv39x4. +// This module is an adaptation of the Sv39 PTW developed +// by Florian Zaruba and David Schaffenrath to the Sv39x4 standard. +// + +/* verilator lint_off WIDTH */ + +module cva6_ptw_sv39x4 + import ariane_pkg::*; +#( + parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, + parameter type dcache_req_i_t = logic, + parameter type dcache_req_o_t = logic, + parameter type tlb_update_t = logic +) ( + input logic clk_i, // Clock + input logic rst_ni, // Asynchronous reset active low + input logic flush_i, // flush everything, we need to do this because + // actually everything we do is speculative at this stage + // e.g.: there could be a CSR instruction that changes everything + output logic ptw_active_o, + output logic walking_instr_o, // set when walking for TLB + output logic ptw_error_o, // set when an error occurred + output logic ptw_error_at_g_st_o, // set when an error occurred at the G-Stage + output logic ptw_err_at_g_int_st_o, // set when an error occurred at the G-Stage during S-Stage translation + output logic ptw_access_exception_o, // set when an PMP access exception occured + input logic enable_translation_i, // CSRs indicate to enable SV39 VS-Stage translation + input logic enable_g_translation_i, // CSRs indicate to enable SV39 G-Stage translation + input logic en_ld_st_translation_i, // enable virtual memory translation for load/stores + input logic en_ld_st_g_translation_i, // enable G-Stage translation for load/stores + input logic v_i, // current virtualization mode bit + input logic ld_st_v_i, // load/store virtualization mode bit + input logic hlvx_inst_i, // is a HLVX load/store instruction + + input logic lsu_is_store_i, // this translation was triggered by a store + // PTW memory interface + input dcache_req_o_t req_port_i, + output dcache_req_i_t req_port_o, + + + // to TLBs, update logic + output tlb_update_t itlb_update_o, + output tlb_update_t dtlb_update_o, + + output logic [CVA6Cfg.VLEN-1:0] update_vaddr_o, + + input logic [CVA6Cfg.ASID_WIDTH-1:0] asid_i, + input logic [CVA6Cfg.ASID_WIDTH-1:0] vs_asid_i, + input logic [CVA6Cfg.VMID_WIDTH-1:0] vmid_i, + // from TLBs + // did we miss? + input logic itlb_access_i, + input logic itlb_hit_i, + input logic [ CVA6Cfg.VLEN-1:0] itlb_vaddr_i, + + input logic dtlb_access_i, + input logic dtlb_hit_i, + input logic [CVA6Cfg.VLEN-1:0] dtlb_vaddr_i, + // from CSR file + input logic [CVA6Cfg.PPNW-1:0] satp_ppn_i, // ppn from satp + input logic [CVA6Cfg.PPNW-1:0] vsatp_ppn_i, // ppn from satp + input logic [CVA6Cfg.PPNW-1:0] hgatp_ppn_i, // ppn from hgatp + input logic mxr_i, + input logic vmxr_i, + // Performance counters + output logic itlb_miss_o, + output logic dtlb_miss_o, + // PMP + + input riscv::pmpcfg_t [15:0] pmpcfg_i, + input logic [15:0][CVA6Cfg.PLEN-3:0] pmpaddr_i, + output logic [CVA6Cfg.GPLEN-1:0] bad_gpaddr_o + +); + + // input registers + logic data_rvalid_q; + logic [63:0] data_rdata_q; + + riscv::pte_t pte; + // register to perform context switch between stages + riscv::pte_t gpte_q, gpte_d; + assign pte = riscv::pte_t'(data_rdata_q); + + enum logic [2:0] { + IDLE, + WAIT_GRANT, + PTE_LOOKUP, + WAIT_RVALID, + PROPAGATE_ERROR, + PROPAGATE_ACCESS_ERROR + } + state_q, state_d; + + // SV39 defines three levels of page tables + enum logic [1:0] { + LVL1, + LVL2, + LVL3 + } + ptw_lvl_q, ptw_lvl_n, gptw_lvl_n, gptw_lvl_q; + + // define 3 PTW stages + // S_STAGE -> S/VS-stage normal translation controlled by the satp/vsatp CSRs + // G_INTERMED_STAGE -> Converts the S/VS-stage non-leaf GPA pointers to HPA (controlled by hgatp) + // G_FINAL_STAGE -> Converts the S/VS-stage final GPA to HPA (controlled by hgatp) + enum logic [1:0] { + S_STAGE, + G_INTERMED_STAGE, + G_FINAL_STAGE + } + ptw_stage_q, ptw_stage_d; + + // is this an instruction page table walk? + logic is_instr_ptw_q, is_instr_ptw_n; + logic global_mapping_q, global_mapping_n; + // latched tag signal + logic tag_valid_n, tag_valid_q; + // register the ASID + logic [CVA6Cfg.ASID_WIDTH-1:0] tlb_update_asid_q, tlb_update_asid_n; + // register the VMID + logic [CVA6Cfg.VMID_WIDTH-1:0] tlb_update_vmid_q, tlb_update_vmid_n; + // register the VPN we need to walk, SV39 defines a 39 bit virtual address + logic [CVA6Cfg.VLEN-1:0] vaddr_q, vaddr_n; + // register the VPN we need to walk, SV39x4 defines a 41 bit virtual address for the G-Stage + logic [CVA6Cfg.GPLEN-1:0] gpaddr_q, gpaddr_n; + // 4 byte aligned physical pointer + logic [CVA6Cfg.PLEN-1:0] ptw_pptr_q, ptw_pptr_n; + logic [CVA6Cfg.PLEN-1:0] gptw_pptr_q, gptw_pptr_n; + + // Assignments + assign update_vaddr_o = vaddr_q; + + assign ptw_active_o = (state_q != IDLE); + assign walking_instr_o = is_instr_ptw_q; + // directly output the correct physical address + assign req_port_o.address_index = ptw_pptr_q[CVA6Cfg.DCACHE_INDEX_WIDTH-1:0]; + assign req_port_o.address_tag = ptw_pptr_q[CVA6Cfg.DCACHE_INDEX_WIDTH+CVA6Cfg.DCACHE_TAG_WIDTH-1:CVA6Cfg.DCACHE_INDEX_WIDTH]; + // we are never going to kill this request + assign req_port_o.kill_req = '0; + // we are never going to write with the HPTW + assign req_port_o.data_wdata = 64'b0; + + // ----------- + // TLB Update + // ----------- + always_comb begin : tlb_update + + itlb_update_o.vpn = {{41 - CVA6Cfg.SVX{1'b0}}, vaddr_q[CVA6Cfg.SVX-1:12]}; + dtlb_update_o.vpn = {{41 - CVA6Cfg.SVX{1'b0}}, vaddr_q[CVA6Cfg.SVX-1:12]}; + // update the correct page table level + if (enable_g_translation_i && enable_translation_i) begin + itlb_update_o.is_s_2M = (gptw_lvl_q == LVL2); + itlb_update_o.is_s_1G = (gptw_lvl_q == LVL1); + itlb_update_o.is_g_2M = (ptw_lvl_q == LVL2); + itlb_update_o.is_g_1G = (ptw_lvl_q == LVL1); + end else if (enable_translation_i) begin + itlb_update_o.is_s_2M = (ptw_lvl_q == LVL2); + itlb_update_o.is_s_1G = (ptw_lvl_q == LVL1); + itlb_update_o.is_g_2M = 1'b0; + itlb_update_o.is_g_1G = 1'b0; + end else begin + itlb_update_o.is_s_2M = 1'b0; + itlb_update_o.is_s_1G = 1'b0; + itlb_update_o.is_g_2M = (ptw_lvl_q == LVL2); + itlb_update_o.is_g_1G = (ptw_lvl_q == LVL1); + end + + if (en_ld_st_g_translation_i && en_ld_st_translation_i) begin + dtlb_update_o.is_s_2M = (gptw_lvl_q == LVL2); + dtlb_update_o.is_s_1G = (gptw_lvl_q == LVL1); + dtlb_update_o.is_g_2M = (ptw_lvl_q == LVL2); + dtlb_update_o.is_g_1G = (ptw_lvl_q == LVL1); + end else if (en_ld_st_translation_i) begin + dtlb_update_o.is_s_2M = (ptw_lvl_q == LVL2); + dtlb_update_o.is_s_1G = (ptw_lvl_q == LVL1); + dtlb_update_o.is_g_2M = 1'b0; + dtlb_update_o.is_g_1G = 1'b0; + end else begin + dtlb_update_o.is_s_2M = 1'b0; + dtlb_update_o.is_s_1G = 1'b0; + dtlb_update_o.is_g_2M = (ptw_lvl_q == LVL2); + dtlb_update_o.is_g_1G = (ptw_lvl_q == LVL1); + end + // output the correct ASID + itlb_update_o.asid = tlb_update_asid_q; + dtlb_update_o.asid = tlb_update_asid_q; + // output the correct VMID + itlb_update_o.vmid = tlb_update_vmid_q; + dtlb_update_o.vmid = tlb_update_vmid_q; + // set the global mapping bit + if (enable_g_translation_i) begin + itlb_update_o.content = gpte_q | (global_mapping_q << 5); + itlb_update_o.g_content = pte; + end else begin + itlb_update_o.content = pte | (global_mapping_q << 5); + itlb_update_o.g_content = '0; + end + if (en_ld_st_g_translation_i) begin + dtlb_update_o.content = gpte_q | (global_mapping_q << 5); + dtlb_update_o.g_content = pte; + end else begin + dtlb_update_o.content = pte | (global_mapping_q << 5); + dtlb_update_o.g_content = '0; + end + end + + assign req_port_o.tag_valid = tag_valid_q; + + logic allow_access; + + assign bad_gpaddr_o = ptw_error_at_g_st_o ? ((ptw_stage_q == G_INTERMED_STAGE) ? gptw_pptr_q[CVA6Cfg.GPLEN:0] : gpaddr_q) : 'b0; + + pmp #( + .CVA6Cfg (CVA6Cfg), + .PLEN (CVA6Cfg.PLEN), + .PMP_LEN (CVA6Cfg.PLEN - 2), + .NR_ENTRIES(CVA6Cfg.NrPMPEntries) + ) i_pmp_ptw ( + .addr_i (ptw_pptr_q), + // PTW access are always checked as if in S-Mode... + .priv_lvl_i (riscv::PRIV_LVL_S), + // ...and they are always loads + .access_type_i(riscv::ACCESS_READ), + // Configuration + .conf_addr_i (pmpaddr_i), + .conf_i (pmpcfg_i), + .allow_o (allow_access) + ); + + //------------------- + // Page table walker + //------------------- + // A virtual address va is translated into a physical address pa as follows: + // 1. Let a be sptbr.ppn × PAGESIZE, and let i = LEVELS-1. (For Sv39, + // PAGESIZE=2^12 and LEVELS=3.) + // 2. Let pte be the value of the PTE at address a+va.vpn[i]×PTESIZE. (For + // Sv32, PTESIZE=4.) + // 3. If pte.v = 0, or if pte.r = 0 and pte.w = 1, or if any bits or encodings + // that are reserved for future standard use are set within pte, stop and raise + // a page-fault exception corresponding to the original access type. + // 4. Otherwise, the PTE is valid. If pte.r = 1 or pte.x = 1, go to step 5. + // Otherwise, this PTE is a pointer to the next level of the page table. + // Let i=i-1. If i < 0, stop and raise an access exception. Otherwise, let + // a = pte.ppn × PAGESIZE and go to step 2. + // 5. A leaf PTE has been found. Determine if the requested memory access + // is allowed by the pte.r, pte.w, and pte.x bits. If not, stop and + // raise an access exception. Otherwise, the translation is successful. + // Set pte.a to 1, and, if the memory access is a store, set pte.d to 1. + // The translated physical address is given as follows: + // - pa.pgoff = va.pgoff. + // - If i > 0, then this is a superpage translation and + // pa.ppn[i-1:0] = va.vpn[i-1:0]. + // - pa.ppn[LEVELS-1:i] = pte.ppn[LEVELS-1:i]. + always_comb begin : ptw + automatic logic [ CVA6Cfg.PLEN-1:0] pptr; + automatic logic [CVA6Cfg.GPLEN-1:0] gpaddr; + // default assignments + // PTW memory interface + tag_valid_n = 1'b0; + req_port_o.data_req = 1'b0; + req_port_o.data_be = 8'hFF; + req_port_o.data_size = 2'b11; + req_port_o.data_we = 1'b0; + ptw_error_o = 1'b0; + ptw_error_at_g_st_o = 1'b0; + ptw_err_at_g_int_st_o = 1'b0; + ptw_access_exception_o = 1'b0; + itlb_update_o.valid = 1'b0; + dtlb_update_o.valid = 1'b0; + is_instr_ptw_n = is_instr_ptw_q; + ptw_lvl_n = ptw_lvl_q; + gptw_lvl_n = gptw_lvl_q; + ptw_pptr_n = ptw_pptr_q; + gptw_pptr_n = gptw_pptr_q; + state_d = state_q; + ptw_stage_d = ptw_stage_q; + gpte_d = gpte_q; + global_mapping_n = global_mapping_q; + // input registers + tlb_update_asid_n = tlb_update_asid_q; + tlb_update_vmid_n = tlb_update_vmid_q; + vaddr_n = vaddr_q; + gpaddr_n = gpaddr_q; + pptr = ptw_pptr_q; + gpaddr = gpaddr_q; + + itlb_miss_o = 1'b0; + dtlb_miss_o = 1'b0; + + case (state_q) + + IDLE: begin + // by default we start with the top-most page table + ptw_lvl_n = LVL1; + gptw_lvl_n = LVL1; + global_mapping_n = 1'b0; + is_instr_ptw_n = 1'b0; + gpaddr_n = '0; + gpte_d = '0; + // if we got an ITLB miss + if ((enable_translation_i | enable_g_translation_i) & itlb_access_i & ~itlb_hit_i & ~dtlb_access_i) begin + if (enable_translation_i && enable_g_translation_i) begin + ptw_stage_d = G_INTERMED_STAGE; + pptr = {vsatp_ppn_i, itlb_vaddr_i[CVA6Cfg.SV-1:30], 3'b0}; + gptw_pptr_n = pptr; + ptw_pptr_n = {hgatp_ppn_i[CVA6Cfg.PPNW-1:2], pptr[CVA6Cfg.SVX-1:30], 3'b0}; + end else if (!enable_translation_i && enable_g_translation_i) begin + ptw_stage_d = G_FINAL_STAGE; + gpaddr_n = itlb_vaddr_i[CVA6Cfg.SVX-1:0]; + ptw_pptr_n = {hgatp_ppn_i[CVA6Cfg.PPNW-1:2], itlb_vaddr_i[CVA6Cfg.SVX-1:30], 3'b0}; + end else begin + ptw_stage_d = S_STAGE; + if (v_i) ptw_pptr_n = {vsatp_ppn_i, itlb_vaddr_i[CVA6Cfg.SV-1:30], 3'b0}; + else ptw_pptr_n = {satp_ppn_i, itlb_vaddr_i[CVA6Cfg.SV-1:30], 3'b0}; + end + is_instr_ptw_n = 1'b1; + tlb_update_asid_n = v_i ? vs_asid_i : asid_i; + tlb_update_vmid_n = vmid_i; + vaddr_n = itlb_vaddr_i; + state_d = WAIT_GRANT; + itlb_miss_o = 1'b1; + // we got an DTLB miss + end else if ((en_ld_st_translation_i || en_ld_st_g_translation_i) & dtlb_access_i & ~dtlb_hit_i) begin + if (en_ld_st_translation_i && en_ld_st_g_translation_i) begin + ptw_stage_d = G_INTERMED_STAGE; + pptr = {vsatp_ppn_i, dtlb_vaddr_i[CVA6Cfg.SV-1:30], 3'b0}; + gptw_pptr_n = pptr; + ptw_pptr_n = {hgatp_ppn_i[CVA6Cfg.PPNW-1:2], pptr[CVA6Cfg.SVX-1:30], 3'b0}; + end else if (!en_ld_st_translation_i && en_ld_st_g_translation_i) begin + ptw_stage_d = G_FINAL_STAGE; + gpaddr_n = dtlb_vaddr_i[CVA6Cfg.SVX-1:0]; + ptw_pptr_n = {hgatp_ppn_i[CVA6Cfg.PPNW-1:2], dtlb_vaddr_i[CVA6Cfg.SVX-1:30], 3'b0}; + end else begin + ptw_stage_d = S_STAGE; + if (ld_st_v_i) ptw_pptr_n = {vsatp_ppn_i, dtlb_vaddr_i[CVA6Cfg.SV-1:30], 3'b0}; + else ptw_pptr_n = {satp_ppn_i, dtlb_vaddr_i[CVA6Cfg.SV-1:30], 3'b0}; + end + tlb_update_asid_n = ld_st_v_i ? vs_asid_i : asid_i; + tlb_update_vmid_n = vmid_i; + vaddr_n = dtlb_vaddr_i; + state_d = WAIT_GRANT; + dtlb_miss_o = 1'b1; + end + end + + WAIT_GRANT: begin + // send a request out + req_port_o.data_req = 1'b1; + // wait for the WAIT_GRANT + if (req_port_i.data_gnt) begin + // send the tag valid signal one cycle later + tag_valid_n = 1'b1; + state_d = PTE_LOOKUP; + end + end + + PTE_LOOKUP: begin + // we wait for the valid signal + if (data_rvalid_q) begin + + // check if the global mapping bit is set + if (pte.g && ptw_stage_q == S_STAGE) global_mapping_n = 1'b1; + + // ------------- + // Invalid PTE + // ------------- + // If pte.v = 0, or if pte.r = 0 and pte.w = 1, stop and raise a page-fault exception. + if (!pte.v || (!pte.r && pte.w) || (|pte.reserved)) state_d = PROPAGATE_ERROR; + // ----------- + // Valid PTE + // ----------- + else begin + state_d = IDLE; + // it is a valid PTE + // if pte.r = 1 or pte.x = 1 it is a valid PTE + if (pte.r || pte.x) begin + case (ptw_stage_q) + S_STAGE: begin + if ((is_instr_ptw_q && enable_g_translation_i) || (!is_instr_ptw_q && en_ld_st_g_translation_i)) begin + state_d = WAIT_GRANT; + ptw_stage_d = G_FINAL_STAGE; + gpte_d = pte; + gptw_lvl_n = ptw_lvl_q; + gpaddr = {pte.ppn[CVA6Cfg.GPPNW-1:0], vaddr_q[11:0]}; + if (ptw_lvl_q == LVL2) gpaddr[20:0] = vaddr_q[20:0]; + if (ptw_lvl_q == LVL1) gpaddr[29:0] = vaddr_q[29:0]; + gpaddr_n = gpaddr; + ptw_pptr_n = {hgatp_ppn_i[CVA6Cfg.PPNW-1:2], gpaddr[CVA6Cfg.SVX-1:30], 3'b0}; + ptw_lvl_n = LVL1; + end + end + G_INTERMED_STAGE: begin + state_d = WAIT_GRANT; + ptw_stage_d = S_STAGE; + ptw_lvl_n = gptw_lvl_q; + pptr = {pte.ppn[CVA6Cfg.GPPNW-1:0], gptw_pptr_q[11:0]}; + if (ptw_lvl_q == LVL2) pptr[20:0] = gptw_pptr_q[20:0]; + if (ptw_lvl_q == LVL1) pptr[29:0] = gptw_pptr_q[29:0]; + ptw_pptr_n = pptr; + end + default: ; + endcase + // Valid translation found (either 1G, 2M or 4K entry) + if (is_instr_ptw_q) begin + // ------------ + // Update ITLB + // ------------ + // If page is not executable, we can directly raise an error. This + // doesn't put a useless entry into the TLB. The same idea applies + // to the access flag since we let the access flag be managed by SW. + if (!pte.x || !pte.a) begin + state_d = PROPAGATE_ERROR; + ptw_stage_d = ptw_stage_q; + end else if ((ptw_stage_q == G_FINAL_STAGE) || !enable_g_translation_i) + itlb_update_o.valid = 1'b1; + + end else begin + // ------------ + // Update DTLB + // ------------ + // Check if the access flag has been set, otherwise throw a page-fault + // and let the software handle those bits. + // If page is not readable (there are no write-only pages) + // we can directly raise an error. This doesn't put a useless + // entry into the TLB. + if (pte.a && ((pte.r && !hlvx_inst_i) || (pte.x && (mxr_i || hlvx_inst_i || (ptw_stage_q == S_STAGE && vmxr_i && ld_st_v_i))))) begin + if ((ptw_stage_q == G_FINAL_STAGE) || !en_ld_st_g_translation_i) + dtlb_update_o.valid = 1'b1; + end else begin + state_d = PROPAGATE_ERROR; + ptw_stage_d = ptw_stage_q; + end + // Request is a store: perform some additional checks + // If the request was a store and the page is not write-able, raise an error + // the same applies if the dirty flag is not set + if (lsu_is_store_i && (!pte.w || !pte.d)) begin + dtlb_update_o.valid = 1'b0; + state_d = PROPAGATE_ERROR; + ptw_stage_d = ptw_stage_q; + end + end + // check if the ppn is correctly aligned: + // 6. If i > 0 and pa.ppn[i − 1 : 0] != 0, this is a misaligned superpage; stop and raise a page-fault + // exception. + if (ptw_lvl_q == LVL1 && pte.ppn[17:0] != '0) begin + state_d = PROPAGATE_ERROR; + ptw_stage_d = ptw_stage_q; + dtlb_update_o.valid = 1'b0; + itlb_update_o.valid = 1'b0; + end else if (ptw_lvl_q == LVL2 && pte.ppn[8:0] != '0) begin + state_d = PROPAGATE_ERROR; + ptw_stage_d = ptw_stage_q; + dtlb_update_o.valid = 1'b0; + itlb_update_o.valid = 1'b0; + end + // check if 63:41 are all zeros + if (((v_i && is_instr_ptw_q) || (ld_st_v_i && !is_instr_ptw_q)) && ptw_stage_q == S_STAGE && !((|pte.ppn[CVA6Cfg.PPNW-1:CVA6Cfg.GPPNW]) == 1'b0)) begin + state_d = PROPAGATE_ERROR; + ptw_stage_d = G_FINAL_STAGE; + end + // this is a pointer to the next TLB level + end else begin + // pointer to next level of page table + if (ptw_lvl_q == LVL1) begin + // we are in the second level now + ptw_lvl_n = LVL2; + case (ptw_stage_q) + S_STAGE: begin + if ((is_instr_ptw_q && enable_g_translation_i) || (!is_instr_ptw_q && en_ld_st_g_translation_i)) begin + ptw_stage_d = G_INTERMED_STAGE; + gpte_d = pte; + gptw_lvl_n = LVL2; + pptr = {pte.ppn, vaddr_q[29:21], 3'b0}; + gptw_pptr_n = pptr; + ptw_pptr_n = {hgatp_ppn_i[CVA6Cfg.PPNW-1:2], pptr[CVA6Cfg.SVX-1:30], 3'b0}; + ptw_lvl_n = LVL1; + end else begin + ptw_pptr_n = {pte.ppn, vaddr_q[29:21], 3'b0}; + end + end + G_INTERMED_STAGE: begin + ptw_pptr_n = {pte.ppn, gptw_pptr_q[29:21], 3'b0}; + end + G_FINAL_STAGE: begin + ptw_pptr_n = {pte.ppn, gpaddr_q[29:21], 3'b0}; + end + default: ; + endcase + end + + if (ptw_lvl_q == LVL2) begin + // here we received a pointer to the third level + ptw_lvl_n = LVL3; + unique case (ptw_stage_q) + S_STAGE: begin + if ((is_instr_ptw_q && enable_g_translation_i) || (!is_instr_ptw_q && en_ld_st_g_translation_i)) begin + ptw_stage_d = G_INTERMED_STAGE; + gpte_d = pte; + gptw_lvl_n = LVL3; + pptr = {pte.ppn, vaddr_q[20:12], 3'b0}; + gptw_pptr_n = pptr; + ptw_pptr_n = {hgatp_ppn_i[CVA6Cfg.PPNW-1:2], pptr[CVA6Cfg.SVX-1:30], 3'b0}; + ptw_lvl_n = LVL1; + end else begin + ptw_pptr_n = {pte.ppn, vaddr_q[20:12], 3'b0}; + end + end + G_INTERMED_STAGE: begin + ptw_pptr_n = {pte.ppn, gptw_pptr_q[20:12], 3'b0}; + end + G_FINAL_STAGE: begin + ptw_pptr_n = {pte.ppn, gpaddr_q[20:12], 3'b0}; + end + default: ; + endcase + end + + state_d = WAIT_GRANT; + // check if reserved bits are cleared for non-leaf entries + if (pte.a || pte.d || pte.u) begin + state_d = PROPAGATE_ERROR; + ptw_stage_d = ptw_stage_q; + end + if (ptw_lvl_q == LVL3) begin + // Should already be the last level page table => Error + ptw_lvl_n = LVL3; + state_d = PROPAGATE_ERROR; + ptw_stage_d = ptw_stage_q; + end + // check if 63:41 are all zeros + if (((v_i && is_instr_ptw_q) || (ld_st_v_i && !is_instr_ptw_q)) && ptw_stage_q == S_STAGE && !((|pte.ppn[CVA6Cfg.PPNW-1:CVA6Cfg.GPPNW]) == 1'b0)) begin + state_d = PROPAGATE_ERROR; + ptw_stage_d = ptw_stage_q; + end + end + end + + // Check if this access was actually allowed from a PMP perspective + if (!allow_access) begin + itlb_update_o.valid = 1'b0; + dtlb_update_o.valid = 1'b0; + // we have to return the failed address in bad_addr + ptw_pptr_n = ptw_pptr_q; + ptw_stage_d = ptw_stage_q; + state_d = PROPAGATE_ACCESS_ERROR; + end + end + // we've got a data WAIT_GRANT so tell the cache that the tag is valid + end + // Propagate error to MMU/LSU + PROPAGATE_ERROR: begin + state_d = IDLE; + ptw_error_o = 1'b1; + ptw_error_at_g_st_o = (ptw_stage_q != S_STAGE) ? 1'b1 : 1'b0; + ptw_err_at_g_int_st_o = (ptw_stage_q == G_INTERMED_STAGE) ? 1'b1 : 1'b0; + end + PROPAGATE_ACCESS_ERROR: begin + state_d = IDLE; + ptw_access_exception_o = 1'b1; + end + // wait for the rvalid before going back to IDLE + WAIT_RVALID: begin + if (data_rvalid_q) state_d = IDLE; + end + default: begin + state_d = IDLE; + end + endcase + + // ------- + // Flush + // ------- + // should we have flushed before we got an rvalid, wait for it until going back to IDLE + if (flush_i) begin + // on a flush check whether we are + // 1. in the PTE Lookup check whether we still need to wait for an rvalid + // 2. waiting for a grant, if so: wait for it + // if not, go back to idle + if (((state_q inside {PTE_LOOKUP, WAIT_RVALID}) && !data_rvalid_q) || + ((state_q == WAIT_GRANT) && req_port_i.data_gnt)) + state_d = WAIT_RVALID; + else state_d = IDLE; + end + end + + // sequential process + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + state_q <= IDLE; + ptw_stage_q <= S_STAGE; + is_instr_ptw_q <= 1'b0; + ptw_lvl_q <= LVL1; + gptw_lvl_q <= LVL1; + tag_valid_q <= 1'b0; + tlb_update_asid_q <= '0; + tlb_update_vmid_q <= '0; + vaddr_q <= '0; + gpaddr_q <= '0; + ptw_pptr_q <= '0; + gptw_pptr_q <= '0; + global_mapping_q <= 1'b0; + data_rdata_q <= '0; + gpte_q <= '0; + data_rvalid_q <= 1'b0; + end else begin + state_q <= state_d; + ptw_stage_q <= ptw_stage_d; + ptw_pptr_q <= ptw_pptr_n; + gptw_pptr_q <= gptw_pptr_n; + is_instr_ptw_q <= is_instr_ptw_n; + ptw_lvl_q <= ptw_lvl_n; + gptw_lvl_q <= gptw_lvl_n; + tag_valid_q <= tag_valid_n; + tlb_update_asid_q <= tlb_update_asid_n; + tlb_update_vmid_q <= tlb_update_vmid_n; + vaddr_q <= vaddr_n; + gpaddr_q <= gpaddr_n; + global_mapping_q <= global_mapping_n; + data_rdata_q <= req_port_i.data_rdata; + gpte_q <= gpte_d; + data_rvalid_q <= req_port_i.data_rvalid; + end + end + +endmodule +/* verilator lint_on WIDTH */ diff --git a/core/mmu_sv39x4/cva6_tlb_sv39x4.sv b/core/mmu_sv39x4/cva6_tlb_sv39x4.sv new file mode 100644 index 0000000000..69b1038905 --- /dev/null +++ b/core/mmu_sv39x4/cva6_tlb_sv39x4.sv @@ -0,0 +1,415 @@ +// Copyright (c) 2022 Bruno Sá and Zero-Day Labs. +// Copyright and related rights are licensed under the Solderpad Hardware +// License, Version 0.51 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law +// or agreed to in writing, software, hardware and materials distributed under +// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// Author: Bruno Sá +// Date: 14/08/2022 +// Acknowledges: Technology Innovation Institute (TII) +// +// Description: Translation Lookaside Buffer, Sv39x4 , fully set-associative +// This module is an adaptation of the Sv39 TLB developed +// by Florian Zaruba and David Schaffenrath to the Sv39x4 standard. + + +module cva6_tlb_sv39x4 + import ariane_pkg::*; +#( + parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, + parameter type tlb_update_t = logic, + parameter int unsigned TLB_ENTRIES = 4 +) ( + input logic clk_i, // Clock + input logic rst_ni, // Asynchronous reset active low + input logic flush_i, // Flush normal translations signal + input logic flush_vvma_i, // Flush vs stage signal + input logic flush_gvma_i, // Flush g stage signal + input logic s_st_enbl_i, // s-stage enabled + input logic g_st_enbl_i, // g-stage enabled + input logic v_i, // virtualization mode + // Update TLB + input tlb_update_t update_i, + // Lookup signals + input logic lu_access_i, + input logic [CVA6Cfg.ASID_WIDTH-1:0] lu_asid_i, + input logic [CVA6Cfg.VMID_WIDTH-1:0] lu_vmid_i, + input logic [CVA6Cfg.VLEN-1:0] lu_vaddr_i, + output logic [CVA6Cfg.GPLEN-1:0] lu_gpaddr_o, + output riscv::pte_t lu_content_o, + output riscv::pte_t lu_g_content_o, + input logic [CVA6Cfg.ASID_WIDTH-1:0] asid_to_be_flushed_i, + input logic [CVA6Cfg.VMID_WIDTH-1:0] vmid_to_be_flushed_i, + input logic [CVA6Cfg.VLEN-1:0] vaddr_to_be_flushed_i, + input logic [CVA6Cfg.GPLEN-1:0] gpaddr_to_be_flushed_i, + output logic lu_is_2M_o, + output logic lu_is_1G_o, + output logic lu_hit_o +); + localparam VPN2 = (CVA6Cfg.VLEN - 31 < 8) ? CVA6Cfg.VLEN - 31 : 8; + localparam GPPN2 = (CVA6Cfg.XLEN == 32) ? CVA6Cfg.VLEN - 33 : 10; + + // SV39 defines three levels of page tables + struct packed { + logic [CVA6Cfg.ASID_WIDTH-1:0] asid; + logic [CVA6Cfg.VMID_WIDTH-1:0] vmid; + logic [GPPN2:0] vpn2; + logic [8:0] vpn1; + logic [8:0] vpn0; + logic is_s_2M; + logic is_s_1G; + logic is_g_2M; + logic is_g_1G; + logic s_st_enbl; // s-stage translation + logic g_st_enbl; // g-stage translation + logic v; // virtualization mode + logic valid; + } [TLB_ENTRIES-1:0] + tags_q, tags_n; + + struct packed { + riscv::pte_t pte; + riscv::pte_t gpte; + } [TLB_ENTRIES-1:0] + content_q, content_n; + + logic [8:0] vpn0, vpn1; + logic [GPPN2:0] vpn2; + logic [TLB_ENTRIES-1:0] lu_hit; // to replacement logic + logic [TLB_ENTRIES-1:0] replace_en; // replace the following entry, set by replacement strategy + logic [TLB_ENTRIES-1:0] match_vmid; + logic [TLB_ENTRIES-1:0] match_asid; + logic [TLB_ENTRIES-1:0] is_1G; + logic [TLB_ENTRIES-1:0] is_2M; + logic [TLB_ENTRIES-1:0] match_stage; + riscv::pte_t g_content; + //------------- + // Translation + //------------- + always_comb begin : translation + automatic logic [GPPN2:0] mask_pn2; + mask_pn2 = s_st_enbl_i ? ((2 ** (VPN2 + 1)) - 1) : ((2 ** (GPPN2 + 1)) - 1); + vpn0 = lu_vaddr_i[20:12]; + vpn1 = lu_vaddr_i[29:21]; + vpn2 = lu_vaddr_i[30+GPPN2:30] & mask_pn2; + + // default assignment + lu_hit = '{default: 0}; + lu_hit_o = 1'b0; + lu_content_o = '{default: 0}; + lu_g_content_o = '{default: 0}; + lu_is_1G_o = 1'b0; + lu_is_2M_o = 1'b0; + match_asid = '{default: 0}; + match_vmid = '{default: 0}; + match_stage = '{default: 0}; + is_1G = '{default: 0}; + is_2M = '{default: 0}; + g_content = '{default: 0}; + lu_gpaddr_o = '{default: 0}; + + + for (int unsigned i = 0; i < TLB_ENTRIES; i++) begin + // first level match, this may be a giga page, check the ASID flags as well + // if the entry is associated to a global address, don't match the ASID (ASID is don't care) + match_asid[i] = (((lu_asid_i == tags_q[i].asid) || content_q[i].pte.g) && s_st_enbl_i) || !s_st_enbl_i; + match_vmid[i] = (lu_vmid_i == tags_q[i].vmid && g_st_enbl_i) || !g_st_enbl_i; + is_1G[i] = is_trans_1G(s_st_enbl_i, g_st_enbl_i, tags_q[i].is_s_1G, tags_q[i].is_g_1G); + is_2M[i] = is_trans_2M( + s_st_enbl_i, + g_st_enbl_i, + tags_q[i].is_s_1G, + tags_q[i].is_s_2M, + tags_q[i].is_g_1G, + tags_q[i].is_g_2M + ); + // check if translation is a: S-Stage and G-Stage, S-Stage only or G-Stage only translation and virtualization mode is on/off + match_stage[i] = (tags_q[i].v == v_i) && (tags_q[i].g_st_enbl == g_st_enbl_i) && (tags_q[i].s_st_enbl == s_st_enbl_i); + if (tags_q[i].valid && match_asid[i] && match_vmid[i] && match_stage[i] && (vpn2 == (tags_q[i].vpn2 & mask_pn2))) begin + lu_gpaddr_o = make_gpaddr(s_st_enbl_i, tags_q[i].is_s_1G, tags_q[i].is_s_2M, lu_vaddr_i, + content_q[i].pte); + if (is_1G[i]) begin + lu_is_1G_o = is_1G[i]; + lu_content_o = content_q[i].pte; + lu_g_content_o = content_q[i].gpte; + lu_hit_o = 1'b1; + lu_hit[i] = 1'b1; + // not a giga page hit so check further + end else if (vpn1 == tags_q[i].vpn1) begin + // this could be a 2 mega page hit or a 4 kB hit + // output accordingly + if (is_2M[i] || vpn0 == tags_q[i].vpn0) begin + lu_is_2M_o = is_2M[i]; + // Compute G-Stage PPN based on the gpaddr + g_content = content_q[i].gpte; + if (tags_q[i].is_g_2M) g_content.ppn[8:0] = lu_gpaddr_o[20:12]; + if (tags_q[i].is_g_1G) g_content.ppn[17:0] = lu_gpaddr_o[29:12]; + // Output G-stage and S-stage content + lu_g_content_o = g_content; + lu_content_o = content_q[i].pte; + lu_hit_o = 1'b1; + lu_hit[i] = 1'b1; + end + end + end + end + end + + + + logic asid_to_be_flushed_is0; // indicates that the ASID provided by SFENCE.VMA (rs2)is 0, active high + logic vaddr_to_be_flushed_is0; // indicates that the VADDR provided by SFENCE.VMA (rs1)is 0, active high + logic vmid_to_be_flushed_is0; // indicates that the VMID provided is 0, active high + logic gpaddr_to_be_flushed_is0; // indicates that the GPADDR provided is 0, active high + logic [TLB_ENTRIES-1:0] vaddr_vpn0_match; + logic [TLB_ENTRIES-1:0] vaddr_vpn1_match; + logic [TLB_ENTRIES-1:0] vaddr_vpn2_match; + logic [TLB_ENTRIES-1:0] gpaddr_gppn0_match; + logic [TLB_ENTRIES-1:0] gpaddr_gppn1_match; + logic [TLB_ENTRIES-1:0] gpaddr_gppn2_match; + logic [TLB_ENTRIES-1:0][(CVA6Cfg.GPPNW-1):0] gppn; + + + assign asid_to_be_flushed_is0 = ~(|asid_to_be_flushed_i); + assign vaddr_to_be_flushed_is0 = ~(|vaddr_to_be_flushed_i); + assign vmid_to_be_flushed_is0 = ~(|vmid_to_be_flushed_i); + assign gpaddr_to_be_flushed_is0 = ~(|gpaddr_to_be_flushed_i); + + // ------------------ + // Update and Flush + // ------------------ + always_comb begin : update_flush + tags_n = tags_q; + content_n = content_q; + + for (int unsigned i = 0; i < TLB_ENTRIES; i++) begin + + vaddr_vpn0_match[i] = (vaddr_to_be_flushed_i[20:12] == tags_q[i].vpn0); + vaddr_vpn1_match[i] = (vaddr_to_be_flushed_i[29:21] == tags_q[i].vpn1); + vaddr_vpn2_match[i] = (vaddr_to_be_flushed_i[30+VPN2:30] == tags_q[i].vpn2[VPN2:0]); + + gppn[i] = make_gppn( + tags_q[i].s_st_enbl, + tags_q[i].is_s_1G, + tags_q[i].is_s_2M, + { + tags_q[i].vpn2, tags_q[i].vpn1, tags_q[i].vpn0 + }, + content_q[i].pte + ); + gpaddr_gppn0_match[i] = (gpaddr_to_be_flushed_i[20:12] == gppn[i][8:0]); + gpaddr_gppn1_match[i] = (gpaddr_to_be_flushed_i[29:21] == gppn[i][17:9]); + gpaddr_gppn2_match[i] = (gpaddr_to_be_flushed_i[30+GPPN2:30] == gppn[i][18+GPPN2:18]); + + if (flush_i) begin + if (!tags_q[i].v) begin + // invalidate logic + // flush everything if ASID is 0 and vaddr is 0 ("SFENCE.VMA x0 x0" case) + if (asid_to_be_flushed_is0 && vaddr_to_be_flushed_is0) tags_n[i].valid = 1'b0; + // flush vaddr in all addressing space ("SFENCE.VMA vaddr x0" case), it should happen only for leaf pages + else if (asid_to_be_flushed_is0 && ((vaddr_vpn0_match[i] && vaddr_vpn1_match[i] && vaddr_vpn2_match[i]) || (vaddr_vpn2_match[i] && tags_q[i].is_s_1G) || (vaddr_vpn1_match[i] && vaddr_vpn2_match[i] && tags_q[i].is_s_2M) ) && (~vaddr_to_be_flushed_is0)) + tags_n[i].valid = 1'b0; + // the entry is flushed if it's not global and asid and vaddr both matches with the entry to be flushed ("SFENCE.VMA vaddr asid" case) + else if ((!content_q[i].pte.g) && ((vaddr_vpn0_match[i] && vaddr_vpn1_match[i] && vaddr_vpn2_match[i]) || (vaddr_vpn2_match[i] && tags_q[i].is_s_1G) || (vaddr_vpn1_match[i] && vaddr_vpn2_match[i] && tags_q[i].is_s_2M)) && (asid_to_be_flushed_i == tags_q[i].asid) && (!vaddr_to_be_flushed_is0) && (!asid_to_be_flushed_is0)) + tags_n[i].valid = 1'b0; + // the entry is flushed if it's not global, and the asid matches and vaddr is 0. ("SFENCE.VMA 0 asid" case) + else if ((!content_q[i].pte.g) && (vaddr_to_be_flushed_is0) && (asid_to_be_flushed_i == tags_q[i].asid) && (!asid_to_be_flushed_is0)) + tags_n[i].valid = 1'b0; + end + end else if (flush_vvma_i) begin + if (tags_q[i].v && tags_q[i].s_st_enbl) begin + // invalidate logic + // flush everything if current VMID matches and ASID is 0 and vaddr is 0 ("SFENCE.VMA/HFENCE.VVMA x0 x0" case) + if (asid_to_be_flushed_is0 && vaddr_to_be_flushed_is0 && ((tags_q[i].g_st_enbl && lu_vmid_i == tags_q[i].vmid) || !tags_q[i].g_st_enbl)) + tags_n[i].valid = 1'b0; + // flush vaddr in all addressing space if current VMID matches ("SFENCE.VMA/HFENCE.VVMA vaddr x0" case), it should happen only for leaf pages + else if (asid_to_be_flushed_is0 && ((vaddr_vpn0_match[i] && vaddr_vpn1_match[i] && vaddr_vpn2_match[i]) || (vaddr_vpn2_match[i] && tags_q[i].is_s_1G) || (vaddr_vpn1_match[i] && vaddr_vpn2_match[i] && tags_q[i].is_s_2M) ) && (~vaddr_to_be_flushed_is0) && ((tags_q[i].g_st_enbl && lu_vmid_i == tags_q[i].vmid) || !tags_q[i].g_st_enbl)) + tags_n[i].valid = 1'b0; + // the entry is flushed if it's not global and asid and vaddr and current VMID matches with the entry to be flushed ("SFENCE.VMA/HFENCE.VVMA vaddr asid" case) + else if ((!content_q[i].pte.g) && ((vaddr_vpn0_match[i] && vaddr_vpn1_match[i] && vaddr_vpn2_match[i]) || (vaddr_vpn2_match[i] && tags_q[i].is_s_1G) || (vaddr_vpn1_match[i] && vaddr_vpn2_match[i] && tags_q[i].is_s_2M)) && (asid_to_be_flushed_i == tags_q[i].asid && ((tags_q[i].g_st_enbl && lu_vmid_i == tags_q[i].vmid) || !tags_q[i].g_st_enbl)) && (!vaddr_to_be_flushed_is0) && (!asid_to_be_flushed_is0)) + tags_n[i].valid = 1'b0; + // the entry is flushed if it's not global, and the asid and the current VMID matches and vaddr is 0. ("SFENCE.VMA/HFENCE.VVMA 0 asid" case) + else if ((!content_q[i].pte.g) && (vaddr_to_be_flushed_is0) && (asid_to_be_flushed_i == tags_q[i].asid && ((tags_q[i].g_st_enbl && lu_vmid_i == tags_q[i].vmid) || !tags_q[i].g_st_enbl)) && (!asid_to_be_flushed_is0)) + tags_n[i].valid = 1'b0; + end + end else if (flush_gvma_i) begin + if (tags_q[i].g_st_enbl) begin + // invalidate logic + // flush everything if vmid is 0 and addr is 0 ("HFENCE.GVMA x0 x0" case) + if (vmid_to_be_flushed_is0 && gpaddr_to_be_flushed_is0) tags_n[i].valid = 1'b0; + // flush gpaddr in all addressing space ("HFENCE.GVMA gpaddr x0" case), it should happen only for leaf pages + else if (vmid_to_be_flushed_is0 && ((gpaddr_gppn0_match[i] && gpaddr_gppn1_match[i] && gpaddr_gppn2_match[i]) || (gpaddr_gppn2_match[i] && tags_q[i].is_g_1G) || (gpaddr_gppn1_match[i] && gpaddr_gppn2_match[i] && tags_q[i].is_g_2M) ) && (~gpaddr_to_be_flushed_is0)) + tags_n[i].valid = 1'b0; + // the entry vmid and gpaddr both matches with the entry to be flushed ("HFENCE.GVMA gpaddr vmid" case) + else if (((gpaddr_gppn0_match[i] && gpaddr_gppn1_match[i] && gpaddr_gppn2_match[i]) || (gpaddr_gppn2_match[i] && tags_q[i].is_g_1G) || (gpaddr_gppn1_match[i] && gpaddr_gppn2_match[i] && tags_q[i].is_g_2M)) && (vmid_to_be_flushed_i == tags_q[i].vmid) && (~gpaddr_to_be_flushed_is0) && (~vmid_to_be_flushed_is0)) + tags_n[i].valid = 1'b0; + // the entry is flushed if the vmid matches and gpaddr is 0. ("HFENCE.GVMA 0 vmid" case) + else if ((gpaddr_to_be_flushed_is0) && (vmid_to_be_flushed_i == tags_q[i].vmid) && (!vmid_to_be_flushed_is0)) + tags_n[i].valid = 1'b0; + end + // normal replacement + end else if (update_i.valid & replace_en[i]) begin + // update tag array + tags_n[i] = '{ + asid: update_i.asid, + vmid: update_i.vmid, + vpn2: update_i.vpn[18+GPPN2:18], + vpn1: update_i.vpn[17:9], + vpn0: update_i.vpn[8:0], + s_st_enbl: s_st_enbl_i, + g_st_enbl: g_st_enbl_i, + v: v_i, + is_s_1G: update_i.is_s_1G, + is_s_2M: update_i.is_s_2M, + is_g_1G: update_i.is_g_1G, + is_g_2M: update_i.is_g_2M, + valid: 1'b1 + }; + // and content as well + content_n[i].pte = update_i.content; + content_n[i].gpte = update_i.g_content; + end + end + end + + // ----------------------------------------------- + // PLRU - Pseudo Least Recently Used Replacement + // ----------------------------------------------- + logic [2*(TLB_ENTRIES-1)-1:0] plru_tree_q, plru_tree_n; + always_comb begin : plru_replacement + plru_tree_n = plru_tree_q; + // The PLRU-tree indexing: + // lvl0 0 + // / \ + // / \ + // lvl1 1 2 + // / \ / \ + // lvl2 3 4 5 6 + // / \ /\/\ /\ + // ... ... ... ... + // Just predefine which nodes will be set/cleared + // E.g. for a TLB with 8 entries, the for-loop is semantically + // equivalent to the following pseudo-code: + // unique case (1'b1) + // lu_hit[7]: plru_tree_n[0, 2, 6] = {1, 1, 1}; + // lu_hit[6]: plru_tree_n[0, 2, 6] = {1, 1, 0}; + // lu_hit[5]: plru_tree_n[0, 2, 5] = {1, 0, 1}; + // lu_hit[4]: plru_tree_n[0, 2, 5] = {1, 0, 0}; + // lu_hit[3]: plru_tree_n[0, 1, 4] = {0, 1, 1}; + // lu_hit[2]: plru_tree_n[0, 1, 4] = {0, 1, 0}; + // lu_hit[1]: plru_tree_n[0, 1, 3] = {0, 0, 1}; + // lu_hit[0]: plru_tree_n[0, 1, 3] = {0, 0, 0}; + // default: begin /* No hit */ end + // endcase + for ( + int unsigned i = 0; i < TLB_ENTRIES; i++ + ) begin + automatic int unsigned idx_base, shift, new_index; + // we got a hit so update the pointer as it was least recently used + if (lu_hit[i] & lu_access_i) begin + // Set the nodes to the values we would expect + for (int unsigned lvl = 0; lvl < $clog2(TLB_ENTRIES); lvl++) begin + idx_base = $unsigned((2 ** lvl) - 1); + // lvl0 <=> MSB, lvl1 <=> MSB-1, ... + shift = $clog2(TLB_ENTRIES) - lvl; + // to circumvent the 32 bit integer arithmetic assignment + new_index = ~((i >> (shift - 1)) & 32'b1); + plru_tree_n[idx_base+(i>>shift)] = new_index[0]; + end + end + end + // Decode tree to write enable signals + // Next for-loop basically creates the following logic for e.g. an 8 entry + // TLB (note: pseudo-code obviously): + // replace_en[7] = &plru_tree_q[ 6, 2, 0]; //plru_tree_q[0,2,6]=={1,1,1} + // replace_en[6] = &plru_tree_q[~6, 2, 0]; //plru_tree_q[0,2,6]=={1,1,0} + // replace_en[5] = &plru_tree_q[ 5,~2, 0]; //plru_tree_q[0,2,5]=={1,0,1} + // replace_en[4] = &plru_tree_q[~5,~2, 0]; //plru_tree_q[0,2,5]=={1,0,0} + // replace_en[3] = &plru_tree_q[ 4, 1,~0]; //plru_tree_q[0,1,4]=={0,1,1} + // replace_en[2] = &plru_tree_q[~4, 1,~0]; //plru_tree_q[0,1,4]=={0,1,0} + // replace_en[1] = &plru_tree_q[ 3,~1,~0]; //plru_tree_q[0,1,3]=={0,0,1} + // replace_en[0] = &plru_tree_q[~3,~1,~0]; //plru_tree_q[0,1,3]=={0,0,0} + // For each entry traverse the tree. If every tree-node matches, + // the corresponding bit of the entry's index, this is + // the next entry to replace. + for (int unsigned i = 0; i < TLB_ENTRIES; i += 1) begin + automatic logic en; + automatic int unsigned idx_base, shift, new_index; + en = 1'b1; + for (int unsigned lvl = 0; lvl < $clog2(TLB_ENTRIES); lvl++) begin + idx_base = $unsigned((2 ** lvl) - 1); + // lvl0 <=> MSB, lvl1 <=> MSB-1, ... + shift = $clog2(TLB_ENTRIES) - lvl; + + // en &= plru_tree_q[idx_base + (i>>shift)] == ((i >> (shift-1)) & 1'b1); + new_index = (i >> (shift - 1)) & 32'b1; + if (new_index[0]) begin + en &= plru_tree_q[idx_base+(i>>shift)]; + end else begin + en &= ~plru_tree_q[idx_base+(i>>shift)]; + end + end + replace_en[i] = en; + end + end + + // sequential process + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + tags_q <= '{default: 0}; + content_q <= '{default: 0}; + plru_tree_q <= '{default: 0}; + end else begin + tags_q <= tags_n; + content_q <= content_n; + plru_tree_q <= plru_tree_n; + end + end + //-------------- + // Sanity checks + //-------------- + + //pragma translate_off +`ifndef VERILATOR + + initial begin : p_assertions + assert ((TLB_ENTRIES % 2 == 0) && (TLB_ENTRIES > 1)) + else begin + $error("TLB size must be a multiple of 2 and greater than 1"); + $stop(); + end + assert (CVA6Cfg.ASID_WIDTH >= 1) + else begin + $error("ASID width must be at least 1"); + $stop(); + end + end + + // Just for checking + function int countSetBits(logic [TLB_ENTRIES-1:0] vector); + automatic int count = 0; + foreach (vector[idx]) begin + count += vector[idx]; + end + return count; + endfunction + + assert property (@(posedge clk_i) (countSetBits(lu_hit) <= 1)) + else begin + $error("More then one hit in TLB!"); + $stop(); + end + assert property (@(posedge clk_i) (countSetBits(replace_en) <= 1)) + else begin + $error("More then one TLB entry selected for next replace!"); + $stop(); + end + +`endif + //pragma translate_on + +endmodule diff --git a/core/store_unit.sv b/core/store_unit.sv index 0112b5f61a..e6a874edc0 100644 --- a/core/store_unit.sv +++ b/core/store_unit.sv @@ -60,6 +60,12 @@ module store_unit output logic [CVA6Cfg.VLEN-1:0] vaddr_o, // RVFI information - RVFI output [CVA6Cfg.PLEN-1:0] rvfi_mem_paddr_o, + // Transformed trap instruction out - TO_BE_COMPLETED + output logic [31:0] tinst_o, + // TO_BE_COMPLETED - TO_BE_COMPLETED + output logic hs_ld_st_inst_o, + // TO_BE_COMPLETED - TO_BE_COMPLETED + output logic hlvx_inst_o, // Physical address - TO_BE_COMPLETED input logic [CVA6Cfg.PLEN-1:0] paddr_i, // Exception raised before store - TO_BE_COMPLETED @@ -127,8 +133,11 @@ module store_unit logic [CVA6Cfg.TRANS_ID_BITS-1:0] trans_id_n, trans_id_q; // output assignments - assign vaddr_o = lsu_ctrl_i.vaddr; // virtual address - assign trans_id_o = trans_id_q; // transaction id from previous cycle + assign vaddr_o = lsu_ctrl_i.vaddr; // virtual address + assign hs_ld_st_inst_o = CVA6Cfg.RVH ? lsu_ctrl_i.hs_ld_st_inst : 1'b0; + assign hlvx_inst_o = CVA6Cfg.RVH ? lsu_ctrl_i.hlvx_inst : 1'b0; + assign tinst_o = CVA6Cfg.RVH ? lsu_ctrl_i.tinst : '0; // transformed instruction + assign trans_id_o = trans_id_q; // transaction id from previous cycle always_comb begin : store_control translation_req_o = 1'b0;