cpu-o3, mem-ruby: Enhance goldenmem

configs/ruby/CHI_config.py

@@ -232,7 +232,7 @@ class CHI_L1Controller(CHI_Cache_Controller):
     Default parameters for a L1 Cache controller
     """
 
-    def __init__(self, ruby_system, sequencer, cache, prefetcher, is_dcache=False):
+    def __init__(self, ruby_system, sequencer, cache, prefetcher, is_dcache=False, enable_difftest=False):
         super().__init__(ruby_system)
         self.sequencer = sequencer
         self.cache = cache
@@ -256,6 +256,7 @@ def __init__(self, ruby_system, sequencer, cache, prefetcher, is_dcache=False):
         self.dealloc_backinv_unique = False
         self.dealloc_backinv_shared = False
         self.is_dcache = is_dcache
+        self.enable_difftest = enable_difftest
         # Some reasonable default TBE params
         self.number_of_TBEs = 32+8
         self.number_of_repl_TBEs = 16
@@ -501,7 +502,7 @@ def __init__(
                 version=Versions.getSeqId(), ruby_system=ruby_system
             )
             cpu.data_sequencer = RubySequencer(
-                version=Versions.getSeqId(), ruby_system=ruby_system
+                version=Versions.getSeqId(), ruby_system=ruby_system, is_data_sequencer=True
             )
 
             self._seqs.append(
@@ -531,7 +532,7 @@ def __init__(
             )
 
             cpu.l1d = CHI_L1Controller(
-                ruby_system, cpu.data_sequencer, l1d_cache, l1d_pf, is_dcache=True
+                ruby_system, cpu.data_sequencer, l1d_cache, l1d_pf, is_dcache=True, enable_difftest = True
             )
 
             cpu.inst_sequencer.dcache = NULL

src/cpu/base.cc

@@ -1149,7 +1149,7 @@ BaseCPU::diffWithNEMU(ThreadID tid, InstSeqNum seq)
                     _goldenMemManager->inPmem(diffInfo.physEffAddr)) {
                     DPRINTF(Diff, "Difference on %s instr found in multicore mode, check in golden memory\n",
                          diffInfo.inst->isLoad() ? "load" : "amo");
-                    uint8_t *golden_ptr = (uint8_t *)_goldenMemManager->guestToHost(diffInfo.physEffAddr);
+                    uint8_t *golden_ptr = diffInfo.goldenValue;
 
                     // a lambda function to sync memory and register from golden results to ref
                     auto sync_mem_reg = [&]() {
@@ -1478,5 +1478,14 @@ BaseCPU::setExceptionGuideExecInfo(uint64_t exception_num, uint64_t mtval,
     // diffAllStates->proxy->update_config(&diffAllStates->diff.dynamic_config);
 }
 
-
+void
+BaseCPU::checkL1DRefill(Addr paddr, const uint8_t* refill_data, size_t size) {
+    assert(size == 64);
+    if (system->multiCore()) {
+        uint8_t *golden_ptr = (uint8_t *)_goldenMemManager->guestToHost(paddr);
+        if (memcmp(golden_ptr, refill_data, size)) {
+            panic("Refill data diff with Golden addr %#lx with size %d\n", paddr, size);
+        }
+    }
+}
 } // namespace gem5

src/cpu/base.hh

@@ -736,6 +736,7 @@ class BaseCPU : public ClockedObject
         bool curInstStrictOrdered{false};
         gem5::Addr physEffAddr;
         gem5::Addr effSize;
+        uint8_t *goldenValue;
         uint64_t amoOldGoldenValue;
         // Register address causing difftest error
         bool errorRegsValue[96];// 32 regs + 32fprs +32 vprs
@@ -797,6 +798,8 @@ class BaseCPU : public ClockedObject
     uint8_t *getGoldenMemPtr() { return goldenMemPtr; }
 
     gem5::GoldenGloablMem *goldenMemManager() { return _goldenMemManager; }
+
+    void checkL1DRefill(Addr paddr, const uint8_t *refill_data, size_t size);
 };
 
 } // namespace gem5

src/cpu/o3/commit.cc

@@ -1335,6 +1335,7 @@ Commit::diffInst(ThreadID tid, const DynInstPtr &inst) {
         inst->strictlyOrdered();
     cpu->diffInfo.physEffAddr = inst->physEffAddr;
     cpu->diffInfo.effSize = inst->effSize;
+    cpu->diffInfo.goldenValue = inst->getGolden();
     cpu->difftestStep(tid, inst->seqNum);
 }
 

src/cpu/o3/dyn_inst.hh

@@ -387,6 +387,9 @@ class DynInst : public ExecContext, public RefCounted
     ssize_t sqIdx = -1;
     typename LSQUnit::SQIterator sqIt;
 
+    /** If load data is from cache then it must be golden */
+    uint8_t goldenData[8] = {0};
+
     int pf_source  = -1; // if load cache line is prefetched
     /////////////////////// TLB Miss //////////////////////
     /**
@@ -1402,6 +1405,12 @@ class DynInst : public ExecContext, public RefCounted
     {
         return pc->as<RiscvISA::PCState>().branching();
     }
+
+    /** set golden */
+    void setGolden(uint8_t *golden) { memcpy(goldenData, golden, effSize); }
+
+    /** get golden */
+    uint8_t *getGolden() { return goldenData; }
 };
 
 } // namespace o3

src/cpu/o3/lsq.cc

@@ -61,6 +61,7 @@
 #include "debug/Schedule.hh"
 #include "debug/StoreBuffer.hh"
 #include "debug/Writeback.hh"
+#include "mem/packet_access.hh"
 #include "params/BaseO3CPU.hh"
 
 namespace gem5
@@ -517,6 +518,11 @@ LSQ::recvFunctionalCustomSignal(PacketPtr pkt, int sig)
     iewStage->loadCancel(request->instruction());
 }
 
+void*
+LSQ::getCPUPtr() {
+    return (void *) cpu;
+}
+
 int
 LSQ::getCount()
 {
@@ -1147,7 +1153,8 @@ LSQ::LSQRequest::LSQRequest(
     _state(State::NotIssued),
     _port(*port), _inst(inst), _data(nullptr),
     _res(nullptr), _addr(0), _size(0), _flags(0),
-    _numOutstandingPackets(0), _amo_op(nullptr)
+    _numOutstandingPackets(0), _amo_op(nullptr),
+    _sbufferBypass(false)
 {
     flags.set(Flag::IsLoad, isLoad);
     if (_inst) {
@@ -1172,7 +1179,8 @@ LSQ::LSQRequest::LSQRequest(
     _flags(flags_),
     _numOutstandingPackets(0),
     _amo_op(std::move(amo_op)),
-    _hasStaleTranslation(stale_translation)
+    _hasStaleTranslation(stale_translation),
+    _sbufferBypass(false)
 {
     flags.set(Flag::IsLoad, isLoad);
     if (_inst) {
@@ -1235,6 +1243,7 @@ LSQ::LSQRequest::forward()
 {
     if (!isLoad() || !needWBToRegister() || forwardPackets.empty()) return;
     DPRINTF(StoreBuffer, "sbuffer forward data\n");
+    _sbufferBypass = true;
     for (auto& p : forwardPackets)
     {
         _inst->memData[p.idx] = p.byte;
@@ -1614,6 +1623,12 @@ LSQ::DcachePort::recvFunctionalCustomSignal(PacketPtr pkt, int sig)
     lsq->recvFunctionalCustomSignal(pkt, sig);
 }
 
+void*
+LSQ::DcachePort::recvGetCPUPtr()
+{
+    return (void *) (lsq->cpu);
+}
+
 void
 LSQ::DcachePort::recvReqRetry()
 {

src/cpu/o3/lsq.hh

@@ -101,6 +101,7 @@ class LSQ
         virtual bool recvTimingResp(PacketPtr pkt);
         virtual void recvTimingSnoopReq(PacketPtr pkt);
         virtual void recvFunctionalCustomSignal(PacketPtr pkt, int sig);
+        virtual void* recvGetCPUPtr();
 
         virtual void
         recvFunctionalSnoop(PacketPtr pkt)
@@ -260,6 +261,7 @@ class LSQ
         uint32_t _numOutstandingPackets;
         AtomicOpFunctorPtr _amo_op;
         bool _hasStaleTranslation;
+        bool _sbufferBypass;
 
         struct FWDPacket
         {
@@ -917,6 +919,8 @@ class LSQ
 
     void recvFunctionalCustomSignal(PacketPtr pkt, int sig);
 
+    void* getCPUPtr();
+
     Fault pushRequest(const DynInstPtr& inst, bool isLoad, uint8_t *data,
                       unsigned int size, Addr addr, Request::Flags flags,
                       uint64_t *res, AtomicOpFunctorPtr amo_op,

src/cpu/o3/lsq_unit.cc

@@ -52,6 +52,7 @@
 #include "cpu/o3/dyn_inst.hh"
 #include "cpu/o3/limits.hh"
 #include "cpu/o3/lsq.hh"
+#include "cpu/utils.hh"
 #include "debug/Activity.hh"
 #include "debug/Diff.hh"
 #include "debug/HtmCpu.hh"
@@ -60,6 +61,7 @@
 #include "debug/O3PipeView.hh"
 #include "debug/StoreBuffer.hh"
 #include "mem/packet.hh"
+#include "mem/packet_access.hh"
 #include "mem/request.hh"
 
 namespace gem5
@@ -243,6 +245,26 @@ LSQUnit::completeDataAccess(PacketPtr pkt)
 
     assert(!cpu->switchedOut());
     if (!inst->isSquashed()) {
+        if (inst->isLoad() || inst->isAtomic()) {
+            Addr addr = pkt->getAddr();
+            auto [enable_diff, diff_all_states] = cpu->getDiffAllStates();
+            if (system->multiCore() && enable_diff && !request->_sbufferBypass &&
+                cpu->goldenMemManager()->inPmem(addr)) {
+                // check data with golden mem
+                uint8_t *golden_data = (uint8_t *)cpu->goldenMemManager()->guestToHost(addr);
+                uint8_t *loaded_data = pkt->getPtr<uint8_t>();
+                size_t size = pkt->getSize();
+                if (memcmp(golden_data, loaded_data, size) == 0) {
+                    assert(size == inst->effSize);
+                    inst->setGolden(golden_data);
+                } else {
+                    panic("Data error at addr %#lx, size %d. %s\n",
+                        addr, size,
+                        goldenDiffStr(loaded_data, golden_data, size).c_str());
+                }
+            }
+        }
+
         if (request->needWBToRegister()) {
             // Only loads, store conditionals and atomics perform the writeback
             // after receving the response from the memory
@@ -306,6 +328,8 @@ LSQUnit::init(CPU *cpu_ptr, IEW *iew_ptr, const BaseO3CPUParams &params,
 
     DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",lsqID);
 
+    system = params.system;
+
     depCheckShift = params.LSQDepCheckShift;
     checkLoads = params.LSQCheckLoads;
     needsTSO = params.needsTSO;

src/cpu/o3/lsq_unit.hh

@@ -556,6 +556,8 @@ class LSQUnit
     BaseMMU *getMMUPtr();
 
   private:
+    System *system;
+
     /** Pointer to the CPU. */
     CPU *cpu;
 

src/cpu/utils.hh

@@ -94,6 +94,18 @@ isAnyActiveElement(const std::vector<bool>::const_iterator& it_start,
     return (it_tmp != it_end);
 }
 
+inline std::string
+goldenDiffStr(uint8_t *dut_ptr, uint8_t* golden_ptr, size_t size) {
+    assert(size <= 8);
+    uint64_t dut_value = 0;
+    uint64_t golden_value = 0;
+    memcpy(&dut_value, dut_ptr, size);
+    memcpy(&golden_value, golden_ptr, size);
+    std::stringstream ss;
+    ss << std::hex << "Dut value: " << dut_value << " , golden value: " << golden_value << " ";
+    return ss.str();
+}
+
 } // namespace gem5
 
 #endif // __CPU_UTILS_HH__

src/mem/port.hh

@@ -325,6 +325,16 @@ class ResponsePort : public Port, public AtomicResponseProtocol,
         }
     }
 
+    void* sendGetCPUPtr()
+    {
+        try {
+            warn("Peer is %s\n", _requestPort->name());
+            return FunctionalResponseProtocol::sendGetCPUPtr(_requestPort);
+        } catch (UnboundPortException) {
+            reportUnbound();
+        }
+    }
+
   public:
     /* The atomic protocol. */
 

src/mem/protocol/functional.cc

@@ -59,6 +59,12 @@ void FunctionalRequestProtocol::sendFunctionalCustomSignal(
     return peer->recvFunctionalCustomSignal(pkt, sig);
 }
 
+void* FunctionalRequestProtocol::recvGetCPUPtr()
+{
+    panic("recvGetCPUPtr not implemented\n");
+    return nullptr;
+}
+
 /* The response protocol. */
 
 void
@@ -75,4 +81,9 @@ void FunctionalResponseProtocol::sendFunctionalCustomSignal(
     return peer->recvFunctionalCustomSignal(pkt, sig);
 }
 
+void* FunctionalResponseProtocol::sendGetCPUPtr(FunctionalRequestProtocol *peer) const
+{
+    return peer->recvGetCPUPtr();
+}
+
 } // namespace gem5

src/mem/protocol/functional.hh

@@ -73,6 +73,8 @@ class FunctionalRequestProtocol
      * Receive a functional custom signals
      */
     virtual void recvFunctionalCustomSignal(PacketPtr pkt, int sig) {};
+
+    virtual void* recvGetCPUPtr();
 };
 
 class FunctionalResponseProtocol
@@ -96,6 +98,8 @@ class FunctionalResponseProtocol
 
     void sendFunctionalCustomSignal(FunctionalRequestProtocol *peer, PacketPtr pkt, int sig) const;
 
+    void* sendGetCPUPtr(FunctionalRequestProtocol *peer) const;
+
     /**
      * Receive a functional custom signals
      */

src/mem/ruby/protocol/RubySlicc_Types.sm

@@ -123,6 +123,7 @@ structure (Sequencer, external = "yes") {
                     Cycles, Cycles, Cycles);
 
   void notifyMissCallback(Addr, bool, bool);
+  void checkL1DRefill(Addr, DataBlock, WriteMask);
   void TBEFullCancel(Addr);
 
   void writeCallback(Addr, DataBlock);

src/mem/ruby/protocol/chi/CHI-cache-actions.sm

@@ -2157,6 +2157,9 @@ action(Receive_ReqDataResp, desc="") {
         tbe.dataBlkValid.clear();
       }
       tbe.dataBlk.copyPartial(in_msg.dataBlk, in_msg.bitMask);
+      if (is_dcache && enable_difftest) {
+        sequencer.checkL1DRefill(tbe.addr, in_msg.dataBlk, in_msg.bitMask);
+      }
       assert(tbe.dataBlkValid.isOverlap(in_msg.bitMask) == false);
       tbe.dataBlkValid.orMask(in_msg.bitMask);
     }

src/mem/ruby/protocol/chi/CHI-cache.sm

@@ -179,7 +179,7 @@ machine(MachineType:Cache, "Cache coherency protocol") :
   bool use_prefetcher, default="false";
 
   bool is_dcache := "False";
-
+  bool enable_difftest := "False";
   // Message Queues
 
   // Interface to the network

src/mem/ruby/system/RubyPort.cc

@@ -46,6 +46,7 @@
 #include "debug/Config.hh"
 #include "debug/Drain.hh"
 #include "debug/Ruby.hh"
+#include "mem/packet_access.hh"
 #include "mem/ruby/protocol/AccessPermission.hh"
 #include "mem/ruby/slicc_interface/AbstractController.hh"
 #include "mem/simple_mem.hh"
@@ -69,7 +70,9 @@ RubyPort::RubyPort(const Params &p)
                    p.ruby_system->getAccessBackingStore(), -1,
                    p.no_retry_on_stall),
       gotAddrRanges(p.port_interrupt_out_port_connection_count),
-      m_isCPUSequencer(p.is_cpu_sequencer)
+      m_isCPUSequencer(p.is_cpu_sequencer),
+      m_isDataSequencer(p.is_data_sequencer),
+      cpu(nullptr)
 {
     assert(m_version != -1);
 
@@ -285,6 +288,12 @@ RubyPort::MemResponsePort::recvTimingReq(PacketPtr pkt)
         }
     }
 
+    // set cpu here
+    // when receiving a request, set sequencer(rubyport)'s cpu
+    if (ruby_port->m_isDataSequencer && !ruby_port->cpu) {
+        ruby_port->cpu = (BaseCPU *)(this->sendGetCPUPtr());
+    }
+
     // Save the port in the sender state object to be used later to
     // route the response
     pkt->pushSenderState(new SenderState(this));