diff --git a/src/main/scala/xiangshan/backend/MemBlock.scala b/src/main/scala/xiangshan/backend/MemBlock.scala index cfe26204d..e1fa96148 100644 --- a/src/main/scala/xiangshan/backend/MemBlock.scala +++ b/src/main/scala/xiangshan/backend/MemBlock.scala @@ -1636,6 +1636,10 @@ class MemBlockImp(outer: MemBlock) extends LazyModuleImp(outer) loadMisalignBuffer.io.overwriteExpBuf.vaddr, storeMisalignBuffer.io.overwriteExpBuf.vaddr ) + val misalignBufExceptionGpaddr = Mux(loadMisalignBuffer.io.overwriteExpBuf.valid, + loadMisalignBuffer.io.overwriteExpBuf.gpaddr, + storeMisalignBuffer.io.overwriteExpBuf.gpaddr + ) val vSegmentException = RegInit(false.B) when (DelayN(redirect.valid, 10) && vSegmentException) { @@ -1648,6 +1652,7 @@ class MemBlockImp(outer: MemBlock) extends LazyModuleImp(outer) val vSegmentExceptionVl = RegEnable(vSegmentUnit.io.exceptionInfo.bits.vl, vSegmentUnit.io.exceptionInfo.valid) val vSegmentExceptionAddress = RegEnable(vSegmentUnit.io.exceptionInfo.bits.vaddr, vSegmentUnit.io.exceptionInfo.valid) val atomicsExceptionGPAddress = RegEnable(atomicsUnit.io.exceptionAddr.bits.gpaddr, atomicsUnit.io.exceptionAddr.valid) + val vSegmentExceptionGPAddress = RegEnable(vSegmentUnit.io.exceptionInfo.bits.gpaddr, vSegmentUnit.io.exceptionInfo.valid) io.mem_to_ooo.lsqio.vaddr := RegNext(Mux( atomicsException, atomicsExceptionAddress, @@ -1671,8 +1676,17 @@ class MemBlockImp(outer: MemBlock) extends LazyModuleImp(outer) ) XSError(atomicsException && atomicsUnit.io.in.valid, "new instruction before exception triggers\n") - io.mem_to_ooo.lsqio.gpaddr := RegNext(Mux(atomicsException, atomicsExceptionGPAddress, lsq.io.exceptionAddr.gpaddr)) - + io.mem_to_ooo.lsqio.gpaddr := RegNext(Mux( + atomicsException, + atomicsExceptionGPAddress, + Mux(misalignBufExceptionOverwrite, + misalignBufExceptionGpaddr, + Mux(vSegmentException, + vSegmentExceptionGPAddress, + lsq.io.exceptionAddr.gpaddr + ) + ) + )) io.mem_to_ooo.topToBackendBypass match { case x => x.hartId := io.hartId x.externalInterrupt.msip := outer.clint_int_sink.in.head._1(0) diff --git a/src/main/scala/xiangshan/mem/lsqueue/LoadExceptionBuffer.scala b/src/main/scala/xiangshan/mem/lsqueue/LoadExceptionBuffer.scala index 2f217bc00..af6ba78a1 100644 --- a/src/main/scala/xiangshan/mem/lsqueue/LoadExceptionBuffer.scala +++ b/src/main/scala/xiangshan/mem/lsqueue/LoadExceptionBuffer.scala @@ -103,7 +103,7 @@ class LqExceptionBuffer(implicit p: Parameters) extends XSModule with HasCircula req := reqSel._2(0) } - io.exceptionAddr.vaddr := req.vaddr + io.exceptionAddr.vaddr := req.vaddr io.exceptionAddr.vstart := req.uop.vpu.vstart io.exceptionAddr.vl := req.uop.vpu.vl io.exceptionAddr.gpaddr := req.gpaddr diff --git a/src/main/scala/xiangshan/mem/lsqueue/LoadMisalignBuffer.scala b/src/main/scala/xiangshan/mem/lsqueue/LoadMisalignBuffer.scala index 63262f5be..863a848bc 100644 --- a/src/main/scala/xiangshan/mem/lsqueue/LoadMisalignBuffer.scala +++ b/src/main/scala/xiangshan/mem/lsqueue/LoadMisalignBuffer.scala @@ -34,7 +34,7 @@ import xiangshan.backend.Bundles.{MemExuOutput, DynInst} class LoadMisalignBuffer(implicit p: Parameters) extends XSModule with HasCircularQueuePtrHelper - with HasLoadHelper + with HasLoadHelper { private val enqPortNum = LoadPipelineWidth private val maxSplitNum = 2 @@ -117,8 +117,9 @@ class LoadMisalignBuffer(implicit p: Parameters) extends XSModule val splitLoadResp = Flipped(Valid(new LqWriteBundle)) val writeBack = Decoupled(new MemExuOutput) val overwriteExpBuf = Output(new XSBundle { - val valid = Bool() - val vaddr = UInt(VAddrBits.W) + val valid = Bool() + val vaddr = UInt(VAddrBits.W) + val gpaddr = UInt(GPAddrBits.W) }) val flushLdExpBuff = Output(Bool()) }) @@ -250,7 +251,7 @@ class LoadMisalignBuffer(implicit p: Parameters) extends XSModule LB -> 0.U, LH -> 1.U, LW -> 3.U, - LD -> 7.U + LD -> 7.U )) + req.vaddr(4, 0) // to see if (vaddr + opSize - 1) and vaddr are in the same 16 bytes region val cross16BytesBoundary = req_valid && (highAddress(4) =/= req.vaddr(4)) @@ -549,7 +550,7 @@ class LoadMisalignBuffer(implicit p: Parameters) extends XSModule io.writeBack.bits.debug.isPerfCnt := false.B io.writeBack.bits.debug.paddr := req.paddr io.writeBack.bits.debug.vaddr := req.vaddr - + val flush = req_valid && req.uop.robIdx.needFlush(io.redirect) when (flush && (bufferState =/= s_idle)) { @@ -564,10 +565,12 @@ class LoadMisalignBuffer(implicit p: Parameters) extends XSModule // NOTE: spectial case (unaligned load cross page, page fault happens in next page) // if exception happens in the higher page address part, overwrite the loadExceptionBuffer vaddr val overwriteExpBuf = GatedValidRegNext(req_valid && cross16BytesBoundary && globalException && (curPtr === 1.U)) - val overwriteAddr = GatedRegNext(splitLoadResp(curPtr).vaddr) + val overwriteVaddr = GatedRegNext(splitLoadResp(curPtr).vaddr) + val overwriteGpaddr = GatedRegNext(splitLoadResp(curPtr).gpaddr) io.overwriteExpBuf.valid := overwriteExpBuf - io.overwriteExpBuf.vaddr := overwriteAddr + io.overwriteExpBuf.vaddr := overwriteVaddr + io.overwriteExpBuf.gpaddr := overwriteGpaddr // when no exception or mmio, flush loadExceptionBuffer at s_wb val flushLdExpBuff = GatedValidRegNext(req_valid && (bufferState === s_wb) && !(globalMMIO || globalException)) diff --git a/src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala b/src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala index 469345645..b3d93a70a 100644 --- a/src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala +++ b/src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala @@ -264,6 +264,7 @@ class LoadQueue(implicit p: Parameters) extends XSModule exceptionBuffer.io.req(LoadPipelineWidth + i).valid := io.vecFeedback(i).valid && io.vecFeedback(i).bits.feedback(VecFeedbacks.FLUSH) // have exception exceptionBuffer.io.req(LoadPipelineWidth + i).bits := DontCare exceptionBuffer.io.req(LoadPipelineWidth + i).bits.vaddr := io.vecFeedback(i).bits.vaddr + exceptionBuffer.io.req(LoadPipelineWidth + i).bits.gpaddr := io.vecFeedback(i).bits.gpaddr exceptionBuffer.io.req(LoadPipelineWidth + i).bits.uop.uopIdx := io.vecFeedback(i).bits.uopidx exceptionBuffer.io.req(LoadPipelineWidth + i).bits.uop.robIdx := io.vecFeedback(i).bits.robidx exceptionBuffer.io.req(LoadPipelineWidth + i).bits.uop.vpu.vstart := io.vecFeedback(i).bits.vstart diff --git a/src/main/scala/xiangshan/mem/lsqueue/StoreMisalignBuffer.scala b/src/main/scala/xiangshan/mem/lsqueue/StoreMisalignBuffer.scala index 885c6e54b..b7f895f5e 100644 --- a/src/main/scala/xiangshan/mem/lsqueue/StoreMisalignBuffer.scala +++ b/src/main/scala/xiangshan/mem/lsqueue/StoreMisalignBuffer.scala @@ -94,6 +94,7 @@ class StoreMisalignBuffer(implicit p: Parameters) extends XSModule val overwriteExpBuf = Output(new XSBundle { val valid = Bool() val vaddr = UInt(VAddrBits.W) + val gpaddr = UInt(GPAddrBits.W) }) val sqControl = new StoreMaBufToSqControlIO }) @@ -591,9 +592,11 @@ class StoreMisalignBuffer(implicit p: Parameters) extends XSModule // if exception happens in the higher page address part, overwrite the storeExceptionBuffer vaddr val overwriteExpBuf = GatedValidRegNext(req_valid && cross16BytesBoundary && globalException && (curPtr === 1.U)) val overwriteAddr = GatedRegNext(splitStoreResp(curPtr).vaddr) + val overwriteGpaddr = GatedRegNext(splitStoreResp(curPtr).gpaddr) io.overwriteExpBuf.valid := overwriteExpBuf io.overwriteExpBuf.vaddr := overwriteAddr + io.overwriteExpBuf.gpaddr := overwriteGpaddr XSPerfAccumulate("alloc", RegNext(!req_valid) && req_valid) XSPerfAccumulate("flush", flush) diff --git a/src/main/scala/xiangshan/mem/lsqueue/StoreQueue.scala b/src/main/scala/xiangshan/mem/lsqueue/StoreQueue.scala index 96d71f109..ab78a9716 100644 --- a/src/main/scala/xiangshan/mem/lsqueue/StoreQueue.scala +++ b/src/main/scala/xiangshan/mem/lsqueue/StoreQueue.scala @@ -229,6 +229,7 @@ class StoreQueue(implicit p: Parameters) extends XSModule exceptionBuffer.io.storeAddrIn(StorePipelineWidth * 2 + i).valid := io.vecFeedback(i).valid && io.vecFeedback(i).bits.feedback(VecFeedbacks.FLUSH) // have exception exceptionBuffer.io.storeAddrIn(StorePipelineWidth * 2 + i).bits := DontCare exceptionBuffer.io.storeAddrIn(StorePipelineWidth * 2 + i).bits.vaddr := io.vecFeedback(i).bits.vaddr + exceptionBuffer.io.storeAddrIn(StorePipelineWidth * 2 + i).bits.gpaddr := io.vecFeedback(i).bits.gpaddr exceptionBuffer.io.storeAddrIn(StorePipelineWidth * 2 + i).bits.uop.uopIdx := io.vecFeedback(i).bits.uopidx exceptionBuffer.io.storeAddrIn(StorePipelineWidth * 2 + i).bits.uop.robIdx := io.vecFeedback(i).bits.robidx exceptionBuffer.io.storeAddrIn(StorePipelineWidth * 2 + i).bits.uop.vpu.vstart := io.vecFeedback(i).bits.vstart diff --git a/src/main/scala/xiangshan/mem/pipeline/LoadUnit.scala b/src/main/scala/xiangshan/mem/pipeline/LoadUnit.scala index 2af9d1810..7fdf00a63 100644 --- a/src/main/scala/xiangshan/mem/pipeline/LoadUnit.scala +++ b/src/main/scala/xiangshan/mem/pipeline/LoadUnit.scala @@ -1554,6 +1554,7 @@ class LoadUnit(implicit p: Parameters) extends XSModule io.vecldout.bits.flushState := DontCare io.vecldout.bits.exceptionVec := ExceptionNO.selectByFu(s3_out.bits.uop.exceptionVec, VlduCfg) io.vecldout.bits.vaddr := s3_in.vaddr + io.vecldout.bits.gpaddr := s3_in.gpaddr io.vecldout.bits.mmio := DontCare io.vecldout.valid := s3_out.valid && !s3_out.bits.uop.robIdx.needFlush(io.redirect) && s3_vecout.isvec || diff --git a/src/main/scala/xiangshan/mem/pipeline/StoreUnit.scala b/src/main/scala/xiangshan/mem/pipeline/StoreUnit.scala index a2bd9f780..59f383fb2 100644 --- a/src/main/scala/xiangshan/mem/pipeline/StoreUnit.scala +++ b/src/main/scala/xiangshan/mem/pipeline/StoreUnit.scala @@ -485,6 +485,7 @@ class StoreUnit(implicit p: Parameters) extends XSModule sx_in(i).mbIndex := s3_in.mbIndex sx_in(i).mask := s3_in.mask sx_in(i).vaddr := s3_in.vaddr + sx_in(i).gpaddr := s3_in.gpaddr sx_ready(i) := !s3_valid(i) || sx_in(i).output.uop.robIdx.needFlush(io.redirect) || (if (TotalDelayCycles == 0) io.stout.ready else sx_ready(i+1)) } else { val cur_kill = sx_in(i).output.uop.robIdx.needFlush(io.redirect) @@ -522,6 +523,7 @@ class StoreUnit(implicit p: Parameters) extends XSModule io.vecstout.bits.alignedType := sx_last_in.alignedType io.vecstout.bits.mask := sx_last_in.mask io.vecstout.bits.vaddr := sx_last_in.vaddr + io.vecstout.bits.gpaddr := sx_last_in.gpaddr // io.vecstout.bits.reg_offset.map(_ := DontCare) // io.vecstout.bits.elemIdx.map(_ := sx_last_in.elemIdx) // io.vecstout.bits.elemIdxInsideVd.map(_ := DontCare) diff --git a/src/main/scala/xiangshan/mem/vector/VMergeBuffer.scala b/src/main/scala/xiangshan/mem/vector/VMergeBuffer.scala index c48f329db..60b466794 100644 --- a/src/main/scala/xiangshan/mem/vector/VMergeBuffer.scala +++ b/src/main/scala/xiangshan/mem/vector/VMergeBuffer.scala @@ -44,6 +44,7 @@ class MBufferBundle(implicit p: Parameters) extends VLSUBundle{ val vstart = UInt(elemIdxBits.W) val vl = UInt(elemIdxBits.W) val vaddr = UInt(VAddrBits.W) + val gpaddr = UInt(GPAddrBits.W) val fof = Bool() val vlmax = UInt(elemIdxBits.W) @@ -102,6 +103,7 @@ abstract class BaseVMergeBuffer(isVStore: Boolean=false)(implicit p: Parameters) sink.feedback(VecFeedbacks.LAST) := true.B sink.vstart := source.vstart // TODO: if lsq need vl for fof? sink.vaddr := source.vaddr + sink.gpaddr := source.gpaddr sink.vl := source.vl sink.exceptionVec := ExceptionNO.selectByFu(source.exceptionVec, fuCfg) sink @@ -241,6 +243,7 @@ abstract class BaseVMergeBuffer(isVStore: Boolean=false)(implicit p: Parameters) entries(wbMbIndex(i)).vstart := selElemInfield entries(wbMbIndex(i)).exceptionVec := ExceptionNO.selectByFu(selExceptionVec, fuCfg) entries(wbMbIndex(i)).vaddr := vaddr + entries(wbMbIndex(i)).gpaddr := selPort(0).gpaddr }.otherwise{ entries(wbMbIndex(i)).vl := selElemInfield } diff --git a/src/main/scala/xiangshan/mem/vector/VSegmentUnit.scala b/src/main/scala/xiangshan/mem/vector/VSegmentUnit.scala index 28d94a6ea..4a483df0c 100644 --- a/src/main/scala/xiangshan/mem/vector/VSegmentUnit.scala +++ b/src/main/scala/xiangshan/mem/vector/VSegmentUnit.scala @@ -49,7 +49,9 @@ class VSegmentBundle(implicit p: Parameters) extends VLSUBundle // for exception val vstart = UInt(elemIdxBits.W) val exceptionVaddr = UInt(VAddrBits.W) + val exceptionGpaddr = UInt(GPAddrBits.W) val exception_va = Bool() + val exception_gpa = Bool() val exception_pa = Bool() val exceptionVstart = UInt(elemIdxBits.W) val exceptionVl = UInt(elemIdxBits.W) @@ -172,6 +174,7 @@ class VSegmentUnit (implicit p: Parameters) extends VLSUModule val splitPtrNext = WireInit(0.U.asTypeOf(new VSegUPtr)) val exception_va = WireInit(false.B) + val exception_gpa = WireInit(false.B) val exception_pa = WireInit(false.B) val maxSegIdx = instMicroOp.vl - 1.U @@ -251,7 +254,7 @@ class VSegmentUnit (implicit p: Parameters) extends VLSUModule }.elsewhen(state === s_pm){ /* if is vStore, send data to sbuffer, so don't need query dcache */ - stateNext := Mux(exception_pa || exception_va, + stateNext := Mux(exception_pa || exception_va || exception_gpa, s_finish, Mux(FuType.isVLoad(instMicroOp.uop.fuType), s_cache_req, s_send_data)) @@ -379,12 +382,15 @@ class VSegmentUnit (implicit p: Parameters) extends VLSUModule val canTriggerException = segmentIdx === 0.U || !instMicroOp.isFof // only elementIdx = 0 or is not fof can trigger // tlb resp when(io.dtlb.resp.fire && state === s_wait_tlb_resp){ - exceptionVec(storePageFault) := io.dtlb.resp.bits.excp(0).pf.st && canTriggerException - exceptionVec(loadPageFault) := io.dtlb.resp.bits.excp(0).pf.ld && canTriggerException - exceptionVec(storeAccessFault) := io.dtlb.resp.bits.excp(0).af.st && canTriggerException - exceptionVec(loadAccessFault) := io.dtlb.resp.bits.excp(0).af.ld && canTriggerException + exceptionVec(storePageFault) := io.dtlb.resp.bits.excp(0).pf.st && canTriggerException + exceptionVec(loadPageFault) := io.dtlb.resp.bits.excp(0).pf.ld && canTriggerException + exceptionVec(storeGuestPageFault) := io.dtlb.resp.bits.excp(0).gpf.st && canTriggerException + exceptionVec(loadGuestPageFault) := io.dtlb.resp.bits.excp(0).gpf.ld && canTriggerException + exceptionVec(storeAccessFault) := io.dtlb.resp.bits.excp(0).af.st && canTriggerException + exceptionVec(loadAccessFault) := io.dtlb.resp.bits.excp(0).af.ld && canTriggerException when(!io.dtlb.resp.bits.miss){ instMicroOp.paddr := io.dtlb.resp.bits.paddr(0) + instMicroOp.exceptionGpaddr := io.dtlb.resp.bits.gpaddr(0) } } // pmp @@ -401,17 +407,19 @@ class VSegmentUnit (implicit p: Parameters) extends VLSUModule exceptionVec(loadAddrMisaligned) := missAligned && FuType.isVLoad(fuType) && canTriggerException exceptionVec(storeAddrMisaligned) := missAligned && FuType.isVStore(fuType) && canTriggerException - exception_va := exceptionVec(storePageFault) || exceptionVec(loadPageFault) || + exception_va := exceptionVec(storePageFault) || exceptionVec(loadPageFault) || exceptionVec(storeAccessFault) || exceptionVec(loadAccessFault) || (missAligned && canTriggerException) - exception_pa := (pmp.st || pmp.ld || pmp.mmio) && canTriggerException + exception_gpa := exceptionVec(storeGuestPageFault) || exceptionVec(loadGuestPageFault) + exception_pa := (pmp.st || pmp.ld || pmp.mmio) && canTriggerException - instMicroOp.exception_pa := exception_pa - instMicroOp.exception_va := exception_va + instMicroOp.exception_pa := exception_pa + instMicroOp.exception_va := exception_va + instMicroOp.exception_gpa := exception_gpa // update storeAccessFault bit. Currently, we don't support vector MMIO exceptionVec(loadAccessFault) := (exceptionVec(loadAccessFault) || pmp.ld || pmp.mmio) && canTriggerException exceptionVec(storeAccessFault) := (exceptionVec(storeAccessFault) || pmp.st || pmp.mmio) && canTriggerException - when(exception_va || exception_pa) { + when(exception_va || exception_gpa || exception_pa) { when(canTriggerException) { instMicroOp.exceptionVaddr := vaddr instMicroOp.exceptionVl := segmentIdx // for exception @@ -653,6 +661,7 @@ class VSegmentUnit (implicit p: Parameters) extends VLSUModule io.exceptionInfo.bits.uopidx := uopq(deqPtr.value).uop.vpu.vuopIdx io.exceptionInfo.bits.vstart := instMicroOp.exceptionVstart io.exceptionInfo.bits.vaddr := instMicroOp.exceptionVaddr + io.exceptionInfo.bits.gpaddr := instMicroOp.exceptionGpaddr io.exceptionInfo.bits.vl := instMicroOp.exceptionVl io.exceptionInfo.valid := (state === s_finish) && instMicroOp.uop.exceptionVec.asUInt.orR && !isEmpty(enqPtr, deqPtr) } diff --git a/src/main/scala/xiangshan/mem/vector/VecBundle.scala b/src/main/scala/xiangshan/mem/vector/VecBundle.scala index 7cea18d75..14029aaf7 100644 --- a/src/main/scala/xiangshan/mem/vector/VecBundle.scala +++ b/src/main/scala/xiangshan/mem/vector/VecBundle.scala @@ -110,6 +110,7 @@ class VecPipelineFeedbackIO(isVStore: Boolean=false) (implicit p: Parameters) ex //val atomic = Bool() val exceptionVec = ExceptionVec() val vaddr = UInt(VAddrBits.W) + val gpaddr = UInt(GPAddrBits.W) //val vec = new OnlyVecExuOutput // feedback val vecFeedback = Bool() @@ -191,6 +192,7 @@ class FeedbackToLsqIO(implicit p: Parameters) extends VLSUBundle{ val robidx = new RobPtr val uopidx = UopIdx() val vaddr = UInt(VAddrBits.W) + val gpaddr = UInt(GPAddrBits.W) val feedback = Vec(VecFeedbacks.allFeedbacks, Bool()) // for exception val vstart = UInt(elemIdxBits.W) diff --git a/src/main/scala/xiangshan/mem/vector/VecCommon.scala b/src/main/scala/xiangshan/mem/vector/VecCommon.scala index a56f035be..327743895 100644 --- a/src/main/scala/xiangshan/mem/vector/VecCommon.scala +++ b/src/main/scala/xiangshan/mem/vector/VecCommon.scala @@ -290,6 +290,7 @@ class VecMemExuOutput(isVector: Boolean = false)(implicit p: Parameters) extends val mbIndex = UInt(vsmBindexBits.W) val mask = UInt(VLENB.W) val vaddr = UInt(VAddrBits.W) + val gpaddr = UInt(GPAddrBits.W) } object MulNum {