root
/
XiangShan
mirror of https://github.com/OpenXiangShan/XiangShan.git
1
0
Fork 0
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge d8433a5a73 into 1e7e38e249

This commit is contained in:
cz4e 2025-04-22 10:05:12 +08:00 committed by GitHub
parent 1e7e38e249 d8433a5a73
commit 910771157b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 81 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/scala/xiangshan/cache/dcache/DCacheWrapper.scala vendored
View File

@ -394,6 +394,7 @@ class DCacheLineReq(implicit p: Parameters) extends DCacheBundle
val data = UInt((cfg.blockBytes * 8).W)
val mask = UInt(cfg.blockBytes.W)
val id = UInt(reqIdWidth.W)
val grow_perm_fail = Bool()
def dump(cond: Bool) = {
XSDebug(cond, "DCacheLineReq: cmd: %x addr: %x data: %x mask: %x id: %d\n",
cmd, addr, data, mask, id)
@ -484,6 +485,7 @@ class DCacheLineResp(implicit p: Parameters) extends DCacheBundle
// cache req nacked, replay it later
val replay = Bool()
val id = UInt(reqIdWidth.W)
val grow_perm_fail = Bool()
def dump(cond: Bool) = {
XSDebug(cond, "DCacheLineResp: data: %x id: %d miss: %b replay: %b\n",
data, id, miss, replay)
@ -589,6 +591,7 @@ class MainPipeResp(implicit p: Parameters) extends DCacheBundle {
val ack_miss_queue = Bool()
val id = UInt(reqIdWidth.W)
val grow_perm_fail = Bool()
def isAMO: Bool = source === AMO_SOURCE.U
def isStore: Bool = source === STORE_SOURCE.U
@ -1516,6 +1519,8 @@ class DCacheImp(outer: DCache) extends LazyModuleImp(outer) with HasDCacheParame
bus.e <> missQueue.io.mem_finish
missQueue.io.probe_addr := bus.b.bits.address
missQueue.io.replace_addr := mainPipe.io.replace_addr
missQueue.io.grow_perm_addr := mainPipe.io.grow_perm_addr
missQueue.io.BtoT_ways_for_set <> mainPipe.io.BtoT_ways_for_set
missQueue.io.main_pipe_resp.valid := RegNext(mainPipe.io.atomic_resp.valid)
missQueue.io.main_pipe_resp.bits := RegEnable(mainPipe.io.atomic_resp.bits, mainPipe.io.atomic_resp.valid)

40
src/main/scala/xiangshan/cache/dcache/mainpipe/MainPipe.scala vendored
View File

@ -73,6 +73,8 @@ class MainPipeReq(implicit p: Parameters) extends DCacheBundle {
val id = UInt(reqIdWidth.W)
val grow_perm_fail = Bool()
def isLoad: Bool = source === LOAD_SOURCE.U
def isStore: Bool = source === STORE_SOURCE.U
def isAMO: Bool = source === AMO_SOURCE.U
@ -95,6 +97,7 @@ class MainPipeReq(implicit p: Parameters) extends DCacheBundle {
req.replace := false.B
req.error := false.B
req.id := store.id
req.grow_perm_fail := store.grow_perm_fail
req
}
}
@ -173,6 +176,9 @@ class MainPipe(implicit p: Parameters) extends DCacheModule with HasPerfEvents w
// find the way to be replaced
val replace_way = new ReplacementWayReqIO
val grow_perm_addr = Output(UInt(PAddrBits.W))
val BtoT_ways_for_set = Input(UInt(nWays.W))
// writeback addr to be replaced
val replace_addr = ValidIO(UInt(PAddrBits.W))
val replace_block = Input(Bool())
@ -359,13 +365,15 @@ class MainPipe(implicit p: Parameters) extends DCacheModule with HasPerfEvents w
XSPerfAccumulate("replace_unused_prefetch", s1_req.replace && isFromL1Prefetch(s1_extra_meta.prefetch) && !s1_extra_meta.access) // may not be accurate
// replacement policy
val alt_replacer = ReplacementPolicy.fromString("random", nWays)
val s1_invalid_vec = wayMap(w => !meta_resp(w).asTypeOf(new Meta).coh.isValid())
val s1_have_invalid_way = s1_invalid_vec.asUInt.orR
val s1_invalid_way_en = ParallelPriorityMux(s1_invalid_vec.zipWithIndex.map(x => x._1 -> UIntToOH(x._2.U(nWays.W))))
val s1_mux_repl_way = Mux(s1_req.grow_perm_fail, alt_replacer.way, io.replace_way.way)
val s1_repl_way_en = WireInit(0.U(nWays.W))
s1_repl_way_en := Mux(
GatedValidRegNext(s0_fire),
UIntToOH(io.replace_way.way),
UIntToOH(s1_mux_repl_way),
RegEnable(s1_repl_way_en, s1_valid)
)
val s1_repl_tag = ParallelMux(s1_repl_way_en.asBools, (0 until nWays).map(w => tag_resp(w)))
@ -373,7 +381,7 @@ class MainPipe(implicit p: Parameters) extends DCacheModule with HasPerfEvents w
val s1_repl_pf = ParallelMux(s1_repl_way_en.asBools, (0 until nWays).map(w => io.extra_meta_resp(w).prefetch))
val s1_repl_way_raw = WireInit(0.U(log2Up(nWays).W))
s1_repl_way_raw := Mux(GatedValidRegNext(s0_fire), io.replace_way.way, RegEnable(s1_repl_way_raw, s1_valid))
s1_repl_way_raw := Mux(GatedValidRegNext(s0_fire), s1_mux_repl_way, RegEnable(s1_repl_way_raw, s1_valid))
val s1_need_replacement = s1_req.miss && !s1_tag_match
val s1_need_eviction = s1_req.miss && !s1_tag_match && s1_repl_coh.state =/= ClientStates.Nothing
@ -389,9 +397,10 @@ class MainPipe(implicit p: Parameters) extends DCacheModule with HasPerfEvents w
XSPerfAccumulate("store_has_invalid_way_but_select_valid_way", io.replace_way.set.valid && wayMap(w => !meta_resp(w).asTypeOf(new Meta).coh.isValid()).asUInt.orR && s1_need_replacement && s1_repl_coh.isValid())
XSPerfAccumulate("store_using_replacement", io.replace_way.set.valid && s1_need_replacement)
val s1_has_permission = s1_hit_coh.onAccess(s1_req.cmd)._1
val (s1_has_permission, s1_shrink_perm, _) = s1_hit_coh.onAccess(s1_req.cmd)
val s1_hit = s1_tag_match && s1_has_permission
val s1_pregen_can_go_to_mq = !s1_req.replace && !s1_req.probe && !s1_req.miss && (s1_req.isStore || s1_req.isAMO && s1_req.cmd =/= M_XSC) && !s1_hit
val s1_store_or_amo = !s1_req.replace && !s1_req.probe && !s1_req.miss && (s1_req.isStore || s1_req.isAMO && s1_req.cmd =/= M_XSC)
val s1_pregen_can_go_to_mq = s1_store_or_amo && !s1_hit
// s2: select data, return resp if this is a store miss
val s2_valid = RegInit(false.B)
@ -399,8 +408,9 @@ class MainPipe(implicit p: Parameters) extends DCacheModule with HasPerfEvents w
val s2_tag_errors = RegEnable(s1_tag_errors, s1_fire)
val s2_tag_match = RegEnable(s1_tag_match, s1_fire)
val s2_tag_match_way = RegEnable(s1_real_tag_match_way, s1_fire)
val s2_tag_ecc_match_way = RegEnable(s1_tag_ecc_match_way, s1_fire)
val s2_hit_coh = RegEnable(s1_hit_coh, s1_fire)
val (s2_has_permission, _, s2_new_hit_coh) = s2_hit_coh.onAccess(s2_req.cmd)
val (s2_has_permission, s2_shrink_perm, s2_new_hit_coh) = s2_hit_coh.onAccess(s2_req.cmd)
val s2_repl_tag = RegEnable(s1_repl_tag, s1_fire)
val s2_repl_coh = RegEnable(s1_repl_coh, s1_fire)
@ -436,6 +446,14 @@ class MainPipe(implicit p: Parameters) extends DCacheModule with HasPerfEvents w
s2_s0_set_conlict := s2_valid && s0_idx === s2_idx
s2_s0_set_conlict_store := s2_valid && store_idx === s2_idx
// Grow permission fail
// Only in case BtoT will both cache and missqueue be occupied
val s2_grow_perm = s2_shrink_perm === BtoT && !s2_has_permission
val s2_grow_perm_fail = PopCount(io.BtoT_ways_for_set) > (nWays-2).U && s2_grow_perm
XSError(s2_valid && s2_grow_perm && io.BtoT_ways_for_set.andR,
"BtoT grow permission, but all ways are BtoT\n"
)
// For a store req, it either hits and goes to s3, or miss and enter miss queue immediately
val s2_req_miss_without_data = Mux(s2_valid, s2_req.miss && !io.refill_info.valid, false.B)
val s2_can_go_to_mq_replay = (s2_req_miss_without_data && RegEnable(s2_req_miss_without_data && !io.mainpipe_info.s2_replay_to_mq, false.B, s2_valid)) || io.replace_block // miss_req in s2 but refill data is invalid, can block 1 cycle
@ -802,21 +820,25 @@ class MainPipe(implicit p: Parameters) extends DCacheModule with HasPerfEvents w
miss_req.cancel := false.B
miss_req.pc := DontCare
miss_req.full_overwrite := s2_req.isStore && s2_req.store_mask.andR
miss_req.isBtoT := s2_grow_perm
miss_req.occupy_way := s2_tag_ecc_match_way
io.wbq_conflict_check.valid := s2_valid && s2_can_go_to_mq
io.wbq_conflict_check.bits := s2_req.addr
io.store_replay_resp.valid := s2_valid && s2_can_go_to_mq && replay && s2_req.isStore
io.store_replay_resp.valid := s2_valid && (s2_can_go_to_mq && replay || s2_grow_perm_fail) && s2_req.isStore
io.store_replay_resp.bits.data := DontCare
io.store_replay_resp.bits.miss := true.B
io.store_replay_resp.bits.replay := true.B
io.store_replay_resp.bits.id := s2_req.id
io.store_replay_resp.bits.grow_perm_fail := s2_grow_perm_fail
io.store_hit_resp.valid := s3_valid && (s3_store_can_go || (s3_miss_can_go && s3_req.isStore))
io.store_hit_resp.bits.data := DontCare
io.store_hit_resp.bits.miss := false.B
io.store_hit_resp.bits.replay := false.B
io.store_hit_resp.bits.id := s3_req.id
io.store_hit_resp.bits.grow_perm_fail := false.B
val atomic_hit_resp = Wire(new MainPipeResp)
atomic_hit_resp.source := s3_req.source
@ -827,6 +849,7 @@ class MainPipe(implicit p: Parameters) extends DCacheModule with HasPerfEvents w
atomic_hit_resp.replay := false.B
atomic_hit_resp.ack_miss_queue := s3_req.miss
atomic_hit_resp.id := lrsc_valid
atomic_hit_resp.grow_perm_fail := false.B
val atomic_replay_resp = Wire(new MainPipeResp)
atomic_replay_resp.source := s2_req.source
atomic_replay_resp.data := DontCare
@ -836,8 +859,9 @@ class MainPipe(implicit p: Parameters) extends DCacheModule with HasPerfEvents w
atomic_replay_resp.replay := true.B
atomic_replay_resp.ack_miss_queue := false.B
atomic_replay_resp.id := DontCare
atomic_replay_resp.grow_perm_fail := s2_grow_perm_fail
val atomic_replay_resp_valid = s2_valid && s2_can_go_to_mq && replay && s2_req.isAMO
val atomic_replay_resp_valid = s2_valid && (s2_can_go_to_mq && replay || s2_grow_perm_fail) && s2_req.isAMO
val atomic_hit_resp_valid = s3_valid && (s3_amo_can_go || s3_miss_can_go && s3_req.isAMO)
io.atomic_resp.valid := atomic_replay_resp_valid || atomic_hit_resp_valid
@ -898,6 +922,8 @@ class MainPipe(implicit p: Parameters) extends DCacheModule with HasPerfEvents w
io.replace_addr.valid := s2_valid && s2_need_eviction
io.replace_addr.bits := get_block_addr(Cat(s2_tag, get_untag(s2_req.vaddr)))
io.grow_perm_addr := get_block_addr(s2_req.addr)
assert(!RegNext(io.tag_write.valid && !io.tag_write_intend))
io.data_write.valid := s3_valid && s3_update_data_cango && update_data

34
src/main/scala/xiangshan/cache/dcache/mainpipe/MissQueue.scala vendored
View File

@ -65,6 +65,17 @@ class MissReqWoStoreData(implicit p: Parameters) extends DCacheBundle {
val req_coh = new ClientMetadata
val id = UInt(reqIdWidth.W)
/**
* isBtoT is used to mark whether the current request requires BtoT permission.
* If so, other requests for BtoT in the same set are blocked. Otherwise,
* they are not blocked.
*/
val isBtoT = Bool()
/**
* The way isBtoT requests to occupy
*/
val occupy_way = UInt(log2Up(nWays).W)
// For now, miss queue entry req is actually valid when req.valid && !cancel
// * req.valid is fast to generate
// * cancel is slow to generate, it will not be used until the last moment
@ -271,6 +282,10 @@ class MissReqPipeRegBundle(edge: TLEdgeOut)(implicit p: Parameters) extends DCac
def block_match(release_addr: UInt): Bool = {
reg_valid() && get_block(req.addr) === get_block(release_addr)
}
def grow_perm_set_match(grow_perm_addr: UInt): Bool = {
reg_valid() && get_idx(req.addr) === get_idx(grow_perm_addr)
}
}
class CMOUnit(edge: TLEdgeOut)(implicit p: Parameters) extends DCacheModule {
@ -379,6 +394,7 @@ class MissEntry(edge: TLEdgeOut, reqNum: Int)(implicit p: Parameters) extends DC
val refill_info = ValidIO(new MissQueueRefillInfo)
val block_addr = ValidIO(UInt(PAddrBits.W))
val occupy_way = Output(UInt(nWays.W))
val req_addr = ValidIO(UInt(PAddrBits.W))
@ -501,8 +517,10 @@ class MissEntry(edge: TLEdgeOut, reqNum: Int)(implicit p: Parameters) extends DC
req_valid := true.B
req := miss_req_pipe_reg_bits.toMissReqWoStoreData()
req_primary_fire := miss_req_pipe_reg_bits.toMissReqWoStoreData()
req.isBtoT := DontCare
req.occupy_way := Mux(miss_req_pipe_reg_bits.isBtoT, 0.U, miss_req_pipe_reg_bits.occupy_way)
req.addr := get_block_addr(miss_req_pipe_reg_bits.addr)
req_primary_fire := miss_req_pipe_reg_bits.toMissReqWoStoreData()
//only load miss need keyword
isKeyword := Mux(miss_req_pipe_reg_bits.isFromLoad, miss_req_pipe_reg_bits.vaddr(5).asBool,false.B)
@ -553,6 +571,8 @@ class MissEntry(edge: TLEdgeOut, reqNum: Int)(implicit p: Parameters) extends DC
when (miss_req_pipe_reg_bits.isFromStore) {
req := miss_req_pipe_reg_bits
req.isBtoT := DontCare
req.occupy_way := Mux(miss_req_pipe_reg_bits.isBtoT, 0.U, miss_req_pipe_reg_bits.occupy_way)
req.addr := get_block_addr(miss_req_pipe_reg_bits.addr)
req_store_mask := miss_req_pipe_reg_bits.store_mask
for (i <- 0 until blockRows) {
@ -818,6 +838,7 @@ class MissEntry(edge: TLEdgeOut, reqNum: Int)(implicit p: Parameters) extends DC
io.main_pipe_req.bits.id := req.id
io.main_pipe_req.bits.pf_source := req.pf_source
io.main_pipe_req.bits.access := access
io.main_pipe_req.bits.grow_perm_fail := false.B
io.block_addr.valid := req_valid && w_grantlast
io.block_addr.bits := req.addr
@ -825,6 +846,8 @@ class MissEntry(edge: TLEdgeOut, reqNum: Int)(implicit p: Parameters) extends DC
io.req_addr.valid := req_valid
io.req_addr.bits := req.addr
io.occupy_way := req.occupy_way
io.refill_info.valid := req_valid && w_grantlast
io.refill_info.bits.store_data := refill_and_store_data.asUInt
io.refill_info.bits.store_mask := ~0.U(blockBytes.W)
@ -928,6 +951,10 @@ class MissQueue(edge: TLEdgeOut, reqNum: Int)(implicit p: Parameters) extends DC
val replace_addr = Flipped(ValidIO(UInt(PAddrBits.W)))
val replace_block = Output(Bool())
// block all way for set to BtoT
val grow_perm_addr = Flipped(UInt(PAddrBits.W))
val BtoT_ways_for_set = Output(UInt(nWays.W))
// req blocked by wbq
val wbq_block_miss_req = Input(Bool())
@ -1150,6 +1177,11 @@ class MissQueue(edge: TLEdgeOut, reqNum: Int)(implicit p: Parameters) extends DC
io.probe_block := Cat(probe_block_vec).orR
io.replace_block := io.replace_addr.valid && Cat(entries.map(e => e.io.req_addr.valid && e.io.req_addr.bits === io.replace_addr.bits) ++ Seq(miss_req_pipe_reg.block_match(io.replace_addr.bits))).orR
val grow_perm_set_hit = entries.map(e => e.io.req_addr.valid && get_idx(e.io.req_addr.bits) === get_idx(io.grow_perm_addr)) ++
Seq(miss_req_pipe_reg.grow_perm_set_match(io.grow_perm_addr))
io.BtoT_ways_for_set := grow_perm_set_hit.zip(entries).map {
case (hit, e) => Fill(nWays, hit) & e.io.occupy_way
}.reduce(_|_)
io.full := ~Cat(entries.map(_.io.primary_ready)).andR

1
src/main/scala/xiangshan/cache/dcache/mainpipe/Probe.scala vendored
View File

@ -105,6 +105,7 @@ class ProbeEntry(implicit p: Parameters) extends DCacheModule {
pipe_req.probe_need_data := req.needData
pipe_req.error := false.B
pipe_req.id := io.id
pipe_req.grow_perm_fail := false.B
when (io.pipe_req.fire) {
state := s_wait_resp

5
src/main/scala/xiangshan/mem/pipeline/AtomicsUnit.scala
View File

@ -80,6 +80,7 @@ class AtomicsUnit(implicit p: Parameters) extends XSModule
// `pdest2` is used to record the pdest of the second uop
val pdest1, pdest2 = Reg(UInt(PhyRegIdxWidth.W))
val pdest1Valid, pdest2Valid = RegInit(false.B)
val grow_perm_fail = RegInit(false.B)
/**
* The # of std uops that an atomic instruction require:
* (1) For AMOs (except AMOCAS) and LR/SC, 1 std uop is wanted: X(rs2) with uopIdx = 0
@ -140,6 +141,7 @@ class AtomicsUnit(implicit p: Parameters) extends XSModule
rs1 := io.in.bits.src_rs1
state := s_tlb_and_flush_sbuffer_req
have_sent_first_tlb_req := false.B
grow_perm_fail := false.B
}
}
@ -358,6 +360,9 @@ class AtomicsUnit(implicit p: Parameters) extends XSModule
when (io.dcache.resp.bits.miss) {
when (io.dcache.resp.bits.replay) {
state := s_cache_req
grow_perm_fail := true.B
} .otherwise {
grow_perm_fail := false.B
}
}.otherwise {
dcache_resp_data := io.dcache.resp.bits.data

4
src/main/scala/xiangshan/mem/sbuffer/Sbuffer.scala
View File

@ -220,6 +220,7 @@ class Sbuffer(implicit p: Parameters)
val stateVec = RegInit(VecInit(Seq.fill(StoreBufferSize)(0.U.asTypeOf(new SbufferEntryState))))
val cohCount = RegInit(VecInit(Seq.fill(StoreBufferSize)(0.U(EvictCountBits.W))))
val missqReplayCount = RegInit(VecInit(Seq.fill(StoreBufferSize)(0.U(MissqReplayCountBits.W))))
val grow_perm_fail = RegInit(VecInit(Seq.fill(StoreBufferSize)(false.B)))
val sbuffer_out_s0_fire = Wire(Bool())
@ -437,6 +438,7 @@ class Sbuffer(implicit p: Parameters)
// missqReplayCount(insertIdx) := 0.U
ptag(entryIdx) := reqptag
vtag(entryIdx) := reqvtag // update vtag if a new sbuffer line is allocated
grow_perm_fail(entryIdx) := false.B
}
})
}
@ -691,6 +693,7 @@ class Sbuffer(implicit p: Parameters)
io.dcache.req.bits.data := data(sbuffer_out_s1_evictionIdx).asUInt
io.dcache.req.bits.mask := mask(sbuffer_out_s1_evictionIdx).asUInt
io.dcache.req.bits.id := sbuffer_out_s1_evictionIdx
io.dcache.req.bits.grow_perm_fail := grow_perm_fail(sbuffer_out_s1_evictionIdx)
XSDebug(sbuffer_out_s1_fire,
p"send buf [$sbuffer_out_s1_evictionIdx] to Dcache, req fire\n"
@ -741,6 +744,7 @@ class Sbuffer(implicit p: Parameters)
when (io.dcache.replay_resp.fire) {
missqReplayCount(replay_resp_id) := 0.U
stateVec(replay_resp_id).w_timeout := true.B
grow_perm_fail(replay_resp_id) := io.dcache.replay_resp.bits.grow_perm_fail
// waiting for timeout
assert(io.dcache.replay_resp.bits.replay)
assert(stateVec(replay_resp_id).state_inflight === true.B)