Support Tile Clusters (tree-like hierarchies)

src/main/scala/devices/debug/Custom.scala

@@ -5,8 +5,7 @@ package freechips.rocketchip.devices.debug
 import chisel3._
 import chisel3.util._
 import chisel3.experimental.SourceInfo
-import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp, NexusNode, RenderedEdge,
-  SimpleNodeImp, SinkNode, SourceNode, ValName}
+import freechips.rocketchip.diplomacy._
 import org.chipsalliance.cde.config.Parameters
 
 case class DebugCustomParams(
@@ -68,7 +67,7 @@ class DebugCustomXbar(
   )
 
   lazy val module = new Impl
-  class Impl extends LazyModuleImp(this) {
+  class Impl extends LazyRawModuleImp(this) {
     // require only one sink
     require(node.out.size == 1, "Must have exactly one sink node, not ${node.out.size}")
     // send address to all sources

src/main/scala/devices/debug/Debug.scala

@@ -693,7 +693,8 @@ class TLDebugModuleOuterAsync(device: Device)(implicit p: Parameters) extends La
   })
 
   val dmOuter = LazyModule( new TLDebugModuleOuter(device))
-  val intnode = IntSyncCrossingSource(alreadyRegistered = true) :*= dmOuter.intnode
+  val intnode = IntSyncIdentityNode()
+  intnode :*= IntSyncCrossingSource(alreadyRegistered = true) :*= dmOuter.intnode
 
   val dmiBypass = LazyModule(new TLBusBypass(beatBytes=4, bufferError=false, maxAtomic=0, maxTransfer=4))
   val dmiInnerNode = TLAsyncCrossingSource() := dmiBypass.node := dmiXbar.node
@@ -727,6 +728,7 @@ class TLDebugModuleOuterAsync(device: Device)(implicit p: Parameters) extends La
 
     childClock := io.dmi_clock
     childReset := io.dmi_reset
+    override def provideImplicitClockToLazyChildren = true
 
     withClockAndReset(childClock, childReset) {
       dmi2tlOpt.foreach { _.module.io.dmi <> io.dmi.get }
@@ -1898,6 +1900,7 @@ class TLDebugModuleInnerAsync(device: Device, getNComponents: () => Int, beatByt
 
     childClock := io.debug_clock
     childReset := io.debug_reset
+    override def provideImplicitClockToLazyChildren = true
 
     val dmactive_synced = withClockAndReset(childClock, childReset) {
       val dmactive_synced = AsyncResetSynchronizerShiftReg(in=io.dmactive, sync=3, name=Some("dmactiveSync"))
@@ -1986,6 +1989,7 @@ class TLDebugModule(beatBytes: Int)(implicit p: Parameters) extends LazyModule {
 
     childClock := io.tl_clock
     childReset := io.tl_reset
+    override def provideImplicitClockToLazyChildren = true
 
     dmOuter.module.io.dmi.foreach { dmOuterDMI =>
       dmOuterDMI <> io.dmi.get.dmi

src/main/scala/devices/debug/Periphery.scala

@@ -99,7 +99,7 @@ trait HasPeripheryDebug { this: BaseSubsystem =>
     tlDM
   }
 
-  lazy val debugNode = debugOpt.map(_.intnode).getOrElse(IntSyncXbar() := NullIntSyncSource())
+  val debugNode = debugOpt.map(_.intnode)
 
   val psd = InModuleBody {
     val psd = IO(new PSDIO)

src/main/scala/devices/tilelink/BootROM.scala

@@ -5,10 +5,9 @@ package freechips.rocketchip.devices.tilelink
 import chisel3._
 import chisel3.util.log2Ceil
 import org.chipsalliance.cde.config.{Field, Parameters}
-import freechips.rocketchip.subsystem.{BaseSubsystem, HierarchicalLocation, HasTiles, TLBusWrapperLocation}
+import freechips.rocketchip.subsystem._
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.tilelink._
-import freechips.rocketchip.prci.{ClockSinkDomain}
 
 import java.nio.ByteBuffer
 import java.nio.file.{Files, Paths}
@@ -66,11 +65,10 @@ object BootROM {
     *    at a configurable location, but also drives the tiles' reset vectors to point
     *    at its 'hang' address parameter value.
     */
-  def attach(params: BootROMParams, subsystem: BaseSubsystem with HasTiles, where: TLBusWrapperLocation)
+  def attach(params: BootROMParams, subsystem: BaseSubsystem with HasHierarchicalElements with HasTileInputConstants, where: TLBusWrapperLocation)
             (implicit p: Parameters): TLROM = {
     val tlbus = subsystem.locateTLBusWrapper(where)
-    val bootROMDomainWrapper = LazyModule(new ClockSinkDomain(take = None))
-    bootROMDomainWrapper.clockNode := tlbus.fixedClockNode
+    val bootROMDomainWrapper = tlbus.generateSynchronousDomain.suggestName("bootrom_domain")
 
     val bootROMResetVectorSourceNode = BundleBridgeSource[UInt]()
     lazy val contents = {

src/main/scala/devices/tilelink/CLINT.scala

@@ -102,18 +102,17 @@ class CLINT(params: CLINTParams, beatBytes: Int)(implicit p: Parameters) extends
 
 /** Trait that will connect a CLINT to a subsystem */
 trait CanHavePeripheryCLINT { this: BaseSubsystem =>
-  val clintOpt = p(CLINTKey).map { params =>
+  val (clintOpt, clintDomainOpt, clintTickOpt) = p(CLINTKey).map { params =>
     val tlbus = locateTLBusWrapper(p(CLINTAttachKey).slaveWhere)
-    val clint = LazyModule(new CLINT(params, cbus.beatBytes))
-    clint.node := tlbus.coupleTo("clint") { TLFragmenter(tlbus) := _ }
-
-    // Override the implicit clock and reset -- could instead include a clockNode in the clint, and make it a RawModuleImp?
-    InModuleBody {
-      clint.module.clock := tlbus.module.clock
-      clint.module.reset := tlbus.module.reset
-    }
-
-    clint
-
-  }
+    val clintDomainWrapper = tlbus.generateSynchronousDomain.suggestName("clint_domain")
+    val clint = clintDomainWrapper { LazyModule(new CLINT(params, cbus.beatBytes)) }
+    clintDomainWrapper { clint.node := tlbus.coupleTo("clint") { TLFragmenter(tlbus) := _ } }
+    val clintTick = clintDomainWrapper { InModuleBody {
+      val tick = IO(Input(Bool()))
+      clint.module.io.rtcTick := tick
+      tick
+    }}
+
+    (clint, clintDomainWrapper, clintTick)
+  }.unzip3
 }

src/main/scala/devices/tilelink/Plic.scala

@@ -12,7 +12,6 @@ import freechips.rocketchip.tilelink._
 import freechips.rocketchip.interrupts._
 import freechips.rocketchip.util._
 import freechips.rocketchip.util.property
-import freechips.rocketchip.prci.{ClockSinkDomain}
 import chisel3.experimental.SourceInfo
 
 import scala.math.min
@@ -355,15 +354,14 @@ class PLICFanIn(nDevices: Int, prioBits: Int) extends Module {
 
 /** Trait that will connect a PLIC to a subsystem */
 trait CanHavePeripheryPLIC { this: BaseSubsystem =>
-  val plicOpt  = p(PLICKey).map { params =>
+  val (plicOpt, plicDomainOpt) = p(PLICKey).map { params =>
     val tlbus = locateTLBusWrapper(p(PLICAttachKey).slaveWhere)
-    val plicDomainWrapper = LazyModule(new ClockSinkDomain(take = None))
-    plicDomainWrapper.clockNode := tlbus.fixedClockNode
+    val plicDomainWrapper = tlbus.generateSynchronousDomain
 
     val plic = plicDomainWrapper { LazyModule(new TLPLIC(params, tlbus.beatBytes)) }
-    plic.node := tlbus.coupleTo("plic") { TLFragmenter(tlbus) := _ }
-    plic.intnode :=* ibus.toPLIC
+    plicDomainWrapper { plic.node := tlbus.coupleTo("plic") { TLFragmenter(tlbus) := _ } }
+    plicDomainWrapper { plic.intnode :=* ibus.toPLIC }
 
-    plic
-  }
+    (plic, plicDomainWrapper)
+  }.unzip
 }

src/main/scala/diplomacy/BundleBridge.scala

@@ -124,7 +124,7 @@ class BundleBridgeNexus[T <: Data](
   val node = BundleBridgeNexusNode[T](default, inputRequiresOutput)
 
   lazy val module = new Impl
-  class Impl extends LazyModuleImp(this) {
+  class Impl extends LazyRawModuleImp(this) {
     val defaultWireOpt = default.map(_())
     val inputs: Seq[T] = node.in.map(_._1)
     inputs.foreach { i => require(DataMirror.checkTypeEquivalence(i, inputs.head),

src/main/scala/diplomacy/LazyModule.scala

@@ -415,7 +415,11 @@ class LazyRawModuleImp(val wrapper: LazyModule) extends RawModule with LazyModul
   // the default is that these are disabled
   childClock := false.B.asClock
   childReset := chisel3.DontCare
-  val (auto, dangles) = withClockAndReset(childClock, childReset) {
+
+  def provideImplicitClockToLazyChildren: Boolean = false
+  val (auto, dangles) = if (provideImplicitClockToLazyChildren) {
+    withClockAndReset(childClock, childReset) { instantiate() }
+  } else {
     instantiate()
   }
 }
@@ -427,6 +431,10 @@ class LazyRawModuleImp(val wrapper: LazyModule) extends RawModule with LazyModul
 class SimpleLazyModule(implicit p: Parameters) extends LazyModule {
   lazy val module = new LazyModuleImp(this)
 }
+class SimpleLazyRawModule(implicit p: Parameters) extends LazyModule {
+  lazy val module = new LazyRawModuleImp(this)
+}
+
 
 /** Allows dynamic creation of [[Module]] hierarchy and "shoving" logic into a [[LazyModule]]. */
 trait LazyScope {

src/main/scala/groundtest/Configs.scala

@@ -30,25 +30,24 @@ class GroundTestBaseConfig extends Config(
     case DebugModuleKey => None
     case CLINTKey => None
     case PLICKey => None
-    case SubsystemExternalResetVectorKey => true
+    case HasTilesExternalResetVectorKey => true
   })
 )
 
 class WithTraceGen(
   n: Int = 2,
-  overrideIdOffset: Option[Int] = None,
   overrideMemOffset: Option[BigInt] = None)(
   params: Seq[DCacheParams] = List.fill(n){ DCacheParams(nSets = 16, nWays = 1) },
   nReqs: Int = 8192
 ) extends Config((site, here, up) => {
   case TilesLocated(InSubsystem) => {
     val prev = up(TilesLocated(InSubsystem), site)
-    val idOffset = overrideIdOffset.getOrElse(prev.size)
+    val idOffset = up(NumTiles)
     val memOffset: BigInt = overrideMemOffset.orElse(site(ExtMem).map(_.master.base)).getOrElse(0x0L)
     params.zipWithIndex.map { case (dcp, i) =>
       TraceGenTileAttachParams(
         tileParams = TraceGenParams(
-          hartId = i + idOffset,
+          tileId = i + idOffset,
           dcache = Some(dcp),
           wordBits = site(XLen),
           addrBits = 32,
@@ -68,4 +67,5 @@ class WithTraceGen(
       )
     } ++ prev
   }
+  case NumTiles => up(NumTiles) + n
 })

src/main/scala/groundtest/GroundTestSubsystem.scala

@@ -4,15 +4,21 @@ package freechips.rocketchip.groundtest
 
 import chisel3._
 import org.chipsalliance.cde.config.{Parameters}
-import freechips.rocketchip.diplomacy.{AddressSet, LazyModule}
-import freechips.rocketchip.interrupts.{IntSinkNode, IntSinkPortSimple}
-import freechips.rocketchip.subsystem.{BaseSubsystem, BaseSubsystemModuleImp, HasTiles, CanHaveMasterAXI4MemPort}
+import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.interrupts._
+import freechips.rocketchip.tile.{NMI}
+import freechips.rocketchip.devices.tilelink.{CLINTConsts}
+import freechips.rocketchip.subsystem._
 import freechips.rocketchip.tilelink.{TLRAM, TLFragmenter}
 import freechips.rocketchip.interrupts.{NullIntSyncSource}
 
 class GroundTestSubsystem(implicit p: Parameters)
   extends BaseSubsystem
-  with HasTiles
+  with InstantiatesHierarchicalElements
+  with HasHierarchicalElementsRootContext
+  with HasHierarchicalElements
+  with HasTileNotificationSinks
+  with HasTileInputConstants
   with CanHaveMasterAXI4MemPort
 {
   val testram = LazyModule(new TLRAM(AddressSet(0x52000000, 0xfff), beatBytes=pbus.beatBytes))
@@ -24,16 +30,18 @@ class GroundTestSubsystem(implicit p: Parameters)
   // No PLIC in ground test; so just sink the interrupts to nowhere
   IntSinkNode(IntSinkPortSimple()) :=* ibus.toPLIC
 
-  val tileStatusNodes = tiles.collect { case t: GroundTestTile => t.statusNode.makeSink() }
-
-  // no debug module
-  val debugNode = NullIntSyncSource()
+  val tileStatusNodes = totalTiles.values.collect { case t: GroundTestTile => t.statusNode.makeSink() }
+  val clintOpt = None
+  val clintDomainOpt = None
+  val debugOpt = None
+  val plicOpt = None
+  val plicDomainOpt = None
 
   override lazy val module = new GroundTestSubsystemModuleImp(this)
 }
 
 class GroundTestSubsystemModuleImp[+L <: GroundTestSubsystem](_outer: L) extends BaseSubsystemModuleImp(_outer) {
   val success = IO(Output(Bool()))
-  val status = dontTouch(DebugCombiner(outer.tileStatusNodes.map(_.bundle)))
+  val status = dontTouch(DebugCombiner(outer.tileStatusNodes.map(_.bundle).toSeq))
   success := outer.tileCeaseSinkNode.in.head._1.asUInt.andR
 }

src/main/scala/groundtest/Tile.scala

@@ -34,7 +34,7 @@ abstract class GroundTestTile(
   with SourcesExternalNotifications
 {
   val cpuDevice: SimpleDevice = new SimpleDevice("groundtest", Nil)
-  val intOutwardNode: IntOutwardNode = IntIdentityNode()
+  val intOutwardNode = None
   val slaveNode: TLInwardNode = TLIdentityNode()
   val statusNode = BundleBridgeSource(() => new GroundTestStatus)
 

src/main/scala/groundtest/TraceGen.scala

@@ -26,7 +26,7 @@ import freechips.rocketchip.diplomacy.{ClockCrossingType}
 import freechips.rocketchip.rocket._
 import freechips.rocketchip.tile._
 import freechips.rocketchip.tilelink._
-import freechips.rocketchip.subsystem.{TileCrossingParamsLike, CanAttachTile}
+import freechips.rocketchip.subsystem.{HierarchicalElementCrossingParamsLike, CanAttachTile}
 import freechips.rocketchip.util._
 import freechips.rocketchip.prci.{ClockSinkParameters}
 
@@ -68,15 +68,15 @@ case class TraceGenParams(
     memStart: BigInt, //p(ExtMem).base
     numGens: Int,
     dcache: Option[DCacheParams] = Some(DCacheParams()),
-    hartId: Int = 0
+    tileId: Int = 0
 ) extends InstantiableTileParams[TraceGenTile] with GroundTestTileParams
 {
-  def instantiate(crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): TraceGenTile = {
+  def instantiate(crossing: HierarchicalElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): TraceGenTile = {
     new TraceGenTile(this, crossing, lookup)
   }
-  val beuAddr = None
   val blockerCtrlAddr = None
-  val name = None
+  val baseName = "tracegentile"
+  val uniqueName = s"${baseName}_$tileId"
   val clockSinkParams = ClockSinkParameters()
 }
 
@@ -105,7 +105,7 @@ trait HasTraceGenParams {
 
 case class TraceGenTileAttachParams(
   tileParams: TraceGenParams,
-  crossingParams: TileCrossingParamsLike
+  crossingParams: HierarchicalElementCrossingParamsLike
 ) extends CanAttachTile {
   type TileType = TraceGenTile
   val lookup: LookupByHartIdImpl = HartsWontDeduplicate(tileParams)
@@ -617,7 +617,7 @@ class TraceGenTile private(
   q: Parameters
 ) extends GroundTestTile(params, crossing, lookup, q)
 {
-  def this(params: TraceGenParams, crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
+  def this(params: TraceGenParams, crossing: HierarchicalElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
     this(params, crossing.crossingType, lookup, p)
 
   val masterNode: TLOutwardNode = TLIdentityNode() := visibilityNode := dcacheOpt.map(_.node).getOrElse(TLTempNode())
@@ -644,5 +644,5 @@ class TraceGenTileModuleImp(outer: TraceGenTile) extends GroundTestTileModuleImp
   status.timeout.bits := 0.U
   status.error.valid := false.B
 
-  assert(!tracegen.io.timeout, s"TraceGen tile ${outer.tileParams.hartId}: request timed out")
+  assert(!tracegen.io.timeout, s"TraceGen tile ${outer.tileParams.tileId}: request timed out")
 }

src/main/scala/interrupts/Crossing.scala

@@ -35,14 +35,16 @@ class IntSyncCrossingSource(alreadyRegistered: Boolean = false)(implicit p: Para
 {
   val node = IntSyncSourceNode(alreadyRegistered)
 
-  lazy val module = new Impl
+  lazy val module = if (alreadyRegistered) (new ImplRegistered) else (new Impl)
+
   class Impl extends LazyModuleImp(this) {
     (node.in zip node.out) foreach { case ((in, edgeIn), (out, edgeOut)) =>
-      if (alreadyRegistered) {
-        out.sync := in
-      } else {
-        out.sync := AsyncResetReg(Cat(in.reverse)).asBools
-      }
+      out.sync := AsyncResetReg(Cat(in.reverse)).asBools
+    }
+  }
+  class ImplRegistered extends LazyRawModuleImp(this) {
+    (node.in zip node.out) foreach { case ((in, edgeIn), (out, edgeOut)) =>
+      out.sync := in
     }
   }
 }
@@ -85,7 +87,7 @@ class IntSyncSyncCrossingSink()(implicit p: Parameters) extends LazyModule
   val node = IntSyncSinkNode(0)
 
   lazy val module = new Impl
-  class Impl extends LazyModuleImp(this) {
+  class Impl extends LazyRawModuleImp(this) {
     (node.in zip node.out) foreach { case ((in, edgeIn), (out, edgeOut)) =>
       out := in.sync
     }

src/main/scala/interrupts/NullIntSource.scala

@@ -12,7 +12,7 @@ class NullIntSource(num: Int = 1, ports: Int = 1, sources: Int = 1)(implicit p:
   val intnode = IntSourceNode(IntSourcePortSimple(num, ports, sources))
 
   lazy val module = new Impl
-  class Impl extends LazyModuleImp(this) {
+  class Impl extends LazyRawModuleImp(this) {
     intnode.out.foreach { case (o, _) => o.foreach { _ := false.B } }
   }
 }
@@ -26,6 +26,6 @@ object NullIntSource {
 
 object NullIntSyncSource {
   def apply(num: Int = 1, ports: Int = 1, sources: Int = 1)(implicit p: Parameters): IntSyncOutwardNode = {
-    IntSyncCrossingSource() := NullIntSource(num, ports, sources)
+    IntSyncCrossingSource(alreadyRegistered = true) := NullIntSource(num, ports, sources)
   }
 }

src/main/scala/interrupts/Xbar.scala

@@ -19,7 +19,7 @@ class IntXbar()(implicit p: Parameters) extends LazyModule
   }
 
   lazy val module = new Impl
-  class Impl extends LazyModuleImp(this) {
+  class Impl extends LazyRawModuleImp(this) {
     val cat = intnode.in.map { case (i, e) => i.take(e.source.num) }.flatten
     intnode.out.foreach { case (o, _) => o := cat }
   }

src/main/scala/prci/ClockBundles.scala

@@ -5,7 +5,7 @@ import chisel3._
 import freechips.rocketchip.util.RecordMap
 
 
-class ClockBundle(val params: ClockBundleParameters) extends Bundle
+class ClockBundle(val params: ClockBundleParameters = ClockBundleParameters()) extends Bundle
 {
   val clock = Output(Clock())
   val reset = Output(Reset())