add benchmark project

.github/workflows/test.yml

@@ -47,6 +47,26 @@ jobs:
         timeout-minutes: 6
         run: sbt ++${{ matrix.scala }} "testOnly integration.**"
 
+  benchmarks:
+    name: Benchmarks
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        scala: [ 2.13.10 ]
+        jvm: [ 8, 11, 21 ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Scala
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'adopt'
+          java-version: ${{ matrix.jvm }}
+      - name: Check for Warnings
+        run: sbt "project benchmark ; set ThisBuild / scalacOptions ++= Seq(\"-Xfatal-warnings\") ; compile"
+      - name: Check Formatting
+        run: sbt "project benchmark ; scalafmtCheckAll"
+
 
   test-mac:
     name: sbt test on mac
@@ -206,7 +226,7 @@ jobs:
   # When adding new jobs, please add them to `needs` below
   all_tests_passed:
     name: "all tests passed"
-    needs: [test, doc, verilator, formal, formal-mac, icarus, test-mac, no-warn, integration-test]
+    needs: [test, doc, verilator, formal, formal-mac, icarus, test-mac, no-warn, integration-test, benchmarks]
     runs-on: ubuntu-latest
     steps:
       - run: echo Success!
@@ -215,7 +235,7 @@ jobs:
   # separate from a Scala versions build matrix to avoid duplicate publishing
   publish:
     # note: we do not require a warning check for publishing!
-    needs: [test, doc, verilator, formal, formal-mac, icarus, test-mac, test-treadle, integration-test]
+    needs: [test, doc, verilator, formal, formal-mac, icarus, test-mac, test-treadle, integration-test, benchmarks]
     runs-on: ubuntu-20.04
     if: github.event_name == 'push'
 

benchmark/.gitignore

@@ -0,0 +1,4 @@
+/project/target
+/project/project
+/target/
+/benchmark.jar

benchmark/src/fsim/Benchmark.scala

@@ -0,0 +1,124 @@
+// Copyright 2023 The Regents of the University of California
+// released under BSD 3-Clause License
+// author: Kevin Laeufer <[email protected]>
+
+package fsim
+
+import firrtl2.options.Dependency
+import firrtl2.stage.Forms
+import scopt.OptionParser
+import treadle2.TreadleTester
+
+case class Config(benches: Seq[String], sim: String, warmupRun: Boolean)
+
+class ArgumentParser extends OptionParser[Config]("synthesizer") {
+  head("fsim benchmark", "0.1")
+  opt[String]("bench").action((a, config) => config.copy(benches = config.benches :+ a))
+  opt[String]("sim").action((a, config) => config.copy(sim = a))
+}
+
+case class Bench(
+  name:           String,
+  getCircuit:     String => String,
+  runTest:        Simulation => Unit,
+  runTreadleTest: TreadleTester => Unit)
+case class Result(nsCompile: Long, nsRun: Long, steps: Int)
+object Benchmark {
+  val Benches = Seq(
+    Bench("gcd_16", _ => GCDBench.circuitSrc(16), GCDBench.fsimTest(_, 10, 500), GCDBench.treadleTest(_, 10, 500)),
+    Bench("gcd_44", _ => GCDBench.circuitSrc(44), GCDBench.fsimTest(_, 10, 500), GCDBench.treadleTest(_, 10, 500)),
+    Bench("gcd_64", _ => GCDBench.circuitSrc(64), GCDBench.fsimTest(_, 10, 500), GCDBench.treadleTest(_, 10, 500))
+  )
+
+  private val DefaultConfig = Config(benches = Seq("gcd_64"), sim = "fsim", warmupRun = true)
+  def main(args: Array[String]): Unit = {
+    val parser = new ArgumentParser()
+    val conf = parser.parse(args, DefaultConfig).get
+    conf.benches.foreach { benchName =>
+      val bench =
+        Benches.find(_.name == benchName).getOrElse(throw new RuntimeException(s"Unknown benchmark: $benchName"))
+      val res = runBench(conf, bench)
+      printResult(bench.name, conf.sim, res)
+    }
+
+  }
+
+  def printResult(bench: String, sim: String, res: Result): Unit = {
+    println(
+      s"${bench} on ${sim}: " +
+        s"${secondString(res.nsRun)}, ${res.steps} cycles, ${freqString(res.nsRun, res.steps)}, " +
+        s"${secondString(res.nsCompile)} to compile"
+    )
+  }
+
+  private def secondString(ns: Long): String = {
+    val elapsedSeconds = ns.toDouble / Giga
+    f"$elapsedSeconds%.6fs"
+  }
+
+  private val Kilo: Double = 1000.0
+  private val Mega: Double = 1000000.0
+  private val Giga: Double = 1000000000.0
+  private def freqString(ns: Long, steps: Int): String = {
+    val elapsedSeconds = ns.toDouble / Giga
+    val hz = steps.toDouble / elapsedSeconds
+    if (hz > Giga) {
+      f"${hz / Giga}%.6fGHz"
+    } else if (hz > Mega) {
+      f"${hz / Mega}%.6fMHz"
+    } else if (hz > Kilo) {
+      f"${hz / Kilo}%.6fkHz"
+    } else {
+      f"${hz}%.6fHz"
+    }
+  }
+
+  def runBench(conf: Config, bench: Bench): Result = {
+    val src = bench.getCircuit(conf.sim)
+    conf.sim match {
+      case "fsim"    => runFSimBench(conf, bench, src)
+      case "treadle" => runTreadleBench(conf, bench, src)
+      case other     => throw new RuntimeException(s"Unsupported simulator: $other")
+    }
+  }
+
+  private def runFSimBench(conf: Config, bench: Bench, src: String): Result = {
+    val compileStart = System.nanoTime()
+    val sim = new Simulation(Compiler.run(FirrtlCompiler.toLow(src)))
+    val compileEnd = System.nanoTime()
+    if (conf.warmupRun) {
+      bench.runTest(sim)
+    }
+    val testStart = System.nanoTime()
+    bench.runTest(sim)
+    val testEnd = System.nanoTime()
+    val steps = sim.getStepCount
+    // TODO: shut down sim
+    Result(nsCompile = compileEnd - compileStart, nsRun = testEnd - testStart, steps = steps)
+  }
+
+  private def runTreadleBench(conf: Config, bench: Bench, src: String): Result = {
+    val compileStart = System.nanoTime()
+    val sim = new Simulation(Compiler.run(FirrtlCompiler.toLow(src)))
+    val compileEnd = System.nanoTime()
+    if (conf.warmupRun) {
+      bench.runTest(sim)
+    }
+    val testStart = System.nanoTime()
+    bench.runTest(sim)
+    val testEnd = System.nanoTime()
+    val steps = sim.getStepCount
+    // TODO: shut down sim
+    Result(nsCompile = compileEnd - compileStart, nsRun = testEnd - testStart, steps = steps)
+  }
+}
+
+object FirrtlCompiler {
+  private val loFirrtlCompiler =
+    new firrtl2.stage.transforms.Compiler(Seq(Dependency[firrtl2.LowFirrtlEmitter]) ++ Forms.LowFormOptimized)
+  def toLow(src: String): firrtl2.ir.Circuit = {
+    val hi = firrtl2.Parser.parse(src)
+    val lo = loFirrtlCompiler.execute(firrtl2.CircuitState(hi))
+    lo.circuit
+  }
+}

benchmark/src/fsim/GCDBench.scala

@@ -0,0 +1,96 @@
+// Copyright 2023 The Regents of the University of California
+// released under BSD 3-Clause License
+// author: Kevin Laeufer <[email protected]>
+
+package fsim
+
+import treadle2.TreadleTester
+
+object GCDBench {
+  def circuitSrc(width: Int): String =
+    s"""
+       |circuit GCD :
+       |  module GCD :
+       |    input clock : Clock
+       |    input reset : UInt<1>
+       |    input io_a : UInt<$width>
+       |    input io_b : UInt<$width>
+       |    input io_e : UInt<1>
+       |    output io_z : UInt<$width>
+       |    output io_v : UInt<1>
+       |    reg x : UInt<$width>, clock with :
+       |      reset => (UInt<1>("h0"), x)
+       |    reg y : UInt<$width>, clock with :
+       |      reset => (UInt<1>("h0"), y)
+       |    node T_13 = gt(x, y)
+       |    node T_14 = sub(x, y)
+       |    node T_15 = tail(T_14, 1)
+       |    node T_17 = eq(T_13, UInt<1>("h0"))
+       |    node T_18 = sub(y, x)
+       |    node T_19 = tail(T_18, 1)
+       |    node T_21 = eq(y, UInt<1>("h0"))
+       |    node GEN_0 = mux(T_13, T_15, x)
+       |    x <= mux(io_e, io_a, GEN_0)
+       |    node GEN_1 = mux(T_17, T_19, y)
+       |    y <= mux(io_e, io_b, GEN_1)
+       |    io_z <= x
+       |    io_v <= T_21
+  """.stripMargin
+
+  private def genValues(from: Long, upTo: Long) =
+    for {
+      x <- from to upTo
+      y <- from to upTo
+    } yield (x, y, BigInt(x).gcd(y).toLong)
+
+  def fsimTest(sim: Simulation, from: Long, upTo: Long): Unit = {
+    val values = genValues(from, upTo)
+    val (io_a, io_b, io_e) = (sim.getSymbolId("io_a"), sim.getSymbolId("io_b"), sim.getSymbolId("io_e"))
+    val (io_v, io_z) = (sim.getSymbolId("io_v"), sim.getSymbolId("io_z"))
+
+    for ((x, y, z) <- values) {
+      sim.step()
+      sim.pokeLong(io_a, x)
+      sim.pokeLong(io_b, y)
+      sim.pokeBool(io_e, true)
+      sim.step()
+
+      sim.pokeBool(io_e, false)
+      sim.step()
+
+      var count = 0
+      while (!sim.peekBool(io_v)) {
+        count += 1
+        sim.step()
+      }
+
+      assert(sim.peekLong(io_z) == z)
+    }
+  }
+
+  private val Big1 = BigInt(1)
+  def treadleTest(tester: TreadleTester, from: Long, upTo: Long): Unit = {
+    val values = genValues(from, upTo)
+    tester.poke("clock", 1)
+
+    for ((x, y, z) <- values) {
+      tester.step()
+      tester.poke("io_a", x)
+      tester.poke("io_b", y)
+      tester.poke("io_e", 1)
+      tester.step()
+
+      tester.poke("io_e", 0)
+      tester.step()
+
+      var count = 0
+      while (tester.peek("io_v") != Big1) {
+        count += 1
+        tester.step()
+      }
+
+      tester.expect("io_z", BigInt(z))
+    }
+  }
+
+}

build.sbt

@@ -81,7 +81,23 @@ lazy val publishSettings = Seq(
   },
 )
 
+lazy val simpleDirectoryLayout = Seq(
+  Compile / scalaSource := baseDirectory.value / "src",
+  Test / scalaSource := baseDirectory.value / "test",
+)
+
 lazy val chiseltest = (project in file("."))
   .settings(commonSettings)
   .settings(chiseltestSettings)
-  .settings(publishSettings)
+  .settings(publishSettings)
+
+lazy val benchmark = (project in file("benchmark"))
+  .dependsOn(chiseltest)
+  .settings(commonSettings)
+  .settings(simpleDirectoryLayout)
+  .settings(
+    name := "benchmark",
+    assembly / assemblyJarName := "benchmark.jar",
+    assembly / test := {},
+    assembly / assemblyOutputPath := file("./benchmark/benchmark.jar")
+  )

project/plugins.sbt

@@ -1,5 +1,5 @@
 logLevel := Level.Warn
 
-addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "5.2.4")
 addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2")
 addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.11")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.3")