diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 158d0906b..37dd79913 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,6 +47,26 @@ jobs: timeout-minutes: 6 run: sbt ++${{ matrix.scala }} "testOnly integration.**" + benchmarks: + name: Benchmarks + runs-on: ubuntu-20.04 + strategy: + matrix: + scala: [ 2.13.10 ] + jvm: [ 8, 11, 21 ] + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup Scala + uses: actions/setup-java@v3 + with: + distribution: 'adopt' + java-version: ${{ matrix.jvm }} + - name: Check for Warnings + run: sbt "project benchmark ; set ThisBuild / scalacOptions ++= Seq(\"-Xfatal-warnings\") ; compile" + - name: Check Formatting + run: sbt "project benchmark ; scalafmtCheckAll" + test-mac: name: sbt test on mac @@ -206,7 +226,7 @@ jobs: # When adding new jobs, please add them to `needs` below all_tests_passed: name: "all tests passed" - needs: [test, doc, verilator, formal, formal-mac, icarus, test-mac, no-warn, integration-test] + needs: [test, doc, verilator, formal, formal-mac, icarus, test-mac, no-warn, integration-test, benchmarks] runs-on: ubuntu-latest steps: - run: echo Success! @@ -215,7 +235,7 @@ jobs: # separate from a Scala versions build matrix to avoid duplicate publishing publish: # note: we do not require a warning check for publishing! - needs: [test, doc, verilator, formal, formal-mac, icarus, test-mac, test-treadle, integration-test] + needs: [test, doc, verilator, formal, formal-mac, icarus, test-mac, test-treadle, integration-test, benchmarks] runs-on: ubuntu-20.04 if: github.event_name == 'push' diff --git a/benchmark/.gitignore b/benchmark/.gitignore new file mode 100644 index 000000000..9cd568afc --- /dev/null +++ b/benchmark/.gitignore @@ -0,0 +1,4 @@ +/project/target +/project/project +/target/ +/benchmark.jar diff --git a/benchmark/src/fsim/Benchmark.scala b/benchmark/src/fsim/Benchmark.scala new file mode 100644 index 000000000..badde6109 --- /dev/null +++ b/benchmark/src/fsim/Benchmark.scala @@ -0,0 +1,124 @@ +// Copyright 2023 The Regents of the University of California +// released under BSD 3-Clause License +// author: Kevin Laeufer + +package fsim + +import firrtl2.options.Dependency +import firrtl2.stage.Forms +import scopt.OptionParser +import treadle2.TreadleTester + +case class Config(benches: Seq[String], sim: String, warmupRun: Boolean) + +class ArgumentParser extends OptionParser[Config]("synthesizer") { + head("fsim benchmark", "0.1") + opt[String]("bench").action((a, config) => config.copy(benches = config.benches :+ a)) + opt[String]("sim").action((a, config) => config.copy(sim = a)) +} + +case class Bench( + name: String, + getCircuit: String => String, + runTest: Simulation => Unit, + runTreadleTest: TreadleTester => Unit) +case class Result(nsCompile: Long, nsRun: Long, steps: Int) +object Benchmark { + val Benches = Seq( + Bench("gcd_16", _ => GCDBench.circuitSrc(16), GCDBench.fsimTest(_, 10, 500), GCDBench.treadleTest(_, 10, 500)), + Bench("gcd_44", _ => GCDBench.circuitSrc(44), GCDBench.fsimTest(_, 10, 500), GCDBench.treadleTest(_, 10, 500)), + Bench("gcd_64", _ => GCDBench.circuitSrc(64), GCDBench.fsimTest(_, 10, 500), GCDBench.treadleTest(_, 10, 500)) + ) + + private val DefaultConfig = Config(benches = Seq("gcd_64"), sim = "fsim", warmupRun = true) + def main(args: Array[String]): Unit = { + val parser = new ArgumentParser() + val conf = parser.parse(args, DefaultConfig).get + conf.benches.foreach { benchName => + val bench = + Benches.find(_.name == benchName).getOrElse(throw new RuntimeException(s"Unknown benchmark: $benchName")) + val res = runBench(conf, bench) + printResult(bench.name, conf.sim, res) + } + + } + + def printResult(bench: String, sim: String, res: Result): Unit = { + println( + s"${bench} on ${sim}: " + + s"${secondString(res.nsRun)}, ${res.steps} cycles, ${freqString(res.nsRun, res.steps)}, " + + s"${secondString(res.nsCompile)} to compile" + ) + } + + private def secondString(ns: Long): String = { + val elapsedSeconds = ns.toDouble / Giga + f"$elapsedSeconds%.6fs" + } + + private val Kilo: Double = 1000.0 + private val Mega: Double = 1000000.0 + private val Giga: Double = 1000000000.0 + private def freqString(ns: Long, steps: Int): String = { + val elapsedSeconds = ns.toDouble / Giga + val hz = steps.toDouble / elapsedSeconds + if (hz > Giga) { + f"${hz / Giga}%.6fGHz" + } else if (hz > Mega) { + f"${hz / Mega}%.6fMHz" + } else if (hz > Kilo) { + f"${hz / Kilo}%.6fkHz" + } else { + f"${hz}%.6fHz" + } + } + + def runBench(conf: Config, bench: Bench): Result = { + val src = bench.getCircuit(conf.sim) + conf.sim match { + case "fsim" => runFSimBench(conf, bench, src) + case "treadle" => runTreadleBench(conf, bench, src) + case other => throw new RuntimeException(s"Unsupported simulator: $other") + } + } + + private def runFSimBench(conf: Config, bench: Bench, src: String): Result = { + val compileStart = System.nanoTime() + val sim = new Simulation(Compiler.run(FirrtlCompiler.toLow(src))) + val compileEnd = System.nanoTime() + if (conf.warmupRun) { + bench.runTest(sim) + } + val testStart = System.nanoTime() + bench.runTest(sim) + val testEnd = System.nanoTime() + val steps = sim.getStepCount + // TODO: shut down sim + Result(nsCompile = compileEnd - compileStart, nsRun = testEnd - testStart, steps = steps) + } + + private def runTreadleBench(conf: Config, bench: Bench, src: String): Result = { + val compileStart = System.nanoTime() + val sim = new Simulation(Compiler.run(FirrtlCompiler.toLow(src))) + val compileEnd = System.nanoTime() + if (conf.warmupRun) { + bench.runTest(sim) + } + val testStart = System.nanoTime() + bench.runTest(sim) + val testEnd = System.nanoTime() + val steps = sim.getStepCount + // TODO: shut down sim + Result(nsCompile = compileEnd - compileStart, nsRun = testEnd - testStart, steps = steps) + } +} + +object FirrtlCompiler { + private val loFirrtlCompiler = + new firrtl2.stage.transforms.Compiler(Seq(Dependency[firrtl2.LowFirrtlEmitter]) ++ Forms.LowFormOptimized) + def toLow(src: String): firrtl2.ir.Circuit = { + val hi = firrtl2.Parser.parse(src) + val lo = loFirrtlCompiler.execute(firrtl2.CircuitState(hi)) + lo.circuit + } +} diff --git a/benchmark/src/fsim/GCDBench.scala b/benchmark/src/fsim/GCDBench.scala new file mode 100644 index 000000000..bb013b8f0 --- /dev/null +++ b/benchmark/src/fsim/GCDBench.scala @@ -0,0 +1,96 @@ +// Copyright 2023 The Regents of the University of California +// released under BSD 3-Clause License +// author: Kevin Laeufer + +package fsim + +import treadle2.TreadleTester + +object GCDBench { + def circuitSrc(width: Int): String = + s""" + |circuit GCD : + | module GCD : + | input clock : Clock + | input reset : UInt<1> + | input io_a : UInt<$width> + | input io_b : UInt<$width> + | input io_e : UInt<1> + | output io_z : UInt<$width> + | output io_v : UInt<1> + | reg x : UInt<$width>, clock with : + | reset => (UInt<1>("h0"), x) + | reg y : UInt<$width>, clock with : + | reset => (UInt<1>("h0"), y) + | node T_13 = gt(x, y) + | node T_14 = sub(x, y) + | node T_15 = tail(T_14, 1) + | node T_17 = eq(T_13, UInt<1>("h0")) + | node T_18 = sub(y, x) + | node T_19 = tail(T_18, 1) + | node T_21 = eq(y, UInt<1>("h0")) + | node GEN_0 = mux(T_13, T_15, x) + | x <= mux(io_e, io_a, GEN_0) + | node GEN_1 = mux(T_17, T_19, y) + | y <= mux(io_e, io_b, GEN_1) + | io_z <= x + | io_v <= T_21 + """.stripMargin + + private def genValues(from: Long, upTo: Long) = + for { + x <- from to upTo + y <- from to upTo + } yield (x, y, BigInt(x).gcd(y).toLong) + + def fsimTest(sim: Simulation, from: Long, upTo: Long): Unit = { + val values = genValues(from, upTo) + val (io_a, io_b, io_e) = (sim.getSymbolId("io_a"), sim.getSymbolId("io_b"), sim.getSymbolId("io_e")) + val (io_v, io_z) = (sim.getSymbolId("io_v"), sim.getSymbolId("io_z")) + + for ((x, y, z) <- values) { + sim.step() + sim.pokeLong(io_a, x) + sim.pokeLong(io_b, y) + sim.pokeBool(io_e, true) + sim.step() + + sim.pokeBool(io_e, false) + sim.step() + + var count = 0 + while (!sim.peekBool(io_v)) { + count += 1 + sim.step() + } + + assert(sim.peekLong(io_z) == z) + } + } + + private val Big1 = BigInt(1) + def treadleTest(tester: TreadleTester, from: Long, upTo: Long): Unit = { + val values = genValues(from, upTo) + tester.poke("clock", 1) + + for ((x, y, z) <- values) { + tester.step() + tester.poke("io_a", x) + tester.poke("io_b", y) + tester.poke("io_e", 1) + tester.step() + + tester.poke("io_e", 0) + tester.step() + + var count = 0 + while (tester.peek("io_v") != Big1) { + count += 1 + tester.step() + } + + tester.expect("io_z", BigInt(z)) + } + } + +} diff --git a/build.sbt b/build.sbt index 3c3fee2aa..70cb5b23b 100644 --- a/build.sbt +++ b/build.sbt @@ -81,7 +81,23 @@ lazy val publishSettings = Seq( }, ) +lazy val simpleDirectoryLayout = Seq( + Compile / scalaSource := baseDirectory.value / "src", + Test / scalaSource := baseDirectory.value / "test", +) + lazy val chiseltest = (project in file(".")) .settings(commonSettings) .settings(chiseltestSettings) - .settings(publishSettings) \ No newline at end of file + .settings(publishSettings) + +lazy val benchmark = (project in file("benchmark")) + .dependsOn(chiseltest) + .settings(commonSettings) + .settings(simpleDirectoryLayout) + .settings( + name := "benchmark", + assembly / assemblyJarName := "benchmark.jar", + assembly / test := {}, + assembly / assemblyOutputPath := file("./benchmark/benchmark.jar") + ) \ No newline at end of file diff --git a/project/plugins.sbt b/project/plugins.sbt index b640eaa35..8bb9f2f23 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,5 +1,5 @@ logLevel := Level.Warn -addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "5.2.4") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2") addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.11") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.3")