diff --git a/bittide-instances/bittide-instances.cabal b/bittide-instances/bittide-instances.cabal index 82288139d..c4664f7a7 100644 --- a/bittide-instances/bittide-instances.cabal +++ b/bittide-instances/bittide-instances.cabal @@ -167,6 +167,7 @@ library Bittide.Instances.Pnr.Si539xSpi Bittide.Instances.Pnr.StabilityChecker Bittide.Instances.Pnr.Synchronizer + Bittide.Instances.Ugns Paths.Bittide.Instances Project.Handle @@ -208,6 +209,7 @@ test-suite unittests other-modules: Tests.ClockControlWb Tests.OverflowResistantDiff + Tests.Ugns Wishbone.Axi Wishbone.CaptureUgn Wishbone.DnaPortE2 diff --git a/bittide-instances/imgs/ugn_calculator.drawio b/bittide-instances/imgs/ugn_calculator.drawio new file mode 100644 index 000000000..74e62b1e7 --- /dev/null +++ b/bittide-instances/imgs/ugn_calculator.drawio @@ -0,0 +1,154 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/bittide-instances/src/Bittide/Instances/Hitl/Setup.hs b/bittide-instances/src/Bittide/Instances/Hitl/Setup.hs index 505781d4d..f64607422 100644 --- a/bittide-instances/src/Bittide/Instances/Hitl/Setup.hs +++ b/bittide-instances/src/Bittide/Instances/Hitl/Setup.hs @@ -55,9 +55,10 @@ neighbors (via the index position in the vector) according to the different hardware interfaces on the boards. -} fpgaSetup :: Vec FpgaCount (FpgaId, Vec LinkCount (Index FpgaCount)) +{- FOURMOLU_DISABLE -} fpgaSetup = - -- FPGA Id SFP0 SFP1 J4 J5 J6 J7 SMA - ("210308B3B272", 3 :> 2 :> 4 :> 5 :> 6 :> 7 :> 1 :> Nil) + -- FPGA Id SFP0 SFP1 J4 J5 J6 J7 SMA + ("210308B3B272", 3 :> 2 :> 4 :> 5 :> 6 :> 7 :> 1 :> Nil) :> ("210308B0992E", 2 :> 3 :> 5 :> 6 :> 7 :> 4 :> 0 :> Nil) :> ("210308B0AE73", 1 :> 0 :> 6 :> 7 :> 4 :> 5 :> 3 :> Nil) :> ("210308B0AE6D", 0 :> 1 :> 7 :> 4 :> 5 :> 6 :> 2 :> Nil) @@ -66,6 +67,7 @@ fpgaSetup = :> ("210308B3A22D", 5 :> 4 :> 2 :> 1 :> 0 :> 3 :> 7 :> Nil) :> ("210308B0B0C2", 4 :> 5 :> 3 :> 2 :> 1 :> 0 :> 6 :> Nil) :> Nil +{- FOURMOLU_ENABLE -} {- | The IDs of the Digilent chips on each of the FPGA boards of the test setup. The indices match the position of each FPGA in the mining rig. diff --git a/bittide-instances/src/Bittide/Instances/Ugns.hs b/bittide-instances/src/Bittide/Instances/Ugns.hs new file mode 100644 index 000000000..5fe893010 --- /dev/null +++ b/bittide-instances/src/Bittide/Instances/Ugns.hs @@ -0,0 +1,263 @@ +-- SPDX-FileCopyrightText: 2025 Google LLC +-- +-- SPDX-License-Identifier: Apache-2.0 +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE NumericUnderscores #-} + +module Bittide.Instances.Ugns where + +import Clash.Prelude + +import Data.Bifunctor (Bifunctor (bimap)) +import Data.Map (Map) +import Data.Maybe (catMaybes) +import GHC.Stack (HasCallStack) + +import Bittide.Instances.Hitl.Setup (FpgaCount, FpgaId, LinkCount) + +import qualified Bittide.Instances.Hitl.Setup as Setup +import qualified Data.Map as Map + +type LocalCounter = BitVector 64 +type RemoteCounter = LocalCounter +type Ugn = LocalCounter + +-- | Local and remote counters captured from a real system. +exampleUgnParts :: Vec FpgaCount (Vec LinkCount (LocalCounter, RemoteCounter)) +{- FOURMOLU_DISABLE -} -- data / tabular format +exampleUgnParts = + -- tx rx link0 tx rx link1 tx rx link2 tx rx link3 tx rx link4 tx rx link5 tx rx link6 + ((128392080, 128080803) :> (128392078, 142179719) :> (128392071, 125040058) :> (128392074, 140853500) :> (128392083, 125839249) :> (128392067, 140121605) :> (128392083, 139381360) :> Nil) + :> ((139388015, 142187569) :> (139388010, 128087421) :> (139388000, 140860114) :> (139388012, 125845867) :> (139388005, 140128232) :> (139387998, 125046674) :> (139388009, 128398664) :> Nil) + :> ((142185175, 139385540) :> (142185182, 128396241) :> (142185181, 125843441) :> (142185171, 140125803) :> (142185172, 125044254) :> (142185179, 140857699) :> (142185187, 128085003) :> Nil) + :> ((128087441, 128398651) :> (128087437, 139387957) :> (128087450, 140128233) :> (128087446, 125046677) :> (128087440, 140860108) :> (128087445, 125845855) :> (128087452, 142187567) :> Nil) + :> ((125047404, 140128921) :> (125047406, 125846552) :> (125047399, 128399343) :> (125047401, 128088101) :> (125047405, 142188255) :> (125047404, 139388659) :> (125047407, 140860811) :> Nil) + :> ((140860941, 125846648) :> (140860950, 140129029) :> (140860936, 139388754) :> (140860935, 128399441) :> (140860941, 128088203) :> (140860942, 142188354) :> (140860936, 125047464) :> Nil) + :> ((125841740, 140855965) :> (125841741, 125042526) :> (125841729, 142183399) :> (125841725, 139383800) :> (125841737, 128394501) :> (125841735, 128083256) :> (125841722, 140124061) :> Nil) + :> ((140118664, 125037078) :> (140118660, 140850511) :> (140118663, 128077812) :> (140118668, 142177966) :> (140118677, 139378381) :> (140118665, 128389058) :> (140118662, 125836256) :> Nil) + :> Nil +{- FOURMOLU_ENABLE -} + +-- | UGNs calculated from 'exampleUgnParts'. +exampleUgns :: Vec FpgaCount (Vec LinkCount Ugn) +exampleUgns = map (map (\(tx, rx) -> tx - rx)) exampleUgnParts + +{- | Mapping from an FPGA's local counter to another's local counter. I.e., for: + +> (counterA, counterB) = exampleCounterMap !! a Map.!! b@ + +a word sent at @counterA@ on FPGA @a@ is received at @counterB@ on FPGA @b@. +-} +exampleCounterMap :: + (HasCallStack) => + Vec FpgaCount (Map (Index FpgaCount) (BitVector 64)) +exampleCounterMap = toCounterMap (toFpgaIndexed Setup.fpgaSetup exampleUgnParts) + +{- | Convert a table (fpga_nr x link_nr) to a vector of maps mapping an fpga index +(instead of a link index) to the value. +-} +toFpgaIndexed :: + forall nNodes a. + (KnownNat nNodes, 1 <= nNodes) => + Vec nNodes (FpgaId, Vec (nNodes - 1) (Index nNodes)) -> + Vec nNodes (Vec (nNodes - 1) a) -> + Vec nNodes (Map (Index nNodes) a) +toFpgaIndexed fpgaSetup table = toMap <$> (goRow <$> indicesI <*> table) + where + goRow nodeNr row = goCell nodeNr <$> indicesI <*> row + goCell nodeNr linkNr a = (snd (fpgaSetup !! nodeNr) !! linkNr, a) + toMap = Map.fromList . toList + +{- | Given a table structured as 'exampleUgnParts' applied to 'toFpgaIndex' +convert it into a table that represents the mapping from an FPGA's local counter +to another FPGA's counter. +-} +toCounterMap :: + forall nNodes a. + (HasCallStack, Num a, KnownNat nNodes, 1 <= nNodes) => + Vec nNodes (Map (Index nNodes) (a, a)) -> + -- | n -> m: c + Vec nNodes (Map (Index nNodes) a) +toCounterMap fpgaIndexed = goSrc <$> indicesI + where + goSrc :: + (HasCallStack) => + Index nNodes -> + Map (Index nNodes) a + goSrc src = Map.fromList (catMaybes (toList (goSrcDst src <$> indicesI))) + + goSrcDst :: + (HasCallStack) => + Index nNodes -> + Index nNodes -> + Maybe (Index nNodes, a) + goSrcDst src dst | src == dst = Nothing + goSrcDst src dst = + let (srcCycle, dstCycle) = fpgaIndexed !! dst Map.! src + in Just (dst, srcCycle - dstCycle) + +uncons :: Vec (n + 1) a -> (a, Vec n a) +uncons (x :> xs) = (x, xs) +uncons Nil = error "uncons: unreachable" + +data PeConfig a b = PeConfig + { startWriteAt :: a + , writeForN :: b + , startReadAt :: a + , readForN :: b + } + deriving (Functor, Eq, Show) + +instance Bifunctor PeConfig where + bimap f g PeConfig{startWriteAt, writeForN, startReadAt, readForN} = + PeConfig + { startWriteAt = f startWriteAt + , writeForN = g writeForN + , startReadAt = f startReadAt + , readForN = g readForN + } + +{- | Create a vector of 'PeConfig's that form a chain of write reads, such that +a PE starts writing as soon as it can after reading values from its +predecessor. +-} +fullChainConfiguration :: + forall nNodes cyclesPerWrite a. + ( KnownNat nNodes + , Bounded a + , Integral a + , 1 <= nNodes + ) => + -- | Cycles per write + SNat cyclesPerWrite -> + -- | FPGA configuration + Vec nNodes (FpgaId, Vec (nNodes - 1) (Index nNodes)) -> + -- | UGN parts + Vec nNodes (Vec (nNodes - 1) (a, a)) -> + -- | Start offset for the first write + a -> + -- | PeConfig for each node in the chain + Vec nNodes (PeConfig a (Index (nNodes + 1))) +fullChainConfiguration = chainConfiguration SNat + +{- | Like 'fullChainConfiguration' but the number of nodes in the chain is +configurable. +-} +chainConfiguration :: + forall chainLength nNodes cyclesPerWrite a. + ( KnownNat nNodes + , KnownNat chainLength + , Bounded a + , Integral a + , 1 <= chainLength + , 1 <= nNodes + , chainLength <= nNodes + ) => + SNat chainLength -> + -- | Cycles per write + SNat cyclesPerWrite -> + -- | FPGA configuration + Vec nNodes (FpgaId, Vec (nNodes - 1) (Index nNodes)) -> + -- | UGN parts + Vec nNodes (Vec (nNodes - 1) (a, a)) -> + -- | Start offset for the first write + a -> + -- | PeConfig for each node in the chain + Vec chainLength (PeConfig a (Index (nNodes + 1))) +chainConfiguration SNat cyclesPerWrite fpgaConfig ugnParts writeOffset = + resultI + where + ugnPartsI = map (map (bimap toInteger toInteger)) ugnParts + resultI = map (bimap fromIntegral go) result + result = + chainConfigurationWorker + fpgaConfig + ugnPartsI + (toInteger writeOffset) + (snatToNum cyclesPerWrite) + + -- XXX: 'chainConfigurationWorker' calculates the number of cycles to write, + -- but our demo PE wants the number of TRI-cycles to write (or really, + -- however many cycles it takes to write a single "packet"). We hence end + -- up dividing it by 'cyclesPerWrite' here. + go :: forall b. (Bounded b, Integral b) => Integer -> b + go n = checkedFromIntegral (n `quot` snatToNum cyclesPerWrite) + +{- | Like 'chainConfiguration' but with 'Integer' types. We might rework this to +be more precise in its types, but be prepared for a boatload of type constraints... +-} +chainConfigurationWorker :: + forall chainLength nNodes. + ( KnownNat nNodes + , 1 <= nNodes + , KnownNat chainLength + , 1 <= chainLength + , chainLength <= nNodes + ) => + -- | FPGA configuration + Vec nNodes (FpgaId, Vec (nNodes - 1) (Index nNodes)) -> + -- | UGN parts + Vec nNodes (Vec (nNodes - 1) (Integer, Integer)) -> + -- | Start offset for the first write + Integer -> + -- | Cycles per write (probably 3) + Integer -> + Vec chainLength (PeConfig Integer Integer) +chainConfigurationWorker fpgaConfig ugnParts writeOffset cyclesPerWrite = + id + $ leToPlus @1 @nNodes + $ leToPlus @1 @chainLength + $ leToPlusKN @chainLength @nNodes + $ let + start = (0, writeOffset) + ((_, startReadAt), configs) = mapAccumL mkLink start (takeI indicesI) + (firstConfig, restConfigs) = uncons configs + readForN = cyclesPerWrite * natToNum @chainLength + in + firstConfig{startReadAt = startReadAt, readForN = readForN} :> restConfigs + where + counterMap = toCounterMap (toFpgaIndexed fpgaConfig ugnParts) + + mkLink :: + (Integer, Integer) -> + Index nNodes -> + ((Integer, Integer), PeConfig Integer Integer) + mkLink (readForN, startReadAt) node = + ((writeForN, nextStartReadAt), peConfig) + where + peConfig = PeConfig{startWriteAt, writeForN, startReadAt, readForN} + k = findK node nextNode startWriteAtClosestTo + startWriteAt = startWriteAtClosestTo + k + startWriteAtClosestTo = startReadAt + readForN + writeForN = readForN + cyclesPerWrite + nextStartReadAt = mapCycle startWriteAt node nextNode + nextNode = satSucc SatWrap node + + mapCycle :: Integer -> Index nNodes -> Index nNodes -> Integer + mapCycle srcCycle src dst = srcCycle + counterMap !! src Map.! dst + + findK :: Index nNodes -> Index nNodes -> Integer -> Integer + findK src dst srcCycle + | dstCycle <= linkStart0 = linkStart0 - dstCycle + | dstCycle <= linkStart1 = linkStart1 - dstCycle + | otherwise = error "findK: unreachable" + where + -- For a graphical representation of what we're doing here, check out: + -- bittide-instances/imgs/ugn_calculator.drawio. For a general introduction + -- see slides "determining read/write cycles" in + -- https://docs.google.com/presentation/d/1AGbAJQ1zhTPtrekKnQcthd0TUPyQs-zowQpV1ux4k-Y + dstCycle = mapCycle srcCycle src dst + linkSelectOffset = offsetsByFpga !! dst Map.! src + fullCalendarsSeen = dstCycle `quot` calendarLength + prevCalendarStart = fullCalendarsSeen * calendarLength + linkStart0 = prevCalendarStart + linkSelectOffset + linkStart1 = linkStart0 + calendarLength + + calendarEntryLength = cyclesPerWrite * natToNum @nNodes + calendarLength = calendarEntryLength * natToNum @nNodes + + offsets :: (KnownNat nNodes, 1 <= nNodes) => Vec nNodes (Vec (nNodes - 1) Integer) + offsets = map ((cyclesPerWrite *) . fromIntegral) <$> repeat indicesI + + offsetsByFpga :: (KnownNat nNodes, 1 <= nNodes) => Vec nNodes (Map (Index nNodes) Integer) + offsetsByFpga = toFpgaIndexed fpgaConfig offsets diff --git a/bittide-instances/tests/Tests/Ugns.hs b/bittide-instances/tests/Tests/Ugns.hs new file mode 100644 index 000000000..d03a6904d --- /dev/null +++ b/bittide-instances/tests/Tests/Ugns.hs @@ -0,0 +1,69 @@ +-- SPDX-FileCopyrightText: 2025 Google LLC +-- +-- SPDX-License-Identifier: Apache-2.0 +{-# LANGUAGE OverloadedLists #-} +{-# LANGUAGE TemplateHaskell #-} + +module Tests.Ugns where + +import Clash.Prelude hiding (indices) + +import Bittide.Instances.Hitl.Setup (FpgaId) +import Bittide.Instances.Ugns ( + PeConfig (..), + chainConfiguration, + toCounterMap, + toFpgaIndexed, + ) + +import Test.Tasty (TestTree, defaultMain) +import Test.Tasty.HUnit (Assertion, testCase, (@?=)) +import Test.Tasty.TH (testGroupGenerator) + +{- FOURMOLU_DISABLE -} -- data / tabular format +parts :: Vec 3 (Vec 2 (Int, Int)) +parts = + ((1, 0) :> (3, 4) :> Nil) + :> ((3, 1) :> (4, 6) :> Nil) + :> ((5, 2) :> (5, 8) :> Nil) + :> Nil + +indices :: Vec 3 (Vec 2 Int) +indices = + (0 :> 1 :> Nil) + :> (2 :> 3 :> Nil) + :> (4 :> 5 :> Nil) + :> Nil + +fpgaSetup :: Vec 3 (FpgaId, Vec 2 (Index 3)) +fpgaSetup = + ("A", 1 :> 2 :> Nil) + :> ("B", 2 :> 0 :> Nil) + :> ("C", 0 :> 1 :> Nil) + :> Nil +{- FOURMOLU_ENABLE -} + +case_toFpgaIndexed :: Assertion +case_toFpgaIndexed = toFpgaIndexed fpgaSetup indices @?= expected + where + expected = [(1, 0), (2, 1)] :> [(2, 2), (0, 3)] :> [(0, 4), (1, 5)] :> Nil + +case_toCounterMap :: Assertion +case_toCounterMap = toCounterMap (toFpgaIndexed fpgaSetup parts) @?= expected + where + expected = [(1, -2), (2, 3)] :> [(0, 1), (2, -3)] :> [(0, -1), (1, 2)] :> Nil + +case_chainConfiguration :: Assertion +case_chainConfiguration = chainConfiguration d3 d3 fpgaSetup parts 100 @?= expected + where + expected = peConfig0 :> peConfig1 :> peConfig2 :> Nil + + peConfig0 = PeConfig{startWriteAt = 113, writeForN = 1, startReadAt = 138, readForN = 3} + peConfig1 = PeConfig{startWriteAt = 114, writeForN = 2, startReadAt = 111, readForN = 1} + peConfig2 = PeConfig{startWriteAt = 139, writeForN = 3, startReadAt = 111, readForN = 2} + +tests :: TestTree +tests = $(testGroupGenerator) + +main :: IO () +main = defaultMain tests diff --git a/bittide-instances/tests/unittests.hs b/bittide-instances/tests/unittests.hs index d14beb531..39d00793e 100644 --- a/bittide-instances/tests/unittests.hs +++ b/bittide-instances/tests/unittests.hs @@ -10,6 +10,7 @@ import Test.Tasty import qualified Tests.ClockControlWb as ClockControlWb import qualified Tests.OverflowResistantDiff as Ord +import qualified Tests.Ugns as Ugns import qualified Wishbone.Axi as Axi import qualified Wishbone.CaptureUgn as CaptureUgn import qualified Wishbone.DnaPortE2 as DnaPortE2 @@ -22,14 +23,15 @@ tests :: TestTree tests = testGroup "Unittests" - [ CaptureUgn.tests + [ Axi.tests + , CaptureUgn.tests , ClockControlWb.tests - , ScatterGather.tests - , SwitchDemoProcessingElement.tests , DnaPortE2.tests , Ord.tests + , ScatterGather.tests + , SwitchDemoProcessingElement.tests , Time.tests - , Axi.tests + , Ugns.tests , Watchdog.tests ]