diff --git a/FEXCore/Source/Interface/Context/Context.h b/FEXCore/Source/Interface/Context/Context.h index b5ce016653..c6c9e80bd2 100644 --- a/FEXCore/Source/Interface/Context/Context.h +++ b/FEXCore/Source/Interface/Context/Context.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/FEXCore/Source/Interface/Core/Core.cpp b/FEXCore/Source/Interface/Core/Core.cpp index 5a30183252..47f9fd661d 100644 --- a/FEXCore/Source/Interface/Core/Core.cpp +++ b/FEXCore/Source/Interface/Core/Core.cpp @@ -26,6 +26,7 @@ desc: Glues Frontend, OpDispatcher and IR Opts & Compilation, LookupCache, Dispa #include "Interface/IR/Passes/RegisterAllocationPass.h" #include "Interface/IR/Passes.h" #include "Interface/IR/PassManager.h" +#include "Interface/IR/RegisterAllocationData.h" #include "Utils/Allocator.h" #include "Utils/Allocator/HostAllocator.h" @@ -38,9 +39,6 @@ desc: Glues Frontend, OpDispatcher and IR Opts & Compilation, LookupCache, Dispa #include #include #include -#include -#include -#include #include #include #include diff --git a/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h b/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h index 69ee572d33..7bc5ce7453 100644 --- a/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h +++ b/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h @@ -2,9 +2,8 @@ #include "Common/SoftFloat.h" #include "Common/SoftFloat-3e/softfloat.h" -#include - #include "Interface/Core/Interpreter/Fallbacks/FallbackOpHandler.h" +#include "Interface/IR/IR.h" namespace FEXCore::CPU { FEXCORE_PRESERVE_ALL_ATTR diff --git a/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h b/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h index c103dd1182..9b7727f57f 100644 --- a/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h +++ b/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h @@ -7,7 +7,6 @@ #include #include -#include namespace FEXCore::IR { class IRListView; diff --git a/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h b/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h index bd07897f46..c3675950ef 100644 --- a/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h +++ b/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h @@ -11,14 +11,15 @@ tags: backend|arm64 #include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h" #include "Interface/Core/CPUBackend.h" #include "Interface/Core/Dispatcher/Dispatcher.h" +#include "Interface/IR/IR.h" +#include "Interface/IR/IntrusiveIRList.h" +#include "Interface/IR/RegisterAllocationData.h" #include #include #include #include -#include -#include #include #include #include diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp b/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp index 2214f4c923..3c42d45fa8 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp @@ -18,7 +18,6 @@ desc: Handles x86/64 ops to IR, no-pf opt, local-flags opt #include #include #include -#include #include #include diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/FEXCore/Source/Interface/Core/OpcodeDispatcher.h index 9c58e50725..552e7b94e0 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher.h +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher.h @@ -9,7 +9,6 @@ #include #include #include -#include #include #include diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp b/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp index b8aa8371fa..5c8ebbd6bf 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp @@ -13,7 +13,6 @@ desc: Handles x86/64 flag generation #include #include #include -#include #include #include diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp b/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp index c72cef15c5..b7200b5dd6 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp @@ -13,7 +13,6 @@ desc: Handles x86/64 Vector instructions to IR #include #include #include -#include #include #include diff --git a/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp b/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp index 48faeee99a..6842b0e713 100644 --- a/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp +++ b/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp @@ -6,13 +6,13 @@ tags: glue|thunks $end_info$ */ +#include "Interface/IR/IR.h" #include "Interface/IR/IREmitter.h" #include #include #include #include -#include #include #include #include diff --git a/FEXCore/Source/Interface/HLE/Thunks/Thunks.h b/FEXCore/Source/Interface/HLE/Thunks/Thunks.h index d7933e5dd4..6d6437f572 100644 --- a/FEXCore/Source/Interface/HLE/Thunks/Thunks.h +++ b/FEXCore/Source/Interface/HLE/Thunks/Thunks.h @@ -7,7 +7,8 @@ tags: glue|thunks #pragma once -#include +#include "Interface/IR/IR.h" + #include #include diff --git a/FEXCore/Source/Interface/IR/AOTIR.cpp b/FEXCore/Source/Interface/IR/AOTIR.cpp index c16ab231a4..a09bdfffeb 100644 --- a/FEXCore/Source/Interface/IR/AOTIR.cpp +++ b/FEXCore/Source/Interface/IR/AOTIR.cpp @@ -2,9 +2,9 @@ #include "FEXHeaderUtils/Filesystem.h" #include "Interface/Context/Context.h" #include "Interface/IR/AOTIR.h" +#include "Interface/IR/IntrusiveIRList.h" +#include "Interface/IR/RegisterAllocationData.h" -#include -#include #include #include #include diff --git a/FEXCore/Source/Interface/IR/AOTIR.h b/FEXCore/Source/Interface/IR/AOTIR.h index 4dfc5319c3..b128b8ab6d 100644 --- a/FEXCore/Source/Interface/IR/AOTIR.h +++ b/FEXCore/Source/Interface/IR/AOTIR.h @@ -1,7 +1,8 @@ // SPDX-License-Identifier: MIT #pragma once -#include "FEXCore/IR/RegisterAllocationData.h" +#include "Interface/IR/RegisterAllocationData.h" + #include #include #include diff --git a/FEXCore/Source/Interface/IR/IR.h b/FEXCore/Source/Interface/IR/IR.h index c92a7a433c..77d1930d2e 100644 --- a/FEXCore/Source/Interface/IR/IR.h +++ b/FEXCore/Source/Interface/IR/IR.h @@ -9,11 +9,628 @@ namespace FEXCore::IR { +class OrderedNode; +class RegisterAllocationPass; class RegisterAllocationData; +/** + * @brief The IROp_Header is an dynamically sized array + * At the end it contains a uint8_t for the number of arguments that Op has + * Then there is an unsized array of NodeWrapper arguments for the number of arguments this op has + * The op structures that are including the header must ensure that they pad themselves correctly to the number of arguments used + */ +struct IROp_Header; + +/** + * @brief Represents the ID of a given IR node. + * + * Intended to provide strong typing from other integer values + * to prevent passing incorrect values to certain API functions. + */ +struct NodeID final { + using value_type = uint32_t; + + constexpr NodeID() noexcept = default; + constexpr explicit NodeID(value_type Value_) noexcept : Value{Value_} {} + + constexpr NodeID(const NodeID&) noexcept = default; + constexpr NodeID& operator=(const NodeID&) noexcept = default; + + constexpr NodeID(NodeID&&) noexcept = default; + constexpr NodeID& operator=(NodeID&&) noexcept = default; + + [[nodiscard]] constexpr bool IsValid() const noexcept { + return Value != 0; + } + [[nodiscard]] constexpr bool IsInvalid() const noexcept { + return !IsValid(); + } + constexpr void Invalidate() noexcept { + Value = 0; + } + + [[nodiscard]] friend constexpr bool operator==(NodeID, NodeID) noexcept = default; + + [[nodiscard]] friend constexpr bool operator<(NodeID lhs, NodeID rhs) noexcept { + return lhs.Value < rhs.Value; + } + [[nodiscard]] friend constexpr bool operator>(NodeID lhs, NodeID rhs) noexcept { + return operator<(rhs, lhs); + } + [[nodiscard]] friend constexpr bool operator<=(NodeID lhs, NodeID rhs) noexcept { + return !operator>(lhs, rhs); + } + [[nodiscard]] friend constexpr bool operator>=(NodeID lhs, NodeID rhs) noexcept { + return !operator<(lhs, rhs); + } + + friend std::ostream& operator<<(std::ostream& out, NodeID ID) { + out << ID.Value; + return out; + } + friend std::istream& operator>>(std::istream& in, NodeID& ID) { + in >> ID.Value; + return in; + } + + value_type Value{}; +}; + +/** + * @brief This is a very simple wrapper for our node pointers + * You probably don't want to use this directly + * Use OpNodeWrapper and OrderedNodeWrapper types below instead + * + * This is necessary to allow two things + * - Reduce memory usage by having the pointer be an 32bit offset rather than the whole 64bit pointer + * - Actually use an offset from a base so we aren't storing pointers for everything + * - Makes IR list copying be as cheap as a memcpy + * Downsides + * - The IR nodes have to be allocated out of a linear array of memory + * - We currently only allow a 32bit offset, so *only* 4 million nodes per list + * - We have to have the base offset live somewhere else + * - Has to be POD and trivially copyable + * - Makes every real node access turn in to a [Base + Offset] access + * - Can be confusing if you're mixing OpNodeWrapper and OrderedNodeWrapper usage + */ +template +struct NodeWrapperBase final { + // On x86-64 using a uint64_t type is more efficient since RIP addressing gives you [ + + ] + // On AArch64 using uint32_t is just more memory efficient. 32bit or 64bit offset doesn't matter + // We use uint32_t to be more memory efficient (Cuts our node list size in half) + using NodeOffsetType = uint32_t; + NodeOffsetType NodeOffset; + + explicit NodeWrapperBase() = default; + + [[nodiscard]] static NodeWrapperBase WrapOffset(NodeOffsetType Offset) { + NodeWrapperBase Wrapped; + Wrapped.NodeOffset = Offset; + return Wrapped; + } + + [[nodiscard]] static NodeWrapperBase WrapPtr(uintptr_t Base, uintptr_t Value) { + NodeWrapperBase Wrapped; + Wrapped.SetOffset(Base, Value); + return Wrapped; + } + + [[nodiscard]] static void *UnwrapNode(uintptr_t Base, NodeWrapperBase Node) { + return Node.GetNode(Base); + } + + [[nodiscard]] NodeID ID() const; + + [[nodiscard]] bool IsInvalid() const { return NodeOffset == 0; } + + [[nodiscard]] Type *GetNode(uintptr_t Base) { + return reinterpret_cast(Base + NodeOffset); + } + [[nodiscard]] const Type *GetNode(uintptr_t Base) const { + return reinterpret_cast(Base + NodeOffset); + } + + void SetOffset(uintptr_t Base, uintptr_t Value) { NodeOffset = Value - Base; } + + [[nodiscard]] friend constexpr bool operator==(const NodeWrapperBase&, const NodeWrapperBase&) = default; +}; + +static_assert(std::is_trivial_v>); + +static_assert(sizeof(NodeWrapperBase) == sizeof(uint32_t)); + +using OpNodeWrapper = NodeWrapperBase; +using OrderedNodeWrapper = NodeWrapperBase; + +struct OrderedNodeHeader { + OpNodeWrapper Value; + OrderedNodeWrapper Next; + OrderedNodeWrapper Previous; +}; + +static_assert(sizeof(OrderedNodeHeader) == sizeof(uint32_t) * 3); + +/** + * @brief This is a node in our IR representation + * Is a doubly linked list node that lives in a representation of a linearly allocated node list + * The links in the nodes can live in a list independent of the data IR data + * + * ex. + * Region1 : ... <-> <-> <-> ... + * | * | + * v v + * Region2 : ...... + * + * In this example the OrderedNodes are allocated in one linear memory region (Not necessarily contiguous with one another linking) + * The second region is contiguous but they don't have any relationship with one another directly + */ +class OrderedNode final { + friend class NodeWrapperIterator; + friend class OrderedList; + public: + // These three values are laid out very specifically to make it fast to access the NodeWrappers specifically + OrderedNodeHeader Header; + uint32_t NumUses; + + using value_type = OrderedNodeWrapper; + + OrderedNode() = default; + + /** + * @brief Appends a node to this current node + * + * Before. <-> <-> + * After. <-> <-> <-> Next + * + * @return Pointer to the node being added + */ + value_type append(uintptr_t Base, value_type Node) { + // Set Next Node's Previous to incoming node + SetPrevious(Base, Header.Next, Node); + + // Set Incoming node's links to this node's links + SetPrevious(Base, Node, Wrapped(Base)); + SetNext(Base, Node, Header.Next); + + // Set this node's next to the incoming node + SetNext(Base, Wrapped(Base), Node); + + // Return the node we are appending + return Node; + } + + OrderedNode *append(uintptr_t Base, OrderedNode *Node) { + value_type WNode = Node->Wrapped(Base); + // Set Next Node's Previous to incoming node + SetPrevious(Base, Header.Next, WNode); + + // Set Incoming node's links to this node's links + SetPrevious(Base, WNode, Wrapped(Base)); + SetNext(Base, WNode, Header.Next); + + // Set this node's next to the incoming node + SetNext(Base, Wrapped(Base), WNode); + + // Return the node we are appending + return Node; + } + + /** + * @brief Prepends a node to the current node + * Before. <-> <-> + * After. <-> <-> <-> Next + * + * @return Pointer to the node being added + */ + value_type prepend(uintptr_t Base, value_type Node) { + // Set the previous node's next to the incoming node + SetNext(Base, Header.Previous, Node); + + // Set the incoming node's links + SetPrevious(Base, Node, Header.Previous); + SetNext(Base, Node, Wrapped(Base)); + + // Set the current node's link + SetPrevious(Base, Wrapped(Base), Node); + + // Return the node we are prepending + return Node; + } + + OrderedNode *prepend(uintptr_t Base, OrderedNode *Node) { + value_type WNode = Node->Wrapped(Base); + // Set the previous node's next to the incoming node + SetNext(Base, Header.Previous, WNode); + + // Set the incoming node's links + SetPrevious(Base, WNode, Header.Previous); + SetNext(Base, WNode, Wrapped(Base)); + + // Set the current node's link + SetPrevious(Base, Wrapped(Base), WNode); + + // Return the node we are prepending + return Node; + } + + /** + * @brief Gets the remaining size of the blocks from this point onward + * + * Doesn't find the head of the list + * + */ + [[nodiscard]] size_t size(uintptr_t Base) const { + size_t Size = 1; + // Walk the list forward until we hit a sentinel + value_type Current = Header.Next; + while (Current.NodeOffset != 0) { + ++Size; + OrderedNode *RealNode = Current.GetNode(Base); + Current = RealNode->Header.Next; + } + return Size; + } + + void Unlink(uintptr_t Base) { + // This removes the node from the list. Orphaning it + // Before: <-> <-> + // After: + SetNext(Base, Header.Previous, Header.Next); + SetPrevious(Base, Header.Next, Header.Previous); + } + + [[nodiscard]] IROp_Header const* Op(uintptr_t Base) const { + return Header.Value.GetNode(Base); + } + [[nodiscard]] IROp_Header *Op(uintptr_t Base) { + return Header.Value.GetNode(Base); + } + + [[nodiscard]] uint32_t GetUses() const { return NumUses; } + + void AddUse() { ++NumUses; } + void RemoveUse() { --NumUses; } + + [[nodiscard]] value_type Wrapped(uintptr_t Base) const { + value_type Tmp; + Tmp.SetOffset(Base, reinterpret_cast(this)); + return Tmp; + } + + private: + [[nodiscard]] value_type WrappedOffset(uint32_t Offset) const { + value_type Tmp; + Tmp.NodeOffset = Offset; + return Tmp; + } + + static void SetPrevious(uintptr_t Base, value_type Node, value_type New) { + OrderedNode *RealNode = Node.GetNode(Base); + RealNode->Header.Previous = New; + } + + static void SetNext(uintptr_t Base, value_type Node, value_type New) { + OrderedNode *RealNode = Node.GetNode(Base); + RealNode->Header.Next = New; + } + + void SetUses(uint32_t Uses) { NumUses = Uses; } +}; + +static_assert(std::is_trivial_v); +static_assert(std::is_trivially_copyable_v); +static_assert(offsetof(OrderedNode, Header) == 0); +static_assert(sizeof(OrderedNode) == (sizeof(OrderedNodeHeader) + sizeof(uint32_t))); + +struct RegisterClassType final { + using value_type = uint32_t; + + value_type Val; + [[nodiscard]] constexpr operator value_type() const { + return Val; + } + [[nodiscard]] friend constexpr bool operator==(const RegisterClassType&, const RegisterClassType&) = default; +}; + +struct CondClassType final { + uint8_t Val; + [[nodiscard]] constexpr operator uint8_t() const { + return Val; + } + [[nodiscard]] friend constexpr bool operator==(const CondClassType&, const CondClassType&) = default; +}; + +struct MemOffsetType final { + uint8_t Val; + [[nodiscard]] constexpr operator uint8_t() const { + return Val; + } + [[nodiscard]] friend constexpr bool operator==(const MemOffsetType&, const MemOffsetType&) = default; +}; + +struct TypeDefinition final { + uint16_t Val; + + [[nodiscard]] constexpr operator uint16_t() const { + return Val; + } + + [[nodiscard]] static constexpr TypeDefinition Create(uint8_t Bytes) { + TypeDefinition Type{}; + Type.Val = Bytes << 8; + return Type; + } + + [[nodiscard]] static constexpr TypeDefinition Create(uint8_t Bytes, uint8_t Elements) { + TypeDefinition Type{}; + Type.Val = (Bytes << 8) | (Elements & 255); + return Type; + } + + [[nodiscard]] constexpr uint8_t Bytes() const { + return Val >> 8; + } + + [[nodiscard]] constexpr uint8_t Elements() const { + return Val & 255; + } + + [[nodiscard]] friend constexpr bool operator==(const TypeDefinition&, const TypeDefinition&) = default; +}; + +static_assert(std::is_trivial_v); + +struct FenceType final { + using value_type = uint8_t; + + value_type Val; + [[nodiscard]] constexpr operator value_type() const { + return Val; + } + [[nodiscard]] friend constexpr bool operator==(const FenceType&, const FenceType&) = default; +}; + +struct RoundType final { + uint8_t Val; + [[nodiscard]] constexpr operator uint8_t() const { + return Val; + } + [[nodiscard]] friend constexpr bool operator==(const RoundType&, const RoundType&) = default; +}; + +class NodeIterator; + +/* This iterator can be used to step though nodes. + * Due to how our IR is laid out, this can be used to either step + * though the CodeBlocks or though the code within a single block. + */ +class NodeIterator { +public: + using value_type = std::tuple; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = value_type*; + using const_pointer = const value_type*; + using iterator = NodeIterator; + using const_iterator = const NodeIterator; + using reverse_iterator = iterator; + using const_reverse_iterator = const_iterator; + using iterator_category = std::bidirectional_iterator_tag; + + NodeIterator(uintptr_t Base, uintptr_t IRBase) : BaseList {Base}, IRList{ IRBase } {} + explicit NodeIterator(uintptr_t Base, uintptr_t IRBase, OrderedNodeWrapper Ptr) : BaseList {Base}, IRList{ IRBase }, Node {Ptr} {} + + [[nodiscard]] bool operator==(const NodeIterator &rhs) const { + return Node.NodeOffset == rhs.Node.NodeOffset; + } + + [[nodiscard]] bool operator!=(const NodeIterator &rhs) const { + return !operator==(rhs); + } + + NodeIterator operator++() { + OrderedNodeHeader *RealNode = reinterpret_cast(Node.GetNode(BaseList)); + Node = RealNode->Next; + return *this; + } + + NodeIterator operator--() { + OrderedNodeHeader *RealNode = reinterpret_cast(Node.GetNode(BaseList)); + Node = RealNode->Previous; + return *this; + } + + [[nodiscard]] value_type operator*() { + OrderedNode *RealNode = Node.GetNode(BaseList); + return { RealNode, RealNode->Op(IRList) }; + } + + [[nodiscard]] value_type operator()() { + OrderedNode *RealNode = Node.GetNode(BaseList); + return { RealNode, RealNode->Op(IRList) }; + } + + [[nodiscard]] NodeID ID() const { + return Node.ID(); + } + + [[nodiscard]] static NodeIterator Invalid() { + return NodeIterator(0, 0); + } + +protected: + uintptr_t BaseList{}; + uintptr_t IRList{}; + OrderedNodeWrapper Node{}; +}; + +// This must directly match bytes to the named opsize. +// Implicit sized IR operations does math to get between sizes. +enum OpSize : uint8_t { + i8Bit = 1, + i16Bit = 2, + i32Bit = 4, + i64Bit = 8, + i128Bit = 16, + i256Bit = 32, +}; + +enum class FloatCompareOp : uint8_t { + EQ = 0, + LT, + LE, + UNO, + NEQ, + ORD, +}; + +enum class ShiftType : uint8_t { + LSL = 0, + LSR, + ASR, + ROR, +}; + + +// Converts a size stored as an integer in to an OpSize enum. +// This is a nop operation and will be eliminated by the compiler. +static inline OpSize SizeToOpSize(uint8_t Size) { + switch (Size) { + case 1: return OpSize::i8Bit; + case 2: return OpSize::i16Bit; + case 4: return OpSize::i32Bit; + case 8: return OpSize::i64Bit; + case 16: return OpSize::i128Bit; + case 32: return OpSize::i256Bit; + default: FEX_UNREACHABLE; + } +} + +#define IROP_ENUM +#define IROP_STRUCTS +#define IROP_SIZES +#define IROP_REG_CLASSES +#include + +/* This iterator can be used to step though every single node in a multi-block in SSA order. + * + * Iterates in the order of: + * + * end <-- CodeBlockA <--> BlockAInst1 <--> BlockAInst2 <--> CodeBlockB <--> BlockBInst1 <--> BlockBInst2 --> end + */ +class AllNodesIterator : public NodeIterator { +public: + AllNodesIterator(uintptr_t Base, uintptr_t IRBase) : NodeIterator(Base, IRBase) {} + explicit AllNodesIterator(uintptr_t Base, uintptr_t IRBase, OrderedNodeWrapper Ptr) : NodeIterator(Base, IRBase, Ptr) {} + AllNodesIterator(NodeIterator other) : NodeIterator(other) {} // Allow NodeIterator to be upgraded + + AllNodesIterator operator++() { + OrderedNodeHeader *RealNode = reinterpret_cast(Node.GetNode(BaseList)); + auto IROp = Node.GetNode(BaseList)->Op(IRList); + + // If this is the last node of a codeblock, we need to continue to the next block + if (IROp->Op == OP_ENDBLOCK) { + auto EndBlock = IROp->C(); + + auto CurrentBlock = EndBlock->BlockHeader.GetNode(BaseList); + Node = CurrentBlock->Header.Next; + } else if (IROp->Op == OP_CODEBLOCK) { + auto CodeBlock = IROp->C(); + + Node = CodeBlock->Begin; + } else { + Node = RealNode->Next; + } + + return *this; + } + + AllNodesIterator operator--() { + auto IROp = Node.GetNode(BaseList)->Op(IRList); + + if (IROp->Op == OP_BEGINBLOCK) { + auto BeginBlock = IROp->C(); + + Node = BeginBlock->BlockHeader; + } else if (IROp->Op == OP_CODEBLOCK) { + auto PrevBlockWrapper = Node.GetNode(BaseList)->Header.Previous; + auto PrevCodeBlock = PrevBlockWrapper.GetNode(BaseList)->Op(IRList)->C(); + + Node = PrevCodeBlock->Last; + } else { + Node = Node.GetNode(BaseList)->Header.Previous; + } + + return *this; + } + + [[nodiscard]] static AllNodesIterator Invalid() { + return AllNodesIterator(0, 0); + } +}; + class IRListView; class IREmitter; +template +inline NodeID NodeWrapperBase::ID() const { + return NodeID(NodeOffset / sizeof(IR::OrderedNode)); +} + +bool IsFragmentExit(FEXCore::IR::IROps Op); +bool IsBlockExit(FEXCore::IR::IROps Op); + void Dump(fextl::stringstream *out, IRListView const* IR, IR::RegisterAllocationData *RAData); fextl::unique_ptr Parse(FEXCore::Utils::IntrusivePooledAllocator &ThreadAllocator, fextl::stringstream &MapsStream); } + +template <> +struct std::hash { + size_t operator()(const FEXCore::IR::NodeID& ID) const noexcept { + return std::hash{}(ID.Value); + } +}; + +template <> +struct fmt::formatter : fmt::formatter { + using Base = fmt::formatter; + + // Pass-through the underlying value, so IDs can + // be formatted like any integral value. + template + auto format(const FEXCore::IR::NodeID& ID, FormatContext& ctx) const { + return Base::format(ID.Value, ctx); + } +}; + +template <> +struct fmt::formatter : fmt::formatter { + using Base = fmt::formatter; + + template + auto format(const FEXCore::IR::RegisterClassType& Class, FormatContext& ctx) const { + return Base::format(Class.Val, ctx); + } +}; + +template <> +struct fmt::formatter : fmt::formatter { + using Base = fmt::formatter; + + template + auto format(const FEXCore::IR::FenceType& Fence, FormatContext& ctx) const { + return Base::format(Fence.Val, ctx); + } +}; + +template <> +struct fmt::formatter : fmt::formatter> { + using Base = fmt::formatter>; + + template + auto format(const FEXCore::IR::OpSize& OpSize, FormatContext& ctx) const { + return Base::format(FEXCore::ToUnderlying(OpSize), ctx); + } +}; diff --git a/FEXCore/Source/Interface/IR/IRDumper.cpp b/FEXCore/Source/Interface/IR/IRDumper.cpp index 5cd4dcce07..b95e348328 100644 --- a/FEXCore/Source/Interface/IR/IRDumper.cpp +++ b/FEXCore/Source/Interface/IR/IRDumper.cpp @@ -6,9 +6,10 @@ tags: ir|dumper $end_info$ */ +#include "Interface/IR/IntrusiveIRList.h" +#include "Interface/IR/RegisterAllocationData.h" + #include -#include -#include #include #include diff --git a/FEXCore/Source/Interface/IR/IREmitter.cpp b/FEXCore/Source/Interface/IR/IREmitter.cpp index d811ce4acf..0da4e8e41d 100644 --- a/FEXCore/Source/Interface/IR/IREmitter.cpp +++ b/FEXCore/Source/Interface/IR/IREmitter.cpp @@ -9,7 +9,6 @@ tags: ir|emitter #include "Interface/IR/IREmitter.h" #include -#include #include #include diff --git a/FEXCore/Source/Interface/IR/IREmitter.h b/FEXCore/Source/Interface/IR/IREmitter.h index 35ab50221b..ebb974671b 100644 --- a/FEXCore/Source/Interface/IR/IREmitter.h +++ b/FEXCore/Source/Interface/IR/IREmitter.h @@ -1,7 +1,10 @@ // SPDX-License-Identifier: MIT #pragma once + +#include "Interface/IR/IR.h" +#include "Interface/IR/IntrusiveIRList.h" + #include -#include #include #include diff --git a/FEXCore/Source/Interface/IR/IRParser.cpp b/FEXCore/Source/Interface/IR/IRParser.cpp index 94c16b4c8b..1c941de865 100644 --- a/FEXCore/Source/Interface/IR/IRParser.cpp +++ b/FEXCore/Source/Interface/IR/IRParser.cpp @@ -7,8 +7,8 @@ tags: ir|parser */ #include "Interface/IR/IREmitter.h" + #include -#include #include #include #include diff --git a/FEXCore/include/FEXCore/IR/IntrusiveIRList.h b/FEXCore/Source/Interface/IR/IntrusiveIRList.h similarity index 99% rename from FEXCore/include/FEXCore/IR/IntrusiveIRList.h rename to FEXCore/Source/Interface/IR/IntrusiveIRList.h index 3b626d3374..c5f1e7e8e2 100644 --- a/FEXCore/include/FEXCore/IR/IntrusiveIRList.h +++ b/FEXCore/Source/Interface/IR/IntrusiveIRList.h @@ -1,7 +1,8 @@ // SPDX-License-Identifier: MIT #pragma once -#include "FEXCore/IR/IR.h" +#include "Interface/IR/IR.h" + #include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp b/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp index 192ae6520c..affa4468bc 100644 --- a/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp +++ b/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp @@ -17,7 +17,6 @@ desc: ConstProp, ZExt elim, addressgen coalesce, const pooling, fcmp reduction, #include "Interface/IR/PassManager.h" #include -#include #include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/DeadCodeElimination.cpp b/FEXCore/Source/Interface/IR/Passes/DeadCodeElimination.cpp index 28ee963c14..14a234a37d 100644 --- a/FEXCore/Source/Interface/IR/Passes/DeadCodeElimination.cpp +++ b/FEXCore/Source/Interface/IR/Passes/DeadCodeElimination.cpp @@ -9,7 +9,6 @@ tags: ir|opts #include "Interface/IR/PassManager.h" #include -#include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/DeadContextStoreElimination.cpp b/FEXCore/Source/Interface/IR/Passes/DeadContextStoreElimination.cpp index ca95b8333f..1fd7eca1b6 100644 --- a/FEXCore/Source/Interface/IR/Passes/DeadContextStoreElimination.cpp +++ b/FEXCore/Source/Interface/IR/Passes/DeadContextStoreElimination.cpp @@ -6,6 +6,7 @@ desc: Transforms ContextLoad/Store to temporaries, similar to mem2reg $end_info$ */ +#include "Interface/IR/IR.h" #include "Interface/IR/IREmitter.h" #include "Interface/IR/Passes.h" #include "Interface/IR/PassManager.h" @@ -13,7 +14,6 @@ desc: Transforms ContextLoad/Store to temporaries, similar to mem2reg #include #include #include -#include #include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/DeadStoreElimination.cpp b/FEXCore/Source/Interface/IR/Passes/DeadStoreElimination.cpp index 9bd5c62324..4b3a2d46b6 100644 --- a/FEXCore/Source/Interface/IR/Passes/DeadStoreElimination.cpp +++ b/FEXCore/Source/Interface/IR/Passes/DeadStoreElimination.cpp @@ -12,7 +12,6 @@ desc: Cross block store-after-store elimination #include #include #include -#include #include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/IRCompaction.cpp b/FEXCore/Source/Interface/IR/Passes/IRCompaction.cpp index 79ae90c7cb..a146ed5578 100644 --- a/FEXCore/Source/Interface/IR/Passes/IRCompaction.cpp +++ b/FEXCore/Source/Interface/IR/Passes/IRCompaction.cpp @@ -11,7 +11,6 @@ desc: Sorts the ssa storage in memory, needed for RA and others #include "Interface/Core/OpcodeDispatcher.h" #include -#include #include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp b/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp index 655b0a96c0..29b696ef8b 100644 --- a/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp +++ b/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp @@ -9,12 +9,11 @@ desc: Sanity checking pass #include "Interface/IR/IR.h" #include "Interface/IR/IREmitter.h" #include "Interface/IR/PassManager.h" +#include "Interface/IR/RegisterAllocationData.h" #include "Interface/IR/Passes/IRValidation.h" #include "Interface/IR/Passes/RegisterAllocationPass.h" #include -#include -#include #include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/InlineCallOptimization.cpp b/FEXCore/Source/Interface/IR/Passes/InlineCallOptimization.cpp index 8aef6ccc0c..8a0e7f4d00 100644 --- a/FEXCore/Source/Interface/IR/Passes/InlineCallOptimization.cpp +++ b/FEXCore/Source/Interface/IR/Passes/InlineCallOptimization.cpp @@ -11,7 +11,6 @@ desc: Removes unused arguments if known syscall number #include "Interface/IR/PassManager.h" #include -#include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/LongDivideRemovalPass.cpp b/FEXCore/Source/Interface/IR/Passes/LongDivideRemovalPass.cpp index 00a3c446bd..03d3cf14f9 100644 --- a/FEXCore/Source/Interface/IR/Passes/LongDivideRemovalPass.cpp +++ b/FEXCore/Source/Interface/IR/Passes/LongDivideRemovalPass.cpp @@ -9,7 +9,6 @@ desc: Long divide elimination pass #include "Interface/IR/IREmitter.h" #include "Interface/IR/PassManager.h" #include -#include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/RAValidation.cpp b/FEXCore/Source/Interface/IR/Passes/RAValidation.cpp index f506d6c619..6f95ecbab1 100644 --- a/FEXCore/Source/Interface/IR/Passes/RAValidation.cpp +++ b/FEXCore/Source/Interface/IR/Passes/RAValidation.cpp @@ -3,12 +3,11 @@ #include "Interface/IR/IR.h" #include "Interface/IR/IREmitter.h" #include "Interface/IR/PassManager.h" +#include "Interface/IR/RegisterAllocationData.h" #include "Interface/IR/Passes/IRValidation.h" #include "Interface/IR/Passes/RegisterAllocationPass.h" #include -#include -#include #include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/RedundantFlagCalculationElimination.cpp b/FEXCore/Source/Interface/IR/Passes/RedundantFlagCalculationElimination.cpp index 2df6270d8d..b3167481dd 100644 --- a/FEXCore/Source/Interface/IR/Passes/RedundantFlagCalculationElimination.cpp +++ b/FEXCore/Source/Interface/IR/Passes/RedundantFlagCalculationElimination.cpp @@ -10,7 +10,6 @@ desc: This is not used right now, possibly broken #include "Interface/IR/IREmitter.h" #include -#include #include #include "Interface/IR/PassManager.h" diff --git a/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp b/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp index e2a2c8de51..fe714b5459 100644 --- a/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp +++ b/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp @@ -9,12 +9,12 @@ tags: ir|opts #include "Interface/IR/Passes/RegisterAllocationPass.h" #include "FEXCore/Core/X86Enums.h" +#include "Interface/IR/IR.h" #include "Interface/IR/IREmitter.h" +#include "Interface/IR/RegisterAllocationData.h" #include "Interface/IR/Passes.h" #include #include -#include -#include #include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/ValueDominanceValidation.cpp b/FEXCore/Source/Interface/IR/Passes/ValueDominanceValidation.cpp index 39f83e8fe0..adf731a885 100644 --- a/FEXCore/Source/Interface/IR/Passes/ValueDominanceValidation.cpp +++ b/FEXCore/Source/Interface/IR/Passes/ValueDominanceValidation.cpp @@ -11,7 +11,6 @@ desc: Sanity Checking #include "Interface/IR/PassManager.h" #include -#include #include #include #include diff --git a/FEXCore/include/FEXCore/IR/RegisterAllocationData.h b/FEXCore/Source/Interface/IR/RegisterAllocationData.h similarity index 100% rename from FEXCore/include/FEXCore/IR/RegisterAllocationData.h rename to FEXCore/Source/Interface/IR/RegisterAllocationData.h diff --git a/FEXCore/include/FEXCore/Core/CoreState.h b/FEXCore/include/FEXCore/Core/CoreState.h index 47a6c09ae6..e3a465f4a0 100644 --- a/FEXCore/include/FEXCore/Core/CoreState.h +++ b/FEXCore/include/FEXCore/Core/CoreState.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: MIT #pragma once -#include "FEXCore/Core/X86Enums.h" -#include "FEXCore/IR/IR.h" +#include +#include #include #include #include diff --git a/FEXCore/include/FEXCore/Debug/InternalThreadState.h b/FEXCore/include/FEXCore/Debug/InternalThreadState.h index 4647796acf..cc1112751f 100644 --- a/FEXCore/include/FEXCore/Debug/InternalThreadState.h +++ b/FEXCore/include/FEXCore/Debug/InternalThreadState.h @@ -3,8 +3,6 @@ #include #include #include -#include -#include #include #include #include diff --git a/FEXCore/include/FEXCore/IR/IR.h b/FEXCore/include/FEXCore/IR/IR.h index 1ca505867e..a09c26885d 100644 --- a/FEXCore/include/FEXCore/IR/IR.h +++ b/FEXCore/include/FEXCore/IR/IR.h @@ -8,13 +8,8 @@ #include #include -#include -#include #include #include -#include -#include -#include #include @@ -24,477 +19,6 @@ class OrderedNode; class RegisterAllocationPass; class RegisterAllocationData; -/** - * @brief The IROp_Header is an dynamically sized array - * At the end it contains a uint8_t for the number of arguments that Op has - * Then there is an unsized array of NodeWrapper arguments for the number of arguments this op has - * The op structures that are including the header must ensure that they pad themselves correctly to the number of arguments used - */ -struct IROp_Header; - -/** - * @brief Represents the ID of a given IR node. - * - * Intended to provide strong typing from other integer values - * to prevent passing incorrect values to certain API functions. - */ -struct NodeID final { - using value_type = uint32_t; - - constexpr NodeID() noexcept = default; - constexpr explicit NodeID(value_type Value_) noexcept : Value{Value_} {} - - constexpr NodeID(const NodeID&) noexcept = default; - constexpr NodeID& operator=(const NodeID&) noexcept = default; - - constexpr NodeID(NodeID&&) noexcept = default; - constexpr NodeID& operator=(NodeID&&) noexcept = default; - - [[nodiscard]] constexpr bool IsValid() const noexcept { - return Value != 0; - } - [[nodiscard]] constexpr bool IsInvalid() const noexcept { - return !IsValid(); - } - constexpr void Invalidate() noexcept { - Value = 0; - } - - [[nodiscard]] friend constexpr bool operator==(NodeID, NodeID) noexcept = default; - - [[nodiscard]] friend constexpr bool operator<(NodeID lhs, NodeID rhs) noexcept { - return lhs.Value < rhs.Value; - } - [[nodiscard]] friend constexpr bool operator>(NodeID lhs, NodeID rhs) noexcept { - return operator<(rhs, lhs); - } - [[nodiscard]] friend constexpr bool operator<=(NodeID lhs, NodeID rhs) noexcept { - return !operator>(lhs, rhs); - } - [[nodiscard]] friend constexpr bool operator>=(NodeID lhs, NodeID rhs) noexcept { - return !operator<(lhs, rhs); - } - - friend std::ostream& operator<<(std::ostream& out, NodeID ID) { - out << ID.Value; - return out; - } - friend std::istream& operator>>(std::istream& in, NodeID& ID) { - in >> ID.Value; - return in; - } - - value_type Value{}; -}; - -/** - * @brief This is a very simple wrapper for our node pointers - * You probably don't want to use this directly - * Use OpNodeWrapper and OrderedNodeWrapper types below instead - * - * This is necessary to allow two things - * - Reduce memory usage by having the pointer be an 32bit offset rather than the whole 64bit pointer - * - Actually use an offset from a base so we aren't storing pointers for everything - * - Makes IR list copying be as cheap as a memcpy - * Downsides - * - The IR nodes have to be allocated out of a linear array of memory - * - We currently only allow a 32bit offset, so *only* 4 million nodes per list - * - We have to have the base offset live somewhere else - * - Has to be POD and trivially copyable - * - Makes every real node access turn in to a [Base + Offset] access - * - Can be confusing if you're mixing OpNodeWrapper and OrderedNodeWrapper usage - */ -template -struct NodeWrapperBase final { - // On x86-64 using a uint64_t type is more efficient since RIP addressing gives you [ + + ] - // On AArch64 using uint32_t is just more memory efficient. 32bit or 64bit offset doesn't matter - // We use uint32_t to be more memory efficient (Cuts our node list size in half) - using NodeOffsetType = uint32_t; - NodeOffsetType NodeOffset; - - explicit NodeWrapperBase() = default; - - [[nodiscard]] static NodeWrapperBase WrapOffset(NodeOffsetType Offset) { - NodeWrapperBase Wrapped; - Wrapped.NodeOffset = Offset; - return Wrapped; - } - - [[nodiscard]] static NodeWrapperBase WrapPtr(uintptr_t Base, uintptr_t Value) { - NodeWrapperBase Wrapped; - Wrapped.SetOffset(Base, Value); - return Wrapped; - } - - [[nodiscard]] static void *UnwrapNode(uintptr_t Base, NodeWrapperBase Node) { - return Node.GetNode(Base); - } - - [[nodiscard]] NodeID ID() const; - - [[nodiscard]] bool IsInvalid() const { return NodeOffset == 0; } - - [[nodiscard]] Type *GetNode(uintptr_t Base) { - return reinterpret_cast(Base + NodeOffset); - } - [[nodiscard]] const Type *GetNode(uintptr_t Base) const { - return reinterpret_cast(Base + NodeOffset); - } - - void SetOffset(uintptr_t Base, uintptr_t Value) { NodeOffset = Value - Base; } - - [[nodiscard]] friend constexpr bool operator==(const NodeWrapperBase&, const NodeWrapperBase&) = default; -}; - -static_assert(std::is_trivial_v>); - -static_assert(sizeof(NodeWrapperBase) == sizeof(uint32_t)); - -using OpNodeWrapper = NodeWrapperBase; -using OrderedNodeWrapper = NodeWrapperBase; - -struct OrderedNodeHeader { - OpNodeWrapper Value; - OrderedNodeWrapper Next; - OrderedNodeWrapper Previous; -}; - -static_assert(sizeof(OrderedNodeHeader) == sizeof(uint32_t) * 3); - -/** - * @brief This is a node in our IR representation - * Is a doubly linked list node that lives in a representation of a linearly allocated node list - * The links in the nodes can live in a list independent of the data IR data - * - * ex. - * Region1 : ... <-> <-> <-> ... - * | * | - * v v - * Region2 : ...... - * - * In this example the OrderedNodes are allocated in one linear memory region (Not necessarily contiguous with one another linking) - * The second region is contiguous but they don't have any relationship with one another directly - */ -class OrderedNode final { - friend class NodeWrapperIterator; - friend class OrderedList; - public: - // These three values are laid out very specifically to make it fast to access the NodeWrappers specifically - OrderedNodeHeader Header; - uint32_t NumUses; - - using value_type = OrderedNodeWrapper; - - OrderedNode() = default; - - /** - * @brief Appends a node to this current node - * - * Before. <-> <-> - * After. <-> <-> <-> Next - * - * @return Pointer to the node being added - */ - value_type append(uintptr_t Base, value_type Node) { - // Set Next Node's Previous to incoming node - SetPrevious(Base, Header.Next, Node); - - // Set Incoming node's links to this node's links - SetPrevious(Base, Node, Wrapped(Base)); - SetNext(Base, Node, Header.Next); - - // Set this node's next to the incoming node - SetNext(Base, Wrapped(Base), Node); - - // Return the node we are appending - return Node; - } - - OrderedNode *append(uintptr_t Base, OrderedNode *Node) { - value_type WNode = Node->Wrapped(Base); - // Set Next Node's Previous to incoming node - SetPrevious(Base, Header.Next, WNode); - - // Set Incoming node's links to this node's links - SetPrevious(Base, WNode, Wrapped(Base)); - SetNext(Base, WNode, Header.Next); - - // Set this node's next to the incoming node - SetNext(Base, Wrapped(Base), WNode); - - // Return the node we are appending - return Node; - } - - /** - * @brief Prepends a node to the current node - * Before. <-> <-> - * After. <-> <-> <-> Next - * - * @return Pointer to the node being added - */ - value_type prepend(uintptr_t Base, value_type Node) { - // Set the previous node's next to the incoming node - SetNext(Base, Header.Previous, Node); - - // Set the incoming node's links - SetPrevious(Base, Node, Header.Previous); - SetNext(Base, Node, Wrapped(Base)); - - // Set the current node's link - SetPrevious(Base, Wrapped(Base), Node); - - // Return the node we are prepending - return Node; - } - - OrderedNode *prepend(uintptr_t Base, OrderedNode *Node) { - value_type WNode = Node->Wrapped(Base); - // Set the previous node's next to the incoming node - SetNext(Base, Header.Previous, WNode); - - // Set the incoming node's links - SetPrevious(Base, WNode, Header.Previous); - SetNext(Base, WNode, Wrapped(Base)); - - // Set the current node's link - SetPrevious(Base, Wrapped(Base), WNode); - - // Return the node we are prepending - return Node; - } - - /** - * @brief Gets the remaining size of the blocks from this point onward - * - * Doesn't find the head of the list - * - */ - [[nodiscard]] size_t size(uintptr_t Base) const { - size_t Size = 1; - // Walk the list forward until we hit a sentinel - value_type Current = Header.Next; - while (Current.NodeOffset != 0) { - ++Size; - OrderedNode *RealNode = Current.GetNode(Base); - Current = RealNode->Header.Next; - } - return Size; - } - - void Unlink(uintptr_t Base) { - // This removes the node from the list. Orphaning it - // Before: <-> <-> - // After: - SetNext(Base, Header.Previous, Header.Next); - SetPrevious(Base, Header.Next, Header.Previous); - } - - [[nodiscard]] IROp_Header const* Op(uintptr_t Base) const { - return Header.Value.GetNode(Base); - } - [[nodiscard]] IROp_Header *Op(uintptr_t Base) { - return Header.Value.GetNode(Base); - } - - [[nodiscard]] uint32_t GetUses() const { return NumUses; } - - void AddUse() { ++NumUses; } - void RemoveUse() { --NumUses; } - - [[nodiscard]] value_type Wrapped(uintptr_t Base) const { - value_type Tmp; - Tmp.SetOffset(Base, reinterpret_cast(this)); - return Tmp; - } - - private: - [[nodiscard]] value_type WrappedOffset(uint32_t Offset) const { - value_type Tmp; - Tmp.NodeOffset = Offset; - return Tmp; - } - - static void SetPrevious(uintptr_t Base, value_type Node, value_type New) { - OrderedNode *RealNode = Node.GetNode(Base); - RealNode->Header.Previous = New; - } - - static void SetNext(uintptr_t Base, value_type Node, value_type New) { - OrderedNode *RealNode = Node.GetNode(Base); - RealNode->Header.Next = New; - } - - void SetUses(uint32_t Uses) { NumUses = Uses; } -}; - -static_assert(std::is_trivial_v); -static_assert(std::is_trivially_copyable_v); -static_assert(offsetof(OrderedNode, Header) == 0); -static_assert(sizeof(OrderedNode) == (sizeof(OrderedNodeHeader) + sizeof(uint32_t))); - -struct RegisterClassType final { - using value_type = uint32_t; - - value_type Val; - [[nodiscard]] constexpr operator value_type() const { - return Val; - } - [[nodiscard]] friend constexpr bool operator==(const RegisterClassType&, const RegisterClassType&) = default; -}; - -struct CondClassType final { - uint8_t Val; - [[nodiscard]] constexpr operator uint8_t() const { - return Val; - } - [[nodiscard]] friend constexpr bool operator==(const CondClassType&, const CondClassType&) = default; -}; - -struct MemOffsetType final { - uint8_t Val; - [[nodiscard]] constexpr operator uint8_t() const { - return Val; - } - [[nodiscard]] friend constexpr bool operator==(const MemOffsetType&, const MemOffsetType&) = default; -}; - -struct TypeDefinition final { - uint16_t Val; - - [[nodiscard]] constexpr operator uint16_t() const { - return Val; - } - - [[nodiscard]] static constexpr TypeDefinition Create(uint8_t Bytes) { - TypeDefinition Type{}; - Type.Val = Bytes << 8; - return Type; - } - - [[nodiscard]] static constexpr TypeDefinition Create(uint8_t Bytes, uint8_t Elements) { - TypeDefinition Type{}; - Type.Val = (Bytes << 8) | (Elements & 255); - return Type; - } - - [[nodiscard]] constexpr uint8_t Bytes() const { - return Val >> 8; - } - - [[nodiscard]] constexpr uint8_t Elements() const { - return Val & 255; - } - - [[nodiscard]] friend constexpr bool operator==(const TypeDefinition&, const TypeDefinition&) = default; -}; - -static_assert(std::is_trivial_v); - -struct FenceType final { - using value_type = uint8_t; - - value_type Val; - [[nodiscard]] constexpr operator value_type() const { - return Val; - } - [[nodiscard]] friend constexpr bool operator==(const FenceType&, const FenceType&) = default; -}; - -struct RoundType final { - uint8_t Val; - [[nodiscard]] constexpr operator uint8_t() const { - return Val; - } - [[nodiscard]] friend constexpr bool operator==(const RoundType&, const RoundType&) = default; -}; - -struct SHA256Sum final { - uint8_t data[32]; - [[nodiscard]] bool operator<(SHA256Sum const &rhs) const { - return memcmp(data, rhs.data, sizeof(data)) < 0; - } - - [[nodiscard]] bool operator==(SHA256Sum const &rhs) const { - return memcmp(data, rhs.data, sizeof(data)) == 0; - } -}; - -typedef void ThunkedFunction(void* ArgsRv); - -struct ThunkDefinition final { - SHA256Sum Sum; - ThunkedFunction *ThunkFunction; -}; - -class NodeIterator; - -/* This iterator can be used to step though nodes. - * Due to how our IR is laid out, this can be used to either step - * though the CodeBlocks or though the code within a single block. - */ -class NodeIterator { -public: - using value_type = std::tuple; - using size_type = std::size_t; - using difference_type = std::ptrdiff_t; - using reference = value_type&; - using const_reference = const value_type&; - using pointer = value_type*; - using const_pointer = const value_type*; - using iterator = NodeIterator; - using const_iterator = const NodeIterator; - using reverse_iterator = iterator; - using const_reverse_iterator = const_iterator; - using iterator_category = std::bidirectional_iterator_tag; - - NodeIterator(uintptr_t Base, uintptr_t IRBase) : BaseList {Base}, IRList{ IRBase } {} - explicit NodeIterator(uintptr_t Base, uintptr_t IRBase, OrderedNodeWrapper Ptr) : BaseList {Base}, IRList{ IRBase }, Node {Ptr} {} - - [[nodiscard]] bool operator==(const NodeIterator &rhs) const { - return Node.NodeOffset == rhs.Node.NodeOffset; - } - - [[nodiscard]] bool operator!=(const NodeIterator &rhs) const { - return !operator==(rhs); - } - - NodeIterator operator++() { - OrderedNodeHeader *RealNode = reinterpret_cast(Node.GetNode(BaseList)); - Node = RealNode->Next; - return *this; - } - - NodeIterator operator--() { - OrderedNodeHeader *RealNode = reinterpret_cast(Node.GetNode(BaseList)); - Node = RealNode->Previous; - return *this; - } - - [[nodiscard]] value_type operator*() { - OrderedNode *RealNode = Node.GetNode(BaseList); - return { RealNode, RealNode->Op(IRList) }; - } - - [[nodiscard]] value_type operator()() { - OrderedNode *RealNode = Node.GetNode(BaseList); - return { RealNode, RealNode->Op(IRList) }; - } - - [[nodiscard]] NodeID ID() const { - return Node.ID(); - } - - [[nodiscard]] static NodeIterator Invalid() { - return NodeIterator(0, 0); - } - -protected: - uintptr_t BaseList{}; - uintptr_t IRList{}; - OrderedNodeWrapper Node{}; -}; - enum class SyscallFlags : uint8_t { DEFAULT = 0, // Syscalldoesn't care about CPUState being serialized up to the syscall instruction. @@ -555,168 +79,22 @@ enum IndexNamedVectorConstant : uint8_t { INDEXED_NAMED_VECTOR_MAX, }; -// This must directly match bytes to the named opsize. -// Implicit sized IR operations does math to get between sizes. -enum OpSize : uint8_t { - i8Bit = 1, - i16Bit = 2, - i32Bit = 4, - i64Bit = 8, - i128Bit = 16, - i256Bit = 32, -}; - -enum class FloatCompareOp : uint8_t { - EQ = 0, - LT, - LE, - UNO, - NEQ, - ORD, -}; - -enum class ShiftType : uint8_t { - LSL = 0, - LSR, - ASR, - ROR, -}; - -// Converts a size stored as an integer in to an OpSize enum. -// This is a nop operation and will be eliminated by the compiler. -static inline OpSize SizeToOpSize(uint8_t Size) { - switch (Size) { - case 1: return OpSize::i8Bit; - case 2: return OpSize::i16Bit; - case 4: return OpSize::i32Bit; - case 8: return OpSize::i64Bit; - case 16: return OpSize::i128Bit; - case 32: return OpSize::i256Bit; - default: FEX_UNREACHABLE; - } -} - -#define IROP_ENUM -#define IROP_STRUCTS -#define IROP_SIZES -#define IROP_REG_CLASSES -#include - -/* This iterator can be used to step though every single node in a multi-block in SSA order. - * - * Iterates in the order of: - * - * end <-- CodeBlockA <--> BlockAInst1 <--> BlockAInst2 <--> CodeBlockB <--> BlockBInst1 <--> BlockBInst2 --> end - */ -class AllNodesIterator : public NodeIterator { -public: - AllNodesIterator(uintptr_t Base, uintptr_t IRBase) : NodeIterator(Base, IRBase) {} - explicit AllNodesIterator(uintptr_t Base, uintptr_t IRBase, OrderedNodeWrapper Ptr) : NodeIterator(Base, IRBase, Ptr) {} - AllNodesIterator(NodeIterator other) : NodeIterator(other) {} // Allow NodeIterator to be upgraded - - AllNodesIterator operator++() { - OrderedNodeHeader *RealNode = reinterpret_cast(Node.GetNode(BaseList)); - auto IROp = Node.GetNode(BaseList)->Op(IRList); - - // If this is the last node of a codeblock, we need to continue to the next block - if (IROp->Op == OP_ENDBLOCK) { - auto EndBlock = IROp->C(); - - auto CurrentBlock = EndBlock->BlockHeader.GetNode(BaseList); - Node = CurrentBlock->Header.Next; - } else if (IROp->Op == OP_CODEBLOCK) { - auto CodeBlock = IROp->C(); - - Node = CodeBlock->Begin; - } else { - Node = RealNode->Next; - } - - return *this; - } - - AllNodesIterator operator--() { - auto IROp = Node.GetNode(BaseList)->Op(IRList); - - if (IROp->Op == OP_BEGINBLOCK) { - auto BeginBlock = IROp->C(); - - Node = BeginBlock->BlockHeader; - } else if (IROp->Op == OP_CODEBLOCK) { - auto PrevBlockWrapper = Node.GetNode(BaseList)->Header.Previous; - auto PrevCodeBlock = PrevBlockWrapper.GetNode(BaseList)->Op(IRList)->C(); - - Node = PrevCodeBlock->Last; - } else { - Node = Node.GetNode(BaseList)->Header.Previous; - } - - return *this; - } - - [[nodiscard]] static AllNodesIterator Invalid() { - return AllNodesIterator(0, 0); - } -}; - -class IRListView; -class IREmitter; - -template -inline NodeID NodeWrapperBase::ID() const { - return NodeID(NodeOffset / sizeof(IR::OrderedNode)); -} - -bool IsFragmentExit(FEXCore::IR::IROps Op); -bool IsBlockExit(FEXCore::IR::IROps Op); - -} // namespace FEXCore::IR - -template <> -struct std::hash { - size_t operator()(const FEXCore::IR::NodeID& ID) const noexcept { - return std::hash{}(ID.Value); - } -}; - -template <> -struct fmt::formatter : fmt::formatter { - using Base = fmt::formatter; - - // Pass-through the underlying value, so IDs can - // be formatted like any integral value. - template - auto format(const FEXCore::IR::NodeID& ID, FormatContext& ctx) const { - return Base::format(ID.Value, ctx); +struct SHA256Sum final { + uint8_t data[32]; + [[nodiscard]] bool operator<(SHA256Sum const &rhs) const { + return memcmp(data, rhs.data, sizeof(data)) < 0; } -}; -template <> -struct fmt::formatter : fmt::formatter { - using Base = fmt::formatter; - - template - auto format(const FEXCore::IR::RegisterClassType& Class, FormatContext& ctx) const { - return Base::format(Class.Val, ctx); + [[nodiscard]] bool operator==(SHA256Sum const &rhs) const { + return memcmp(data, rhs.data, sizeof(data)) == 0; } }; -template <> -struct fmt::formatter : fmt::formatter { - using Base = fmt::formatter; +typedef void ThunkedFunction(void* ArgsRv); - template - auto format(const FEXCore::IR::FenceType& Fence, FormatContext& ctx) const { - return Base::format(Fence.Val, ctx); - } +struct ThunkDefinition final { + SHA256Sum Sum; + ThunkedFunction *ThunkFunction; }; -template <> -struct fmt::formatter : fmt::formatter> { - using Base = fmt::formatter>; - - template - auto format(const FEXCore::IR::OpSize& OpSize, FormatContext& ctx) const { - return Base::format(FEXCore::ToUnderlying(OpSize), ctx); - } -}; +} // namespace FEXCore::IR