From 2f926342d5faa95ae9d9f2d95f511e8b532613a4 Mon Sep 17 00:00:00 2001 From: Anton Tokar Date: Thu, 13 Apr 2023 11:14:00 -0600 Subject: [PATCH] Initial Commit --- .gitignore | 8 + .swift-format | 7 + CHANGELOG.md | 12 + CODE_OF_CONDUCT.md | 133 ++++ LICENSE | 29 + Package.swift | 64 ++ README.md | 40 ++ .../StructureKit/Metal/STKARKitGeometry.metal | 57 ++ .../Metal/STKARKitOverlayRenderer.swift | 100 +++ Sources/StructureKit/Metal/STKBindings.swift | 78 +++ .../StructureKit/Metal/STKColorFrame.metal | 61 ++ .../Metal/STKColorFrameRenderer.swift | 97 +++ Sources/StructureKit/Metal/STKCommon.swift | 297 +++++++++ .../StructureKit/Metal/STKDepthFrame.metal | 69 +++ .../StructureKit/Metal/STKDepthOverlay.metal | 90 +++ .../StructureKit/Metal/STKDepthRenderer.swift | 302 +++++++++ .../Metal/STKDrawableObject.swift | 54 ++ Sources/StructureKit/Metal/STKLine.metal | 62 ++ .../StructureKit/Metal/STKLineOverDepth.metal | 83 +++ .../StructureKit/Metal/STKLineRenderer.swift | 180 ++++++ .../StructureKit/Metal/STKMeshBuffers.swift | 267 ++++++++ .../StructureKit/Metal/STKMeshRenderers.swift | 581 ++++++++++++++++++ Sources/StructureKit/Metal/STKMetalCommon.h | 34 + .../StructureKit/Metal/STKMetalCommon.metal | 56 ++ Sources/StructureKit/Metal/STKMetalData.h | 152 +++++ .../Metal/STKMetalLibLoader.swift | 42 ++ .../StructureKit/Metal/STKMetalRenderer.swift | 358 +++++++++++ .../Metal/STKPerVertexColor.metal | 53 ++ .../StructureKit/Metal/STKPointCloud.metal | 56 ++ .../StructureKit/Metal/STKSolidColor.metal | 58 ++ Sources/StructureKit/Metal/STKTexture.metal | 61 ++ Sources/StructureKit/Metal/STKWireframe.metal | 61 ++ .../STKMixedActivityItemSource.swift | 75 +++ .../StructureKitCTypes/StructureKitCTypes.mm | 29 + .../include/StructureKitCTypes.h | 29 + 35 files changed, 3735 insertions(+) create mode 100644 .gitignore create mode 100644 .swift-format create mode 100644 CHANGELOG.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 LICENSE create mode 100644 Package.swift create mode 100644 README.md create mode 100644 Sources/StructureKit/Metal/STKARKitGeometry.metal create mode 100644 Sources/StructureKit/Metal/STKARKitOverlayRenderer.swift create mode 100644 Sources/StructureKit/Metal/STKBindings.swift create mode 100644 Sources/StructureKit/Metal/STKColorFrame.metal create mode 100644 Sources/StructureKit/Metal/STKColorFrameRenderer.swift create mode 100644 Sources/StructureKit/Metal/STKCommon.swift create mode 100644 Sources/StructureKit/Metal/STKDepthFrame.metal create mode 100644 Sources/StructureKit/Metal/STKDepthOverlay.metal create mode 100644 Sources/StructureKit/Metal/STKDepthRenderer.swift create mode 100644 Sources/StructureKit/Metal/STKDrawableObject.swift create mode 100644 Sources/StructureKit/Metal/STKLine.metal create mode 100644 Sources/StructureKit/Metal/STKLineOverDepth.metal create mode 100644 Sources/StructureKit/Metal/STKLineRenderer.swift create mode 100644 Sources/StructureKit/Metal/STKMeshBuffers.swift create mode 100644 Sources/StructureKit/Metal/STKMeshRenderers.swift create mode 100644 Sources/StructureKit/Metal/STKMetalCommon.h create mode 100644 Sources/StructureKit/Metal/STKMetalCommon.metal create mode 100644 Sources/StructureKit/Metal/STKMetalData.h create mode 100644 Sources/StructureKit/Metal/STKMetalLibLoader.swift create mode 100644 Sources/StructureKit/Metal/STKMetalRenderer.swift create mode 100644 Sources/StructureKit/Metal/STKPerVertexColor.metal create mode 100644 Sources/StructureKit/Metal/STKPointCloud.metal create mode 100644 Sources/StructureKit/Metal/STKSolidColor.metal create mode 100644 Sources/StructureKit/Metal/STKTexture.metal create mode 100644 Sources/StructureKit/Metal/STKWireframe.metal create mode 100644 Sources/StructureKit/STKMixedActivityItemSource.swift create mode 100644 Sources/StructureKitCTypes/StructureKitCTypes.mm create mode 100644 Sources/StructureKitCTypes/include/StructureKitCTypes.h diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b63bf7f --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.DS_Store +/.build +/Packages +/*.xcodeproj +xcuserdata/ +DerivedData/ +.swiftpm/ +.netrc diff --git a/.swift-format b/.swift-format new file mode 100644 index 0000000..d20201e --- /dev/null +++ b/.swift-format @@ -0,0 +1,7 @@ +{ + "version": 1, + "lineLength": 120, + "indentation": { + "spaces": 2 + } +} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a8bdccb --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). + +## 1.0.0 +Initial release + +### Added + +* Visualization helpers. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..4aec9d1 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,133 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +opensourcereports@structure.io. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..618e5fe --- /dev/null +++ b/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2022, XRPro, LLC. https://structure.io +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Package.swift b/Package.swift new file mode 100644 index 0000000..cd23f8e --- /dev/null +++ b/Package.swift @@ -0,0 +1,64 @@ +// swift-tools-version: 5.8 +// +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import PackageDescription + +let package = Package( + name: "StructureKit", + platforms: [ + .iOS(.v13) + ], + products: [ + .library( + name: "StructureKit", + targets: ["StructureKit"]) + ], + dependencies: [], + targets: [ + .target( + name: "StructureKit", + dependencies: ["StructureKitCTypes"], + linkerSettings: [ + .linkedFramework("ARKit"), + .linkedFramework("Accelerate"), + .linkedFramework("CoreMedia"), + .linkedFramework("CoreVideo"), + .linkedFramework("GLKit"), + .linkedFramework("Metal"), + .linkedFramework("MetalKit"), + .linkedFramework("UIKit"), + .linkedLibrary("swiftsimd"), + ] + ), + .target( + name: "StructureKitCTypes" + ), + ] +) diff --git a/README.md b/README.md new file mode 100644 index 0000000..6c6d384 --- /dev/null +++ b/README.md @@ -0,0 +1,40 @@ +# StructureKit + +A collection of helper tools for [Structure SDK](https://structure.io/developers) + +## Installation +You can consume it as a Swift Package: +[Adding package dependencies to your app](https://developer.apple.com/documentation/xcode/adding-package-dependencies-to-your-app) + +## Usage +Import into your Swift source code: +```swift +import StructurKit +``` + +### Integration with Structure SDK +StructureKit based on Structure SDK - compatible interfaces. + +Current version is compatible with Structure SDK `2.*` + +To seamlessly use Structure SDK types together with StructureKit, add the following code in your project: + +```swift +import StructureKit + +extension STMesh : STKMesh { +} + +extension STColorFrame : STKColorFrame { +} + +extension STIntrinsics : STKIntrinsics { +} + +extension STDepthFrame : STKDepthFrame { + public func intrinsics() -> STKIntrinsics { + let i : STIntrinsics = self.intrinsics() + return i; + } +} +``` diff --git a/Sources/StructureKit/Metal/STKARKitGeometry.metal b/Sources/StructureKit/Metal/STKARKitGeometry.metal new file mode 100644 index 0000000..50020f5 --- /dev/null +++ b/Sources/StructureKit/Metal/STKARKitGeometry.metal @@ -0,0 +1,57 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#include "STKMetalData.h" + +#include + +using namespace metal; + +struct VertexOut +{ + float4 position [[position]]; + float4 color; +}; + +vertex VertexOut vertexARKit( + const device float3* vertex_array [[buffer(0)]], + const device STKUniformsMesh& uniforms [[buffer(1)]], + unsigned int vid [[vertex_id]]) +{ + float4 posIn = float4(vertex_array[vid], 1); + + VertexOut out; + out.position = uniforms.projectionMatrix * uniforms.modelViewMatrix * posIn; + out.color = uniforms.color; + return out; +} + +fragment float4 fragmentARKit(VertexOut in [[stage_in]]) +{ + return in.color; +} diff --git a/Sources/StructureKit/Metal/STKARKitOverlayRenderer.swift b/Sources/StructureKit/Metal/STKARKitOverlayRenderer.swift new file mode 100644 index 0000000..60d77b4 --- /dev/null +++ b/Sources/StructureKit/Metal/STKARKitOverlayRenderer.swift @@ -0,0 +1,100 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import Metal +import MetalKit +import StructureKitCTypes + +// Draws an ARKit face geometry as a white transparent mesh +class STKARKitOverlayRenderer { + var arkitToWorld = simd_float4x4() + private var depthStencilARKitState: MTLDepthStencilState + private var renderARKitState: MTLRenderPipelineState + + init(view: MTKView, device: MTLDevice) { + let depthStencilDescriptor = MTLDepthStencilDescriptor() + depthStencilDescriptor.depthCompareFunction = .less + depthStencilDescriptor.isDepthWriteEnabled = true + depthStencilARKitState = device.makeDepthStencilState(descriptor: depthStencilDescriptor)! + + let vertexDescriptor = MTLVertexDescriptor() + vertexDescriptor.attributes[0] = MTLVertexAttributeDescriptor(bufferIndex: 0, offset: 0, format: .float3) // vertices + vertexDescriptor.layouts[0].stride = MemoryLayout.stride + + let library = STKMetalLibLoader.load(device: device) + let pipelineMeshDescriptor = MTLRenderPipelineDescriptor() + pipelineMeshDescriptor.sampleCount = 1 + pipelineMeshDescriptor.colorAttachments[0].pixelFormat = view.colorPixelFormat + pipelineMeshDescriptor.depthAttachmentPixelFormat = view.depthStencilPixelFormat + pipelineMeshDescriptor.vertexDescriptor = vertexDescriptor + + pipelineMeshDescriptor.vertexFunction = library.makeFunction(name: "vertexARKit") + pipelineMeshDescriptor.fragmentFunction = library.makeFunction(name: "fragmentARKit") + + // blending + pipelineMeshDescriptor.colorAttachments[0].isBlendingEnabled = true + pipelineMeshDescriptor.colorAttachments[0].rgbBlendOperation = .add + pipelineMeshDescriptor.colorAttachments[0].sourceRGBBlendFactor = .sourceAlpha + pipelineMeshDescriptor.colorAttachments[0].destinationRGBBlendFactor = .oneMinusSourceAlpha + + renderARKitState = try! device.makeRenderPipelineState(descriptor: pipelineMeshDescriptor) + } + + func renderARkitGeom( + _ commandEncoder: MTLRenderCommandEncoder, + mesh: STKDrawableObject, + cameraPosition: float4x4, + projection: float4x4, + orientation: float4x4 + ) { + guard let vertexBuffer = mesh.vertices(), + let indexBuffer = mesh.indices() + else { return } + + commandEncoder.pushDebugGroup("RenderARKitGeometry") + commandEncoder.setDepthStencilState(depthStencilARKitState) + commandEncoder.setRenderPipelineState(renderARKitState) + + commandEncoder.setVertexBuffer(vertexBuffer, offset: 0, index: Int(STKVertexAttrPosition.rawValue)) + + var uniforms = STKUniformsMesh( + modelViewMatrix: float4x4.makeRotationZ(Float.pi) * cameraPosition.inverse * arkitToWorld, + projectionMatrix: orientation * projection, + color: vector_float4(1, 1, 1, 0.5)) + commandEncoder.setVertexBytes(&uniforms, length: MemoryLayout.stride, index: 1) + + commandEncoder.drawIndexedPrimitives( + type: .triangle, + indexCount: mesh.triangleCount() * 3, + indexType: .uint16, + indexBuffer: indexBuffer, + indexBufferOffset: 0) + + commandEncoder.popDebugGroup() + } +} diff --git a/Sources/StructureKit/Metal/STKBindings.swift b/Sources/StructureKit/Metal/STKBindings.swift new file mode 100644 index 0000000..0646234 --- /dev/null +++ b/Sources/StructureKit/Metal/STKBindings.swift @@ -0,0 +1,78 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import CoreMedia +import CoreVideo +import GLKit + +public protocol STKColorFrame { + var sampleBuffer: CMSampleBuffer! { get } + func glProjectionMatrix() -> GLKMatrix4 +} + +public protocol STKDepthFrame { + var width: Int32 { get } + var height: Int32 { get } + var depthInMillimeters: UnsafeMutablePointer! { get } + func intrinsics() -> STKIntrinsics + func glProjectionMatrix() -> GLKMatrix4 +} + +/// Bridged minimal STMesh prototol from Structure SDK. +/// extension STIntrinsics : STKIntrinsics { +/// } +public protocol STKIntrinsics { + var width: Int32 { get set } + var height: Int32 { get set } + var fx: Float { get set } + var fy: Float { get set } + var cx: Float { get set } + var cy: Float { get set } + var k1: Float { get set } + var k2: Float { get set } +} + +/// Bridged minimal STMesh prototol from Structure SDK. +/// extension STMesh : STKMesh { +/// } +public protocol STKMesh { + func numberOfMeshes() -> Int32 + func number(ofMeshFaces meshIndex: Int32) -> Int32 + func number(ofMeshVertices meshIndex: Int32) -> Int32 + func number(ofMeshLines meshIndex: Int32) -> Int32 + func meshVertices(_ meshIndex: Int32) -> UnsafeMutablePointer! + func hasPerVertexNormals() -> Bool + func meshPerVertexNormals(_ meshIndex: Int32) -> UnsafeMutablePointer! + func hasPerVertexColors() -> Bool + func meshPerVertexColors(_ meshIndex: Int32) -> UnsafeMutablePointer! + func hasPerVertexUVTextureCoords() -> Bool + func meshPerVertexUVTextureCoords(_ meshIndex: Int32) -> UnsafeMutablePointer! + func meshLines(_ meshIndex: Int32) -> UnsafeMutablePointer! + func meshFaces(_ meshIndex: Int32) -> UnsafeMutablePointer! + func meshYCbCrTexture() -> Unmanaged! +} diff --git a/Sources/StructureKit/Metal/STKColorFrame.metal b/Sources/StructureKit/Metal/STKColorFrame.metal new file mode 100644 index 0000000..4ddc607 --- /dev/null +++ b/Sources/StructureKit/Metal/STKColorFrame.metal @@ -0,0 +1,61 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#include "STKMetalCommon.h" +#include "STKMetalData.h" + +#include + +using namespace metal; + +using VertexOut = STKVertexTexOut; +vertex VertexOut vertexTex( + const device STKVertexTex* vertex_array [[buffer(0)]], + const device STKUniformsColorTexture& uniforms [[buffer(1)]], + unsigned int vid [[vertex_id]]) +{ + const STKVertexTex vertexIn = vertex_array[vid]; + const float4x4 orientation = uniforms.projection; + + VertexOut VertexOut; + VertexOut.position = float4(vertexIn.position, 1); + float4 texTrans = orientation * float4(vertexIn.texCoord, 0, 1); + VertexOut.texCoord = float2(texTrans.x, texTrans.y); + return VertexOut; +} + +fragment float4 fragmentTex( + VertexOut interpolated [[stage_in]], + texture2d tex2dY [[texture(0)]], + texture2d tex2dCbCr [[texture(1)]], + sampler sampler2D [[sampler(0)]]) +{ + const float x = tex2dY.sample(sampler2D, interpolated.texCoord).r; + const float2 yz = tex2dCbCr.sample(sampler2D, interpolated.texCoord).rg - float2(0.5, 0.5); + return calcYCbCrColor(x, yz); +} diff --git a/Sources/StructureKit/Metal/STKColorFrameRenderer.swift b/Sources/StructureKit/Metal/STKColorFrameRenderer.swift new file mode 100644 index 0000000..14172b8 --- /dev/null +++ b/Sources/StructureKit/Metal/STKColorFrameRenderer.swift @@ -0,0 +1,97 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import CoreMedia +import Metal +import MetalKit +import StructureKitCTypes + +// Draws a color texture in the view using 2 triangles, so we can apply a transformation to correct the screen orientation of the device +public class STKColorFrameRenderer { + private var vertexBuffer: MTLBuffer! + private var textureCache: CVMetalTextureCache! + private var textureY: MTLTexture? + private var textureCbCr: MTLTexture? + private var renderTextureState: MTLRenderPipelineState + private var samplerState: MTLSamplerState! + + init(view: MTKView, device: MTLDevice) { + let library = STKMetalLibLoader.load(device: device) + let pipelineDescriptor = MTLRenderPipelineDescriptor() + pipelineDescriptor.sampleCount = 1 + pipelineDescriptor.colorAttachments[0].pixelFormat = view.colorPixelFormat + pipelineDescriptor.depthAttachmentPixelFormat = view.depthStencilPixelFormat + pipelineDescriptor.vertexFunction = library.makeFunction(name: "vertexTex") + pipelineDescriptor.fragmentFunction = library.makeFunction(name: "fragmentTex") + renderTextureState = try! device.makeRenderPipelineState(descriptor: pipelineDescriptor) + + samplerState = makeDefaultSampler(device) + + CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, device, nil, &textureCache) + + let vertexData = makeRectangularVertices() + let dataSize = vertexData.count * MemoryLayout.size(ofValue: vertexData[0]) + vertexBuffer = device.makeBuffer(bytes: vertexData, length: dataSize, options: [])! + } + + func uploadColorTexture(_ colorFrame: STKColorFrame) { + guard let imageBuffer = CMSampleBufferGetImageBuffer(colorFrame.sampleBuffer) else { return } + textureY = try! makeTexture( + pixelBuffer: imageBuffer, textureCache: textureCache, planeIndex: 0, pixelFormat: .r8Unorm) + textureCbCr = try! makeTexture( + pixelBuffer: imageBuffer, textureCache: textureCache, planeIndex: 1, pixelFormat: .rg8Unorm) + } + + func renderCameraImage( + _ commandEncoder: MTLRenderCommandEncoder, + orientation: float4x4 + ) { + guard let textureY = textureY, let textureCbCr = textureCbCr else { return } + + commandEncoder.pushDebugGroup("RenderColorFrame") + commandEncoder.setRenderPipelineState(renderTextureState) + + // rotate and mirror: + // move to [-0.5, 0.5] coordinates + // rotate + // mirror and apply scale to fix the ratio + // move back to [0, 1] coordinates + let projection = float4x4.makeTranslation(0.5, 0.5, 0) * orientation * float4x4.makeTranslation(-0.5, -0.5, 0) + var uniforms = STKUniformsColorTexture( + projection: projection) + commandEncoder.setVertexBuffer(vertexBuffer, offset: 0, index: 0) + commandEncoder.setVertexBytes(&uniforms, length: MemoryLayout.stride, index: 1) + + commandEncoder.setFragmentTexture(textureY, index: 0) // [[ texture(0) ]], + commandEncoder.setFragmentTexture(textureCbCr, index: 1) // [[ texture(1) ]], + commandEncoder.setFragmentSamplerState(samplerState, index: 0) // [[ sampler(0) ]] + commandEncoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 6, instanceCount: 1) + commandEncoder.popDebugGroup() + } + +} diff --git a/Sources/StructureKit/Metal/STKCommon.swift b/Sources/StructureKit/Metal/STKCommon.swift new file mode 100644 index 0000000..f1b0d15 --- /dev/null +++ b/Sources/StructureKit/Metal/STKCommon.swift @@ -0,0 +1,297 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import GLKit +import Metal +import MetalKit +import SceneKit +import StructureKitCTypes + +extension String: Error {} + +public class STKShaderManager { + static let pixelFormat = MTLPixelFormat.bgra8Unorm + static let depthFormat = MTLPixelFormat.depth32Float + static let device = MTLCreateSystemDefaultDevice()! + + static let solid = STKMeshRendererSolid(colorFormat: pixelFormat, depthFormat: depthFormat, device: device) + static let wireframe = STKMeshRendererWireframe(colorFormat: pixelFormat, depthFormat: depthFormat, device: device) + static let vertexColor = STKMeshRendererColor(colorFormat: pixelFormat, depthFormat: depthFormat, device: device) + static let textureShader = STKMeshRendererTexture(colorFormat: pixelFormat, depthFormat: depthFormat, device: device) + static let pointCloud = STKMeshRendererPoints(colorFormat: pixelFormat, depthFormat: depthFormat, device: device) + static let lines = STKMeshRendererLines(colorFormat: pixelFormat, depthFormat: depthFormat, device: device) +} + +extension float4x4 { + public init() { self = float4x4.identity() } + + public init(_ m: GLKMatrix4) { self = unsafeBitCast(m, to: float4x4.self) } + + public func toGLK() -> GLKMatrix4 { SCNMatrix4ToGLKMatrix4(SCNMatrix4(self)) } + + public mutating func translate(_ x: Float, _ y: Float, _ z: Float) -> float4x4 { + self = self * float4x4.makeTranslation(x, y, z) + return self + } + + public static func identity() -> float4x4 { unsafeBitCast(GLKMatrix4Identity, to: float4x4.self) } + + public static func makeTranslation(_ vec: vector_float3) -> float4x4 { + unsafeBitCast(GLKMatrix4MakeTranslation(vec.x, vec.y, vec.z), to: float4x4.self) + } + + public static func makeTranslation(_ x: Float, _ y: Float, _ z: Float) -> float4x4 { + unsafeBitCast(GLKMatrix4MakeTranslation(x, y, z), to: float4x4.self) + } + + public static func makeRotationX(_ radians: Float) -> float4x4 { + unsafeBitCast(GLKMatrix4MakeXRotation(radians), to: float4x4.self) + } + + public static func makeRotationY(_ radians: Float) -> float4x4 { + unsafeBitCast(GLKMatrix4MakeYRotation(radians), to: float4x4.self) + } + + public static func makeRotationZ(_ radians: Float) -> float4x4 { + unsafeBitCast(GLKMatrix4MakeZRotation(radians), to: float4x4.self) + } + + public static func makeRotation(_ radians: Float, _ x: Float, _ y: Float, _ z: Float) -> float4x4 { + unsafeBitCast(GLKMatrix4MakeRotation(radians, x, y, z), to: float4x4.self) + } + + public static func makeRotation(_ radians: Float, _ vec: vector_float3) -> float4x4 { + float4x4.makeRotation(radians, vec.x, vec.y, vec.z) + } + + public static func makeScale(_ s: Float) -> float4x4 { + unsafeBitCast(GLKMatrix4MakeScale(s, s, s), to: float4x4.self) + } + + public static func makeScale(_ x: Float, _ y: Float, _ z: Float) -> float4x4 { + unsafeBitCast(GLKMatrix4MakeScale(x, y, z), to: float4x4.self) + } + + public static func makePerspective(_ fovyRadians: Float, _ aspect: Float, _ nearZ: Float, _ farZ: Float) -> float4x4 { + unsafeBitCast(GLKMatrix4MakePerspective(fovyRadians, aspect, nearZ, farZ), to: float4x4.self) + } + + public var translation: vector_float4 { return self.columns.3 } +} + +extension float3x3 { + public static func makeRotationZ(_ radians: Float) -> float3x3 { + float3x3([ + simd_float3(cos(radians), -sin(radians), 0), simd_float3(sin(radians), cos(radians), 0), simd_float3(0, 0, 1), + ]) + } + + public static func makeTranslation(x: Float, y: Float) -> float3x3 { + float3x3([simd_float3(1, 0, 0), simd_float3(0, 1, 0), simd_float3(x, y, 1)]) + } + + public static func makeScale(x: Float, y: Float, z: Float) -> float3x3 { float3x3(diagonal: simd_float3(x, y, z)) } +} + +extension float2x2 { + public static func makeRotation(_ radians: Float) -> float2x2 { + float2x2([simd_float2(cos(radians), -sin(radians)), simd_float2(sin(radians), cos(radians))]) + } + + public static func makeScale(x: Float, y: Float) -> float2x2 { float2x2(diagonal: simd_float2(x, y)) } +} + +extension simd_float4 { + public func norm2() -> Float { length_squared(self) } + + public func norm() -> Float { length(self) } + + public func toGLK() -> GLKVector4 { GLKVector4Make(x, y, z, w) } + + public var xyz: vector_float3 { vector_float3(self) } +} + +extension simd_float3 { + public init(_ v: GLKVector3) { self = simd_float3(v.x, v.y, v.z) } + + public init(_ v: simd_float4) { self = simd_float3(v.x, v.y, v.z) } + + public func norm2() -> Float { length_squared(self) } + + public func norm() -> Float { length(self) } + + public func toGLK() -> GLKVector3 { GLKVector3Make(x, y, z) } +} + +extension simd_float2 { + public init(_ v: GLKVector2) { self = simd_float2(v.x, v.y) } + + public func norm2() -> Float { length_squared(self) } + + public func norm() -> Float { length(self) } + + public func toGLK() -> GLKVector2 { GLKVector2Make(x, y) } +} + +public func makeRectangularVertices(z: Float = 0.0) -> [STKVertexTex] { + let vertexData: [STKVertexTex] = [ + // a b + // c d + // -1.0, 1.0, 0.0, 0.0, 0.0, // a + // 1.0, 1.0, 0.0, 1.0, 0.0, // b + // 1.0, -1.0, 0.0, 1.0, 1.0, // d + // + // -1.0, 1.0, 0.0, 0.0, 0.0, // a + // 1.0, -1.0, 0.0, 1.0, 1.0, // d + // -1.0, -1.0, 0.0, 0.0, 1.0 // c + STKVertexTex(position: vector_float3(x: -1.0, y: 1.0, z: z), texCoord: vector_float2(0.0, 0.0)), // a + STKVertexTex(position: vector_float3(x: 1.0, y: 1.0, z: z), texCoord: vector_float2(1.0, 0.0)), // b + STKVertexTex(position: vector_float3(x: 1.0, y: -1.0, z: z), texCoord: vector_float2(1.0, 1.0)), // d + + STKVertexTex(position: vector_float3(x: -1.0, y: 1.0, z: z), texCoord: vector_float2(0.0, 0.0)), // a + STKVertexTex(position: vector_float3(x: 1.0, y: -1.0, z: z), texCoord: vector_float2(1.0, 1.0)), // d + STKVertexTex(position: vector_float3(x: -1.0, y: -1.0, z: z), texCoord: vector_float2(0.0, 1.0)), // c + ] + return vertexData +} + +public func makeDefaultSampler(_ device: MTLDevice) -> MTLSamplerState { + let sampler = MTLSamplerDescriptor() + sampler.minFilter = MTLSamplerMinMagFilter.nearest + sampler.magFilter = MTLSamplerMinMagFilter.nearest + sampler.mipFilter = MTLSamplerMipFilter.notMipmapped + sampler.maxAnisotropy = 1 + sampler.sAddressMode = MTLSamplerAddressMode.clampToEdge + sampler.tAddressMode = MTLSamplerAddressMode.clampToEdge + sampler.rAddressMode = MTLSamplerAddressMode.clampToEdge + sampler.normalizedCoordinates = true + sampler.lodMinClamp = 0 + sampler.lodMaxClamp = Float.greatestFiniteMagnitude + return device.makeSamplerState(descriptor: sampler)! +} + +extension MTLVertexAttributeDescriptor { + public convenience init(bufferIndex: Int, offset: Int, format: MTLVertexFormat) { + self.init() + self.format = format + self.offset = offset + self.bufferIndex = bufferIndex + } +} + +extension MTLRenderPipelineDescriptor { + public convenience init( + _ view: MTKView, + _ device: MTLDevice, + _ vertex: String, + _ fragment: String, + _ vertexDescriptor: MTLVertexDescriptor + ) { + self.init() + + let library = STKMetalLibLoader.load(device: device) + self.vertexFunction = library.makeFunction(name: vertex) + self.fragmentFunction = library.makeFunction(name: fragment) + self.colorAttachments[0].pixelFormat = view.colorPixelFormat + self.depthAttachmentPixelFormat = view.depthStencilPixelFormat + self.vertexDescriptor = vertexDescriptor + } + +} + +public func makeDepthStencilState(_ device: MTLDevice) -> MTLDepthStencilState { + let depthStencilDescriptor = MTLDepthStencilDescriptor() + depthStencilDescriptor.depthCompareFunction = .less + depthStencilDescriptor.isDepthWriteEnabled = true + return device.makeDepthStencilState(descriptor: depthStencilDescriptor)! +} + +public func makePipeline( + _ device: MTLDevice, + _ vertex: String, + _ fragment: String, + _ vertexDescriptor: MTLVertexDescriptor, + _ pixelFormat: MTLPixelFormat, + _ depthFormat: MTLPixelFormat, + blending: Bool +) -> MTLRenderPipelineState { + let pipelineDescriptor = MTLRenderPipelineDescriptor() + let library = STKMetalLibLoader.load(device: device) + pipelineDescriptor.vertexFunction = library.makeFunction(name: vertex) + pipelineDescriptor.fragmentFunction = library.makeFunction(name: fragment) + pipelineDescriptor.colorAttachments[0].pixelFormat = pixelFormat + pipelineDescriptor.depthAttachmentPixelFormat = depthFormat + pipelineDescriptor.vertexDescriptor = vertexDescriptor + + if blending { + pipelineDescriptor.colorAttachments[0].isBlendingEnabled = true + pipelineDescriptor.colorAttachments[0].rgbBlendOperation = .add + pipelineDescriptor.colorAttachments[0].sourceRGBBlendFactor = .sourceAlpha + pipelineDescriptor.colorAttachments[0].destinationRGBBlendFactor = .oneMinusSourceAlpha + } + + return try! device.makeRenderPipelineState(descriptor: pipelineDescriptor) +} + +public func makePipeline( + _ view: MTKView, + _ device: MTLDevice, + _ vertex: String, + _ fragment: String, + _ vertexDescriptor: MTLVertexDescriptor, + blending: Bool +) -> MTLRenderPipelineState { + makePipeline( + device, vertex, fragment, vertexDescriptor, view.colorPixelFormat, view.depthStencilPixelFormat, blending: blending) +} + +public func makeTexture( + pixelBuffer: CVPixelBuffer, textureCache: CVMetalTextureCache, planeIndex: Int, pixelFormat: MTLPixelFormat +) throws -> MTLTexture { + var imageTexture: CVMetalTexture? + let isPlanar = CVPixelBufferIsPlanar(pixelBuffer) + let width = isPlanar ? CVPixelBufferGetWidthOfPlane(pixelBuffer, planeIndex) : CVPixelBufferGetWidth(pixelBuffer) + let height = isPlanar ? CVPixelBufferGetHeightOfPlane(pixelBuffer, planeIndex) : CVPixelBufferGetHeight(pixelBuffer) + let result = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, textureCache, pixelBuffer, nil, pixelFormat, width, height, planeIndex, &imageTexture) + guard + let unwrappedImageTexture = imageTexture, + let texture = CVMetalTextureGetTexture(unwrappedImageTexture), + result == kCVReturnSuccess + else { throw "cannot create texture" } + return texture +} + +public func calcVisualizationDistance(cameraPoint: vector_float3, cubeSize: vector_float3) -> (min: Float, max: Float) { + // we need to evenly distribute the colormap across the whole depth range inside the cube. lets approximate it with a sphere + let cubeCenter = cubeSize / 2 + let maxDiagonal = cubeSize.max() + let maxDistMM = ((cubeCenter - cameraPoint).norm() + maxDiagonal / 2) + let minDistMM = max((cubeCenter - cameraPoint).norm() - maxDiagonal / 2, 0) + return (minDistMM, maxDistMM) +} diff --git a/Sources/StructureKit/Metal/STKDepthFrame.metal b/Sources/StructureKit/Metal/STKDepthFrame.metal new file mode 100644 index 0000000..bdb67cb --- /dev/null +++ b/Sources/StructureKit/Metal/STKDepthFrame.metal @@ -0,0 +1,69 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#include "STKMetalCommon.h" +#include "STKMetalData.h" + +#include + +using namespace metal; + +using VertexOut = STKVertexTexOut; +vertex VertexOut vertexTexDepthFrame( + const device STKVertexTex* vertex_array [[buffer(0)]], + const device STKUniformsDepthTexture& uniforms [[buffer(1)]], + unsigned int vid [[vertex_id]]) +{ + const STKVertexTex vertexIn = vertex_array[vid]; + const float4x4 orientation = uniforms.projection; + + VertexOut VertexOut; + VertexOut.position = float4(vertexIn.position, 1); + float4 texTrans = orientation * float4(vertexIn.texCoord, 0, 1); + VertexOut.texCoord = float2(texTrans.x, texTrans.y); + return VertexOut; +} + +fragment float4 fragmentDepthFrame( + VertexOut interpolated [[stage_in]], + texture2d depthMap [[texture(0)]], + texture2d colors [[texture(1)]], + sampler sampler2D [[sampler(0)]], + const device STKUniformsDepthTexture& uniforms [[buffer(0)]]) +{ + const float depth = depthMap.sample(sampler2D, interpolated.texCoord).r; + if (isnan(depth)) + return float4(0); + if (depth < uniforms.depthMin || depth > uniforms.depthMax) + return float4(0); + + // calculate the depth color + float4 finalColor = calcDepthColor(depth, float2(uniforms.depthMin, uniforms.depthMax), colors); + finalColor.w = uniforms.alpha; + return finalColor; +} diff --git a/Sources/StructureKit/Metal/STKDepthOverlay.metal b/Sources/StructureKit/Metal/STKDepthOverlay.metal new file mode 100644 index 0000000..82cfbc9 --- /dev/null +++ b/Sources/StructureKit/Metal/STKDepthOverlay.metal @@ -0,0 +1,90 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#include "STKMetalCommon.h" +#include "STKMetalData.h" + +#include + +using namespace metal; + +vertex STKVertexTexOut vertexDepthOverlay( + const device STKVertexTex* vertex_array [[buffer(0)]], + const device STKUniformsDepthOverlay& uniforms [[buffer(1)]], + unsigned int vid [[vertex_id]]) +{ + const STKVertexTex VertexIn = vertex_array[vid]; + const float4 texTrans = uniforms.projection * float4(VertexIn.texCoord, 0, 1); + + STKVertexTexOut VertexOut; + VertexOut.position = float4(VertexIn.position, 1); + VertexOut.texCoord = float2(texTrans.x, texTrans.y); + return VertexOut; +} + +fragment float4 fragmentDepthOverlay( + STKVertexTexOut interpolated [[stage_in]], + const device STKUniformsDepthOverlay& uniforms [[buffer(0)]], + texture2d texDepth [[texture(0)]], + texture2d colors [[texture(1)]], + sampler sampler2D [[sampler(0)]]) +{ + // depth udefined + const float depth = texDepth.sample(sampler2D, interpolated.texCoord).r; + if (isnan(depth)) + return float4(0); + + // calculate position of the depth pixel in the world CS using intrinsics + const auto intrinsics = uniforms.cameraIntrinsics; + const float u = interpolated.texCoord.x * intrinsics.width; + const float v = interpolated.texCoord.y * intrinsics.height; + const float depthM = depth / 1000; + const float4 cameraPoint( + depthM * (u - intrinsics.cx) / intrinsics.fx, + depthM * (v - intrinsics.cy) / intrinsics.fy, + depthM, + 1); + const float4 worldPoint = uniforms.cameraPose * cameraPoint; + const float4 cubePoint = uniforms.cubeModelInv * worldPoint; + + // the cube borders + // const float eps1 = 0.003; + // if((abs(cubePoint.x) < eps1 || abs(cubePoint.x - 1) < eps1) + // || (abs(cubePoint.y) < eps1 || abs(cubePoint.y - 1) < eps1) + // || (abs(cubePoint.z) < eps1 || abs(cubePoint.z - 1) < eps1)) + // return float4(0, 1, 0, uniforms.alpha); + + // outside the box + if (cubePoint.x < 0 || cubePoint.x > 1 || cubePoint.y < 0 || cubePoint.y > 1 || cubePoint.z < 0 || cubePoint.z > 1) + return float4(0); + + // calculate the depth color + float4 finalColor = calcDepthColor(depth, float2(uniforms.depthMin, uniforms.depthMax), colors); + finalColor.w = uniforms.alpha; + return finalColor; +} diff --git a/Sources/StructureKit/Metal/STKDepthRenderer.swift b/Sources/StructureKit/Metal/STKDepthRenderer.swift new file mode 100644 index 0000000..c55e742 --- /dev/null +++ b/Sources/StructureKit/Metal/STKDepthRenderer.swift @@ -0,0 +1,302 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import Metal +import MetalKit +import StructureKitCTypes + +// Draws various objects which require depth texture: +// 1. the depth frame +// 2. the cube +// 3. the depth overlay inside the cube +class STKDepthRenderer { + private var mtkView: MTKView + private var renderDepthState: MTLRenderPipelineState + private var renderDepthFrameState: MTLRenderPipelineState + private var textureDepth: MTLTexture? + private var textureColor: MTLTexture? + private var vertexBuffer: MTLBuffer + private var samplerState: MTLSamplerState + private var device: MTLDevice + private var intr: STKIntrinsics? + var depthRenderingColors = [simd_float4(1, 0, 0, 1), simd_float4(1, 1, 0, 1)] { // red and yellow + didSet { updateColorTexture() } + } + + // cube rendering + private var _renderCubeState: MTLRenderPipelineState! + private var _vertexCubeBuffer: MTLBuffer! + private var _indexCubeBuffer: MTLBuffer! + + init(view: MTKView, device: MTLDevice) { + mtkView = view + self.device = device + + let library = STKMetalLibLoader.load(device: device) + let pipelineDepthDescriptor = MTLRenderPipelineDescriptor() + pipelineDepthDescriptor.sampleCount = 1 + pipelineDepthDescriptor.colorAttachments[0].pixelFormat = mtkView.colorPixelFormat + pipelineDepthDescriptor.depthAttachmentPixelFormat = mtkView.depthStencilPixelFormat + pipelineDepthDescriptor.vertexFunction = library.makeFunction(name: "vertexDepthOverlay") + pipelineDepthDescriptor.fragmentFunction = library.makeFunction(name: "fragmentDepthOverlay") + // blending + pipelineDepthDescriptor.colorAttachments[0].isBlendingEnabled = true + pipelineDepthDescriptor.colorAttachments[0].rgbBlendOperation = .add + pipelineDepthDescriptor.colorAttachments[0].sourceRGBBlendFactor = .sourceAlpha + pipelineDepthDescriptor.colorAttachments[0].destinationRGBBlendFactor = .oneMinusSourceAlpha + + renderDepthState = try! device.makeRenderPipelineState(descriptor: pipelineDepthDescriptor) + + let vertexData = makeRectangularVertices() + let dataSize = vertexData.count * MemoryLayout.size(ofValue: vertexData[0]) + vertexBuffer = device.makeBuffer(bytes: vertexData, length: dataSize, options: [])! + + samplerState = makeDefaultSampler(device) + + let pipelineDepthFrameDescriptor = MTLRenderPipelineDescriptor() + pipelineDepthFrameDescriptor.sampleCount = 1 + pipelineDepthFrameDescriptor.colorAttachments[0].pixelFormat = mtkView.colorPixelFormat + pipelineDepthFrameDescriptor.depthAttachmentPixelFormat = mtkView.depthStencilPixelFormat + pipelineDepthFrameDescriptor.vertexFunction = library.makeFunction(name: "vertexTexDepthFrame") + pipelineDepthFrameDescriptor.fragmentFunction = library.makeFunction(name: "fragmentDepthFrame") + // blending + pipelineDepthFrameDescriptor.colorAttachments[0].isBlendingEnabled = true + pipelineDepthFrameDescriptor.colorAttachments[0].rgbBlendOperation = .add + pipelineDepthFrameDescriptor.colorAttachments[0].sourceRGBBlendFactor = .sourceAlpha + pipelineDepthFrameDescriptor.colorAttachments[0].destinationRGBBlendFactor = .oneMinusSourceAlpha + renderDepthFrameState = try! device.makeRenderPipelineState(descriptor: pipelineDepthFrameDescriptor) + + let pipelineCubeDescriptor = MTLRenderPipelineDescriptor() + pipelineCubeDescriptor.sampleCount = 1 + pipelineCubeDescriptor.colorAttachments[0].pixelFormat = view.colorPixelFormat + pipelineCubeDescriptor.depthAttachmentPixelFormat = view.depthStencilPixelFormat + pipelineCubeDescriptor.vertexFunction = library.makeFunction(name: "vertexLineOverDepth") + pipelineCubeDescriptor.fragmentFunction = library.makeFunction(name: "fragmentLineOverDepth") + // blending + pipelineCubeDescriptor.colorAttachments[0].isBlendingEnabled = true + pipelineCubeDescriptor.colorAttachments[0].rgbBlendOperation = .add + pipelineCubeDescriptor.colorAttachments[0].sourceRGBBlendFactor = .sourceAlpha + pipelineCubeDescriptor.colorAttachments[0].destinationRGBBlendFactor = .oneMinusSourceAlpha + + _renderCubeState = try! device.makeRenderPipelineState(descriptor: pipelineCubeDescriptor) + + let cubeVertices: [Float] = [ + 0, 0, 0, 1, 1, 1, // a + 1, 0, 0, 1, 1, 1, // b + 1, 1, 0, 1, 1, 1, // c + 0, 1, 0, 1, 1, 1, // d + + 0, 0, 1, 1, 1, 1, // e + 1, 0, 1, 1, 1, 1, // f + 1, 1, 1, 1, 1, 1, // g + 0, 1, 1, 1, 1, 1, // h + ] + + let cubeIndices: [UInt32] = [ + 0, 1, + 1, 2, + 2, 3, + 3, 0, + + 4, 5, + 5, 6, + 6, 7, + 7, 4, + + 0, 4, + 1, 5, + 2, 6, + 3, 7, + ] + + _vertexCubeBuffer = device.makeBuffer( + bytes: cubeVertices, length: MemoryLayout.stride * cubeVertices.count, options: [])! + _indexCubeBuffer = device.makeBuffer( + bytes: cubeIndices, length: MemoryLayout.stride * cubeIndices.count, options: [])! + + updateColorTexture() + } + + private func updateColorTexture() { + let colorTextureDescriptor = MTLTextureDescriptor.texture2DDescriptor( + pixelFormat: .rgba32Float, width: Int(depthRenderingColors.count), height: Int(1), mipmapped: false) + colorTextureDescriptor.usage = MTLTextureUsage( + rawValue: MTLTextureUsage.renderTarget.rawValue | MTLTextureUsage.shaderRead.rawValue) + textureColor = device.makeTexture(descriptor: colorTextureDescriptor) + + let bytesPerRow: Int = depthRenderingColors.count * MemoryLayout.stride + let region = MTLRegionMake2D(0, 0, depthRenderingColors.count, 1) + textureColor?.replace(region: region, mipmapLevel: 0, withBytes: &depthRenderingColors, bytesPerRow: bytesPerRow) + } + + func uploadColorTextureFromDepth(_ depthFrame: STKDepthFrame) { + if let texture = textureDepth { + if texture.width != depthFrame.width || texture.height != depthFrame.height { + textureDepth = nil // invalidate texture + } + } + + if textureDepth == nil { + let textureDescriptor = MTLTextureDescriptor.texture2DDescriptor( + pixelFormat: .r32Float, width: Int(depthFrame.width), height: Int(depthFrame.height), mipmapped: false) + textureDescriptor.usage = MTLTextureUsage( + rawValue: MTLTextureUsage.renderTarget.rawValue | MTLTextureUsage.shaderRead.rawValue) + textureDepth = device.makeTexture(descriptor: textureDescriptor) + } + + intr = depthFrame.intrinsics() + let depthMap = depthFrame.depthInMillimeters + assert(MemoryLayout.size == MemoryLayout.size) + + let bytesPerRow: Int = Int(depthFrame.width) * MemoryLayout.stride + let region = MTLRegionMake2D(0, 0, Int(depthFrame.width), Int(depthFrame.height)) + textureDepth?.replace(region: region, mipmapLevel: 0, withBytes: depthMap!, bytesPerRow: bytesPerRow) + } + + func renderDepthOverlay( + _ commandEncoder: MTLRenderCommandEncoder, + volumeSizeInMeters: simd_float3, + cameraPosition: float4x4, + textureOrientation: float4x4, + alpha: Float + ) { + guard let texture = textureDepth, + let intr = intr, + let textureColor = textureColor + else { return } + + commandEncoder.pushDebugGroup("RenderDepthOverlay") + commandEncoder.setRenderPipelineState(renderDepthState) + + // rotate and mirror: + // move to [-0.5, 0.5] coordinates + // rotate + // mirror and apply scale to fix the ratio + // move back to [0, 1] coordinates + let projection = + float4x4.makeTranslation(0.5, 0.5, 0) * textureOrientation * float4x4.makeTranslation(-0.5, -0.5, 0) + let (minDistM, maxDistM) = calcVisualizationDistance( + cameraPoint: cameraPosition.translation.xyz, + cubeSize: volumeSizeInMeters) + + let intrinsics = STKIntrinsicsMetal( + cx: intr.cx, cy: intr.cy, fx: intr.fx, fy: intr.fy, width: UInt32(texture.width), height: UInt32(texture.height)) + + var uniforms = STKUniformsDepthOverlay( + projection: projection, + cameraPose: cameraPosition, + cameraIntrinsics: intrinsics, + cubeModelInv: float4x4.makeScale(volumeSizeInMeters.x, volumeSizeInMeters.y, volumeSizeInMeters.z).inverse, + depthMin: minDistM * 1000, + depthMax: maxDistM * 1000, + alpha: alpha + ) + + commandEncoder.setVertexBuffer(vertexBuffer, offset: 0, index: 0) + commandEncoder.setVertexBytes(&uniforms, length: MemoryLayout.stride, index: 1) + + commandEncoder.setFragmentBytes(&uniforms, length: MemoryLayout.stride, index: 0) + commandEncoder.setFragmentTexture(texture, index: 0) // [[ texture(0) ]], + commandEncoder.setFragmentTexture(textureColor, index: 1) // [[ texture(1) ]], + commandEncoder.setFragmentSamplerState(samplerState, index: 0) // [[ sampler(0) ]] + commandEncoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 6, instanceCount: 1) + commandEncoder.popDebugGroup() + } + + func renderDepthFrame( + _ commandEncoder: MTLRenderCommandEncoder, + orientation: float4x4, + minDepth: Float, + maxDepth: Float, + alpha: Float = 1 + ) { + guard let textureDepth = textureDepth, + let textureColor = textureColor + else { return } + + commandEncoder.pushDebugGroup("RenderDepthFrame") + commandEncoder.setRenderPipelineState(renderDepthFrameState) + + let projection = float4x4.makeTranslation(0.5, 0.5, 0) * orientation * float4x4.makeTranslation(-0.5, -0.5, 0) + var uniforms = STKUniformsDepthTexture(projection: projection, depthMin: minDepth, depthMax: maxDepth, alpha: alpha) + commandEncoder.setVertexBuffer(vertexBuffer, offset: 0, index: 0) + commandEncoder.setVertexBytes(&uniforms, length: MemoryLayout.stride, index: 1) + + commandEncoder.setFragmentTexture(textureDepth, index: 0) // [[ texture(0) ]], + commandEncoder.setFragmentTexture(textureColor, index: 1) // [[ texture(1) ]], + commandEncoder.setFragmentBytes(&uniforms, length: MemoryLayout.stride, index: 0) + + commandEncoder.setFragmentSamplerState(samplerState, index: 0) // [[ sampler(0) ]] + commandEncoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 6, instanceCount: 1) + commandEncoder.popDebugGroup() + + } + + func renderCubeOutline( + _ commandEncoder: MTLRenderCommandEncoder, + volumeSizeInMeters: simd_float3, + cameraPosition: float4x4, + projection: float4x4, + orientation: float4x4, + useOcclusion: Bool + ) { + guard let textureDepth = textureDepth, + let intr = intr + else { return } + + var uniformsCube = STKUniformsCube( + model: float4x4.makeScale(volumeSizeInMeters.x, volumeSizeInMeters.y, volumeSizeInMeters.z), + view: cameraPosition.inverse, + projection: orientation * projection, + cameraIntrinsics: STKIntrinsicsMetal( + cx: intr.cx, cy: intr.cy, + fx: intr.fx, fy: intr.fy, + width: UInt32(textureDepth.width), height: UInt32(textureDepth.height)), + useOcclusion: useOcclusion + ) + + commandEncoder.pushDebugGroup("RenderCube") + commandEncoder.setRenderPipelineState(_renderCubeState) + + commandEncoder.setVertexBuffer(_vertexCubeBuffer, offset: 0, index: 0) + commandEncoder.setVertexBytes(&uniformsCube, length: MemoryLayout.stride, index: 1) + + commandEncoder.setFragmentTexture(textureDepth, index: 0) // [[ texture(0) ]], + commandEncoder.setFragmentBytes(&uniformsCube, length: MemoryLayout.stride, index: 0) + commandEncoder.setFragmentSamplerState(samplerState, index: 0) // [[ sampler(0) ]] + + commandEncoder.drawIndexedPrimitives( + type: .line, + indexCount: 24, + indexType: .uint32, + indexBuffer: _indexCubeBuffer, + indexBufferOffset: 0) + commandEncoder.popDebugGroup() + } + +} diff --git a/Sources/StructureKit/Metal/STKDrawableObject.swift b/Sources/StructureKit/Metal/STKDrawableObject.swift new file mode 100644 index 0000000..e56430b --- /dev/null +++ b/Sources/StructureKit/Metal/STKDrawableObject.swift @@ -0,0 +1,54 @@ +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import Metal +import MetalKit +import simd + +// A protocol for a generic drawable object +public protocol STKDrawableObject: AnyObject { + var vertexType: Any { get } + var indexType: Any { get } + + func vertices() -> MTLBuffer? + func indices() -> MTLBuffer? + func lines() -> MTLBuffer? + func normals() -> MTLBuffer? + func colors() -> MTLBuffer? + func texCoords() -> MTLBuffer? + func textureY() -> MTLTexture? + func textureCbCr() -> MTLTexture? + func modelMatrix() -> float4x4 + func vertexCount() -> Int + func triangleCount() -> Int + func lineCount() -> Int + + func vertexArray() -> [simd_float3]? + func boundingBox() -> (min: simd_float3, max: simd_float3)? + + func updateBuffers() +} diff --git a/Sources/StructureKit/Metal/STKLine.metal b/Sources/StructureKit/Metal/STKLine.metal new file mode 100644 index 0000000..34b3677 --- /dev/null +++ b/Sources/StructureKit/Metal/STKLine.metal @@ -0,0 +1,62 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#include "STKMetalData.h" + +#include + +using namespace metal; + +struct VertexIn +{ + packed_float3 position; + packed_float3 color; +}; + +struct VertexOut +{ + float4 position [[position]]; + float4 color; +}; + +vertex VertexOut vertexLine( + const device VertexIn* vertex_array [[buffer(0)]], + const device STKUniformsLine& uniforms [[buffer(1)]], + unsigned int vid [[vertex_id]]) +{ + VertexIn in = vertex_array[vid]; + VertexOut out; + out.position = uniforms.projection * uniforms.view * uniforms.model * float4(in.position, 1.0); + out.color = float4(in.color, 1); + return out; +} + +fragment float4 fragmentLine(VertexOut in [[stage_in]]) +{ + return in.color; +} diff --git a/Sources/StructureKit/Metal/STKLineOverDepth.metal b/Sources/StructureKit/Metal/STKLineOverDepth.metal new file mode 100644 index 0000000..5ab0974 --- /dev/null +++ b/Sources/StructureKit/Metal/STKLineOverDepth.metal @@ -0,0 +1,83 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#include "STKMetalData.h" + +#include + +using namespace metal; + +struct VertexIn +{ + packed_float3 position; + packed_float3 color; +}; + +struct VertexOut +{ + float4 position [[position]]; + float4 cameraPosition; + float4 color; +}; + +vertex VertexOut vertexLineOverDepth( + const device VertexIn* vertex_array [[buffer(0)]], + const device STKUniformsCube& uniforms [[buffer(1)]], + unsigned int vid [[vertex_id]]) +{ + VertexIn in = vertex_array[vid]; + VertexOut out; + + out.position = uniforms.projection * uniforms.view * uniforms.model * float4(in.position, 1.0); + if (uniforms.useOcclusion) + out.cameraPosition = uniforms.view * uniforms.model * float4(in.position, 1.0); + out.color = float4(in.color, 1); + return out; +} + +fragment float4 fragmentLineOverDepth( + VertexOut in [[stage_in]], + const device STKUniformsCube& uniforms [[buffer(0)]], + texture2d texDepth [[texture(0)]], + sampler sampler2D [[sampler(0)]]) +{ + if (uniforms.useOcclusion) + { + // project cube point to depth image + const float4 point = in.cameraPosition; // screen + const float2 proj( + uniforms.cameraIntrinsics.cx + (point.x * uniforms.cameraIntrinsics.fx) / point.z, + uniforms.cameraIntrinsics.cy + (point.y * uniforms.cameraIntrinsics.fy) / point.z); + + // const float depth = texDepth.read(uint2(texDepth.get_width() - proj.x, texDepth.get_height() - proj.y)).r; + const float depth = texDepth.read(uint2(proj.x, proj.y)).r; + if (in.cameraPosition.z * 1000 > depth) + return float4(0); + } + return in.color; +} diff --git a/Sources/StructureKit/Metal/STKLineRenderer.swift b/Sources/StructureKit/Metal/STKLineRenderer.swift new file mode 100644 index 0000000..335549d --- /dev/null +++ b/Sources/StructureKit/Metal/STKLineRenderer.swift @@ -0,0 +1,180 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import Metal +import MetalKit +import StructureKitCTypes + +// Draws colored lines in the world coordinate system +class STKLineRenderer { + private var renderLineState: MTLRenderPipelineState! + private var vertexCubeBuffer: MTLBuffer! + private var indexCubeBuffer: MTLBuffer! + private var indexTriadBuffer: MTLBuffer! + private var mtkView: MTKView + var colorCameraGLProjectionMatrix = float4x4() + var depthCameraGLProjectionMatrix = float4x4() + + private let cubeVertices4: [simd_float4] = [ + simd_float4(0, 0, 0, 1), + simd_float4(1, 0, 0, 1), + simd_float4(1, 1, 0, 1), + simd_float4(0, 1, 0, 1), + + simd_float4(0, 0, 1, 1), + simd_float4(1, 0, 1, 1), + simd_float4(1, 1, 1, 1), + simd_float4(0, 1, 1, 1), + ] + + init(view: MTKView, device: MTLDevice) { + mtkView = view + let library = STKMetalLibLoader.load(device: device) + let pipelineCubeDescriptor = MTLRenderPipelineDescriptor() + pipelineCubeDescriptor.sampleCount = 1 + pipelineCubeDescriptor.colorAttachments[0].pixelFormat = view.colorPixelFormat + pipelineCubeDescriptor.depthAttachmentPixelFormat = view.depthStencilPixelFormat + pipelineCubeDescriptor.vertexFunction = library.makeFunction(name: "vertexLine") + pipelineCubeDescriptor.fragmentFunction = library.makeFunction(name: "fragmentLine") + renderLineState = try! device.makeRenderPipelineState(descriptor: pipelineCubeDescriptor) + + let cubeVertices: [Float] = [ + 0, 0, 0, 1, 1, 1, // a + 1, 0, 0, 1, 1, 1, // b + 1, 1, 0, 1, 1, 1, // c + 0, 1, 0, 1, 1, 1, // d + + 0, 0, 1, 1, 1, 1, // e + 1, 0, 1, 1, 1, 1, // f + 1, 1, 1, 1, 1, 1, // g + 0, 1, 1, 1, 1, 1, // h + ] + + let cubeIndices: [UInt32] = [ + 0, 1, + 1, 2, + 2, 3, + 3, 0, + + 4, 5, + 5, 6, + 6, 7, + 7, 4, + + 0, 4, + 1, 5, + 2, 6, + 3, 7, + ] + + vertexCubeBuffer = device.makeBuffer( + bytes: cubeVertices, length: MemoryLayout.stride * cubeVertices.count, options: [])! + indexCubeBuffer = device.makeBuffer( + bytes: cubeIndices, length: MemoryLayout.stride * cubeIndices.count, options: [])! + + let triadIndices: [UInt32] = [ + 0, 1, + 2, 3, + 4, 5, + ] + indexTriadBuffer = device.makeBuffer( + bytes: triadIndices, length: MemoryLayout.stride * triadIndices.count, options: [])! + } + + func renderCubeOutline( + _ commandEncoder: MTLRenderCommandEncoder, + volumeSizeInMeters: simd_float3, + cameraPosition: float4x4, + projection: float4x4, + orientation: float4x4 + ) { + var uniformsCube = STKUniformsLine( + model: float4x4.makeScale(volumeSizeInMeters.x, volumeSizeInMeters.y, volumeSizeInMeters.z), + view: cameraPosition.inverse, + projection: orientation * projection + ) + + commandEncoder.pushDebugGroup("RenderCube") + commandEncoder.setRenderPipelineState(renderLineState) + commandEncoder.setVertexBuffer(vertexCubeBuffer, offset: 0, index: 0) + commandEncoder.setVertexBytes(&uniformsCube, length: MemoryLayout.stride, index: 1) + commandEncoder.drawIndexedPrimitives( + type: .line, + indexCount: 24, + indexType: .uint32, + indexBuffer: indexCubeBuffer, + indexBufferOffset: 0) + commandEncoder.popDebugGroup() + + renderAnchors( + commandEncoder, + anchors: [simd_float4x4()], + cameraPosition: cameraPosition, + projection: projection, + orientation: orientation, + triadSize: volumeSizeInMeters.x + ) + } + + func renderAnchors( + _ commandEncoder: MTLRenderCommandEncoder, + anchors: [simd_float4x4], + cameraPosition: float4x4, + projection: float4x4, + orientation: float4x4, + triadSize: Float = 0.05 + ) { + for anchor in anchors { + var uniforms = STKUniformsLine( + model: anchor, + view: cameraPosition.inverse, + projection: orientation * projection) + commandEncoder.pushDebugGroup("RenderAnchor") + commandEncoder.setRenderPipelineState(renderLineState) + + var triadVertices: [Float] = [ + 0, 0, 0, 1, 0, 0, // ox + triadSize, 0, 0, 1, 0, 0, // x + 0, 0, 0, 0, 0, 0, // oy + 0, triadSize, 0, 0, 1, 0, // y + 0, 0, 0, 0, 0, 1, // oz + 0, 0, triadSize, 0, 0, 1, // z + ] + commandEncoder.setVertexBytes(&triadVertices, length: MemoryLayout.stride * triadVertices.count, index: 0) + commandEncoder.setVertexBytes(&uniforms, length: MemoryLayout.stride, index: 1) + commandEncoder.drawIndexedPrimitives( + type: .line, + indexCount: 3 * 2, + indexType: .uint32, + indexBuffer: indexTriadBuffer, + indexBufferOffset: 0) + commandEncoder.popDebugGroup() + } + } + +} diff --git a/Sources/StructureKit/Metal/STKMeshBuffers.swift b/Sources/StructureKit/Metal/STKMeshBuffers.swift new file mode 100644 index 0000000..9691f6e --- /dev/null +++ b/Sources/StructureKit/Metal/STKMeshBuffers.swift @@ -0,0 +1,267 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import ARKit +import Metal +import MetalKit + +// Implementation of the STKDrawableObject protocol for STKMesh and ARFaceGeometry +public class STKMeshBuffers: STKDrawableObject { + public var mesh: STKMesh? + public var arkitMesh: ARFaceGeometry? + + public var vertexType: Any = GLKVector3() + public var indexType: Any = UInt32() + + fileprivate var device: MTLDevice + fileprivate var textureCache: CVMetalTextureCache! + fileprivate var vertexBuffer: MTLBuffer? + fileprivate var indexBuffer: MTLBuffer? + fileprivate var lineBuffer: MTLBuffer? + fileprivate var normalBuffer: MTLBuffer? + fileprivate var colorBuffer: MTLBuffer? + fileprivate var texcoordBuffer: MTLBuffer? + fileprivate var textureYinternal: MTLTexture? + fileprivate var textureCbCrinternal: MTLTexture? + fileprivate var nTriangle: Int = 0 + fileprivate var nVertex: Int = 0 + fileprivate var nLine: Int = 0 + + public init(_ device: MTLDevice) { + self.device = device + CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, device, nil, &textureCache) + } + + fileprivate func clear() { + vertexBuffer = nil + indexBuffer = nil + lineBuffer = nil + normalBuffer = nil + colorBuffer = nil + texcoordBuffer = nil + textureYinternal = nil + textureCbCrinternal = nil + nTriangle = 0 + nVertex = 0 + nLine = 0 + self.mesh = nil + arkitMesh = nil + } + + public func updateMesh(_ mesh: STKMesh) { + self.mesh = mesh + updateBuffers() + } + + public func updateMesh(arkitFace mesh: ARFaceGeometry) { + clear() + vertexType = vector_float3() + indexType = UInt16() + + guard mesh.vertices.count > 0 else { return } + + nTriangle = mesh.triangleCount + nVertex = mesh.vertices.count + + vertexBuffer = device.makeBuffer( + bytes: mesh.vertices, length: mesh.vertices.count * MemoryLayout.stride, options: []) + + // convert Int16 to UInt16 + var indices = [UInt16](repeating: 0, count: mesh.triangleIndices.count) + for i in 0...stride + indexBuffer = device.makeBuffer(bytes: indices, length: indexSize, options: []) + } + + public func vertexArray() -> [simd_float3]? { + if let mesh = mesh { + let numVertices: Int = Int(mesh.number(ofMeshVertices: Int32(0))) + let buff = UnsafeMutableBufferPointer(start: mesh.meshVertices(0), count: numVertices) + let glk: [GLKVector3] = [GLKVector3](buff) + let simd: [simd_float3] = glk.map { simd_float3($0) } + return simd + } else if let mesh = arkitMesh { + return mesh.vertices + } + return nil + } + + public func boundingBox() -> (min: simd_float3, max: simd_float3)? { + var min = simd_float3(repeating: Float.greatestFiniteMagnitude) + var max = simd_float3(repeating: -Float.greatestFiniteMagnitude) + + if let mesh = mesh { + for i in 0.. 0 else { return } + let numVertices: Int = Int(mesh.number(ofMeshVertices: Int32(0))) + vertexBuffer = device.makeBuffer( + bytes: mesh.meshVertices(0)!, length: numVertices * MemoryLayout.stride, options: []) + + if mesh.hasPerVertexNormals(), let bytes = mesh.meshPerVertexNormals(0) { + normalBuffer = device.makeBuffer( + bytes: bytes, length: numVertices * MemoryLayout.stride, options: []) + } + if mesh.hasPerVertexColors(), let bytes = mesh.meshPerVertexColors(0) { + colorBuffer = device.makeBuffer( + bytes: bytes, length: numVertices * MemoryLayout.stride, options: []) + } + if mesh.hasPerVertexUVTextureCoords(), let bytes = mesh.meshPerVertexUVTextureCoords(0) { + texcoordBuffer = device.makeBuffer( + bytes: bytes, length: numVertices * MemoryLayout.stride, options: []) + } + + nLine = Int(mesh.number(ofMeshLines: 0)) + if nLine > 0, let bytes = mesh.meshLines(0) { + lineBuffer = device.makeBuffer(bytes: bytes, length: nLine * MemoryLayout.stride * 2, options: []) + } + + let indexSize = Int(mesh.number(ofMeshFaces: Int32(0))) * MemoryLayout.stride * 3 + if indexSize > 0, let bytes = mesh.meshFaces(0) { + indexBuffer = device.makeBuffer(bytes: bytes, length: indexSize, options: []) + } + + nTriangle = Int(mesh.number(ofMeshFaces: 0)) + nVertex = Int(mesh.number(ofMeshVertices: 0)) + + // texture + if let pixelBuffer: CVPixelBuffer = mesh.meshYCbCrTexture()?.takeUnretainedValue() { + textureYinternal = try! makeTexture( + pixelBuffer: pixelBuffer, textureCache: textureCache, planeIndex: 0, pixelFormat: .r8Unorm) + textureCbCrinternal = try! makeTexture( + pixelBuffer: pixelBuffer, textureCache: textureCache, planeIndex: 1, pixelFormat: .rg8Unorm) + } + } + } + + public func vertices() -> MTLBuffer? { vertexBuffer } + + public func indices() -> MTLBuffer? { indexBuffer } + + public func lines() -> MTLBuffer? { lineBuffer } + + public func normals() -> MTLBuffer? { normalBuffer } + + public func colors() -> MTLBuffer? { colorBuffer } + + public func texCoords() -> MTLBuffer? { texcoordBuffer } + + public func textureY() -> MTLTexture? { textureYinternal } + + public func textureCbCr() -> MTLTexture? { textureCbCrinternal } + + public func modelMatrix() -> float4x4 { float4x4() } + + public func vertexCount() -> Int { nVertex } + + public func triangleCount() -> Int { nTriangle } + + public func lineCount() -> Int { nLine } +} + +extension STKMeshBuffers { + public func update(polyline: [vector_float3], colors: [vector_float3], closed: Bool = false) { + clear() + + vertexType = vector_float3() + indexType = UInt32() + + guard polyline.count > 1 else { return } + + // TODO(GD): important + // MemoryLayout.stride means there should be + // vertexDescriptor.layouts[0].stride = MemoryLayout.stride in the vertex descriptor!!! + // make type fixed in the drawable ? + + nVertex = polyline.count + vertexBuffer = device.makeBuffer( + bytes: polyline, length: polyline.count * MemoryLayout.stride, options: []) + + if colors.count == polyline.count { + colorBuffer = device.makeBuffer( + bytes: colors, length: colors.count * MemoryLayout.stride, options: []) + } + + nLine = polyline.count - 1 + if nLine > 0 { + var segmentIndices: [vector_uint2] = [] + for i in stride(from: 0, through: polyline.count - 1, by: 1) { + segmentIndices.append(vector_uint2(x: UInt32(i), y: UInt32(i + 1))) + } + if closed { + nLine += 1 + segmentIndices.append(vector_uint2(x: UInt32(polyline.count - 1), y: 0)) + } + lineBuffer = device.makeBuffer( + bytes: segmentIndices, length: nLine * MemoryLayout.stride * 2, options: []) + } + } + +} diff --git a/Sources/StructureKit/Metal/STKMeshRenderers.swift b/Sources/StructureKit/Metal/STKMeshRenderers.swift new file mode 100644 index 0000000..d588ea5 --- /dev/null +++ b/Sources/StructureKit/Metal/STKMeshRenderers.swift @@ -0,0 +1,581 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import GLKit +import MetalKit +import StructureKitCTypes + +public protocol STKShader { + func render( + _ commandEncoder: MTLRenderCommandEncoder, + node: STKDrawableObject, + worldModelMatrix: float4x4, + projectionMatrix: float4x4 + ) +} + +// Renders a mesh as a solid body, in grayscale by default, uses the mesh normals to calculate light +public class STKMeshRendererSolid: STKShader { + private var depthStencilState: MTLDepthStencilState + private var pipelineState: MTLRenderPipelineState + + public init(colorFormat: MTLPixelFormat, depthFormat: MTLPixelFormat, device: MTLDevice) { + depthStencilState = makeDepthStencilState(device) + + let vertexDescriptor = MTLVertexDescriptor() + vertexDescriptor.attributes[0] = MTLVertexAttributeDescriptor(bufferIndex: 0, offset: 0, format: .float3) // vertices + vertexDescriptor.attributes[1] = MTLVertexAttributeDescriptor( + bufferIndex: Int(STKVertexAttrAddition.rawValue), offset: 0, format: .float3) // normals + vertexDescriptor.layouts[0].stride = MemoryLayout.stride + vertexDescriptor.layouts[1].stride = MemoryLayout.stride + + pipelineState = makePipeline( + device, + "vertexSolid", + "fragmentSolid", + vertexDescriptor, + colorFormat, + depthFormat, + blending: true) + } + + public convenience init(view: MTKView, device: MTLDevice) { + self.init(colorFormat: view.colorPixelFormat, depthFormat: view.depthStencilPixelFormat, device: device) + } + + public func render( + _ commandEncoder: MTLRenderCommandEncoder, + node: STKDrawableObject, + worldModelMatrix: float4x4, + projectionMatrix: float4x4 + ) { + render( + commandEncoder, + node: node, + worldModelMatrix: worldModelMatrix, + projectionMatrix: projectionMatrix, + color: vector_float4(1, 1, 1, 1)) + } + + public func render( + _ commandEncoder: MTLRenderCommandEncoder, + node: STKDrawableObject, + worldModelMatrix: float4x4, + projectionMatrix: float4x4, + color: vector_float4 = vector_float4(1, 1, 1, 1) + ) { + guard node.vertexType is GLKVector3, + node.indexType is UInt32 + else { + assertionFailure("Type mismatch") + return + } + guard let vertexBuffer = node.vertices(), + let indexBuffer = node.indices(), + let normalsBuffer = node.normals() + else { return } + + commandEncoder.pushDebugGroup("RenderMeshLightedGrey") + + commandEncoder.setDepthStencilState(depthStencilState) + commandEncoder.setRenderPipelineState(pipelineState) + + // buffers + commandEncoder.setVertexBuffer(vertexBuffer, offset: 0, index: Int(STKVertexAttrPosition.rawValue)) + commandEncoder.setVertexBuffer(normalsBuffer, offset: 0, index: Int(STKVertexAttrAddition.rawValue)) + let nodeModelMatrix = worldModelMatrix * node.modelMatrix() + var uniforms = STKUniformsMesh(modelViewMatrix: nodeModelMatrix, projectionMatrix: projectionMatrix, color: color) + commandEncoder.setVertexBytes( + &uniforms, length: MemoryLayout.stride, index: Int(STKVertexBufferIndexUniforms.rawValue)) + + commandEncoder.drawIndexedPrimitives( + type: .triangle, + indexCount: node.triangleCount() * 3, + indexType: .uint32, + indexBuffer: indexBuffer, + indexBufferOffset: 0) + commandEncoder.popDebugGroup() + } +} + +// Renders mesh edges, uses the mesh normals to calculate light +public class STKMeshRendererWireframe: STKShader { + private var pipelineState: MTLRenderPipelineState + + public init(colorFormat: MTLPixelFormat, depthFormat: MTLPixelFormat, device: MTLDevice) { + let vertexDescriptor = MTLVertexDescriptor() + vertexDescriptor.attributes[0] = MTLVertexAttributeDescriptor(bufferIndex: 0, offset: 0, format: .float3) // vertices + vertexDescriptor.attributes[1] = MTLVertexAttributeDescriptor( + bufferIndex: Int(STKVertexAttrAddition.rawValue), offset: 0, format: .float3) // normals + vertexDescriptor.layouts[0].stride = MemoryLayout.stride + vertexDescriptor.layouts[1].stride = MemoryLayout.stride + + pipelineState = makePipeline( + device, + "vertexWireframe", + "fragmentWireframe", + vertexDescriptor, + colorFormat, + depthFormat, + blending: false) + } + + public convenience init(view: MTKView, device: MTLDevice) { + self.init(colorFormat: view.colorPixelFormat, depthFormat: view.depthStencilPixelFormat, device: device) + } + + public func render( + _ commandEncoder: MTLRenderCommandEncoder, + node: STKDrawableObject, + worldModelMatrix: float4x4, + projectionMatrix: float4x4 + ) { + render( + commandEncoder, node: node, worldModelMatrix: worldModelMatrix, projectionMatrix: projectionMatrix, useXray: true) + } + + public func render( + _ commandEncoder: MTLRenderCommandEncoder, + node: STKDrawableObject, + worldModelMatrix: float4x4, + projectionMatrix: float4x4, + useXray: Bool = true, + color: vector_float4 = vector_float4(1, 1, 1, 1) + ) { + guard node.vertexType is GLKVector3, + node.indexType is UInt32 + else { + assertionFailure("Type mismatch") + return + } + + guard let vertexBuffer = node.vertices(), + let lineIndexBuffer = node.lines(), + let normalsBuffer = node.normals() + else { return } + + commandEncoder.pushDebugGroup("RenderMeshXray") + commandEncoder.setRenderPipelineState(pipelineState) + + // set buffers + commandEncoder.setVertexBuffer(vertexBuffer, offset: 0, index: Int(STKVertexAttrPosition.rawValue)) + commandEncoder.setVertexBuffer(normalsBuffer, offset: 0, index: Int(STKVertexAttrAddition.rawValue)) + let nodeModelMatrix = worldModelMatrix * node.modelMatrix() + var uniforms = STKUniformsMeshWireframe( + modelViewMatrix: nodeModelMatrix, projectionMatrix: projectionMatrix, + color: color, + useXray: useXray) + + commandEncoder.setVertexBytes( + &uniforms, length: MemoryLayout.stride, + index: Int(STKVertexBufferIndexUniforms.rawValue)) + + commandEncoder.drawIndexedPrimitives( + type: .line, + indexCount: node.lineCount() * 2, + indexType: .uint32, + indexBuffer: lineIndexBuffer, + indexBufferOffset: 0) + + commandEncoder.popDebugGroup() + } +} + +// Renders a colored mesh, uses the vertex colors +public class STKMeshRendererColor: STKShader { + var depthStencilState: MTLDepthStencilState + private var pipelineState: MTLRenderPipelineState + + public init(colorFormat: MTLPixelFormat, depthFormat: MTLPixelFormat, device: MTLDevice) { + depthStencilState = makeDepthStencilState(device) + + let vertexDescriptor = MTLVertexDescriptor() + vertexDescriptor.attributes[0] = MTLVertexAttributeDescriptor( + bufferIndex: Int(STKVertexAttrPosition.rawValue), offset: 0, format: .float3) // vertices + vertexDescriptor.attributes[1] = MTLVertexAttributeDescriptor( + bufferIndex: Int(STKVertexAttrAddition.rawValue), offset: 0, format: .float3) // colors + vertexDescriptor.layouts[0].stride = MemoryLayout.stride + vertexDescriptor.layouts[1].stride = MemoryLayout.stride + + pipelineState = makePipeline( + device, + "vertexColor", + "fragmentColor", + vertexDescriptor, + colorFormat, + depthFormat, + blending: false) + } + + public convenience init(view: MTKView, device: MTLDevice) { + self.init(colorFormat: view.colorPixelFormat, depthFormat: view.depthStencilPixelFormat, device: device) + } + + public func render( + _ commandEncoder: MTLRenderCommandEncoder, + node: STKDrawableObject, + worldModelMatrix: float4x4, + projectionMatrix: float4x4 + ) { + guard node.vertexType is GLKVector3, + node.indexType is UInt32 + else { + assertionFailure("Type mismatch") + return + } + + guard let vertexBuffer = node.vertices(), + let indexBuffer = node.indices(), + let colorsBuffer = node.colors(), + indexBuffer.length > 0 + else { return } + + commandEncoder.pushDebugGroup("RenderMeshColor") + commandEncoder.setRenderPipelineState(pipelineState) + + commandEncoder.setCullMode(MTLCullMode.front) + commandEncoder.setDepthStencilState(depthStencilState) + commandEncoder.setRenderPipelineState(pipelineState) + + commandEncoder.setVertexBuffer(vertexBuffer, offset: 0, index: Int(STKVertexAttrPosition.rawValue)) + commandEncoder.setVertexBuffer(colorsBuffer, offset: 0, index: Int(STKVertexAttrAddition.rawValue)) + + // set uniforms + let nodeModelMatrix = worldModelMatrix * node.modelMatrix() + var uniforms = STKUniformsMesh( + modelViewMatrix: nodeModelMatrix, projectionMatrix: projectionMatrix, color: vector_float4(1, 1, 1, 1)) + commandEncoder.setVertexBytes( + &uniforms, length: MemoryLayout.stride, index: Int(STKVertexBufferIndexUniforms.rawValue)) + + commandEncoder.drawIndexedPrimitives( + type: .triangle, + indexCount: node.triangleCount() * 3, + indexType: .uint32, + indexBuffer: indexBuffer, + indexBufferOffset: 0) + + commandEncoder.popDebugGroup() + } +} + +// Renders a textured mesh +public class STKMeshRendererTexture: STKShader { + private var depthStencilState: MTLDepthStencilState + private var samplerState: MTLSamplerState + private var pipelineState: MTLRenderPipelineState + + public init(colorFormat: MTLPixelFormat, depthFormat: MTLPixelFormat, device: MTLDevice) { + depthStencilState = makeDepthStencilState(device) + samplerState = makeDefaultSampler(device) + + let vertexDescriptor = MTLVertexDescriptor() + vertexDescriptor.attributes[0] = MTLVertexAttributeDescriptor( + bufferIndex: Int(STKVertexAttrPosition.rawValue), offset: 0, format: .float3) // vertices + vertexDescriptor.attributes[1] = MTLVertexAttributeDescriptor( + bufferIndex: Int(STKVertexAttrAddition.rawValue), offset: 0, format: .float2) // texture coordinates + vertexDescriptor.layouts[0].stride = MemoryLayout.stride + vertexDescriptor.layouts[1].stride = MemoryLayout.stride + + pipelineState = makePipeline( + device, + "vertexTexture", + "fragmentTexture", + vertexDescriptor, + colorFormat, + depthFormat, + blending: false) + } + + public convenience init(view: MTKView, device: MTLDevice) { + self.init(colorFormat: view.colorPixelFormat, depthFormat: view.depthStencilPixelFormat, device: device) + } + + public func render( + _ commandEncoder: MTLRenderCommandEncoder, + node: STKDrawableObject, + worldModelMatrix: float4x4, + projectionMatrix: float4x4 + ) { + guard node.vertexType is GLKVector3, + node.indexType is UInt32 + else { + assertionFailure("Type mismatch") + return + } + + guard let vertexBuffer = node.vertices(), + let indexBuffer = node.indices(), + let texcoordBuffer = node.texCoords(), + let textureY = node.textureY(), + let textureCbCr = node.textureCbCr() + else { return } + + commandEncoder.pushDebugGroup("RenderMeshTexture") + + commandEncoder.setCullMode(MTLCullMode.front) + commandEncoder.setDepthStencilState(depthStencilState) + commandEncoder.setRenderPipelineState(pipelineState) + + commandEncoder.setVertexBuffer(vertexBuffer, offset: 0, index: Int(STKVertexAttrPosition.rawValue)) + commandEncoder.setVertexBuffer(texcoordBuffer, offset: 0, index: Int(STKVertexAttrAddition.rawValue)) + + // set uniforms + let nodeModelMatrix = worldModelMatrix * node.modelMatrix() + var uniforms = STKUniformsMesh( + modelViewMatrix: nodeModelMatrix, projectionMatrix: projectionMatrix, color: vector_float4(1, 1, 1, 1)) + commandEncoder.setVertexBytes( + &uniforms, length: MemoryLayout.stride, index: Int(STKVertexBufferIndexUniforms.rawValue)) + + commandEncoder.setFragmentTexture(textureY, index: 0) // [[ texture(0) ]] + commandEncoder.setFragmentTexture(textureCbCr, index: 1) // [[ texture(0) ]] + commandEncoder.setFragmentSamplerState(samplerState, index: 0) // [[ sampler(0) ]] + + commandEncoder.drawIndexedPrimitives( + type: .triangle, + indexCount: node.triangleCount() * 3, + indexType: .uint32, + indexBuffer: indexBuffer, + indexBufferOffset: 0) + + commandEncoder.popDebugGroup() + } +} + +// Renders a mesh in grayscale, uses the mesh normals to calculate light +class STKMeshRendererPoints: STKShader { + private var depthStencilState: MTLDepthStencilState + private var pipelineState: MTLRenderPipelineState + + public init(colorFormat: MTLPixelFormat, depthFormat: MTLPixelFormat, device: MTLDevice) { + depthStencilState = makeDepthStencilState(device) + + let vertexDescriptor = MTLVertexDescriptor() + vertexDescriptor.attributes[0] = MTLVertexAttributeDescriptor(bufferIndex: 0, offset: 0, format: .float3) // vertices + vertexDescriptor.attributes[1] = MTLVertexAttributeDescriptor( + bufferIndex: Int(STKVertexAttrAddition.rawValue), offset: 0, format: .float3) // colors + vertexDescriptor.layouts[0].stride = MemoryLayout.stride + vertexDescriptor.layouts[1].stride = MemoryLayout.stride + + pipelineState = makePipeline( + device, + "vertexColorPoints", + "fragmentColorPoints", + vertexDescriptor, + colorFormat, + depthFormat, + blending: false) + } + + public convenience init(view: MTKView, device: MTLDevice) { + self.init(colorFormat: view.colorPixelFormat, depthFormat: view.depthStencilPixelFormat, device: device) + } + + public func render( + _ commandEncoder: MTLRenderCommandEncoder, + node: STKDrawableObject, + worldModelMatrix: float4x4, + projectionMatrix: float4x4 + ) { + render( + commandEncoder, + node: node, + worldModelMatrix: worldModelMatrix, + projectionMatrix: projectionMatrix, + alpha: 1.0) + } + + public func render( + _ commandEncoder: MTLRenderCommandEncoder, + node: STKDrawableObject, + worldModelMatrix: float4x4, + projectionMatrix: float4x4, + alpha: Float = 1.0 + ) { + guard node.vertexType is GLKVector3 else { + assertionFailure("Type mismatch") + return + } + + guard let vertexBuffer = node.vertices(), + let colorsBuffer = node.colors() + else { return } + + commandEncoder.pushDebugGroup("RenderPoints") + commandEncoder.setRenderPipelineState(pipelineState) + + commandEncoder.setCullMode(MTLCullMode.front) + commandEncoder.setDepthStencilState(depthStencilState) + commandEncoder.setRenderPipelineState(pipelineState) + + commandEncoder.setVertexBuffer(vertexBuffer, offset: 0, index: Int(STKVertexAttrPosition.rawValue)) + commandEncoder.setVertexBuffer(colorsBuffer, offset: 0, index: Int(STKVertexAttrAddition.rawValue)) + + // set uniforms + let nodeModelMatrix = worldModelMatrix * node.modelMatrix() + var uniforms = STKUniformsMesh( + modelViewMatrix: nodeModelMatrix, projectionMatrix: projectionMatrix, color: vector_float4(1, 1, 1, alpha)) + commandEncoder.setVertexBytes( + &uniforms, length: MemoryLayout.stride, index: Int(STKVertexBufferIndexUniforms.rawValue)) + + commandEncoder.drawPrimitives(type: .point, vertexStart: 0, vertexCount: node.vertexCount()) + + commandEncoder.popDebugGroup() + } +} + +public class STKMeshRendererLines: STKShader { + private var depthStencilState: MTLDepthStencilState + private var pipelineState: MTLRenderPipelineState + + public init(colorFormat: MTLPixelFormat, depthFormat: MTLPixelFormat, device: MTLDevice) { + depthStencilState = makeDepthStencilState(device) + + let vertexDescriptor = MTLVertexDescriptor() + vertexDescriptor.attributes[0] = MTLVertexAttributeDescriptor(bufferIndex: 0, offset: 0, format: .float3) // vertices + vertexDescriptor.attributes[1] = MTLVertexAttributeDescriptor( + bufferIndex: Int(STKVertexAttrAddition.rawValue), offset: 0, format: .float3) // colors + vertexDescriptor.layouts[0].stride = MemoryLayout.stride + vertexDescriptor.layouts[1].stride = MemoryLayout.stride + + pipelineState = makePipeline( + device, + "vertexColorPoints", + "fragmentColorPoints", + vertexDescriptor, + colorFormat, + depthFormat, + blending: false) + } + + public convenience init(view: MTKView, device: MTLDevice) { + self.init(colorFormat: view.colorPixelFormat, depthFormat: view.depthStencilPixelFormat, device: device) + } + + public func render( + _ commandEncoder: MTLRenderCommandEncoder, + node: STKDrawableObject, + worldModelMatrix: float4x4, + projectionMatrix: float4x4 + ) { + render( + commandEncoder, + node: node, + worldModelMatrix: worldModelMatrix, + projectionMatrix: projectionMatrix, + alpha: 1.0) + } + + public func render( + _ commandEncoder: MTLRenderCommandEncoder, + node: STKDrawableObject, + worldModelMatrix: float4x4, + projectionMatrix: float4x4, + alpha: Float = 1.0 + ) { + guard node.vertexType is vector_float3, + node.indexType is UInt32 + else { + assertionFailure("Type mismatch") + return + } + + guard let vertexBuffer = node.vertices(), + let colorsBuffer = node.colors(), + let lineIndexBuffer = node.lines() + else { return } + + commandEncoder.pushDebugGroup("RenderLines") + commandEncoder.setRenderPipelineState(pipelineState) + + commandEncoder.setCullMode(MTLCullMode.front) + commandEncoder.setDepthStencilState(depthStencilState) + commandEncoder.setRenderPipelineState(pipelineState) + + commandEncoder.setVertexBuffer(vertexBuffer, offset: 0, index: Int(STKVertexAttrPosition.rawValue)) + commandEncoder.setVertexBuffer(colorsBuffer, offset: 0, index: Int(STKVertexAttrAddition.rawValue)) + + // set uniforms + let nodeModelMatrix = worldModelMatrix * node.modelMatrix() + var uniforms = STKUniformsMesh( + modelViewMatrix: nodeModelMatrix, projectionMatrix: projectionMatrix, color: vector_float4(1, 1, 1, alpha)) + commandEncoder.setVertexBytes( + &uniforms, length: MemoryLayout.stride, index: Int(STKVertexBufferIndexUniforms.rawValue)) + + commandEncoder.drawIndexedPrimitives( + type: .line, + indexCount: node.lineCount() * 2, + indexType: .uint32, + indexBuffer: lineIndexBuffer, + indexBufferOffset: 0) + + commandEncoder.popDebugGroup() + } +} + +public class STKScanMeshRenderer { + private var solid: STKMeshRendererSolid + private var wireframe: STKMeshRendererWireframe + + public init(view: MTKView, device: MTLDevice) { + solid = STKShaderManager.solid + wireframe = STKShaderManager.wireframe + } + + public func render( + _ commandEncoder: MTLRenderCommandEncoder, + node: STKDrawableObject, + cameraPosition: float4x4, + projection: float4x4, + orientation: float4x4, + color: vector_float4, + style: STKMeshRenderingStyle + ) { + let modelViewMatrix = cameraPosition.inverse + let projectionMatrix = orientation * projection + + switch style { + case .solid: + solid.render( + commandEncoder, + node: node, + worldModelMatrix: modelViewMatrix, + projectionMatrix: projectionMatrix, + color: color + ) + case .wireframe: + wireframe.render( + commandEncoder, + node: node, + worldModelMatrix: modelViewMatrix, + projectionMatrix: projectionMatrix, + useXray: false, + color: color) + } + } + +} diff --git a/Sources/StructureKit/Metal/STKMetalCommon.h b/Sources/StructureKit/Metal/STKMetalCommon.h new file mode 100644 index 0000000..e4178b0 --- /dev/null +++ b/Sources/StructureKit/Metal/STKMetalCommon.h @@ -0,0 +1,34 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include + +float4 calcDepthColor(float depth, float2 range, metal::texture2d colors); +float4 calcYCbCrColor(float y, float2 cbcr); diff --git a/Sources/StructureKit/Metal/STKMetalCommon.metal b/Sources/StructureKit/Metal/STKMetalCommon.metal new file mode 100644 index 0000000..a36b405 --- /dev/null +++ b/Sources/StructureKit/Metal/STKMetalCommon.metal @@ -0,0 +1,56 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#include "STKMetalCommon.h" + +#include + +using namespace metal; + +float4 calcDepthColor(float depth, float2 range, texture2d colors) +{ + const float minDepth = range.x; + const float maxDepth = range.y; + const float normalized = clamp((depth - minDepth) / (maxDepth - minDepth), 0.f, 0.999f); // [0, 1) + + const uint colorCoord = colors.get_width() * normalized; + const uint colorCoordNext = colorCoord < (colors.get_width() - 1) ? colorCoord + 1 : colorCoord; + const float fraction = float(colors.get_width()) * normalized - colorCoord; + + const float4 color1 = colors.read(uint2(colorCoord, 0)); + const float4 color2 = colors.read(uint2(colorCoordNext, 0)); + float4 finalColor = color1 * (1 - fraction) + color2 * fraction; + return finalColor; +} + +float4 calcYCbCrColor(float y, float2 cbcr) +{ + const float3x3 yuv2rgb(1, 1, 1, 0, -.18732, 1.8556, 1.57481, -.46813, 0); + const float3 rgb = yuv2rgb * float3(y, cbcr); + return float4(rgb, 1.0); +} diff --git a/Sources/StructureKit/Metal/STKMetalData.h b/Sources/StructureKit/Metal/STKMetalData.h new file mode 100644 index 0000000..0ba3fb7 --- /dev/null +++ b/Sources/StructureKit/Metal/STKMetalData.h @@ -0,0 +1,152 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#import + +#ifdef __METAL_VERSION__ + #define ST_IS_METAL 1 + #define ST_IF_METAL(x) x +#else + #define ST_IS_METAL 0 + #define ST_IF_METAL(x) + #include +#endif + +struct STKIntrinsicsMetal +{ + float cx; + float cy; + float fx; + float fy; + uint32_t width; + uint32_t height; +}; + +struct STKUniformsColorTexture +{ + matrix_float4x4 projection; +}; + +struct STKUniformsDepthTexture +{ + matrix_float4x4 projection; + float depthMin; + float depthMax; + float alpha; +}; + +struct STKUniformsDepthOverlay +{ + matrix_float4x4 projection; + matrix_float4x4 cameraPose; + struct STKIntrinsicsMetal cameraIntrinsics; + matrix_float4x4 cubeModelInv; + float depthMin; + float depthMax; + float alpha; +}; + +struct STKUniformsLine +{ + matrix_float4x4 model; + matrix_float4x4 view; + matrix_float4x4 projection; +}; + +struct STKUniformsCube +{ + matrix_float4x4 model; + matrix_float4x4 view; + matrix_float4x4 projection; + struct STKIntrinsicsMetal cameraIntrinsics; + bool useOcclusion; +}; + +struct STKUniformsMesh +{ + matrix_float4x4 modelViewMatrix; + matrix_float4x4 projectionMatrix; + vector_float4 color; +}; + +struct STKUniformsMeshWireframe +{ + matrix_float4x4 modelViewMatrix; + matrix_float4x4 projectionMatrix; + vector_float4 color; + bool useXray; +}; + +struct STKVertexTex +{ + vector_float3 position; + vector_float2 texCoord; +}; + +// The following enums are added to keep in order indexes of the buffers for mesh rendering shaders. +enum STKVertexAttr +{ + STKVertexAttrPosition = 0, + STKVertexAttrAddition = 1 +}; + +// STKVertexBufferIndex is an addition to STKVertexAttr, so it starts from 2 +enum STKVertexBufferIndex +{ + STKVertexBufferIndexUniforms = 2 +}; + +struct STKVertexNormal +{ + vector_float3 position ST_IF_METAL([[attribute(STKVertexAttrPosition)]]); + vector_float3 normal ST_IF_METAL([[attribute(STKVertexAttrAddition)]]); +}; + +struct STKVertexColor +{ + vector_float3 position ST_IF_METAL([[attribute(STKVertexAttrPosition)]]); + vector_float3 color ST_IF_METAL([[attribute(STKVertexAttrAddition)]]); +}; + +struct STKVertexTexModel +{ + vector_float3 position ST_IF_METAL([[attribute(STKVertexAttrPosition)]]); + vector_float2 texCoord ST_IF_METAL([[attribute(STKVertexAttrAddition)]]); +}; + +#if ST_IS_METAL + +struct STKVertexTexOut +{ + float4 position [[position]]; + float2 texCoord; +}; + +#endif diff --git a/Sources/StructureKit/Metal/STKMetalLibLoader.swift b/Sources/StructureKit/Metal/STKMetalLibLoader.swift new file mode 100644 index 0000000..9ef6525 --- /dev/null +++ b/Sources/StructureKit/Metal/STKMetalLibLoader.swift @@ -0,0 +1,42 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import MetalKit + +class STKMetalLibLoader { + class func load(device: MTLDevice) -> MTLLibrary { + guard + let url = Bundle.main.url( + forResource: "default", withExtension: "metallib", subdirectory: "StructureKit_StructureKit.bundle"), + let library = try? device.makeLibrary(URL: url) + else { + fatalError("Unable to load Metal library") + } + return library + } +} diff --git a/Sources/StructureKit/Metal/STKMetalRenderer.swift b/Sources/StructureKit/Metal/STKMetalRenderer.swift new file mode 100644 index 0000000..ac6851b --- /dev/null +++ b/Sources/StructureKit/Metal/STKMetalRenderer.swift @@ -0,0 +1,358 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import ARKit +import Accelerate +import CoreVideo +import MetalKit + +public enum STKMeshRenderingStyle { + case solid + case wireframe +} + +// MARK: Metal rendering API +protocol STKRenderer: AnyObject { + // MARK: update functions + /** A depth frame for visualization + @param depthFrame The depth frame. + */ + func setDepthFrame(_ depthFrame: STKDepthFrame) + + /** A color frame for visualization + @param colorFrame The color frame. + */ + func setColorFrame(_ colorFrame: STKColorFrame) + + /** A mesh for visualization of current scanning progress + @param mesh The mesh. + */ + func setScanningMesh(_ mesh: STKMesh) + + /** An ARKit face mesh for visualization + @param mesh The mesh. + */ + func setARKitMesh(_ mesh: ARFaceGeometry) + + /** Transformation matrices describing coordinate systems for visualization + @param anchors Set of transformation matrices. + */ + func setARKitAnchors(_ anchors: [simd_float4x4]) + + /** Specify the cube size. + @param sizeInMeters The current volume size in meters. + */ + func adjustCubeSize(_ sizeInMeters: simd_float3) + + /** + Set custom colors used for depth rendering. + @param baseColors Array of base colors for gradient. + Colors are specified in RGB space using float values in range [0, 1]. Outlier will be clamped to that range. + These colors will be spaced evenly and the final color will be lineary interpolated between nearest neighbours. + */ + func setDepthRenderingColors(_ baseColors: [simd_float4]) + + // MARK: rendering functions + /** + Initialized metal rendering pipeline. Must be called before other rendering functions. + */ + func startRendering() + + /** Highlight the depth frame area which fits inside the cube. + @param cameraPose the viewpoint to use for rendering. + @param alpha transparency factor between 0 (fully transparent) and 1 (fully opaque). + @param textureOrientation orientation of the depth texture relatively to the metal view. + */ + func renderHighlightedDepth(cameraPose: simd_float4x4, alpha: Float, textureOrientation: simd_float4x4) + + /** + Render the cube wireframe outline at the given pose. + @param cameraPose the viewpoint to use for rendering. + @param occlusionTestEnabled whether to use the current depth frame to do occlusion testing. You can turn this off for + better performance. + @param orientation orientation of the cube relatively to the metal view. + @param drawTriad whether to draw the origin of the coordinate system. + */ + func renderCubeOutline(cameraPose: simd_float4x4, occlusionTest: Bool, orientation: simd_float4x4, drawTriad: Bool) + + /** + Render the color frame in the view. + @param orientation orientation of the color texture relatively to the metal view. + */ + func renderColorFrame(orientation textureOrientation: simd_float4x4) + + /** + Render the depth frame in the view. + @param orientation orientation of the depth texture relatively to the metal view. + @param range minimum and maximum depth in mm for colorization. + */ + func renderDepthFrame(orientation textureOrientation: simd_float4x4, range: simd_float2) + + /** + Render the current mesh in the view. + @param cameraPose the viewpoint to use for rendering. + @param meshOrientation orientation of the mesh relatively to the metal view. + */ + func renderScanningMesh( + cameraPose: simd_float4x4, meshOrientation: simd_float4x4, color: vector_float4, style: STKMeshRenderingStyle) + + /** + Render the ARKit face mesh in the view. + @param cameraPose the viewpoint to use for rendering. + @param orientation orientation of the mesh relatively to the metal view. + */ + func renderARKitMesh(cameraPose: simd_float4x4, orientation: simd_float4x4) + + /** + Render the triads in the view. + @param cameraPose the viewpoint to use for rendering. + @param orientation orientation of the triads relatively to the metal view. + */ + func renderARKitAnchors(cameraPose: simd_float4x4, orientation: simd_float4x4) + + /** + Finalizes the rendering pipeline and presents the resulting texture in the metal view. + */ + func presentDrawable() + + /** + Access to the Command Encoder of the current pipeline. Available between calls to startRendering() and presentDrawable() + */ + var commandEncoder: MTLRenderCommandEncoder? { get } +} + +// Implementation of STKRenderer protocol. Calculates viewport and projection matrix and stores the metal buffers to render meshes. +// Starts and finalizes the rendering pipeline. +public class STKMetalRenderer: NSObject, STKRenderer { + // data to visualize + private var _scanMesh: STKMeshBuffers + private var _arkitMesh: STKMeshBuffers + private var _anchors: [simd_float4x4] = [] + private var _volumeSize = simd_float3(1, 1, 1) + private var colorCameraGLProjectionMatrix = float4x4() + private var depthCameraGLProjectionMatrix = float4x4() + + // metal general + private var _mtkView: MTKView + private var _device: MTLDevice + private var _commandQueue: MTLCommandQueue + + // specific renderers + private var _colorFrameRenderer: STKColorFrameRenderer + private var _arkitRenderer: STKARKitOverlayRenderer + private var _anchorRenderer: STKLineRenderer + private var _depthOverlayRenderer: STKDepthRenderer + private var _meshRenderer: STKScanMeshRenderer + + // rendering state + private var _commandEncoder: MTLRenderCommandEncoder? + private var _commandBuffer: MTLCommandBuffer? + private var _currentDrawable: CAMetalDrawable? + + // projection + private var projection: float4x4 { colorCameraGLProjectionMatrix } + private var frameRatio: Float { + abs(colorCameraGLProjectionMatrix.columns.0[0] / colorCameraGLProjectionMatrix.columns.1[1]) + } + private var viewport: MTLViewport { + let z = (near: 0.0, far: 1.0) + let frame = _mtkView.bounds.size + let sc: CGFloat = UIScreen.main.scale + let screen = simd_float2(x: Float(frame.width * sc), y: Float(frame.height * sc)) + + let isPortraitOrientation = frame.height > frame.width + if isPortraitOrientation { + let width = screen.y * frameRatio + let overflow = Double(width) - Double(screen.x) + let viewport = MTLViewport.init( + originX: -overflow / 2, originY: 0, width: Double(width), height: Double(screen.y), znear: z.near, zfar: z.far) + return viewport + } else { + let height = screen.x * frameRatio + let overflow = Double(height) - Double(screen.y) + let viewport = MTLViewport.init( + originX: 0, originY: -overflow / 2, width: Double(screen.x), height: Double(height), znear: z.near, zfar: z.far) + return viewport + } + } + + var commandEncoder: MTLRenderCommandEncoder? { _commandEncoder } + + public init(view: MTKView, device: MTLDevice, mesh: STKMesh) { + _mtkView = view + _device = device + _commandQueue = device.makeCommandQueue()! + + _colorFrameRenderer = STKColorFrameRenderer(view: view, device: device) + _arkitRenderer = STKARKitOverlayRenderer(view: view, device: device) + _anchorRenderer = STKLineRenderer(view: view, device: device) + _depthOverlayRenderer = STKDepthRenderer(view: view, device: device) + _meshRenderer = STKScanMeshRenderer(view: view, device: device) + + _scanMesh = STKMeshBuffers(_device) + _scanMesh.mesh = mesh + _arkitMesh = STKMeshBuffers(_device) + super.init() + } + + public func setDepthFrame(_ depthFrame: STKDepthFrame) { + _depthOverlayRenderer.uploadColorTextureFromDepth(depthFrame) + depthCameraGLProjectionMatrix = float4x4(depthFrame.glProjectionMatrix()) + } + + public func setColorFrame(_ colorFrame: STKColorFrame) { + _colorFrameRenderer.uploadColorTexture(colorFrame) + colorCameraGLProjectionMatrix = float4x4(colorFrame.glProjectionMatrix()) + } + + public func setScanningMesh(_ mesh: STKMesh) { _scanMesh.updateMesh(mesh) } + + public func setARKitMesh(_ mesh: ARFaceGeometry) { _arkitMesh.updateMesh(arkitFace: mesh) } + + public func setARKitAnchors(_ anchors: [simd_float4x4]) { _anchors = anchors } + + public func adjustCubeSize(_ sizeInMeters: simd_float3) { _volumeSize = sizeInMeters } + + public func setARKitTransformation(_ transformation: simd_float4x4) { _arkitRenderer.arkitToWorld = transformation } + + public func setDepthRenderingColors(_ baseColors: [simd_float4]) { + _depthOverlayRenderer.depthRenderingColors = baseColors + } + + public func startRendering() { + guard let commandBuffer = _commandQueue.makeCommandBuffer(), + let currentRenderPassDescriptor = _mtkView.currentRenderPassDescriptor, + let currentDrawable = _mtkView.currentDrawable, + let commandEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: currentRenderPassDescriptor) + else { + return + } + _commandBuffer = commandBuffer + _currentDrawable = currentDrawable + _commandEncoder = commandEncoder + + commandEncoder.setViewport(viewport) + } + + public func presentDrawable() { + guard let commandBuffer = _commandBuffer, + let currentDrawable = _currentDrawable, + let commandEncoder = _commandEncoder + else { + return + } + + commandEncoder.endEncoding() + commandBuffer.present(currentDrawable) + commandBuffer.commit() + + _commandBuffer = nil + _commandBuffer = nil + _commandEncoder = nil + } + + public func renderColorFrame(orientation textureOrientation: simd_float4x4) { + guard let commandEncoder = _commandEncoder else { return } + _colorFrameRenderer.renderCameraImage(commandEncoder, orientation: textureOrientation) + } + + public func renderCubeOutline( + cameraPose: simd_float4x4, occlusionTest: Bool, orientation: simd_float4x4, drawTriad: Bool + ) { + guard let commandEncoder = _commandEncoder else { return } + _depthOverlayRenderer.renderCubeOutline( + commandEncoder, + volumeSizeInMeters: _volumeSize, + cameraPosition: cameraPose, + projection: projection, + orientation: orientation, + useOcclusion: occlusionTest + ) + + if drawTriad { + _anchorRenderer.renderAnchors( + commandEncoder, + anchors: [simd_float4x4()], + cameraPosition: cameraPose, + projection: projection, + orientation: orientation, + triadSize: _volumeSize.x) + } + } + + public func renderHighlightedDepth(cameraPose: simd_float4x4, alpha: Float, textureOrientation: simd_float4x4) { + guard let commandEncoder = _commandEncoder else { return } + _depthOverlayRenderer.renderDepthOverlay( + commandEncoder, + volumeSizeInMeters: _volumeSize, + cameraPosition: cameraPose, + textureOrientation: textureOrientation, + alpha: alpha) + } + + public func renderDepthFrame(orientation textureOrientation: simd_float4x4, range: simd_float2) { + guard let commandEncoder = _commandEncoder else { return } + _depthOverlayRenderer.renderDepthFrame( + commandEncoder, orientation: textureOrientation, minDepth: range.x, maxDepth: range.y, alpha: 0.5) + } + + public func renderScanningMesh( + cameraPose: simd_float4x4, meshOrientation: simd_float4x4, color: vector_float4, style: STKMeshRenderingStyle + ) { + guard let commandEncoder = _commandEncoder else { return } + _meshRenderer.render( + commandEncoder, + node: _scanMesh, + cameraPosition: cameraPose, + projection: projection, + orientation: meshOrientation, + color: color, + style: style) + } + + public func renderARKitMesh(cameraPose: simd_float4x4, orientation: simd_float4x4) { + guard let commandEncoder = _commandEncoder else { return } + _arkitRenderer.renderARkitGeom( + commandEncoder, + mesh: _arkitMesh, + cameraPosition: cameraPose, + projection: projection, + orientation: orientation) + } + + public func renderARKitAnchors(cameraPose: simd_float4x4, orientation: simd_float4x4) { + guard let commandEncoder = _commandEncoder else { return } + _anchorRenderer.renderAnchors( + commandEncoder, + anchors: _anchors, + cameraPosition: cameraPose, + projection: projection, + orientation: orientation) + } + +} diff --git a/Sources/StructureKit/Metal/STKPerVertexColor.metal b/Sources/StructureKit/Metal/STKPerVertexColor.metal new file mode 100644 index 0000000..834a33f --- /dev/null +++ b/Sources/StructureKit/Metal/STKPerVertexColor.metal @@ -0,0 +1,53 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#include "STKMetalData.h" + +#include + +using namespace metal; + +struct VertexOut +{ + float4 position [[position]]; + float4 color; +}; + +vertex VertexOut +vertexColor(STKVertexColor in [[stage_in]], const device STKUniformsMesh& uniforms [[buffer(STKVertexBufferIndexUniforms)]]) +{ + VertexOut out; + out.position = uniforms.projectionMatrix * uniforms.modelViewMatrix * float4(in.position, 1); + out.color = float4(in.color, 1); + return out; +} + +fragment float4 fragmentColor(VertexOut in [[stage_in]]) +{ + return in.color; +} diff --git a/Sources/StructureKit/Metal/STKPointCloud.metal b/Sources/StructureKit/Metal/STKPointCloud.metal new file mode 100644 index 0000000..505888f --- /dev/null +++ b/Sources/StructureKit/Metal/STKPointCloud.metal @@ -0,0 +1,56 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#include "STKMetalData.h" + +#include + +using namespace metal; + +struct VertexOut +{ + float4 position [[position]]; + float4 color; + float pointSize [[point_size]]; +}; + +vertex VertexOut vertexColorPoints( + STKVertexColor in [[stage_in]], + const device STKUniformsMesh& uniforms [[buffer(STKVertexBufferIndexUniforms)]]) +{ + VertexOut out; + out.position = uniforms.projectionMatrix * uniforms.modelViewMatrix * float4(in.position, 1); + out.color = float4(in.color, 1); + out.pointSize = 5.f; + return out; +} + +fragment float4 fragmentColorPoints(VertexOut in [[stage_in]]) +{ + return in.color; +} diff --git a/Sources/StructureKit/Metal/STKSolidColor.metal b/Sources/StructureKit/Metal/STKSolidColor.metal new file mode 100644 index 0000000..01f960b --- /dev/null +++ b/Sources/StructureKit/Metal/STKSolidColor.metal @@ -0,0 +1,58 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#include "STKMetalData.h" + +#include + +using namespace metal; + +struct VertexOut +{ + float4 position [[position]]; + float4 color; +}; + +vertex VertexOut +vertexSolid(STKVertexNormal in [[stage_in]], const device STKUniformsMesh& uniforms [[buffer(STKVertexBufferIndexUniforms)]]) +{ + const float4 normal = + uniforms.modelViewMatrix * float4(in.normal, 0); // Directional lighting that moves with the camera + const float luminance = 0.5 * abs(normal.z) + 0.5; // Slightly reducing the effect of the lighting + + VertexOut out; + out.position = uniforms.projectionMatrix * uniforms.modelViewMatrix * float4(in.position, 1); + out.color = uniforms.color * float4(luminance); + out.color.w = uniforms.color.w; + return out; +} + +fragment float4 fragmentSolid(VertexOut in [[stage_in]]) +{ + return in.color; +} diff --git a/Sources/StructureKit/Metal/STKTexture.metal b/Sources/StructureKit/Metal/STKTexture.metal new file mode 100644 index 0000000..ecc8c01 --- /dev/null +++ b/Sources/StructureKit/Metal/STKTexture.metal @@ -0,0 +1,61 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#include "STKMetalCommon.h" +#include "STKMetalData.h" + +#include + +using namespace metal; + +struct VertexOut +{ + float4 position [[position]]; + float2 texCoord; +}; + +vertex VertexOut vertexTexture( + STKVertexTexModel in [[stage_in]], + const device STKUniformsMesh& uniforms [[buffer(STKVertexBufferIndexUniforms)]]) +{ + VertexOut out; + out.position = uniforms.projectionMatrix * uniforms.modelViewMatrix * float4(in.position, 1); + out.texCoord = in.texCoord; + return out; +} + +fragment float4 fragmentTexture( + VertexOut in [[stage_in]], + texture2d tex2dY [[texture(0)]], + texture2d tex2dCbCr [[texture(1)]], + sampler sampler2D [[sampler(0)]]) +{ + const float x = tex2dY.sample(sampler2D, in.texCoord).r; + const float2 yz = tex2dCbCr.sample(sampler2D, in.texCoord).rg - float2(0.5, 0.5); + return calcYCbCrColor(x, yz); +} diff --git a/Sources/StructureKit/Metal/STKWireframe.metal b/Sources/StructureKit/Metal/STKWireframe.metal new file mode 100644 index 0000000..77a19df --- /dev/null +++ b/Sources/StructureKit/Metal/STKWireframe.metal @@ -0,0 +1,61 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#include "STKMetalData.h" + +#include + +using namespace metal; + +struct VertexOut +{ + float4 position [[position]]; + float4 color; +}; + +vertex VertexOut vertexWireframe( + STKVertexNormal in [[stage_in]], + const device STKUniformsMeshWireframe& uniforms [[buffer(STKVertexBufferIndexUniforms)]]) +{ + VertexOut out; + out.position = uniforms.projectionMatrix * uniforms.modelViewMatrix * float4(in.position, 1); + out.color = uniforms.color; + if (uniforms.useXray) + { + const float4 normal = + uniforms.modelViewMatrix * float4(in.normal, 0); // Directional lighting that moves with the camera + const float luminance = 1.0 - abs(normal.z); // Slightly reducing the effect of the lighting + out.color *= luminance; + } + return out; +} + +fragment float4 fragmentWireframe(VertexOut in [[stage_in]]) +{ + return in.color; +} diff --git a/Sources/StructureKit/STKMixedActivityItemSource.swift b/Sources/StructureKit/STKMixedActivityItemSource.swift new file mode 100644 index 0000000..6abde86 --- /dev/null +++ b/Sources/StructureKit/STKMixedActivityItemSource.swift @@ -0,0 +1,75 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import UIKit + +public class STKMixedActivityItemSource: NSObject, UIActivityItemSource { + + public enum Item { + case image(image: UIImage) + case archieve(file: URL) + case disclaimer + } + + let item: Item + + var disclaimer: String { + return """ + More info about the Structure SDK: http://structure.io/developers + """ + } + + public init(item: Item) { + self.item = item + super.init() + } + + public func activityViewControllerPlaceholderItem(_ activityViewController: UIActivityViewController) -> Any { + switch item { + case .archieve(let fileUrl): + return fileUrl + case .image(let image): + return image + case .disclaimer: + return disclaimer + } + } + + public func activityViewController( + _ activityViewController: UIActivityViewController, itemForActivityType activityType: UIActivity.ActivityType? + ) -> Any? { + switch item { + case .archieve(let fileUrl): + return fileUrl + case .image(let image): + return activityType == .airDrop ? nil : image + case .disclaimer: + return activityType == .airDrop ? nil : disclaimer + } + } +} diff --git a/Sources/StructureKitCTypes/StructureKitCTypes.mm b/Sources/StructureKitCTypes/StructureKitCTypes.mm new file mode 100644 index 0000000..1c154a5 --- /dev/null +++ b/Sources/StructureKitCTypes/StructureKitCTypes.mm @@ -0,0 +1,29 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#import "StructureKitCTypes.h" diff --git a/Sources/StructureKitCTypes/include/StructureKitCTypes.h b/Sources/StructureKitCTypes/include/StructureKitCTypes.h new file mode 100644 index 0000000..21bb3bc --- /dev/null +++ b/Sources/StructureKitCTypes/include/StructureKitCTypes.h @@ -0,0 +1,29 @@ +// StructureKit - A collection of extension utilities for Structure SDK +// Copyright 2022 XRPro, LLC. All rights reserved. +// http://structure.io +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of XRPro, LLC nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#import "../../StructureKit/Metal/STKMetalData.h"