Camera node holistic record & replay

cmake/Depthai/DepthaiDeviceRVC4Config.cmake

@@ -4,4 +4,4 @@ set(DEPTHAI_DEVICE_RVC4_MATURITY "snapshot")
 
 # "version if applicable"
 # set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+93f7b75a885aa32f44c5e9f53b74470c49d2b1af")
-set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+b5ae8d5ca8e08345c88873fba39e572bdde16175")
+set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+93adc1673067a84ce4a91544e6c975895e6d32ba")

cmake/Depthai/DepthaiDeviceSideConfig.cmake

@@ -2,7 +2,7 @@
 set(DEPTHAI_DEVICE_SIDE_MATURITY "snapshot")
 
 # "full commit hash of device side binary"
-set(DEPTHAI_DEVICE_SIDE_COMMIT "c26ee202de4561360d7b08ce2966a506d08b8c89")
+set(DEPTHAI_DEVICE_SIDE_COMMIT "c18db93d519c0cf57e52205509492db7b45490d1")
 
 # "version if applicable"
 set(DEPTHAI_DEVICE_SIDE_VERSION "")

examples/cpp/CMakeLists.txt

@@ -477,6 +477,7 @@ dai_add_example(camera_multiple_outputs RVC4/Camera/camera_multiple_outputs.cpp
 
 # Host nodes
 dai_add_example(rgb_video RVC2/ColorCamera/rgb_video.cpp ON OFF)
+dai_add_example(rgb_video2 RVC2/Camera/rgb_video2.cpp ON OFF)
 dai_add_example(host_node HostNodes/host_node.cpp ON OFF)
 dai_add_example(threaded_host_node HostNodes/threaded_host_node.cpp ON OFF)
 

examples/cpp/RVC2/Camera/rgb_video2.cpp

@@ -0,0 +1,31 @@
+// Includes common necessary includes for development using depthai library
+#include "depthai/common/CameraBoardSocket.hpp"
+#include "depthai/depthai.hpp"
+#include "depthai/pipeline/datatype/ImgFrame.hpp"
+#include "depthai/pipeline/node/host/Replay.hpp"
+
+int main(int argc, char** argv) {
+    // Create pipeline
+    auto device = argc > 1 ? std::make_shared<dai::Device>(argv[1]) : std::make_shared<dai::Device>();
+    dai::Pipeline pipeline(device);
+
+    // Define source and output
+    auto camRgb = pipeline.create<dai::node::Camera>()->build(dai::CameraBoardSocket::CAM_A);
+
+    auto outputQueue = camRgb->requestOutput({1280, 720}, dai::ImgFrame::Type::NV12)->createOutputQueue();
+
+    pipeline.start();
+    while(pipeline.isRunning()) {
+        auto videoIn = outputQueue->get<dai::ImgFrame>();
+
+        // Get BGR frame from NV12 encoded video frame to show with opencv
+        // Visualizing the frame on slower hosts might have overhead
+        cv::imshow("video", videoIn->getCvFrame());
+
+        int key = cv::waitKey(1);
+        if(key == 'q' || key == 'Q') {
+            pipeline.stop();
+        }
+    }
+    return 0;
+}

include/depthai/pipeline/Node.hpp

@@ -311,12 +311,16 @@ class Node : public std::enable_shared_from_this<Node> {
     };
 
     class Input : public MessageQueue {
+        friend class Output;
+        friend class OutputMap;
        public:
         enum class Type { SReceiver, MReceiver };  // TODO(Morato) - refactor, make the MReceiver a separate class (shouldn't inherit from MessageQueue)
 
+       protected:
+        std::vector<Output*> connectedOutputs;
+
        private:
         std::reference_wrapper<Node> parent;
-        std::vector<Output*> connectedOutputs;
         // Options - more information about the input
         bool waitForMessage{false};
         std::string group;
@@ -393,6 +397,11 @@ class Node : public std::enable_shared_from_this<Node> {
          */
         std::string getGroup() const;
 
+        /**
+         * Check if this input is connected
+         */
+        bool isConnected() const;
+
         /** Default value for the blocking argument in the createInputQueue method */
         static constexpr bool INPUT_QUEUE_DEFAULT_BLOCKING = false;
 

include/depthai/pipeline/node/Camera.hpp

@@ -60,6 +60,12 @@ class Camera : public DeviceNodeCRTP<DeviceNode, Camera, CameraProperties>, publ
     Input inputControl{
         *this, {"inputControl", DEFAULT_GROUP, DEFAULT_BLOCKING, DEFAULT_QUEUE_SIZE, {{{DatatypeEnum::CameraControl, false}}}, DEFAULT_WAIT_FOR_MESSAGE}};
 
+    /**
+     * Input for mocking 'isp' functionality on RVC2.
+     * Default queue is blocking with size 8
+     */
+    Input mockIsp{*this, {"mockIsp", DEFAULT_GROUP, true, 8, {{{DatatypeEnum::ImgFrame, false}}}, DEFAULT_WAIT_FOR_MESSAGE}};
+
     /**
      * Retrieves which board socket to use
      * @returns Board socket to use
@@ -90,10 +96,14 @@ class Camera : public DeviceNodeCRTP<DeviceNode, Camera, CameraProperties>, publ
 
     void buildStage1() override;
 
+    std::pair<size_t, size_t> getMaxRequestedSize() const;
+    float getMaxRequestedFps() const;
+
    protected:
     Properties& getProperties() override;
     bool isSourceNode() const override;
     NodeRecordParams getNodeRecordParams() const override;
+    Input& getReplayInput() override;
 
    private:
     bool isBuilt = false;

include/depthai/properties/CameraProperties.hpp

@@ -118,6 +118,11 @@ struct CameraProperties : PropertiesSerializable<Properties, CameraProperties> {
      */
     int32_t mockIspHeight = AUTO;
 
+    /**
+     * Select the mock isp fps. Overrides fps if mockIsp is connected.
+     */
+    float mockIspFps = AUTO;
+
     /**
      * Camera sensor FPS
      */

include/depthai/utility/RecordReplay.hpp

@@ -31,9 +31,9 @@ struct RecordConfig {
     enum class CompressionLevel : uint8_t { NONE, FASTEST, FAST, DEFAULT, SLOW, SLOWEST };
 
     struct VideoEncoding {
-        bool enabled = true;
+        bool enabled = false;
         int bitrate = 0;
-        Profile profile = Profile::H264_MAIN;
+        Profile profile = Profile::MJPEG;
         bool lossless = false;
         int quality = 80;
     };

src/opencv/HolisticRecordReplay.cpp

@@ -19,7 +19,8 @@
 namespace dai {
 namespace utility {
 
-bool setupHolisticRecord(Pipeline& pipeline, const std::string& mxId, RecordConfig& recordConfig, std::unordered_map<std::string, std::string>& outFilenames) {
+bool setupHolisticRecord(
+    Pipeline& pipeline, const std::string& mxId, RecordConfig& recordConfig, std::unordered_map<std::string, std::string>& outFilenames, bool legacy) {
     auto sources = pipeline.getSourceNodes();
     const auto recordPath = recordConfig.outputDir;
     try {
@@ -35,9 +36,15 @@ bool setupHolisticRecord(Pipeline& pipeline, const std::string& mxId, RecordConf
             outFilenames[nodeName] = filePath;
             if(std::dynamic_pointer_cast<node::Camera>(node) != nullptr || std::dynamic_pointer_cast<node::ColorCamera>(node) != nullptr
                || std::dynamic_pointer_cast<node::MonoCamera>(node) != nullptr) {
+                Node::Output* output;
                 if(std::dynamic_pointer_cast<node::Camera>(node) != nullptr) {
-                    // TODO(asahtik)
-                    throw std::runtime_error("Holistic record with Camera node is not yet supported.");
+                    auto cam = std::dynamic_pointer_cast<dai::node::Camera>(node);
+                    auto fps = cam->getMaxRequestedFps();
+                    const auto [width, height] = cam->getMaxRequestedSize();
+                    // TODO(asahtik): RVC4 H264 video encoder only supports native resolutions. Get min larger native resolution (also removes upscaling).
+                    output = cam->requestOutput({width, height}, dai::ImgFrame::Type::NV12, dai::ImgResizeMode::CROP, fps);
+                } else {
+                    output = &nodeS->getRecordOutput();
                 }
                 auto recordNode = pipeline.create<dai::node::RecordVideo>();
                 recordNode->setRecordMetadataFile(filePath + ".mcap");
@@ -49,8 +56,8 @@ bool setupHolisticRecord(Pipeline& pipeline, const std::string& mxId, RecordConf
                     videnc->setLossless(recordConfig.videoEncoding.lossless);
                     videnc->setBitrate(recordConfig.videoEncoding.bitrate);
                     videnc->setQuality(recordConfig.videoEncoding.quality);
-                    int maxOutputFrameSize = 3110400;
-                    if(std::dynamic_pointer_cast<node::Camera>(node) != nullptr || std::dynamic_pointer_cast<node::ColorCamera>(node) != nullptr) {
+                    if((std::dynamic_pointer_cast<node::Camera>(node) != nullptr || std::dynamic_pointer_cast<node::ColorCamera>(node) != nullptr) && legacy) {
+                        int maxOutputFrameSize = 3110400;
                         if(std::dynamic_pointer_cast<node::ColorCamera>(node) != nullptr) {
                             auto cam = std::dynamic_pointer_cast<dai::node::ColorCamera>(node);
                             maxOutputFrameSize = std::get<0>(cam->getIspSize()) * std::get<1>(cam->getIspSize()) * 3;
@@ -59,14 +66,15 @@ bool setupHolisticRecord(Pipeline& pipeline, const std::string& mxId, RecordConf
                         imageManip->initialConfig.setFrameType(ImgFrame::Type::NV12);
                         imageManip->setMaxOutputFrameSize(maxOutputFrameSize);
 
-                        nodeS->getRecordOutput().link(imageManip->inputImage);
+                        output->link(imageManip->inputImage);
                         imageManip->out.link(videnc->input);
                     } else {
-                        nodeS->getRecordOutput().link(videnc->input);
+                        // TODO(asahtik): RVC4 video encoder only supports native resolutions.
+                        output->link(videnc->input);
                     }
                     videnc->out.link(recordNode->input);
                 } else {
-                    nodeS->getRecordOutput().link(recordNode->input);
+                    output->link(recordNode->input);
                 }
             } else {
                 auto recordNode = pipeline.create<dai::node::RecordMetadataOnly>();
@@ -97,7 +105,8 @@ bool setupHolisticReplay(Pipeline& pipeline,
                          std::string replayPath,
                          const std::string& mxId,
                          RecordConfig& recordConfig,
-                         std::unordered_map<std::string, std::string>& outFilenames) {
+                         std::unordered_map<std::string, std::string>& outFilenames,
+                         bool legacy) {
     const std::string rootPath = platform::getDirFromPath(replayPath);
     auto sources = pipeline.getSourceNodes();
     try {
@@ -207,15 +216,15 @@ bool setupHolisticReplay(Pipeline& pipeline,
                 replay->setReplayMetadataFile(platform::joinPaths(rootPath, nodeName + ".mcap"));
                 // replay->setReplayVideo(platform::joinPaths(rootPath, (mxId + "_").append(nodeName).append(".mp4")));
                 replay->setReplayVideoFile(platform::joinPaths(rootPath, nodeName + ".mp4"));
-                replay->setOutFrameType(ImgFrame::Type::YUV420p);
+                replay->setOutFrameType(legacy ? ImgFrame::Type::YUV420p : ImgFrame::Type::NV12);
 
                 auto videoSize = BytePlayer::getVideoSize(replay->getReplayMetadataFile().string());
                 if(videoSize.has_value()) {
                     auto [width, height] = videoSize.value();
                     if(std::dynamic_pointer_cast<node::Camera>(node) != nullptr) {
-                        // TODO(asahtik)
-                        /*auto cam = std::dynamic_pointer_cast<dai::node::Camera>(node);*/
-                        /*cam->setMockIspSize(width, height);*/
+                        auto cam = std::dynamic_pointer_cast<dai::node::Camera>(node);
+                        cam->properties.mockIspWidth = width;
+                        cam->properties.mockIspHeight = height;
                     } else if(std::dynamic_pointer_cast<node::ColorCamera>(node) != nullptr) {
                         auto cam = std::dynamic_pointer_cast<dai::node::ColorCamera>(node);
                         cam->setMockIspSize(width, height);

src/pipeline/Node.cpp

@@ -80,6 +80,10 @@ std::string Node::Input::getGroup() const {
     return group;
 }
 
+bool Node::Input::isConnected() const {
+    return !connectedOutputs.empty();
+}
+
 std::vector<Node::ConnectionInternal> Node::Output::getConnections() {
     std::vector<Node::ConnectionInternal> myConnections;
     for(const auto& conn : parent.get().connections) {
@@ -153,6 +157,7 @@ void Node::Output::link(Input& in) {
     parent.get().connections.insert(connection);
     // Add the shared_ptr to the input directly for host side
     connectedInputs.push_back(&in);
+    in.connectedOutputs.push_back(this);
 }
 
 std::shared_ptr<dai::MessageQueue> Node::Output::createOutputQueue(unsigned int maxSize, bool blocking) {
@@ -211,6 +216,7 @@ void Node::Output::unlink(Input& in) {
 
     // Remove the shared_ptr to the input directly for host side
     connectedInputs.erase(std::remove(connectedInputs.begin(), connectedInputs.end(), &in), connectedInputs.end());
+    in.connectedOutputs.erase(std::remove(in.connectedOutputs.begin(), in.connectedOutputs.end(), this), in.connectedOutputs.end());
 }
 
 void Node::Output::send(const std::shared_ptr<ADatatype>& msg) {

src/pipeline/Pipeline.cpp

@@ -605,7 +605,8 @@ void PipelineImpl::build() {
         std::string replayPath = utility::getEnv("DEPTHAI_REPLAY");
 
         if(defaultDevice->getDeviceInfo().platform == XLinkPlatform_t::X_LINK_MYRIAD_2
-           || defaultDevice->getDeviceInfo().platform == XLinkPlatform_t::X_LINK_MYRIAD_X) {
+           || defaultDevice->getDeviceInfo().platform == XLinkPlatform_t::X_LINK_MYRIAD_X
+           || defaultDevice->getDeviceInfo().platform == XLinkPlatform_t::X_LINK_RVC4) {
             try {
 #ifdef DEPTHAI_MERGED_TARGET
                 if(enableHolisticRecordReplay) {
@@ -631,7 +632,12 @@ void PipelineImpl::build() {
                 } else if(!recordPath.empty()) {
                     if(enableHolisticRecordReplay || utility::checkRecordConfig(recordPath, recordConfig)) {
                         if(platform::checkWritePermissions(recordPath)) {
-                            if(utility::setupHolisticRecord(parent, defaultDeviceMxId, recordConfig, recordReplayFilenames)) {
+                            if(utility::setupHolisticRecord(parent,
+                                                            defaultDeviceMxId,
+                                                            recordConfig,
+                                                            recordReplayFilenames,
+                                                            defaultDevice->getDeviceInfo().platform == XLinkPlatform_t::X_LINK_MYRIAD_2
+                                                                || defaultDevice->getDeviceInfo().platform == XLinkPlatform_t::X_LINK_MYRIAD_X)) {
                                 recordConfig.state = RecordConfig::RecordReplayState::RECORD;
                                 Logging::getInstance().logger.info("Record enabled.");
                             } else {
@@ -646,7 +652,13 @@ void PipelineImpl::build() {
                 } else if(!replayPath.empty()) {
                     if(platform::checkPathExists(replayPath)) {
                         if(platform::checkWritePermissions(replayPath)) {
-                            if(utility::setupHolisticReplay(parent, replayPath, defaultDeviceMxId, recordConfig, recordReplayFilenames)) {
+                            if(utility::setupHolisticReplay(parent,
+                                                            replayPath,
+                                                            defaultDeviceMxId,
+                                                            recordConfig,
+                                                            recordReplayFilenames,
+                                                            defaultDevice->getDeviceInfo().platform == XLinkPlatform_t::X_LINK_MYRIAD_2
+                                                                || defaultDevice->getDeviceInfo().platform == XLinkPlatform_t::X_LINK_MYRIAD_X)) {
                                 recordConfig.state = RecordConfig::RecordReplayState::REPLAY;
                                 Logging::getInstance().logger.info("Replay enabled.");
                             } else {

src/pipeline/node/Camera.cpp

@@ -3,6 +3,7 @@
 #include <fstream>
 #include <memory>
 #include <stdexcept>
+#include <utility>
 
 // libraries
 #include <spimpl.h>
@@ -178,6 +179,46 @@ NodeRecordParams Camera::getNodeRecordParams() const {
     params.name = "Camera" + toString(properties.boardSocket);
     return params;
 }
+Camera::Input& Camera::getReplayInput() {
+    return mockIsp;
+}
+
+std::pair<size_t, size_t> Camera::getMaxRequestedSize() const {
+    uint32_t maxWidth = 0;
+    uint32_t maxHeight = 0;
+    for(const auto& outputRequest : pimpl->outputRequests) {
+        if(outputRequest.capability.size.value) {
+            if(const auto* size = std::get_if<std::pair<uint32_t, uint32_t>>(&(*outputRequest.capability.size.value))) {
+                maxWidth = std::max(maxWidth, size->first);
+                maxHeight = std::max(maxHeight, size->second);
+            } else {
+                throw std::runtime_error("Unsupported size value");
+            }
+        }
+    }
+    if(maxWidth == 0 || maxHeight == 0) {
+        throw std::runtime_error("Invalid max width or height");
+    }
+    return std::make_pair(maxWidth, maxHeight);
+};
+float Camera::getMaxRequestedFps() const {
+    float maxFps = 0;
+    for(const auto& outputRequest : pimpl->outputRequests) {
+        if(outputRequest.capability.fps.value) {
+            if(const auto* fps = std::get_if<float>(&(*outputRequest.capability.fps.value))) {
+                maxFps = std::max(maxFps, *fps);
+            } else if(const auto* fps = std::get_if<std::pair<float, float>>(&(*outputRequest.capability.fps.value))) {
+                maxFps = std::max(maxFps, std::get<1>(*fps));
+            } else if(const auto* fps = std::get_if<std::vector<float>>(&(*outputRequest.capability.fps.value))) {
+                DAI_CHECK(fps->size() > 0, "When passing a vector to ImgFrameCapability->fps, please pass a non empty vector!");
+                maxFps = std::max(maxFps, (*fps)[0]);
+            } else {
+                throw std::runtime_error("Unsupported fps value");
+            }
+        }
+    }
+    return maxFps == 0 ? 30 : maxFps;
+}
 
 /*
 Camera::Output& Camera::getRecordOutput() {

src/utility/HolisticRecordReplay.hpp

@@ -11,12 +11,14 @@ namespace dai {
 namespace utility {
 
 #ifdef DEPTHAI_HAVE_OPENCV_SUPPORT
-bool setupHolisticRecord(Pipeline& pipeline, const std::string& mxId, RecordConfig& recordConfig, std::unordered_map<std::string, std::string>& outFilenames);
+bool setupHolisticRecord(
+    Pipeline& pipeline, const std::string& mxId, RecordConfig& recordConfig, std::unordered_map<std::string, std::string>& outFilenames, bool legacy = false);
 bool setupHolisticReplay(Pipeline& pipeline,
                          std::string replayPath,
                          const std::string& mxId,
                          RecordConfig& recordConfig,
-                         std::unordered_map<std::string, std::string>& outFilenames);
+                         std::unordered_map<std::string, std::string>& outFilenames,
+                         bool legacy = false);
 #endif
 
 }  // namespace utility