Merge branch 'feature/usb_stream_support_h264_format' into 'master'

feat: usb_stream support frame based

Closes AEG-1079

See merge request ae_group/esp-iot-solution!918

.gitlab/ci/build.yml

@@ -457,6 +457,7 @@ build_example_usb_host_usb_camera_lcd_display:
     - .rules:build:example_usb_host_usb_camera_lcd_display
   parallel:
     matrix:
+      - IMAGE: espressif/idf:release-v5.1
       - IMAGE: espressif/idf:release-v5.1
         EXAMPLE_CONFIG: "sdkconfig.psram_octal_120m=default"
   variables:

components/usb/usb_stream/CHANGELOG.md

@@ -1,5 +1,9 @@
 # ChangeLog
 
+## v1.3.0 - 2023-12-14
+
+* Support UVC frame type: frame based, which can be used to transmit H264 streams.
+
 ## v1.2.1 - 2023-11-27
 
 ### Bug Fixes:

components/usb/usb_stream/descriptor.c

@@ -10,6 +10,7 @@
 #include <inttypes.h>
 
 #include "esp_log.h"
+#include "libuvc_def.h"
 #include "usb/usb_host.h"
 #include "usb/usb_types_ch9.h"
 #include "usb_stream_descriptor.h"
@@ -90,12 +91,106 @@ void print_uvc_header_desc(const uint8_t *buff, uint8_t sub_class)
 #endif
 }
 
-void parse_vs_format_mjpeg_desc(const uint8_t *buff, uint8_t *format_idx, uint8_t *frame_num)
+struct format_table_entry {
+    enum uvc_frame_format format;
+    uint8_t abstract_fmt;
+    uint8_t guid[16];
+    int children_count;
+    enum uvc_frame_format *children;
+};
+
+struct format_table_entry *_get_format_entry(enum uvc_frame_format format)
+{
+#define ABS_FMT(_fmt, _num, ...) \
+        case _fmt: { \
+        static enum uvc_frame_format _fmt##_children[] = __VA_ARGS__; \
+        static struct format_table_entry _fmt##_entry = { \
+            _fmt, 0, {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}, _num, _fmt##_children }; \
+        return &_fmt##_entry; }
+
+#define FMT(_fmt, ...) \
+        case _fmt: { \
+        static struct format_table_entry _fmt##_entry = { \
+            _fmt, 0, __VA_ARGS__, 0, NULL }; \
+        return &_fmt##_entry; }
+
+    switch (format) {
+        /* Define new formats here */
+        ABS_FMT(UVC_FRAME_FORMAT_ANY, 2,
+        {UVC_FRAME_FORMAT_UNCOMPRESSED, UVC_FRAME_FORMAT_COMPRESSED})
+
+        ABS_FMT(UVC_FRAME_FORMAT_UNCOMPRESSED, 8, {
+            UVC_FRAME_FORMAT_YUYV, UVC_FRAME_FORMAT_UYVY, UVC_FRAME_FORMAT_GRAY8,
+            UVC_FRAME_FORMAT_GRAY16, UVC_FRAME_FORMAT_NV12, UVC_FRAME_FORMAT_P010,
+            UVC_FRAME_FORMAT_BGR, UVC_FRAME_FORMAT_RGB
+        })
+        FMT(UVC_FRAME_FORMAT_YUYV,
+        {'Y',  'U',  'Y',  '2', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71})
+        FMT(UVC_FRAME_FORMAT_UYVY,
+        {'U',  'Y',  'V',  'Y', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71})
+        FMT(UVC_FRAME_FORMAT_GRAY8,
+        {'Y',  '8',  '0',  '0', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71})
+        FMT(UVC_FRAME_FORMAT_GRAY16,
+        {'Y',  '1',  '6',  ' ', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71})
+        FMT(UVC_FRAME_FORMAT_NV12,
+        {'N',  'V',  '1',  '2', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71})
+        FMT(UVC_FRAME_FORMAT_P010,
+        {'P',  '0',  '1',  '0', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71})
+        FMT(UVC_FRAME_FORMAT_BGR,
+        {0x7d, 0xeb, 0x36, 0xe4, 0x4f, 0x52, 0xce, 0x11, 0x9f, 0x53, 0x00, 0x20, 0xaf, 0x0b, 0xa7, 0x70})
+        FMT(UVC_FRAME_FORMAT_RGB,
+        {0x7e, 0xeb, 0x36, 0xe4, 0x4f, 0x52, 0xce, 0x11, 0x9f, 0x53, 0x00, 0x20, 0xaf, 0x0b, 0xa7, 0x70})
+        FMT(UVC_FRAME_FORMAT_BY8,
+        {'B',  'Y',  '8',  ' ', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71})
+        FMT(UVC_FRAME_FORMAT_BA81,
+        {'B',  'A',  '8',  '1', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71})
+        FMT(UVC_FRAME_FORMAT_SGRBG8,
+        {'G',  'R',  'B',  'G', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71})
+        FMT(UVC_FRAME_FORMAT_SGBRG8,
+        {'G',  'B',  'R',  'G', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71})
+        FMT(UVC_FRAME_FORMAT_SRGGB8,
+        {'R',  'G',  'G',  'B', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71})
+        FMT(UVC_FRAME_FORMAT_SBGGR8,
+        {'B',  'G',  'G',  'R', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71})
+        ABS_FMT(UVC_FRAME_FORMAT_COMPRESSED, 2,
+        {UVC_FRAME_FORMAT_MJPEG, UVC_FRAME_FORMAT_H264})
+        FMT(UVC_FRAME_FORMAT_MJPEG,
+        {'M',  'J',  'P',  'G'})
+        FMT(UVC_FRAME_FORMAT_H264,
+        {'H',  '2',  '6',  '4', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71})
+
+    default:
+        return NULL;
+    }
+
+#undef ABS_FMT
+#undef FMT
+}
+
+static enum uvc_frame_format uvc_frame_format_for_guid(const uint8_t guid[16])
+{
+    struct format_table_entry *format;
+    enum uvc_frame_format fmt;
+
+    for (fmt = 0; fmt < UVC_FRAME_FORMAT_COUNT; ++fmt) {
+        format = _get_format_entry(fmt);
+        if (!format || format->abstract_fmt) {
+            continue;
+        }
+        if (!memcmp(format->guid, guid, 16)) {
+            return format->format;
+        }
+    }
+
+    return UVC_FRAME_FORMAT_UNKNOWN;
+}
+
+void parse_vs_format_mjpeg_desc(const uint8_t *buff, uint8_t *format_idx, uint8_t *frame_num, enum uvc_frame_format *fmt)
 {
     if (buff == NULL) {
         return;
     }
-    const vs_format_desc_t *desc = (const vs_format_desc_t *) buff;
+    const vs_format_mjpeg_desc_t *desc = (const vs_format_mjpeg_desc_t *) buff;
 #ifdef CONFIG_UVC_PRINT_DESC
     printf("\t*** VS Format MJPEG Descriptor ***\n");
 #ifdef CONFIG_UVC_PRINT_DESC_VERBOSE
@@ -122,14 +217,17 @@ void parse_vs_format_mjpeg_desc(const uint8_t *buff, uint8_t *format_idx, uint8_
     if (frame_num) {
         *frame_num = desc->bNumFrameDescriptors;
     }
+    if (fmt) {
+        *fmt = UVC_FRAME_FORMAT_MJPEG;
+    }
 }
 
 void parse_vs_frame_mjpeg_desc(const uint8_t *buff, uint8_t *frame_idx, uint16_t *width, uint16_t *heigh, uint8_t *interval_type, const uint32_t **pp_interval, uint32_t *dflt_interval)
 {
     if (buff == NULL) {
         return;
     }
-    const vs_frame_desc_t *desc = (const vs_frame_desc_t *) buff;
+    const vs_frame_mjpeg_desc_t *desc = (const vs_frame_mjpeg_desc_t *) buff;
 #ifdef CONFIG_UVC_PRINT_DESC
     printf("\t*** VS MJPEG Frame Descriptor ***\n");
 #ifdef CONFIG_UVC_PRINT_DESC_VERBOSE
@@ -159,6 +257,104 @@ void parse_vs_frame_mjpeg_desc(const uint8_t *buff, uint8_t *frame_idx, uint16_t
     } else {
         // Discrete Frame Intervals
         size_t num_of_intervals = (desc->bLength - 26) / 4;
+        assert(num_of_intervals == desc->bFrameIntervalType);  // num_of_intervals should same as bFrameIntervalType
+        uint32_t *interval = (uint32_t *)&desc->dwFrameInterval;
+        for (int i = 0; i < num_of_intervals; ++i) {
+            printf("\tFrameInterval[%d] %"PRIu32"\n", i, interval[i]);
+        }
+    }
+#endif
+    if (width) {
+        *width = desc->wWidth;
+    }
+    if (heigh) {
+        *heigh = desc->wHeigh;
+    }
+    if (frame_idx) {
+        *frame_idx = desc->bFrameIndex;
+    }
+    if (interval_type) {
+        *interval_type = desc->bFrameIntervalType;
+    }
+    if (pp_interval) {
+        *pp_interval = &(desc->dwFrameInterval);
+    }
+    if (dflt_interval) {
+        *dflt_interval = desc->dwDefaultFrameInterval;
+    }
+}
+
+void parse_vs_format_frame_based_desc(const uint8_t *buff, uint8_t *format_idx, uint8_t *frame_num, enum uvc_frame_format *fmt)
+{
+    if (buff == NULL) {
+        return;
+    }
+    const vs_format_frame_based_desc_t *desc = (const vs_format_frame_based_desc_t *) buff;
+#ifdef CONFIG_UVC_PRINT_DESC
+    printf("\t*** VS Format Frame-Based Descriptor ***\n");
+#ifdef CONFIG_UVC_PRINT_DESC_VERBOSE
+    printf("\tbLength 0x%x\n", desc->bLength);
+    printf("\tbDescriptorType 0x%x\n", desc->bDescriptorType);
+    printf("\tbDescriptorSubType 0x%x\n", desc->bDescriptorSubType);
+#endif
+    printf("\tbFormatIndex 0x%x\n", desc->bFormatIndex);
+    printf("\tbNumFrameDescriptors %u\n", desc->bNumFrameDescriptors);
+    printf("\tguidFormat %.*s\n", 16, desc->guidFormat);
+    printf("\tbDefaultFrameIndex %u\n", desc->bDefaultFrameIndex);
+#ifdef CONFIG_UVC_PRINT_DESC_VERBOSE
+    printf("\tbAspectRatioX %u\n", desc->bAspectRatioX);
+    printf("\tbAspectRatioY %u\n", desc->bAspectRatioY);
+    printf("\tbmInterlaceFlags 0x%x\n", desc->bmInterlaceFlags);
+    printf("\tbCopyProtect %u\n", desc->bCopyProtect);
+#endif
+#endif
+    if (format_idx) {
+        *format_idx = desc->bFormatIndex;
+    }
+    if (frame_num) {
+        *frame_num = desc->bNumFrameDescriptors;
+    }
+    if (fmt) {
+        *fmt = uvc_frame_format_for_guid(desc->guidFormat);
+    }
+}
+
+void parse_vs_frame_frame_based_desc(const uint8_t *buff, uint8_t *frame_idx, uint16_t *width, uint16_t *heigh, uint8_t *interval_type, const uint32_t **pp_interval, uint32_t *dflt_interval)
+{
+    if (buff == NULL) {
+        return;
+    }
+    const vs_frame_frame_based_desc_t *desc = (const vs_frame_frame_based_desc_t *) buff;
+#ifdef CONFIG_UVC_PRINT_DESC
+    printf("\t*** VS Frame-Based Frame Descriptor ***\n");
+#ifdef CONFIG_UVC_PRINT_DESC_VERBOSE
+    printf("\tbLength 0x%x\n", desc->bLength);
+    printf("\tbDescriptorType 0x%x\n", desc->bDescriptorType);
+    printf("\tbDescriptorSubType 0x%x\n", desc->bDescriptorSubType);
+#endif
+    printf("\tbFrameIndex 0x%x\n", desc->bFrameIndex);
+#ifdef CONFIG_UVC_PRINT_DESC_VERBOSE
+    printf("\tbmCapabilities 0x%x\n", desc->bmCapabilities);
+#endif
+    printf("\twWidth %u\n", desc->wWidth);
+    printf("\twHeigh %u\n", desc->wHeigh);
+#ifdef CONFIG_UVC_PRINT_DESC_VERBOSE
+    printf("\tdwMinBitRate %"PRIu32"\n", desc->dwMinBitRate);
+    printf("\tdwMaxBitRate %"PRIu32"\n", desc->dwMaxBitRate);
+    printf("\tdwDefaultFrameInterval %"PRIu32"\n", desc->dwDefaultFrameInterval);
+    printf("\tbFrameIntervalType %u\n", desc->bFrameIntervalType);
+    printf("\tdwBytesPerLine %"PRIu32"\n", desc->dwBytesPerLine);
+#endif
+
+    if (desc->bFrameIntervalType == 0) {
+        // Continuous Frame Intervals
+        printf("\tdwMinFrameInterval %"PRIu32"\n",  desc->dwMinFrameInterval);
+        printf("\tdwMaxFrameInterval %"PRIu32"\n",  desc->dwMaxFrameInterval);
+        printf("\tdwFrameIntervalStep %"PRIu32"\n", desc->dwFrameIntervalStep);
+    } else {
+        // Discrete Frame Intervals
+        size_t num_of_intervals = (desc->bLength - 26) / 4;
+        assert(num_of_intervals == desc->bFrameIntervalType);  // num_of_intervals should same as bFrameIntervalType
         uint32_t *interval = (uint32_t *)&desc->dwFrameInterval;
         for (int i = 0; i < num_of_intervals; ++i) {
             printf("\tFrameInterval[%d] %"PRIu32"\n", i, interval[i]);

components/usb/usb_stream/idf_component.yml

@@ -1,4 +1,4 @@
-version: "1.2.1"
+version: "1.3.0"
 targets:
   - esp32s2
   - esp32s3

components/usb/usb_stream/include/libuvc_def.h

@@ -87,6 +87,8 @@ enum uvc_frame_format {
     UVC_FRAME_FORMAT_SBGGR8,
     /** YUV420: NV12 */
     UVC_FRAME_FORMAT_NV12,
+    /** YUV: P010 */
+    UVC_FRAME_FORMAT_P010,
     /** Number of formats understood */
     UVC_FRAME_FORMAT_COUNT,
 };

components/usb/usb_stream/include/usb_stream.h

@@ -42,6 +42,17 @@ typedef enum {
     UVC_XFER_UNKNOWN,   /*!< Unknown Mode */
 } uvc_xfer_t;
 
+/**
+ * @brief UVC stream format type, default using MJPEG format,
+ */
+/** @cond **/
+typedef enum {
+    UVC_FORMAT_MJPEG = 0,    /*!< Default MJPEG format */
+    UVC_FORMAT_FRAME_BASED,  /*!< Frame-based format */
+    UVC_FORMAT_MAX,         /*!< Unknown format */
+} uvc_format_t;
+/** @endcond **/
+
 /**
  * @brief Stream id, used for control
  *
@@ -90,10 +101,11 @@ typedef struct uvc_config {
     uint8_t *frame_buffer;          /*!< Buffer for one frame */
     uvc_frame_callback_t frame_cb;  /*!< callback function to handle incoming frame */
     void *frame_cb_arg;             /*!< callback function arg */
+    uvc_format_t format;            /*!< (optional) UVC stream format, default using MJPEG */
     /*!< Optional configs, Users need to specify parameters manually when they want to
     skip the get and process descriptors steps (used to speed up startup)*/
     uvc_xfer_t xfer_type;           /*!< (optional) UVC stream transfer type, UVC_XFER_ISOC or UVC_XFER_BULK */
-    uint8_t format_index;           /*!< (optional) Format index of MJPEG */
+    uint8_t format_index;           /*!< (optional) Format index */
     uint8_t frame_index;            /*!< (optional) Frame index, to choose resolution */
     uint16_t interface;             /*!< (optional) UVC stream interface number */
     uint16_t interface_alt;         /*!< (optional) UVC stream alternate interface, to choose MPS (Max Packet Size), bulk fix to 0*/

components/usb/usb_stream/private_include/usb_stream_descriptor.h

@@ -8,6 +8,7 @@
 
 #include <stdio.h>
 #include <string.h>
+#include "libuvc_def.h"
 #include "usb/usb_types_ch9.h"
 #include "usb/usb_types_stack.h"
 
@@ -213,7 +214,7 @@ typedef struct {
     uint8_t  bAspectRatioY;
     uint8_t  bmInterlaceFlags;
     uint8_t  bCopyProtect;
-} USB_DESC_ATTR vs_format_desc_t;
+} USB_DESC_ATTR vs_format_mjpeg_desc_t;
 
 typedef struct {
     uint8_t  bLength;
@@ -236,7 +237,46 @@ typedef struct {
             uint32_t dwFrameIntervalStep;
         };
     };
-} USB_DESC_ATTR vs_frame_desc_t;
+} USB_DESC_ATTR vs_frame_mjpeg_desc_t;
+
+typedef struct {
+    uint8_t  bLength;
+    uint8_t  bDescriptorType;
+    uint8_t  bDescriptorSubType;
+    uint8_t  bFormatIndex;
+    uint8_t  bNumFrameDescriptors;
+    uint8_t  guidFormat[16];
+    uint8_t  bBitsPerPixel;
+    uint8_t  bDefaultFrameIndex;
+    uint8_t  bAspectRatioX;
+    uint8_t  bAspectRatioY;
+    uint8_t  bmInterlaceFlags;
+    uint8_t  bCopyProtect;
+    uint8_t  bVariableSize;
+} USB_DESC_ATTR vs_format_frame_based_desc_t;
+
+typedef struct {
+    uint8_t  bLength;
+    uint8_t  bDescriptorType;
+    uint8_t  bDescriptorSubType;
+    uint8_t  bFrameIndex;
+    uint8_t  bmCapabilities;
+    uint16_t wWidth;
+    uint16_t wHeigh;
+    uint32_t dwMinBitRate;
+    uint32_t dwMaxBitRate;
+    uint32_t dwDefaultFrameInterval;
+    uint8_t  bFrameIntervalType;
+    uint32_t dwBytesPerLine;
+    union {
+        uint32_t dwFrameInterval;
+        struct {
+            uint32_t dwMinFrameInterval;
+            uint32_t dwMaxFrameInterval;
+            uint32_t dwFrameIntervalStep;
+        };
+    };
+} USB_DESC_ATTR vs_frame_frame_based_desc_t;
 
 typedef struct {
     uint8_t bLength;
@@ -385,8 +425,10 @@ void print_cfg_desc(const uint8_t *buff);
 void print_assoc_desc(const uint8_t *buff);
 void print_intf_desc(const uint8_t *buff);
 void parse_ep_desc(const uint8_t *buff, uint16_t *ep_mps, uint8_t *ep_addr, uint8_t *ep_attr);
-void parse_vs_format_mjpeg_desc(const uint8_t *buff, uint8_t *format_idx, uint8_t *frame_num);
+void parse_vs_format_mjpeg_desc(const uint8_t *buff, uint8_t *format_idx, uint8_t *frame_num, enum uvc_frame_format *fmt);
 void parse_vs_frame_mjpeg_desc(const uint8_t *buff, uint8_t *frame_idx, uint16_t *width, uint16_t *heigh, uint8_t *interval_type, const uint32_t **pp_interval, uint32_t *dflt_interval);
+void parse_vs_format_frame_based_desc(const uint8_t *buff, uint8_t *format_idx, uint8_t *frame_num, enum uvc_frame_format *fmt);
+void parse_vs_frame_frame_based_desc(const uint8_t *buff, uint8_t *frame_idx, uint16_t *width, uint16_t *heigh, uint8_t *interval_type, const uint32_t **pp_interval, uint32_t *dflt_interval);
 void print_uvc_header_desc(const uint8_t *buff, uint8_t sub_class);
 void print_device_descriptor(const uint8_t *buff);
 void print_ep_desc(const uint8_t *buff);