diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index f7eb832d0..cad4494ef 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -457,6 +457,7 @@ build_example_usb_host_usb_camera_lcd_display: - .rules:build:example_usb_host_usb_camera_lcd_display parallel: matrix: + - IMAGE: espressif/idf:release-v5.1 - IMAGE: espressif/idf:release-v5.1 EXAMPLE_CONFIG: "sdkconfig.psram_octal_120m=default" variables: diff --git a/components/usb/usb_stream/CHANGELOG.md b/components/usb/usb_stream/CHANGELOG.md index 1f64c9fc4..ac78c7d27 100644 --- a/components/usb/usb_stream/CHANGELOG.md +++ b/components/usb/usb_stream/CHANGELOG.md @@ -1,5 +1,9 @@ # ChangeLog +## v1.3.0 - 2023-12-14 + +* Support UVC frame type: frame based, which can be used to transmit H264 streams. + ## v1.2.1 - 2023-11-27 ### Bug Fixes: diff --git a/components/usb/usb_stream/descriptor.c b/components/usb/usb_stream/descriptor.c index e84516538..50caf08ab 100644 --- a/components/usb/usb_stream/descriptor.c +++ b/components/usb/usb_stream/descriptor.c @@ -10,6 +10,7 @@ #include #include "esp_log.h" +#include "libuvc_def.h" #include "usb/usb_host.h" #include "usb/usb_types_ch9.h" #include "usb_stream_descriptor.h" @@ -90,12 +91,106 @@ void print_uvc_header_desc(const uint8_t *buff, uint8_t sub_class) #endif } -void parse_vs_format_mjpeg_desc(const uint8_t *buff, uint8_t *format_idx, uint8_t *frame_num) +struct format_table_entry { + enum uvc_frame_format format; + uint8_t abstract_fmt; + uint8_t guid[16]; + int children_count; + enum uvc_frame_format *children; +}; + +struct format_table_entry *_get_format_entry(enum uvc_frame_format format) +{ +#define ABS_FMT(_fmt, _num, ...) \ + case _fmt: { \ + static enum uvc_frame_format _fmt##_children[] = __VA_ARGS__; \ + static struct format_table_entry _fmt##_entry = { \ + _fmt, 0, {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}, _num, _fmt##_children }; \ + return &_fmt##_entry; } + +#define FMT(_fmt, ...) \ + case _fmt: { \ + static struct format_table_entry _fmt##_entry = { \ + _fmt, 0, __VA_ARGS__, 0, NULL }; \ + return &_fmt##_entry; } + + switch (format) { + /* Define new formats here */ + ABS_FMT(UVC_FRAME_FORMAT_ANY, 2, + {UVC_FRAME_FORMAT_UNCOMPRESSED, UVC_FRAME_FORMAT_COMPRESSED}) + + ABS_FMT(UVC_FRAME_FORMAT_UNCOMPRESSED, 8, { + UVC_FRAME_FORMAT_YUYV, UVC_FRAME_FORMAT_UYVY, UVC_FRAME_FORMAT_GRAY8, + UVC_FRAME_FORMAT_GRAY16, UVC_FRAME_FORMAT_NV12, UVC_FRAME_FORMAT_P010, + UVC_FRAME_FORMAT_BGR, UVC_FRAME_FORMAT_RGB + }) + FMT(UVC_FRAME_FORMAT_YUYV, + {'Y', 'U', 'Y', '2', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}) + FMT(UVC_FRAME_FORMAT_UYVY, + {'U', 'Y', 'V', 'Y', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}) + FMT(UVC_FRAME_FORMAT_GRAY8, + {'Y', '8', '0', '0', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}) + FMT(UVC_FRAME_FORMAT_GRAY16, + {'Y', '1', '6', ' ', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}) + FMT(UVC_FRAME_FORMAT_NV12, + {'N', 'V', '1', '2', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}) + FMT(UVC_FRAME_FORMAT_P010, + {'P', '0', '1', '0', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}) + FMT(UVC_FRAME_FORMAT_BGR, + {0x7d, 0xeb, 0x36, 0xe4, 0x4f, 0x52, 0xce, 0x11, 0x9f, 0x53, 0x00, 0x20, 0xaf, 0x0b, 0xa7, 0x70}) + FMT(UVC_FRAME_FORMAT_RGB, + {0x7e, 0xeb, 0x36, 0xe4, 0x4f, 0x52, 0xce, 0x11, 0x9f, 0x53, 0x00, 0x20, 0xaf, 0x0b, 0xa7, 0x70}) + FMT(UVC_FRAME_FORMAT_BY8, + {'B', 'Y', '8', ' ', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}) + FMT(UVC_FRAME_FORMAT_BA81, + {'B', 'A', '8', '1', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}) + FMT(UVC_FRAME_FORMAT_SGRBG8, + {'G', 'R', 'B', 'G', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}) + FMT(UVC_FRAME_FORMAT_SGBRG8, + {'G', 'B', 'R', 'G', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}) + FMT(UVC_FRAME_FORMAT_SRGGB8, + {'R', 'G', 'G', 'B', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}) + FMT(UVC_FRAME_FORMAT_SBGGR8, + {'B', 'G', 'G', 'R', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}) + ABS_FMT(UVC_FRAME_FORMAT_COMPRESSED, 2, + {UVC_FRAME_FORMAT_MJPEG, UVC_FRAME_FORMAT_H264}) + FMT(UVC_FRAME_FORMAT_MJPEG, + {'M', 'J', 'P', 'G'}) + FMT(UVC_FRAME_FORMAT_H264, + {'H', '2', '6', '4', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}) + + default: + return NULL; + } + +#undef ABS_FMT +#undef FMT +} + +static enum uvc_frame_format uvc_frame_format_for_guid(const uint8_t guid[16]) +{ + struct format_table_entry *format; + enum uvc_frame_format fmt; + + for (fmt = 0; fmt < UVC_FRAME_FORMAT_COUNT; ++fmt) { + format = _get_format_entry(fmt); + if (!format || format->abstract_fmt) { + continue; + } + if (!memcmp(format->guid, guid, 16)) { + return format->format; + } + } + + return UVC_FRAME_FORMAT_UNKNOWN; +} + +void parse_vs_format_mjpeg_desc(const uint8_t *buff, uint8_t *format_idx, uint8_t *frame_num, enum uvc_frame_format *fmt) { if (buff == NULL) { return; } - const vs_format_desc_t *desc = (const vs_format_desc_t *) buff; + const vs_format_mjpeg_desc_t *desc = (const vs_format_mjpeg_desc_t *) buff; #ifdef CONFIG_UVC_PRINT_DESC printf("\t*** VS Format MJPEG Descriptor ***\n"); #ifdef CONFIG_UVC_PRINT_DESC_VERBOSE @@ -122,6 +217,9 @@ void parse_vs_format_mjpeg_desc(const uint8_t *buff, uint8_t *format_idx, uint8_ if (frame_num) { *frame_num = desc->bNumFrameDescriptors; } + if (fmt) { + *fmt = UVC_FRAME_FORMAT_MJPEG; + } } void parse_vs_frame_mjpeg_desc(const uint8_t *buff, uint8_t *frame_idx, uint16_t *width, uint16_t *heigh, uint8_t *interval_type, const uint32_t **pp_interval, uint32_t *dflt_interval) @@ -129,7 +227,7 @@ void parse_vs_frame_mjpeg_desc(const uint8_t *buff, uint8_t *frame_idx, uint16_t if (buff == NULL) { return; } - const vs_frame_desc_t *desc = (const vs_frame_desc_t *) buff; + const vs_frame_mjpeg_desc_t *desc = (const vs_frame_mjpeg_desc_t *) buff; #ifdef CONFIG_UVC_PRINT_DESC printf("\t*** VS MJPEG Frame Descriptor ***\n"); #ifdef CONFIG_UVC_PRINT_DESC_VERBOSE @@ -159,6 +257,104 @@ void parse_vs_frame_mjpeg_desc(const uint8_t *buff, uint8_t *frame_idx, uint16_t } else { // Discrete Frame Intervals size_t num_of_intervals = (desc->bLength - 26) / 4; + assert(num_of_intervals == desc->bFrameIntervalType); // num_of_intervals should same as bFrameIntervalType + uint32_t *interval = (uint32_t *)&desc->dwFrameInterval; + for (int i = 0; i < num_of_intervals; ++i) { + printf("\tFrameInterval[%d] %"PRIu32"\n", i, interval[i]); + } + } +#endif + if (width) { + *width = desc->wWidth; + } + if (heigh) { + *heigh = desc->wHeigh; + } + if (frame_idx) { + *frame_idx = desc->bFrameIndex; + } + if (interval_type) { + *interval_type = desc->bFrameIntervalType; + } + if (pp_interval) { + *pp_interval = &(desc->dwFrameInterval); + } + if (dflt_interval) { + *dflt_interval = desc->dwDefaultFrameInterval; + } +} + +void parse_vs_format_frame_based_desc(const uint8_t *buff, uint8_t *format_idx, uint8_t *frame_num, enum uvc_frame_format *fmt) +{ + if (buff == NULL) { + return; + } + const vs_format_frame_based_desc_t *desc = (const vs_format_frame_based_desc_t *) buff; +#ifdef CONFIG_UVC_PRINT_DESC + printf("\t*** VS Format Frame-Based Descriptor ***\n"); +#ifdef CONFIG_UVC_PRINT_DESC_VERBOSE + printf("\tbLength 0x%x\n", desc->bLength); + printf("\tbDescriptorType 0x%x\n", desc->bDescriptorType); + printf("\tbDescriptorSubType 0x%x\n", desc->bDescriptorSubType); +#endif + printf("\tbFormatIndex 0x%x\n", desc->bFormatIndex); + printf("\tbNumFrameDescriptors %u\n", desc->bNumFrameDescriptors); + printf("\tguidFormat %.*s\n", 16, desc->guidFormat); + printf("\tbDefaultFrameIndex %u\n", desc->bDefaultFrameIndex); +#ifdef CONFIG_UVC_PRINT_DESC_VERBOSE + printf("\tbAspectRatioX %u\n", desc->bAspectRatioX); + printf("\tbAspectRatioY %u\n", desc->bAspectRatioY); + printf("\tbmInterlaceFlags 0x%x\n", desc->bmInterlaceFlags); + printf("\tbCopyProtect %u\n", desc->bCopyProtect); +#endif +#endif + if (format_idx) { + *format_idx = desc->bFormatIndex; + } + if (frame_num) { + *frame_num = desc->bNumFrameDescriptors; + } + if (fmt) { + *fmt = uvc_frame_format_for_guid(desc->guidFormat); + } +} + +void parse_vs_frame_frame_based_desc(const uint8_t *buff, uint8_t *frame_idx, uint16_t *width, uint16_t *heigh, uint8_t *interval_type, const uint32_t **pp_interval, uint32_t *dflt_interval) +{ + if (buff == NULL) { + return; + } + const vs_frame_frame_based_desc_t *desc = (const vs_frame_frame_based_desc_t *) buff; +#ifdef CONFIG_UVC_PRINT_DESC + printf("\t*** VS Frame-Based Frame Descriptor ***\n"); +#ifdef CONFIG_UVC_PRINT_DESC_VERBOSE + printf("\tbLength 0x%x\n", desc->bLength); + printf("\tbDescriptorType 0x%x\n", desc->bDescriptorType); + printf("\tbDescriptorSubType 0x%x\n", desc->bDescriptorSubType); +#endif + printf("\tbFrameIndex 0x%x\n", desc->bFrameIndex); +#ifdef CONFIG_UVC_PRINT_DESC_VERBOSE + printf("\tbmCapabilities 0x%x\n", desc->bmCapabilities); +#endif + printf("\twWidth %u\n", desc->wWidth); + printf("\twHeigh %u\n", desc->wHeigh); +#ifdef CONFIG_UVC_PRINT_DESC_VERBOSE + printf("\tdwMinBitRate %"PRIu32"\n", desc->dwMinBitRate); + printf("\tdwMaxBitRate %"PRIu32"\n", desc->dwMaxBitRate); + printf("\tdwDefaultFrameInterval %"PRIu32"\n", desc->dwDefaultFrameInterval); + printf("\tbFrameIntervalType %u\n", desc->bFrameIntervalType); + printf("\tdwBytesPerLine %"PRIu32"\n", desc->dwBytesPerLine); +#endif + + if (desc->bFrameIntervalType == 0) { + // Continuous Frame Intervals + printf("\tdwMinFrameInterval %"PRIu32"\n", desc->dwMinFrameInterval); + printf("\tdwMaxFrameInterval %"PRIu32"\n", desc->dwMaxFrameInterval); + printf("\tdwFrameIntervalStep %"PRIu32"\n", desc->dwFrameIntervalStep); + } else { + // Discrete Frame Intervals + size_t num_of_intervals = (desc->bLength - 26) / 4; + assert(num_of_intervals == desc->bFrameIntervalType); // num_of_intervals should same as bFrameIntervalType uint32_t *interval = (uint32_t *)&desc->dwFrameInterval; for (int i = 0; i < num_of_intervals; ++i) { printf("\tFrameInterval[%d] %"PRIu32"\n", i, interval[i]); diff --git a/components/usb/usb_stream/idf_component.yml b/components/usb/usb_stream/idf_component.yml index bec8c00ac..2eca1dacd 100644 --- a/components/usb/usb_stream/idf_component.yml +++ b/components/usb/usb_stream/idf_component.yml @@ -1,4 +1,4 @@ -version: "1.2.1" +version: "1.3.0" targets: - esp32s2 - esp32s3 diff --git a/components/usb/usb_stream/include/libuvc_def.h b/components/usb/usb_stream/include/libuvc_def.h index 8efb93d72..ca9b56359 100644 --- a/components/usb/usb_stream/include/libuvc_def.h +++ b/components/usb/usb_stream/include/libuvc_def.h @@ -87,6 +87,8 @@ enum uvc_frame_format { UVC_FRAME_FORMAT_SBGGR8, /** YUV420: NV12 */ UVC_FRAME_FORMAT_NV12, + /** YUV: P010 */ + UVC_FRAME_FORMAT_P010, /** Number of formats understood */ UVC_FRAME_FORMAT_COUNT, }; diff --git a/components/usb/usb_stream/include/usb_stream.h b/components/usb/usb_stream/include/usb_stream.h index f5ef78222..8c02f7e50 100644 --- a/components/usb/usb_stream/include/usb_stream.h +++ b/components/usb/usb_stream/include/usb_stream.h @@ -42,6 +42,17 @@ typedef enum { UVC_XFER_UNKNOWN, /*!< Unknown Mode */ } uvc_xfer_t; +/** + * @brief UVC stream format type, default using MJPEG format, + */ +/** @cond **/ +typedef enum { + UVC_FORMAT_MJPEG = 0, /*!< Default MJPEG format */ + UVC_FORMAT_FRAME_BASED, /*!< Frame-based format */ + UVC_FORMAT_MAX, /*!< Unknown format */ +} uvc_format_t; +/** @endcond **/ + /** * @brief Stream id, used for control * @@ -90,10 +101,11 @@ typedef struct uvc_config { uint8_t *frame_buffer; /*!< Buffer for one frame */ uvc_frame_callback_t frame_cb; /*!< callback function to handle incoming frame */ void *frame_cb_arg; /*!< callback function arg */ + uvc_format_t format; /*!< (optional) UVC stream format, default using MJPEG */ /*!< Optional configs, Users need to specify parameters manually when they want to skip the get and process descriptors steps (used to speed up startup)*/ uvc_xfer_t xfer_type; /*!< (optional) UVC stream transfer type, UVC_XFER_ISOC or UVC_XFER_BULK */ - uint8_t format_index; /*!< (optional) Format index of MJPEG */ + uint8_t format_index; /*!< (optional) Format index */ uint8_t frame_index; /*!< (optional) Frame index, to choose resolution */ uint16_t interface; /*!< (optional) UVC stream interface number */ uint16_t interface_alt; /*!< (optional) UVC stream alternate interface, to choose MPS (Max Packet Size), bulk fix to 0*/ diff --git a/components/usb/usb_stream/private_include/usb_stream_descriptor.h b/components/usb/usb_stream/private_include/usb_stream_descriptor.h index a58a681de..edbe2c0a5 100644 --- a/components/usb/usb_stream/private_include/usb_stream_descriptor.h +++ b/components/usb/usb_stream/private_include/usb_stream_descriptor.h @@ -8,6 +8,7 @@ #include #include +#include "libuvc_def.h" #include "usb/usb_types_ch9.h" #include "usb/usb_types_stack.h" @@ -213,7 +214,7 @@ typedef struct { uint8_t bAspectRatioY; uint8_t bmInterlaceFlags; uint8_t bCopyProtect; -} USB_DESC_ATTR vs_format_desc_t; +} USB_DESC_ATTR vs_format_mjpeg_desc_t; typedef struct { uint8_t bLength; @@ -236,7 +237,46 @@ typedef struct { uint32_t dwFrameIntervalStep; }; }; -} USB_DESC_ATTR vs_frame_desc_t; +} USB_DESC_ATTR vs_frame_mjpeg_desc_t; + +typedef struct { + uint8_t bLength; + uint8_t bDescriptorType; + uint8_t bDescriptorSubType; + uint8_t bFormatIndex; + uint8_t bNumFrameDescriptors; + uint8_t guidFormat[16]; + uint8_t bBitsPerPixel; + uint8_t bDefaultFrameIndex; + uint8_t bAspectRatioX; + uint8_t bAspectRatioY; + uint8_t bmInterlaceFlags; + uint8_t bCopyProtect; + uint8_t bVariableSize; +} USB_DESC_ATTR vs_format_frame_based_desc_t; + +typedef struct { + uint8_t bLength; + uint8_t bDescriptorType; + uint8_t bDescriptorSubType; + uint8_t bFrameIndex; + uint8_t bmCapabilities; + uint16_t wWidth; + uint16_t wHeigh; + uint32_t dwMinBitRate; + uint32_t dwMaxBitRate; + uint32_t dwDefaultFrameInterval; + uint8_t bFrameIntervalType; + uint32_t dwBytesPerLine; + union { + uint32_t dwFrameInterval; + struct { + uint32_t dwMinFrameInterval; + uint32_t dwMaxFrameInterval; + uint32_t dwFrameIntervalStep; + }; + }; +} USB_DESC_ATTR vs_frame_frame_based_desc_t; typedef struct { uint8_t bLength; @@ -385,8 +425,10 @@ void print_cfg_desc(const uint8_t *buff); void print_assoc_desc(const uint8_t *buff); void print_intf_desc(const uint8_t *buff); void parse_ep_desc(const uint8_t *buff, uint16_t *ep_mps, uint8_t *ep_addr, uint8_t *ep_attr); -void parse_vs_format_mjpeg_desc(const uint8_t *buff, uint8_t *format_idx, uint8_t *frame_num); +void parse_vs_format_mjpeg_desc(const uint8_t *buff, uint8_t *format_idx, uint8_t *frame_num, enum uvc_frame_format *fmt); void parse_vs_frame_mjpeg_desc(const uint8_t *buff, uint8_t *frame_idx, uint16_t *width, uint16_t *heigh, uint8_t *interval_type, const uint32_t **pp_interval, uint32_t *dflt_interval); +void parse_vs_format_frame_based_desc(const uint8_t *buff, uint8_t *format_idx, uint8_t *frame_num, enum uvc_frame_format *fmt); +void parse_vs_frame_frame_based_desc(const uint8_t *buff, uint8_t *frame_idx, uint16_t *width, uint16_t *heigh, uint8_t *interval_type, const uint32_t **pp_interval, uint32_t *dflt_interval); void print_uvc_header_desc(const uint8_t *buff, uint8_t sub_class); void print_device_descriptor(const uint8_t *buff); void print_ep_desc(const uint8_t *buff); diff --git a/components/usb/usb_stream/usb_stream.c b/components/usb/usb_stream/usb_stream.c index 041661614..d44c1639e 100644 --- a/components/usb/usb_stream/usb_stream.c +++ b/components/usb/usb_stream/usb_stream.c @@ -411,6 +411,7 @@ typedef struct { typedef struct { _stream_ifc_t *vs_ifc; _uvc_stream_handle_t *uvc_stream_hdl; + enum uvc_frame_format frame_format; uint8_t format_index; // dynamic values should be protect uvc_frame_size_t *frame_size; @@ -800,10 +801,12 @@ static esp_err_t _update_config_from_descriptor(const usb_config_desc_t *cfg_des int offset = 0; bool already_next = false; uint16_t wTotalLength = cfg_desc->wTotalLength; - /* flags indicate if required format and frame found */ - bool mjpeg_format_found = false; - uint8_t mjpeg_format_idx = 0; - uint8_t mjpeg_frame_num = 0; + /* flags indicate if required setting format and frame found */ + bool format_set_found = false; + uint8_t format_idx = 0; + uint8_t frame_num = 0; + enum uvc_frame_format format = UVC_FRAME_FORMAT_UNKNOWN; + /* flags user definded frame found */ bool user_frame_found = false; uint8_t user_frame_idx = 0; /* flags indicate if suitable audio stream interface found */ @@ -968,19 +971,26 @@ static esp_err_t _update_config_from_descriptor(const usb_config_desc_t *cfg_des print_uvc_header_desc((const uint8_t *)next_desc, VIDEO_SUBCLASS_STREAMING); break; case VIDEO_CS_ITF_VS_FORMAT_MJPEG: - parse_vs_format_mjpeg_desc((const uint8_t *)next_desc, &mjpeg_format_idx, &mjpeg_frame_num); + if (usb_dev->uvc_cfg.format != UVC_FORMAT_MJPEG) { + break; + } + parse_vs_format_mjpeg_desc((const uint8_t *)next_desc, &format_idx, &frame_num, &format); if (uvc_dev) { uvc_frame_size_t *frame_size = uvc_dev->frame_size; - frame_size = (uvc_frame_size_t *)heap_caps_realloc(frame_size, mjpeg_frame_num * sizeof(uvc_frame_size_t), MALLOC_CAP_DEFAULT); + frame_size = (uvc_frame_size_t *)heap_caps_realloc(frame_size, frame_num * sizeof(uvc_frame_size_t), MALLOC_CAP_DEFAULT); UVC_CHECK(frame_size, "alloc uvc frame size failed", ESP_ERR_NO_MEM); UVC_ENTER_CRITICAL(); - uvc_dev->frame_num = mjpeg_frame_num; + uvc_dev->frame_num = frame_num; uvc_dev->frame_size = frame_size; + uvc_dev->frame_format = format; UVC_EXIT_CRITICAL(); } - mjpeg_format_found = true; + format_set_found = true; break; case VIDEO_CS_ITF_VS_FRAME_MJPEG: { + if (usb_dev->uvc_cfg.format != UVC_FORMAT_MJPEG) { + break; + } uint8_t interval_type = 0; const uint32_t *pp_interval = NULL; uint32_t dflt_interval = 0; @@ -1043,6 +1053,89 @@ static esp_err_t _update_config_from_descriptor(const usb_config_desc_t *cfg_des } break; } + case VIDEO_CS_ITF_VS_FORMAT_FRAME_BASED: + if (usb_dev->uvc_cfg.format != UVC_FORMAT_FRAME_BASED) { + break; + } + parse_vs_format_frame_based_desc((const uint8_t *)next_desc, &format_idx, &frame_num, &format); + if (uvc_dev) { + uvc_frame_size_t *frame_size = uvc_dev->frame_size; + frame_size = (uvc_frame_size_t *)heap_caps_realloc(frame_size, frame_num * sizeof(uvc_frame_size_t), MALLOC_CAP_DEFAULT); + UVC_CHECK(frame_size, "alloc uvc frame size failed", ESP_ERR_NO_MEM); + UVC_ENTER_CRITICAL(); + uvc_dev->frame_num = frame_num; + uvc_dev->frame_size = frame_size; + uvc_dev->frame_format = format; + UVC_EXIT_CRITICAL(); + } + format_set_found = true; + break; + case VIDEO_CS_ITF_VS_FRAME_FRAME_BASED: { + if (usb_dev->uvc_cfg.format != UVC_FORMAT_FRAME_BASED) { + break; + } + uint8_t interval_type = 0; + const uint32_t *pp_interval = NULL; + uint32_t dflt_interval = 0; + uint32_t max_interval = 0; + uint32_t min_interval = 0; + uint32_t step_interval = 0; + uint32_t final_interval = 0; + parse_vs_frame_frame_based_desc((const uint8_t *)next_desc, &_frame_idx, &_frame_width, &_frame_heigh, &interval_type, &pp_interval, &dflt_interval); + if (interval_type) { + min_interval = pp_interval[0]; + for (size_t i = 0; i < interval_type; i++) { + if (usb_dev->uvc_cfg.frame_interval == pp_interval[i]) { + final_interval = pp_interval[i]; + } + if (pp_interval[i] > max_interval) { + max_interval = pp_interval[i]; + } + if (pp_interval[i] < min_interval) { + min_interval = pp_interval[i]; + } + } + } else { + min_interval = pp_interval[0]; + max_interval = pp_interval[1]; + step_interval = pp_interval[2]; + if (usb_dev->uvc_cfg.frame_interval >= min_interval && usb_dev->uvc_cfg.frame_interval <= max_interval) { + for (uint32_t i = min_interval; i < max_interval; i += step_interval) { + if (usb_dev->uvc_cfg.frame_interval >= i && usb_dev->uvc_cfg.frame_interval < (i + step_interval)) { + final_interval = i; + } + } + } + } + if (final_interval == 0) { + final_interval = dflt_interval; + ESP_LOGD(TAG, "UVC frame interval %" PRIu32 " not found, using default = %" PRIu32, usb_dev->uvc_cfg.frame_interval, final_interval); + } else { + ESP_LOGD(TAG, "UVC frame interval %" PRIu32 " found = %" PRIu32, usb_dev->uvc_cfg.frame_interval, final_interval); + } + if (uvc_dev) { + assert((_frame_idx - 1) < uvc_dev->frame_num); //should not happen + UVC_ENTER_CRITICAL(); + uvc_dev->frame_size[_frame_idx - 1].width = _frame_width; + uvc_dev->frame_size[_frame_idx - 1].height = _frame_heigh; + uvc_dev->frame_size[_frame_idx - 1].interval = final_interval; + uvc_dev->frame_size[_frame_idx - 1].interval_min = min_interval; + uvc_dev->frame_size[_frame_idx - 1].interval_max = max_interval; + uvc_dev->frame_size[_frame_idx - 1].interval_step = step_interval; + UVC_EXIT_CRITICAL(); + } + if (user_frame_found == true) { + break; + } + if (((_frame_width == usb_dev->uvc_cfg.frame_width) || (FRAME_RESOLUTION_ANY == usb_dev->uvc_cfg.frame_width)) + && ((_frame_heigh == usb_dev->uvc_cfg.frame_height) || (FRAME_RESOLUTION_ANY == usb_dev->uvc_cfg.frame_height))) { + user_frame_found = true; + user_frame_idx = _frame_idx; + } else if ((_frame_width == usb_dev->uvc_cfg.frame_height) && (_frame_heigh == usb_dev->uvc_cfg.frame_width)) { + ESP_LOGW(TAG, "found width*heigh %u * %u , orientation swap?", _frame_heigh, _frame_width); + } + break; + } default: break; } @@ -1337,15 +1430,15 @@ static esp_err_t _update_config_from_descriptor(const usb_config_desc_t *cfg_des ESP_LOGW(TAG, "VS Interface(MPS <= %d) NOT found", USB_EP_ISOC_IN_MAX_MPS); ESP_LOGW(TAG, "Try with first alt-interface config"); } - if (mjpeg_format_found) { - ESP_LOGI(TAG, "Actual MJPEG format index = %u, contains %u frames", mjpeg_format_idx, mjpeg_frame_num); - uvc_dev->format_index = mjpeg_format_idx; + if (format_set_found) { + ESP_LOGI(TAG, "Actual %s format index, format index = %u, contains %u frames", usb_dev->uvc_cfg.format == UVC_FORMAT_FRAME_BASED ? "Frame Based" : "MJPEG", format_idx, frame_num); + uvc_dev->format_index = format_idx; } else if (usb_dev->uvc_cfg.format_index) { - ESP_LOGW(TAG, "MJPEG format NOT found"); + ESP_LOGW(TAG, "Setting format: %d NOT found", usb_dev->uvc_cfg.format); ESP_LOGW(TAG, "Try with user's config"); uvc_dev->format_index = usb_dev->uvc_cfg.format_index; } else { - ESP_LOGE(TAG, "MJPEG format NOT found"); + ESP_LOGE(TAG, "Setting format: %d NOT found", usb_dev->uvc_cfg.format); // We treat MJPEG format as mandatory vs_intf_found = false; } @@ -1353,9 +1446,9 @@ static esp_err_t _update_config_from_descriptor(const usb_config_desc_t *cfg_des UVC_ENTER_CRITICAL(); uvc_dev->frame_index = user_frame_idx; UVC_EXIT_CRITICAL(); - ESP_LOGI(TAG, "Actual MJPEG width*heigh: %u*%u, frame index = %u", usb_dev->uvc_cfg.frame_width, usb_dev->uvc_cfg.frame_height, user_frame_idx); + ESP_LOGI(TAG, "Actual Frame: %d, width*heigh: %u*%u, frame index = %u", uvc_dev->frame_format, usb_dev->uvc_cfg.frame_width, usb_dev->uvc_cfg.frame_height, user_frame_idx); } else if (usb_dev->uvc_cfg.frame_index) { - ESP_LOGW(TAG, "MJPEG width*heigh: %u*%u, NOT found", usb_dev->uvc_cfg.frame_width, usb_dev->uvc_cfg.frame_height); + ESP_LOGW(TAG, "Frame: %d, width*heigh: %u*%u, NOT found", uvc_dev->frame_format, usb_dev->uvc_cfg.frame_width, usb_dev->uvc_cfg.frame_height); ESP_LOGW(TAG, "Try with user's config"); UVC_ENTER_CRITICAL(); uvc_dev->frame_index = usb_dev->uvc_cfg.frame_index; @@ -1364,7 +1457,7 @@ static esp_err_t _update_config_from_descriptor(const usb_config_desc_t *cfg_des UVC_EXIT_CRITICAL(); } else { // No suitable frame found, we need suspend UVC interface during start - ESP_LOGW(TAG, "MJPEG width*heigh: %u*%u, NOT found", usb_dev->uvc_cfg.frame_width, usb_dev->uvc_cfg.frame_height); + ESP_LOGW(TAG, "Frame: %d, width*heigh: %u*%u, NOT found", uvc_dev->frame_format, usb_dev->uvc_cfg.frame_width, usb_dev->uvc_cfg.frame_height); vs_intf_found = false; } } @@ -2006,6 +2099,7 @@ static uvc_error_t uvc_stream_open_ctrl(uvc_device_handle_t *devh, _uvc_stream_h strmh->outbuf = s_usb_dev.uvc_cfg.xfer_buffer_a; strmh->holdbuf = s_usb_dev.uvc_cfg.xfer_buffer_b; strmh->frame.data = s_usb_dev.uvc_cfg.frame_buffer; + strmh->frame_format = s_usb_dev.uvc->frame_format; strmh->cb_mutex = xSemaphoreCreateMutex(); @@ -2049,7 +2143,6 @@ static uvc_error_t uvc_stream_start(_uvc_stream_handle_t *strmh, uvc_frame_callb strmh->fid = 0; strmh->pts = 0; strmh->last_scr = 0; - strmh->frame_format = UVC_FRAME_FORMAT_MJPEG; strmh->user_cb = cb; strmh->user_ptr = user_ptr; @@ -2444,7 +2537,7 @@ static esp_err_t _uvc_streaming_resume(void) frame_size.width = uvc_dev->frame_width; frame_size.height = uvc_dev->frame_height; UVC_EXIT_CRITICAL(); - ESP_LOGI(TAG, "Probe Format(%u) MJPEG, Frame(%u) %u*%u, interval(%"PRIu32")", ctrl_set.bFormatIndex, + ESP_LOGI(TAG, "Probe Format(%u), Frame(%u) %u*%u, interval(%"PRIu32")", ctrl_set.bFormatIndex, ctrl_set.bFrameIndex, frame_size.width, frame_size.height, ctrl_set.dwFrameInterval); ESP_LOGI(TAG, "Probe payload size = %"PRIu32, ctrl_set.dwMaxPayloadTransferSize); esp_err_t ret = _uvc_vs_commit_control(&ctrl_set, &ctrl_probed); @@ -2456,6 +2549,7 @@ static esp_err_t _uvc_streaming_resume(void) uvc_error_t uvc_ret = UVC_SUCCESS; uvc_ret = uvc_stream_open_ctrl(NULL, &uvc_dev->uvc_stream_hdl, &ctrl_probed); UVC_CHECK(uvc_ret == UVC_SUCCESS, "open uvc stream failed", ESP_FAIL); + uvc_ret = uvc_stream_start(uvc_dev->uvc_stream_hdl, s_usb_dev.uvc_cfg.frame_cb, s_usb_dev.uvc_cfg.frame_cb_arg, 0); UVC_CHECK_GOTO(uvc_ret == UVC_SUCCESS, "start uvc stream failed", free_stream_); if (uvc_dev->vs_ifc->xfer_type == UVC_XFER_ISOC) { @@ -3495,6 +3589,7 @@ esp_err_t uvc_streaming_config(const uvc_config_t *config) UVC_CHECK(config != NULL, "config can't NULL", ESP_ERR_INVALID_ARG); UVC_CHECK((config->frame_interval >= FRAME_MIN_INTERVAL && config->frame_interval <= FRAME_MAX_INTERVAL), "frame_interval Support 333333~2000000", ESP_ERR_INVALID_ARG); + UVC_CHECK(config->format < UVC_FORMAT_MAX, "format can't larger than UVC_FORMAT_MAX", ESP_ERR_INVALID_ARG); UVC_CHECK(config->frame_height != 0, "frame_height can't 0", ESP_ERR_INVALID_ARG); UVC_CHECK(config->frame_width != 0, "frame_width can't 0", ESP_ERR_INVALID_ARG); UVC_CHECK(config->frame_buffer_size != 0, "frame_buffer_size can't 0", ESP_ERR_INVALID_ARG);