From b64deee429cd3baa660513bae8019845ae87106e Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Mon, 5 Feb 2024 20:16:09 -0500 Subject: [PATCH 1/4] Add NV12 support (V4L2 needs work) --- FlashCap.Core/Internal/BitmapTranscoder.cs | 228 +++++++++++++++------ FlashCap.Core/Internal/NativeMethods.cs | 8 +- FlashCap.Core/VideoCharacteristics.cs | 2 + FlashCap.V4L2Generator/Program.cs | 1 + 4 files changed, 180 insertions(+), 59 deletions(-) diff --git a/FlashCap.Core/Internal/BitmapTranscoder.cs b/FlashCap.Core/Internal/BitmapTranscoder.cs index 1c06536..68995c3 100644 --- a/FlashCap.Core/Internal/BitmapTranscoder.cs +++ b/FlashCap.Core/Internal/BitmapTranscoder.cs @@ -17,6 +17,11 @@ internal static class BitmapTranscoder { private static readonly int scatteringBase = Environment.ProcessorCount; + struct ConversionConstants + { + public int multY, multUB, multUG, multVG, multVR, offsetY; + } + // Prefered article: https://docs.microsoft.com/en-us/windows/win32/medfound/recommended-8-bit-yuv-formats-for-video-rendering#420-formats-16-bits-per-pixel // some interesting code references: @@ -25,11 +30,11 @@ internal static class BitmapTranscoder // TranscodeFormats WITH FullRange suffix means that we suppose Y U and V are in [0..255] range private static unsafe void TranscodeFromYUVInternal( int width, int height, - TranscodeFormats conversionStandard, bool isUYVY, + TranscodeFormats conversionStandard, NativeMethods.Compression compression, byte* pFrom, byte* pTo) { // constants for color conversion - int multY, multUB, multUG, multVG, multVR, offsetY; + ConversionConstants conversionConstants; // select constants for the color conversion switch (conversionStandard) @@ -52,21 +57,28 @@ private static unsafe void TranscodeFromYUVInternal( // multiply Y by 1.16438 // multiply UV by 1.13839 - multY = 298; - multUB = 587; - multUG = 114; - multVG = 237; - multVR = 466; - offsetY = 16; + conversionConstants = new ConversionConstants + { + multY = 298, + multUB = 587, + multUG = 114, + multVG = 237, + multVR = 466, + offsetY = 16 + }; break; case TranscodeFormats.BT601FullRange: - multY = 255; - multUB = 516; - multUG = 100; - multVG = 208; - multVR = 409; - offsetY = 0; + + conversionConstants = new ConversionConstants + { + multY = 255, + multUB = 516, + multUG = 100, + multVG = 208, + multVR = 409, + offsetY = 0 + }; break; ////////////////////////////////////////////////// @@ -87,21 +99,28 @@ private static unsafe void TranscodeFromYUVInternal( // multiply Y by 1.16438 // multiply UV by 1.13839 - multY = 298; - multUB = 541; - multUG = 55; - multVG = 137; - multVR = 459; - offsetY = 16; + conversionConstants = new ConversionConstants + { + multY = 298, + multUB = 541, + multUG = 55, + multVG = 137, + multVR = 459, + offsetY = 16 + }; break; case TranscodeFormats.BT709FullRange: - multY = 255; - multUB = 475; - multUG = 48; - multVG = 120; - multVR = 403; - offsetY = 0; + + conversionConstants = new ConversionConstants + { + multY = 255, + multUB = 475, + multUG = 48, + multVG = 120, + multVR = 403, + offsetY = 0 + }; break; ////////////////////////////////////////////////// @@ -113,21 +132,29 @@ private static unsafe void TranscodeFromYUVInternal( // multiply Y by 1.16438 // multiply UV by 1.13839 - multY = 298; - multUB = 549; - multUG = 48; - multVG = 166; - multVR = 429; - offsetY = 16; + + conversionConstants = new ConversionConstants + { + multY = 298, + multUB = 549, + multUG = 48, + multVG = 166, + multVR = 429, + offsetY = 16 + }; break; case TranscodeFormats.BT2020FullRange: - multY = 255; - multUB = 482; - multUG = 42; - multVG = 146; - multVR = 377; - offsetY = 0; + + conversionConstants = new ConversionConstants + { + multY = 255, + multUB = 482, + multUG = 42, + multVG = 146, + multVR = 377, + offsetY = 0 + }; break; ////////////////////////////////////////////////// @@ -136,6 +163,88 @@ private static unsafe void TranscodeFromYUVInternal( throw new ArgumentException(nameof(conversionStandard)); } + switch(compression) + { + case NativeMethods.Compression.UYVY: + case NativeMethods.Compression.HDYC: + YUY2_UYVY_to_RGB24(true, width, height, conversionConstants, compression, pFrom, pTo); + break; + case NativeMethods.Compression.YUYV: + case NativeMethods.Compression.YUY2: + YUY2_UYVY_to_RGB24(false, width, height, conversionConstants, compression, pFrom, pTo); + break; + case NativeMethods.Compression.NV12: + NV12_to_RGB24(width, height, conversionConstants, pFrom, pTo); + break; + default: + throw new ArgumentException(nameof(compression)); + } + } + + private static unsafe void NV12_to_RGB24( + int width, int height, + ConversionConstants cconsts, + byte* pFrom, byte* pTo) + { + var scatter = height / scatteringBase; + Parallel.For(0, (height + scatter - 1) / scatter, ys => + { + var y = ys * scatter; + var myi = Math.Min(height - y, scatter); + + for (var yi = 0; yi < myi; yi++) + { + byte* pFromY = pFrom + (y + yi) * width; + byte* pFromUV = pFrom + (height + (y + yi) / 2) * width; + + byte* pToBase = pTo + (height - (y + yi) - 1) * width * 3; + + + for (var x = 0; x < width - 1; x += 2) + { + int c1 = pFromY[0] - cconsts.offsetY; // Y1 + int c2 = pFromY[1] - cconsts.offsetY; // Y2 + int d = pFromUV[0] - 128; // U + int e = pFromUV[1] - 128; // V + + int cc1 = cconsts.multY * c1; + int cc2 = cconsts.multY * c2; + + *pToBase++ = Clip((cc1 + cconsts.multUB * d + 128) >> 8); // B1 + *pToBase++ = Clip((cc1 - cconsts.multUG * d - cconsts.multVG * e + 128) >> 8); // G1 + *pToBase++ = Clip((cc1 + cconsts.multVR * e + 128) >> 8); // R1 + + *pToBase++ = Clip((cc2 + cconsts.multUB * d + 128) >> 8); // B2 + *pToBase++ = Clip((cc2 - cconsts.multUG * d - cconsts.multVG * e + 128) >> 8); // G2 + *pToBase++ = Clip((cc2 + cconsts.multVR * e + 128) >> 8); // R2 + + pFromY += 2; + pFromUV += 2; + } + + if ((width & 1) != 0) + { + int c1 = pFromY[0] - cconsts.offsetY; // Y1 + int d = pFromUV[0] - 128; // U + int e = pFromUV[1] - 128; // V + + int cc1 = cconsts.multY * c1; + + *pToBase++ = Clip((cc1 + cconsts.multUB * d + 128) >> 8); // B1 + *pToBase++ = Clip((cc1 - cconsts.multUG * d - cconsts.multVG * e + 128) >> 8); // G1 + *pToBase++ = Clip((cc1 + cconsts.multVR * e + 128) >> 8); // R1 + } + } + }); + } + + private static unsafe void YUY2_UYVY_to_RGB24( + bool isUYVY, + int width, int height, + ConversionConstants cconsts, + NativeMethods.Compression compression, + byte* pFrom, byte* pTo) + { var scatter = height / scatteringBase; Parallel.For(0, (height + scatter - 1) / scatter, ys => { @@ -150,31 +259,33 @@ private static unsafe void TranscodeFromYUVInternal( for (var x = 0; x < width; x += 2) { + // UYVY, HDYC if (isUYVY) { d = pFromBase[0] - 128; // U - c1 = pFromBase[1] - offsetY; // Y1 + c1 = pFromBase[1] - cconsts.offsetY; // Y1 e = pFromBase[2] - 128; // V - c2 = pFromBase[3] - offsetY; // Y2 + c2 = pFromBase[3] - cconsts.offsetY; // Y2 } + // YUY2, YUYV else { - c1 = pFromBase[0] - offsetY; // Y1 + c1 = pFromBase[0] - cconsts.offsetY; // Y1 d = pFromBase[1] - 128; // U - c2 = pFromBase[2] - offsetY; // Y2 + c2 = pFromBase[2] - cconsts.offsetY; // Y2 e = pFromBase[3] - 128; // V } - cc1 = multY * c1; - cc2 = multY * c2; + cc1 = cconsts.multY * c1; + cc2 = cconsts.multY * c2; - *pToBase++ = Clip((cc1 + multUB * d + 128) >> 8); // B1 - *pToBase++ = Clip((cc1 - multUG * d - multVG * e + 128) >> 8); // G1 - *pToBase++ = Clip((cc1 + multVR * e + 128) >> 8); // R1 + *pToBase++ = Clip((cc1 + cconsts.multUB * d + 128) >> 8); // B1 + *pToBase++ = Clip((cc1 - cconsts.multUG * d - cconsts.multVG * e + 128) >> 8); // G1 + *pToBase++ = Clip((cc1 + cconsts.multVR * e + 128) >> 8); // R1 - *pToBase++ = Clip((cc2 + multUB * d + 128) >> 8); // B2 - *pToBase++ = Clip((cc2 - multUG * d - multVG * e + 128) >> 8); // G2 - *pToBase++ = Clip((cc2 + multVR * e + 128) >> 8); // R2 + *pToBase++ = Clip((cc2 + cconsts.multUB * d + 128) >> 8); // B2 + *pToBase++ = Clip((cc2 - cconsts.multUG * d - cconsts.multVG * e + 128) >> 8); // G2 + *pToBase++ = Clip((cc2 + cconsts.multVR * e + 128) >> 8); // R2 pFromBase += 4; } @@ -184,7 +295,8 @@ private static unsafe void TranscodeFromYUVInternal( private static unsafe void TranscodeFromYUV( int width, int height, - TranscodeFormats transcodeFormat, bool isUYVY, + TranscodeFormats transcodeFormat, + NativeMethods.Compression compression, byte* pFrom, byte* pTo) { switch (transcodeFormat) @@ -195,16 +307,16 @@ private static unsafe void TranscodeFromYUV( case TranscodeFormats.BT709FullRange: case TranscodeFormats.BT2020: case TranscodeFormats.BT2020FullRange: - TranscodeFromYUVInternal(width, height, transcodeFormat, isUYVY, pFrom, pTo); + TranscodeFromYUVInternal(width, height, transcodeFormat, compression, pFrom, pTo); break; case TranscodeFormats.Auto: // determine the color conversion based on the width and height of the frame if (width > 1920 || height > 1080) // UHD or larger - TranscodeFromYUVInternal(width, height, TranscodeFormats.BT2020, isUYVY, pFrom, pTo); + TranscodeFromYUVInternal(width, height, TranscodeFormats.BT2020, compression, pFrom, pTo); else if (width > 720 || height > 576) // HD - TranscodeFromYUVInternal(width, height, TranscodeFormats.BT709, isUYVY, pFrom, pTo); + TranscodeFromYUVInternal(width, height, TranscodeFormats.BT709, compression, pFrom, pTo); else // SD - TranscodeFromYUVInternal(width, height, TranscodeFormats.BT601, isUYVY, pFrom, pTo); + TranscodeFromYUVInternal(width, height, TranscodeFormats.BT601, compression, pFrom, pTo); break; default: throw new ArgumentException(nameof(transcodeFormat)); @@ -228,6 +340,7 @@ private static byte Clip(int value) => case NativeMethods.Compression.YUYV: case NativeMethods.Compression.YUY2: case NativeMethods.Compression.HDYC: + case NativeMethods.Compression.NV12: return width * height * 3; default: return null; @@ -244,11 +357,10 @@ public static unsafe void Transcode( { case NativeMethods.Compression.UYVY: case NativeMethods.Compression.HDYC: - TranscodeFromYUV(width, height, transcodeFormat, true, pFrom, pTo); - break; case NativeMethods.Compression.YUYV: case NativeMethods.Compression.YUY2: - TranscodeFromYUV(width, height, transcodeFormat, false, pFrom, pTo); + case NativeMethods.Compression.NV12: + TranscodeFromYUV(width, height, transcodeFormat, compression, pFrom, pTo); break; default: throw new ArgumentException(nameof(compression)); diff --git a/FlashCap.Core/Internal/NativeMethods.cs b/FlashCap.Core/Internal/NativeMethods.cs index 1374e1f..35bbc1c 100644 --- a/FlashCap.Core/Internal/NativeMethods.cs +++ b/FlashCap.Core/Internal/NativeMethods.cs @@ -199,7 +199,8 @@ public enum Compression YUYV = 0x56595559, // FOURCC UYVY = 0x59565955, // FOURCC MJPG = 0x47504A4D, // FOURCC - HDYC = 0x43594448 // FOURCC (BlackMagic input (UYVY)) + HDYC = 0x43594448, // FOURCC (BlackMagic input (UYVY)) + NV12 = 0x3231564E, // FOURCC } private static int CalculateClrUsed( @@ -506,6 +507,7 @@ static PixelFormats GetRGBPixelFormat(int clrBits) => Compression.YUYV => PixelFormats.YUYV, Compression.YUY2 => PixelFormats.YUYV, Compression.HDYC => PixelFormats.YUYV, + Compression.NV12 => PixelFormats.NV12, _ => PixelFormats.Unknown, }; @@ -579,6 +581,10 @@ public static bool GetCompressionAndBitCount( compression = Compression.YUYV; bitCount = 16; return true; + case PixelFormats.NV12: + compression = Compression.NV12; + bitCount = 12; + return true; default: compression = default; bitCount = 0; diff --git a/FlashCap.Core/VideoCharacteristics.cs b/FlashCap.Core/VideoCharacteristics.cs index 37712cf..2b30f4a 100644 --- a/FlashCap.Core/VideoCharacteristics.cs +++ b/FlashCap.Core/VideoCharacteristics.cs @@ -25,6 +25,7 @@ public enum PixelFormats PNG, UYVY, YUYV, + NV12 } public sealed class VideoCharacteristics : @@ -80,6 +81,7 @@ public VideoCharacteristics( PixelFormats.ARGB32 => 32, PixelFormats.UYVY => 16, PixelFormats.YUYV => 16, + PixelFormats.NV12 => 12, _ => 0, }; diff --git a/FlashCap.V4L2Generator/Program.cs b/FlashCap.V4L2Generator/Program.cs index f345c99..be08c70 100644 --- a/FlashCap.V4L2Generator/Program.cs +++ b/FlashCap.V4L2Generator/Program.cs @@ -76,6 +76,7 @@ public SymbolName(string name, bool withComment = false) "V4L2_PIX_FMT_UYVY", "V4L2_PIX_FMT_XRGB32", "V4L2_PIX_FMT_YUYV", + "V4L2_PIX_FMT_NV12", "VIDIOC_DQBUF", "VIDIOC_ENUM_FMT", "VIDIOC_ENUM_FRAMEINTERVALS", From f4ad557274b74af9669d9524c09bdec5d0355262 Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Mon, 5 Feb 2024 21:29:08 -0500 Subject: [PATCH 2/4] Regenerate NativeMethods_V4L2_Interop_x86_64.cs --- .../Internal/V4L2/NativeMethods_V4L2_Interop_x86_64.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/FlashCap.Core/Internal/V4L2/NativeMethods_V4L2_Interop_x86_64.cs b/FlashCap.Core/Internal/V4L2/NativeMethods_V4L2_Interop_x86_64.cs index 99ae4d2..1ad7fd5 100644 --- a/FlashCap.Core/Internal/V4L2/NativeMethods_V4L2_Interop_x86_64.cs +++ b/FlashCap.Core/Internal/V4L2/NativeMethods_V4L2_Interop_x86_64.cs @@ -1,6 +1,6 @@ -// This is auto generated code by FlashCap.V4L2Generator [0.14.6]. Do not edit. -// Linux version 5.13.0-39-generic (buildd@lcy02-amd64-080) (gcc (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0, GNU ld (GNU Binutils for Ubuntu) 2.34) #44~20.04.1-Ubuntu SMP Thu Mar 24 16:43:35 UTC 2022 -// Fri, 15 Apr 2022 03:59:31 GMT +// This is auto generated code by FlashCap.V4L2Generator [0.0.301]. Do not edit. +// Linux version 5.4.0-169-generic (buildd@lcy02-amd64-102) (gcc version 9.4.0 (Ubuntu 9.4.0-1ubuntu1~20.04.2)) #187-Ubuntu SMP Thu Nov 23 14:52:28 UTC 2023 +// Tue, 06 Feb 2024 02:26:04 GMT using System; using System.Runtime.InteropServices; @@ -10,7 +10,7 @@ namespace FlashCap.Internal.V4L2 internal sealed class NativeMethods_V4L2_Interop_x86_64 : NativeMethods_V4L2_Interop { // Common - public override string Label => "Linux version 5.13.0-39-generic (buildd@lcy02-amd64-080) (gcc (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0, GNU ld (GNU Binutils for Ubuntu) 2.34) #44~20.04.1-Ubuntu SMP Thu Mar 24 16:43:35 UTC 2022"; + public override string Label => "Linux version 5.4.0-169-generic (buildd@lcy02-amd64-102) (gcc version 9.4.0 (Ubuntu 9.4.0-1ubuntu1~20.04.2)) #187-Ubuntu SMP Thu Nov 23 14:52:28 UTC 2023"; public override string Architecture => "x86_64"; public override int sizeof_size_t => 8; public override int sizeof_off_t => 8; @@ -21,6 +21,7 @@ internal sealed class NativeMethods_V4L2_Interop_x86_64 : NativeMethods_V4L2_Int public override uint V4L2_PIX_FMT_ARGB32 => 875708738U; public override uint V4L2_PIX_FMT_JPEG => 1195724874U; public override uint V4L2_PIX_FMT_MJPEG => 1196444237U; + public override uint V4L2_PIX_FMT_NV12 => 842094158U; public override uint V4L2_PIX_FMT_RGB24 => 859981650U; public override uint V4L2_PIX_FMT_RGB332 => 826427218U; public override uint V4L2_PIX_FMT_RGB565 => 1346520914U; From 51ac1e490460db3947537f04b8d59eb998c175bc Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Mon, 5 Feb 2024 21:40:04 -0500 Subject: [PATCH 3/4] Regenerated NativeMethods_V4L2_Interop.cs --- .../V4L2/NativeMethods_V4L2_Interop.cs | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/FlashCap.Core/Internal/V4L2/NativeMethods_V4L2_Interop.cs b/FlashCap.Core/Internal/V4L2/NativeMethods_V4L2_Interop.cs index 6af22cc..a92ae8b 100644 --- a/FlashCap.Core/Internal/V4L2/NativeMethods_V4L2_Interop.cs +++ b/FlashCap.Core/Internal/V4L2/NativeMethods_V4L2_Interop.cs @@ -1,6 +1,6 @@ -// This is auto generated code by FlashCap.V4L2Generator [1.8.0]. Do not edit. -// Linux version 4.19.0-19-loongson-3 (abuild@10.40.52.160) (gcc version 8.3.0 (Loongnix 8.3.0-6.lnd.vec.36)) #1 SMP 4.19.190.8.14 Thu Aug 24 08:54:20 UTC 2023 -// Thu, 14 Dec 2023 01:30:19 GMT +// This is auto generated code by FlashCap.V4L2Generator [0.0.301]. Do not edit. +// Linux version 5.4.0-169-generic (buildd@lcy02-amd64-102) (gcc version 9.4.0 (Ubuntu 9.4.0-1ubuntu1~20.04.2)) #187-Ubuntu SMP Thu Nov 23 14:52:28 UTC 2023 +// Tue, 06 Feb 2024 02:38:45 GMT using System; using System.Runtime.InteropServices; @@ -21,6 +21,7 @@ internal abstract partial class NativeMethods_V4L2_Interop public virtual uint V4L2_PIX_FMT_ARGB32 => throw new NotImplementedException(); public virtual uint V4L2_PIX_FMT_JPEG => throw new NotImplementedException(); public virtual uint V4L2_PIX_FMT_MJPEG => throw new NotImplementedException(); + public virtual uint V4L2_PIX_FMT_NV12 => throw new NotImplementedException(); public virtual uint V4L2_PIX_FMT_RGB24 => throw new NotImplementedException(); public virtual uint V4L2_PIX_FMT_RGB332 => throw new NotImplementedException(); public virtual uint V4L2_PIX_FMT_RGB565 => throw new NotImplementedException(); @@ -56,6 +57,7 @@ public enum v4l2_buf_type SDR_CAPTURE = 11, SDR_OUTPUT = 12, META_CAPTURE = 13, + META_OUTPUT = 14, PRIVATE = 128, } @@ -223,6 +225,12 @@ uint reserved2 set; } + int request_fd + { + get; + set; + } + uint reserved { get; @@ -890,6 +898,12 @@ uint memory set; } + uint capabilities + { + get; + set; + } + uint[] reserved { get; From f5e95d4c984de5a35e5089f490d14bf992ec831e Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Mon, 5 Feb 2024 22:26:15 -0500 Subject: [PATCH 4/4] Add NV12 to supported pixel formats in NativeMethods_V4L2.cs --- FlashCap.Core/Internal/NativeMethods_V4L2.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/FlashCap.Core/Internal/NativeMethods_V4L2.cs b/FlashCap.Core/Internal/NativeMethods_V4L2.cs index 7722ce9..cf40402 100644 --- a/FlashCap.Core/Internal/NativeMethods_V4L2.cs +++ b/FlashCap.Core/Internal/NativeMethods_V4L2.cs @@ -91,6 +91,7 @@ static NativeMethods_V4L2() pixelFormats.Add(Interop.V4L2_PIX_FMT_UYVY, PixelFormats.UYVY); pixelFormats.Add(Interop.V4L2_PIX_FMT_YUYV, PixelFormats.YUYV); pixelFormats.Add(Interop.V4L2_PIX_FMT_YUY2, PixelFormats.YUYV); + pixelFormats.Add(Interop.V4L2_PIX_FMT_NV12, PixelFormats.NV12); } public static bool IsKnownPixelFormat(uint pix_fmt) => @@ -324,6 +325,8 @@ public static uint[] GetPixelFormats( return new[] { Interop.V4L2_PIX_FMT_UYVY }; case PixelFormats.YUYV: return new[] { Interop.V4L2_PIX_FMT_YUYV, Interop.V4L2_PIX_FMT_YUY2 }; + case PixelFormats.NV12: + return new[] { Interop.V4L2_PIX_FMT_NV12 }; case PixelFormats.JPEG: return new[] { Interop.V4L2_PIX_FMT_MJPEG, Interop.V4L2_PIX_FMT_JPEG, (uint)NativeMethods.Compression.BI_JPEG }; case PixelFormats.PNG: