Merge pull request #9 from kilbouri/h264-packet-upgrade

resolution encoded in h264packet

client/window/window.thread.cpp

@@ -128,8 +128,7 @@ namespace StudentSync::Client {
 					}
 				};
 
-				auto
-					packet = encoder.ReceivePacket();
+				auto packet = encoder.ReceivePacket();
 				if (!packet) {
 					PushLogMessage(std::format("Failed to read packet from encoder: %d", static_cast<int>(packet.error())));
 					if (packet.error() != H264Encoder::ReceivePacketError::InsufficientInput) {
@@ -139,7 +138,9 @@ namespace StudentSync::Client {
 				}
 
 				Message::H264Packet message{
-					.imageData = *packet
+					.imageData = *packet,
+					.frameWidth = width,
+					.frameHeight = height
 				};
 
 				if (!message.ToTLVMessage().Send(connection.socket)) {

common/ffmpeg/decoders/h264Decoder.cpp

@@ -12,15 +12,6 @@ static inline constexpr T* throwIfNull(T* ptr, std::string_view error) {
 	return ptr;
 }
 
-static inline size_t getBitsPerPixel(AVPixelFormat format) {
-	const AVPixFmtDescriptor* descriptor = av_pix_fmt_desc_get(format);
-	if (!descriptor) {
-		throw std::format("Unknown pixel format {}", static_cast<std::underlying_type_t<AVPixelFormat>>(format));
-	}
-
-	return static_cast<size_t>(av_get_bits_per_pixel(descriptor));
-}
-
 namespace StudentSync::Common::FFmpeg::Decoders {
 	H264Decoder::H264Decoder(AVPixelFormat outputFormat)
 		: codec{

common/ffmpeg/decoders/h264Decoder.hpp

@@ -19,8 +19,8 @@ namespace StudentSync::Common::FFmpeg::Decoders {
 		ffmpeg_struct_ptr<AVFrame> frame;
 		ffmpeg_struct_ptr<AVPacket> packet;
 
-		size_t incomingBytesPerPixel;
-		size_t outgoingBytesPerPixel;
+		int incomingBytesPerPixel;
+		int outgoingBytesPerPixel;
 		AVPixelFormat outputFormat;
 
 	public:

common/ffmpeg/encoders/h264Encoder.cpp

@@ -9,15 +9,6 @@ static inline constexpr T* throwIfNull(T* ptr, std::string_view error) {
 	return ptr;
 }
 
-static inline int getBitsPerPixel(AVPixelFormat format) {
-	const AVPixFmtDescriptor* descriptor = av_pix_fmt_desc_get(format);
-	if (!descriptor) {
-		throw std::format("Unknown pixel format {}", static_cast<std::underlying_type_t<AVPixelFormat>>(format));
-	}
-
-	return av_get_bits_per_pixel(descriptor);
-}
-
 namespace StudentSync::Common::FFmpeg::Encoders {
 	H264Encoder::H264Encoder(int width, int height, int fps, AVPixelFormat incomingFormat)
 		: codec{

ffmpegincludes.h

@@ -1,8 +1,22 @@
+#pragma once
+
 extern "C" {
 	#include <libavcodec/avcodec.h>
 	#include <libavformat/avformat.h>
 	#include <libavutil/avutil.h>
 	#include <libavutil/pixdesc.h>
 	#include <libavdevice/avdevice.h>
 	#include <libswscale/swscale.h>
+}
+
+#include <format>
+
+// util function to compute num bytes per pixel
+static inline int getBitsPerPixel(AVPixelFormat format) {
+	const AVPixFmtDescriptor* descriptor = av_pix_fmt_desc_get(format);
+	if (!descriptor) {
+		throw std::format("Unknown pixel format {}", static_cast<std::underlying_type_t<AVPixelFormat>>(format));
+	}
+
+	return av_get_bits_per_pixel(descriptor);
 }

networking/message/h264packetmessage.hpp

@@ -1,22 +1,86 @@
 #pragma once
+
+#include <iterator>
+
 #include "../tlvmessage/tlvmessage.hpp"
+#include "../../ffmpegincludes.h"
 
 namespace StudentSync::Networking::Message {
 	struct H264Packet {
+
+	public:
 		std::vector<uint8_t> imageData;
 
+		int frameWidth;
+		int frameHeight;
+
+	private:
+		static constexpr size_t MIN_BYTES_SIZE = sizeof(frameWidth) + sizeof(frameHeight);
+
+	public:
 		static std::optional<H264Packet> FromTLVMessage(const TLVMessage& netMessage) noexcept {
 			if (netMessage.tag != TLVMessage::Tag::H264Packet) {
 				return std::nullopt;
 			}
 
+			if (netMessage.data.size() < MIN_BYTES_SIZE) {
+				return std::nullopt;
+			}
+
+			const int* frameDimPtr = reinterpret_cast<const int*>(netMessage.data.data());
+
+			int frameWidth = *frameDimPtr;
+			int frameHeight = *(frameDimPtr + 1);
+
+			const void* imageDataStart = frameDimPtr + 2;
+
+			std::vector<uint8_t> data{};
+			std::move(
+				netMessage.data.begin() + sizeof(frameWidth) + sizeof(frameHeight),
+				netMessage.data.end(),
+				std::back_inserter(data)
+			);
+
+
 			return H264Packet{
-				.imageData = netMessage.data
+				.imageData = data,
+				.frameWidth = static_cast<int>(ntohl(frameWidth)),
+				.frameHeight = static_cast<int>(ntohl(frameHeight)),
 			};
 		}
 
 		TLVMessage ToTLVMessage() const noexcept {
-			return TLVMessage(TLVMessage::Tag::H264Packet, imageData);
+
+			std::vector<uint8_t> networkValue;
+			networkValue.reserve(MIN_BYTES_SIZE + imageData.size());
+
+			// copy in width and height
+			int netWidth = static_cast<int>(htonl(frameWidth));
+			int netHeight = static_cast<int>(htonl(frameHeight));
+
+			const uint8_t* wPtr = reinterpret_cast<const uint8_t*>(&netWidth);
+			const uint8_t* hPtr = reinterpret_cast<const uint8_t*>(&netHeight);
+
+			std::copy(
+				wPtr,
+				wPtr + sizeof(netWidth),
+				std::back_inserter(networkValue)
+			);
+
+			std::copy(
+				hPtr,
+				hPtr + sizeof(netHeight),
+				std::back_inserter(networkValue)
+			);
+
+			// copy image data
+			std::copy(
+				imageData.begin(),
+				imageData.end(),
+				std::back_inserter(networkValue)
+			);
+
+			return TLVMessage(TLVMessage::Tag::H264Packet, networkValue);
 		}
 	};
 }

server/window/eventdispatcher/eventdispatcher.cpp

@@ -66,7 +66,7 @@ namespace StudentSync::Server {
 
 		std::memcpy(bmpDataPtr, rgbPixelData.data(), pixelDataByteLength);
 
-		wxImage image{ 2256, 1504, reinterpret_cast<unsigned char*>(bmpDataPtr), false };
+		wxImage image{ packet.frameWidth, packet.frameHeight, reinterpret_cast<unsigned char*>(bmpDataPtr), false };
 		wxBitmap bmp{ image };
 
 		if (!bmp.IsOk()) {