Add clock sync module for per-frame delay calculation.

Author: taste1981

api/video/encoded_image.h

@@ -184,6 +184,14 @@ class RTC_EXPORT EncodedImage {
     int64_t receive_finish_ms = 0;
   } timing_;
 
+#if defined(WEBRTC_WIN)
+  struct BWEStats {
+    double start_duration_ = 0;
+    double last_duration_ = 0;
+    int32_t packets_lost_ = 0;
+  }bwe_stats_;
+#endif
+
  private:
   size_t capacity() const {
     return buffer_ ? capacity_ : (encoded_data_ ? encoded_data_->size() : 0);

modules/video_coding/frame_object.h

@@ -49,7 +49,15 @@ class RtpFrameObject : public EncodedFrame {
   const RTPVideoHeader& GetRtpVideoHeader() const;
 
   uint8_t* mutable_data() { return image_buffer_->data(); }
-
+#if defined(WEBRTC_WIN)
+  void SetBWETiming(double start_duration,
+      double last_duration,
+      int32_t packets_lost) {
+    bwe_stats_.start_duration_ = start_duration;
+    bwe_stats_.last_duration_ = last_duration;
+    bwe_stats_.packets_lost_ = packets_lost;
+  }
+#endif
  private:
   // Reference for mutable access.
   rtc::scoped_refptr<EncodedImageBuffer> image_buffer_;

modules/video_coding/packet_buffer.cc

@@ -99,7 +99,9 @@ PacketBuffer::InsertResult PacketBuffer::InsertPacket(
 
     first_seq_num_ = seq_num;
   }
-
+#if defined(WEBRTC_WIN)
+  QueryPerformanceCounter((LARGE_INTEGER*)&packet->time_ticks);
+#endif
   if (buffer_[index] != nullptr) {
     // Duplicate packet, just delete the payload.
     if (buffer_[index]->seq_num == packet->seq_num) {

modules/video_coding/packet_buffer.h

@@ -70,6 +70,9 @@ class PacketBuffer {
     RTPVideoHeader video_header;
 
     RtpPacketInfo packet_info;
+#if defined(WEBRTC_WIN)
+    int64_t time_ticks;
+#endif
   };
   struct InsertResult {
     std::vector<std::unique_ptr<Packet>> packets;

rtc_base/time/BUILD.gn

@@ -17,4 +17,8 @@ rtc_library("timestamp_extrapolator") {
     "timestamp_extrapolator.cc",
     "timestamp_extrapolator.h",
   ]
+
+  if(is_win) {
+    sources += [ "ptp_clock_sync.h" ]
+  }
 }

rtc_base/time/ptp_clock_sync.h

@@ -0,0 +1,57 @@
+// Copyright (C) <2021> Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+#ifndef RTC_BASE_TIME_PTP_CLOCK_SYNC_H_
+#define RTC_BASE_TIME_PTP_CLOCK_SYNC_H_
+
+#include <windows.h>
+#include <cstdint>
+
+#define MICROSECONDS_FACTOR 1000000.0
+#define OFFSET_FACTOR 200000
+#define SERVER_FREQUENCY 0.09  // RTP/NTP timestamp runs at 90KHz clock
+
+// A Windows implementation for PTP using timestamp from RTP and local
+// timestamp. We may need to implement something like QueryPerformanceFrequency
+// for this to work on Linux platforms.
+namespace webrtc {
+class PTPClockSync {
+ public:
+  PTPClockSync()
+      : m_server_point(0), m_server_freq(0), m_client_point(0), m_last_ts(0) {
+    uint64_t freq; // Performance counter frequency in a second.
+    QueryPerformanceFrequency((LARGE_INTEGER*)&freq);
+    m_client_freq = (double)freq / MICROSECONDS_FACTOR;
+    m_server_freq = SERVER_FREQUENCY;
+  }
+  ~PTPClockSync() {}
+
+  void Sync(uint32_t ts, uint64_t tc) {
+    if (GetDuration(ts, tc) < 0 ||
+        (ts - m_last_ts) > OFFSET_FACTOR * m_server_freq) {
+      UpdateSync(ts, tc);
+    }
+    m_last_ts = ts;
+  }
+
+  double GetDuration(uint32_t ts, uint64_t tc) {
+    int ds = (int)(ts - m_server_point);
+    int dc = (int)(tc - m_client_point);
+    return (double)dc / m_client_freq - (double)ds / m_server_freq;
+  }
+
+ protected:
+  uint32_t m_server_point;
+  double m_server_freq;  // count per us
+  uint64_t m_client_point;
+  double m_client_freq;  // count per us
+  uint32_t m_last_ts;
+
+  void UpdateSync(uint32_t ts, uint64_t tc) {
+    m_client_point = tc;
+    m_server_point = ts;
+  }
+};
+}  // namespace webrtc
+#endif

video/rtp_video_stream_receiver2.cc

@@ -729,6 +729,11 @@ void RtpVideoStreamReceiver2::OnInsertedPacket(
   std::vector<rtc::ArrayView<const uint8_t>> payloads;
   RtpPacketInfos::vector_type packet_infos;
 
+// Add timing information required by sender-side BWE.
+#if defined(WEBRTC_WIN)
+  int64_t max_tc = 0, min_tc = 0;
+  double start_duration = 0, last_duration = 0;
+#endif
   bool frame_boundary = true;
   for (auto& packet : result.packets) {
     // PacketBuffer promisses frame boundaries are correctly set on each
@@ -741,18 +746,30 @@ void RtpVideoStreamReceiver2::OnInsertedPacket(
       max_recv_time = packet->packet_info.receive_time_ms();
       payloads.clear();
       packet_infos.clear();
+#if defined(WEBRTC_WIN)
+      max_tc = min_tc = packet->time_ticks;
+#endif
     } else {
       max_nack_count = std::max(max_nack_count, packet->times_nacked);
       min_recv_time =
           std::min(min_recv_time, packet->packet_info.receive_time_ms());
       max_recv_time =
           std::max(max_recv_time, packet->packet_info.receive_time_ms());
+#if defined(WEBRTC_WIN)
+      max_tc = std::max(max_tc, packet->time_ticks);
+      min_tc = std::min(min_tc, packet->time_ticks);
+#endif
     }
     payloads.emplace_back(packet->video_payload);
     packet_infos.push_back(packet->packet_info);
 
     frame_boundary = packet->is_last_packet_in_frame();
     if (packet->is_last_packet_in_frame()) {
+#if defined(WEBRTC_WIN)
+      clock_sync_.Sync(packet->timestamp, min_tc);
+      start_duration = clock_sync_.GetDuration(packet->timestamp, min_tc);
+      last_duration = clock_sync_.GetDuration(packet->timestamp, max_tc);
+#endif
       auto depacketizer_it = payload_type_map_.find(first_packet->payload_type);
       RTC_CHECK(depacketizer_it != payload_type_map_.end());
 
@@ -764,25 +781,37 @@ void RtpVideoStreamReceiver2::OnInsertedPacket(
       }
 
       const video_coding::PacketBuffer::Packet& last_packet = *packet;
-      OnAssembledFrame(std::make_unique<video_coding::RtpFrameObject>(
-          first_packet->seq_num,                    //
-          last_packet.seq_num,                      //
-          last_packet.marker_bit,                   //
-          max_nack_count,                           //
-          min_recv_time,                            //
-          max_recv_time,                            //
-          first_packet->timestamp,                  //
-          first_packet->ntp_time_ms,                //
-          last_packet.video_header.video_timing,    //
-          first_packet->payload_type,               //
-          first_packet->codec(),                    //
-          last_packet.video_header.rotation,        //
-          last_packet.video_header.content_type,    //
-          first_packet->video_header,               //
-          last_packet.video_header.color_space,     //
-          RtpPacketInfos(std::move(packet_infos)),  //
-          std::move(bitstream)));
+      std::unique_ptr<video_coding::RtpFrameObject> frame =
+          std::make_unique<video_coding::RtpFrameObject>(
+              first_packet->seq_num,                    //
+              last_packet.seq_num,                      //
+              last_packet.marker_bit,                   //
+              max_nack_count,                           //
+              min_recv_time,                            //
+              max_recv_time,                            //
+              first_packet->timestamp,                  //
+              first_packet->ntp_time_ms,                //
+              last_packet.video_header.video_timing,    //
+              first_packet->payload_type,               //
+              first_packet->codec(),                    //
+              last_packet.video_header.rotation,        //
+              last_packet.video_header.content_type,    //
+              first_packet->video_header,               //
+              last_packet.video_header.color_space,     //
+              RtpPacketInfos(std::move(packet_infos)),  //
+              std::move(bitstream));
+#if defined(WEBRTC_WIN)
+      StreamStatistician* ss =
+          rtp_receive_statistics_->GetStatistician(config_.rtp.remote_ssrc);
+      int32_t packets_lost = 0;
+      if (ss != nullptr) {
+        packets_lost = ss->GetStats().packets_lost;
+        frame->SetBWETiming(start_duration, last_duration, packets_lost);
+      }
+#endif
+      OnAssembledFrame(std::move(frame));
     }
+
   }
   RTC_DCHECK(frame_boundary);
   if (result.buffer_cleared) {

video/rtp_video_stream_receiver2.h

@@ -42,6 +42,9 @@
 #include "rtc_base/constructor_magic.h"
 #include "rtc_base/experiments/field_trial_parser.h"
 #include "rtc_base/numerics/sequence_number_util.h"
+#if defined(WEBRTC_WIN)
+#include "rtc_base/time/ptp_clock_sync.h"
+#endif
 #include "rtc_base/synchronization/sequence_checker.h"
 #include "rtc_base/thread_annotations.h"
 #include "video/buffered_frame_decryptor.h"
@@ -271,6 +274,9 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender,
   void OnAssembledFrame(std::unique_ptr<video_coding::RtpFrameObject> frame);
 
   Clock* const clock_;
+#if defined(WEBRTC_WIN)
+  PTPClockSync clock_sync_;
+#endif
   // Ownership of this object lies with VideoReceiveStream, which owns |this|.
   const VideoReceiveStream::Config& config_;
   PacketRouter* const packet_router_;