issue: 1471639 Add TCP Rx timestamp support

Unlike UDP, each recv call may include multiple receive packets with
different timestamps. in this case the timestamp should be the
timestamp of the last byte.

Signed-off-by: Liran Oz <[email protected]>

src/vma/dev/cq_mgr.cpp

@@ -611,10 +611,10 @@ void cq_mgr::reclaim_recv_buffer_helper(mem_buf_desc_t* buff)
 				temp->rx.flow_tag_id = 0;
 				temp->rx.tcp.p_ip_h = NULL;
 				temp->rx.tcp.p_tcp_h = NULL;
-				temp->rx.sw_timestamp.tv_nsec = 0;
-				temp->rx.sw_timestamp.tv_sec = 0;
-				temp->rx.hw_timestamp.tv_nsec = 0;
-				temp->rx.hw_timestamp.tv_sec = 0;
+				temp->rx.timestamps.sw.tv_nsec = 0;
+				temp->rx.timestamps.sw.tv_sec = 0;
+				temp->rx.timestamps.hw.tv_nsec = 0;
+				temp->rx.timestamps.hw.tv_sec = 0;
 				temp->rx.hw_raw_timestamp = 0;
 				free_lwip_pbuf(&temp->lwip_pbuf);
 				m_rx_pool.push_back(temp);
@@ -647,10 +647,10 @@ void cq_mgr::socketxtreme_reclaim_recv_buffer_helper(mem_buf_desc_t* buff)
 				temp->rx.flow_tag_id = 0;
 				temp->rx.tcp.p_ip_h = NULL;
 				temp->rx.tcp.p_tcp_h = NULL;
-				temp->rx.sw_timestamp.tv_nsec = 0;
-				temp->rx.sw_timestamp.tv_sec = 0;
-				temp->rx.hw_timestamp.tv_nsec = 0;
-				temp->rx.hw_timestamp.tv_sec = 0;
+				temp->rx.timestamps.sw.tv_nsec = 0;
+				temp->rx.timestamps.sw.tv_sec = 0;
+				temp->rx.timestamps.hw.tv_nsec = 0;
+				temp->rx.timestamps.hw.tv_sec = 0;
 				temp->rx.hw_raw_timestamp = 0;
 				free_lwip_pbuf(&temp->lwip_pbuf);
 				m_rx_pool.push_back(temp);

src/vma/dev/ring_tap.cpp

@@ -667,10 +667,10 @@ bool ring_tap::reclaim_recv_buffers(mem_buf_desc_t *buff)
 				temp->rx.flow_tag_id = 0;
 				temp->rx.tcp.p_ip_h = NULL;
 				temp->rx.tcp.p_tcp_h = NULL;
-				temp->rx.sw_timestamp.tv_nsec = 0;
-				temp->rx.sw_timestamp.tv_sec = 0;
-				temp->rx.hw_timestamp.tv_nsec = 0;
-				temp->rx.hw_timestamp.tv_sec = 0;
+				temp->rx.timestamps.sw.tv_nsec = 0;
+				temp->rx.timestamps.sw.tv_sec = 0;
+				temp->rx.timestamps.hw.tv_nsec = 0;
+				temp->rx.timestamps.hw.tv_sec = 0;
 				temp->rx.hw_raw_timestamp = 0;
 				free_lwip_pbuf(&temp->lwip_pbuf);
 				m_rx_pool.push_back(temp);

src/vma/proto/mem_buf_desc.h

@@ -41,6 +41,12 @@
 
 class ring_slave;
 
+struct timestamps_t
+{
+	struct timespec sw;
+	struct timespec	hw;
+};
+
 /**
  * mem_buf_desc_t struct is used as the mapping of the wr_id in the wce to:
  * (1) p_desc_owner - to notify the owner of this mem_buf_desc of a completion of this WR
@@ -79,8 +85,7 @@ class mem_buf_desc_t {
 			iovec 		frag; // Datagram part base address and length
 			size_t		sz_payload; // This is the total amount of data of the packet, if (sz_payload>sz_data) means fragmented packet.
 			uint64_t	hw_raw_timestamp;
-			struct timespec sw_timestamp;
-			struct timespec	hw_timestamp;
+			timestamps_t	timestamps;
 			void* 		context;
 			uint32_t	flow_tag_id; // Flow Tag ID of this received packet
 

src/vma/sock/sockinfo.cpp

@@ -1524,15 +1524,15 @@ void sockinfo::process_timestamps(mem_buf_desc_t* p_desc)
 	if ((m_b_rcvtstamp ||
 		 (m_n_tsing_flags &
 		  (SOF_TIMESTAMPING_RX_SOFTWARE | SOF_TIMESTAMPING_SOFTWARE))) &&
-		!p_desc->rx.sw_timestamp.tv_sec) {
-		clock_gettime(CLOCK_REALTIME, &(p_desc->rx.sw_timestamp));
+		!p_desc->rx.timestamps.sw.tv_sec) {
+		clock_gettime(CLOCK_REALTIME, &(p_desc->rx.timestamps.sw));
 	}
 
 	// convert hw timestamp to system time
 	if (m_n_tsing_flags & SOF_TIMESTAMPING_RAW_HARDWARE) {
 		ring_simple* owner_ring = (ring_simple*) p_desc->p_desc_owner;
 		if (owner_ring) {
-			owner_ring->convert_hw_time_to_system_time(p_desc->rx.hw_raw_timestamp, &p_desc->rx.hw_timestamp);
+			owner_ring->convert_hw_time_to_system_time(p_desc->rx.hw_raw_timestamp, &p_desc->rx.timestamps.hw);
 		}
 	}
 }
@@ -1547,13 +1547,8 @@ void sockinfo::handle_recv_timestamping(struct cmsg_state *cm_state)
 
 	memset(&tsing, 0, sizeof(tsing));
 
-	mem_buf_desc_t* packet = get_front_m_rx_pkt_ready_list();
-	if (unlikely(!packet)) {
-		si_logdbg("m_rx_pkt_ready_list empty");
-		return ;
-	}
-
-	struct timespec* packet_systime = &packet->rx.sw_timestamp;
+	timestamps_t* packet_timestamps = get_socket_timestamps();
+	struct timespec* packet_systime = &packet_timestamps->sw;
 
 	// Only fill in SO_TIMESTAMPNS if both requested.
 	// This matches the kernel behavior.
@@ -1574,11 +1569,11 @@ void sockinfo::handle_recv_timestamping(struct cmsg_state *cm_state)
 	}
 
 	if (m_n_tsing_flags & SOF_TIMESTAMPING_SOFTWARE) {
-		tsing.systime = packet->rx.sw_timestamp;
+		tsing.systime = packet_timestamps->sw;
 	}
 
 	if (m_n_tsing_flags & SOF_TIMESTAMPING_RAW_HARDWARE) {
-		tsing.hwtimeraw = packet->rx.hw_timestamp;
+		tsing.hwtimeraw = packet_timestamps->hw;
 	}
 
 	insert_cmsg(cm_state, SOL_SOCKET, SO_TIMESTAMPING, &tsing, sizeof(tsing));

src/vma/sock/sockinfo.h

@@ -283,7 +283,8 @@ class sockinfo : public socket_fd_api, public pkt_rcvr_sink, public pkt_sndr_sou
 	virtual void            set_dst_entry_ttl() = 0;
 	virtual mem_buf_desc_t *get_next_desc (mem_buf_desc_t *p_desc) = 0;
 	virtual	mem_buf_desc_t* get_next_desc_peek(mem_buf_desc_t *p_desc, int& rx_pkt_ready_list_idx) = 0;
-
+	virtual timestamps_t* get_socket_timestamps() = 0;
+	virtual void          update_socket_timestamps(timestamps_t * ts) = 0;
 	virtual void 	post_deqeue (bool release_buff) = 0;
 
 	virtual int 	zero_copy_rx (iovec *p_iov, mem_buf_desc_t *pdesc, int *p_flags) = 0;
@@ -437,10 +438,11 @@ class sockinfo : public socket_fd_api, public pkt_rcvr_sink, public pkt_sndr_sou
 					m_rx_pkt_ready_offset += nbytes;
 					bytes_left -= nbytes;
 					iov_base = (uint8_t*)iov_base + nbytes;
+					if (m_b_rcvtstamp || m_n_tsing_flags) update_socket_timestamps(&pdesc->rx.timestamps);
 					if(bytes_left <= 0) {
 						if (unlikely(is_peek)) {
 							pdesc = get_next_desc_peek(pdesc, rx_pkt_ready_list_idx);
-						}else {
+						} else {
 							pdesc = get_next_desc(pdesc);
 						}
 						m_rx_pkt_ready_offset = 0;

src/vma/sock/sockinfo_tcp.cpp

@@ -240,6 +240,8 @@ sockinfo_tcp::sockinfo_tcp(int fd):
 	m_protocol = PROTO_TCP;
 	m_p_socket_stats->socket_type = SOCK_STREAM;
 
+	memset(&m_rx_timestamps, 0, sizeof(m_rx_timestamps));
+
 	m_sock_state = TCP_SOCK_INITED;
 	m_conn_state = TCP_CONN_INIT;
 	m_conn_timeout = CONNECT_DEFAULT_TIMEOUT_MS;
@@ -1568,6 +1570,7 @@ err_t sockinfo_tcp::rx_lwip_cb(void *arg, struct tcp_pcb *pcb,
 		p_curr_desc->rx.frag.iov_base = p_curr_buff->payload;
 		p_curr_desc->rx.frag.iov_len = p_curr_buff->len;
 		p_curr_desc->p_next_desc = (mem_buf_desc_t *)p_curr_buff->next;
+		conn->process_timestamps(p_curr_desc);
 		p_curr_buff = p_curr_buff->next;
 		p_curr_desc = p_curr_desc->p_next_desc;
 	}
@@ -1586,6 +1589,13 @@ err_t sockinfo_tcp::rx_lwip_cb(void *arg, struct tcp_pcb *pcb,
 		pkt_info.socket_ready_queue_pkt_count = conn->m_p_socket_stats->n_rx_ready_pkt_count;
 		pkt_info.socket_ready_queue_byte_count = conn->m_p_socket_stats->n_rx_ready_byte_count;
 
+		if (conn->m_n_tsing_flags & SOF_TIMESTAMPING_RAW_HARDWARE) {
+			pkt_info.hw_timestamp = p_first_desc->rx.timestamps.hw;
+		}
+		if (p_first_desc->rx.timestamps.sw.tv_sec) {
+			pkt_info.sw_timestamp = p_first_desc->rx.timestamps.sw;
+		}
+
 		// fill io vector array with data buffer pointers
 		iovec iov[p_first_desc->rx.n_frags];
 		nr_frags = 0;
@@ -1623,6 +1633,11 @@ err_t sockinfo_tcp::rx_lwip_cb(void *arg, struct tcp_pcb *pcb,
 			completion->packet.total_len = p->tot_len;
 			completion->src = p_first_desc->rx.src;
 			completion->packet.num_bufs = p_first_desc->rx.n_frags;
+
+			if (conn->m_n_tsing_flags & SOF_TIMESTAMPING_RAW_HARDWARE) {
+				completion->packet.hw_timestamp = p_first_desc->rx.timestamps.hw;
+			}
+
 			NOTIFY_ON_EVENTS(conn, VMA_SOCKETXTREME_PACKET);
 			conn->save_stats_rx_offload(completion->packet.total_len);
 		}
@@ -1805,6 +1820,7 @@ ssize_t sockinfo_tcp::rx(const rx_call_t call_type, iovec* p_iov, ssize_t sz_iov
 	si_tcp_logfunc("something in rx queues: %d %p", m_n_rx_pkt_ready_list_count, m_rx_pkt_ready_list.front());
 
 	total_rx = dequeue_packet(p_iov, sz_iov, (sockaddr_in *)__from, __fromlen, in_flags, &out_flags);
+	if (__msg) handle_cmsg(__msg);
 
 	/*
 	* RCVBUFF Accounting: Going 'out' of the internal buffer: if some bytes are not tcp_recved yet  - do that.
@@ -4069,6 +4085,11 @@ mem_buf_desc_t* sockinfo_tcp::get_next_desc_peek(mem_buf_desc_t *pdesc, int& rx_
 	return pdesc;
 }
 
+timestamps_t* sockinfo_tcp::get_socket_timestamps()
+{
+	return &m_rx_timestamps;
+}
+
 void sockinfo_tcp::post_deqeue(bool release_buff)
 {
 	NOT_IN_USE(release_buff);

src/vma/sock/sockinfo_tcp.h

@@ -216,6 +216,11 @@ class sockinfo_tcp : public sockinfo, public timer_handler
 		return m_sock_state == TCP_SOCK_ACCEPT_READY || m_sock_state == TCP_SOCK_ACCEPT_SHUT;
 	}
 
+	virtual void update_socket_timestamps(timestamps_t * ts)
+	{
+		m_rx_timestamps = *ts;
+	}
+
 	static const int CONNECT_DEFAULT_TIMEOUT_MS = 10000;
 	virtual inline fd_type_t get_type()
 	{
@@ -237,6 +242,7 @@ class sockinfo_tcp : public sockinfo, public timer_handler
 	//lwip specific things
 	struct tcp_pcb m_pcb;
 	socket_options_list_t m_socket_options_list;
+	timestamps_t m_rx_timestamps;
 	tcp_sock_offload_e m_sock_offload;
 	tcp_sock_state_e m_sock_state;
 	sockinfo_tcp *m_parent;
@@ -383,6 +389,7 @@ class sockinfo_tcp : public sockinfo, public timer_handler
 	inline void 	reuse_buffer(mem_buf_desc_t *buff);
 	virtual mem_buf_desc_t *get_next_desc(mem_buf_desc_t *p_desc);
 	virtual	mem_buf_desc_t* get_next_desc_peek(mem_buf_desc_t *p_desc, int& rx_pkt_ready_list_idx);
+	virtual timestamps_t* get_socket_timestamps();
 	virtual void 	post_deqeue(bool release_buff);
 	virtual int 	zero_copy_rx(iovec *p_iov, mem_buf_desc_t *pdesc, int *p_flags);
 	struct tcp_pcb* get_syn_received_pcb(const flow_tuple &key) const;

src/vma/sock/sockinfo_udp.cpp

@@ -1901,10 +1901,10 @@ inline vma_recv_callback_retval_t sockinfo_udp::inspect_by_user_cb(mem_buf_desc_
 	pkt_info.socket_ready_queue_byte_count = m_p_socket_stats->n_rx_ready_byte_count;
 
 	if (m_n_tsing_flags & SOF_TIMESTAMPING_RAW_HARDWARE) {
-		pkt_info.hw_timestamp = p_desc->rx.hw_timestamp;
+		pkt_info.hw_timestamp = p_desc->rx.timestamps.hw;
 	}
-	if (p_desc->rx.sw_timestamp.tv_sec) {
-		pkt_info.sw_timestamp = p_desc->rx.sw_timestamp;
+	if (p_desc->rx.timestamps.sw.tv_sec) {
+		pkt_info.sw_timestamp = p_desc->rx.timestamps.sw;
 	}
 
 	// fill io vector array with data buffer pointers
@@ -1941,7 +1941,7 @@ inline void sockinfo_udp::fill_completion(mem_buf_desc_t* p_desc)
 	completion->src = p_desc->rx.src;
 
 	if (m_n_tsing_flags & SOF_TIMESTAMPING_RAW_HARDWARE) {
-		completion->packet.hw_timestamp = p_desc->rx.hw_timestamp;
+		completion->packet.hw_timestamp = p_desc->rx.timestamps.hw;
 	}
 
 	for(mem_buf_desc_t *tmp_p=p_desc; tmp_p; tmp_p=tmp_p->p_next_desc) {
@@ -2490,6 +2490,15 @@ mem_buf_desc_t* sockinfo_udp::get_next_desc_peek(mem_buf_desc_t *p_desc, int& rx
 	return p_desc->p_next_desc;
 }
 
+timestamps_t* sockinfo_udp::get_socket_timestamps()
+{
+	if (unlikely(m_rx_pkt_ready_list.empty())) {
+		si_udp_logdbg("m_rx_pkt_ready_list empty");
+		return NULL;
+	}
+	return &m_rx_pkt_ready_list.front()->rx.timestamps;
+}
+
 void sockinfo_udp::post_deqeue(bool release_buff)
 {
 	mem_buf_desc_t *to_resue = m_rx_pkt_ready_list.get_and_pop_front();

src/vma/sock/sockinfo_udp.h

@@ -266,6 +266,8 @@ class sockinfo_udp : public sockinfo
 	virtual inline void			reuse_buffer(mem_buf_desc_t *buff);
 	virtual 	mem_buf_desc_t*	get_next_desc (mem_buf_desc_t *p_desc);
 	virtual		mem_buf_desc_t* get_next_desc_peek(mem_buf_desc_t *p_desc, int& rx_pkt_ready_list_idx);
+	virtual		timestamps_t* get_socket_timestamps();
+	virtual		void update_socket_timestamps(timestamps_t *) {};
 
 	inline bool	rx_process_udp_packet_full(mem_buf_desc_t* p_desc, void* pv_fd_ready_array);
 	inline bool	rx_process_udp_packet_partial(mem_buf_desc_t* p_desc, void* pv_fd_ready_array);