From 4e2fde989fef3a7a069cfc48ef1e4410db22c770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Falgueras=20Garc=C3=ADa?= Date: Mon, 28 Oct 2024 15:01:48 +0100 Subject: [PATCH 01/10] Implement rtph266pay This is a first draft of a RTP payloder for H.266. - [X]: parse NALUs (byte-stream) - [X]: UP - [X]: FU - [X]: Mark - [?]: Timestamps - [X]: EOS/Drain/Flush - [X]: _sink_event() - [X]: PAUSED state - [.]: XPS cache - [ ]: Latency - [ ]: Discont - [?]: Delta - [ ]: sprop - [ ]: AP - [ ]: DONL - [ ]: stream-format={vvc1,???} - [ ]: codec_data - [ ]: parse NALUs (vvc1) - [ ]: Trim trailing padding NOTE: This commit is not intended to be merged upstream, but to be squashed first with future fixups. --- subprojects/gst-plugins-good/gst/rtp/gstrtp.c | 1 + .../gst-plugins-good/gst/rtp/gstrtpelements.h | 1 + .../gst-plugins-good/gst/rtp/gstrtph266pay.c | 629 ++++++++++++++++++ .../gst-plugins-good/gst/rtp/gstrtph266pay.h | 33 + .../gst-plugins-good/gst/rtp/meson.build | 1 + 5 files changed, 665 insertions(+) create mode 100644 subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c create mode 100644 subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.h diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtp.c b/subprojects/gst-plugins-good/gst/rtp/gstrtp.c index 180a3f634dc..2724aeca21c 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtp.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtp.c @@ -77,6 +77,7 @@ plugin_init (GstPlugin * plugin) ret |= GST_ELEMENT_REGISTER (rtph264pay, plugin); ret |= GST_ELEMENT_REGISTER (rtph265depay, plugin); ret |= GST_ELEMENT_REGISTER (rtph265pay, plugin); + ret |= GST_ELEMENT_REGISTER (rtph266pay, plugin); ret |= GST_ELEMENT_REGISTER (rtpj2kdepay, plugin); ret |= GST_ELEMENT_REGISTER (rtpj2kpay, plugin); ret |= GST_ELEMENT_REGISTER (rtpjpegdepay, plugin); diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpelements.h b/subprojects/gst-plugins-good/gst/rtp/gstrtpelements.h index ec9663730e7..0db0d785a93 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpelements.h +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpelements.h @@ -77,6 +77,7 @@ GST_ELEMENT_REGISTER_DECLARE (rtph264depay); GST_ELEMENT_REGISTER_DECLARE (rtph264pay); GST_ELEMENT_REGISTER_DECLARE (rtph265depay); GST_ELEMENT_REGISTER_DECLARE (rtph265pay); +GST_ELEMENT_REGISTER_DECLARE (rtph266pay); GST_ELEMENT_REGISTER_DECLARE (rtpj2kdepay); GST_ELEMENT_REGISTER_DECLARE (rtpj2kpay); GST_ELEMENT_REGISTER_DECLARE (rtpjpegdepay); diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c new file mode 100644 index 00000000000..6f7b517b42f --- /dev/null +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c @@ -0,0 +1,629 @@ +/* GStreamer + * Copyright (C) <2024> Carlos Falgueras García + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/* references: + * - RTP Payload Format for Versatile Video Coding (VVC) + * https://www.ietf.org/rfc/rfc9328.txt + * - RTP: A Transport Protocol for Real-Time Applications + * https://www.ietf.org/rfc/rfc3550.txt + * - H.266: Versatile video coding + * https://www.itu.int/rec/T-REC-H.266-202309-I + */ + +#include "gstrtph266pay.h" +#include +#include +#include +#include +#include + +#define GST_CAT_DEFAULT rtph266pay_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +typedef enum +{ + NALU_TYPE_VPS = 14, + NALU_TYPE_SPS = 15, + NALU_TYPE_PPS = 16, + NALU_TYPE_PAPS = 17, + NALU_TYPE_SAPS = 18, + NALU_TYPE_AUD = 20, +} NaluType; + +#define NALU_SC_MSK 0xffffff00 +#define NALU_SC_VAL 0x00000100 +#define NALU_SC_LEN 3 +#define NALU_HDR_LEN 2 +#define NALU_INVALID_XPS 0xFF +#define NALU_IS_PARAMETER_SET(nalu) \ + (((nalu)->type >= NALU_TYPE_VPS) && ((nalu)->type <= NALU_TYPE_SAPS)) + +#define FU_TYPE 29 +#define FU_HDR_LEN (NALU_HDR_LEN + 1) // PayloadHdr + FU header + +typedef struct +{ + GstBuffer *nalu_buf; + GstBuffer *hdr_buf; + GstBuffer *rbsp_buf; + guint16 size; // Size of the NALU (inluding its header) + NaluType type; // nal_unit_type + guint8 xps_id; // {vps_video,sps_seq,pps_pic,aps_adaptation}_parameter_set_id + gboolean au_start; + gboolean au_end; +} Nalu; + +#define NALU_PTR_FORMAT \ + "p, size: %u, type: %u, xps_id: %u, au_start: %s, au_end: %s" +#define NALU_ARGS(nalu) \ + (nalu), (nalu)->size, (nalu)->type, (nalu)->xps_id, (nalu)->au_start ? "true" : "false", (nalu)->au_end ? "true" : "false" + +typedef enum +{ + ALIGNMENT_AU, + ALIGNMENT_NAL, + ALIGNMENT_UNKOWN, +} Alignment; + +struct _GstRtpH266Pay +{ + GstRTPBasePayload payload; + + GstAdapter *adapter; + GQueue nalus; + Alignment alignment; +}; + +#define gst_rtp_h266_pay_parent_class parent_class +G_DEFINE_TYPE (GstRtpH266Pay, gst_rtp_h266_pay, GST_TYPE_RTP_BASE_PAYLOAD); +GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (rtph266pay, "rtph266pay", + GST_RANK_SECONDARY, GST_TYPE_RTP_H266_PAY, rtp_element_init (plugin)); + +static GstStaticPadTemplate sink_template = +GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, + GST_STATIC_CAPS ("video/x-h266, stream-format = (string) byte-stream, " + "alignment = (string) { nal, au }")); + +static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("application/x-rtp, " + "media = (string) \"video\", " + "payload = (int) " GST_RTP_PAYLOAD_DYNAMIC_STRING ", " + "clock-rate = (int) 90000, " "encoding-name = (string) \"H266\"") + ); + +enum +{ + PROP_0, +}; + +// GObject methods +static void _finalize (GObject * object); +static void _set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static void _get_property (GObject * object, guint prop_id, GValue * value, + GParamSpec * pspec); + +// GstElement methods +static GstStateChangeReturn _change_state (GstElement * element, + GstStateChange transition); + +// GstRTPBasePayload methods +static gboolean _set_caps (GstRTPBasePayload * rtpbasepay, GstCaps * caps); +static GstFlowReturn _handle_buffer (GstRTPBasePayload * rtpbasepay, + GstBuffer * buffer); +static gboolean _sink_event (GstRTPBasePayload * rtpbasepay, GstEvent * event); + +// GstRtpH266Pay methods +static void _process_nalu (GstRtpH266Pay * rtph266pay, GstBuffer * nalu_buf); +static void _set_au_boundaries (GstRtpH266Pay * rtph266pay); +static GstFlowReturn _push_pending_data (GstRtpH266Pay * rtph266pay, + gboolean eos); +static GstFlowReturn _push_up (GstRtpH266Pay * rtph266pay, const Nalu * nalu); +static GstFlowReturn _push_fu (GstRtpH266Pay * rtph266pay, const Nalu * nalu); +//static GstFlowReturn _push_ap (GstRtpH266Pay * rtph266pay, GQueue *nalus); +static gboolean _up_fits_in_mtu (const GstRtpH266Pay * rtph266pay, + const Nalu * nalu); +static GstBuffer *_extract_fu (GstRtpH266Pay * rtph266pay, const Nalu * nalu, + gsize offset, gsize size, gboolean last); +static void _clear_nalu_queue (GstRtpH266Pay * rtph266pay); + +// Nalu methods +static Nalu *_nalu_new (GstBuffer * nalu_buf); +static void _nalu_free (Nalu * nalu); + +static void +gst_rtp_h266_pay_class_init (GstRtpH266PayClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *gstelement_class; + GstRTPBasePayloadClass *gstrtpbasepayload_class; + + gobject_class = (GObjectClass *) klass; + gstelement_class = (GstElementClass *) klass; + gstrtpbasepayload_class = (GstRTPBasePayloadClass *) klass; + + gobject_class->finalize = GST_DEBUG_FUNCPTR (_finalize); + gobject_class->set_property = GST_DEBUG_FUNCPTR (_set_property); + gobject_class->get_property = GST_DEBUG_FUNCPTR (_get_property); + + gstelement_class->change_state = GST_DEBUG_FUNCPTR (_change_state); + + gstrtpbasepayload_class->set_caps = GST_DEBUG_FUNCPTR (_set_caps); + gstrtpbasepayload_class->handle_buffer = GST_DEBUG_FUNCPTR (_handle_buffer); + gstrtpbasepayload_class->sink_event = GST_DEBUG_FUNCPTR (_sink_event); + + gst_element_class_add_static_pad_template (gstelement_class, &src_template); + gst_element_class_add_static_pad_template (gstelement_class, &sink_template); + + gst_element_class_set_static_metadata (gstelement_class, "RTP H266 payloader", + "Codec/Payloader/Network/RTP", + "Payload-encode H266 video into RTP packets (RFC 9328)", + "Carlos Falgueras García "); + + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "rtph266pay", 0, + "H266 RTP Payloader"); +} + +static void +gst_rtp_h266_pay_init (GstRtpH266Pay * rtph266pay) +{ + rtph266pay->adapter = gst_adapter_new (); + g_queue_init (&rtph266pay->nalus); + rtph266pay->alignment = ALIGNMENT_UNKOWN; +} + +static void +_finalize (GObject * object) +{ + GstRtpH266Pay *rtph266pay = GST_RTP_H266_PAY (object); + + g_clear_pointer (&rtph266pay->adapter, g_object_unref); + _clear_nalu_queue (rtph266pay); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ +} + +static void +_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ +} + +static GstStateChangeReturn +_change_state (GstElement * element, GstStateChange transition) +{ + GstRtpH266Pay *rtph266pay = GST_RTP_H266_PAY (element); + GstStateChangeReturn ret; + + if (transition == GST_STATE_CHANGE_READY_TO_PAUSED) { + gst_adapter_clear (rtph266pay->adapter); + _clear_nalu_queue (rtph266pay); + } + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + + return ret; +} + +static gboolean +_set_caps (GstRTPBasePayload * rtpbasepay, GstCaps * caps) +{ + GstRtpH266Pay *rtph266pay = GST_RTP_H266_PAY (rtpbasepay); + const gchar *alignment_str; + GstStructure *s; + + gst_rtp_base_payload_set_options (rtpbasepay, "video", TRUE, "H266", 90000); + + s = gst_caps_get_structure (caps, 0); + g_assert (s); + alignment_str = gst_structure_get_string (s, "alignment"); + if (alignment_str) { + if (g_str_equal (alignment_str, "au")) { + rtph266pay->alignment = ALIGNMENT_AU; + } else if (g_str_equal (alignment_str, "nal")) { + rtph266pay->alignment = ALIGNMENT_NAL; + } else { + rtph266pay->alignment = ALIGNMENT_UNKOWN; + } + } + + return TRUE; +} + +static GstFlowReturn +_handle_buffer (GstRTPBasePayload * rtpbasepay, GstBuffer * buffer) +{ + GstRtpH266Pay *rtph266pay = GST_RTP_H266_PAY (rtpbasepay); + GstAdapter *adapter = rtph266pay->adapter; + gsize adapter_size; + + GST_DEBUG_OBJECT (rtph266pay, "New buffer %" GST_PTR_FORMAT, buffer); + gst_adapter_push (adapter, buffer); + + // Advance until the first start code to skip heading zeroes + adapter_size = gst_adapter_available (adapter); + gssize next_nalu_start = + gst_adapter_masked_scan_uint32 (adapter, NALU_SC_MSK, NALU_SC_VAL, 0, + adapter_size); + if (next_nalu_start < 0) { + GST_WARNING_OBJECT (rtph266pay, "Not NALU found"); + gst_adapter_flush (adapter, adapter_size); + return GST_FLOW_OK; + } + gst_adapter_flush (adapter, next_nalu_start); + + // For each NALU + while ((adapter_size = gst_adapter_available (adapter)) > NALU_SC_LEN) { + // Find next start code, skipping the actual one + gssize next_nalu_start = + gst_adapter_masked_scan_uint32 (adapter, NALU_SC_MSK, NALU_SC_VAL, + NALU_SC_LEN, adapter_size - NALU_SC_LEN); + + // If no start code found, the length of the NALU is the remaining size + gssize nalu_len = (next_nalu_start < 0) ? adapter_size : next_nalu_start; + + if (nalu_len <= NALU_SC_LEN) { + GST_WARNING_OBJECT (rtph266pay, "NALU too small %ld, skipping it", + nalu_len); + gst_adapter_flush (adapter, nalu_len); + continue; + } + + // Remove the start code + nalu_len -= NALU_SC_LEN; // Here, it'll always be > 0 + gst_adapter_flush (adapter, NALU_SC_LEN); + + GstBuffer *nalu_buf = gst_adapter_take_buffer (adapter, nalu_len); + // nalu_buf will never be aligned with an incoming buffer, therefore + // gst_adapter_take_buffer() will never set its timestamps. + GST_BUFFER_PTS (nalu_buf) = gst_adapter_prev_pts (adapter, NULL); + GST_BUFFER_DTS (nalu_buf) = gst_adapter_prev_dts (adapter, NULL); + _process_nalu (rtph266pay, nalu_buf); + } + + return _push_pending_data (rtph266pay, FALSE); +} + +static gboolean +_sink_event (GstRTPBasePayload * rtpbasepay, GstEvent * event) +{ + GstRtpH266Pay *rtph266pay = GST_RTP_H266_PAY (rtpbasepay); + GstFlowReturn ret = GST_FLOW_OK; + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_FLUSH_STOP: + gst_adapter_clear (rtph266pay->adapter); + _clear_nalu_queue (rtph266pay); + break; + case GST_EVENT_EOS: + GST_DEBUG_OBJECT (rtph266pay, "EOS: Draining"); + ret = _push_pending_data (rtph266pay, TRUE); + break; + default: + break; + } + + if (ret != GST_FLOW_OK) + return FALSE; + + return GST_RTP_BASE_PAYLOAD_CLASS (parent_class)->sink_event (rtpbasepay, + event); +} + +static void +_process_nalu (GstRtpH266Pay * rtph266pay, GstBuffer * nalu_buf) +{ + Nalu *nalu; + + g_assert (nalu_buf); + + // Parse NALU + nalu = _nalu_new (nalu_buf); + if (!nalu) { + GST_WARNING_OBJECT (rtph266pay, + "Couldn't decode NALU %" GST_PTR_FORMAT ". Dropping it.", nalu_buf); + return; + } + GST_DEBUG_OBJECT (rtph266pay, "NALU decoded: %" NALU_PTR_FORMAT, + NALU_ARGS (nalu)); + + g_queue_push_tail (&rtph266pay->nalus, nalu); +} + +static void +_set_au_boundaries (GstRtpH266Pay * rtph266pay) +{ + GList *head = g_queue_peek_head_link (&rtph266pay->nalus); + Nalu *last_nalu = g_queue_peek_tail (&rtph266pay->nalus); + + for (GList * l = head; l != NULL; l = l->next) { + Nalu *nalu = l->data; + Nalu *prev_nalu = l->prev ? l->prev->data : NULL; + // TODO gboolean discont = GST_BUFFER_IS_DISCONT (nalu->nalu_buf); + GstClockTime prev_pts = GST_CLOCK_TIME_NONE; + GstClockTime prev_dts = GST_CLOCK_TIME_NONE; + GstClockTime pts = GST_BUFFER_PTS (nalu->nalu_buf); + GstClockTime dts = GST_BUFFER_DTS (nalu->nalu_buf); + if (prev_nalu) { + prev_pts = GST_BUFFER_PTS (prev_nalu->nalu_buf); + prev_dts = GST_BUFFER_DTS (prev_nalu->nalu_buf); + } + gboolean aud = nalu->type == NALU_TYPE_AUD; + gboolean new_ts = (prev_pts != pts) || (prev_dts != dts); + + nalu->au_start = aud || new_ts /*|| discont */ ; + if (prev_nalu && nalu->au_start) { + prev_nalu->au_end = TRUE; + GST_DEBUG_OBJECT (rtph266pay, "AU start found -> previous AU finished"); + } + } + + // In some cases, we already know where is the end of an AU. Mark it now to + // avoid waiting for the next NALU. + if (last_nalu) { + gboolean au_alignment = rtph266pay->alignment == ALIGNMENT_AU; + gboolean marker = + GST_BUFFER_FLAG_IS_SET (last_nalu->nalu_buf, GST_BUFFER_FLAG_MARKER); + last_nalu->au_end = last_nalu->au_end || au_alignment || marker; + } +} + +static GstFlowReturn +_push_pending_data (GstRtpH266Pay * rtph266pay, gboolean eos) +{ + GQueue *nalus = &rtph266pay->nalus; + GstFlowReturn ret = GST_FLOW_OK; + Nalu *nalu; + + // TODO: handle AP + // TODO: send XPS if needed + + // TODO: Maybe it's worth to avoid iterating the NALU list twice, maybe not + _set_au_boundaries (rtph266pay); + + // Try to push all NALUs + while ((nalu = g_queue_pop_head (nalus))) { + gboolean is_last = g_queue_get_length (nalus) == 0; + + // Can't push the last NALU without knowing if it's the end of an AU, + // because setting the M bit could be necessary. But have to push it if + // we're on EOS + if (!eos && is_last && !nalu->au_end) { + GST_DEBUG_OBJECT (rtph266pay, "Keeping last NALU: %" NALU_PTR_FORMAT, + NALU_ARGS (nalu)); + g_queue_push_head (nalus, nalu); + break; + } + + GST_DEBUG_OBJECT (rtph266pay, "Pushing NALU: %" NALU_PTR_FORMAT, + NALU_ARGS (nalu)); + + if (_up_fits_in_mtu (rtph266pay, nalu)) + ret = _push_up (rtph266pay, nalu); + else + ret = _push_fu (rtph266pay, nalu); + + _nalu_free (nalu); + if (ret != GST_FLOW_OK) + break; + } + + return ret; +} + +static GstFlowReturn +_push_up (GstRtpH266Pay * rtph266pay, const Nalu * nalu) +{ + GstRTPBasePayload *rtpbasepay = GST_RTP_BASE_PAYLOAD (rtph266pay); + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + GstBuffer *out_buf; + + // Allocate just the RTP header. We'll add the buffer payload later. This way + // we avoid unnecessary copies + out_buf = gst_rtp_base_payload_allocate_output_buffer (rtpbasepay, 0, 0, 0); + if (!gst_rtp_buffer_map (out_buf, GST_MAP_WRITE, &rtp)) + goto error; + + // Copy buffer metadata + gst_buffer_copy_into (out_buf, nalu->nalu_buf, + GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS, 0, -1); + gst_rtp_buffer_set_marker (&rtp, nalu->au_end); + + // Append the payload to the output buffer + out_buf = gst_buffer_append (out_buf, gst_buffer_ref (nalu->hdr_buf)); + // TODO: Conditionally append DONL + out_buf = gst_buffer_append (out_buf, gst_buffer_ref (nalu->rbsp_buf)); + g_assert (out_buf); + + gst_rtp_buffer_unmap (&rtp); + GST_DEBUG_OBJECT (rtph266pay, "Pushing UP %" GST_PTR_FORMAT, out_buf); + return gst_rtp_base_payload_push (rtpbasepay, out_buf); +error: + gst_buffer_unref (out_buf); + return GST_FLOW_ERROR; +} + +static GstFlowReturn +_push_fu (GstRtpH266Pay * rtph266pay, const Nalu * nalu) +{ + GstRTPBasePayload *rtpbasepay = GST_RTP_BASE_PAYLOAD (rtph266pay); + GstBufferList *out_buflist = gst_buffer_list_new (); + + // TODO: Consider DONL + guint mtu = GST_RTP_BASE_PAYLOAD_MTU (rtpbasepay); + guint rtp_pyl_max_size = gst_rtp_buffer_calc_payload_len (mtu, 0, 0); + guint fu_pyl_max_size = rtp_pyl_max_size - FU_HDR_LEN; + gsize rbsp_size = nalu->size - NALU_HDR_LEN; + g_assert_cmpuint (fu_pyl_max_size, <, rbsp_size); + + // Split the NALU into several FU + for (gsize offset = 0; offset < rbsp_size; offset += fu_pyl_max_size) { + gsize remaining = rbsp_size - offset; + gboolean last = remaining <= fu_pyl_max_size; + gsize size = last ? remaining : fu_pyl_max_size; + + GstBuffer *fu_buf = _extract_fu (rtph266pay, nalu, offset, size, last); + GST_DEBUG_OBJECT (rtph266pay, "FU [%lu, %lu] %" GST_PTR_FORMAT, + offset, offset + size, fu_buf); + gst_buffer_list_add (out_buflist, fu_buf); + } + + GST_DEBUG_OBJECT (rtph266pay, "Pushing FU GstBufferList %" GST_PTR_FORMAT, + out_buflist); + return gst_rtp_base_payload_push_list (rtpbasepay, out_buflist); +} + +static gboolean +_up_fits_in_mtu (const GstRtpH266Pay * rtph266pay, const Nalu * nalu) +{ + guint mtu = GST_RTP_BASE_PAYLOAD_MTU (rtph266pay); + guint payload_size = nalu->size; // TODO: Consider DONL + gboolean fits = gst_rtp_buffer_calc_packet_len (payload_size, 0, 0) <= mtu; + + if (!fits) { + GST_DEBUG_OBJECT (rtph266pay, + "NALU does not fit into %u MTU: %" GST_PTR_FORMAT, mtu, nalu->nalu_buf); + } + return fits; +} + +static GstBuffer * +_extract_fu (GstRtpH266Pay * rtph266pay, const Nalu * nalu, + gsize offset, gsize size, gboolean last) +{ + GstRTPBasePayload *rtpbasepay = GST_RTP_BASE_PAYLOAD (rtph266pay); + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + gboolean au_end = last && nalu->au_end; + gboolean first = (offset == 0); + GstBuffer *fu_pyl_buf; + GstBuffer *fu_buf; + guint8 *fu_hdr; + + fu_pyl_buf = gst_buffer_copy_region (nalu->rbsp_buf, GST_BUFFER_COPY_MEMORY, + offset, size); + g_assert (fu_pyl_buf); + + // Allocate just the RTP header + PayloadHdr + FU header. We'll add the buffer + // payload later. This way we avoid unnecessary copies + fu_buf = + gst_rtp_base_payload_allocate_output_buffer (rtpbasepay, FU_HDR_LEN, 0, + 0); + gboolean ok = gst_rtp_buffer_map (fu_buf, GST_MAP_WRITE, &rtp); + g_assert (ok); + + // Copy required buffer metadata + gst_buffer_copy_into (fu_buf, nalu->nalu_buf, + GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS, 0, -1); + gst_rtp_buffer_set_marker (&rtp, au_end); + + fu_hdr = gst_rtp_buffer_get_payload (&rtp); + g_assert (fu_hdr); + + // Setup PayloadHdr and FU Header + // | PayloadHdr (NALU HDR) | FU HEADER | + // |-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-|-+-+-+-+-+-+-+-| + // |F|Z| LayerID | Type(29)| TID |S|E|P| FuType | + // +---------------+---------------|---------------+ + guint8 read = gst_buffer_extract (nalu->hdr_buf, 0, fu_hdr, NALU_HDR_LEN); + g_assert_cmpuint (read, >=, NALU_HDR_LEN); + fu_hdr[1] = (FU_TYPE << 3) | (fu_hdr[1] & 0x07); // set Type = 29 + fu_hdr[2] = (!!first << 7) | (!!last << 6) | (!!au_end << 5) | nalu->type; + + gst_rtp_buffer_unmap (&rtp); + + // TODO: Conditionally append DONL + fu_buf = gst_buffer_append (fu_buf, fu_pyl_buf); + g_assert (fu_buf); + + return fu_buf; +} + +static void +_clear_nalu_queue (GstRtpH266Pay * rtph266pay) +{ + g_queue_clear_full (&rtph266pay->nalus, (GDestroyNotify) _nalu_free); +} + +static Nalu * +_nalu_new (GstBuffer * nalu_buf) +{ + // | NALU HDR | 2 first bytes of XPS + // |-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-|-+-+-+-+-+-+-+-:-+-+-+-+-+-+-+-:-+-+ + // |F|Z| LayerID | Type | TID | [VPS, PPS, SPS, PAPS, SAPS] : ... + // +---------------+---------------|---------------:---------------:---- + // VPS/SPS: |X X X X . . . .:. . . . . . . .: + // PPS: |X X X X X X . .:. . . . . . . .: + // PAPS/SAPS: |. . . . . . X X:X X . . . . . .: + guint8 data[NALU_HDR_LEN + 2]; + Nalu *nalu; + + if (gst_buffer_extract (nalu_buf, 0, data, sizeof (data)) < sizeof (data)) { + gst_buffer_unref (nalu_buf); + return NULL; + } + + nalu = g_new (Nalu, 1); + nalu->size = gst_buffer_get_size (nalu_buf); + nalu->type = data[1] >> 3; + switch (nalu->type) { + case NALU_TYPE_VPS: + case NALU_TYPE_SPS: + nalu->xps_id = data[2] >> 4; + break; + case NALU_TYPE_PPS: + nalu->xps_id = data[2] >> 2; + break; + case NALU_TYPE_PAPS: + case NALU_TYPE_SAPS: + nalu->xps_id = ((data[2] & 0x03) << 2) | ((data[3] & 0xC0) >> 6); + break; + default: + nalu->xps_id = NALU_INVALID_XPS; + } + + nalu->nalu_buf = nalu_buf; + nalu->hdr_buf = + gst_buffer_copy_region (nalu_buf, GST_BUFFER_COPY_MEMORY, 0, + NALU_HDR_LEN); + nalu->rbsp_buf = + gst_buffer_copy_region (nalu_buf, GST_BUFFER_COPY_MEMORY, NALU_HDR_LEN, + -1); + + nalu->au_start = FALSE; + nalu->au_end = FALSE; + + return nalu; +} + +static void +_nalu_free (Nalu * nalu) +{ + gst_buffer_unref (nalu->nalu_buf); + gst_buffer_unref (nalu->hdr_buf); + gst_buffer_unref (nalu->rbsp_buf); + g_free (nalu); +} diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.h b/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.h new file mode 100644 index 00000000000..70276df980c --- /dev/null +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.h @@ -0,0 +1,33 @@ +/* GStreamer + * Copyright (C) <2024> Carlos Falgueras García + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_RTP_H266_PAY_H__ +#define __GST_RTP_H266_PAY_H__ + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_RTP_H266_PAY gst_rtp_h266_pay_get_type() +G_DECLARE_FINAL_TYPE(GstRtpH266Pay, gst_rtp_h266_pay, GST, RTP_H266_PAY, + GstRTPBasePayload); + +G_END_DECLS +#endif /* __GST_RTP_H266_PAY_H__ */ diff --git a/subprojects/gst-plugins-good/gst/rtp/meson.build b/subprojects/gst-plugins-good/gst/rtp/meson.build index 000b91d7ef6..18c6848227d 100644 --- a/subprojects/gst-plugins-good/gst/rtp/meson.build +++ b/subprojects/gst-plugins-good/gst/rtp/meson.build @@ -54,6 +54,7 @@ rtp_sources = [ 'gstrtph264pay.c', 'gstrtph265depay.c', 'gstrtph265pay.c', + 'gstrtph266pay.c', 'gstrtpj2kdepay.c', 'gstrtpj2kpay.c', 'gstrtpjpegdepay.c', From 5fd4ee8b2ecf67001ca006e8ef1be9e9038a48cc Mon Sep 17 00:00:00 2001 From: Fabian Orccon Date: Thu, 21 Nov 2024 18:59:43 +0100 Subject: [PATCH 02/10] rtp: Add rtph266depay element --- subprojects/gst-plugins-good/gst/rtp/gstrtp.c | 1 + .../gst-plugins-good/gst/rtp/gstrtpelements.h | 1 + .../gst/rtp/gstrtph266depay.c | 670 ++++++++++++++++++ .../gst/rtp/gstrtph266depay.h | 102 +++ .../gst-plugins-good/gst/rtp/meson.build | 1 + 5 files changed, 775 insertions(+) create mode 100644 subprojects/gst-plugins-good/gst/rtp/gstrtph266depay.c create mode 100644 subprojects/gst-plugins-good/gst/rtp/gstrtph266depay.h diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtp.c b/subprojects/gst-plugins-good/gst/rtp/gstrtp.c index 2724aeca21c..e76cbf99085 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtp.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtp.c @@ -77,6 +77,7 @@ plugin_init (GstPlugin * plugin) ret |= GST_ELEMENT_REGISTER (rtph264pay, plugin); ret |= GST_ELEMENT_REGISTER (rtph265depay, plugin); ret |= GST_ELEMENT_REGISTER (rtph265pay, plugin); + ret |= GST_ELEMENT_REGISTER (rtph266depay, plugin); ret |= GST_ELEMENT_REGISTER (rtph266pay, plugin); ret |= GST_ELEMENT_REGISTER (rtpj2kdepay, plugin); ret |= GST_ELEMENT_REGISTER (rtpj2kpay, plugin); diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpelements.h b/subprojects/gst-plugins-good/gst/rtp/gstrtpelements.h index 0db0d785a93..1ab6daedc38 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpelements.h +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpelements.h @@ -77,6 +77,7 @@ GST_ELEMENT_REGISTER_DECLARE (rtph264depay); GST_ELEMENT_REGISTER_DECLARE (rtph264pay); GST_ELEMENT_REGISTER_DECLARE (rtph265depay); GST_ELEMENT_REGISTER_DECLARE (rtph265pay); +GST_ELEMENT_REGISTER_DECLARE (rtph266depay); GST_ELEMENT_REGISTER_DECLARE (rtph266pay); GST_ELEMENT_REGISTER_DECLARE (rtpj2kdepay); GST_ELEMENT_REGISTER_DECLARE (rtpj2kpay); diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph266depay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtph266depay.c new file mode 100644 index 00000000000..e8504884e98 --- /dev/null +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph266depay.c @@ -0,0 +1,670 @@ +/* GStreamer + * Copyright (C) <2006> Wim Taymans + * Copyright (C) <2014> Jurgen Slowack + * Copyright (C) <2024> César Fabián Orccón Chipana + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/* references: + * - RTP Payload Format for Versatile Video Coding (VVC) + * https://www.ietf.org/rfc/rfc9328.txt + * - RTP: A Transport Protocol for Real-Time Applications + * https://www.ietf.org/rfc/rfc3550.txt + * - H.266: Versatile video coding + * https://www.itu.int/rec/T-REC-H.266-202309-I + */ + +#include "gstrtph266depay.h" +#include +#include +#include +#include + +#define NAL_TYPE_IS_PARAMETER_SET(nt) (((nt) == GST_H266_NAL_VPS)\ + || ((nt) == GST_H266_NAL_SPS)\ + || ((nt) == GST_H266_NAL_PPS) ) +#define NAL_TYPE_IS_KEY(nt) (NAL_TYPE_IS_PARAMETER_SET(nt)) + +#define DEFAULT_CONFIG_INTERVAL 0 +#define DEFAULT_CLOCK_RATE 90000 + +#define GST_CAT_DEFAULT rtph266depay_debug +#define gst_rtp_h266_depay_parent_class parent_class + +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); +G_DEFINE_TYPE (GstRtpH266Depay, gst_rtp_h266_depay, + GST_TYPE_RTP_BASE_DEPAYLOAD); +GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (rtph266depay, "rtph266depay", + GST_RANK_SECONDARY, GST_TYPE_RTP_H266_DEPAY, rtp_element_init (plugin)); + +static GstStaticPadTemplate gst_rtp_h266_depay_src_template = +GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS + ("video/x-h266, stream-format = (string) byte-stream, alignment = (string) au ")); + +static GstStaticPadTemplate gst_rtp_h266_depay_sink_template = +GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("application/x-rtp, " + "media = (string) \"video\", " + "clock-rate = (int) 90000, " "encoding-name = (string) \"H266\"")); + +enum +{ + PROP_0, +}; + +/* 3 zero bytes syncword */ +static const guint8 sync_bytes[] = { 0, 0, 0, 1 }; + +static gboolean +gst_rtp_h266_depay_negotiate (GstRtpH266Depay * rtph266depay) +{ + GstCaps *caps; + gboolean ret = FALSE; + + caps = + gst_pad_get_allowed_caps (GST_RTP_BASE_DEPAYLOAD_SRCPAD (rtph266depay)); + + GST_DEBUG_OBJECT (rtph266depay, "allowed caps: %" GST_PTR_FORMAT, caps); + + if (!caps) { + GST_ERROR_OBJECT (rtph266depay, "Caps not found."); + return ret; + } + + if (gst_caps_get_size (caps) > 0) { + GstStructure *s = gst_caps_get_structure (caps, 0); + const gchar *str; + + GST_DEBUG_OBJECT (rtph266depay, "get stream-format"); + str = gst_structure_get_string (s, "stream-format"); + if (g_strcmp0 (str, "byte-stream") != 0) { + GST_ERROR_OBJECT (rtph266depay, "only byte-stream supported: %s", str); + goto beach; + } + + GST_DEBUG_OBJECT (rtph266depay, "st: %" GST_PTR_FORMAT, s); + + + GST_DEBUG_OBJECT (rtph266depay, "get alignment"); + str = gst_structure_get_string (s, "alignment"); + if (g_strcmp0 (str, "au") == 0) { + rtph266depay->alignment = GST_H266_ALIGNMENT_AU; + } else { + GST_ERROR_OBJECT (rtph266depay, "alignment not supported: %s", str); + goto beach; + } + } + + ret = TRUE; + +beach: + gst_caps_unref (caps); + return ret; +} + +static GstBuffer * +gst_rtp_h266_depay_allocate_output_buffer (GstRtpH266Depay * depay, gsize size) +{ + GstBuffer *buffer = NULL; + + GST_LOG_OBJECT (depay, "want output buffer of %u bytes", (guint) size); + + g_return_val_if_fail (size > 0, NULL); + + // TODO: Forward allocator. + buffer = gst_buffer_new_allocate (NULL, size, NULL); + + return buffer; +} + +static GstBuffer * +gst_rtp_h266_complete_au (GstRtpH266Depay * rtph266depay, + GstClockTime * out_timestamp, gboolean * out_keyframe) +{ + GstBufferList *list; + GstMapInfo outmap; + GstBuffer *outbuf; + guint outsize, offset = 0; + gint b, n_bufs, m, n_mem; + + /* we had a picture in the adapter and we completed it */ + GST_DEBUG_OBJECT (rtph266depay, "taking completed AU"); + outsize = gst_adapter_available (rtph266depay->picture_adapter); + + GST_DEBUG_OBJECT (rtph266depay, "will allocate buffer of size %d", outsize); + outbuf = gst_rtp_h266_depay_allocate_output_buffer (rtph266depay, outsize); + + if (G_UNLIKELY (outbuf == NULL)) + return NULL; + + if (!gst_buffer_map (outbuf, &outmap, GST_MAP_WRITE)) + return NULL; + + list = gst_adapter_take_buffer_list (rtph266depay->picture_adapter, outsize); + + n_bufs = gst_buffer_list_length (list); + for (b = 0; b < n_bufs; ++b) { + GstBuffer *buf = gst_buffer_list_get (list, b); + + n_mem = gst_buffer_n_memory (buf); + for (m = 0; m < n_mem; ++m) { + GstMemory *mem = gst_buffer_peek_memory (buf, m); + gsize mem_size = gst_memory_get_sizes (mem, NULL, NULL); + GstMapInfo mem_map; + + if (gst_memory_map (mem, &mem_map, GST_MAP_READ)) { + memcpy (outmap.data + offset, mem_map.data, mem_size); + gst_memory_unmap (mem, &mem_map); + } else { + memset (outmap.data + offset, 0, mem_size); + } + offset += mem_size; + } + + gst_rtp_copy_video_meta (rtph266depay, outbuf, buf); + } + gst_buffer_list_unref (list); + gst_buffer_unmap (outbuf, &outmap); + + *out_timestamp = rtph266depay->last_ts; + *out_keyframe = rtph266depay->last_keyframe; + + rtph266depay->last_keyframe = FALSE; + + return outbuf; +} + + +static void +gst_rtp_h266_depay_push (GstRtpH266Depay * rtph266depay, GstBuffer * outbuf, + gboolean keyframe, GstClockTime timestamp, gboolean marker) +{ + GST_DEBUG_OBJECT (rtph266depay, "To push buffer"); + + outbuf = gst_buffer_make_writable (outbuf); + + gst_rtp_drop_non_video_meta (rtph266depay, outbuf); + + GST_BUFFER_PTS (outbuf) = timestamp; + + if (keyframe) + GST_BUFFER_FLAG_UNSET (outbuf, GST_BUFFER_FLAG_DELTA_UNIT); + else + GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_DELTA_UNIT); + + if (marker) + GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_MARKER); + + gst_rtp_base_depayload_push (GST_RTP_BASE_DEPAYLOAD (rtph266depay), outbuf); +} + + +static void +gst_rtp_h266_depay_handle_nal (GstRtpH266Depay * rtph266depay, GstBuffer * nal, + GstClockTime in_timestamp, gboolean marker) +{ + GstRTPBaseDepayload *depayload = GST_RTP_BASE_DEPAYLOAD (rtph266depay); + GstBuffer *outbuf = NULL; + GstMapInfo map; + gboolean keyframe, out_keyframe; + GstClockTime out_timestamp; + guint8 nal_unit_type; + + gst_buffer_map (nal, &map, GST_MAP_READ); + if (G_UNLIKELY (map.size <= sizeof (sync_bytes))) + goto short_nal; + + GST_MEMDUMP_OBJECT (rtph266depay, "nal data: ", map.data, map.size); + + nal_unit_type = (map.data[sizeof (sync_bytes) + 1] >> 3) & 0x1F; + GST_DEBUG_OBJECT (rtph266depay, "Process nal with type %d", nal_unit_type); + + g_assert (rtph266depay->alignment == GST_H266_ALIGNMENT_AU); + + keyframe = NAL_TYPE_IS_PARAMETER_SET (nal_unit_type); + out_keyframe = keyframe; + out_timestamp = in_timestamp; + +#if 0 + /* Assume payloader always sets the (marker) M bit (whether 0 or 1). */ + if (!marker) { + /* ... */ + } +#endif + + /* add to adapter */ + gst_buffer_unmap (nal, &map); + GST_DEBUG_OBJECT (depayload, "adding NAL to picture adapter"); + gst_adapter_push (rtph266depay->picture_adapter, nal); + rtph266depay->last_ts = in_timestamp; + rtph266depay->last_keyframe = rtph266depay->last_keyframe || keyframe; + + if (marker) + outbuf = gst_rtp_h266_complete_au (rtph266depay, &out_timestamp, + &out_keyframe); + + if (outbuf) { + gst_rtp_h266_depay_push (rtph266depay, outbuf, out_keyframe, out_timestamp, + marker); + } + + return; + + /* ERRORS */ +short_nal: + { + GST_WARNING_OBJECT (depayload, "dropping short NAL"); + gst_buffer_unmap (nal, &map); + gst_buffer_unref (nal); + return; + } +} + +static void +gst_rtp_h266_finish_fragmentation_unit (GstRtpH266Depay * rtph266depay) +{ + guint outsize; + GstBuffer *outbuf; + + outsize = gst_adapter_available (rtph266depay->adapter); + g_assert (outsize >= sizeof (sync_bytes)); + + outbuf = gst_adapter_take_buffer (rtph266depay->adapter, outsize); + GST_DEBUG_OBJECT (rtph266depay, "output %d bytes", outsize); + +#if 0 + { + GstMapInfo map; + + gst_buffer_map (outbuf, &map, GST_MAP_READ); + g_assert (g_memcmp (map.data, sync_bytes, sizeof (sync_bytes)) == 0); + gst_buffer_unmap (outbuf, &map); + } +#endif + + rtph266depay->current_fu_type = 0; + + gst_rtp_h266_depay_handle_nal (rtph266depay, outbuf, + rtph266depay->fu_timestamp, rtph266depay->fu_marker); +} + +static GstBuffer * +gst_rtp_h266_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) +{ + GstRtpH266Depay *rtph266depay = GST_RTP_H266_DEPAY (depayload); + GstBuffer *outbuf = NULL; + GstMapInfo map; + gint payload_len; + guint8 *payload; + guint8 nal_unit_type, nuh_layer_id, nuh_temporal_id_plus1; + guint header_len; + GstClockTime timestamp; + gboolean marker; + guint outsize, nalu_size; + + GST_DEBUG_OBJECT (depayload, "Start processing."); + + payload_len = gst_rtp_buffer_get_payload_len (rtp); + payload = gst_rtp_buffer_get_payload (rtp); + + GST_DEBUG_OBJECT (rtph266depay, "receiving %d bytes", payload_len); + + if (payload_len == 0) + goto empty_packet; + + // +---------------+---------------+ + // |0|1|2|3|4|5|6|7|0|1|2|3|4|5|6|7| + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // |F|Z| LayerID | Type | TID | + // +---------------+---------------+ + + nal_unit_type = (payload[1] >> 3) & 0x1F; + nuh_layer_id = payload[0] & 0x3F; + nuh_temporal_id_plus1 = payload[1] & 0x07; + + header_len = 2; + + timestamp = GST_BUFFER_PTS (rtp->buffer); + marker = gst_rtp_buffer_get_marker (rtp); + + GST_DEBUG_OBJECT (rtph266depay, "marker: %d", marker); + GST_DEBUG_OBJECT (rtph266depay, + "NAL header nal_unit_type %d, nuh_temporal_id_plus1 %d", nal_unit_type, + nuh_temporal_id_plus1); + GST_DEBUG_OBJECT (depayload, "is discont %d", + GST_BUFFER_IS_DISCONT (rtp->buffer)); + +#if 0 + GST_FIXME_OBJECT (rtph266depay, "Assuming DONL field is not present"); +#endif + + /* If FU unit was being processed, but the current nal is of a different + * type. Assume that the remote payloader is buggy (didn't set the end bit + * when the FU ended) and send out what we gathered thusfar */ + if (G_UNLIKELY (rtph266depay->current_fu_type != 0 && + nal_unit_type != rtph266depay->current_fu_type)) { + gst_rtp_base_depayload_delayed (depayload); + gst_rtp_h266_finish_fragmentation_unit (rtph266depay); + } + + switch (nal_unit_type) { + case GST_H266_NAL_AP: + { + goto not_implemented; + } + case GST_H266_NAL_FU: + { + guint8 S, E, P, FUType; + guint16 nal_header; + guint16 seqnum; + + GST_DEBUG_OBJECT (rtph266depay, "Processing Fragmentation Unit"); + // Fragmentation units (FUs) Section 4.3.3 + // + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | PayloadHdr (Type=29) | FU header | DONL (cond) | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-| + // | DONL (cond) | | + // |-+-+-+-+-+-+-+-+ | + // | FU payload | + // | | + // | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | :...OPTIONAL RTP padding | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // + // FU Header + // +---------------+ + // |0|1|2|3|4|5|6|7| + // +-+-+-+-+-+-+-+-+ + // |S|E|P| FuType | + // +---------------+ + + /* strip headers */ + payload += header_len; // Set pointer to the start of FU header. + payload_len -= header_len; + + /* processing FU header */ + S = (payload[0] & 0x80) == 0x80; + E = (payload[0] & 0x40) == 0x40; + P = (payload[0] & 0x20) == 0x20; + FUType = payload[0] & 0x1F; + + GST_DEBUG_OBJECT (rtph266depay, + "FU header with S %d, E %d, P %d FUType %d", S, E, P, FUType); + + seqnum = gst_rtp_buffer_get_seq (rtp); + + if (S) { // Start of NAL unit. + /* If a new FU unit started, while still processing an older one. + * Assume that the remote payloader is buggy (doesn't set the end + * bit) and send out what we've gathered thusfar */ + if (G_UNLIKELY (rtph266depay->current_fu_type != 0)) { + gst_rtp_base_depayload_delayed (depayload); + gst_rtp_h266_finish_fragmentation_unit (rtph266depay); + } + + rtph266depay->current_fu_type = nal_unit_type; + rtph266depay->fu_timestamp = timestamp; + rtph266depay->last_fu_seqnum = seqnum; + + /* reconstruct NAL header */ + nal_header = + (((guint16) FUType) << 3) | (((guint16) nuh_layer_id) << 8) | + (guint16) nuh_temporal_id_plus1; + GST_MEMDUMP_OBJECT (rtph266depay, "nal_header", (guint8 *) & nal_header, + sizeof (nal_header)); + + /* go back one byte so we can copy the payload + two bytes more in the front which + * will be overwritten by the nal_header + */ + payload -= 1; + payload_len += 1; + + nalu_size = payload_len; + outsize = nalu_size + sizeof (sync_bytes); + outbuf = gst_buffer_new_and_alloc (outsize); + + gst_buffer_map (outbuf, &map, GST_MAP_WRITE); + memcpy (map.data, sync_bytes, sizeof (sync_bytes)); + memcpy (map.data + sizeof (sync_bytes), payload, nalu_size); + map.data[sizeof (sync_bytes)] = nal_header >> 8; + map.data[sizeof (sync_bytes) + 1] = nal_header & 0xff; + gst_buffer_unmap (outbuf, &map); + + } else { + if (rtph266depay->current_fu_type == 0) { + /* previous FU packet missing start bit? */ + GST_WARNING_OBJECT (rtph266depay, "missing FU start bit on an " + "earlier packet. Dropping."); + gst_rtp_base_depayload_flush (depayload, FALSE); + gst_adapter_clear (rtph266depay->adapter); + return NULL; + } + if (gst_rtp_buffer_compare_seqnum (rtph266depay->last_fu_seqnum, + seqnum) != 1) { + /* jump in sequence numbers within an FU is cause for discarding */ + GST_WARNING_OBJECT (rtph266depay, "Jump in sequence numbers from " + "%u to %u within Fragmentation Unit. Data was lost, dropping " + "stored.", rtph266depay->last_fu_seqnum, seqnum); + gst_rtp_base_depayload_flush (depayload, FALSE); + gst_adapter_clear (rtph266depay->adapter); + return NULL; + } + + rtph266depay->last_fu_seqnum = seqnum; + + GST_DEBUG_OBJECT (rtph266depay, "FU seqnum: %d", + rtph266depay->last_fu_seqnum); + + /* strip off FU header byte: Ignore DONL */ + payload += 1; + payload_len -= 1; + + outsize = payload_len; + outbuf = gst_buffer_new_and_alloc (outsize); + gst_buffer_fill (outbuf, 0, payload, outsize); + } + + gst_rtp_copy_video_meta (rtph266depay, outbuf, rtp->buffer); + GST_DEBUG_OBJECT (rtph266depay, "queueing %d bytes", outsize); + /* and assemble in the adapter */ + gst_adapter_push (rtph266depay->adapter, outbuf); + outbuf = NULL; + + rtph266depay->fu_marker = marker; + /* if NAL unit ends, flush the adapter */ + if (E) { + gst_rtp_h266_finish_fragmentation_unit (rtph266depay); + GST_DEBUG_OBJECT (rtph266depay, "End of Fragmentation Unit"); + } + + break; + } + default: + GST_DEBUG_OBJECT (rtph266depay, "Processing Single NAL Unit packet"); + nalu_size = payload_len; + outsize = nalu_size + sizeof (sync_bytes); + outbuf = gst_buffer_new_and_alloc (outsize); + + gst_buffer_map (outbuf, &map, GST_MAP_WRITE); + + /* Assume byte-stream format. This is what template caps accepts right now. */ + memcpy (map.data, sync_bytes, sizeof (sync_bytes)); + memcpy (map.data + 4, payload, nalu_size); + gst_buffer_unmap (outbuf, &map); + + gst_rtp_copy_video_meta (rtph266depay, outbuf, rtp->buffer); + gst_rtp_h266_depay_handle_nal (rtph266depay, outbuf, timestamp, marker); + break; + } + + return NULL; + + /* ERRORS */ +empty_packet: + { + GST_DEBUG_OBJECT (rtph266depay, "empty packet"); + gst_rtp_base_depayload_dropped (depayload); + return NULL; + } +not_implemented: + { + GST_ELEMENT_ERROR (rtph266depay, STREAM, FORMAT, + (NULL), ("NAL unit type %d not supported yet", nal_unit_type)); + gst_rtp_base_depayload_dropped (depayload); + return NULL; + } + +} + +static void +gst_rtp_h266_depay_drain (GstRtpH266Depay * rtph266depay) +{ + GstClockTime timestamp; + gboolean keyframe; + GstBuffer *outbuf; + + outbuf = gst_rtp_h266_complete_au (rtph266depay, ×tamp, &keyframe); + if (outbuf) + gst_rtp_h266_depay_push (rtph266depay, outbuf, keyframe, timestamp, FALSE); +} + +static void +gst_rtp_h266_depay_reset (GstRtpH266Depay * rtph266depay) +{ + gst_adapter_clear (rtph266depay->adapter); + gst_adapter_clear (rtph266depay->picture_adapter); + + rtph266depay->last_keyframe = FALSE; + rtph266depay->last_ts = 0; + rtph266depay->current_fu_type = 0; +} + +static gboolean +gst_rtp_h266_depay_setcaps (GstRTPBaseDepayload * depayload, GstCaps * caps) +{ + GstRtpH266Depay *rtph266depay = GST_RTP_H266_DEPAY (depayload); + GstStructure *structure = gst_caps_get_structure (caps, 0); + gint clock_rate; + + if (!gst_structure_get_int (structure, "clock-rate", &clock_rate)) + clock_rate = DEFAULT_CLOCK_RATE; + depayload->clock_rate = clock_rate; + + if (!gst_rtp_h266_depay_negotiate (rtph266depay)) + return FALSE; + + GST_DEBUG_OBJECT (rtph266depay, "set caps"); + + return TRUE; +} + +static gboolean +gst_rtp_h266_depay_handle_event (GstRTPBaseDepayload * depay, GstEvent * event) +{ + GstRtpH266Depay *rtph266depay = GST_RTP_H266_DEPAY (depay); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_FLUSH_STOP: + gst_rtp_h266_depay_reset (rtph266depay); + break; + case GST_EVENT_EOS: + GST_DEBUG_OBJECT (rtph266depay, "EOS..."); + gst_rtp_h266_depay_drain (rtph266depay); + break; + default: + break; + } + + return GST_RTP_BASE_DEPAYLOAD_CLASS (parent_class)->handle_event (depay, + event); +} + +static GstStateChangeReturn +gst_rtp_h266_depay_change_state (GstElement * element, + GstStateChange transition) +{ + GstRtpH266Depay *rtph266depay = GST_RTP_H266_DEPAY (element); + GstStateChangeReturn ret; + + if (transition == GST_STATE_CHANGE_READY_TO_PAUSED) + gst_rtp_h266_depay_reset (rtph266depay); + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + + if (transition == GST_STATE_CHANGE_PAUSED_TO_READY) + gst_rtp_h266_depay_reset (rtph266depay); + + return ret; +} + +static void +gst_rtp_h266_depay_finalize (GObject * object) +{ + GstRtpH266Depay *rtph266depay = GST_RTP_H266_DEPAY (object); + + g_clear_pointer (&rtph266depay->adapter, g_object_unref); + g_clear_pointer (&rtph266depay->picture_adapter, g_object_unref); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_rtp_h266_depay_class_init (GstRtpH266DepayClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *gstelement_class; + GstRTPBaseDepayloadClass *gstrtpbasedepayload_class; + + gobject_class = (GObjectClass *) klass; + gstelement_class = (GstElementClass *) klass; + gstrtpbasedepayload_class = (GstRTPBaseDepayloadClass *) klass; + + gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_rtp_h266_depay_finalize); + + gst_element_class_add_static_pad_template (gstelement_class, + &gst_rtp_h266_depay_src_template); + gst_element_class_add_static_pad_template (gstelement_class, + &gst_rtp_h266_depay_sink_template); + + gstelement_class->change_state = gst_rtp_h266_depay_change_state; + gstrtpbasedepayload_class->process_rtp_packet = gst_rtp_h266_depay_process; + gstrtpbasedepayload_class->set_caps = gst_rtp_h266_depay_setcaps; + gstrtpbasedepayload_class->handle_event = gst_rtp_h266_depay_handle_event; + + gst_element_class_set_static_metadata (gstelement_class, + "RTP H266 depayloader", "Codec/Depayloader/Network/RTP", + "Extracts H266 video from RTP packets (RFC 9328)", + "César Fabián Orccón Chipana "); + + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "rtph266depay", 0, + "H266 RTP Depayloader"); +} + +static void +gst_rtp_h266_depay_init (GstRtpH266Depay * rtph266depay) +{ + rtph266depay->adapter = gst_adapter_new (); + rtph266depay->picture_adapter = gst_adapter_new (); +} diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph266depay.h b/subprojects/gst-plugins-good/gst/rtp/gstrtph266depay.h new file mode 100644 index 00000000000..2224b763694 --- /dev/null +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph266depay.h @@ -0,0 +1,102 @@ +/* GStreamer + * Copyright (C) <2006> Wim Taymans + * Copyright (C) <2014> Jurgen Slowack + * Copyright (C) <2021> Intel Corporation + * Copyright (C) <2024> César Fabián Orccón Chipana + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_RTP_H266_DEPAY_H__ +#define __GST_RTP_H266_DEPAY_H__ + +#include +#include +#include + +G_BEGIN_DECLS + +typedef enum +{ + GST_H266_ALIGNMENT_AU, + GST_H266_ALIGNMENT_NAL, + GST_H266_ALIGNMENT_UNKOWN, +} GstH266Alignment; + +typedef enum +{ + GST_H266_STREAM_FORMAT_UNKNOWN, + GST_H266_STREAM_FORMAT_BYTESTREAM, + GST_H266_STREAM_FORMAT_HVC1, + GST_H266_STREAM_FORMAT_HEV1 +} GstH266StreamFormat; + +/* Imported from gsth266parse */ +/* *INDENT-OFF* */ +typedef enum +{ + GST_H266_NAL_SLICE_TRAIL = 0, + GST_H266_NAL_SLICE_STSA = 1, + GST_H266_NAL_SLICE_RADL = 2, + GST_H266_NAL_SLICE_RASL = 3, + GST_H266_NAL_SLICE_IDR_W_RADL = 7, + GST_H266_NAL_SLICE_IDR_N_LP = 8, + GST_H266_NAL_SLICE_CRA = 9, + GST_H266_NAL_SLICE_GDR = 10, + GST_H266_NAL_OPI = 12, + GST_H266_NAL_DCI = 13, + GST_H266_NAL_VPS = 14, + GST_H266_NAL_SPS = 15, + GST_H266_NAL_PPS = 16, + GST_H266_NAL_PREFIX_APS = 17, + GST_H266_NAL_SUFFIX_APS = 18, + GST_H266_NAL_PH = 19, + GST_H266_NAL_AUD = 20, + GST_H266_NAL_EOS = 21, + GST_H266_NAL_EOB = 22, + GST_H266_NAL_PREFIX_SEI = 23, + GST_H266_NAL_SUFFIX_SEI = 24, + GST_H266_NAL_FD = 25, + GST_H266_NAL_AP = 28, + GST_H266_NAL_FU = 29, +} GstH266NalUnitType; +/* *INDENT-ON* */ + +#define GST_TYPE_RTP_H266_DEPAY gst_rtp_h266_depay_get_type() +G_DECLARE_FINAL_TYPE(GstRtpH266Depay, gst_rtp_h266_depay, GST, RTP_H266_DEPAY, + GstRTPBaseDepayload); + +struct _GstRtpH266Depay +{ + GstRTPBaseDepayload depayload; + + GstAdapter *adapter; + + /* nal merging */ + GstH266Alignment alignment; + GstAdapter *picture_adapter; + GstClockTime last_ts; + gboolean last_keyframe; + + /* FU */ + guint8 current_fu_type; + guint16 last_fu_seqnum; + GstClockTime fu_timestamp; + gboolean fu_marker; +}; + +G_END_DECLS +#endif /* __GST_RTP_H266_DEPAY_H__ */ diff --git a/subprojects/gst-plugins-good/gst/rtp/meson.build b/subprojects/gst-plugins-good/gst/rtp/meson.build index 18c6848227d..b8070671e69 100644 --- a/subprojects/gst-plugins-good/gst/rtp/meson.build +++ b/subprojects/gst-plugins-good/gst/rtp/meson.build @@ -54,6 +54,7 @@ rtp_sources = [ 'gstrtph264pay.c', 'gstrtph265depay.c', 'gstrtph265pay.c', + 'gstrtph266depay.c', 'gstrtph266pay.c', 'gstrtpj2kdepay.c', 'gstrtpj2kpay.c', From 83b83c55563862f508b36584288cef235e052a19 Mon Sep 17 00:00:00 2001 From: Fabian Orccon Date: Thu, 21 Nov 2024 18:59:43 +0100 Subject: [PATCH 03/10] rtp: Add rtph266depay element --- subprojects/gst-plugins-good/gst/rtp/gstrtp.c | 1 + .../gst-plugins-good/gst/rtp/gstrtpelements.h | 1 + .../gst/rtp/gstrtph266depay.c | 670 ++++++++++++++++++ .../gst/rtp/gstrtph266depay.h | 102 +++ .../gst-plugins-good/gst/rtp/meson.build | 1 + 5 files changed, 775 insertions(+) create mode 100644 subprojects/gst-plugins-good/gst/rtp/gstrtph266depay.c create mode 100644 subprojects/gst-plugins-good/gst/rtp/gstrtph266depay.h diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtp.c b/subprojects/gst-plugins-good/gst/rtp/gstrtp.c index 2724aeca21c..e76cbf99085 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtp.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtp.c @@ -77,6 +77,7 @@ plugin_init (GstPlugin * plugin) ret |= GST_ELEMENT_REGISTER (rtph264pay, plugin); ret |= GST_ELEMENT_REGISTER (rtph265depay, plugin); ret |= GST_ELEMENT_REGISTER (rtph265pay, plugin); + ret |= GST_ELEMENT_REGISTER (rtph266depay, plugin); ret |= GST_ELEMENT_REGISTER (rtph266pay, plugin); ret |= GST_ELEMENT_REGISTER (rtpj2kdepay, plugin); ret |= GST_ELEMENT_REGISTER (rtpj2kpay, plugin); diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpelements.h b/subprojects/gst-plugins-good/gst/rtp/gstrtpelements.h index 0db0d785a93..1ab6daedc38 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpelements.h +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpelements.h @@ -77,6 +77,7 @@ GST_ELEMENT_REGISTER_DECLARE (rtph264depay); GST_ELEMENT_REGISTER_DECLARE (rtph264pay); GST_ELEMENT_REGISTER_DECLARE (rtph265depay); GST_ELEMENT_REGISTER_DECLARE (rtph265pay); +GST_ELEMENT_REGISTER_DECLARE (rtph266depay); GST_ELEMENT_REGISTER_DECLARE (rtph266pay); GST_ELEMENT_REGISTER_DECLARE (rtpj2kdepay); GST_ELEMENT_REGISTER_DECLARE (rtpj2kpay); diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph266depay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtph266depay.c new file mode 100644 index 00000000000..e8504884e98 --- /dev/null +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph266depay.c @@ -0,0 +1,670 @@ +/* GStreamer + * Copyright (C) <2006> Wim Taymans + * Copyright (C) <2014> Jurgen Slowack + * Copyright (C) <2024> César Fabián Orccón Chipana + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/* references: + * - RTP Payload Format for Versatile Video Coding (VVC) + * https://www.ietf.org/rfc/rfc9328.txt + * - RTP: A Transport Protocol for Real-Time Applications + * https://www.ietf.org/rfc/rfc3550.txt + * - H.266: Versatile video coding + * https://www.itu.int/rec/T-REC-H.266-202309-I + */ + +#include "gstrtph266depay.h" +#include +#include +#include +#include + +#define NAL_TYPE_IS_PARAMETER_SET(nt) (((nt) == GST_H266_NAL_VPS)\ + || ((nt) == GST_H266_NAL_SPS)\ + || ((nt) == GST_H266_NAL_PPS) ) +#define NAL_TYPE_IS_KEY(nt) (NAL_TYPE_IS_PARAMETER_SET(nt)) + +#define DEFAULT_CONFIG_INTERVAL 0 +#define DEFAULT_CLOCK_RATE 90000 + +#define GST_CAT_DEFAULT rtph266depay_debug +#define gst_rtp_h266_depay_parent_class parent_class + +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); +G_DEFINE_TYPE (GstRtpH266Depay, gst_rtp_h266_depay, + GST_TYPE_RTP_BASE_DEPAYLOAD); +GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (rtph266depay, "rtph266depay", + GST_RANK_SECONDARY, GST_TYPE_RTP_H266_DEPAY, rtp_element_init (plugin)); + +static GstStaticPadTemplate gst_rtp_h266_depay_src_template = +GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS + ("video/x-h266, stream-format = (string) byte-stream, alignment = (string) au ")); + +static GstStaticPadTemplate gst_rtp_h266_depay_sink_template = +GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("application/x-rtp, " + "media = (string) \"video\", " + "clock-rate = (int) 90000, " "encoding-name = (string) \"H266\"")); + +enum +{ + PROP_0, +}; + +/* 3 zero bytes syncword */ +static const guint8 sync_bytes[] = { 0, 0, 0, 1 }; + +static gboolean +gst_rtp_h266_depay_negotiate (GstRtpH266Depay * rtph266depay) +{ + GstCaps *caps; + gboolean ret = FALSE; + + caps = + gst_pad_get_allowed_caps (GST_RTP_BASE_DEPAYLOAD_SRCPAD (rtph266depay)); + + GST_DEBUG_OBJECT (rtph266depay, "allowed caps: %" GST_PTR_FORMAT, caps); + + if (!caps) { + GST_ERROR_OBJECT (rtph266depay, "Caps not found."); + return ret; + } + + if (gst_caps_get_size (caps) > 0) { + GstStructure *s = gst_caps_get_structure (caps, 0); + const gchar *str; + + GST_DEBUG_OBJECT (rtph266depay, "get stream-format"); + str = gst_structure_get_string (s, "stream-format"); + if (g_strcmp0 (str, "byte-stream") != 0) { + GST_ERROR_OBJECT (rtph266depay, "only byte-stream supported: %s", str); + goto beach; + } + + GST_DEBUG_OBJECT (rtph266depay, "st: %" GST_PTR_FORMAT, s); + + + GST_DEBUG_OBJECT (rtph266depay, "get alignment"); + str = gst_structure_get_string (s, "alignment"); + if (g_strcmp0 (str, "au") == 0) { + rtph266depay->alignment = GST_H266_ALIGNMENT_AU; + } else { + GST_ERROR_OBJECT (rtph266depay, "alignment not supported: %s", str); + goto beach; + } + } + + ret = TRUE; + +beach: + gst_caps_unref (caps); + return ret; +} + +static GstBuffer * +gst_rtp_h266_depay_allocate_output_buffer (GstRtpH266Depay * depay, gsize size) +{ + GstBuffer *buffer = NULL; + + GST_LOG_OBJECT (depay, "want output buffer of %u bytes", (guint) size); + + g_return_val_if_fail (size > 0, NULL); + + // TODO: Forward allocator. + buffer = gst_buffer_new_allocate (NULL, size, NULL); + + return buffer; +} + +static GstBuffer * +gst_rtp_h266_complete_au (GstRtpH266Depay * rtph266depay, + GstClockTime * out_timestamp, gboolean * out_keyframe) +{ + GstBufferList *list; + GstMapInfo outmap; + GstBuffer *outbuf; + guint outsize, offset = 0; + gint b, n_bufs, m, n_mem; + + /* we had a picture in the adapter and we completed it */ + GST_DEBUG_OBJECT (rtph266depay, "taking completed AU"); + outsize = gst_adapter_available (rtph266depay->picture_adapter); + + GST_DEBUG_OBJECT (rtph266depay, "will allocate buffer of size %d", outsize); + outbuf = gst_rtp_h266_depay_allocate_output_buffer (rtph266depay, outsize); + + if (G_UNLIKELY (outbuf == NULL)) + return NULL; + + if (!gst_buffer_map (outbuf, &outmap, GST_MAP_WRITE)) + return NULL; + + list = gst_adapter_take_buffer_list (rtph266depay->picture_adapter, outsize); + + n_bufs = gst_buffer_list_length (list); + for (b = 0; b < n_bufs; ++b) { + GstBuffer *buf = gst_buffer_list_get (list, b); + + n_mem = gst_buffer_n_memory (buf); + for (m = 0; m < n_mem; ++m) { + GstMemory *mem = gst_buffer_peek_memory (buf, m); + gsize mem_size = gst_memory_get_sizes (mem, NULL, NULL); + GstMapInfo mem_map; + + if (gst_memory_map (mem, &mem_map, GST_MAP_READ)) { + memcpy (outmap.data + offset, mem_map.data, mem_size); + gst_memory_unmap (mem, &mem_map); + } else { + memset (outmap.data + offset, 0, mem_size); + } + offset += mem_size; + } + + gst_rtp_copy_video_meta (rtph266depay, outbuf, buf); + } + gst_buffer_list_unref (list); + gst_buffer_unmap (outbuf, &outmap); + + *out_timestamp = rtph266depay->last_ts; + *out_keyframe = rtph266depay->last_keyframe; + + rtph266depay->last_keyframe = FALSE; + + return outbuf; +} + + +static void +gst_rtp_h266_depay_push (GstRtpH266Depay * rtph266depay, GstBuffer * outbuf, + gboolean keyframe, GstClockTime timestamp, gboolean marker) +{ + GST_DEBUG_OBJECT (rtph266depay, "To push buffer"); + + outbuf = gst_buffer_make_writable (outbuf); + + gst_rtp_drop_non_video_meta (rtph266depay, outbuf); + + GST_BUFFER_PTS (outbuf) = timestamp; + + if (keyframe) + GST_BUFFER_FLAG_UNSET (outbuf, GST_BUFFER_FLAG_DELTA_UNIT); + else + GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_DELTA_UNIT); + + if (marker) + GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_MARKER); + + gst_rtp_base_depayload_push (GST_RTP_BASE_DEPAYLOAD (rtph266depay), outbuf); +} + + +static void +gst_rtp_h266_depay_handle_nal (GstRtpH266Depay * rtph266depay, GstBuffer * nal, + GstClockTime in_timestamp, gboolean marker) +{ + GstRTPBaseDepayload *depayload = GST_RTP_BASE_DEPAYLOAD (rtph266depay); + GstBuffer *outbuf = NULL; + GstMapInfo map; + gboolean keyframe, out_keyframe; + GstClockTime out_timestamp; + guint8 nal_unit_type; + + gst_buffer_map (nal, &map, GST_MAP_READ); + if (G_UNLIKELY (map.size <= sizeof (sync_bytes))) + goto short_nal; + + GST_MEMDUMP_OBJECT (rtph266depay, "nal data: ", map.data, map.size); + + nal_unit_type = (map.data[sizeof (sync_bytes) + 1] >> 3) & 0x1F; + GST_DEBUG_OBJECT (rtph266depay, "Process nal with type %d", nal_unit_type); + + g_assert (rtph266depay->alignment == GST_H266_ALIGNMENT_AU); + + keyframe = NAL_TYPE_IS_PARAMETER_SET (nal_unit_type); + out_keyframe = keyframe; + out_timestamp = in_timestamp; + +#if 0 + /* Assume payloader always sets the (marker) M bit (whether 0 or 1). */ + if (!marker) { + /* ... */ + } +#endif + + /* add to adapter */ + gst_buffer_unmap (nal, &map); + GST_DEBUG_OBJECT (depayload, "adding NAL to picture adapter"); + gst_adapter_push (rtph266depay->picture_adapter, nal); + rtph266depay->last_ts = in_timestamp; + rtph266depay->last_keyframe = rtph266depay->last_keyframe || keyframe; + + if (marker) + outbuf = gst_rtp_h266_complete_au (rtph266depay, &out_timestamp, + &out_keyframe); + + if (outbuf) { + gst_rtp_h266_depay_push (rtph266depay, outbuf, out_keyframe, out_timestamp, + marker); + } + + return; + + /* ERRORS */ +short_nal: + { + GST_WARNING_OBJECT (depayload, "dropping short NAL"); + gst_buffer_unmap (nal, &map); + gst_buffer_unref (nal); + return; + } +} + +static void +gst_rtp_h266_finish_fragmentation_unit (GstRtpH266Depay * rtph266depay) +{ + guint outsize; + GstBuffer *outbuf; + + outsize = gst_adapter_available (rtph266depay->adapter); + g_assert (outsize >= sizeof (sync_bytes)); + + outbuf = gst_adapter_take_buffer (rtph266depay->adapter, outsize); + GST_DEBUG_OBJECT (rtph266depay, "output %d bytes", outsize); + +#if 0 + { + GstMapInfo map; + + gst_buffer_map (outbuf, &map, GST_MAP_READ); + g_assert (g_memcmp (map.data, sync_bytes, sizeof (sync_bytes)) == 0); + gst_buffer_unmap (outbuf, &map); + } +#endif + + rtph266depay->current_fu_type = 0; + + gst_rtp_h266_depay_handle_nal (rtph266depay, outbuf, + rtph266depay->fu_timestamp, rtph266depay->fu_marker); +} + +static GstBuffer * +gst_rtp_h266_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) +{ + GstRtpH266Depay *rtph266depay = GST_RTP_H266_DEPAY (depayload); + GstBuffer *outbuf = NULL; + GstMapInfo map; + gint payload_len; + guint8 *payload; + guint8 nal_unit_type, nuh_layer_id, nuh_temporal_id_plus1; + guint header_len; + GstClockTime timestamp; + gboolean marker; + guint outsize, nalu_size; + + GST_DEBUG_OBJECT (depayload, "Start processing."); + + payload_len = gst_rtp_buffer_get_payload_len (rtp); + payload = gst_rtp_buffer_get_payload (rtp); + + GST_DEBUG_OBJECT (rtph266depay, "receiving %d bytes", payload_len); + + if (payload_len == 0) + goto empty_packet; + + // +---------------+---------------+ + // |0|1|2|3|4|5|6|7|0|1|2|3|4|5|6|7| + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // |F|Z| LayerID | Type | TID | + // +---------------+---------------+ + + nal_unit_type = (payload[1] >> 3) & 0x1F; + nuh_layer_id = payload[0] & 0x3F; + nuh_temporal_id_plus1 = payload[1] & 0x07; + + header_len = 2; + + timestamp = GST_BUFFER_PTS (rtp->buffer); + marker = gst_rtp_buffer_get_marker (rtp); + + GST_DEBUG_OBJECT (rtph266depay, "marker: %d", marker); + GST_DEBUG_OBJECT (rtph266depay, + "NAL header nal_unit_type %d, nuh_temporal_id_plus1 %d", nal_unit_type, + nuh_temporal_id_plus1); + GST_DEBUG_OBJECT (depayload, "is discont %d", + GST_BUFFER_IS_DISCONT (rtp->buffer)); + +#if 0 + GST_FIXME_OBJECT (rtph266depay, "Assuming DONL field is not present"); +#endif + + /* If FU unit was being processed, but the current nal is of a different + * type. Assume that the remote payloader is buggy (didn't set the end bit + * when the FU ended) and send out what we gathered thusfar */ + if (G_UNLIKELY (rtph266depay->current_fu_type != 0 && + nal_unit_type != rtph266depay->current_fu_type)) { + gst_rtp_base_depayload_delayed (depayload); + gst_rtp_h266_finish_fragmentation_unit (rtph266depay); + } + + switch (nal_unit_type) { + case GST_H266_NAL_AP: + { + goto not_implemented; + } + case GST_H266_NAL_FU: + { + guint8 S, E, P, FUType; + guint16 nal_header; + guint16 seqnum; + + GST_DEBUG_OBJECT (rtph266depay, "Processing Fragmentation Unit"); + // Fragmentation units (FUs) Section 4.3.3 + // + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | PayloadHdr (Type=29) | FU header | DONL (cond) | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-| + // | DONL (cond) | | + // |-+-+-+-+-+-+-+-+ | + // | FU payload | + // | | + // | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | :...OPTIONAL RTP padding | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // + // FU Header + // +---------------+ + // |0|1|2|3|4|5|6|7| + // +-+-+-+-+-+-+-+-+ + // |S|E|P| FuType | + // +---------------+ + + /* strip headers */ + payload += header_len; // Set pointer to the start of FU header. + payload_len -= header_len; + + /* processing FU header */ + S = (payload[0] & 0x80) == 0x80; + E = (payload[0] & 0x40) == 0x40; + P = (payload[0] & 0x20) == 0x20; + FUType = payload[0] & 0x1F; + + GST_DEBUG_OBJECT (rtph266depay, + "FU header with S %d, E %d, P %d FUType %d", S, E, P, FUType); + + seqnum = gst_rtp_buffer_get_seq (rtp); + + if (S) { // Start of NAL unit. + /* If a new FU unit started, while still processing an older one. + * Assume that the remote payloader is buggy (doesn't set the end + * bit) and send out what we've gathered thusfar */ + if (G_UNLIKELY (rtph266depay->current_fu_type != 0)) { + gst_rtp_base_depayload_delayed (depayload); + gst_rtp_h266_finish_fragmentation_unit (rtph266depay); + } + + rtph266depay->current_fu_type = nal_unit_type; + rtph266depay->fu_timestamp = timestamp; + rtph266depay->last_fu_seqnum = seqnum; + + /* reconstruct NAL header */ + nal_header = + (((guint16) FUType) << 3) | (((guint16) nuh_layer_id) << 8) | + (guint16) nuh_temporal_id_plus1; + GST_MEMDUMP_OBJECT (rtph266depay, "nal_header", (guint8 *) & nal_header, + sizeof (nal_header)); + + /* go back one byte so we can copy the payload + two bytes more in the front which + * will be overwritten by the nal_header + */ + payload -= 1; + payload_len += 1; + + nalu_size = payload_len; + outsize = nalu_size + sizeof (sync_bytes); + outbuf = gst_buffer_new_and_alloc (outsize); + + gst_buffer_map (outbuf, &map, GST_MAP_WRITE); + memcpy (map.data, sync_bytes, sizeof (sync_bytes)); + memcpy (map.data + sizeof (sync_bytes), payload, nalu_size); + map.data[sizeof (sync_bytes)] = nal_header >> 8; + map.data[sizeof (sync_bytes) + 1] = nal_header & 0xff; + gst_buffer_unmap (outbuf, &map); + + } else { + if (rtph266depay->current_fu_type == 0) { + /* previous FU packet missing start bit? */ + GST_WARNING_OBJECT (rtph266depay, "missing FU start bit on an " + "earlier packet. Dropping."); + gst_rtp_base_depayload_flush (depayload, FALSE); + gst_adapter_clear (rtph266depay->adapter); + return NULL; + } + if (gst_rtp_buffer_compare_seqnum (rtph266depay->last_fu_seqnum, + seqnum) != 1) { + /* jump in sequence numbers within an FU is cause for discarding */ + GST_WARNING_OBJECT (rtph266depay, "Jump in sequence numbers from " + "%u to %u within Fragmentation Unit. Data was lost, dropping " + "stored.", rtph266depay->last_fu_seqnum, seqnum); + gst_rtp_base_depayload_flush (depayload, FALSE); + gst_adapter_clear (rtph266depay->adapter); + return NULL; + } + + rtph266depay->last_fu_seqnum = seqnum; + + GST_DEBUG_OBJECT (rtph266depay, "FU seqnum: %d", + rtph266depay->last_fu_seqnum); + + /* strip off FU header byte: Ignore DONL */ + payload += 1; + payload_len -= 1; + + outsize = payload_len; + outbuf = gst_buffer_new_and_alloc (outsize); + gst_buffer_fill (outbuf, 0, payload, outsize); + } + + gst_rtp_copy_video_meta (rtph266depay, outbuf, rtp->buffer); + GST_DEBUG_OBJECT (rtph266depay, "queueing %d bytes", outsize); + /* and assemble in the adapter */ + gst_adapter_push (rtph266depay->adapter, outbuf); + outbuf = NULL; + + rtph266depay->fu_marker = marker; + /* if NAL unit ends, flush the adapter */ + if (E) { + gst_rtp_h266_finish_fragmentation_unit (rtph266depay); + GST_DEBUG_OBJECT (rtph266depay, "End of Fragmentation Unit"); + } + + break; + } + default: + GST_DEBUG_OBJECT (rtph266depay, "Processing Single NAL Unit packet"); + nalu_size = payload_len; + outsize = nalu_size + sizeof (sync_bytes); + outbuf = gst_buffer_new_and_alloc (outsize); + + gst_buffer_map (outbuf, &map, GST_MAP_WRITE); + + /* Assume byte-stream format. This is what template caps accepts right now. */ + memcpy (map.data, sync_bytes, sizeof (sync_bytes)); + memcpy (map.data + 4, payload, nalu_size); + gst_buffer_unmap (outbuf, &map); + + gst_rtp_copy_video_meta (rtph266depay, outbuf, rtp->buffer); + gst_rtp_h266_depay_handle_nal (rtph266depay, outbuf, timestamp, marker); + break; + } + + return NULL; + + /* ERRORS */ +empty_packet: + { + GST_DEBUG_OBJECT (rtph266depay, "empty packet"); + gst_rtp_base_depayload_dropped (depayload); + return NULL; + } +not_implemented: + { + GST_ELEMENT_ERROR (rtph266depay, STREAM, FORMAT, + (NULL), ("NAL unit type %d not supported yet", nal_unit_type)); + gst_rtp_base_depayload_dropped (depayload); + return NULL; + } + +} + +static void +gst_rtp_h266_depay_drain (GstRtpH266Depay * rtph266depay) +{ + GstClockTime timestamp; + gboolean keyframe; + GstBuffer *outbuf; + + outbuf = gst_rtp_h266_complete_au (rtph266depay, ×tamp, &keyframe); + if (outbuf) + gst_rtp_h266_depay_push (rtph266depay, outbuf, keyframe, timestamp, FALSE); +} + +static void +gst_rtp_h266_depay_reset (GstRtpH266Depay * rtph266depay) +{ + gst_adapter_clear (rtph266depay->adapter); + gst_adapter_clear (rtph266depay->picture_adapter); + + rtph266depay->last_keyframe = FALSE; + rtph266depay->last_ts = 0; + rtph266depay->current_fu_type = 0; +} + +static gboolean +gst_rtp_h266_depay_setcaps (GstRTPBaseDepayload * depayload, GstCaps * caps) +{ + GstRtpH266Depay *rtph266depay = GST_RTP_H266_DEPAY (depayload); + GstStructure *structure = gst_caps_get_structure (caps, 0); + gint clock_rate; + + if (!gst_structure_get_int (structure, "clock-rate", &clock_rate)) + clock_rate = DEFAULT_CLOCK_RATE; + depayload->clock_rate = clock_rate; + + if (!gst_rtp_h266_depay_negotiate (rtph266depay)) + return FALSE; + + GST_DEBUG_OBJECT (rtph266depay, "set caps"); + + return TRUE; +} + +static gboolean +gst_rtp_h266_depay_handle_event (GstRTPBaseDepayload * depay, GstEvent * event) +{ + GstRtpH266Depay *rtph266depay = GST_RTP_H266_DEPAY (depay); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_FLUSH_STOP: + gst_rtp_h266_depay_reset (rtph266depay); + break; + case GST_EVENT_EOS: + GST_DEBUG_OBJECT (rtph266depay, "EOS..."); + gst_rtp_h266_depay_drain (rtph266depay); + break; + default: + break; + } + + return GST_RTP_BASE_DEPAYLOAD_CLASS (parent_class)->handle_event (depay, + event); +} + +static GstStateChangeReturn +gst_rtp_h266_depay_change_state (GstElement * element, + GstStateChange transition) +{ + GstRtpH266Depay *rtph266depay = GST_RTP_H266_DEPAY (element); + GstStateChangeReturn ret; + + if (transition == GST_STATE_CHANGE_READY_TO_PAUSED) + gst_rtp_h266_depay_reset (rtph266depay); + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + + if (transition == GST_STATE_CHANGE_PAUSED_TO_READY) + gst_rtp_h266_depay_reset (rtph266depay); + + return ret; +} + +static void +gst_rtp_h266_depay_finalize (GObject * object) +{ + GstRtpH266Depay *rtph266depay = GST_RTP_H266_DEPAY (object); + + g_clear_pointer (&rtph266depay->adapter, g_object_unref); + g_clear_pointer (&rtph266depay->picture_adapter, g_object_unref); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_rtp_h266_depay_class_init (GstRtpH266DepayClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *gstelement_class; + GstRTPBaseDepayloadClass *gstrtpbasedepayload_class; + + gobject_class = (GObjectClass *) klass; + gstelement_class = (GstElementClass *) klass; + gstrtpbasedepayload_class = (GstRTPBaseDepayloadClass *) klass; + + gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_rtp_h266_depay_finalize); + + gst_element_class_add_static_pad_template (gstelement_class, + &gst_rtp_h266_depay_src_template); + gst_element_class_add_static_pad_template (gstelement_class, + &gst_rtp_h266_depay_sink_template); + + gstelement_class->change_state = gst_rtp_h266_depay_change_state; + gstrtpbasedepayload_class->process_rtp_packet = gst_rtp_h266_depay_process; + gstrtpbasedepayload_class->set_caps = gst_rtp_h266_depay_setcaps; + gstrtpbasedepayload_class->handle_event = gst_rtp_h266_depay_handle_event; + + gst_element_class_set_static_metadata (gstelement_class, + "RTP H266 depayloader", "Codec/Depayloader/Network/RTP", + "Extracts H266 video from RTP packets (RFC 9328)", + "César Fabián Orccón Chipana "); + + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "rtph266depay", 0, + "H266 RTP Depayloader"); +} + +static void +gst_rtp_h266_depay_init (GstRtpH266Depay * rtph266depay) +{ + rtph266depay->adapter = gst_adapter_new (); + rtph266depay->picture_adapter = gst_adapter_new (); +} diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph266depay.h b/subprojects/gst-plugins-good/gst/rtp/gstrtph266depay.h new file mode 100644 index 00000000000..2224b763694 --- /dev/null +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph266depay.h @@ -0,0 +1,102 @@ +/* GStreamer + * Copyright (C) <2006> Wim Taymans + * Copyright (C) <2014> Jurgen Slowack + * Copyright (C) <2021> Intel Corporation + * Copyright (C) <2024> César Fabián Orccón Chipana + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_RTP_H266_DEPAY_H__ +#define __GST_RTP_H266_DEPAY_H__ + +#include +#include +#include + +G_BEGIN_DECLS + +typedef enum +{ + GST_H266_ALIGNMENT_AU, + GST_H266_ALIGNMENT_NAL, + GST_H266_ALIGNMENT_UNKOWN, +} GstH266Alignment; + +typedef enum +{ + GST_H266_STREAM_FORMAT_UNKNOWN, + GST_H266_STREAM_FORMAT_BYTESTREAM, + GST_H266_STREAM_FORMAT_HVC1, + GST_H266_STREAM_FORMAT_HEV1 +} GstH266StreamFormat; + +/* Imported from gsth266parse */ +/* *INDENT-OFF* */ +typedef enum +{ + GST_H266_NAL_SLICE_TRAIL = 0, + GST_H266_NAL_SLICE_STSA = 1, + GST_H266_NAL_SLICE_RADL = 2, + GST_H266_NAL_SLICE_RASL = 3, + GST_H266_NAL_SLICE_IDR_W_RADL = 7, + GST_H266_NAL_SLICE_IDR_N_LP = 8, + GST_H266_NAL_SLICE_CRA = 9, + GST_H266_NAL_SLICE_GDR = 10, + GST_H266_NAL_OPI = 12, + GST_H266_NAL_DCI = 13, + GST_H266_NAL_VPS = 14, + GST_H266_NAL_SPS = 15, + GST_H266_NAL_PPS = 16, + GST_H266_NAL_PREFIX_APS = 17, + GST_H266_NAL_SUFFIX_APS = 18, + GST_H266_NAL_PH = 19, + GST_H266_NAL_AUD = 20, + GST_H266_NAL_EOS = 21, + GST_H266_NAL_EOB = 22, + GST_H266_NAL_PREFIX_SEI = 23, + GST_H266_NAL_SUFFIX_SEI = 24, + GST_H266_NAL_FD = 25, + GST_H266_NAL_AP = 28, + GST_H266_NAL_FU = 29, +} GstH266NalUnitType; +/* *INDENT-ON* */ + +#define GST_TYPE_RTP_H266_DEPAY gst_rtp_h266_depay_get_type() +G_DECLARE_FINAL_TYPE(GstRtpH266Depay, gst_rtp_h266_depay, GST, RTP_H266_DEPAY, + GstRTPBaseDepayload); + +struct _GstRtpH266Depay +{ + GstRTPBaseDepayload depayload; + + GstAdapter *adapter; + + /* nal merging */ + GstH266Alignment alignment; + GstAdapter *picture_adapter; + GstClockTime last_ts; + gboolean last_keyframe; + + /* FU */ + guint8 current_fu_type; + guint16 last_fu_seqnum; + GstClockTime fu_timestamp; + gboolean fu_marker; +}; + +G_END_DECLS +#endif /* __GST_RTP_H266_DEPAY_H__ */ diff --git a/subprojects/gst-plugins-good/gst/rtp/meson.build b/subprojects/gst-plugins-good/gst/rtp/meson.build index 18c6848227d..b8070671e69 100644 --- a/subprojects/gst-plugins-good/gst/rtp/meson.build +++ b/subprojects/gst-plugins-good/gst/rtp/meson.build @@ -54,6 +54,7 @@ rtp_sources = [ 'gstrtph264pay.c', 'gstrtph265depay.c', 'gstrtph265pay.c', + 'gstrtph266depay.c', 'gstrtph266pay.c', 'gstrtpj2kdepay.c', 'gstrtpj2kpay.c', From c7c307398df187d52eb20e4987a11354d9d2a86a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Falgueras=20Garc=C3=ADa?= Date: Tue, 19 Nov 2024 08:20:16 +0100 Subject: [PATCH 04/10] rtph266pay: Add config-interval property Keep a cache of all Parameter Sets received, then, send them when it's appropriated according with the `config-interval` parameter. Issue: OCP_6005 --- .../gst-plugins-good/gst/rtp/gstrtph266pay.c | 271 ++++++++++++++++-- 1 file changed, 255 insertions(+), 16 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c index 6f7b517b42f..90adcae3ce5 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c @@ -38,12 +38,35 @@ GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); typedef enum { - NALU_TYPE_VPS = 14, - NALU_TYPE_SPS = 15, - NALU_TYPE_PPS = 16, - NALU_TYPE_PAPS = 17, - NALU_TYPE_SAPS = 18, - NALU_TYPE_AUD = 20, + NALU_TYPE_TRAIL_NUT = 0, + NALU_TYPE_STSA_NUT = 1, + NALU_TYPE_RADL_NUT = 2, + NALU_TYPE_RASL_NUT = 3, + NALU_TYPE_RSV_VCL_4 = 4, + NALU_TYPE_RSV_VCL_5 = 5, + NALU_TYPE_RSV_VCL_6 = 6, + NALU_TYPE_IDR_W_RADL = 7, + NALU_TYPE_IDR_N_LP = 8, + NALU_TYPE_CRA_NUT = 9, + NALU_TYPE_GDR_NUT = 10, + NALU_TYPE_RSV_IRAP_11 = 11, + NALU_TYPE_OPI_NUT = 12, + NALU_TYPE_DCI_NUT = 13, + NALU_TYPE_VPS_NUT = 14, + NALU_TYPE_SPS_NUT = 15, + NALU_TYPE_PPS_NUT = 16, + NALU_TYPE_PREFIX_APS_NUT = 17, + NALU_TYPE_SUFFIX_APS_NUT = 18, + NALU_TYPE_PH_NUT = 19, + NALU_TYPE_AUD_NUT = 20, + NALU_TYPE_EOS_NUT = 21, + NALU_TYPE_EOB_NUT = 22, + NALU_TYPE_PREFIX_SEI_NUT = 23, + NALU_TYPE_SUFFIX_SEI_NUT = 24, + NALU_TYPE_FD_NUT = 25, + NALU_TYPE_RSV_NVCL_26 = 26, + NALU_TYPE_RSV_NVCL_27 = 27, + _NALU_TYPE_MAX } NaluType; #define NALU_SC_MSK 0xffffff00 @@ -51,8 +74,12 @@ typedef enum #define NALU_SC_LEN 3 #define NALU_HDR_LEN 2 #define NALU_INVALID_XPS 0xFF -#define NALU_IS_PARAMETER_SET(nalu) \ - (((nalu)->type >= NALU_TYPE_VPS) && ((nalu)->type <= NALU_TYPE_SAPS)) +#define NALU_IS_XPS(nalu) \ + (((nalu)->type >= NALU_TYPE_VPS_NUT) && ((nalu)->type <= NALU_TYPE_SUFFIX_APS_NUT)) +#define NALU_IS_VCL(nalu) \ + (((nalu)->type >= NALU_TYPE_TRAIL_NUT) && ((nalu)->type <= NALU_TYPE_RSV_IRAP_11)) +#define NALU_IS_IDR(nalu) \ + (((nalu)->type == NALU_TYPE_IDR_W_RADL) || ((nalu)->type == NALU_TYPE_IDR_N_LP)) #define FU_TYPE 29 #define FU_HDR_LEN (NALU_HDR_LEN + 1) // PayloadHdr + FU header @@ -87,7 +114,13 @@ struct _GstRtpH266Pay GstAdapter *adapter; GQueue nalus; + GHashTable *xps_id_nalu_map; + Alignment alignment; + GstClockTime ts_last_xps_to_sent; + + // Properties + gint config_interval; }; #define gst_rtp_h266_pay_parent_class parent_class @@ -109,9 +142,12 @@ static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", "clock-rate = (int) 90000, " "encoding-name = (string) \"H266\"") ); +#define DEFAULT_CONFIG_INTERVAL 0 + enum { PROP_0, + PROP_CONFIG_INTERVAL, }; // GObject methods @@ -133,6 +169,10 @@ static gboolean _sink_event (GstRTPBasePayload * rtpbasepay, GstEvent * event); // GstRtpH266Pay methods static void _process_nalu (GstRtpH266Pay * rtph266pay, GstBuffer * nalu_buf); +static void _update_xps_cache (GstRtpH266Pay * rtph266pay, const Nalu * nalu); +static void _clear_xps_cache (GstRtpH266Pay * rtph266pay); +static gboolean _can_insert_xps (GstRtpH266Pay * rtph266pay, const Nalu * nalu); +static void _insert_xps_cache (GstRtpH266Pay * rtph266pay); static void _set_au_boundaries (GstRtpH266Pay * rtph266pay); static GstFlowReturn _push_pending_data (GstRtpH266Pay * rtph266pay, gboolean eos); @@ -144,10 +184,15 @@ static gboolean _up_fits_in_mtu (const GstRtpH266Pay * rtph266pay, static GstBuffer *_extract_fu (GstRtpH266Pay * rtph266pay, const Nalu * nalu, gsize offset, gsize size, gboolean last); static void _clear_nalu_queue (GstRtpH266Pay * rtph266pay); +static GstClockTime _get_nalu_running_time (const GstRtpH266Pay * rtph266pay, + const Nalu * nalu); // Nalu methods static Nalu *_nalu_new (GstBuffer * nalu_buf); static void _nalu_free (Nalu * nalu); +static Nalu *_nalu_copy (const Nalu * nalu); +static Nalu *_nalu_copy_ts (Nalu * dst, const Nalu * src); +static gboolean _nalu_is_rsv (const Nalu * nalu); static void gst_rtp_h266_pay_class_init (GstRtpH266PayClass * klass) @@ -164,6 +209,16 @@ gst_rtp_h266_pay_class_init (GstRtpH266PayClass * klass) gobject_class->set_property = GST_DEBUG_FUNCPTR (_set_property); gobject_class->get_property = GST_DEBUG_FUNCPTR (_get_property); + g_object_class_install_property (gobject_class, + PROP_CONFIG_INTERVAL, + g_param_spec_int ("config-interval", + "Parameter Set send interval", + "Send VPS, SPS, PPS and APS at this interval (in seconds)" + "(0 = disabled, -1 = send with every IDR frame)", + -1, 3600, DEFAULT_CONFIG_INTERVAL, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS) + ); + gstelement_class->change_state = GST_DEBUG_FUNCPTR (_change_state); gstrtpbasepayload_class->set_caps = GST_DEBUG_FUNCPTR (_set_caps); @@ -187,7 +242,12 @@ gst_rtp_h266_pay_init (GstRtpH266Pay * rtph266pay) { rtph266pay->adapter = gst_adapter_new (); g_queue_init (&rtph266pay->nalus); + rtph266pay->xps_id_nalu_map = + g_hash_table_new_full (g_direct_hash, g_direct_equal, NULL, + (GDestroyNotify) _nalu_free); rtph266pay->alignment = ALIGNMENT_UNKOWN; + rtph266pay->ts_last_xps_to_sent = GST_CLOCK_TIME_NONE; + rtph266pay->config_interval = DEFAULT_CONFIG_INTERVAL; } static void @@ -196,6 +256,7 @@ _finalize (GObject * object) GstRtpH266Pay *rtph266pay = GST_RTP_H266_PAY (object); g_clear_pointer (&rtph266pay->adapter, g_object_unref); + g_clear_pointer (&rtph266pay->xps_id_nalu_map, g_hash_table_destroy); _clear_nalu_queue (rtph266pay); G_OBJECT_CLASS (parent_class)->finalize (object); @@ -205,12 +266,36 @@ static void _set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { + GstRtpH266Pay *rtph266pay = GST_RTP_H266_PAY (object); + + GST_OBJECT_LOCK (rtph266pay); + switch (prop_id) { + case PROP_CONFIG_INTERVAL: + rtph266pay->config_interval = g_value_get_int (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } + GST_OBJECT_UNLOCK (rtph266pay); } static void _get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { + GstRtpH266Pay *rtph266pay = GST_RTP_H266_PAY (object); + + GST_OBJECT_LOCK (rtph266pay); + switch (prop_id) { + case PROP_CONFIG_INTERVAL: + g_value_set_int (value, rtph266pay->config_interval); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } + GST_OBJECT_UNLOCK (rtph266pay); } static GstStateChangeReturn @@ -226,6 +311,11 @@ _change_state (GstElement * element, GstStateChange transition) ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + // Chain up to the parent class first to avoid a race condition + // Check commit df724c410b02b82bd7db893d24e8572a06c2fcb1 + if (transition == GST_STATE_CHANGE_PAUSED_TO_READY) + _clear_xps_cache (rtph266pay); + return ret; } @@ -323,6 +413,9 @@ _sink_event (GstRTPBasePayload * rtpbasepay, GstEvent * event) GST_DEBUG_OBJECT (rtph266pay, "EOS: Draining"); ret = _push_pending_data (rtph266pay, TRUE); break; + case GST_EVENT_STREAM_START: + _clear_xps_cache (rtph266pay); + break; default: break; } @@ -351,9 +444,111 @@ _process_nalu (GstRtpH266Pay * rtph266pay, GstBuffer * nalu_buf) GST_DEBUG_OBJECT (rtph266pay, "NALU decoded: %" NALU_PTR_FORMAT, NALU_ARGS (nalu)); + _update_xps_cache (rtph266pay, nalu); g_queue_push_tail (&rtph266pay->nalus, nalu); } +static void +_update_xps_cache (GstRtpH266Pay * rtph266pay, const Nalu * nalu) +{ + GHashTable *map = rtph266pay->xps_id_nalu_map; + guint8 type = nalu->type; + guint8 id = nalu->xps_id; + Nalu *new_nalu; + gpointer *key; + + if (!NALU_IS_XPS (nalu)) + return; + + new_nalu = _nalu_copy (nalu); + new_nalu->au_start = FALSE; + new_nalu->au_end = FALSE; + key = GINT_TO_POINTER ((type << 8) | id); + if (g_hash_table_insert (map, key, new_nalu)) + GST_DEBUG_OBJECT (rtph266pay, "XPS(%u,%u) cached", type, id); + else + GST_DEBUG_OBJECT (rtph266pay, "XPS(%u,%u) replaced", type, id); + + rtph266pay->ts_last_xps_to_sent = _get_nalu_running_time (rtph266pay, nalu); +} + +static void +_clear_xps_cache (GstRtpH266Pay * rtph266pay) +{ + rtph266pay->ts_last_xps_to_sent = GST_CLOCK_TIME_NONE; + g_hash_table_remove_all (rtph266pay->xps_id_nalu_map); +} + +static gboolean +_can_insert_xps (GstRtpH266Pay * rtph266pay, const Nalu * nalu) +{ + if (_nalu_is_rsv (nalu) || !NALU_IS_VCL (nalu)) + return FALSE; // Can't insert before this kind of NALU + + GstClockTime ts_nalu = _get_nalu_running_time (rtph266pay, nalu); + GstClockTime ts_last_xps = rtph266pay->ts_last_xps_to_sent; + gboolean ts_last_xps_valid = GST_CLOCK_TIME_IS_VALID (ts_last_xps); + gboolean automatic_interval = rtph266pay->config_interval < 0; + gboolean xps_already_sent = ts_last_xps_valid && (ts_last_xps == ts_nalu); + + if (automatic_interval) { + if (!NALU_IS_IDR (nalu) || xps_already_sent) + return FALSE; + GST_DEBUG_OBJECT (rtph266pay, "IDR detected: XPS can be sent"); + return TRUE; + } else { + if (!ts_last_xps_valid) + return FALSE; // Haven't saw any XPS yet + + GstClockTime xps_period = rtph266pay->config_interval * GST_SECOND; + GstClockTime ts_next_xps = ts_last_xps + xps_period; + + if (ts_next_xps <= ts_nalu) { + GST_DEBUG_OBJECT (rtph266pay, "config-interval starved: XPS can be sent"); + return TRUE; + } + } + + return FALSE; +} + +static void +_insert_xps_cache (GstRtpH266Pay * rtph266pay) +{ + if (g_hash_table_size (rtph266pay->xps_id_nalu_map) == 0) + return; + if (rtph266pay->config_interval == 0) + return; // Disabled + + GQueue *nalus = &rtph266pay->nalus; + GList *head = g_queue_peek_head_link (nalus); + + for (GList * l = head; l != NULL; l = l->next) { + Nalu *current_nalu = l->data; + + if (!_can_insert_xps (rtph266pay, current_nalu)) + continue; + + // Insert all cached parameter sets before the current nalu + GHashTableIter iter; + Nalu *xps_nalu; + g_hash_table_iter_init (&iter, rtph266pay->xps_id_nalu_map); + while (g_hash_table_iter_next (&iter, NULL, (gpointer) & xps_nalu)) { + Nalu *xps_nalu_copy = _nalu_copy (xps_nalu); + xps_nalu_copy = _nalu_copy_ts (xps_nalu_copy, current_nalu); + GST_DEBUG_OBJECT (rtph266pay, "XPS(%u,%u) queued to be sent", + xps_nalu_copy->type, xps_nalu_copy->xps_id); + g_queue_insert_before (nalus, l, xps_nalu_copy); + } + + // Update ts_last_xps_to_sent with the PTS of the current NALU + rtph266pay->ts_last_xps_to_sent = + _get_nalu_running_time (rtph266pay, current_nalu); + + break; // XPS already inserted + } +} + static void _set_au_boundaries (GstRtpH266Pay * rtph266pay) { @@ -372,7 +567,7 @@ _set_au_boundaries (GstRtpH266Pay * rtph266pay) prev_pts = GST_BUFFER_PTS (prev_nalu->nalu_buf); prev_dts = GST_BUFFER_DTS (prev_nalu->nalu_buf); } - gboolean aud = nalu->type == NALU_TYPE_AUD; + gboolean aud = nalu->type == NALU_TYPE_AUD_NUT; gboolean new_ts = (prev_pts != pts) || (prev_dts != dts); nalu->au_start = aud || new_ts /*|| discont */ ; @@ -400,9 +595,9 @@ _push_pending_data (GstRtpH266Pay * rtph266pay, gboolean eos) Nalu *nalu; // TODO: handle AP - // TODO: send XPS if needed - // TODO: Maybe it's worth to avoid iterating the NALU list twice, maybe not + // FIXME: Maybe it's worth to avoid iterating the NALUs several times, maybe not + _insert_xps_cache (rtph266pay); _set_au_boundaries (rtph266pay); // Try to push all NALUs @@ -568,6 +763,14 @@ _clear_nalu_queue (GstRtpH266Pay * rtph266pay) g_queue_clear_full (&rtph266pay->nalus, (GDestroyNotify) _nalu_free); } +static GstClockTime +_get_nalu_running_time (const GstRtpH266Pay * rtph266pay, const Nalu * nalu) +{ + GstSegment *segment = &GST_RTP_BASE_PAYLOAD (rtph266pay)->segment; + GstClockTime pts = GST_BUFFER_PTS (nalu->nalu_buf); + return gst_segment_to_running_time (segment, GST_FORMAT_TIME, pts); +} + static Nalu * _nalu_new (GstBuffer * nalu_buf) { @@ -590,15 +793,15 @@ _nalu_new (GstBuffer * nalu_buf) nalu->size = gst_buffer_get_size (nalu_buf); nalu->type = data[1] >> 3; switch (nalu->type) { - case NALU_TYPE_VPS: - case NALU_TYPE_SPS: + case NALU_TYPE_VPS_NUT: + case NALU_TYPE_SPS_NUT: nalu->xps_id = data[2] >> 4; break; - case NALU_TYPE_PPS: + case NALU_TYPE_PPS_NUT: nalu->xps_id = data[2] >> 2; break; - case NALU_TYPE_PAPS: - case NALU_TYPE_SAPS: + case NALU_TYPE_PREFIX_APS_NUT: + case NALU_TYPE_SUFFIX_APS_NUT: nalu->xps_id = ((data[2] & 0x03) << 2) | ((data[3] & 0xC0) >> 6); break; default: @@ -627,3 +830,39 @@ _nalu_free (Nalu * nalu) gst_buffer_unref (nalu->rbsp_buf); g_free (nalu); } + +static Nalu * +_nalu_copy (const Nalu * nalu) +{ + Nalu *new_nalu = g_new (Nalu, 1); + *new_nalu = *nalu; + new_nalu->nalu_buf = gst_buffer_ref (nalu->nalu_buf); + new_nalu->hdr_buf = gst_buffer_ref (nalu->hdr_buf); + new_nalu->rbsp_buf = gst_buffer_ref (nalu->rbsp_buf); + return new_nalu; +} + +static Nalu * +_nalu_copy_ts (Nalu * dst, const Nalu * src) +{ + dst->nalu_buf = gst_buffer_make_writable (dst->nalu_buf); + gst_buffer_copy_into (dst->nalu_buf, src->nalu_buf, + GST_BUFFER_COPY_TIMESTAMPS, 0, -1); + return dst; +} + +static gboolean +_nalu_is_rsv (const Nalu * nalu) +{ + switch (nalu->type) { + case NALU_TYPE_RSV_VCL_4: + case NALU_TYPE_RSV_VCL_5: + case NALU_TYPE_RSV_VCL_6: + case NALU_TYPE_RSV_IRAP_11: + case NALU_TYPE_RSV_NVCL_26: + case NALU_TYPE_RSV_NVCL_27: + return TRUE; + default: + return FALSE; + } +} From b3d67f4bceaabcd033e06daebe46bf2ecb5c9461 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Falgueras=20Garc=C3=ADa?= Date: Fri, 22 Nov 2024 16:07:15 +0100 Subject: [PATCH 05/10] rtph266pay: Add support for Aggregation Packages Add support for two types of aggregation: * zero-latency: Only aggregate Parameter Set NALUs * max: Try to aggregate entires NALUs Issue: OCP_6005 --- .../gst-plugins-good/gst/rtp/gstrtph266pay.c | 365 ++++++++++++++++-- 1 file changed, 332 insertions(+), 33 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c index 90adcae3ce5..6be9281d311 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c @@ -81,6 +81,7 @@ typedef enum #define NALU_IS_IDR(nalu) \ (((nalu)->type == NALU_TYPE_IDR_W_RADL) || ((nalu)->type == NALU_TYPE_IDR_N_LP)) +#define AP_TYPE 28 #define FU_TYPE 29 #define FU_HDR_LEN (NALU_HDR_LEN + 1) // PayloadHdr + FU header @@ -90,6 +91,8 @@ typedef struct GstBuffer *hdr_buf; GstBuffer *rbsp_buf; guint16 size; // Size of the NALU (inluding its header) + gboolean f_bit; // forbidden_zero_bit + guint8 layer_id; // nuh_layer_id NaluType type; // nal_unit_type guint8 xps_id; // {vps_video,sps_seq,pps_pic,aps_adaptation}_parameter_set_id gboolean au_start; @@ -108,19 +111,39 @@ typedef enum ALIGNMENT_UNKOWN, } Alignment; +typedef enum +{ + AGGREGATE_NONE, + AGGREGATE_ZERO_LATENCY, + AGGREGATE_MAX, +} AggregateMode; +#define TYPE_AGGREGATE_MODE _aggregate_mode_get_type () + +typedef struct +{ + GQueue nalus; + gsize nalu_size_sum; + guint min_layer_id; + gboolean f_bit; +} AggregatedNalus; + struct _GstRtpH266Pay { GstRTPBasePayload payload; GstAdapter *adapter; GQueue nalus; + AggregatedNalus aggregated_nalus; GHashTable *xps_id_nalu_map; Alignment alignment; GstClockTime ts_last_xps_to_sent; + gint fps_n; + gint fps_d; // Properties gint config_interval; + AggregateMode aggregate_mode; }; #define gst_rtp_h266_pay_parent_class parent_class @@ -143,11 +166,13 @@ static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", ); #define DEFAULT_CONFIG_INTERVAL 0 +#define DEFAULT_AGGREGATE_MODE AGGREGATE_NONE enum { PROP_0, PROP_CONFIG_INTERVAL, + PROP_AGGREGATE_MODE, }; // GObject methods @@ -176,14 +201,17 @@ static void _insert_xps_cache (GstRtpH266Pay * rtph266pay); static void _set_au_boundaries (GstRtpH266Pay * rtph266pay); static GstFlowReturn _push_pending_data (GstRtpH266Pay * rtph266pay, gboolean eos); -static GstFlowReturn _push_up (GstRtpH266Pay * rtph266pay, const Nalu * nalu); -static GstFlowReturn _push_fu (GstRtpH266Pay * rtph266pay, const Nalu * nalu); -//static GstFlowReturn _push_ap (GstRtpH266Pay * rtph266pay, GQueue *nalus); +static GstFlowReturn _push_unit_pkt (GstRtpH266Pay * rtph266pay, Nalu * nalu); +static GstFlowReturn _push_fragmented (GstRtpH266Pay * rtph266pay, Nalu * nalu); +static GstFlowReturn _push_aggregated (GstRtpH266Pay * rtph266pay); static gboolean _up_fits_in_mtu (const GstRtpH266Pay * rtph266pay, const Nalu * nalu); +static gboolean _ap_fits_in_mtu (const GstRtpH266Pay * rtph266pay, + const Nalu * nalu); +static gboolean _can_aggregate_nalu (GstRtpH266Pay * rtph266pay, Nalu * nalu); static GstBuffer *_extract_fu (GstRtpH266Pay * rtph266pay, const Nalu * nalu, gsize offset, gsize size, gboolean last); -static void _clear_nalu_queue (GstRtpH266Pay * rtph266pay); +static void _clear_nalu_queues (GstRtpH266Pay * rtph266pay); static GstClockTime _get_nalu_running_time (const GstRtpH266Pay * rtph266pay, const Nalu * nalu); @@ -194,6 +222,18 @@ static Nalu *_nalu_copy (const Nalu * nalu); static Nalu *_nalu_copy_ts (Nalu * dst, const Nalu * src); static gboolean _nalu_is_rsv (const Nalu * nalu); +// AggregatedNalus methods +static void _aggregated_nalus_init (AggregatedNalus * aggregated_nalus); +static void _aggregated_nalus_clear (AggregatedNalus * aggregated_nalus); +static void _aggregated_nalus_add (AggregatedNalus * aggregated_nalus, + Nalu * nalu); +static gboolean _aggregated_nalus_is_empty (AggregatedNalus * aggregated_nalus); +static gboolean _aggregated_nalus_have_au (AggregatedNalus * aggregated_nalus); + +// Others +static GType _aggregate_mode_get_type (void); +static gboolean _src_query (GstPad * pad, GstObject * parent, GstQuery * query); + static void gst_rtp_h266_pay_class_init (GstRtpH266PayClass * klass) { @@ -219,6 +259,15 @@ gst_rtp_h266_pay_class_init (GstRtpH266PayClass * klass) G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS) ); + g_object_class_install_property (G_OBJECT_CLASS (klass), + PROP_AGGREGATE_MODE, + g_param_spec_enum ("aggregate-mode", + "Attempt to use aggregate packets", + "Bundle suitable Parameter Set NAL units into aggregate packets.", + TYPE_AGGREGATE_MODE, + DEFAULT_AGGREGATE_MODE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS) + ); + gstelement_class->change_state = GST_DEBUG_FUNCPTR (_change_state); gstrtpbasepayload_class->set_caps = GST_DEBUG_FUNCPTR (_set_caps); @@ -242,12 +291,17 @@ gst_rtp_h266_pay_init (GstRtpH266Pay * rtph266pay) { rtph266pay->adapter = gst_adapter_new (); g_queue_init (&rtph266pay->nalus); + _aggregated_nalus_init (&rtph266pay->aggregated_nalus); rtph266pay->xps_id_nalu_map = g_hash_table_new_full (g_direct_hash, g_direct_equal, NULL, (GDestroyNotify) _nalu_free); rtph266pay->alignment = ALIGNMENT_UNKOWN; rtph266pay->ts_last_xps_to_sent = GST_CLOCK_TIME_NONE; rtph266pay->config_interval = DEFAULT_CONFIG_INTERVAL; + rtph266pay->aggregate_mode = DEFAULT_AGGREGATE_MODE; + + gst_pad_set_query_function (GST_RTP_BASE_PAYLOAD_SRCPAD (rtph266pay), + _src_query); } static void @@ -257,7 +311,7 @@ _finalize (GObject * object) g_clear_pointer (&rtph266pay->adapter, g_object_unref); g_clear_pointer (&rtph266pay->xps_id_nalu_map, g_hash_table_destroy); - _clear_nalu_queue (rtph266pay); + _clear_nalu_queues (rtph266pay); G_OBJECT_CLASS (parent_class)->finalize (object); } @@ -273,6 +327,9 @@ _set_property (GObject * object, guint prop_id, case PROP_CONFIG_INTERVAL: rtph266pay->config_interval = g_value_get_int (value); break; + case PROP_AGGREGATE_MODE: + rtph266pay->aggregate_mode = g_value_get_enum (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -291,6 +348,9 @@ _get_property (GObject * object, guint prop_id, case PROP_CONFIG_INTERVAL: g_value_set_int (value, rtph266pay->config_interval); break; + case PROP_AGGREGATE_MODE: + g_value_set_enum (value, rtph266pay->aggregate_mode); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -306,7 +366,7 @@ _change_state (GstElement * element, GstStateChange transition) if (transition == GST_STATE_CHANGE_READY_TO_PAUSED) { gst_adapter_clear (rtph266pay->adapter); - _clear_nalu_queue (rtph266pay); + _clear_nalu_queues (rtph266pay); } ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); @@ -341,6 +401,12 @@ _set_caps (GstRTPBasePayload * rtpbasepay, GstCaps * caps) } } + gint fps_n = 0; + gint fps_d = 0; + gst_structure_get_fraction (s, "framerate", &fps_n, &fps_d); + rtph266pay->fps_n = fps_n; + rtph266pay->fps_d = fps_d; + return TRUE; } @@ -407,7 +473,7 @@ _sink_event (GstRTPBasePayload * rtpbasepay, GstEvent * event) switch (GST_EVENT_TYPE (event)) { case GST_EVENT_FLUSH_STOP: gst_adapter_clear (rtph266pay->adapter); - _clear_nalu_queue (rtph266pay); + _clear_nalu_queues (rtph266pay); break; case GST_EVENT_EOS: GST_DEBUG_OBJECT (rtph266pay, "EOS: Draining"); @@ -554,11 +620,13 @@ _set_au_boundaries (GstRtpH266Pay * rtph266pay) { GList *head = g_queue_peek_head_link (&rtph266pay->nalus); Nalu *last_nalu = g_queue_peek_tail (&rtph266pay->nalus); + // Take into account the last aggregated NALU + Nalu *prev_nalu = g_queue_peek_tail (&rtph266pay->aggregated_nalus.nalus); for (GList * l = head; l != NULL; l = l->next) { Nalu *nalu = l->data; - Nalu *prev_nalu = l->prev ? l->prev->data : NULL; - // TODO gboolean discont = GST_BUFFER_IS_DISCONT (nalu->nalu_buf); + prev_nalu = l->prev ? l->prev->data : prev_nalu; + //gboolean discont = GST_BUFFER_IS_DISCONT (nalu->nalu_buf); GstClockTime prev_pts = GST_CLOCK_TIME_NONE; GstClockTime prev_dts = GST_CLOCK_TIME_NONE; GstClockTime pts = GST_BUFFER_PTS (nalu->nalu_buf); @@ -590,48 +658,62 @@ _set_au_boundaries (GstRtpH266Pay * rtph266pay) static GstFlowReturn _push_pending_data (GstRtpH266Pay * rtph266pay, gboolean eos) { + AggregatedNalus *aggregated_nalus = &rtph266pay->aggregated_nalus; GQueue *nalus = &rtph266pay->nalus; GstFlowReturn ret = GST_FLOW_OK; Nalu *nalu; - // TODO: handle AP - // FIXME: Maybe it's worth to avoid iterating the NALUs several times, maybe not _insert_xps_cache (rtph266pay); _set_au_boundaries (rtph266pay); // Try to push all NALUs while ((nalu = g_queue_pop_head (nalus))) { - gboolean is_last = g_queue_get_length (nalus) == 0; - - // Can't push the last NALU without knowing if it's the end of an AU, - // because setting the M bit could be necessary. But have to push it if - // we're on EOS - if (!eos && is_last && !nalu->au_end) { - GST_DEBUG_OBJECT (rtph266pay, "Keeping last NALU: %" NALU_PTR_FORMAT, + if (_can_aggregate_nalu (rtph266pay, nalu)) { + _aggregated_nalus_add (aggregated_nalus, nalu); + GST_DEBUG_OBJECT (rtph266pay, "NALU aggregated: %" NALU_PTR_FORMAT, NALU_ARGS (nalu)); - g_queue_push_head (nalus, nalu); - break; + continue; } - GST_DEBUG_OBJECT (rtph266pay, "Pushing NALU: %" NALU_PTR_FORMAT, - NALU_ARGS (nalu)); + // vvv Can't aggregate anymore vvv + if (!_aggregated_nalus_is_empty (aggregated_nalus)) { + g_queue_push_head (nalus, nalu); // Keep this one for later + ret = _push_aggregated (rtph266pay); // Push what we have right now + if (ret != GST_FLOW_OK) + return ret; + } else { // Not aggregation path + gboolean is_last = g_queue_get_length (nalus) == 0; + // Can't push the last NALU without knowing if it's the end of an AU, + // because setting the M bit could be necessary. But have to push it if + // we're on EOS + if (!eos && is_last && !nalu->au_end) { + GST_DEBUG_OBJECT (rtph266pay, "Keeping last NALU: %" NALU_PTR_FORMAT, + NALU_ARGS (nalu)); + g_queue_push_head (nalus, nalu); + return GST_FLOW_OK; + } + + GST_DEBUG_OBJECT (rtph266pay, "Pushing NALU: %" NALU_PTR_FORMAT, + NALU_ARGS (nalu)); - if (_up_fits_in_mtu (rtph266pay, nalu)) - ret = _push_up (rtph266pay, nalu); - else - ret = _push_fu (rtph266pay, nalu); + if (_up_fits_in_mtu (rtph266pay, nalu)) + ret = _push_unit_pkt (rtph266pay, nalu); + else + ret = _push_fragmented (rtph266pay, nalu); - _nalu_free (nalu); - if (ret != GST_FLOW_OK) - break; + if (ret != GST_FLOW_OK) + return ret; + } } - return ret; + if (_aggregated_nalus_have_au (aggregated_nalus)) + return _push_aggregated (rtph266pay); + return GST_FLOW_OK; } static GstFlowReturn -_push_up (GstRtpH266Pay * rtph266pay, const Nalu * nalu) +_push_unit_pkt (GstRtpH266Pay * rtph266pay, Nalu * nalu) { GstRTPBasePayload *rtpbasepay = GST_RTP_BASE_PAYLOAD (rtph266pay); GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; @@ -655,15 +737,17 @@ _push_up (GstRtpH266Pay * rtph266pay, const Nalu * nalu) g_assert (out_buf); gst_rtp_buffer_unmap (&rtp); + _nalu_free (nalu); GST_DEBUG_OBJECT (rtph266pay, "Pushing UP %" GST_PTR_FORMAT, out_buf); return gst_rtp_base_payload_push (rtpbasepay, out_buf); error: + _nalu_free (nalu); gst_buffer_unref (out_buf); return GST_FLOW_ERROR; } static GstFlowReturn -_push_fu (GstRtpH266Pay * rtph266pay, const Nalu * nalu) +_push_fragmented (GstRtpH266Pay * rtph266pay, Nalu * nalu) { GstRTPBasePayload *rtpbasepay = GST_RTP_BASE_PAYLOAD (rtph266pay); GstBufferList *out_buflist = gst_buffer_list_new (); @@ -689,9 +773,80 @@ _push_fu (GstRtpH266Pay * rtph266pay, const Nalu * nalu) GST_DEBUG_OBJECT (rtph266pay, "Pushing FU GstBufferList %" GST_PTR_FORMAT, out_buflist); + _nalu_free (nalu); return gst_rtp_base_payload_push_list (rtpbasepay, out_buflist); } +static GstFlowReturn +_push_aggregated (GstRtpH266Pay * rtph266pay) +{ + GstRTPBasePayload *rtpbasepay = GST_RTP_BASE_PAYLOAD (rtph266pay); + AggregatedNalus *aggregated_nalus = &rtph266pay->aggregated_nalus; + GQueue *agg_nalus = &aggregated_nalus->nalus; + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + GstBuffer *out_buf; + guint num_nalus; + Nalu *nalu; + + // Can't send an AP with a single NALU + num_nalus = g_queue_get_length (agg_nalus); + if (num_nalus == 1) { + GST_DEBUG_OBJECT (rtph266pay, "Can't send an AP with a single NALU"); + nalu = g_queue_pop_head (agg_nalus); + _aggregated_nalus_clear (aggregated_nalus); + return _push_unit_pkt (rtph266pay, nalu); + } + + // Allocate just the RTP header. We'll add the buffer payload later. This way + // we avoid unnecessary copies + out_buf = gst_rtp_base_payload_allocate_output_buffer (rtpbasepay, 0, 0, 0); + if (!gst_rtp_buffer_map (out_buf, GST_MAP_WRITE, &rtp)) + goto error; + + // Copy buffer metadata from the last nalu to aggregate + nalu = g_queue_peek_tail (agg_nalus); + gst_buffer_copy_into (out_buf, nalu->nalu_buf, + GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS, 0, -1); + gst_rtp_buffer_set_marker (&rtp, nalu->au_end); + + // Append PayloadHdr + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // |F|Z| MinLayerID| Type(28)| TID | + // +---------------+---------------+ + guint16 f_bit_msk = (!!aggregated_nalus->f_bit) << (7 + 8); + guint16 min_layer_id_msk = ((aggregated_nalus->min_layer_id & 0x3F) << 8); + guint16 type_msk = AP_TYPE << 3; + guint16 payload_hdr = g_htons (0 | f_bit_msk | min_layer_id_msk | type_msk); + GstBuffer *payload_hdr_buf = + gst_buffer_new_memdup (&payload_hdr, sizeof (payload_hdr)); + out_buf = gst_buffer_append (out_buf, payload_hdr_buf); + + // TODO: Conditionally append DONL + + // Append all NALUs with their sizes and headers + while ((nalu = g_queue_pop_head (agg_nalus))) { + guint16 nalu_size = g_htons (nalu->size); + GstBuffer *nalu_size_buf = + gst_buffer_new_memdup (&nalu_size, sizeof (nalu_size)); + + out_buf = gst_buffer_append (out_buf, nalu_size_buf); + out_buf = gst_buffer_append (out_buf, gst_buffer_ref (nalu->hdr_buf)); + out_buf = gst_buffer_append (out_buf, gst_buffer_ref (nalu->rbsp_buf)); + _nalu_free (nalu); + } + g_assert (out_buf); + + gst_rtp_buffer_unmap (&rtp); + GST_DEBUG_OBJECT (rtph266pay, "Pushing AP [%d] %" GST_PTR_FORMAT, num_nalus, + out_buf); + _aggregated_nalus_clear (aggregated_nalus); + return gst_rtp_base_payload_push (rtpbasepay, out_buf); +error: + _aggregated_nalus_clear (aggregated_nalus); + gst_buffer_unref (out_buf); + return GST_FLOW_ERROR; +} + static gboolean _up_fits_in_mtu (const GstRtpH266Pay * rtph266pay, const Nalu * nalu) { @@ -706,6 +861,53 @@ _up_fits_in_mtu (const GstRtpH266Pay * rtph266pay, const Nalu * nalu) return fits; } +static gboolean +_ap_fits_in_mtu (const GstRtpH266Pay * rtph266pay, const Nalu * nalu) +{ + const AggregatedNalus *aggregated_nalus = &rtph266pay->aggregated_nalus; + guint mtu = GST_RTP_BASE_PAYLOAD_MTU (rtph266pay); + guint num_nalus = g_queue_get_length ((GQueue *) & aggregated_nalus->nalus); + gsize nalu_size_sum = aggregated_nalus->nalu_size_sum; + guint nalu_size_field_sum = (num_nalus + 1) * sizeof (guint16); // +1 for the current nalu + // TODO: Consider DONL + guint payload_size = + NALU_HDR_LEN + nalu_size_field_sum + nalu_size_sum + nalu->size; + + gboolean fits = gst_rtp_buffer_calc_packet_len (payload_size, 0, 0) <= mtu; + + if (!fits) { + GST_DEBUG_OBJECT (rtph266pay, + "NALU does not fit into the current AP [%u/%u]: %" GST_PTR_FORMAT, + payload_size, mtu, nalu->nalu_buf); + } + return fits; +} + +static gboolean +_can_aggregate_nalu (GstRtpH266Pay * rtph266pay, Nalu * nalu) +{ + AggregateMode aggregate_mode = rtph266pay->aggregate_mode; + + if (aggregate_mode == AGGREGATE_NONE) + return FALSE; + + if (aggregate_mode == AGGREGATE_ZERO_LATENCY) { + if (NALU_IS_VCL (nalu)) { + GST_DEBUG_OBJECT (rtph266pay, "VLC NALU -> can't aggregate"); + return FALSE; + } + } + + gboolean have_aggregated_nalus = + !_aggregated_nalus_is_empty (&rtph266pay->aggregated_nalus); + if (nalu->au_start && have_aggregated_nalus) { + GST_DEBUG_OBJECT (rtph266pay, "More than 1 AU -> can't aggregate"); + return FALSE; + } + + return _ap_fits_in_mtu (rtph266pay, nalu); +} + static GstBuffer * _extract_fu (GstRtpH266Pay * rtph266pay, const Nalu * nalu, gsize offset, gsize size, gboolean last) @@ -758,9 +960,10 @@ _extract_fu (GstRtpH266Pay * rtph266pay, const Nalu * nalu, } static void -_clear_nalu_queue (GstRtpH266Pay * rtph266pay) +_clear_nalu_queues (GstRtpH266Pay * rtph266pay) { g_queue_clear_full (&rtph266pay->nalus, (GDestroyNotify) _nalu_free); + _aggregated_nalus_clear (&rtph266pay->aggregated_nalus); } static GstClockTime @@ -791,6 +994,8 @@ _nalu_new (GstBuffer * nalu_buf) nalu = g_new (Nalu, 1); nalu->size = gst_buffer_get_size (nalu_buf); + nalu->f_bit = !!(data[0] & 0x80); + nalu->layer_id = data[0] & 0x3F; nalu->type = data[1] >> 3; switch (nalu->type) { case NALU_TYPE_VPS_NUT: @@ -866,3 +1071,97 @@ _nalu_is_rsv (const Nalu * nalu) return FALSE; } } + +static void +_aggregated_nalus_init (AggregatedNalus * aggregated_nalus) +{ + g_queue_init (&aggregated_nalus->nalus); + aggregated_nalus->nalu_size_sum = 0; + aggregated_nalus->min_layer_id = 0; + aggregated_nalus->f_bit = FALSE; +} + +static void +_aggregated_nalus_clear (AggregatedNalus * aggregated_nalus) +{ + g_queue_clear_full (&aggregated_nalus->nalus, (GDestroyNotify) _nalu_free); + aggregated_nalus->nalu_size_sum = 0; + aggregated_nalus->min_layer_id = 0; + aggregated_nalus->f_bit = FALSE; +} + +static void +_aggregated_nalus_add (AggregatedNalus * aggregated_nalus, Nalu * nalu) +{ + g_queue_push_head (&aggregated_nalus->nalus, nalu); + aggregated_nalus->nalu_size_sum += nalu->size; + if (nalu->layer_id < aggregated_nalus->min_layer_id) + aggregated_nalus->min_layer_id = nalu->layer_id; + aggregated_nalus->f_bit = aggregated_nalus->f_bit || nalu->f_bit; +} + +static gboolean +_aggregated_nalus_is_empty (AggregatedNalus * aggregated_nalus) +{ + return g_queue_get_length (&aggregated_nalus->nalus) == 0; +} + +static gboolean +_aggregated_nalus_have_au (AggregatedNalus * aggregated_nalus) +{ + Nalu *last_nalu = g_queue_peek_tail (&aggregated_nalus->nalus); + return last_nalu && last_nalu->au_end; +} + +static GType +_aggregate_mode_get_type (void) +{ + static GType type = 0; + static const GEnumValue values[] = { + {AGGREGATE_NONE, "Do not aggregate NAL units", "none"}, + {AGGREGATE_ZERO_LATENCY, + "Aggregate NAL units until a VCL or suffix unit is included", + "zero-latency"}, + {AGGREGATE_MAX, + "Aggregate all NAL units with the same timestamp (adds one frame of latency)", + "max"}, + {0, NULL, NULL}, + }; + + if (!type) { + type = g_enum_register_static ("GstRtpH266AggregateMode", values); + } + return type; +} + +static gboolean +_src_query (GstPad * pad, GstObject * parent, GstQuery * query) +{ + GstRtpH266Pay *rtph266pay = GST_RTP_H266_PAY (parent); + + if (GST_QUERY_TYPE (query) != GST_QUERY_LATENCY) + return gst_pad_query_default (pad, parent, query); + + if (rtph266pay->alignment == ALIGNMENT_UNKNOWN) + return FALSE; + + gboolean aggregate_max = rtph266pay->aggregate_mode == AGGREGATE_MAX; + gboolean au_alignment = rtph266pay->alignment == ALIGNMENT_AU; + gint fps_n = rtph266pay->fps_n; + gint fps_d = rtph266pay->fps_d; + gboolean configured = fps_n && fps_d; + if (!aggregate_max || !au_alignment || !configured) + return gst_pad_query_default (pad, parent, query); + + GstClockTime min_latency; + GstClockTime max_latency; + gboolean live; + gst_query_parse_latency (query, &live, &min_latency, &max_latency); + + GstClockTime one_frame = gst_util_uint64_scale_int (GST_SECOND, fps_d, fps_n); + min_latency += one_frame; + max_latency += one_frame; + + gst_query_set_latency (query, live, min_latency, max_latency); + return gst_pad_query_default (pad, parent, query); +} From 5400b41341bc7a874096f23b6dd1a77c5e5e158a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Falgueras=20Garc=C3=ADa?= Date: Fri, 22 Nov 2024 18:07:48 +0100 Subject: [PATCH 06/10] rtph266pay: Amend errata UNKOWN -> UNKNOWN Issue: OCP_6005 --- subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c index 6be9281d311..ee60d2fcfea 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c @@ -108,7 +108,7 @@ typedef enum { ALIGNMENT_AU, ALIGNMENT_NAL, - ALIGNMENT_UNKOWN, + ALIGNMENT_UNKNOWN, } Alignment; typedef enum @@ -295,7 +295,7 @@ gst_rtp_h266_pay_init (GstRtpH266Pay * rtph266pay) rtph266pay->xps_id_nalu_map = g_hash_table_new_full (g_direct_hash, g_direct_equal, NULL, (GDestroyNotify) _nalu_free); - rtph266pay->alignment = ALIGNMENT_UNKOWN; + rtph266pay->alignment = ALIGNMENT_UNKNOWN; rtph266pay->ts_last_xps_to_sent = GST_CLOCK_TIME_NONE; rtph266pay->config_interval = DEFAULT_CONFIG_INTERVAL; rtph266pay->aggregate_mode = DEFAULT_AGGREGATE_MODE; @@ -397,7 +397,7 @@ _set_caps (GstRTPBasePayload * rtpbasepay, GstCaps * caps) } else if (g_str_equal (alignment_str, "nal")) { rtph266pay->alignment = ALIGNMENT_NAL; } else { - rtph266pay->alignment = ALIGNMENT_UNKOWN; + rtph266pay->alignment = ALIGNMENT_UNKNOWN; } } @@ -682,7 +682,7 @@ _push_pending_data (GstRtpH266Pay * rtph266pay, gboolean eos) ret = _push_aggregated (rtph266pay); // Push what we have right now if (ret != GST_FLOW_OK) return ret; - } else { // Not aggregation path + } else { // no-aggregation path gboolean is_last = g_queue_get_length (nalus) == 0; // Can't push the last NALU without knowing if it's the end of an AU, // because setting the M bit could be necessary. But have to push it if From 2e9dd5a0d87b93f60e1d9d8b8102fa47e2ac7c0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Falgueras=20Garc=C3=ADa?= Date: Mon, 2 Dec 2024 07:56:46 +0100 Subject: [PATCH 07/10] rtph266pay: XPS -> PS refactor Remove X from XPS. Now it's only Package Set. Issue: OCP_6005 --- .../gst-plugins-good/gst/rtp/gstrtph266pay.c | 118 +++++++++--------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c index ee60d2fcfea..00e9a72dbbd 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c @@ -73,8 +73,8 @@ typedef enum #define NALU_SC_VAL 0x00000100 #define NALU_SC_LEN 3 #define NALU_HDR_LEN 2 -#define NALU_INVALID_XPS 0xFF -#define NALU_IS_XPS(nalu) \ +#define NALU_INVALID_PS 0xFF +#define NALU_IS_PS(nalu) \ (((nalu)->type >= NALU_TYPE_VPS_NUT) && ((nalu)->type <= NALU_TYPE_SUFFIX_APS_NUT)) #define NALU_IS_VCL(nalu) \ (((nalu)->type >= NALU_TYPE_TRAIL_NUT) && ((nalu)->type <= NALU_TYPE_RSV_IRAP_11)) @@ -94,15 +94,15 @@ typedef struct gboolean f_bit; // forbidden_zero_bit guint8 layer_id; // nuh_layer_id NaluType type; // nal_unit_type - guint8 xps_id; // {vps_video,sps_seq,pps_pic,aps_adaptation}_parameter_set_id + guint8 ps_id; // {vps_video,sps_seq,pps_pic,aps_adaptation}_parameter_set_id gboolean au_start; gboolean au_end; } Nalu; #define NALU_PTR_FORMAT \ - "p, size: %u, type: %u, xps_id: %u, au_start: %s, au_end: %s" + "p, size: %u, type: %u, ps_id: %u, au_start: %s, au_end: %s" #define NALU_ARGS(nalu) \ - (nalu), (nalu)->size, (nalu)->type, (nalu)->xps_id, (nalu)->au_start ? "true" : "false", (nalu)->au_end ? "true" : "false" + (nalu), (nalu)->size, (nalu)->type, (nalu)->ps_id, (nalu)->au_start ? "true" : "false", (nalu)->au_end ? "true" : "false" typedef enum { @@ -134,10 +134,10 @@ struct _GstRtpH266Pay GstAdapter *adapter; GQueue nalus; AggregatedNalus aggregated_nalus; - GHashTable *xps_id_nalu_map; + GHashTable *ps_id_nalu_map; Alignment alignment; - GstClockTime ts_last_xps_to_sent; + GstClockTime ts_last_ps_to_sent; gint fps_n; gint fps_d; @@ -194,10 +194,10 @@ static gboolean _sink_event (GstRTPBasePayload * rtpbasepay, GstEvent * event); // GstRtpH266Pay methods static void _process_nalu (GstRtpH266Pay * rtph266pay, GstBuffer * nalu_buf); -static void _update_xps_cache (GstRtpH266Pay * rtph266pay, const Nalu * nalu); -static void _clear_xps_cache (GstRtpH266Pay * rtph266pay); -static gboolean _can_insert_xps (GstRtpH266Pay * rtph266pay, const Nalu * nalu); -static void _insert_xps_cache (GstRtpH266Pay * rtph266pay); +static void _update_ps_cache (GstRtpH266Pay * rtph266pay, const Nalu * nalu); +static void _clear_ps_cache (GstRtpH266Pay * rtph266pay); +static gboolean _can_insert_ps (GstRtpH266Pay * rtph266pay, const Nalu * nalu); +static void _insert_ps_cache (GstRtpH266Pay * rtph266pay); static void _set_au_boundaries (GstRtpH266Pay * rtph266pay); static GstFlowReturn _push_pending_data (GstRtpH266Pay * rtph266pay, gboolean eos); @@ -292,11 +292,11 @@ gst_rtp_h266_pay_init (GstRtpH266Pay * rtph266pay) rtph266pay->adapter = gst_adapter_new (); g_queue_init (&rtph266pay->nalus); _aggregated_nalus_init (&rtph266pay->aggregated_nalus); - rtph266pay->xps_id_nalu_map = + rtph266pay->ps_id_nalu_map = g_hash_table_new_full (g_direct_hash, g_direct_equal, NULL, (GDestroyNotify) _nalu_free); rtph266pay->alignment = ALIGNMENT_UNKNOWN; - rtph266pay->ts_last_xps_to_sent = GST_CLOCK_TIME_NONE; + rtph266pay->ts_last_ps_to_sent = GST_CLOCK_TIME_NONE; rtph266pay->config_interval = DEFAULT_CONFIG_INTERVAL; rtph266pay->aggregate_mode = DEFAULT_AGGREGATE_MODE; @@ -310,7 +310,7 @@ _finalize (GObject * object) GstRtpH266Pay *rtph266pay = GST_RTP_H266_PAY (object); g_clear_pointer (&rtph266pay->adapter, g_object_unref); - g_clear_pointer (&rtph266pay->xps_id_nalu_map, g_hash_table_destroy); + g_clear_pointer (&rtph266pay->ps_id_nalu_map, g_hash_table_destroy); _clear_nalu_queues (rtph266pay); G_OBJECT_CLASS (parent_class)->finalize (object); @@ -374,7 +374,7 @@ _change_state (GstElement * element, GstStateChange transition) // Chain up to the parent class first to avoid a race condition // Check commit df724c410b02b82bd7db893d24e8572a06c2fcb1 if (transition == GST_STATE_CHANGE_PAUSED_TO_READY) - _clear_xps_cache (rtph266pay); + _clear_ps_cache (rtph266pay); return ret; } @@ -480,7 +480,7 @@ _sink_event (GstRTPBasePayload * rtpbasepay, GstEvent * event) ret = _push_pending_data (rtph266pay, TRUE); break; case GST_EVENT_STREAM_START: - _clear_xps_cache (rtph266pay); + _clear_ps_cache (rtph266pay); break; default: break; @@ -510,20 +510,20 @@ _process_nalu (GstRtpH266Pay * rtph266pay, GstBuffer * nalu_buf) GST_DEBUG_OBJECT (rtph266pay, "NALU decoded: %" NALU_PTR_FORMAT, NALU_ARGS (nalu)); - _update_xps_cache (rtph266pay, nalu); + _update_ps_cache (rtph266pay, nalu); g_queue_push_tail (&rtph266pay->nalus, nalu); } static void -_update_xps_cache (GstRtpH266Pay * rtph266pay, const Nalu * nalu) +_update_ps_cache (GstRtpH266Pay * rtph266pay, const Nalu * nalu) { - GHashTable *map = rtph266pay->xps_id_nalu_map; + GHashTable *map = rtph266pay->ps_id_nalu_map; guint8 type = nalu->type; - guint8 id = nalu->xps_id; + guint8 id = nalu->ps_id; Nalu *new_nalu; gpointer *key; - if (!NALU_IS_XPS (nalu)) + if (!NALU_IS_PS (nalu)) return; new_nalu = _nalu_copy (nalu); @@ -531,46 +531,46 @@ _update_xps_cache (GstRtpH266Pay * rtph266pay, const Nalu * nalu) new_nalu->au_end = FALSE; key = GINT_TO_POINTER ((type << 8) | id); if (g_hash_table_insert (map, key, new_nalu)) - GST_DEBUG_OBJECT (rtph266pay, "XPS(%u,%u) cached", type, id); + GST_DEBUG_OBJECT (rtph266pay, "PS(%u,%u) cached", type, id); else - GST_DEBUG_OBJECT (rtph266pay, "XPS(%u,%u) replaced", type, id); + GST_DEBUG_OBJECT (rtph266pay, "PS(%u,%u) replaced", type, id); - rtph266pay->ts_last_xps_to_sent = _get_nalu_running_time (rtph266pay, nalu); + rtph266pay->ts_last_ps_to_sent = _get_nalu_running_time (rtph266pay, nalu); } static void -_clear_xps_cache (GstRtpH266Pay * rtph266pay) +_clear_ps_cache (GstRtpH266Pay * rtph266pay) { - rtph266pay->ts_last_xps_to_sent = GST_CLOCK_TIME_NONE; - g_hash_table_remove_all (rtph266pay->xps_id_nalu_map); + rtph266pay->ts_last_ps_to_sent = GST_CLOCK_TIME_NONE; + g_hash_table_remove_all (rtph266pay->ps_id_nalu_map); } static gboolean -_can_insert_xps (GstRtpH266Pay * rtph266pay, const Nalu * nalu) +_can_insert_ps (GstRtpH266Pay * rtph266pay, const Nalu * nalu) { if (_nalu_is_rsv (nalu) || !NALU_IS_VCL (nalu)) return FALSE; // Can't insert before this kind of NALU GstClockTime ts_nalu = _get_nalu_running_time (rtph266pay, nalu); - GstClockTime ts_last_xps = rtph266pay->ts_last_xps_to_sent; - gboolean ts_last_xps_valid = GST_CLOCK_TIME_IS_VALID (ts_last_xps); + GstClockTime ts_last_ps = rtph266pay->ts_last_ps_to_sent; + gboolean ts_last_ps_valid = GST_CLOCK_TIME_IS_VALID (ts_last_ps); gboolean automatic_interval = rtph266pay->config_interval < 0; - gboolean xps_already_sent = ts_last_xps_valid && (ts_last_xps == ts_nalu); + gboolean ps_already_sent = ts_last_ps_valid && (ts_last_ps == ts_nalu); if (automatic_interval) { - if (!NALU_IS_IDR (nalu) || xps_already_sent) + if (!NALU_IS_IDR (nalu) || ps_already_sent) return FALSE; - GST_DEBUG_OBJECT (rtph266pay, "IDR detected: XPS can be sent"); + GST_DEBUG_OBJECT (rtph266pay, "IDR detected: PS can be sent"); return TRUE; } else { - if (!ts_last_xps_valid) - return FALSE; // Haven't saw any XPS yet + if (!ts_last_ps_valid) + return FALSE; // Haven't saw any PS yet - GstClockTime xps_period = rtph266pay->config_interval * GST_SECOND; - GstClockTime ts_next_xps = ts_last_xps + xps_period; + GstClockTime ps_period = rtph266pay->config_interval * GST_SECOND; + GstClockTime ts_next_ps = ts_last_ps + ps_period; - if (ts_next_xps <= ts_nalu) { - GST_DEBUG_OBJECT (rtph266pay, "config-interval starved: XPS can be sent"); + if (ts_next_ps <= ts_nalu) { + GST_DEBUG_OBJECT (rtph266pay, "config-interval starved: PS can be sent"); return TRUE; } } @@ -579,9 +579,9 @@ _can_insert_xps (GstRtpH266Pay * rtph266pay, const Nalu * nalu) } static void -_insert_xps_cache (GstRtpH266Pay * rtph266pay) +_insert_ps_cache (GstRtpH266Pay * rtph266pay) { - if (g_hash_table_size (rtph266pay->xps_id_nalu_map) == 0) + if (g_hash_table_size (rtph266pay->ps_id_nalu_map) == 0) return; if (rtph266pay->config_interval == 0) return; // Disabled @@ -592,26 +592,26 @@ _insert_xps_cache (GstRtpH266Pay * rtph266pay) for (GList * l = head; l != NULL; l = l->next) { Nalu *current_nalu = l->data; - if (!_can_insert_xps (rtph266pay, current_nalu)) + if (!_can_insert_ps (rtph266pay, current_nalu)) continue; // Insert all cached parameter sets before the current nalu GHashTableIter iter; - Nalu *xps_nalu; - g_hash_table_iter_init (&iter, rtph266pay->xps_id_nalu_map); - while (g_hash_table_iter_next (&iter, NULL, (gpointer) & xps_nalu)) { - Nalu *xps_nalu_copy = _nalu_copy (xps_nalu); - xps_nalu_copy = _nalu_copy_ts (xps_nalu_copy, current_nalu); - GST_DEBUG_OBJECT (rtph266pay, "XPS(%u,%u) queued to be sent", - xps_nalu_copy->type, xps_nalu_copy->xps_id); - g_queue_insert_before (nalus, l, xps_nalu_copy); + Nalu *ps_nalu; + g_hash_table_iter_init (&iter, rtph266pay->ps_id_nalu_map); + while (g_hash_table_iter_next (&iter, NULL, (gpointer) & ps_nalu)) { + Nalu *ps_nalu_copy = _nalu_copy (ps_nalu); + ps_nalu_copy = _nalu_copy_ts (ps_nalu_copy, current_nalu); + GST_DEBUG_OBJECT (rtph266pay, "PS(%u,%u) queued to be sent", + ps_nalu_copy->type, ps_nalu_copy->ps_id); + g_queue_insert_before (nalus, l, ps_nalu_copy); } - // Update ts_last_xps_to_sent with the PTS of the current NALU - rtph266pay->ts_last_xps_to_sent = + // Update ts_last_ps_to_sent with the PTS of the current NALU + rtph266pay->ts_last_ps_to_sent = _get_nalu_running_time (rtph266pay, current_nalu); - break; // XPS already inserted + break; // PS already inserted } } @@ -664,7 +664,7 @@ _push_pending_data (GstRtpH266Pay * rtph266pay, gboolean eos) Nalu *nalu; // FIXME: Maybe it's worth to avoid iterating the NALUs several times, maybe not - _insert_xps_cache (rtph266pay); + _insert_ps_cache (rtph266pay); _set_au_boundaries (rtph266pay); // Try to push all NALUs @@ -977,7 +977,7 @@ _get_nalu_running_time (const GstRtpH266Pay * rtph266pay, const Nalu * nalu) static Nalu * _nalu_new (GstBuffer * nalu_buf) { - // | NALU HDR | 2 first bytes of XPS + // | NALU HDR | 2 first bytes of PS // |-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-|-+-+-+-+-+-+-+-:-+-+-+-+-+-+-+-:-+-+ // |F|Z| LayerID | Type | TID | [VPS, PPS, SPS, PAPS, SAPS] : ... // +---------------+---------------|---------------:---------------:---- @@ -1000,17 +1000,17 @@ _nalu_new (GstBuffer * nalu_buf) switch (nalu->type) { case NALU_TYPE_VPS_NUT: case NALU_TYPE_SPS_NUT: - nalu->xps_id = data[2] >> 4; + nalu->ps_id = data[2] >> 4; break; case NALU_TYPE_PPS_NUT: - nalu->xps_id = data[2] >> 2; + nalu->ps_id = data[2] >> 2; break; case NALU_TYPE_PREFIX_APS_NUT: case NALU_TYPE_SUFFIX_APS_NUT: - nalu->xps_id = ((data[2] & 0x03) << 2) | ((data[3] & 0xC0) >> 6); + nalu->ps_id = ((data[2] & 0x03) << 2) | ((data[3] & 0xC0) >> 6); break; default: - nalu->xps_id = NALU_INVALID_XPS; + nalu->ps_id = NALU_INVALID_PS; } nalu->nalu_buf = nalu_buf; From 57e25868734ef0c8dd86b1b46004ba5c4742a0c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Falgueras=20Garc=C3=ADa?= Date: Mon, 2 Dec 2024 08:15:57 +0100 Subject: [PATCH 08/10] rtph266pay: Add explanatory commentaries Explain the more complex part of the algorithm, as it was proveen to be hard to understand. Issue: OCP_6005 --- .../gst-plugins-good/gst/rtp/gstrtph266pay.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c index 00e9a72dbbd..e2ba3d346de 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c @@ -667,8 +667,18 @@ _push_pending_data (GstRtpH266Pay * rtph266pay, gboolean eos) _insert_ps_cache (rtph266pay); _set_au_boundaries (rtph266pay); - // Try to push all NALUs + // 1. Aggregate NALUs while possible + // ---- vvv Can't aggregate anymore vvv ---- + // 2. If it has been possible to **aggregate something**: + // 1. Keep the current NALU, it'll be processed later + // 2. Send the current Aggregation Packet + // 3. If it has been possible to **aggregate nothing**: + // * If the packet fits into a MTP -> Send a Unit Packet + // * Else -> Send two or more Fragmentation Units + // + // Note: _push_aggregated will send a UP when there is only 1 NALU aggregated while ((nalu = g_queue_pop_head (nalus))) { + // Aggregate NALUs while possible if (_can_aggregate_nalu (rtph266pay, nalu)) { _aggregated_nalus_add (aggregated_nalus, nalu); GST_DEBUG_OBJECT (rtph266pay, "NALU aggregated: %" NALU_PTR_FORMAT, @@ -707,6 +717,8 @@ _push_pending_data (GstRtpH266Pay * rtph266pay, gboolean eos) } } + // If we already have an Access Unit aggregated, sent it right now instead of + // waiting for the next buffer. An AP can't have more than 1 AU aggregated. if (_aggregated_nalus_have_au (aggregated_nalus)) return _push_aggregated (rtph266pay); return GST_FLOW_OK; From cf25ff121dc495eb51dc32f00684e321d8ffe234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Falgueras=20Garc=C3=ADa?= Date: Mon, 2 Dec 2024 08:18:37 +0100 Subject: [PATCH 09/10] rtph266pay: Update NALU_PTR_FORMAT and NALU_ARGS Add missing Nalu fields to these parameters. Issue: OCP_6005 --- subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c index e2ba3d346de..6beca6b9fd7 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c @@ -100,9 +100,9 @@ typedef struct } Nalu; #define NALU_PTR_FORMAT \ - "p, size: %u, type: %u, ps_id: %u, au_start: %s, au_end: %s" + "p, size: %u, f_bit: %d, layer_id: %d, type: %u, ps_id: %u, au_start: %s, au_end: %s" #define NALU_ARGS(nalu) \ - (nalu), (nalu)->size, (nalu)->type, (nalu)->ps_id, (nalu)->au_start ? "true" : "false", (nalu)->au_end ? "true" : "false" + (nalu), (nalu)->size, (nalu)->f_bit, (nalu)->layer_id, (nalu)->type, (nalu)->ps_id, (nalu)->au_start ? "true" : "false", (nalu)->au_end ? "true" : "false" typedef enum { From 0874ee813c93a98ccd2145cd633f5112d32f8f2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Falgueras=20Garc=C3=ADa?= Date: Wed, 4 Dec 2024 08:00:55 +0100 Subject: [PATCH 10/10] rtph266pay: Add references to section numbers Add commentaries with references to section numbers of the standards. Issue: OCP_6005 --- subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c index 6beca6b9fd7..8604ae5cf41 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph266pay.c @@ -821,7 +821,7 @@ _push_aggregated (GstRtpH266Pay * rtph266pay) GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS, 0, -1); gst_rtp_buffer_set_marker (&rtp, nalu->au_end); - // Append PayloadHdr + // Append PayloadHdr (RFC 9328 4.3.2) // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ // |F|Z| MinLayerID| Type(28)| TID | // +---------------+---------------+ @@ -952,7 +952,7 @@ _extract_fu (GstRtpH266Pay * rtph266pay, const Nalu * nalu, fu_hdr = gst_rtp_buffer_get_payload (&rtp); g_assert (fu_hdr); - // Setup PayloadHdr and FU Header + // Setup PayloadHdr and FU Header (RFC 9328 4.3.3) // | PayloadHdr (NALU HDR) | FU HEADER | // |-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-|-+-+-+-+-+-+-+-| // |F|Z| LayerID | Type(29)| TID |S|E|P| FuType | @@ -989,6 +989,7 @@ _get_nalu_running_time (const GstRtpH266Pay * rtph266pay, const Nalu * nalu) static Nalu * _nalu_new (GstBuffer * nalu_buf) { + // ITU-T H.266 V3: 7.3.1.2, 7.3.2.3, 7.3.2.4, 7.3.2.5, 7.3.2.6 // | NALU HDR | 2 first bytes of PS // |-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-|-+-+-+-+-+-+-+-:-+-+-+-+-+-+-+-:-+-+ // |F|Z| LayerID | Type | TID | [VPS, PPS, SPS, PAPS, SAPS] : ...