From: Michael Froman Date: Thu, 27 Jul 2023 12:42:44 -0500 Subject: Bug 1838080: Store the rid in TransformableVideoSenderFrame. r=ng,webrtc-reviewers This is necessary to reliably detect what rid a given keyframe is for, for the purposes of resolving promises from RTCRtpScriptTransformer.generateKeyFrame. Differential Revision: https://phabricator.services.mozilla.com/D180737 Mercurial Revision: https://hg.mozilla.org/mozilla-central/rev/2f1a0ba74bf71cfa0bc4e77714b8a5276a70cc36 --- api/frame_transformer_interface.h | 1 + modules/rtp_rtcp/source/rtp_sender.h | 4 ++++ modules/rtp_rtcp/source/rtp_sender_video.cc | 1 + ...rtp_sender_video_frame_transformer_delegate.cc | 15 +++++++++++---- .../rtp_sender_video_frame_transformer_delegate.h | 2 ++ ..._stream_receiver_frame_transformer_delegate.cc | 5 +++++ 6 files changed, 24 insertions(+), 4 deletions(-) diff --git a/api/frame_transformer_interface.h b/api/frame_transformer_interface.h index db3a8cdac3..f6b884fd7d 100644 --- a/api/frame_transformer_interface.h +++ b/api/frame_transformer_interface.h @@ -113,6 +113,7 @@ class TransformableVideoFrameInterface : public TransformableFrameInterface { RTC_EXPORT explicit TransformableVideoFrameInterface(Passkey passkey); virtual ~TransformableVideoFrameInterface() = default; virtual bool IsKeyFrame() const = 0; + virtual const std::string& GetRid() const = 0; virtual VideoFrameMetadata Metadata() const = 0; diff --git a/modules/rtp_rtcp/source/rtp_sender.h b/modules/rtp_rtcp/source/rtp_sender.h index 80c1c84043..2c900dde28 100644 --- a/modules/rtp_rtcp/source/rtp_sender.h +++ b/modules/rtp_rtcp/source/rtp_sender.h @@ -140,6 +140,10 @@ class RTPSender { uint32_t SSRC() const RTC_LOCKS_EXCLUDED(send_mutex_) { return ssrc_; } + const std::string& Rid() const RTC_LOCKS_EXCLUDED(send_mutex_) { + return rid_; + } + std::optional FlexfecSsrc() const RTC_LOCKS_EXCLUDED(send_mutex_) { return flexfec_ssrc_; } diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc index 8503b0db75..d19bb9d3ae 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -189,6 +189,7 @@ RTPSenderVideo::RTPSenderVideo(const Config& config) this, config.frame_transformer, rtp_sender_->SSRC(), + rtp_sender_->Rid(), config.task_queue_factory) : nullptr) { if (frame_transformer_delegate_) diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc index 7f9a80ed4e..9fe88fdabd 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc @@ -55,7 +55,8 @@ class TransformableVideoSenderFrame : public TransformableVideoFrameInterface { uint32_t rtp_timestamp, TimeDelta expected_retransmission_time, uint32_t ssrc, - std::vector csrcs) + std::vector csrcs, + const std::string& rid) : TransformableVideoFrameInterface(Passkey()), encoded_data_(encoded_image.GetEncodedData()), pre_transform_payload_size_(encoded_image.size()), @@ -68,7 +69,8 @@ class TransformableVideoSenderFrame : public TransformableVideoFrameInterface { presentation_timestamp_(encoded_image.PresentationTimestamp()), expected_retransmission_time_(expected_retransmission_time), ssrc_(ssrc), - csrcs_(csrcs) { + csrcs_(csrcs), + rid_(rid) { RTC_DCHECK_GE(payload_type_, 0); RTC_DCHECK_LE(payload_type_, 127); } @@ -141,6 +143,8 @@ class TransformableVideoSenderFrame : public TransformableVideoFrameInterface { return std::nullopt; } + const std::string& GetRid() const override { return rid_; } + private: scoped_refptr encoded_data_; const size_t pre_transform_payload_size_; @@ -155,16 +159,19 @@ class TransformableVideoSenderFrame : public TransformableVideoFrameInterface { uint32_t ssrc_; std::vector csrcs_; + const std::string rid_; }; RTPSenderVideoFrameTransformerDelegate::RTPSenderVideoFrameTransformerDelegate( RTPVideoFrameSenderInterface* sender, scoped_refptr frame_transformer, uint32_t ssrc, + const std::string& rid, TaskQueueFactory* task_queue_factory) : sender_(sender), frame_transformer_(std::move(frame_transformer)), ssrc_(ssrc), + rid_(rid), transformation_queue_(task_queue_factory->CreateTaskQueue( "video_frame_transformer", TaskQueueFactory::Priority::NORMAL)) {} @@ -194,7 +201,7 @@ bool RTPSenderVideoFrameTransformerDelegate::TransformFrame( } frame_transformer_->Transform(std::make_unique( encoded_image, video_header, payload_type, codec_type, rtp_timestamp, - expected_retransmission_time, ssrc_, csrcs)); + expected_retransmission_time, ssrc_, csrcs, rid_)); return true; } @@ -297,7 +304,7 @@ std::unique_ptr CloneSenderVideoFrame( return std::make_unique( encoded_image, new_header, original->GetPayloadType(), new_header.codec, original->GetTimestamp(), kDefaultRetransmissionsTime, - original->GetSsrc(), metadata.GetCsrcs()); + original->GetSsrc(), metadata.GetCsrcs(), original->GetRid()); } } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h index 2e84e1f074..9c20a6ce78 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h @@ -67,6 +67,7 @@ class RTPSenderVideoFrameTransformerDelegate : public TransformedFrameCallback { RTPVideoFrameSenderInterface* sender, scoped_refptr frame_transformer, uint32_t ssrc, + const std::string& rid, TaskQueueFactory* send_transport_queue); void Init(); @@ -116,6 +117,7 @@ class RTPSenderVideoFrameTransformerDelegate : public TransformedFrameCallback { RTPVideoFrameSenderInterface* sender_ RTC_GUARDED_BY(sender_lock_); scoped_refptr frame_transformer_; const uint32_t ssrc_; + const std::string rid_; // Used when the encoded frames arrives without a current task queue. This can // happen if a hardware encoder was used. std::unique_ptr transformation_queue_; diff --git a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc index b3a77c197f..59c2e9a875 100644 --- a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc +++ b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc @@ -75,6 +75,11 @@ class TransformableVideoReceiverFrame return frame_->FrameType() == VideoFrameType::kVideoFrameKey; } + const std::string& GetRid() const override { + static const std::string empty; + return empty; + } + VideoFrameMetadata Metadata() const override { return metadata_; } void SetMetadata(const VideoFrameMetadata& metadata) override {