File webrtc-aom3.8-AV1E_SET_MAX_CONSEC_FRAME_DROP_CBR.patch of Package nodejs-electron
From f7a15067031cae07b1dac031fa263a9f01902f1c Mon Sep 17 00:00:00 2001
From: Sergey Silkin <ssilkin@webrtc.org>
Date: Tue, 9 Jul 2024 15:03:10 +0200
Subject: [PATCH] Adjust max consecutive drops depending on target frame rate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Current thresholds were tuned to guarantee no buffer overshoot in an extreme scenario (encoding a high complexity video in a low bitrate).
Bug: b/337757868, webrtc:351644568
Change-Id: I832b2564af6f18f06550338cc9b3618f8acdf831
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/356580
Reviewed-by: Dan Tan <dwtan@google.com>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42620}
---
experiments/field_trials.py | 6 +--
modules/video_coding/codecs/av1/BUILD.gn | 2 +
.../codecs/av1/libaom_av1_encoder.cc | 42 +++++++++------
.../codecs/av1/libaom_av1_encoder_unittest.cc | 53 ++++++++++++++-----
4 files changed, 69 insertions(+), 34 deletions(-)
diff --git a/modules/video_coding/codecs/av1/BUILD.gn b/modules/video_coding/codecs/av1/BUILD.gn
index 7d93de7a3a..197e1f3e19 100644
--- a/third_party/webrtc/modules/video_coding/codecs/av1/BUILD.gn
+++ b/third_party/webrtc/modules/video_coding/codecs/av1/BUILD.gn
@@ -60,6 +60,7 @@ rtc_library("libaom_av1_encoder") {
"../../../../api/video_codecs:scalability_mode",
"../../../../api/video_codecs:video_codecs_api",
"../../../../common_video",
+ "../../../../modules/rtp_rtcp:rtp_rtcp_format",
"../../../../rtc_base:checks",
"../../../../rtc_base:logging",
"../../../../rtc_base:rtc_numerics",
@@ -104,6 +105,7 @@ if (rtc_include_tests) {
"../../../../api/units:data_size",
"../../../../api/units:time_delta",
"../../../../api/video:video_frame",
+ "../../../../modules/rtp_rtcp:rtp_rtcp_format",
"../../../../test:scoped_key_value_config",
"../../svc:scalability_mode_util",
"../../svc:scalability_structures",
diff --git a/modules/video_coding/codecs/av1/libaom_av1_encoder.cc b/modules/video_coding/codecs/av1/libaom_av1_encoder.cc
index 3a8830252d..258d3d3b88 100644
--- a/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_encoder.cc
+++ b/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_encoder.cc
@@ -30,6 +30,7 @@
#include "api/video_codecs/scalability_mode.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_encoder.h"
+#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "modules/video_coding/svc/create_scalability_structure.h"
@@ -65,7 +66,6 @@ constexpr int kLowQindex = 145; // Low qindex threshold for QP scaling.
constexpr int kHighQindex = 205; // High qindex threshold for QP scaling.
constexpr int kBitDepth = 8;
constexpr int kLagInFrames = 0; // No look ahead.
-constexpr int kRtpTicksPerSecond = 90000;
constexpr double kMinFrameRateFps = 1.0;
aom_superblock_size_t GetSuperblockSize(int width, int height, int threads) {
@@ -133,7 +133,9 @@ class LibaomAv1Encoder final : public VideoEncoder {
double framerate_fps_; // Current target frame rate.
int64_t timestamp_;
const LibaomAv1EncoderInfoSettings encoder_info_override_;
- int max_consec_frame_drop_;
+ // TODO(webrtc:351644568): Remove this kill-switch after the feature is fully
+ // deployed.
+ bool adaptive_max_consec_drops_;
};
int32_t VerifyCodecSettings(const VideoCodec& codec_settings) {
@@ -164,12 +166,12 @@ int32_t VerifyCodecSettings(const VideoCodec& codec_settings) {
return WEBRTC_VIDEO_CODEC_OK;
}
-int GetMaxConsecutiveFrameDrop(const FieldTrialsView& field_trials) {
- webrtc::FieldTrialParameter<int> maxdrop("maxdrop", 0);
- webrtc::ParseFieldTrial(
- {&maxdrop},
- field_trials.Lookup("WebRTC-LibaomAv1Encoder-MaxConsecFrameDrop"));
- return maxdrop;
+int GetMaxConsecDrops(double framerate_fps) {
+ // Consecutive frame drops result in a video freeze. We want to minimize the
+ // max number of consecutive drops and, at the same time, keep the value high
+ // enough to let encoder drain the buffer at overshoot.
+ constexpr double kMaxFreezeSeconds = 0.25;
+ return std::ceil(kMaxFreezeSeconds * framerate_fps);
}
LibaomAv1Encoder::LibaomAv1Encoder(const Environment& env,
@@ -182,7 +184,8 @@ LibaomAv1Encoder::LibaomAv1Encoder(const Environment& env,
framerate_fps_(0),
timestamp_(0),
encoder_info_override_(env.field_trials()),
- max_consec_frame_drop_(GetMaxConsecutiveFrameDrop(env.field_trials())) {}
+ adaptive_max_consec_drops_(!env.field_trials().IsDisabled(
+ "WebRTC-LibaomAv1Encoder-AdaptiveMaxConsecDrops")) {}
LibaomAv1Encoder::~LibaomAv1Encoder() {
Release();
@@ -242,7 +245,7 @@ int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings,
cfg_.g_threads =
NumberOfThreads(cfg_.g_w, cfg_.g_h, settings.number_of_cores);
cfg_.g_timebase.num = 1;
- cfg_.g_timebase.den = kRtpTicksPerSecond;
+ cfg_.g_timebase.den = kVideoPayloadTypeFrequency;
cfg_.rc_target_bitrate = encoder_settings_.startBitrate; // kilobits/sec.
cfg_.rc_dropframe_thresh = encoder_settings_.GetFrameDropEnabled() ? 30 : 0;
cfg_.g_input_bit_depth = kBitDepth;
@@ -304,12 +307,6 @@ int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings,
SET_ENCODER_PARAM_OR_RETURN_ERROR(AV1E_SET_ENABLE_PALETTE, 0);
}
- if (codec_settings->mode == VideoCodecMode::kRealtimeVideo &&
- encoder_settings_.GetFrameDropEnabled() && max_consec_frame_drop_ > 0) {
- SET_ENCODER_PARAM_OR_RETURN_ERROR(AV1E_SET_MAX_CONSEC_FRAME_DROP_CBR,
- max_consec_frame_drop_);
- }
-
if (cfg_.g_threads == 8) {
// Values passed to AV1E_SET_TILE_ROWS and AV1E_SET_TILE_COLUMNS are log2()
// based.
@@ -659,7 +656,7 @@ int32_t LibaomAv1Encoder::Encode(
return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
}
- const uint32_t duration = kRtpTicksPerSecond / framerate_fps_;
+ const uint32_t duration = kVideoPayloadTypeFrequency / framerate_fps_;
timestamp_ += duration;
const size_t num_spatial_layers =
@@ -836,6 +833,17 @@ void LibaomAv1Encoder::SetRates(const RateControlParameters& parameters) {
SetEncoderControlParameters(AV1E_SET_SVC_PARAMS, &*svc_params_);
}
+ if (adaptive_max_consec_drops_ &&
+ (!rates_configured_ || framerate_fps_ != parameters.framerate_fps)) {
+ int max_consec_drops = GetMaxConsecDrops(parameters.framerate_fps);
+ if (!SetEncoderControlParameters(AV1E_SET_MAX_CONSEC_FRAME_DROP_CBR,
+ max_consec_drops)) {
+ RTC_LOG(LS_WARNING)
+ << "Failed to set AV1E_SET_MAX_CONSEC_FRAME_DROP_CBR to "
+ << max_consec_drops;
+ }
+ }
+
framerate_fps_ = parameters.framerate_fps;
rates_configured_ = true;
diff --git a/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc b/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc
index abb6fce0cf..a00b03aeda 100644
--- a/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc
+++ b/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc
@@ -10,6 +10,7 @@
#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
+#include <algorithm>
#include <limits>
#include <memory>
#include <utility>
@@ -22,6 +23,7 @@
#include "api/test/frame_generator_interface.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_encoder.h"
+#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/video_coding/codecs/test/encoded_video_frame_producer.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "test/gmock.h"
@@ -37,6 +39,7 @@ using ::testing::Eq;
using ::testing::Field;
using ::testing::IsEmpty;
using ::testing::SizeIs;
+using ::testing::Values;
VideoCodec DefaultCodecSettings() {
VideoCodec codec_settings;
@@ -199,32 +202,54 @@ TEST(LibaomAv1EncoderTest, CheckOddDimensionsWithSpatialLayers) {
ASSERT_THAT(encoded_frames, SizeIs(6));
}
-TEST(LibaomAv1EncoderTest, WithMaximumConsecutiveFrameDrop) {
- auto field_trials = std::make_unique<ScopedKeyValueConfig>(
- "WebRTC-LibaomAv1Encoder-MaxConsecFrameDrop/maxdrop:2/");
- const Environment env = CreateEnvironment(std::move(field_trials));
+class LibaomAv1EncoderMaxConsecDropTest
+ : public ::testing::TestWithParam</*framerate_fps=*/int> {};
+
+TEST_P(LibaomAv1EncoderMaxConsecDropTest, MaxConsecDrops) {
VideoBitrateAllocation allocation;
- allocation.SetBitrate(0, 0, 1000); // some very low bitrate
- std::unique_ptr<VideoEncoder> encoder = CreateLibaomAv1Encoder(env);
+ allocation.SetBitrate(0, 0,
+ 1000); // Very low bitrate to provoke frame drops.
+ std::unique_ptr<VideoEncoder> encoder =
+ CreateLibaomAv1Encoder(CreateEnvironment());
VideoCodec codec_settings = DefaultCodecSettings();
codec_settings.SetFrameDropEnabled(true);
codec_settings.SetScalabilityMode(ScalabilityMode::kL1T1);
codec_settings.startBitrate = allocation.get_sum_kbps();
+ codec_settings.maxFramerate = GetParam();
ASSERT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()),
WEBRTC_VIDEO_CODEC_OK);
encoder->SetRates(VideoEncoder::RateControlParameters(
allocation, codec_settings.maxFramerate));
- EncodedVideoFrameProducer evfp(*encoder);
- evfp.SetResolution(
- RenderResolution{codec_settings.width, codec_settings.height});
- // We should code the first frame, skip two, then code another frame.
std::vector<EncodedVideoFrameProducer::EncodedFrame> encoded_frames =
- evfp.SetNumInputFrames(4).Encode();
- ASSERT_THAT(encoded_frames, SizeIs(2));
- // The 4 frames have default Rtp-timestamps of 1000, 4000, 7000, 10000.
- ASSERT_THAT(encoded_frames[1].encoded_image.RtpTimestamp(), 10000);
+ EncodedVideoFrameProducer(*encoder)
+ .SetNumInputFrames(60)
+ .SetFramerateFps(codec_settings.maxFramerate)
+ .SetResolution(RenderResolution{320, 180})
+ .Encode();
+ ASSERT_GE(encoded_frames.size(), 2u);
+
+ int max_consec_drops = 0;
+ for (size_t i = 1; i < encoded_frames.size(); ++i) {
+ uint32_t frame_duration_rtp =
+ encoded_frames[i].encoded_image.RtpTimestamp() -
+ encoded_frames[i - 1].encoded_image.RtpTimestamp();
+ // X consecutive drops result in a freeze of (X + 1) frame duration.
+ // Subtract 1 to get pure number of drops.
+ int num_drops = frame_duration_rtp * codec_settings.maxFramerate /
+ kVideoPayloadTypeFrequency -
+ 1;
+ max_consec_drops = std::max(max_consec_drops, num_drops);
+ }
+
+ const int expected_max_consec_drops =
+ std::ceil(0.25 * codec_settings.maxFramerate);
+ EXPECT_EQ(max_consec_drops, expected_max_consec_drops);
}
+INSTANTIATE_TEST_SUITE_P(LibaomAv1EncoderMaxConsecDropTests,
+ LibaomAv1EncoderMaxConsecDropTest,
+ Values(1, 2, 5, 15, 30, 60));
+
TEST(LibaomAv1EncoderTest, EncoderInfoWithoutResolutionBitrateLimits) {
std::unique_ptr<VideoEncoder> encoder =
CreateLibaomAv1Encoder(CreateEnvironment());
