Repo created
This commit is contained in:
parent
81b91f4139
commit
f8c34fa5ee
22732 changed files with 4815320 additions and 2 deletions
|
|
@ -0,0 +1,884 @@
|
|||
/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/video_coding/codecs/vp8/default_temporal_layers.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <memory>
|
||||
#include <set>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "modules/video_coding/include/video_codec_interface.h"
|
||||
#include "rtc_base/arraysize.h"
|
||||
#include "rtc_base/checks.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "system_wrappers/include/field_trial.h"
|
||||
|
||||
namespace webrtc {
|
||||
DefaultTemporalLayers::PendingFrame::PendingFrame() = default;
|
||||
DefaultTemporalLayers::PendingFrame::PendingFrame(
|
||||
uint32_t timestamp,
|
||||
bool expired,
|
||||
uint8_t updated_buffers_mask,
|
||||
const DependencyInfo& dependency_info)
|
||||
: timestamp(timestamp),
|
||||
expired(expired),
|
||||
updated_buffer_mask(updated_buffers_mask),
|
||||
dependency_info(dependency_info) {}
|
||||
|
||||
namespace {
|
||||
using BufferFlags = Vp8FrameConfig::BufferFlags;
|
||||
using FreezeEntropy = Vp8FrameConfig::FreezeEntropy;
|
||||
using Vp8BufferReference = Vp8FrameConfig::Vp8BufferReference;
|
||||
|
||||
constexpr BufferFlags kNone = BufferFlags::kNone;
|
||||
constexpr BufferFlags kReference = BufferFlags::kReference;
|
||||
constexpr BufferFlags kUpdate = BufferFlags::kUpdate;
|
||||
constexpr BufferFlags kReferenceAndUpdate = BufferFlags::kReferenceAndUpdate;
|
||||
constexpr FreezeEntropy kFreezeEntropy = FreezeEntropy::kFreezeEntropy;
|
||||
|
||||
static constexpr uint8_t kUninitializedPatternIndex =
|
||||
std::numeric_limits<uint8_t>::max();
|
||||
static constexpr std::array<Vp8BufferReference, 3> kAllBuffers = {
|
||||
{Vp8BufferReference::kLast, Vp8BufferReference::kGolden,
|
||||
Vp8BufferReference::kAltref}};
|
||||
|
||||
std::vector<unsigned int> GetTemporalIds(size_t num_layers) {
|
||||
switch (num_layers) {
|
||||
case 1:
|
||||
// Temporal layer structure (single layer):
|
||||
// 0 0 0 0 ...
|
||||
return {0};
|
||||
case 2:
|
||||
// Temporal layer structure:
|
||||
// 1 1 ...
|
||||
// 0 0 ...
|
||||
return {0, 1};
|
||||
case 3:
|
||||
// Temporal layer structure:
|
||||
// 2 2 2 2 ...
|
||||
// 1 1 ...
|
||||
// 0 0 ...
|
||||
return {0, 2, 1, 2};
|
||||
case 4:
|
||||
// Temporal layer structure:
|
||||
// 3 3 3 3 3 3 3 3 ...
|
||||
// 2 2 2 2 ...
|
||||
// 1 1 ...
|
||||
// 0 0 ...
|
||||
return {0, 3, 2, 3, 1, 3, 2, 3};
|
||||
default:
|
||||
RTC_DCHECK_NOTREACHED();
|
||||
break;
|
||||
}
|
||||
RTC_DCHECK_NOTREACHED();
|
||||
return {0};
|
||||
}
|
||||
|
||||
uint8_t GetUpdatedBuffers(const Vp8FrameConfig& config) {
|
||||
uint8_t flags = 0;
|
||||
if (config.last_buffer_flags & BufferFlags::kUpdate) {
|
||||
flags |= static_cast<uint8_t>(Vp8BufferReference::kLast);
|
||||
}
|
||||
if (config.golden_buffer_flags & BufferFlags::kUpdate) {
|
||||
flags |= static_cast<uint8_t>(Vp8BufferReference::kGolden);
|
||||
}
|
||||
if (config.arf_buffer_flags & BufferFlags::kUpdate) {
|
||||
flags |= static_cast<uint8_t>(Vp8BufferReference::kAltref);
|
||||
}
|
||||
return flags;
|
||||
}
|
||||
|
||||
size_t BufferToIndex(Vp8BufferReference buffer) {
|
||||
switch (buffer) {
|
||||
case Vp8FrameConfig::Vp8BufferReference::kLast:
|
||||
return 0;
|
||||
case Vp8FrameConfig::Vp8BufferReference::kGolden:
|
||||
return 1;
|
||||
case Vp8FrameConfig::Vp8BufferReference::kAltref:
|
||||
return 2;
|
||||
case Vp8FrameConfig::Vp8BufferReference::kNone:
|
||||
RTC_CHECK_NOTREACHED();
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
constexpr size_t DefaultTemporalLayers::kNumReferenceBuffers;
|
||||
|
||||
std::vector<DefaultTemporalLayers::DependencyInfo>
|
||||
DefaultTemporalLayers::GetDependencyInfo(size_t num_layers) {
|
||||
// For indexing in the patterns described below (which temporal layers they
|
||||
// belong to), see the diagram above.
|
||||
// Layer sync is done similarly for all patterns (except single stream) and
|
||||
// happens every 8 frames:
|
||||
// TL1 layer syncs by periodically by only referencing TL0 ('last'), but still
|
||||
// updating 'golden', so it can be used as a reference by future TL1 frames.
|
||||
// TL2 layer syncs just before TL1 by only depending on TL0 (and not depending
|
||||
// on TL1's buffer before TL1 has layer synced).
|
||||
// TODO(pbos): Consider cyclically updating 'arf' (and 'golden' for 1TL) for
|
||||
// the base layer in 1-3TL instead of 'last' periodically on long intervals,
|
||||
// so that if scene changes occur (user walks between rooms or rotates webcam)
|
||||
// the 'arf' (or 'golden' respectively) is not stuck on a no-longer relevant
|
||||
// keyframe.
|
||||
|
||||
switch (num_layers) {
|
||||
case 1:
|
||||
// Always reference and update the same buffer.
|
||||
return {{"S", {kReferenceAndUpdate, kNone, kNone}}};
|
||||
case 2:
|
||||
// All layers can reference but not update the 'alt' buffer, this means
|
||||
// that the 'alt' buffer reference is effectively the last keyframe.
|
||||
// TL0 also references and updates the 'last' buffer.
|
||||
// TL1 also references 'last' and references and updates 'golden'.
|
||||
if (!field_trial::IsDisabled("WebRTC-UseShortVP8TL2Pattern")) {
|
||||
// Shortened 4-frame pattern:
|
||||
// 1---1 1---1 ...
|
||||
// / / / /
|
||||
// 0---0---0---0 ...
|
||||
return {{"SS", {kReferenceAndUpdate, kNone, kNone}},
|
||||
{"-S", {kReference, kUpdate, kNone}},
|
||||
{"SR", {kReferenceAndUpdate, kNone, kNone}},
|
||||
{"-D", {kReference, kReference, kNone, kFreezeEntropy}}};
|
||||
} else {
|
||||
// "Default" 8-frame pattern:
|
||||
// 1---1---1---1 1---1---1---1 ...
|
||||
// / / / / / / / /
|
||||
// 0---0---0---0---0---0---0---0 ...
|
||||
return {{"SS", {kReferenceAndUpdate, kNone, kNone}},
|
||||
{"-S", {kReference, kUpdate, kNone}},
|
||||
{"SR", {kReferenceAndUpdate, kNone, kNone}},
|
||||
{"-R", {kReference, kReferenceAndUpdate, kNone}},
|
||||
{"SR", {kReferenceAndUpdate, kNone, kNone}},
|
||||
{"-R", {kReference, kReferenceAndUpdate, kNone}},
|
||||
{"SR", {kReferenceAndUpdate, kNone, kNone}},
|
||||
{"-D", {kReference, kReference, kNone, kFreezeEntropy}}};
|
||||
}
|
||||
case 3:
|
||||
if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) {
|
||||
// This field trial is intended to check if it is worth using a shorter
|
||||
// temporal pattern, trading some coding efficiency for less risk of
|
||||
// dropped frames.
|
||||
// The coding efficiency will decrease somewhat since the higher layer
|
||||
// state is more volatile, but it will be offset slightly by updating
|
||||
// the altref buffer with TL2 frames, instead of just referencing lower
|
||||
// layers.
|
||||
// If a frame is dropped in a higher layer, the jitter
|
||||
// buffer on the receive side won't be able to decode any higher layer
|
||||
// frame until the next sync frame. So we expect a noticeable decrease
|
||||
// in frame drops on links with high packet loss.
|
||||
|
||||
// TL0 references and updates the 'last' buffer.
|
||||
// TL1 references 'last' and references and updates 'golden'.
|
||||
// TL2 references both 'last' & 'golden' and references and updates
|
||||
// 'arf'.
|
||||
// 2-------2 2-------2 2
|
||||
// / __/ / __/ /
|
||||
// / __1 / __1 /
|
||||
// /___/ /___/ /
|
||||
// 0---------------0---------------0-----
|
||||
// 0 1 2 3 4 5 6 7 8 9 ...
|
||||
return {{"SSS", {kReferenceAndUpdate, kNone, kNone}},
|
||||
{"--S", {kReference, kNone, kUpdate}},
|
||||
{"-DR", {kReference, kUpdate, kNone}},
|
||||
{"--D", {kReference, kReference, kReference, kFreezeEntropy}}};
|
||||
} else {
|
||||
// All layers can reference but not update the 'alt' buffer, this means
|
||||
// that the 'alt' buffer reference is effectively the last keyframe.
|
||||
// TL0 also references and updates the 'last' buffer.
|
||||
// TL1 also references 'last' and references and updates 'golden'.
|
||||
// TL2 references both 'last' and 'golden' but updates no buffer.
|
||||
// 2 __2 _____2 __2 2
|
||||
// / /____/ / / /
|
||||
// / 1---------/-----1 /
|
||||
// /_____/ /_____/ /
|
||||
// 0---------------0---------------0-----
|
||||
// 0 1 2 3 4 5 6 7 8 9 ...
|
||||
return {{"SSS", {kReferenceAndUpdate, kNone, kNone}},
|
||||
{"--D", {kReference, kNone, kNone, kFreezeEntropy}},
|
||||
{"-SS", {kReference, kUpdate, kNone}},
|
||||
{"--D", {kReference, kReference, kNone, kFreezeEntropy}},
|
||||
{"SRR", {kReferenceAndUpdate, kNone, kNone}},
|
||||
{"--D", {kReference, kReference, kNone, kFreezeEntropy}},
|
||||
{"-DS", {kReference, kReferenceAndUpdate, kNone}},
|
||||
{"--D", {kReference, kReference, kNone, kFreezeEntropy}}};
|
||||
}
|
||||
case 4:
|
||||
// TL0 references and updates only the 'last' buffer.
|
||||
// TL1 references 'last' and updates and references 'golden'.
|
||||
// TL2 references 'last' and 'golden', and references and updates 'arf'.
|
||||
// TL3 references all buffers but update none of them.
|
||||
// TODO(philipel): Set decode target information for this structure.
|
||||
return {{"----", {kReferenceAndUpdate, kNone, kNone}},
|
||||
{"----", {kReference, kNone, kNone, kFreezeEntropy}},
|
||||
{"----", {kReference, kNone, kUpdate}},
|
||||
{"----", {kReference, kNone, kReference, kFreezeEntropy}},
|
||||
{"----", {kReference, kUpdate, kNone}},
|
||||
{"----", {kReference, kReference, kReference, kFreezeEntropy}},
|
||||
{"----", {kReference, kReference, kReferenceAndUpdate}},
|
||||
{"----", {kReference, kReference, kReference, kFreezeEntropy}},
|
||||
{"----", {kReferenceAndUpdate, kNone, kNone}},
|
||||
{"----", {kReference, kReference, kReference, kFreezeEntropy}},
|
||||
{"----", {kReference, kReference, kReferenceAndUpdate}},
|
||||
{"----", {kReference, kReference, kReference, kFreezeEntropy}},
|
||||
{"----", {kReference, kReferenceAndUpdate, kNone}},
|
||||
{"----", {kReference, kReference, kReference, kFreezeEntropy}},
|
||||
{"----", {kReference, kReference, kReferenceAndUpdate}},
|
||||
{"----", {kReference, kReference, kReference, kFreezeEntropy}}};
|
||||
default:
|
||||
RTC_DCHECK_NOTREACHED();
|
||||
break;
|
||||
}
|
||||
RTC_DCHECK_NOTREACHED();
|
||||
return {{"", {kNone, kNone, kNone}}};
|
||||
}
|
||||
|
||||
std::bitset<DefaultTemporalLayers::kNumReferenceBuffers>
|
||||
DefaultTemporalLayers::DetermineStaticBuffers(
|
||||
const std::vector<DependencyInfo>& temporal_pattern) {
|
||||
std::bitset<kNumReferenceBuffers> buffers;
|
||||
buffers.set();
|
||||
for (const DependencyInfo& info : temporal_pattern) {
|
||||
uint8_t updated_buffers = GetUpdatedBuffers(info.frame_config);
|
||||
|
||||
for (Vp8BufferReference buffer : kAllBuffers) {
|
||||
if (static_cast<uint8_t>(buffer) & updated_buffers) {
|
||||
buffers.reset(BufferToIndex(buffer));
|
||||
}
|
||||
}
|
||||
}
|
||||
return buffers;
|
||||
}
|
||||
|
||||
DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers)
|
||||
: num_layers_(std::max(1, number_of_temporal_layers)),
|
||||
temporal_ids_(GetTemporalIds(num_layers_)),
|
||||
temporal_pattern_(GetDependencyInfo(num_layers_)),
|
||||
is_static_buffer_(DetermineStaticBuffers(temporal_pattern_)),
|
||||
pattern_idx_(kUninitializedPatternIndex),
|
||||
new_bitrates_bps_(std::vector<uint32_t>(num_layers_, 0u)) {
|
||||
RTC_CHECK_GE(kMaxTemporalStreams, number_of_temporal_layers);
|
||||
RTC_CHECK_GE(number_of_temporal_layers, 0);
|
||||
RTC_CHECK_LE(number_of_temporal_layers, 4);
|
||||
// pattern_idx_ wraps around temporal_pattern_.size, this is incorrect if
|
||||
// temporal_ids_ are ever longer. If this is no longer correct it needs to
|
||||
// wrap at max(temporal_ids_.size(), temporal_pattern_.size()).
|
||||
RTC_DCHECK_LE(temporal_ids_.size(), temporal_pattern_.size());
|
||||
|
||||
RTC_DCHECK(
|
||||
checker_ = TemporalLayersChecker::CreateTemporalLayersChecker(
|
||||
Vp8TemporalLayersType::kFixedPattern, number_of_temporal_layers));
|
||||
|
||||
// Always need to start with a keyframe, so pre-populate all frame counters.
|
||||
frames_since_buffer_refresh_.fill(0);
|
||||
}
|
||||
|
||||
DefaultTemporalLayers::~DefaultTemporalLayers() = default;
|
||||
|
||||
void DefaultTemporalLayers::SetQpLimits(size_t stream_index,
|
||||
int min_qp,
|
||||
int max_qp) {
|
||||
RTC_DCHECK_LT(stream_index, StreamCount());
|
||||
// Ignore.
|
||||
}
|
||||
|
||||
size_t DefaultTemporalLayers::StreamCount() const {
|
||||
return 1;
|
||||
}
|
||||
|
||||
bool DefaultTemporalLayers::SupportsEncoderFrameDropping(
|
||||
size_t stream_index) const {
|
||||
RTC_DCHECK_LT(stream_index, StreamCount());
|
||||
// This class allows the encoder drop frames as it sees fit.
|
||||
return true;
|
||||
}
|
||||
|
||||
void DefaultTemporalLayers::OnRatesUpdated(
|
||||
size_t stream_index,
|
||||
const std::vector<uint32_t>& bitrates_bps,
|
||||
int framerate_fps) {
|
||||
RTC_DCHECK_LT(stream_index, StreamCount());
|
||||
RTC_DCHECK_GT(bitrates_bps.size(), 0);
|
||||
RTC_DCHECK_LE(bitrates_bps.size(), num_layers_);
|
||||
// `bitrates_bps` uses individual rate per layer, but Vp8EncoderConfig wants
|
||||
// the accumulated rate, so sum them up.
|
||||
new_bitrates_bps_ = bitrates_bps;
|
||||
new_bitrates_bps_->resize(num_layers_);
|
||||
for (size_t i = 1; i < num_layers_; ++i) {
|
||||
(*new_bitrates_bps_)[i] += (*new_bitrates_bps_)[i - 1];
|
||||
}
|
||||
}
|
||||
|
||||
Vp8EncoderConfig DefaultTemporalLayers::UpdateConfiguration(
|
||||
size_t stream_index) {
|
||||
RTC_DCHECK_LT(stream_index, StreamCount());
|
||||
|
||||
Vp8EncoderConfig config;
|
||||
|
||||
if (!new_bitrates_bps_) {
|
||||
return config;
|
||||
}
|
||||
|
||||
config.temporal_layer_config.emplace();
|
||||
Vp8EncoderConfig::TemporalLayerConfig& ts_config =
|
||||
config.temporal_layer_config.value();
|
||||
|
||||
for (size_t i = 0; i < num_layers_; ++i) {
|
||||
ts_config.ts_target_bitrate[i] = (*new_bitrates_bps_)[i] / 1000;
|
||||
// ..., 4, 2, 1
|
||||
ts_config.ts_rate_decimator[i] = 1 << (num_layers_ - i - 1);
|
||||
}
|
||||
|
||||
ts_config.ts_number_layers = num_layers_;
|
||||
ts_config.ts_periodicity = temporal_ids_.size();
|
||||
std::copy(temporal_ids_.begin(), temporal_ids_.end(),
|
||||
ts_config.ts_layer_id.begin());
|
||||
|
||||
new_bitrates_bps_.reset();
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
bool DefaultTemporalLayers::IsSyncFrame(const Vp8FrameConfig& config) const {
|
||||
// Since we always assign TL0 to 'last' in these patterns, we can infer layer
|
||||
// sync by checking if temporal id > 0 and we only reference TL0 or buffers
|
||||
// containing the last key-frame.
|
||||
if (config.packetizer_temporal_idx == 0) {
|
||||
// TL0 frames are per definition not sync frames.
|
||||
return false;
|
||||
}
|
||||
|
||||
if ((config.last_buffer_flags & BufferFlags::kReference) == 0) {
|
||||
// Sync frames must reference TL0.
|
||||
return false;
|
||||
}
|
||||
|
||||
if ((config.golden_buffer_flags & BufferFlags::kReference) &&
|
||||
!is_static_buffer_[BufferToIndex(Vp8BufferReference::kGolden)]) {
|
||||
// Referencing a golden frame that contains a non-(base layer|key frame).
|
||||
return false;
|
||||
}
|
||||
if ((config.arf_buffer_flags & BufferFlags::kReference) &&
|
||||
!is_static_buffer_[BufferToIndex(Vp8BufferReference::kAltref)]) {
|
||||
// Referencing an altref frame that contains a non-(base layer|key frame).
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
Vp8FrameConfig DefaultTemporalLayers::NextFrameConfig(size_t stream_index,
|
||||
uint32_t timestamp) {
|
||||
RTC_DCHECK_LT(stream_index, StreamCount());
|
||||
RTC_DCHECK_GT(num_layers_, 0);
|
||||
RTC_DCHECK_GT(temporal_pattern_.size(), 0);
|
||||
|
||||
RTC_DCHECK_GT(kUninitializedPatternIndex, temporal_pattern_.size());
|
||||
const bool first_frame = (pattern_idx_ == kUninitializedPatternIndex);
|
||||
|
||||
pattern_idx_ = (pattern_idx_ + 1) % temporal_pattern_.size();
|
||||
DependencyInfo dependency_info = temporal_pattern_[pattern_idx_];
|
||||
Vp8FrameConfig& tl_config = dependency_info.frame_config;
|
||||
tl_config.encoder_layer_id = tl_config.packetizer_temporal_idx =
|
||||
temporal_ids_[pattern_idx_ % temporal_ids_.size()];
|
||||
|
||||
if (pattern_idx_ == 0) {
|
||||
// Start of new pattern iteration, set up clear state by invalidating any
|
||||
// pending frames, so that we don't make an invalid reference to a buffer
|
||||
// containing data from a previous iteration.
|
||||
for (auto& frame : pending_frames_) {
|
||||
frame.expired = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (first_frame) {
|
||||
tl_config = Vp8FrameConfig::GetIntraFrameConfig();
|
||||
} else {
|
||||
// Last is always ok to reference as it contains the base layer. For other
|
||||
// buffers though, we need to check if the buffer has actually been
|
||||
// refreshed this cycle of the temporal pattern. If the encoder dropped
|
||||
// a frame, it might not have.
|
||||
ValidateReferences(&tl_config.golden_buffer_flags,
|
||||
Vp8BufferReference::kGolden);
|
||||
ValidateReferences(&tl_config.arf_buffer_flags,
|
||||
Vp8BufferReference::kAltref);
|
||||
// Update search order to let the encoder know which buffers contains the
|
||||
// most recent data.
|
||||
UpdateSearchOrder(&tl_config);
|
||||
// Figure out if this a sync frame (non-base-layer frame with only
|
||||
// base-layer references).
|
||||
tl_config.layer_sync = IsSyncFrame(tl_config);
|
||||
|
||||
// Increment frame age, this needs to be in sync with `pattern_idx_`,
|
||||
// so must update it here. Resetting age to 0 must be done when encoding is
|
||||
// complete though, and so in the case of pipelining encoder it might lag.
|
||||
// To prevent this data spill over into the next iteration,
|
||||
// the `pedning_frames_` map is reset in loops. If delay is constant,
|
||||
// the relative age should still be OK for the search order.
|
||||
for (size_t& n : frames_since_buffer_refresh_) {
|
||||
++n;
|
||||
}
|
||||
}
|
||||
|
||||
// Add frame to set of pending frames, awaiting completion.
|
||||
pending_frames_.emplace_back(timestamp, false, GetUpdatedBuffers(tl_config),
|
||||
dependency_info);
|
||||
|
||||
// Checker does not yet support encoder frame dropping, so validate flags
|
||||
// here before they can be dropped.
|
||||
// TODO(sprang): Update checker to support dropping.
|
||||
RTC_DCHECK(checker_->CheckTemporalConfig(first_frame, tl_config));
|
||||
|
||||
return tl_config;
|
||||
}
|
||||
|
||||
void DefaultTemporalLayers::ValidateReferences(BufferFlags* flags,
|
||||
Vp8BufferReference ref) const {
|
||||
// Check if the buffer specified by `ref` is actually referenced, and if so
|
||||
// if it also a dynamically updating one (buffers always just containing
|
||||
// keyframes are always safe to reference).
|
||||
if ((*flags & BufferFlags::kReference) &&
|
||||
!is_static_buffer_[BufferToIndex(ref)]) {
|
||||
if (NumFramesSinceBufferRefresh(ref) >= pattern_idx_) {
|
||||
// No valid buffer state, or buffer contains frame that is older than the
|
||||
// current pattern. This reference is not valid, so remove it.
|
||||
*flags = static_cast<BufferFlags>(*flags & ~BufferFlags::kReference);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void DefaultTemporalLayers::UpdateSearchOrder(Vp8FrameConfig* config) {
|
||||
// Figure out which of the buffers we can reference, and order them so that
|
||||
// the most recently refreshed is first. Otherwise prioritize last first,
|
||||
// golden second, and altref third.
|
||||
using BufferRefAge = std::pair<Vp8BufferReference, size_t>;
|
||||
std::vector<BufferRefAge> eligible_buffers;
|
||||
if (config->last_buffer_flags & BufferFlags::kReference) {
|
||||
eligible_buffers.emplace_back(
|
||||
Vp8BufferReference::kLast,
|
||||
NumFramesSinceBufferRefresh(Vp8BufferReference::kLast));
|
||||
}
|
||||
if (config->golden_buffer_flags & BufferFlags::kReference) {
|
||||
eligible_buffers.emplace_back(
|
||||
Vp8BufferReference::kGolden,
|
||||
NumFramesSinceBufferRefresh(Vp8BufferReference::kGolden));
|
||||
}
|
||||
if (config->arf_buffer_flags & BufferFlags::kReference) {
|
||||
eligible_buffers.emplace_back(
|
||||
Vp8BufferReference::kAltref,
|
||||
NumFramesSinceBufferRefresh(Vp8BufferReference::kAltref));
|
||||
}
|
||||
|
||||
std::sort(eligible_buffers.begin(), eligible_buffers.end(),
|
||||
[](const BufferRefAge& lhs, const BufferRefAge& rhs) {
|
||||
if (lhs.second != rhs.second) {
|
||||
// Lower count has highest precedence.
|
||||
return lhs.second < rhs.second;
|
||||
}
|
||||
return lhs.first < rhs.first;
|
||||
});
|
||||
|
||||
// Populate the search order fields where possible.
|
||||
if (!eligible_buffers.empty()) {
|
||||
config->first_reference = eligible_buffers.front().first;
|
||||
if (eligible_buffers.size() > 1)
|
||||
config->second_reference = eligible_buffers[1].first;
|
||||
}
|
||||
}
|
||||
|
||||
size_t DefaultTemporalLayers::NumFramesSinceBufferRefresh(
|
||||
Vp8FrameConfig::Vp8BufferReference ref) const {
|
||||
return frames_since_buffer_refresh_[BufferToIndex(ref)];
|
||||
}
|
||||
|
||||
void DefaultTemporalLayers::ResetNumFramesSinceBufferRefresh(
|
||||
Vp8FrameConfig::Vp8BufferReference ref) {
|
||||
frames_since_buffer_refresh_[BufferToIndex(ref)] = 0;
|
||||
}
|
||||
|
||||
void DefaultTemporalLayers::CullPendingFramesBefore(uint32_t timestamp) {
|
||||
while (!pending_frames_.empty() &&
|
||||
pending_frames_.front().timestamp != timestamp) {
|
||||
pending_frames_.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
void DefaultTemporalLayers::OnEncodeDone(size_t stream_index,
|
||||
uint32_t rtp_timestamp,
|
||||
size_t size_bytes,
|
||||
bool is_keyframe,
|
||||
int qp,
|
||||
CodecSpecificInfo* info) {
|
||||
RTC_DCHECK_LT(stream_index, StreamCount());
|
||||
RTC_DCHECK_GT(num_layers_, 0);
|
||||
|
||||
if (size_bytes == 0) {
|
||||
RTC_LOG(LS_WARNING) << "Empty frame; treating as dropped.";
|
||||
OnFrameDropped(stream_index, rtp_timestamp);
|
||||
return;
|
||||
}
|
||||
|
||||
CullPendingFramesBefore(rtp_timestamp);
|
||||
RTC_CHECK(!pending_frames_.empty());
|
||||
PendingFrame& frame = pending_frames_.front();
|
||||
RTC_DCHECK_EQ(frame.timestamp, rtp_timestamp);
|
||||
const Vp8FrameConfig& frame_config = frame.dependency_info.frame_config;
|
||||
if (is_keyframe) {
|
||||
// Signal key-frame so checker resets state.
|
||||
RTC_DCHECK(checker_->CheckTemporalConfig(true, frame_config));
|
||||
}
|
||||
|
||||
CodecSpecificInfoVP8& vp8_info = info->codecSpecific.VP8;
|
||||
if (num_layers_ == 1) {
|
||||
vp8_info.temporalIdx = kNoTemporalIdx;
|
||||
vp8_info.layerSync = false;
|
||||
} else {
|
||||
if (is_keyframe) {
|
||||
// Restart the temporal pattern on keyframes.
|
||||
pattern_idx_ = 0;
|
||||
vp8_info.temporalIdx = 0;
|
||||
vp8_info.layerSync = true; // Keyframes are always sync frames.
|
||||
|
||||
for (Vp8BufferReference buffer : kAllBuffers) {
|
||||
if (is_static_buffer_[BufferToIndex(buffer)]) {
|
||||
// Update frame count of all kf-only buffers, regardless of state of
|
||||
// `pending_frames_`.
|
||||
ResetNumFramesSinceBufferRefresh(buffer);
|
||||
} else {
|
||||
// Key-frames update all buffers, this should be reflected when
|
||||
// updating state in FrameEncoded().
|
||||
frame.updated_buffer_mask |= static_cast<uint8_t>(buffer);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Delta frame, update codec specifics with temporal id and sync flag.
|
||||
vp8_info.temporalIdx = frame_config.packetizer_temporal_idx;
|
||||
vp8_info.layerSync = frame_config.layer_sync;
|
||||
}
|
||||
}
|
||||
|
||||
vp8_info.useExplicitDependencies = true;
|
||||
RTC_DCHECK_EQ(vp8_info.referencedBuffersCount, 0u);
|
||||
RTC_DCHECK_EQ(vp8_info.updatedBuffersCount, 0u);
|
||||
|
||||
GenericFrameInfo& generic_frame_info = info->generic_frame_info.emplace();
|
||||
|
||||
for (int i = 0; i < static_cast<int>(Vp8FrameConfig::Buffer::kCount); ++i) {
|
||||
bool references = false;
|
||||
bool updates = is_keyframe;
|
||||
|
||||
if (!is_keyframe &&
|
||||
frame_config.References(static_cast<Vp8FrameConfig::Buffer>(i))) {
|
||||
RTC_DCHECK_LT(vp8_info.referencedBuffersCount,
|
||||
arraysize(CodecSpecificInfoVP8::referencedBuffers));
|
||||
references = true;
|
||||
vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i;
|
||||
}
|
||||
|
||||
if (is_keyframe ||
|
||||
frame_config.Updates(static_cast<Vp8FrameConfig::Buffer>(i))) {
|
||||
RTC_DCHECK_LT(vp8_info.updatedBuffersCount,
|
||||
arraysize(CodecSpecificInfoVP8::updatedBuffers));
|
||||
updates = true;
|
||||
vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i;
|
||||
}
|
||||
|
||||
if (references || updates) {
|
||||
generic_frame_info.encoder_buffers.emplace_back(i, references, updates);
|
||||
}
|
||||
}
|
||||
|
||||
// The templates are always present on keyframes, and then refered to by
|
||||
// subsequent frames.
|
||||
if (is_keyframe) {
|
||||
info->template_structure = GetTemplateStructure(num_layers_);
|
||||
generic_frame_info.decode_target_indications =
|
||||
temporal_pattern_.front().decode_target_indications;
|
||||
generic_frame_info.temporal_id = 0;
|
||||
} else {
|
||||
generic_frame_info.decode_target_indications =
|
||||
frame.dependency_info.decode_target_indications;
|
||||
generic_frame_info.temporal_id = frame_config.packetizer_temporal_idx;
|
||||
}
|
||||
|
||||
if (!frame.expired) {
|
||||
for (Vp8BufferReference buffer : kAllBuffers) {
|
||||
if (frame.updated_buffer_mask & static_cast<uint8_t>(buffer)) {
|
||||
ResetNumFramesSinceBufferRefresh(buffer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pending_frames_.pop_front();
|
||||
}
|
||||
|
||||
void DefaultTemporalLayers::OnFrameDropped(size_t stream_index,
|
||||
uint32_t rtp_timestamp) {
|
||||
CullPendingFramesBefore(rtp_timestamp);
|
||||
RTC_CHECK(!pending_frames_.empty());
|
||||
RTC_DCHECK_EQ(pending_frames_.front().timestamp, rtp_timestamp);
|
||||
pending_frames_.pop_front();
|
||||
}
|
||||
|
||||
void DefaultTemporalLayers::OnPacketLossRateUpdate(float packet_loss_rate) {}
|
||||
|
||||
void DefaultTemporalLayers::OnRttUpdate(int64_t rtt_ms) {}
|
||||
|
||||
void DefaultTemporalLayers::OnLossNotification(
|
||||
const VideoEncoder::LossNotification& loss_notification) {}
|
||||
|
||||
FrameDependencyStructure DefaultTemporalLayers::GetTemplateStructure(
|
||||
int num_layers) const {
|
||||
RTC_CHECK_LT(num_layers, 5);
|
||||
RTC_CHECK_GT(num_layers, 0);
|
||||
|
||||
FrameDependencyStructure template_structure;
|
||||
template_structure.num_decode_targets = num_layers;
|
||||
|
||||
switch (num_layers) {
|
||||
case 1: {
|
||||
template_structure.templates.resize(2);
|
||||
template_structure.templates[0].T(0).Dtis("S");
|
||||
template_structure.templates[1].T(0).Dtis("S").FrameDiffs({1});
|
||||
return template_structure;
|
||||
}
|
||||
case 2: {
|
||||
template_structure.templates.resize(5);
|
||||
template_structure.templates[0].T(0).Dtis("SS");
|
||||
template_structure.templates[1].T(0).Dtis("SS").FrameDiffs({2});
|
||||
template_structure.templates[2].T(0).Dtis("SR").FrameDiffs({2});
|
||||
template_structure.templates[3].T(1).Dtis("-S").FrameDiffs({1});
|
||||
template_structure.templates[4].T(1).Dtis("-D").FrameDiffs({2, 1});
|
||||
return template_structure;
|
||||
}
|
||||
case 3: {
|
||||
if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) {
|
||||
template_structure.templates.resize(5);
|
||||
template_structure.templates[0].T(0).Dtis("SSS");
|
||||
template_structure.templates[1].T(0).Dtis("SSS").FrameDiffs({4});
|
||||
template_structure.templates[2].T(1).Dtis("-DR").FrameDiffs({2});
|
||||
template_structure.templates[3].T(2).Dtis("--S").FrameDiffs({1});
|
||||
template_structure.templates[4].T(2).Dtis("--D").FrameDiffs({2, 1});
|
||||
} else {
|
||||
template_structure.templates.resize(7);
|
||||
template_structure.templates[0].T(0).Dtis("SSS");
|
||||
template_structure.templates[1].T(0).Dtis("SSS").FrameDiffs({4});
|
||||
template_structure.templates[2].T(0).Dtis("SRR").FrameDiffs({4});
|
||||
template_structure.templates[3].T(1).Dtis("-SS").FrameDiffs({2});
|
||||
template_structure.templates[4].T(1).Dtis("-DS").FrameDiffs({4, 2});
|
||||
template_structure.templates[5].T(2).Dtis("--D").FrameDiffs({1});
|
||||
template_structure.templates[6].T(2).Dtis("--D").FrameDiffs({3, 1});
|
||||
}
|
||||
return template_structure;
|
||||
}
|
||||
case 4: {
|
||||
template_structure.templates.resize(8);
|
||||
template_structure.templates[0].T(0).Dtis("SSSS");
|
||||
template_structure.templates[1].T(0).Dtis("SSSS").FrameDiffs({8});
|
||||
template_structure.templates[2].T(1).Dtis("-SRR").FrameDiffs({4});
|
||||
template_structure.templates[3].T(1).Dtis("-SRR").FrameDiffs({4, 8});
|
||||
template_structure.templates[4].T(2).Dtis("--SR").FrameDiffs({2});
|
||||
template_structure.templates[5].T(2).Dtis("--SR").FrameDiffs({2, 4});
|
||||
template_structure.templates[6].T(3).Dtis("---D").FrameDiffs({1});
|
||||
template_structure.templates[7].T(3).Dtis("---D").FrameDiffs({1, 3});
|
||||
return template_structure;
|
||||
}
|
||||
default:
|
||||
RTC_DCHECK_NOTREACHED();
|
||||
// To make the compiler happy!
|
||||
return template_structure;
|
||||
}
|
||||
}
|
||||
|
||||
// Returns list of temporal dependencies for each frame in the temporal pattern.
|
||||
// Values are lists of indecies in the pattern.
|
||||
std::vector<std::set<uint8_t>> GetTemporalDependencies(
|
||||
int num_temporal_layers) {
|
||||
switch (num_temporal_layers) {
|
||||
case 1:
|
||||
return {{0}};
|
||||
case 2:
|
||||
if (!field_trial::IsDisabled("WebRTC-UseShortVP8TL2Pattern")) {
|
||||
return {{2}, {0}, {0}, {1, 2}};
|
||||
} else {
|
||||
return {{6}, {0}, {0}, {1, 2}, {2}, {3, 4}, {4}, {5, 6}};
|
||||
}
|
||||
case 3:
|
||||
if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) {
|
||||
return {{0}, {0}, {0}, {0, 1, 2}};
|
||||
} else {
|
||||
return {{4}, {0}, {0}, {0, 2}, {0}, {2, 4}, {2, 4}, {4, 6}};
|
||||
}
|
||||
case 4:
|
||||
return {{8}, {0}, {0}, {0, 2},
|
||||
{0}, {0, 2, 4}, {0, 2, 4}, {0, 4, 6},
|
||||
{0}, {4, 6, 8}, {4, 6, 8}, {4, 8, 10},
|
||||
{4, 8}, {8, 10, 12}, {8, 10, 12}, {8, 12, 14}};
|
||||
default:
|
||||
RTC_DCHECK_NOTREACHED();
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
DefaultTemporalLayersChecker::DefaultTemporalLayersChecker(
|
||||
int num_temporal_layers)
|
||||
: TemporalLayersChecker(num_temporal_layers),
|
||||
num_layers_(std::max(1, num_temporal_layers)),
|
||||
temporal_ids_(GetTemporalIds(num_layers_)),
|
||||
temporal_dependencies_(GetTemporalDependencies(num_layers_)),
|
||||
pattern_idx_(255) {
|
||||
int i = 0;
|
||||
while (temporal_ids_.size() < temporal_dependencies_.size()) {
|
||||
temporal_ids_.push_back(temporal_ids_[i++]);
|
||||
}
|
||||
}
|
||||
|
||||
DefaultTemporalLayersChecker::~DefaultTemporalLayersChecker() = default;
|
||||
|
||||
bool DefaultTemporalLayersChecker::CheckTemporalConfig(
|
||||
bool frame_is_keyframe,
|
||||
const Vp8FrameConfig& frame_config) {
|
||||
if (!TemporalLayersChecker::CheckTemporalConfig(frame_is_keyframe,
|
||||
frame_config)) {
|
||||
return false;
|
||||
}
|
||||
if (frame_config.drop_frame) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (frame_is_keyframe) {
|
||||
pattern_idx_ = 0;
|
||||
last_ = BufferState();
|
||||
golden_ = BufferState();
|
||||
arf_ = BufferState();
|
||||
return true;
|
||||
}
|
||||
|
||||
++pattern_idx_;
|
||||
if (pattern_idx_ == temporal_ids_.size()) {
|
||||
// All non key-frame buffers should be updated each pattern cycle.
|
||||
if (!last_.is_keyframe && !last_.is_updated_this_cycle) {
|
||||
RTC_LOG(LS_ERROR) << "Last buffer was not updated during pattern cycle.";
|
||||
return false;
|
||||
}
|
||||
if (!arf_.is_keyframe && !arf_.is_updated_this_cycle) {
|
||||
RTC_LOG(LS_ERROR) << "Arf buffer was not updated during pattern cycle.";
|
||||
return false;
|
||||
}
|
||||
if (!golden_.is_keyframe && !golden_.is_updated_this_cycle) {
|
||||
RTC_LOG(LS_ERROR)
|
||||
<< "Golden buffer was not updated during pattern cycle.";
|
||||
return false;
|
||||
}
|
||||
last_.is_updated_this_cycle = false;
|
||||
arf_.is_updated_this_cycle = false;
|
||||
golden_.is_updated_this_cycle = false;
|
||||
pattern_idx_ = 0;
|
||||
}
|
||||
uint8_t expected_tl_idx = temporal_ids_[pattern_idx_];
|
||||
if (frame_config.packetizer_temporal_idx != expected_tl_idx) {
|
||||
RTC_LOG(LS_ERROR) << "Frame has an incorrect temporal index. Expected: "
|
||||
<< static_cast<int>(expected_tl_idx) << " Actual: "
|
||||
<< static_cast<int>(frame_config.packetizer_temporal_idx);
|
||||
return false;
|
||||
}
|
||||
|
||||
bool need_sync = temporal_ids_[pattern_idx_] > 0 &&
|
||||
temporal_ids_[pattern_idx_] != kNoTemporalIdx;
|
||||
std::vector<int> dependencies;
|
||||
|
||||
if (frame_config.last_buffer_flags & BufferFlags::kReference) {
|
||||
uint8_t referenced_layer = temporal_ids_[last_.pattern_idx];
|
||||
if (referenced_layer > 0) {
|
||||
need_sync = false;
|
||||
}
|
||||
if (!last_.is_keyframe) {
|
||||
dependencies.push_back(last_.pattern_idx);
|
||||
}
|
||||
} else if (frame_config.first_reference == Vp8BufferReference::kLast ||
|
||||
frame_config.second_reference == Vp8BufferReference::kLast) {
|
||||
RTC_LOG(LS_ERROR)
|
||||
<< "Last buffer not referenced, but present in search order.";
|
||||
return false;
|
||||
}
|
||||
|
||||
if (frame_config.arf_buffer_flags & BufferFlags::kReference) {
|
||||
uint8_t referenced_layer = temporal_ids_[arf_.pattern_idx];
|
||||
if (referenced_layer > 0) {
|
||||
need_sync = false;
|
||||
}
|
||||
if (!arf_.is_keyframe) {
|
||||
dependencies.push_back(arf_.pattern_idx);
|
||||
}
|
||||
} else if (frame_config.first_reference == Vp8BufferReference::kAltref ||
|
||||
frame_config.second_reference == Vp8BufferReference::kAltref) {
|
||||
RTC_LOG(LS_ERROR)
|
||||
<< "Altret buffer not referenced, but present in search order.";
|
||||
return false;
|
||||
}
|
||||
|
||||
if (frame_config.golden_buffer_flags & BufferFlags::kReference) {
|
||||
uint8_t referenced_layer = temporal_ids_[golden_.pattern_idx];
|
||||
if (referenced_layer > 0) {
|
||||
need_sync = false;
|
||||
}
|
||||
if (!golden_.is_keyframe) {
|
||||
dependencies.push_back(golden_.pattern_idx);
|
||||
}
|
||||
} else if (frame_config.first_reference == Vp8BufferReference::kGolden ||
|
||||
frame_config.second_reference == Vp8BufferReference::kGolden) {
|
||||
RTC_LOG(LS_ERROR)
|
||||
<< "Golden buffer not referenced, but present in search order.";
|
||||
return false;
|
||||
}
|
||||
|
||||
if (need_sync != frame_config.layer_sync) {
|
||||
RTC_LOG(LS_ERROR) << "Sync bit is set incorrectly on a frame. Expected: "
|
||||
<< need_sync << " Actual: " << frame_config.layer_sync;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!frame_is_keyframe) {
|
||||
size_t i;
|
||||
for (i = 0; i < dependencies.size(); ++i) {
|
||||
if (temporal_dependencies_[pattern_idx_].find(dependencies[i]) ==
|
||||
temporal_dependencies_[pattern_idx_].end()) {
|
||||
RTC_LOG(LS_ERROR)
|
||||
<< "Illegal temporal dependency out of defined pattern "
|
||||
"from position "
|
||||
<< static_cast<int>(pattern_idx_) << " to position "
|
||||
<< static_cast<int>(dependencies[i]);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (frame_config.last_buffer_flags & BufferFlags::kUpdate) {
|
||||
last_.is_updated_this_cycle = true;
|
||||
last_.pattern_idx = pattern_idx_;
|
||||
last_.is_keyframe = false;
|
||||
}
|
||||
if (frame_config.arf_buffer_flags & BufferFlags::kUpdate) {
|
||||
arf_.is_updated_this_cycle = true;
|
||||
arf_.pattern_idx = pattern_idx_;
|
||||
arf_.is_keyframe = false;
|
||||
}
|
||||
if (frame_config.golden_buffer_flags & BufferFlags::kUpdate) {
|
||||
golden_.is_updated_this_cycle = true;
|
||||
golden_.pattern_idx = pattern_idx_;
|
||||
golden_.is_keyframe = false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,168 @@
|
|||
/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
/*
|
||||
* This file defines classes for doing temporal layers with VP8.
|
||||
*/
|
||||
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_DEFAULT_TEMPORAL_LAYERS_H_
|
||||
#define MODULES_VIDEO_CODING_CODECS_VP8_DEFAULT_TEMPORAL_LAYERS_H_
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <bitset>
|
||||
#include <deque>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <set>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "api/video_codecs/vp8_frame_config.h"
|
||||
#include "api/video_codecs/vp8_temporal_layers.h"
|
||||
#include "modules/video_coding/codecs/vp8/include/temporal_layers_checker.h"
|
||||
#include "modules/video_coding/include/video_codec_interface.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
class DefaultTemporalLayers final : public Vp8FrameBufferController {
|
||||
public:
|
||||
explicit DefaultTemporalLayers(int number_of_temporal_layers);
|
||||
~DefaultTemporalLayers() override;
|
||||
|
||||
void SetQpLimits(size_t stream_index, int min_qp, int max_qp) override;
|
||||
|
||||
size_t StreamCount() const override;
|
||||
|
||||
bool SupportsEncoderFrameDropping(size_t stream_index) const override;
|
||||
|
||||
// Returns the recommended VP8 encode flags needed. May refresh the decoder
|
||||
// and/or update the reference buffers.
|
||||
Vp8FrameConfig NextFrameConfig(size_t stream_index,
|
||||
uint32_t timestamp) override;
|
||||
|
||||
// New target bitrate, per temporal layer.
|
||||
void OnRatesUpdated(size_t stream_index,
|
||||
const std::vector<uint32_t>& bitrates_bps,
|
||||
int framerate_fps) override;
|
||||
|
||||
Vp8EncoderConfig UpdateConfiguration(size_t stream_index) override;
|
||||
|
||||
// Callbacks methods on frame completion. OnEncodeDone() or OnFrameDropped()
|
||||
// should be called once for each NextFrameConfig() call (using the RTP
|
||||
// timestamp as ID), and the calls MUST be in the same order.
|
||||
void OnEncodeDone(size_t stream_index,
|
||||
uint32_t rtp_timestamp,
|
||||
size_t size_bytes,
|
||||
bool is_keyframe,
|
||||
int qp,
|
||||
CodecSpecificInfo* info) override;
|
||||
void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) override;
|
||||
|
||||
void OnPacketLossRateUpdate(float packet_loss_rate) override;
|
||||
|
||||
void OnRttUpdate(int64_t rtt_ms) override;
|
||||
|
||||
void OnLossNotification(
|
||||
const VideoEncoder::LossNotification& loss_notification) override;
|
||||
|
||||
private:
|
||||
static constexpr size_t kNumReferenceBuffers = 3; // Last, golden, altref.
|
||||
struct DependencyInfo {
|
||||
DependencyInfo() = default;
|
||||
DependencyInfo(absl::string_view indication_symbols,
|
||||
Vp8FrameConfig frame_config)
|
||||
: decode_target_indications(
|
||||
webrtc_impl::StringToDecodeTargetIndications(indication_symbols)),
|
||||
frame_config(frame_config) {}
|
||||
|
||||
absl::InlinedVector<DecodeTargetIndication, 10> decode_target_indications;
|
||||
Vp8FrameConfig frame_config;
|
||||
};
|
||||
struct PendingFrame {
|
||||
PendingFrame();
|
||||
PendingFrame(uint32_t timestamp,
|
||||
bool expired,
|
||||
uint8_t updated_buffers_mask,
|
||||
const DependencyInfo& dependency_info);
|
||||
uint32_t timestamp = 0;
|
||||
// Flag indicating if this frame has expired, ie it belongs to a previous
|
||||
// iteration of the temporal pattern.
|
||||
bool expired = false;
|
||||
// Bitmask of Vp8BufferReference flags, indicating which buffers this frame
|
||||
// updates.
|
||||
uint8_t updated_buffer_mask = 0;
|
||||
// The frame config returned by NextFrameConfig() for this frame.
|
||||
DependencyInfo dependency_info;
|
||||
};
|
||||
|
||||
static std::vector<DependencyInfo> GetDependencyInfo(size_t num_layers);
|
||||
static std::bitset<kNumReferenceBuffers> DetermineStaticBuffers(
|
||||
const std::vector<DependencyInfo>& temporal_pattern);
|
||||
bool IsSyncFrame(const Vp8FrameConfig& config) const;
|
||||
void ValidateReferences(Vp8FrameConfig::BufferFlags* flags,
|
||||
Vp8FrameConfig::Vp8BufferReference ref) const;
|
||||
void UpdateSearchOrder(Vp8FrameConfig* config);
|
||||
size_t NumFramesSinceBufferRefresh(
|
||||
Vp8FrameConfig::Vp8BufferReference ref) const;
|
||||
void ResetNumFramesSinceBufferRefresh(Vp8FrameConfig::Vp8BufferReference ref);
|
||||
void CullPendingFramesBefore(uint32_t timestamp);
|
||||
|
||||
const size_t num_layers_;
|
||||
const std::vector<unsigned int> temporal_ids_;
|
||||
const std::vector<DependencyInfo> temporal_pattern_;
|
||||
// Per reference buffer flag indicating if it is static, meaning it is only
|
||||
// updated by key-frames.
|
||||
const std::bitset<kNumReferenceBuffers> is_static_buffer_;
|
||||
FrameDependencyStructure GetTemplateStructure(int num_layers) const;
|
||||
|
||||
uint8_t pattern_idx_;
|
||||
// Updated cumulative bitrates, per temporal layer.
|
||||
absl::optional<std::vector<uint32_t>> new_bitrates_bps_;
|
||||
|
||||
// Status for each pending frame, in
|
||||
std::deque<PendingFrame> pending_frames_;
|
||||
|
||||
// One counter per reference buffer, indicating number of frames since last
|
||||
// refresh. For non-base-layer frames (ie golden, altref buffers), this is
|
||||
// reset when the pattern loops.
|
||||
std::array<size_t, kNumReferenceBuffers> frames_since_buffer_refresh_;
|
||||
|
||||
// Optional utility used to verify reference validity.
|
||||
std::unique_ptr<TemporalLayersChecker> checker_;
|
||||
};
|
||||
|
||||
class DefaultTemporalLayersChecker : public TemporalLayersChecker {
|
||||
public:
|
||||
explicit DefaultTemporalLayersChecker(int number_of_temporal_layers);
|
||||
~DefaultTemporalLayersChecker() override;
|
||||
|
||||
bool CheckTemporalConfig(bool frame_is_keyframe,
|
||||
const Vp8FrameConfig& frame_config) override;
|
||||
|
||||
private:
|
||||
struct BufferState {
|
||||
BufferState()
|
||||
: is_updated_this_cycle(false), is_keyframe(true), pattern_idx(0) {}
|
||||
|
||||
bool is_updated_this_cycle;
|
||||
bool is_keyframe;
|
||||
uint8_t pattern_idx;
|
||||
};
|
||||
const size_t num_layers_;
|
||||
std::vector<unsigned int> temporal_ids_;
|
||||
const std::vector<std::set<uint8_t>> temporal_dependencies_;
|
||||
BufferState last_;
|
||||
BufferState arf_;
|
||||
BufferState golden_;
|
||||
uint8_t pattern_idx_;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
#endif // MODULES_VIDEO_CODING_CODECS_VP8_DEFAULT_TEMPORAL_LAYERS_H_
|
||||
|
|
@ -0,0 +1,781 @@
|
|||
/*
|
||||
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/video_coding/codecs/vp8/default_temporal_layers.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
|
||||
#include "api/video/video_bitrate_allocation.h"
|
||||
#include "api/video_codecs/video_codec.h"
|
||||
#include "api/video_codecs/vp8_frame_config.h"
|
||||
#include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h"
|
||||
#include "modules/video_coding/include/video_codec_interface.h"
|
||||
#include "modules/video_coding/utility/simulcast_rate_allocator.h"
|
||||
#include "test/field_trial.h"
|
||||
#include "test/gmock.h"
|
||||
#include "test/gtest.h"
|
||||
#include "vpx/vp8cx.h"
|
||||
|
||||
// TODO(bugs.webrtc.org/10582): Test the behavior of UpdateConfiguration().
|
||||
|
||||
namespace webrtc {
|
||||
namespace test {
|
||||
namespace {
|
||||
|
||||
using ::testing::Each;
|
||||
|
||||
enum {
|
||||
kTemporalUpdateLast = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
|
||||
VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF,
|
||||
kTemporalUpdateGoldenWithoutDependency =
|
||||
VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF |
|
||||
VP8_EFLAG_NO_UPD_LAST,
|
||||
kTemporalUpdateGolden =
|
||||
VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
|
||||
kTemporalUpdateAltrefWithoutDependency =
|
||||
VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF |
|
||||
VP8_EFLAG_NO_UPD_LAST,
|
||||
kTemporalUpdateAltref = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_LAST,
|
||||
kTemporalUpdateNone = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
|
||||
VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
|
||||
kTemporalUpdateNoneNoRefAltRef =
|
||||
VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
|
||||
VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
|
||||
kTemporalUpdateNoneNoRefGolden =
|
||||
VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
|
||||
VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
|
||||
kTemporalUpdateNoneNoRefGoldenAltRef =
|
||||
VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_REF_ARF |
|
||||
VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
|
||||
kTemporalUpdateGoldenWithoutDependencyRefAltRef =
|
||||
VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
|
||||
kTemporalUpdateGoldenRefAltRef = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
|
||||
kTemporalUpdateLastRefAltRef =
|
||||
VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF,
|
||||
kTemporalUpdateLastAndGoldenRefAltRef =
|
||||
VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF,
|
||||
};
|
||||
|
||||
using BufferFlags = Vp8FrameConfig::BufferFlags;
|
||||
using Vp8BufferReference = Vp8FrameConfig::Vp8BufferReference;
|
||||
|
||||
constexpr uint8_t kNone = static_cast<uint8_t>(Vp8BufferReference::kNone);
|
||||
constexpr uint8_t kLast = static_cast<uint8_t>(Vp8BufferReference::kLast);
|
||||
constexpr uint8_t kGolden = static_cast<uint8_t>(Vp8BufferReference::kGolden);
|
||||
constexpr uint8_t kAltref = static_cast<uint8_t>(Vp8BufferReference::kAltref);
|
||||
constexpr uint8_t kAll = kLast | kGolden | kAltref;
|
||||
|
||||
constexpr int ToVp8CodecFlags(uint8_t referenced_buffers,
|
||||
uint8_t updated_buffers,
|
||||
bool update_entropy) {
|
||||
return (((referenced_buffers & kLast) == 0) ? VP8_EFLAG_NO_REF_LAST : 0) |
|
||||
(((referenced_buffers & kGolden) == 0) ? VP8_EFLAG_NO_REF_GF : 0) |
|
||||
(((referenced_buffers & kAltref) == 0) ? VP8_EFLAG_NO_REF_ARF : 0) |
|
||||
(((updated_buffers & kLast) == 0) ? VP8_EFLAG_NO_UPD_LAST : 0) |
|
||||
(((updated_buffers & kGolden) == 0) ? VP8_EFLAG_NO_UPD_GF : 0) |
|
||||
(((updated_buffers & kAltref) == 0) ? VP8_EFLAG_NO_UPD_ARF : 0) |
|
||||
(update_entropy ? 0 : VP8_EFLAG_NO_UPD_ENTROPY);
|
||||
}
|
||||
|
||||
constexpr int kKeyFrameFlags = ToVp8CodecFlags(kNone, kAll, true);
|
||||
|
||||
std::vector<uint32_t> GetTemporalLayerRates(int target_bitrate_kbps,
|
||||
int framerate_fps,
|
||||
int num_temporal_layers) {
|
||||
VideoCodec codec;
|
||||
codec.codecType = VideoCodecType::kVideoCodecVP8;
|
||||
codec.numberOfSimulcastStreams = 1;
|
||||
codec.maxBitrate = target_bitrate_kbps;
|
||||
codec.maxFramerate = framerate_fps;
|
||||
codec.simulcastStream[0].targetBitrate = target_bitrate_kbps;
|
||||
codec.simulcastStream[0].maxBitrate = target_bitrate_kbps;
|
||||
codec.simulcastStream[0].numberOfTemporalLayers = num_temporal_layers;
|
||||
codec.simulcastStream[0].active = true;
|
||||
SimulcastRateAllocator allocator(codec);
|
||||
return allocator
|
||||
.Allocate(
|
||||
VideoBitrateAllocationParameters(target_bitrate_kbps, framerate_fps))
|
||||
.GetTemporalLayerAllocation(0);
|
||||
}
|
||||
|
||||
constexpr int kDefaultBitrateBps = 500;
|
||||
constexpr int kDefaultFramerate = 30;
|
||||
constexpr int kDefaultBytesPerFrame =
|
||||
(kDefaultBitrateBps / 8) / kDefaultFramerate;
|
||||
constexpr int kDefaultQp = 2;
|
||||
} // namespace
|
||||
|
||||
class TemporalLayersTest : public ::testing::Test {
|
||||
public:
|
||||
~TemporalLayersTest() override = default;
|
||||
|
||||
CodecSpecificInfo* IgnoredCodecSpecificInfo() {
|
||||
codec_specific_info_ = std::make_unique<CodecSpecificInfo>();
|
||||
return codec_specific_info_.get();
|
||||
}
|
||||
|
||||
private:
|
||||
std::unique_ptr<CodecSpecificInfo> codec_specific_info_;
|
||||
};
|
||||
|
||||
TEST_F(TemporalLayersTest, 2Layers) {
|
||||
constexpr int kNumLayers = 2;
|
||||
DefaultTemporalLayers tl(kNumLayers);
|
||||
DefaultTemporalLayersChecker checker(kNumLayers);
|
||||
tl.OnRatesUpdated(0,
|
||||
GetTemporalLayerRates(kDefaultBytesPerFrame,
|
||||
kDefaultFramerate, kNumLayers),
|
||||
kDefaultFramerate);
|
||||
tl.UpdateConfiguration(0);
|
||||
|
||||
constexpr size_t kPatternSize = 4;
|
||||
constexpr size_t kRepetitions = 4;
|
||||
|
||||
const int expected_flags[kPatternSize] = {
|
||||
ToVp8CodecFlags(kLast, kLast, true),
|
||||
ToVp8CodecFlags(kLast, kGolden, true),
|
||||
ToVp8CodecFlags(kLast, kLast, true),
|
||||
ToVp8CodecFlags(kLast | kGolden, kNone, false),
|
||||
};
|
||||
const int expected_temporal_idx[kPatternSize] = {0, 1, 0, 1};
|
||||
const bool expected_layer_sync[kPatternSize] = {false, true, false, false};
|
||||
|
||||
uint32_t timestamp = 0;
|
||||
for (size_t i = 0; i < kPatternSize * kRepetitions; ++i) {
|
||||
const size_t ind = i % kPatternSize;
|
||||
const bool is_keyframe = (i == 0);
|
||||
CodecSpecificInfo info;
|
||||
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
|
||||
EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[ind],
|
||||
LibvpxVp8Encoder::EncodeFlags(tl_config))
|
||||
<< i;
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe,
|
||||
kDefaultQp, &info);
|
||||
EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config));
|
||||
EXPECT_EQ(expected_temporal_idx[ind], info.codecSpecific.VP8.temporalIdx);
|
||||
EXPECT_EQ(expected_temporal_idx[ind], tl_config.packetizer_temporal_idx);
|
||||
EXPECT_EQ(expected_temporal_idx[ind], tl_config.encoder_layer_id);
|
||||
EXPECT_EQ(is_keyframe || expected_layer_sync[ind],
|
||||
info.codecSpecific.VP8.layerSync);
|
||||
EXPECT_EQ(expected_layer_sync[ind], tl_config.layer_sync);
|
||||
timestamp += 3000;
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TemporalLayersTest, 3Layers) {
|
||||
constexpr int kNumLayers = 3;
|
||||
DefaultTemporalLayers tl(kNumLayers);
|
||||
DefaultTemporalLayersChecker checker(kNumLayers);
|
||||
tl.OnRatesUpdated(0,
|
||||
GetTemporalLayerRates(kDefaultBytesPerFrame,
|
||||
kDefaultFramerate, kNumLayers),
|
||||
kDefaultFramerate);
|
||||
tl.UpdateConfiguration(0);
|
||||
|
||||
int expected_flags[16] = {
|
||||
kTemporalUpdateLast,
|
||||
kTemporalUpdateNoneNoRefGoldenAltRef,
|
||||
kTemporalUpdateGoldenWithoutDependency,
|
||||
kTemporalUpdateNoneNoRefAltRef,
|
||||
kTemporalUpdateLast,
|
||||
kTemporalUpdateNoneNoRefAltRef,
|
||||
kTemporalUpdateGolden,
|
||||
kTemporalUpdateNoneNoRefAltRef,
|
||||
kTemporalUpdateLast,
|
||||
kTemporalUpdateNoneNoRefGoldenAltRef,
|
||||
kTemporalUpdateGoldenWithoutDependency,
|
||||
kTemporalUpdateNoneNoRefAltRef,
|
||||
kTemporalUpdateLast,
|
||||
kTemporalUpdateNoneNoRefAltRef,
|
||||
kTemporalUpdateGolden,
|
||||
kTemporalUpdateNoneNoRefAltRef,
|
||||
};
|
||||
int expected_temporal_idx[16] = {0, 2, 1, 2, 0, 2, 1, 2,
|
||||
0, 2, 1, 2, 0, 2, 1, 2};
|
||||
|
||||
bool expected_layer_sync[16] = {false, true, true, false, false, false,
|
||||
false, false, false, true, true, false,
|
||||
false, false, false, false};
|
||||
|
||||
unsigned int timestamp = 0;
|
||||
for (int i = 0; i < 16; ++i) {
|
||||
const bool is_keyframe = (i == 0);
|
||||
CodecSpecificInfo info;
|
||||
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
|
||||
EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[i],
|
||||
LibvpxVp8Encoder::EncodeFlags(tl_config))
|
||||
<< i;
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe,
|
||||
kDefaultQp, &info);
|
||||
EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config));
|
||||
EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx);
|
||||
EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
|
||||
EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
|
||||
EXPECT_EQ(is_keyframe || expected_layer_sync[i],
|
||||
info.codecSpecific.VP8.layerSync);
|
||||
EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
|
||||
timestamp += 3000;
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TemporalLayersTest, Alternative3Layers) {
|
||||
constexpr int kNumLayers = 3;
|
||||
ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
|
||||
DefaultTemporalLayers tl(kNumLayers);
|
||||
DefaultTemporalLayersChecker checker(kNumLayers);
|
||||
tl.OnRatesUpdated(0,
|
||||
GetTemporalLayerRates(kDefaultBytesPerFrame,
|
||||
kDefaultFramerate, kNumLayers),
|
||||
kDefaultFramerate);
|
||||
tl.UpdateConfiguration(0);
|
||||
|
||||
int expected_flags[8] = {kTemporalUpdateLast,
|
||||
kTemporalUpdateAltrefWithoutDependency,
|
||||
kTemporalUpdateGoldenWithoutDependency,
|
||||
kTemporalUpdateNone,
|
||||
kTemporalUpdateLast,
|
||||
kTemporalUpdateAltrefWithoutDependency,
|
||||
kTemporalUpdateGoldenWithoutDependency,
|
||||
kTemporalUpdateNone};
|
||||
int expected_temporal_idx[8] = {0, 2, 1, 2, 0, 2, 1, 2};
|
||||
|
||||
bool expected_layer_sync[8] = {false, true, true, false,
|
||||
false, true, true, false};
|
||||
|
||||
unsigned int timestamp = 0;
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
const bool is_keyframe = (i == 0);
|
||||
CodecSpecificInfo info;
|
||||
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
|
||||
EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[i],
|
||||
LibvpxVp8Encoder::EncodeFlags(tl_config))
|
||||
<< i;
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe,
|
||||
kDefaultQp, &info);
|
||||
EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config));
|
||||
EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx);
|
||||
EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
|
||||
EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
|
||||
EXPECT_EQ(is_keyframe || expected_layer_sync[i],
|
||||
info.codecSpecific.VP8.layerSync);
|
||||
EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
|
||||
timestamp += 3000;
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TemporalLayersTest, SearchOrder) {
|
||||
constexpr int kNumLayers = 3;
|
||||
ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
|
||||
DefaultTemporalLayers tl(kNumLayers);
|
||||
DefaultTemporalLayersChecker checker(kNumLayers);
|
||||
tl.OnRatesUpdated(0,
|
||||
GetTemporalLayerRates(kDefaultBytesPerFrame,
|
||||
kDefaultFramerate, kNumLayers),
|
||||
kDefaultFramerate);
|
||||
tl.UpdateConfiguration(0);
|
||||
|
||||
// Use a repeating pattern of tl 0, 2, 1, 2.
|
||||
// Tl 0, 1, 2 update last, golden, altref respectively.
|
||||
|
||||
// Start with a key-frame. tl_config flags can be ignored.
|
||||
uint32_t timestamp = 0;
|
||||
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
|
||||
// TL2 frame. First one only references TL0. Updates altref.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
|
||||
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
|
||||
|
||||
// TL1 frame. Can only reference TL0. Updated golden.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
|
||||
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
|
||||
|
||||
// TL2 frame. Can reference all three buffers. Golden was the last to be
|
||||
// updated, the next to last was altref.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kGolden);
|
||||
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kAltref);
|
||||
}
|
||||
|
||||
TEST_F(TemporalLayersTest, SearchOrderWithDrop) {
|
||||
constexpr int kNumLayers = 3;
|
||||
ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
|
||||
DefaultTemporalLayers tl(kNumLayers);
|
||||
DefaultTemporalLayersChecker checker(kNumLayers);
|
||||
tl.OnRatesUpdated(0,
|
||||
GetTemporalLayerRates(kDefaultBytesPerFrame,
|
||||
kDefaultFramerate, kNumLayers),
|
||||
kDefaultFramerate);
|
||||
tl.UpdateConfiguration(0);
|
||||
|
||||
// Use a repeating pattern of tl 0, 2, 1, 2.
|
||||
// Tl 0, 1, 2 update last, golden, altref respectively.
|
||||
|
||||
// Start with a key-frame. tl_config flags can be ignored.
|
||||
uint32_t timestamp = 0;
|
||||
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
|
||||
// TL2 frame. First one only references TL0. Updates altref.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
|
||||
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
|
||||
|
||||
// Dropped TL1 frame. Can only reference TL0. Should have updated golden.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
|
||||
|
||||
// TL2 frame. Can normally reference all three buffers, but golden has not
|
||||
// been populated this cycle. Altref was last to be updated, before that last.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kAltref);
|
||||
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kLast);
|
||||
}
|
||||
|
||||
TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) {
|
||||
constexpr int kNumLayers = 3;
|
||||
// Use a repeating pattern of tl 0, 2, 1, 2.
|
||||
// Tl 0, 1, 2 update last, golden, altref respectively.
|
||||
ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
|
||||
DefaultTemporalLayers tl(kNumLayers);
|
||||
DefaultTemporalLayersChecker checker(kNumLayers);
|
||||
tl.OnRatesUpdated(0,
|
||||
GetTemporalLayerRates(kDefaultBytesPerFrame,
|
||||
kDefaultFramerate, kNumLayers),
|
||||
kDefaultFramerate);
|
||||
tl.UpdateConfiguration(0);
|
||||
|
||||
// Start with a keyframe.
|
||||
uint32_t timestamp = 0;
|
||||
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
|
||||
// Dropped TL2 frame.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
|
||||
|
||||
// Dropped TL1 frame.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
|
||||
|
||||
// TL2 frame. Can reference all three buffers, valid since golden and altref
|
||||
// both contain the last keyframe.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
|
||||
EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference);
|
||||
EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference);
|
||||
|
||||
// Restart of cycle!
|
||||
|
||||
// TL0 base layer frame, updating and referencing last.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
|
||||
// TL2 frame, updating altref.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
|
||||
// TL1 frame, updating golden.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
|
||||
// TL2 frame. Can still reference all buffer since they have been update this
|
||||
// cycle.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
|
||||
EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference);
|
||||
EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference);
|
||||
|
||||
// Restart of cycle!
|
||||
|
||||
// TL0 base layer frame, updating and referencing last.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
|
||||
// Dropped TL2 frame.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
|
||||
|
||||
// Dropped TL1 frame.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
|
||||
|
||||
// TL2 frame. This time golden and altref contain data from the previous cycle
|
||||
// and cannot be referenced.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
|
||||
EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference);
|
||||
EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
|
||||
}
|
||||
|
||||
TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExist) {
|
||||
constexpr int kNumLayers = 3;
|
||||
// Use a repeating pattern of tl 0, 2, 1, 2.
|
||||
// Tl 0, 1 updates last, golden respectively. Altref is always last keyframe.
|
||||
DefaultTemporalLayers tl(kNumLayers);
|
||||
DefaultTemporalLayersChecker checker(kNumLayers);
|
||||
tl.OnRatesUpdated(0,
|
||||
GetTemporalLayerRates(kDefaultBytesPerFrame,
|
||||
kDefaultFramerate, kNumLayers),
|
||||
kDefaultFramerate);
|
||||
tl.UpdateConfiguration(0);
|
||||
|
||||
// Start with a keyframe.
|
||||
uint32_t timestamp = 0;
|
||||
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
|
||||
// Do a full cycle of the pattern.
|
||||
for (int i = 0; i < 7; ++i) {
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
}
|
||||
|
||||
// TL0 base layer frame, starting the cycle over.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
|
||||
// TL2 frame.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
|
||||
// Encoder has a hiccup and builds a queue, so frame encoding is delayed.
|
||||
// TL1 frame, updating golden.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
|
||||
// TL2 frame, that should be referencing golden, but we can't be certain it's
|
||||
// not going to be dropped, so that is not allowed.
|
||||
tl_config = tl.NextFrameConfig(0, timestamp + 1);
|
||||
EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
|
||||
EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference);
|
||||
EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
|
||||
|
||||
// TL0 base layer frame.
|
||||
tl_config = tl.NextFrameConfig(0, timestamp + 2);
|
||||
|
||||
// The previous four enqueued frames finally get encoded, and the updated
|
||||
// buffers are now OK to reference.
|
||||
// Enqueued TL1 frame ready.
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
// Enqueued TL2 frame.
|
||||
tl.OnEncodeDone(0, ++timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
// Enqueued TL0 frame.
|
||||
tl.OnEncodeDone(0, ++timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
|
||||
// TL2 frame, all buffers are now in a known good state, OK to reference.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp + 1);
|
||||
EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
|
||||
EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference);
|
||||
EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
|
||||
}
|
||||
|
||||
TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExistLongDelay) {
|
||||
constexpr int kNumLayers = 3;
|
||||
// Use a repeating pattern of tl 0, 2, 1, 2.
|
||||
// Tl 0, 1 updates last, golden, altref respectively.
|
||||
ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
|
||||
DefaultTemporalLayers tl(kNumLayers);
|
||||
DefaultTemporalLayersChecker checker(kNumLayers);
|
||||
tl.OnRatesUpdated(0,
|
||||
GetTemporalLayerRates(kDefaultBytesPerFrame,
|
||||
kDefaultFramerate, kNumLayers),
|
||||
kDefaultFramerate);
|
||||
tl.UpdateConfiguration(0);
|
||||
|
||||
// Start with a keyframe.
|
||||
uint32_t timestamp = 0;
|
||||
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
|
||||
// Do a full cycle of the pattern.
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
}
|
||||
|
||||
// TL0 base layer frame, starting the cycle over.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
|
||||
// TL2 frame.
|
||||
tl_config = tl.NextFrameConfig(0, ++timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
|
||||
// Encoder has a hiccup and builds a queue, so frame encoding is delayed.
|
||||
// Encoded, but delayed frames in TL 1, 2.
|
||||
tl_config = tl.NextFrameConfig(0, timestamp + 1);
|
||||
tl_config = tl.NextFrameConfig(0, timestamp + 2);
|
||||
|
||||
// Restart of the pattern!
|
||||
|
||||
// Encoded, but delayed frames in TL 2, 1.
|
||||
tl_config = tl.NextFrameConfig(0, timestamp + 3);
|
||||
tl_config = tl.NextFrameConfig(0, timestamp + 4);
|
||||
|
||||
// TL1 frame from last cycle is ready.
|
||||
tl.OnEncodeDone(0, timestamp + 1, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
// TL2 frame from last cycle is ready.
|
||||
tl.OnEncodeDone(0, timestamp + 2, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
|
||||
// TL2 frame, that should be referencing all buffers, but altref and golden
|
||||
// haven not been updated this cycle. (Don't be fooled by the late frames from
|
||||
// the last cycle!)
|
||||
tl_config = tl.NextFrameConfig(0, timestamp + 5);
|
||||
EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
|
||||
EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference);
|
||||
EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
|
||||
}
|
||||
|
||||
TEST_F(TemporalLayersTest, KeyFrame) {
|
||||
constexpr int kNumLayers = 3;
|
||||
DefaultTemporalLayers tl(kNumLayers);
|
||||
DefaultTemporalLayersChecker checker(kNumLayers);
|
||||
tl.OnRatesUpdated(0,
|
||||
GetTemporalLayerRates(kDefaultBytesPerFrame,
|
||||
kDefaultFramerate, kNumLayers),
|
||||
kDefaultFramerate);
|
||||
tl.UpdateConfiguration(0);
|
||||
|
||||
int expected_flags[8] = {
|
||||
kTemporalUpdateLastRefAltRef,
|
||||
kTemporalUpdateNoneNoRefGoldenAltRef,
|
||||
kTemporalUpdateGoldenWithoutDependency,
|
||||
kTemporalUpdateNoneNoRefAltRef,
|
||||
kTemporalUpdateLast,
|
||||
kTemporalUpdateNoneNoRefAltRef,
|
||||
kTemporalUpdateGolden,
|
||||
kTemporalUpdateNone,
|
||||
};
|
||||
int expected_temporal_idx[8] = {0, 2, 1, 2, 0, 2, 1, 2};
|
||||
bool expected_layer_sync[8] = {true, true, true, false,
|
||||
false, false, false, false};
|
||||
|
||||
uint32_t timestamp = 0;
|
||||
for (int i = 0; i < 7; ++i) {
|
||||
// Temporal pattern starts from 0 after key frame. Let the first `i` - 1
|
||||
// frames be delta frames, and the `i`th one key frame.
|
||||
for (int j = 1; j <= i; ++j) {
|
||||
// Since last frame was always a keyframe and thus index 0 in the pattern,
|
||||
// this loop starts at index 1.
|
||||
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
|
||||
EXPECT_EQ(expected_flags[j], LibvpxVp8Encoder::EncodeFlags(tl_config))
|
||||
<< j;
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
EXPECT_TRUE(checker.CheckTemporalConfig(false, tl_config));
|
||||
EXPECT_EQ(expected_temporal_idx[j], tl_config.packetizer_temporal_idx);
|
||||
EXPECT_EQ(expected_temporal_idx[j], tl_config.encoder_layer_id);
|
||||
EXPECT_EQ(expected_layer_sync[j], tl_config.layer_sync);
|
||||
timestamp += 3000;
|
||||
}
|
||||
|
||||
CodecSpecificInfo info;
|
||||
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
|
||||
tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
|
||||
&info);
|
||||
EXPECT_TRUE(info.codecSpecific.VP8.layerSync)
|
||||
<< "Key frame should be marked layer sync.";
|
||||
EXPECT_EQ(0, info.codecSpecific.VP8.temporalIdx)
|
||||
<< "Key frame should always be packetized as layer 0";
|
||||
EXPECT_EQ(0, info.generic_frame_info->temporal_id)
|
||||
<< "Key frame should always be packetized as layer 0";
|
||||
EXPECT_THAT(info.generic_frame_info->decode_target_indications,
|
||||
Each(DecodeTargetIndication::kSwitch))
|
||||
<< "Key frame is universal switch";
|
||||
EXPECT_TRUE(checker.CheckTemporalConfig(true, tl_config));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TemporalLayersTest, SetsTlCountOnFirstConfigUpdate) {
|
||||
// Create an instance and fetch config update without setting any rate.
|
||||
constexpr int kNumLayers = 2;
|
||||
DefaultTemporalLayers tl(kNumLayers);
|
||||
Vp8EncoderConfig config = tl.UpdateConfiguration(0);
|
||||
|
||||
// Config should indicate correct number of temporal layers, but zero bitrate.
|
||||
ASSERT_TRUE(config.temporal_layer_config.has_value());
|
||||
EXPECT_EQ(config.temporal_layer_config->ts_number_layers,
|
||||
uint32_t{kNumLayers});
|
||||
std::array<uint32_t, Vp8EncoderConfig::TemporalLayerConfig::kMaxLayers>
|
||||
kZeroRate = {};
|
||||
EXPECT_EQ(config.temporal_layer_config->ts_target_bitrate, kZeroRate);
|
||||
|
||||
// On second call, no new update.
|
||||
config = tl.UpdateConfiguration(0);
|
||||
EXPECT_FALSE(config.temporal_layer_config.has_value());
|
||||
}
|
||||
|
||||
class TemporalLayersReferenceTest : public TemporalLayersTest,
|
||||
public ::testing::WithParamInterface<int> {
|
||||
public:
|
||||
TemporalLayersReferenceTest()
|
||||
: timestamp_(1),
|
||||
last_sync_timestamp_(timestamp_),
|
||||
tl0_reference_(nullptr) {}
|
||||
virtual ~TemporalLayersReferenceTest() {}
|
||||
|
||||
protected:
|
||||
static const int kMaxPatternLength = 32;
|
||||
|
||||
struct BufferState {
|
||||
BufferState() : BufferState(-1, 0, false) {}
|
||||
BufferState(int temporal_idx, uint32_t timestamp, bool sync)
|
||||
: temporal_idx(temporal_idx), timestamp(timestamp), sync(sync) {}
|
||||
int temporal_idx;
|
||||
uint32_t timestamp;
|
||||
bool sync;
|
||||
};
|
||||
|
||||
bool UpdateSyncRefState(const BufferFlags& flags, BufferState* buffer_state) {
|
||||
if (flags & BufferFlags::kReference) {
|
||||
if (buffer_state->temporal_idx == -1)
|
||||
return true; // References key-frame.
|
||||
if (buffer_state->temporal_idx == 0) {
|
||||
// No more than one reference to TL0 frame.
|
||||
EXPECT_EQ(nullptr, tl0_reference_);
|
||||
tl0_reference_ = buffer_state;
|
||||
return true;
|
||||
}
|
||||
return false; // References higher layer.
|
||||
}
|
||||
return true; // No reference, does not affect sync frame status.
|
||||
}
|
||||
|
||||
void ValidateReference(const BufferFlags& flags,
|
||||
const BufferState& buffer_state,
|
||||
int temporal_layer) {
|
||||
if (flags & BufferFlags::kReference) {
|
||||
if (temporal_layer > 0 && buffer_state.timestamp > 0) {
|
||||
// Check that high layer reference does not go past last sync frame.
|
||||
EXPECT_GE(buffer_state.timestamp, last_sync_timestamp_);
|
||||
}
|
||||
// No reference to buffer in higher layer.
|
||||
EXPECT_LE(buffer_state.temporal_idx, temporal_layer);
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t timestamp_ = 1;
|
||||
uint32_t last_sync_timestamp_ = timestamp_;
|
||||
BufferState* tl0_reference_;
|
||||
|
||||
BufferState last_state;
|
||||
BufferState golden_state;
|
||||
BufferState altref_state;
|
||||
};
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(DefaultTemporalLayersTest,
|
||||
TemporalLayersReferenceTest,
|
||||
::testing::Range(1, kMaxTemporalStreams + 1));
|
||||
|
||||
TEST_P(TemporalLayersReferenceTest, ValidFrameConfigs) {
|
||||
const int num_layers = GetParam();
|
||||
DefaultTemporalLayers tl(num_layers);
|
||||
tl.OnRatesUpdated(
|
||||
0, GetTemporalLayerRates(kDefaultBytesPerFrame, kDefaultFramerate, 1),
|
||||
kDefaultFramerate);
|
||||
tl.UpdateConfiguration(0);
|
||||
|
||||
// Run through the pattern and store the frame dependencies, plus keep track
|
||||
// of the buffer state; which buffers references which temporal layers (if
|
||||
// (any). If a given buffer is never updated, it is legal to reference it
|
||||
// even for sync frames. In order to be general, don't assume TL0 always
|
||||
// updates `last`.
|
||||
std::vector<Vp8FrameConfig> tl_configs(kMaxPatternLength);
|
||||
for (int i = 0; i < kMaxPatternLength; ++i) {
|
||||
Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp_);
|
||||
tl.OnEncodeDone(0, timestamp_, kDefaultBytesPerFrame, i == 0, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
++timestamp_;
|
||||
EXPECT_FALSE(tl_config.drop_frame);
|
||||
tl_configs.push_back(tl_config);
|
||||
int temporal_idx = tl_config.encoder_layer_id;
|
||||
// For the default layers, always keep encoder and rtp layers in sync.
|
||||
EXPECT_EQ(tl_config.packetizer_temporal_idx, temporal_idx);
|
||||
|
||||
// Determine if this frame is in a higher layer but references only TL0
|
||||
// or untouched buffers, if so verify it is marked as a layer sync.
|
||||
bool is_sync_frame = true;
|
||||
tl0_reference_ = nullptr;
|
||||
if (temporal_idx <= 0) {
|
||||
is_sync_frame = false; // TL0 by definition not a sync frame.
|
||||
} else if (!UpdateSyncRefState(tl_config.last_buffer_flags, &last_state)) {
|
||||
is_sync_frame = false;
|
||||
} else if (!UpdateSyncRefState(tl_config.golden_buffer_flags,
|
||||
&golden_state)) {
|
||||
is_sync_frame = false;
|
||||
} else if (!UpdateSyncRefState(tl_config.arf_buffer_flags, &altref_state)) {
|
||||
is_sync_frame = false;
|
||||
}
|
||||
if (is_sync_frame) {
|
||||
// Cache timestamp for last found sync frame, so that we can verify no
|
||||
// references back past this frame.
|
||||
ASSERT_TRUE(tl0_reference_);
|
||||
last_sync_timestamp_ = tl0_reference_->timestamp;
|
||||
}
|
||||
EXPECT_EQ(tl_config.layer_sync, is_sync_frame);
|
||||
|
||||
// Validate no reference from lower to high temporal layer, or backwards
|
||||
// past last reference frame.
|
||||
ValidateReference(tl_config.last_buffer_flags, last_state, temporal_idx);
|
||||
ValidateReference(tl_config.golden_buffer_flags, golden_state,
|
||||
temporal_idx);
|
||||
ValidateReference(tl_config.arf_buffer_flags, altref_state, temporal_idx);
|
||||
|
||||
// Update the current layer state.
|
||||
BufferState state = {temporal_idx, timestamp_, is_sync_frame};
|
||||
if (tl_config.last_buffer_flags & BufferFlags::kUpdate)
|
||||
last_state = state;
|
||||
if (tl_config.golden_buffer_flags & BufferFlags::kUpdate)
|
||||
golden_state = state;
|
||||
if (tl_config.arf_buffer_flags & BufferFlags::kUpdate)
|
||||
altref_state = state;
|
||||
}
|
||||
}
|
||||
} // namespace test
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
/*
|
||||
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_TEMPORAL_LAYERS_CHECKER_H_
|
||||
#define MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_TEMPORAL_LAYERS_CHECKER_H_
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "api/video_codecs/vp8_frame_config.h"
|
||||
#include "api/video_codecs/vp8_temporal_layers.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
// Interface for a class that verifies correctness of temporal layer
|
||||
// configurations (dependencies, sync flag, etc).
|
||||
// Intended to be used in tests as well as with real apps in debug mode.
|
||||
class TemporalLayersChecker {
|
||||
public:
|
||||
explicit TemporalLayersChecker(int num_temporal_layers);
|
||||
virtual ~TemporalLayersChecker() {}
|
||||
|
||||
virtual bool CheckTemporalConfig(bool frame_is_keyframe,
|
||||
const Vp8FrameConfig& frame_config);
|
||||
|
||||
static std::unique_ptr<TemporalLayersChecker> CreateTemporalLayersChecker(
|
||||
Vp8TemporalLayersType type,
|
||||
int num_temporal_layers);
|
||||
|
||||
private:
|
||||
struct BufferState {
|
||||
BufferState() : is_keyframe(true), temporal_layer(0), sequence_number(0) {}
|
||||
bool is_keyframe;
|
||||
uint8_t temporal_layer;
|
||||
uint32_t sequence_number;
|
||||
};
|
||||
bool CheckAndUpdateBufferState(BufferState* state,
|
||||
bool* need_sync,
|
||||
bool frame_is_keyframe,
|
||||
uint8_t temporal_layer,
|
||||
Vp8FrameConfig::BufferFlags flags,
|
||||
uint32_t sequence_number,
|
||||
uint32_t* lowest_sequence_referenced);
|
||||
BufferState last_;
|
||||
BufferState arf_;
|
||||
BufferState golden_;
|
||||
int num_temporal_layers_;
|
||||
uint32_t sequence_number_;
|
||||
uint32_t last_sync_sequence_number_;
|
||||
uint32_t last_tl0_sequence_number_;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_TEMPORAL_LAYERS_CHECKER_H_
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
/*
|
||||
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_H_
|
||||
#define MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_H_
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "api/environment/environment.h"
|
||||
#include "api/video_codecs/video_encoder.h"
|
||||
#include "api/video_codecs/vp8_frame_buffer_controller.h"
|
||||
#include "modules/video_coding/include/video_codec_interface.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
// TODO(brandtr): Move these interfaces to the api/ folder.
|
||||
class VP8Encoder {
|
||||
public:
|
||||
struct Settings {
|
||||
// Allows for overriding the Vp8FrameBufferController used by the encoder.
|
||||
// If unset, a default Vp8FrameBufferController will be instantiated
|
||||
// internally.
|
||||
std::unique_ptr<Vp8FrameBufferControllerFactory>
|
||||
frame_buffer_controller_factory = nullptr;
|
||||
|
||||
// Allows for overriding the resolution/bitrate limits exposed through
|
||||
// VideoEncoder::GetEncoderInfo(). No override is done if empty.
|
||||
std::vector<VideoEncoder::ResolutionBitrateLimits>
|
||||
resolution_bitrate_limits = {};
|
||||
};
|
||||
|
||||
static std::unique_ptr<VideoEncoder> Create();
|
||||
static std::unique_ptr<VideoEncoder> Create(Settings settings);
|
||||
};
|
||||
|
||||
// TODO: bugs.webrtc.org/15791 - Deprecate and delete in favor of the
|
||||
// CreateVp8Decoder function.
|
||||
class VP8Decoder {
|
||||
public:
|
||||
static std::unique_ptr<VideoDecoder> Create();
|
||||
};
|
||||
|
||||
std::unique_ptr<VideoDecoder> CreateVp8Decoder(const Environment& env);
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_H_
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
/*
|
||||
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
// This file contains codec dependent definitions that are needed in
|
||||
// order to compile the WebRTC codebase, even if this codec is not used.
|
||||
|
||||
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_GLOBALS_H_
|
||||
#define MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_GLOBALS_H_
|
||||
|
||||
#include "modules/video_coding/codecs/interface/common_constants.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
struct RTPVideoHeaderVP8 {
|
||||
void InitRTPVideoHeaderVP8() {
|
||||
nonReference = false;
|
||||
pictureId = kNoPictureId;
|
||||
tl0PicIdx = kNoTl0PicIdx;
|
||||
temporalIdx = kNoTemporalIdx;
|
||||
layerSync = false;
|
||||
keyIdx = kNoKeyIdx;
|
||||
partitionId = 0;
|
||||
beginningOfPartition = false;
|
||||
}
|
||||
|
||||
friend bool operator==(const RTPVideoHeaderVP8& lhs,
|
||||
const RTPVideoHeaderVP8& rhs) {
|
||||
return lhs.nonReference == rhs.nonReference &&
|
||||
lhs.pictureId == rhs.pictureId && lhs.tl0PicIdx == rhs.tl0PicIdx &&
|
||||
lhs.temporalIdx == rhs.temporalIdx &&
|
||||
lhs.layerSync == rhs.layerSync && lhs.keyIdx == rhs.keyIdx &&
|
||||
lhs.partitionId == rhs.partitionId &&
|
||||
lhs.beginningOfPartition == rhs.beginningOfPartition;
|
||||
}
|
||||
|
||||
friend bool operator!=(const RTPVideoHeaderVP8& lhs,
|
||||
const RTPVideoHeaderVP8& rhs) {
|
||||
return !(lhs == rhs);
|
||||
}
|
||||
|
||||
bool nonReference; // Frame is discardable.
|
||||
int16_t pictureId; // Picture ID index, 15 bits;
|
||||
// kNoPictureId if PictureID does not exist.
|
||||
int16_t tl0PicIdx; // TL0PIC_IDX, 8 bits;
|
||||
// kNoTl0PicIdx means no value provided.
|
||||
uint8_t temporalIdx; // Temporal layer index, or kNoTemporalIdx.
|
||||
bool layerSync; // This frame is a layer sync frame.
|
||||
// Disabled if temporalIdx == kNoTemporalIdx.
|
||||
int keyIdx; // 5 bits; kNoKeyIdx means not used.
|
||||
int partitionId; // VP8 partition ID
|
||||
bool beginningOfPartition; // True if this packet is the first
|
||||
// in a VP8 partition. Otherwise false
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_GLOBALS_H_
|
||||
|
|
@ -0,0 +1,356 @@
|
|||
/*
|
||||
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "api/environment/environment.h"
|
||||
#include "api/field_trials_view.h"
|
||||
#include "api/scoped_refptr.h"
|
||||
#include "api/transport/field_trial_based_config.h"
|
||||
#include "api/video/i420_buffer.h"
|
||||
#include "api/video/video_frame.h"
|
||||
#include "api/video/video_frame_buffer.h"
|
||||
#include "api/video/video_rotation.h"
|
||||
#include "modules/video_coding/codecs/vp8/include/vp8.h"
|
||||
#include "modules/video_coding/include/video_error_codes.h"
|
||||
#include "rtc_base/checks.h"
|
||||
#include "rtc_base/numerics/exp_filter.h"
|
||||
#include "rtc_base/time_utils.h"
|
||||
#include "system_wrappers/include/metrics.h"
|
||||
#include "third_party/libyuv/include/libyuv/convert.h"
|
||||
#include <vpx/vp8.h>
|
||||
#include <vpx/vp8dx.h>
|
||||
#include <vpx/vpx_decoder.h>
|
||||
|
||||
namespace webrtc {
|
||||
namespace {
|
||||
// vpx_decoder.h documentation indicates decode deadline is time in us, with
|
||||
// "Set to zero for unlimited.", but actual implementation requires this to be
|
||||
// a mode with 0 meaning allow delay and 1 not allowing it.
|
||||
constexpr long kDecodeDeadlineRealtime = 1; // NOLINT
|
||||
|
||||
const char kVp8PostProcArmFieldTrial[] = "WebRTC-VP8-Postproc-Config-Arm";
|
||||
const char kVp8PostProcFieldTrial[] = "WebRTC-VP8-Postproc-Config";
|
||||
|
||||
#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
|
||||
defined(WEBRTC_ANDROID)
|
||||
constexpr bool kIsArm = true;
|
||||
#else
|
||||
constexpr bool kIsArm = false;
|
||||
#endif
|
||||
|
||||
absl::optional<LibvpxVp8Decoder::DeblockParams> DefaultDeblockParams() {
|
||||
return LibvpxVp8Decoder::DeblockParams(/*max_level=*/8,
|
||||
/*degrade_qp=*/60,
|
||||
/*min_qp=*/30);
|
||||
}
|
||||
|
||||
absl::optional<LibvpxVp8Decoder::DeblockParams>
|
||||
GetPostProcParamsFromFieldTrialGroup(const FieldTrialsView& field_trials) {
|
||||
std::string group = field_trials.Lookup(kIsArm ? kVp8PostProcArmFieldTrial
|
||||
: kVp8PostProcFieldTrial);
|
||||
if (group.empty()) {
|
||||
return DefaultDeblockParams();
|
||||
}
|
||||
|
||||
LibvpxVp8Decoder::DeblockParams params;
|
||||
if (sscanf(group.c_str(), "Enabled-%d,%d,%d", ¶ms.max_level,
|
||||
¶ms.min_qp, ¶ms.degrade_qp) != 3) {
|
||||
return DefaultDeblockParams();
|
||||
}
|
||||
|
||||
if (params.max_level < 0 || params.max_level > 16) {
|
||||
return DefaultDeblockParams();
|
||||
}
|
||||
|
||||
if (params.min_qp < 0 || params.degrade_qp <= params.min_qp) {
|
||||
return DefaultDeblockParams();
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::unique_ptr<VideoDecoder> VP8Decoder::Create() {
|
||||
return std::make_unique<LibvpxVp8Decoder>();
|
||||
}
|
||||
|
||||
std::unique_ptr<VideoDecoder> CreateVp8Decoder(const Environment& env) {
|
||||
return std::make_unique<LibvpxVp8Decoder>(env);
|
||||
}
|
||||
|
||||
class LibvpxVp8Decoder::QpSmoother {
|
||||
public:
|
||||
QpSmoother() : last_sample_ms_(rtc::TimeMillis()), smoother_(kAlpha) {}
|
||||
|
||||
int GetAvg() const {
|
||||
float value = smoother_.filtered();
|
||||
return (value == rtc::ExpFilter::kValueUndefined) ? 0
|
||||
: static_cast<int>(value);
|
||||
}
|
||||
|
||||
void Add(float sample) {
|
||||
int64_t now_ms = rtc::TimeMillis();
|
||||
smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample);
|
||||
last_sample_ms_ = now_ms;
|
||||
}
|
||||
|
||||
void Reset() { smoother_.Reset(kAlpha); }
|
||||
|
||||
private:
|
||||
const float kAlpha = 0.95f;
|
||||
int64_t last_sample_ms_;
|
||||
rtc::ExpFilter smoother_;
|
||||
};
|
||||
|
||||
LibvpxVp8Decoder::LibvpxVp8Decoder()
|
||||
: LibvpxVp8Decoder(FieldTrialBasedConfig()) {}
|
||||
|
||||
LibvpxVp8Decoder::LibvpxVp8Decoder(const Environment& env)
|
||||
: LibvpxVp8Decoder(env.field_trials()) {}
|
||||
|
||||
LibvpxVp8Decoder::LibvpxVp8Decoder(const FieldTrialsView& field_trials)
|
||||
: use_postproc_(kIsArm ? field_trials.IsEnabled(kVp8PostProcArmFieldTrial)
|
||||
: true),
|
||||
buffer_pool_(false, 300 /* max_number_of_buffers*/),
|
||||
decode_complete_callback_(NULL),
|
||||
inited_(false),
|
||||
decoder_(NULL),
|
||||
last_frame_width_(0),
|
||||
last_frame_height_(0),
|
||||
key_frame_required_(true),
|
||||
deblock_params_(use_postproc_
|
||||
? GetPostProcParamsFromFieldTrialGroup(field_trials)
|
||||
: absl::nullopt),
|
||||
qp_smoother_(use_postproc_ ? new QpSmoother() : nullptr) {}
|
||||
|
||||
LibvpxVp8Decoder::~LibvpxVp8Decoder() {
|
||||
inited_ = true; // in order to do the actual release
|
||||
Release();
|
||||
}
|
||||
|
||||
bool LibvpxVp8Decoder::Configure(const Settings& settings) {
|
||||
if (Release() < 0) {
|
||||
return false;
|
||||
}
|
||||
if (decoder_ == NULL) {
|
||||
decoder_ = new vpx_codec_ctx_t;
|
||||
memset(decoder_, 0, sizeof(*decoder_));
|
||||
}
|
||||
vpx_codec_dec_cfg_t cfg;
|
||||
// Setting number of threads to a constant value (1)
|
||||
cfg.threads = 1;
|
||||
cfg.h = cfg.w = 0; // set after decode
|
||||
|
||||
vpx_codec_flags_t flags = use_postproc_ ? VPX_CODEC_USE_POSTPROC : 0;
|
||||
|
||||
if (vpx_codec_dec_init(decoder_, vpx_codec_vp8_dx(), &cfg, flags)) {
|
||||
delete decoder_;
|
||||
decoder_ = nullptr;
|
||||
return false;
|
||||
}
|
||||
|
||||
inited_ = true;
|
||||
|
||||
// Always start with a complete key frame.
|
||||
key_frame_required_ = true;
|
||||
if (absl::optional<int> buffer_pool_size = settings.buffer_pool_size()) {
|
||||
if (!buffer_pool_.Resize(*buffer_pool_size)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
int LibvpxVp8Decoder::Decode(const EncodedImage& input_image,
|
||||
int64_t render_time_ms) {
|
||||
return Decode(input_image, /*missing_frames=*/false, render_time_ms);
|
||||
}
|
||||
|
||||
int LibvpxVp8Decoder::Decode(const EncodedImage& input_image,
|
||||
bool /*missing_frames*/,
|
||||
int64_t /*render_time_ms*/) {
|
||||
if (!inited_) {
|
||||
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
|
||||
}
|
||||
if (decode_complete_callback_ == NULL) {
|
||||
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
|
||||
}
|
||||
if (input_image.data() == NULL && input_image.size() > 0) {
|
||||
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
|
||||
}
|
||||
|
||||
// Post process configurations.
|
||||
if (use_postproc_) {
|
||||
vp8_postproc_cfg_t ppcfg;
|
||||
// MFQE enabled to reduce key frame popping.
|
||||
ppcfg.post_proc_flag = VP8_MFQE;
|
||||
|
||||
if (kIsArm) {
|
||||
RTC_DCHECK(deblock_params_.has_value());
|
||||
}
|
||||
if (deblock_params_.has_value()) {
|
||||
// For low resolutions, use stronger deblocking filter.
|
||||
int last_width_x_height = last_frame_width_ * last_frame_height_;
|
||||
if (last_width_x_height > 0 && last_width_x_height <= 320 * 240) {
|
||||
// Enable the deblock and demacroblocker based on qp thresholds.
|
||||
RTC_DCHECK(qp_smoother_);
|
||||
int qp = qp_smoother_->GetAvg();
|
||||
if (qp > deblock_params_->min_qp) {
|
||||
int level = deblock_params_->max_level;
|
||||
if (qp < deblock_params_->degrade_qp) {
|
||||
// Use lower level.
|
||||
level = deblock_params_->max_level *
|
||||
(qp - deblock_params_->min_qp) /
|
||||
(deblock_params_->degrade_qp - deblock_params_->min_qp);
|
||||
}
|
||||
// Deblocking level only affects VP8_DEMACROBLOCK.
|
||||
ppcfg.deblocking_level = std::max(level, 1);
|
||||
ppcfg.post_proc_flag |= VP8_DEBLOCK | VP8_DEMACROBLOCK;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Non-arm with no explicit deblock params set.
|
||||
ppcfg.post_proc_flag |= VP8_DEBLOCK;
|
||||
// For VGA resolutions and lower, enable the demacroblocker postproc.
|
||||
if (last_frame_width_ * last_frame_height_ <= 640 * 360) {
|
||||
ppcfg.post_proc_flag |= VP8_DEMACROBLOCK;
|
||||
}
|
||||
// Strength of deblocking filter. Valid range:[0,16]
|
||||
ppcfg.deblocking_level = 3;
|
||||
}
|
||||
|
||||
vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg);
|
||||
}
|
||||
|
||||
// Always start with a complete key frame.
|
||||
if (key_frame_required_) {
|
||||
if (input_image._frameType != VideoFrameType::kVideoFrameKey)
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
key_frame_required_ = false;
|
||||
}
|
||||
|
||||
const uint8_t* buffer = input_image.data();
|
||||
if (input_image.size() == 0) {
|
||||
buffer = NULL; // Triggers full frame concealment.
|
||||
}
|
||||
if (vpx_codec_decode(decoder_, buffer, input_image.size(), 0,
|
||||
kDecodeDeadlineRealtime)) {
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
|
||||
vpx_codec_iter_t iter = NULL;
|
||||
vpx_image_t* img = vpx_codec_get_frame(decoder_, &iter);
|
||||
int qp;
|
||||
vpx_codec_err_t vpx_ret =
|
||||
vpx_codec_control(decoder_, VPXD_GET_LAST_QUANTIZER, &qp);
|
||||
RTC_DCHECK_EQ(vpx_ret, VPX_CODEC_OK);
|
||||
int ret = ReturnFrame(img, input_image.RtpTimestamp(), qp,
|
||||
input_image.ColorSpace());
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
int LibvpxVp8Decoder::ReturnFrame(
|
||||
const vpx_image_t* img,
|
||||
uint32_t timestamp,
|
||||
int qp,
|
||||
const webrtc::ColorSpace* explicit_color_space) {
|
||||
if (img == NULL) {
|
||||
// Decoder OK and NULL image => No show frame
|
||||
return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
|
||||
}
|
||||
if (qp_smoother_) {
|
||||
if (last_frame_width_ != static_cast<int>(img->d_w) ||
|
||||
last_frame_height_ != static_cast<int>(img->d_h)) {
|
||||
qp_smoother_->Reset();
|
||||
}
|
||||
qp_smoother_->Add(qp);
|
||||
}
|
||||
last_frame_width_ = img->d_w;
|
||||
last_frame_height_ = img->d_h;
|
||||
// Allocate memory for decoded image.
|
||||
rtc::scoped_refptr<VideoFrameBuffer> buffer;
|
||||
|
||||
rtc::scoped_refptr<I420Buffer> i420_buffer =
|
||||
buffer_pool_.CreateI420Buffer(img->d_w, img->d_h);
|
||||
buffer = i420_buffer;
|
||||
if (i420_buffer.get()) {
|
||||
libyuv::I420Copy(img->planes[VPX_PLANE_Y], img->stride[VPX_PLANE_Y],
|
||||
img->planes[VPX_PLANE_U], img->stride[VPX_PLANE_U],
|
||||
img->planes[VPX_PLANE_V], img->stride[VPX_PLANE_V],
|
||||
i420_buffer->MutableDataY(), i420_buffer->StrideY(),
|
||||
i420_buffer->MutableDataU(), i420_buffer->StrideU(),
|
||||
i420_buffer->MutableDataV(), i420_buffer->StrideV(),
|
||||
img->d_w, img->d_h);
|
||||
}
|
||||
|
||||
if (!buffer.get()) {
|
||||
// Pool has too many pending frames.
|
||||
RTC_HISTOGRAM_BOOLEAN("WebRTC.Video.LibvpxVp8Decoder.TooManyPendingFrames",
|
||||
1);
|
||||
return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
|
||||
}
|
||||
|
||||
VideoFrame decoded_image = VideoFrame::Builder()
|
||||
.set_video_frame_buffer(buffer)
|
||||
.set_timestamp_rtp(timestamp)
|
||||
.set_color_space(explicit_color_space)
|
||||
.build();
|
||||
decode_complete_callback_->Decoded(decoded_image, absl::nullopt, qp);
|
||||
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
int LibvpxVp8Decoder::RegisterDecodeCompleteCallback(
|
||||
DecodedImageCallback* callback) {
|
||||
decode_complete_callback_ = callback;
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
int LibvpxVp8Decoder::Release() {
|
||||
int ret_val = WEBRTC_VIDEO_CODEC_OK;
|
||||
|
||||
if (decoder_ != NULL) {
|
||||
if (inited_) {
|
||||
if (vpx_codec_destroy(decoder_)) {
|
||||
ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
|
||||
}
|
||||
}
|
||||
delete decoder_;
|
||||
decoder_ = NULL;
|
||||
}
|
||||
buffer_pool_.Release();
|
||||
inited_ = false;
|
||||
return ret_val;
|
||||
}
|
||||
|
||||
VideoDecoder::DecoderInfo LibvpxVp8Decoder::GetDecoderInfo() const {
|
||||
DecoderInfo info;
|
||||
info.implementation_name = "libvpx";
|
||||
info.is_hardware_accelerated = false;
|
||||
return info;
|
||||
}
|
||||
|
||||
const char* LibvpxVp8Decoder::ImplementationName() const {
|
||||
return "libvpx";
|
||||
}
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
/*
|
||||
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_DECODER_H_
|
||||
#define MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_DECODER_H_
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "api/environment/environment.h"
|
||||
#include "api/field_trials_view.h"
|
||||
#include "api/video/encoded_image.h"
|
||||
#include "api/video_codecs/video_decoder.h"
|
||||
#include "common_video/include/video_frame_buffer_pool.h"
|
||||
#include "modules/video_coding/codecs/vp8/include/vp8.h"
|
||||
#include "modules/video_coding/include/video_codec_interface.h"
|
||||
#include <vpx/vp8dx.h>
|
||||
#include <vpx/vpx_decoder.h>
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
class LibvpxVp8Decoder : public VideoDecoder {
|
||||
public:
|
||||
// TODO: bugs.webrtc.org/15791 - Delete default constructor when
|
||||
// Environment is always propagated.
|
||||
LibvpxVp8Decoder();
|
||||
explicit LibvpxVp8Decoder(const Environment& env);
|
||||
~LibvpxVp8Decoder() override;
|
||||
|
||||
bool Configure(const Settings& settings) override;
|
||||
int Decode(const EncodedImage& input_image,
|
||||
int64_t /*render_time_ms*/) override;
|
||||
|
||||
// TODO(bugs.webrtc.org/15444): Remove once all subclasses have been migrated
|
||||
// to expecting calls Decode without a missing_frames param.
|
||||
int Decode(const EncodedImage& input_image,
|
||||
bool missing_frames,
|
||||
int64_t /*render_time_ms*/) override;
|
||||
|
||||
int RegisterDecodeCompleteCallback(DecodedImageCallback* callback) override;
|
||||
int Release() override;
|
||||
|
||||
DecoderInfo GetDecoderInfo() const override;
|
||||
const char* ImplementationName() const override;
|
||||
|
||||
struct DeblockParams {
|
||||
DeblockParams() : max_level(6), degrade_qp(1), min_qp(0) {}
|
||||
DeblockParams(int max_level, int degrade_qp, int min_qp)
|
||||
: max_level(max_level), degrade_qp(degrade_qp), min_qp(min_qp) {}
|
||||
int max_level; // Deblocking strength: [0, 16].
|
||||
int degrade_qp; // If QP value is below, start lowering `max_level`.
|
||||
int min_qp; // If QP value is below, turn off deblocking.
|
||||
};
|
||||
|
||||
private:
|
||||
class QpSmoother;
|
||||
explicit LibvpxVp8Decoder(const FieldTrialsView& field_trials);
|
||||
int ReturnFrame(const vpx_image_t* img,
|
||||
uint32_t timeStamp,
|
||||
int qp,
|
||||
const webrtc::ColorSpace* explicit_color_space);
|
||||
const bool use_postproc_;
|
||||
|
||||
VideoFrameBufferPool buffer_pool_;
|
||||
DecodedImageCallback* decode_complete_callback_;
|
||||
bool inited_;
|
||||
vpx_codec_ctx_t* decoder_;
|
||||
int last_frame_width_;
|
||||
int last_frame_height_;
|
||||
bool key_frame_required_;
|
||||
const absl::optional<DeblockParams> deblock_params_;
|
||||
const std::unique_ptr<QpSmoother> qp_smoother_;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_DECODER_H_
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,164 @@
|
|||
/*
|
||||
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_ENCODER_H_
|
||||
#define MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_ENCODER_H_
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "api/fec_controller_override.h"
|
||||
#include "api/units/time_delta.h"
|
||||
#include "api/units/timestamp.h"
|
||||
#include "api/video/encoded_image.h"
|
||||
#include "api/video/video_frame.h"
|
||||
#include "api/video_codecs/video_encoder.h"
|
||||
#include "api/video_codecs/vp8_frame_buffer_controller.h"
|
||||
#include "api/video_codecs/vp8_frame_config.h"
|
||||
#include "modules/video_coding/codecs/interface/libvpx_interface.h"
|
||||
#include "modules/video_coding/codecs/vp8/include/vp8.h"
|
||||
#include "modules/video_coding/include/video_codec_interface.h"
|
||||
#include "modules/video_coding/utility/framerate_controller_deprecated.h"
|
||||
#include "modules/video_coding/utility/vp8_constants.h"
|
||||
#include "rtc_base/experiments/cpu_speed_experiment.h"
|
||||
#include "rtc_base/experiments/encoder_info_settings.h"
|
||||
#include "rtc_base/experiments/rate_control_settings.h"
|
||||
#include <vpx/vp8cx.h>
|
||||
#include <vpx/vpx_encoder.h>
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
class LibvpxVp8Encoder : public VideoEncoder {
|
||||
public:
|
||||
LibvpxVp8Encoder(std::unique_ptr<LibvpxInterface> interface,
|
||||
VP8Encoder::Settings settings);
|
||||
~LibvpxVp8Encoder() override;
|
||||
|
||||
int Release() override;
|
||||
|
||||
void SetFecControllerOverride(
|
||||
FecControllerOverride* fec_controller_override) override;
|
||||
|
||||
int InitEncode(const VideoCodec* codec_settings,
|
||||
const VideoEncoder::Settings& settings) override;
|
||||
|
||||
int Encode(const VideoFrame& input_image,
|
||||
const std::vector<VideoFrameType>* frame_types) override;
|
||||
|
||||
int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override;
|
||||
|
||||
void SetRates(const RateControlParameters& parameters) override;
|
||||
|
||||
void OnPacketLossRateUpdate(float packet_loss_rate) override;
|
||||
|
||||
void OnRttUpdate(int64_t rtt_ms) override;
|
||||
|
||||
void OnLossNotification(const LossNotification& loss_notification) override;
|
||||
|
||||
EncoderInfo GetEncoderInfo() const override;
|
||||
|
||||
static vpx_enc_frame_flags_t EncodeFlags(const Vp8FrameConfig& references);
|
||||
|
||||
private:
|
||||
// Get the cpu_speed setting for encoder based on resolution and/or platform.
|
||||
int GetCpuSpeed(int width, int height);
|
||||
|
||||
// Determine number of encoder threads to use.
|
||||
int NumberOfThreads(int width, int height, int number_of_cores);
|
||||
|
||||
// Call encoder initialize function and set control settings.
|
||||
int InitAndSetControlSettings();
|
||||
|
||||
void PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
|
||||
const vpx_codec_cx_pkt& pkt,
|
||||
int stream_idx,
|
||||
int encoder_idx,
|
||||
uint32_t timestamp);
|
||||
|
||||
int GetEncodedPartitions(const VideoFrame& input_image,
|
||||
bool retransmission_allowed);
|
||||
|
||||
// Set the stream state for stream `stream_idx`.
|
||||
void SetStreamState(bool send_stream, int stream_idx);
|
||||
|
||||
uint32_t MaxIntraTarget(uint32_t optimal_buffer_size);
|
||||
|
||||
uint32_t FrameDropThreshold(size_t spatial_idx) const;
|
||||
|
||||
size_t SteadyStateSize(int sid, int tid);
|
||||
|
||||
bool UpdateVpxConfiguration(size_t stream_index);
|
||||
|
||||
void MaybeUpdatePixelFormat(vpx_img_fmt fmt);
|
||||
// Prepares `raw_image_` to reference image data of `buffer`, or of mapped or
|
||||
// scaled versions of `buffer`. Returns a list of buffers that got referenced
|
||||
// as a result, allowing the caller to keep references to them until after
|
||||
// encoding has finished. On failure to convert the buffer, an empty list is
|
||||
// returned.
|
||||
std::vector<rtc::scoped_refptr<VideoFrameBuffer>> PrepareBuffers(
|
||||
rtc::scoped_refptr<VideoFrameBuffer> buffer);
|
||||
|
||||
const std::unique_ptr<LibvpxInterface> libvpx_;
|
||||
|
||||
const CpuSpeedExperiment experimental_cpu_speed_config_arm_;
|
||||
const RateControlSettings rate_control_settings_;
|
||||
|
||||
EncodedImageCallback* encoded_complete_callback_ = nullptr;
|
||||
VideoCodec codec_;
|
||||
bool inited_ = false;
|
||||
int64_t timestamp_ = 0;
|
||||
int qp_max_ = 56;
|
||||
int cpu_speed_default_ = -6;
|
||||
int number_of_cores_ = 0;
|
||||
uint32_t rc_max_intra_target_ = 0;
|
||||
int num_active_streams_ = 0;
|
||||
const std::unique_ptr<Vp8FrameBufferControllerFactory>
|
||||
frame_buffer_controller_factory_;
|
||||
std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_;
|
||||
const std::vector<VideoEncoder::ResolutionBitrateLimits>
|
||||
resolution_bitrate_limits_;
|
||||
std::vector<bool> key_frame_request_;
|
||||
std::vector<bool> send_stream_;
|
||||
std::vector<int> cpu_speed_;
|
||||
std::vector<vpx_image_t> raw_images_;
|
||||
std::vector<EncodedImage> encoded_images_;
|
||||
std::vector<vpx_codec_ctx_t> encoders_;
|
||||
std::vector<vpx_codec_enc_cfg_t> vpx_configs_;
|
||||
std::vector<Vp8EncoderConfig> config_overrides_;
|
||||
std::vector<vpx_rational_t> downsampling_factors_;
|
||||
std::vector<Timestamp> last_encoder_output_time_;
|
||||
|
||||
// Variable frame-rate screencast related fields and methods.
|
||||
const struct VariableFramerateExperiment {
|
||||
bool enabled = false;
|
||||
// Framerate is limited to this value in steady state.
|
||||
float framerate_limit = 5.0;
|
||||
// This qp or below is considered a steady state.
|
||||
int steady_state_qp = kVp8SteadyStateQpThreshold;
|
||||
// Frames of at least this percentage below ideal for configured bitrate are
|
||||
// considered in a steady state.
|
||||
int steady_state_undershoot_percentage = 30;
|
||||
} variable_framerate_experiment_;
|
||||
static VariableFramerateExperiment ParseVariableFramerateConfig(
|
||||
std::string group_name);
|
||||
FramerateControllerDeprecated framerate_controller_;
|
||||
int num_steady_state_frames_ = 0;
|
||||
|
||||
FecControllerOverride* fec_controller_override_ = nullptr;
|
||||
|
||||
const LibvpxVp8EncoderInfoSettings encoder_info_override_;
|
||||
|
||||
absl::optional<TimeDelta> max_frame_drop_interval_;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_ENCODER_H_
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
/*
|
||||
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "api/test/create_simulcast_test_fixture.h"
|
||||
#include "api/test/simulcast_test_fixture.h"
|
||||
#include "api/test/video/function_video_decoder_factory.h"
|
||||
#include "api/test/video/function_video_encoder_factory.h"
|
||||
#include "modules/video_coding/codecs/vp8/include/vp8.h"
|
||||
#include "test/gtest.h"
|
||||
|
||||
namespace webrtc {
|
||||
namespace test {
|
||||
|
||||
namespace {
|
||||
std::unique_ptr<SimulcastTestFixture> CreateSpecificSimulcastTestFixture() {
|
||||
std::unique_ptr<VideoEncoderFactory> encoder_factory =
|
||||
std::make_unique<FunctionVideoEncoderFactory>(
|
||||
[]() { return VP8Encoder::Create(); });
|
||||
std::unique_ptr<VideoDecoderFactory> decoder_factory =
|
||||
std::make_unique<FunctionVideoDecoderFactory>(
|
||||
[](const Environment& env, const SdpVideoFormat& format) {
|
||||
return CreateVp8Decoder(env);
|
||||
});
|
||||
return CreateSimulcastTestFixture(std::move(encoder_factory),
|
||||
std::move(decoder_factory),
|
||||
SdpVideoFormat("VP8"));
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST(LibvpxVp8SimulcastTest, TestKeyFrameRequestsOnAllStreams) {
|
||||
auto fixture = CreateSpecificSimulcastTestFixture();
|
||||
fixture->TestKeyFrameRequestsOnAllStreams();
|
||||
}
|
||||
|
||||
TEST(LibvpxVp8SimulcastTest, TestKeyFrameRequestsOnSpecificStreams) {
|
||||
GTEST_SKIP() << "Not applicable to VP8.";
|
||||
}
|
||||
|
||||
TEST(LibvpxVp8SimulcastTest, TestPaddingAllStreams) {
|
||||
auto fixture = CreateSpecificSimulcastTestFixture();
|
||||
fixture->TestPaddingAllStreams();
|
||||
}
|
||||
|
||||
TEST(LibvpxVp8SimulcastTest, TestPaddingTwoStreams) {
|
||||
auto fixture = CreateSpecificSimulcastTestFixture();
|
||||
fixture->TestPaddingTwoStreams();
|
||||
}
|
||||
|
||||
TEST(LibvpxVp8SimulcastTest, TestPaddingTwoStreamsOneMaxedOut) {
|
||||
auto fixture = CreateSpecificSimulcastTestFixture();
|
||||
fixture->TestPaddingTwoStreamsOneMaxedOut();
|
||||
}
|
||||
|
||||
TEST(LibvpxVp8SimulcastTest, TestPaddingOneStream) {
|
||||
auto fixture = CreateSpecificSimulcastTestFixture();
|
||||
fixture->TestPaddingOneStream();
|
||||
}
|
||||
|
||||
TEST(LibvpxVp8SimulcastTest, TestPaddingOneStreamTwoMaxedOut) {
|
||||
auto fixture = CreateSpecificSimulcastTestFixture();
|
||||
fixture->TestPaddingOneStreamTwoMaxedOut();
|
||||
}
|
||||
|
||||
TEST(LibvpxVp8SimulcastTest, TestSendAllStreams) {
|
||||
auto fixture = CreateSpecificSimulcastTestFixture();
|
||||
fixture->TestSendAllStreams();
|
||||
}
|
||||
|
||||
TEST(LibvpxVp8SimulcastTest, TestDisablingStreams) {
|
||||
auto fixture = CreateSpecificSimulcastTestFixture();
|
||||
fixture->TestDisablingStreams();
|
||||
}
|
||||
|
||||
TEST(LibvpxVp8SimulcastTest, TestActiveStreams) {
|
||||
auto fixture = CreateSpecificSimulcastTestFixture();
|
||||
fixture->TestActiveStreams();
|
||||
}
|
||||
|
||||
TEST(LibvpxVp8SimulcastTest, TestSwitchingToOneStream) {
|
||||
auto fixture = CreateSpecificSimulcastTestFixture();
|
||||
fixture->TestSwitchingToOneStream();
|
||||
}
|
||||
|
||||
TEST(LibvpxVp8SimulcastTest, TestSwitchingToOneOddStream) {
|
||||
auto fixture = CreateSpecificSimulcastTestFixture();
|
||||
fixture->TestSwitchingToOneOddStream();
|
||||
}
|
||||
|
||||
TEST(LibvpxVp8SimulcastTest, TestSwitchingToOneSmallStream) {
|
||||
auto fixture = CreateSpecificSimulcastTestFixture();
|
||||
fixture->TestSwitchingToOneSmallStream();
|
||||
}
|
||||
|
||||
TEST(LibvpxVp8SimulcastTest, TestSpatioTemporalLayers333PatternEncoder) {
|
||||
auto fixture = CreateSpecificSimulcastTestFixture();
|
||||
fixture->TestSpatioTemporalLayers333PatternEncoder();
|
||||
}
|
||||
|
||||
TEST(LibvpxVp8SimulcastTest, TestStrideEncodeDecode) {
|
||||
auto fixture = CreateSpecificSimulcastTestFixture();
|
||||
fixture->TestStrideEncodeDecode();
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,624 @@
|
|||
/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/video_coding/codecs/vp8/screenshare_layers.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <memory>
|
||||
|
||||
#include "modules/video_coding/include/video_codec_interface.h"
|
||||
#include "rtc_base/arraysize.h"
|
||||
#include "rtc_base/checks.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "rtc_base/time_utils.h"
|
||||
#include "system_wrappers/include/metrics.h"
|
||||
|
||||
namespace webrtc {
|
||||
namespace {
|
||||
using BufferFlags = Vp8FrameConfig::BufferFlags;
|
||||
|
||||
constexpr BufferFlags kNone = Vp8FrameConfig::BufferFlags::kNone;
|
||||
constexpr BufferFlags kReference = Vp8FrameConfig::BufferFlags::kReference;
|
||||
constexpr BufferFlags kUpdate = Vp8FrameConfig::BufferFlags::kUpdate;
|
||||
constexpr BufferFlags kReferenceAndUpdate =
|
||||
Vp8FrameConfig::BufferFlags::kReferenceAndUpdate;
|
||||
|
||||
constexpr int kOneSecond90Khz = 90000;
|
||||
constexpr int kMinTimeBetweenSyncs = kOneSecond90Khz * 2;
|
||||
constexpr int kMaxTimeBetweenSyncs = kOneSecond90Khz * 4;
|
||||
constexpr int kQpDeltaThresholdForSync = 8;
|
||||
constexpr int kMinBitrateKbpsForQpBoost = 500;
|
||||
constexpr auto kSwitch = DecodeTargetIndication::kSwitch;
|
||||
} // namespace
|
||||
|
||||
const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5;
|
||||
const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0;
|
||||
|
||||
constexpr int ScreenshareLayers::kMaxNumTemporalLayers;
|
||||
|
||||
// Always emit a frame with certain interval, even if bitrate targets have
|
||||
// been exceeded. This prevents needless keyframe requests.
|
||||
const int ScreenshareLayers::kMaxFrameIntervalMs = 2750;
|
||||
|
||||
ScreenshareLayers::ScreenshareLayers(int num_temporal_layers)
|
||||
: number_of_temporal_layers_(
|
||||
std::min(kMaxNumTemporalLayers, num_temporal_layers)),
|
||||
active_layer_(-1),
|
||||
last_timestamp_(-1),
|
||||
last_sync_timestamp_(-1),
|
||||
last_emitted_tl0_timestamp_(-1),
|
||||
last_frame_time_ms_(-1),
|
||||
max_debt_bytes_(0),
|
||||
encode_framerate_(1000.0f, 1000.0f), // 1 second window, second scale.
|
||||
bitrate_updated_(false),
|
||||
checker_(TemporalLayersChecker::CreateTemporalLayersChecker(
|
||||
Vp8TemporalLayersType::kBitrateDynamic,
|
||||
num_temporal_layers)) {
|
||||
RTC_CHECK_GT(number_of_temporal_layers_, 0);
|
||||
RTC_CHECK_LE(number_of_temporal_layers_, kMaxNumTemporalLayers);
|
||||
}
|
||||
|
||||
ScreenshareLayers::~ScreenshareLayers() {
|
||||
UpdateHistograms();
|
||||
}
|
||||
|
||||
void ScreenshareLayers::SetQpLimits(size_t stream_index,
|
||||
int min_qp,
|
||||
int max_qp) {
|
||||
RTC_DCHECK_LT(stream_index, StreamCount());
|
||||
// 0 < min_qp <= max_qp
|
||||
RTC_DCHECK_LT(0, min_qp);
|
||||
RTC_DCHECK_LE(min_qp, max_qp);
|
||||
|
||||
RTC_DCHECK_EQ(min_qp_.has_value(), max_qp_.has_value());
|
||||
if (!min_qp_.has_value()) {
|
||||
min_qp_ = min_qp;
|
||||
max_qp_ = max_qp;
|
||||
} else {
|
||||
RTC_DCHECK_EQ(min_qp, min_qp_.value());
|
||||
RTC_DCHECK_EQ(max_qp, max_qp_.value());
|
||||
}
|
||||
}
|
||||
|
||||
size_t ScreenshareLayers::StreamCount() const {
|
||||
return 1;
|
||||
}
|
||||
|
||||
bool ScreenshareLayers::SupportsEncoderFrameDropping(
|
||||
size_t stream_index) const {
|
||||
RTC_DCHECK_LT(stream_index, StreamCount());
|
||||
// Frame dropping is handled internally by this class.
|
||||
return false;
|
||||
}
|
||||
|
||||
Vp8FrameConfig ScreenshareLayers::NextFrameConfig(size_t stream_index,
|
||||
uint32_t timestamp) {
|
||||
RTC_DCHECK_LT(stream_index, StreamCount());
|
||||
|
||||
auto it = pending_frame_configs_.find(timestamp);
|
||||
if (it != pending_frame_configs_.end()) {
|
||||
// Drop and re-encode, reuse the previous config.
|
||||
return it->second.frame_config;
|
||||
}
|
||||
|
||||
if (number_of_temporal_layers_ <= 1) {
|
||||
// No flags needed for 1 layer screenshare.
|
||||
// TODO(pbos): Consider updating only last, and not all buffers.
|
||||
DependencyInfo dependency_info{
|
||||
"S", {kReferenceAndUpdate, kReferenceAndUpdate, kReferenceAndUpdate}};
|
||||
pending_frame_configs_[timestamp] = dependency_info;
|
||||
return dependency_info.frame_config;
|
||||
}
|
||||
|
||||
const int64_t now_ms = rtc::TimeMillis();
|
||||
|
||||
int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
|
||||
int64_t ts_diff;
|
||||
if (last_timestamp_ == -1) {
|
||||
ts_diff = kOneSecond90Khz / capture_framerate_.value_or(*target_framerate_);
|
||||
} else {
|
||||
ts_diff = unwrapped_timestamp - last_timestamp_;
|
||||
}
|
||||
|
||||
if (target_framerate_) {
|
||||
// If input frame rate exceeds target frame rate, either over a one second
|
||||
// averaging window, or if frame interval is below 90% of desired value,
|
||||
// drop frame.
|
||||
if (encode_framerate_.Rate(now_ms).value_or(0) > *target_framerate_)
|
||||
return Vp8FrameConfig(kNone, kNone, kNone);
|
||||
|
||||
// Primarily check if frame interval is too short using frame timestamps,
|
||||
// as if they are correct they won't be affected by queuing in webrtc.
|
||||
const int64_t expected_frame_interval_90khz =
|
||||
kOneSecond90Khz / *target_framerate_;
|
||||
if (last_timestamp_ != -1 && ts_diff > 0) {
|
||||
if (ts_diff < 85 * expected_frame_interval_90khz / 100) {
|
||||
return Vp8FrameConfig(kNone, kNone, kNone);
|
||||
}
|
||||
} else {
|
||||
// Timestamps looks off, use realtime clock here instead.
|
||||
const int64_t expected_frame_interval_ms = 1000 / *target_framerate_;
|
||||
if (last_frame_time_ms_ != -1 &&
|
||||
now_ms - last_frame_time_ms_ <
|
||||
(85 * expected_frame_interval_ms) / 100) {
|
||||
return Vp8FrameConfig(kNone, kNone, kNone);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (stats_.first_frame_time_ms_ == -1)
|
||||
stats_.first_frame_time_ms_ = now_ms;
|
||||
|
||||
// Make sure both frame droppers leak out bits.
|
||||
layers_[0].UpdateDebt(ts_diff / 90);
|
||||
layers_[1].UpdateDebt(ts_diff / 90);
|
||||
last_timestamp_ = timestamp;
|
||||
last_frame_time_ms_ = now_ms;
|
||||
|
||||
TemporalLayerState layer_state = TemporalLayerState::kDrop;
|
||||
|
||||
if (active_layer_ == -1 ||
|
||||
layers_[active_layer_].state != TemporalLayer::State::kDropped) {
|
||||
if (last_emitted_tl0_timestamp_ != -1 &&
|
||||
(unwrapped_timestamp - last_emitted_tl0_timestamp_) / 90 >
|
||||
kMaxFrameIntervalMs) {
|
||||
// Too long time has passed since the last frame was emitted, cancel
|
||||
// enough debt to allow a single frame.
|
||||
layers_[0].debt_bytes_ = max_debt_bytes_ - 1;
|
||||
}
|
||||
if (layers_[0].debt_bytes_ > max_debt_bytes_) {
|
||||
// Must drop TL0, encode TL1 instead.
|
||||
if (layers_[1].debt_bytes_ > max_debt_bytes_) {
|
||||
// Must drop both TL0 and TL1.
|
||||
active_layer_ = -1;
|
||||
} else {
|
||||
active_layer_ = 1;
|
||||
}
|
||||
} else {
|
||||
active_layer_ = 0;
|
||||
}
|
||||
}
|
||||
|
||||
switch (active_layer_) {
|
||||
case 0:
|
||||
layer_state = TemporalLayerState::kTl0;
|
||||
last_emitted_tl0_timestamp_ = unwrapped_timestamp;
|
||||
break;
|
||||
case 1:
|
||||
if (layers_[1].state != TemporalLayer::State::kDropped) {
|
||||
if (TimeToSync(unwrapped_timestamp) ||
|
||||
layers_[1].state == TemporalLayer::State::kKeyFrame) {
|
||||
last_sync_timestamp_ = unwrapped_timestamp;
|
||||
layer_state = TemporalLayerState::kTl1Sync;
|
||||
} else {
|
||||
layer_state = TemporalLayerState::kTl1;
|
||||
}
|
||||
} else {
|
||||
layer_state = last_sync_timestamp_ == unwrapped_timestamp
|
||||
? TemporalLayerState::kTl1Sync
|
||||
: TemporalLayerState::kTl1;
|
||||
}
|
||||
break;
|
||||
case -1:
|
||||
layer_state = TemporalLayerState::kDrop;
|
||||
++stats_.num_dropped_frames_;
|
||||
break;
|
||||
default:
|
||||
RTC_DCHECK_NOTREACHED();
|
||||
}
|
||||
|
||||
DependencyInfo dependency_info;
|
||||
// TODO(pbos): Consider referencing but not updating the 'alt' buffer for all
|
||||
// layers.
|
||||
switch (layer_state) {
|
||||
case TemporalLayerState::kDrop:
|
||||
dependency_info = {"", {kNone, kNone, kNone}};
|
||||
break;
|
||||
case TemporalLayerState::kTl0:
|
||||
// TL0 only references and updates 'last'.
|
||||
dependency_info = {"SS", {kReferenceAndUpdate, kNone, kNone}};
|
||||
dependency_info.frame_config.packetizer_temporal_idx = 0;
|
||||
break;
|
||||
case TemporalLayerState::kTl1:
|
||||
// TL1 references both 'last' and 'golden' but only updates 'golden'.
|
||||
dependency_info = {"-R", {kReference, kReferenceAndUpdate, kNone}};
|
||||
dependency_info.frame_config.packetizer_temporal_idx = 1;
|
||||
break;
|
||||
case TemporalLayerState::kTl1Sync:
|
||||
// Predict from only TL0 to allow participants to switch to the high
|
||||
// bitrate stream. Updates 'golden' so that TL1 can continue to refer to
|
||||
// and update 'golden' from this point on.
|
||||
dependency_info = {"-S", {kReference, kUpdate, kNone}};
|
||||
dependency_info.frame_config.packetizer_temporal_idx = 1;
|
||||
dependency_info.frame_config.layer_sync = true;
|
||||
break;
|
||||
}
|
||||
|
||||
pending_frame_configs_[timestamp] = dependency_info;
|
||||
return dependency_info.frame_config;
|
||||
}
|
||||
|
||||
void ScreenshareLayers::OnRatesUpdated(
|
||||
size_t stream_index,
|
||||
const std::vector<uint32_t>& bitrates_bps,
|
||||
int framerate_fps) {
|
||||
RTC_DCHECK_LT(stream_index, StreamCount());
|
||||
RTC_DCHECK_GT(framerate_fps, 0);
|
||||
RTC_DCHECK_GE(bitrates_bps.size(), 1);
|
||||
RTC_DCHECK_LE(bitrates_bps.size(), 2);
|
||||
|
||||
// `bitrates_bps` uses individual rates per layer, but we want to use the
|
||||
// accumulated rate here.
|
||||
uint32_t tl0_kbps = bitrates_bps[0] / 1000;
|
||||
uint32_t tl1_kbps = tl0_kbps;
|
||||
if (bitrates_bps.size() > 1) {
|
||||
tl1_kbps += bitrates_bps[1] / 1000;
|
||||
}
|
||||
|
||||
if (!target_framerate_) {
|
||||
// First OnRatesUpdated() is called during construction, with the
|
||||
// configured targets as parameters.
|
||||
target_framerate_ = framerate_fps;
|
||||
capture_framerate_ = target_framerate_;
|
||||
bitrate_updated_ = true;
|
||||
} else {
|
||||
if ((capture_framerate_ &&
|
||||
framerate_fps != static_cast<int>(*capture_framerate_)) ||
|
||||
(tl0_kbps != layers_[0].target_rate_kbps_) ||
|
||||
(tl1_kbps != layers_[1].target_rate_kbps_)) {
|
||||
bitrate_updated_ = true;
|
||||
}
|
||||
|
||||
if (framerate_fps < 0) {
|
||||
capture_framerate_.reset();
|
||||
} else {
|
||||
capture_framerate_ = framerate_fps;
|
||||
}
|
||||
}
|
||||
|
||||
layers_[0].target_rate_kbps_ = tl0_kbps;
|
||||
layers_[1].target_rate_kbps_ = tl1_kbps;
|
||||
}
|
||||
|
||||
void ScreenshareLayers::OnEncodeDone(size_t stream_index,
|
||||
uint32_t rtp_timestamp,
|
||||
size_t size_bytes,
|
||||
bool is_keyframe,
|
||||
int qp,
|
||||
CodecSpecificInfo* info) {
|
||||
RTC_DCHECK_LT(stream_index, StreamCount());
|
||||
|
||||
if (size_bytes == 0) {
|
||||
RTC_LOG(LS_WARNING) << "Empty frame; treating as dropped.";
|
||||
OnFrameDropped(stream_index, rtp_timestamp);
|
||||
return;
|
||||
}
|
||||
|
||||
absl::optional<DependencyInfo> dependency_info;
|
||||
auto it = pending_frame_configs_.find(rtp_timestamp);
|
||||
if (it != pending_frame_configs_.end()) {
|
||||
dependency_info = it->second;
|
||||
pending_frame_configs_.erase(it);
|
||||
|
||||
if (checker_) {
|
||||
RTC_DCHECK(checker_->CheckTemporalConfig(is_keyframe,
|
||||
dependency_info->frame_config));
|
||||
}
|
||||
}
|
||||
|
||||
CodecSpecificInfoVP8& vp8_info = info->codecSpecific.VP8;
|
||||
GenericFrameInfo& generic_frame_info = info->generic_frame_info.emplace();
|
||||
|
||||
if (number_of_temporal_layers_ == 1) {
|
||||
vp8_info.temporalIdx = kNoTemporalIdx;
|
||||
vp8_info.layerSync = false;
|
||||
generic_frame_info.temporal_id = 0;
|
||||
generic_frame_info.decode_target_indications = {kSwitch};
|
||||
generic_frame_info.encoder_buffers.emplace_back(
|
||||
0, /*referenced=*/!is_keyframe, /*updated=*/true);
|
||||
} else {
|
||||
int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(rtp_timestamp);
|
||||
if (dependency_info) {
|
||||
vp8_info.temporalIdx =
|
||||
dependency_info->frame_config.packetizer_temporal_idx;
|
||||
vp8_info.layerSync = dependency_info->frame_config.layer_sync;
|
||||
generic_frame_info.temporal_id = vp8_info.temporalIdx;
|
||||
generic_frame_info.decode_target_indications =
|
||||
dependency_info->decode_target_indications;
|
||||
} else {
|
||||
RTC_DCHECK(is_keyframe);
|
||||
}
|
||||
|
||||
if (is_keyframe) {
|
||||
vp8_info.temporalIdx = 0;
|
||||
last_sync_timestamp_ = unwrapped_timestamp;
|
||||
vp8_info.layerSync = true;
|
||||
layers_[0].state = TemporalLayer::State::kKeyFrame;
|
||||
layers_[1].state = TemporalLayer::State::kKeyFrame;
|
||||
active_layer_ = 1;
|
||||
info->template_structure =
|
||||
GetTemplateStructure(number_of_temporal_layers_);
|
||||
generic_frame_info.temporal_id = vp8_info.temporalIdx;
|
||||
generic_frame_info.decode_target_indications = {kSwitch, kSwitch};
|
||||
} else if (active_layer_ >= 0 && layers_[active_layer_].state ==
|
||||
TemporalLayer::State::kKeyFrame) {
|
||||
layers_[active_layer_].state = TemporalLayer::State::kNormal;
|
||||
}
|
||||
|
||||
vp8_info.useExplicitDependencies = true;
|
||||
RTC_DCHECK_EQ(vp8_info.referencedBuffersCount, 0u);
|
||||
RTC_DCHECK_EQ(vp8_info.updatedBuffersCount, 0u);
|
||||
|
||||
// Note that `frame_config` is not derefernced if `is_keyframe`,
|
||||
// meaning it's never dereferenced if the optional may be unset.
|
||||
for (int i = 0; i < static_cast<int>(Vp8FrameConfig::Buffer::kCount); ++i) {
|
||||
bool references = false;
|
||||
bool updates = is_keyframe;
|
||||
if (!is_keyframe && dependency_info->frame_config.References(
|
||||
static_cast<Vp8FrameConfig::Buffer>(i))) {
|
||||
RTC_DCHECK_LT(vp8_info.referencedBuffersCount,
|
||||
arraysize(CodecSpecificInfoVP8::referencedBuffers));
|
||||
references = true;
|
||||
vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i;
|
||||
}
|
||||
|
||||
if (is_keyframe || dependency_info->frame_config.Updates(
|
||||
static_cast<Vp8FrameConfig::Buffer>(i))) {
|
||||
RTC_DCHECK_LT(vp8_info.updatedBuffersCount,
|
||||
arraysize(CodecSpecificInfoVP8::updatedBuffers));
|
||||
updates = true;
|
||||
vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i;
|
||||
}
|
||||
|
||||
if (references || updates)
|
||||
generic_frame_info.encoder_buffers.emplace_back(i, references, updates);
|
||||
}
|
||||
}
|
||||
|
||||
encode_framerate_.Update(1, rtc::TimeMillis());
|
||||
|
||||
if (number_of_temporal_layers_ == 1)
|
||||
return;
|
||||
|
||||
RTC_DCHECK_NE(-1, active_layer_);
|
||||
if (layers_[active_layer_].state == TemporalLayer::State::kDropped) {
|
||||
layers_[active_layer_].state = TemporalLayer::State::kQualityBoost;
|
||||
}
|
||||
|
||||
if (qp != -1)
|
||||
layers_[active_layer_].last_qp = qp;
|
||||
|
||||
if (active_layer_ == 0) {
|
||||
layers_[0].debt_bytes_ += size_bytes;
|
||||
layers_[1].debt_bytes_ += size_bytes;
|
||||
++stats_.num_tl0_frames_;
|
||||
stats_.tl0_target_bitrate_sum_ += layers_[0].target_rate_kbps_;
|
||||
stats_.tl0_qp_sum_ += qp;
|
||||
} else if (active_layer_ == 1) {
|
||||
layers_[1].debt_bytes_ += size_bytes;
|
||||
++stats_.num_tl1_frames_;
|
||||
stats_.tl1_target_bitrate_sum_ += layers_[1].target_rate_kbps_;
|
||||
stats_.tl1_qp_sum_ += qp;
|
||||
}
|
||||
}
|
||||
|
||||
void ScreenshareLayers::OnFrameDropped(size_t stream_index,
|
||||
uint32_t rtp_timestamp) {
|
||||
layers_[active_layer_].state = TemporalLayer::State::kDropped;
|
||||
++stats_.num_overshoots_;
|
||||
}
|
||||
|
||||
void ScreenshareLayers::OnPacketLossRateUpdate(float packet_loss_rate) {}
|
||||
|
||||
void ScreenshareLayers::OnRttUpdate(int64_t rtt_ms) {}
|
||||
|
||||
void ScreenshareLayers::OnLossNotification(
|
||||
const VideoEncoder::LossNotification& loss_notification) {}
|
||||
|
||||
FrameDependencyStructure ScreenshareLayers::GetTemplateStructure(
|
||||
int num_layers) const {
|
||||
RTC_CHECK_LT(num_layers, 3);
|
||||
RTC_CHECK_GT(num_layers, 0);
|
||||
|
||||
FrameDependencyStructure template_structure;
|
||||
template_structure.num_decode_targets = num_layers;
|
||||
|
||||
switch (num_layers) {
|
||||
case 1: {
|
||||
template_structure.templates.resize(2);
|
||||
template_structure.templates[0].T(0).Dtis("S");
|
||||
template_structure.templates[1].T(0).Dtis("S").FrameDiffs({1});
|
||||
return template_structure;
|
||||
}
|
||||
case 2: {
|
||||
template_structure.templates.resize(3);
|
||||
template_structure.templates[0].T(0).Dtis("SS");
|
||||
template_structure.templates[1].T(0).Dtis("SS").FrameDiffs({1});
|
||||
template_structure.templates[2].T(1).Dtis("-S").FrameDiffs({1});
|
||||
return template_structure;
|
||||
}
|
||||
default:
|
||||
RTC_DCHECK_NOTREACHED();
|
||||
// To make the compiler happy!
|
||||
return template_structure;
|
||||
}
|
||||
}
|
||||
|
||||
bool ScreenshareLayers::TimeToSync(int64_t timestamp) const {
|
||||
RTC_DCHECK_EQ(1, active_layer_);
|
||||
RTC_DCHECK_NE(-1, layers_[0].last_qp);
|
||||
if (layers_[1].last_qp == -1) {
|
||||
// First frame in TL1 should only depend on TL0 since there are no
|
||||
// previous frames in TL1.
|
||||
return true;
|
||||
}
|
||||
|
||||
RTC_DCHECK_NE(-1, last_sync_timestamp_);
|
||||
int64_t timestamp_diff = timestamp - last_sync_timestamp_;
|
||||
if (timestamp_diff > kMaxTimeBetweenSyncs) {
|
||||
// After a certain time, force a sync frame.
|
||||
return true;
|
||||
} else if (timestamp_diff < kMinTimeBetweenSyncs) {
|
||||
// If too soon from previous sync frame, don't issue a new one.
|
||||
return false;
|
||||
}
|
||||
// Issue a sync frame if difference in quality between TL0 and TL1 isn't too
|
||||
// large.
|
||||
if (layers_[0].last_qp - layers_[1].last_qp < kQpDeltaThresholdForSync)
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t ScreenshareLayers::GetCodecTargetBitrateKbps() const {
|
||||
uint32_t target_bitrate_kbps = layers_[0].target_rate_kbps_;
|
||||
|
||||
if (number_of_temporal_layers_ > 1) {
|
||||
// Calculate a codec target bitrate. This may be higher than TL0, gaining
|
||||
// quality at the expense of frame rate at TL0. Constraints:
|
||||
// - TL0 frame rate no less than framerate / kMaxTL0FpsReduction.
|
||||
// - Target rate * kAcceptableTargetOvershoot should not exceed TL1 rate.
|
||||
target_bitrate_kbps =
|
||||
std::min(layers_[0].target_rate_kbps_ * kMaxTL0FpsReduction,
|
||||
layers_[1].target_rate_kbps_ / kAcceptableTargetOvershoot);
|
||||
}
|
||||
|
||||
return std::max(layers_[0].target_rate_kbps_, target_bitrate_kbps);
|
||||
}
|
||||
|
||||
Vp8EncoderConfig ScreenshareLayers::UpdateConfiguration(size_t stream_index) {
|
||||
RTC_DCHECK_LT(stream_index, StreamCount());
|
||||
RTC_DCHECK(min_qp_.has_value());
|
||||
RTC_DCHECK(max_qp_.has_value());
|
||||
|
||||
const uint32_t target_bitrate_kbps = GetCodecTargetBitrateKbps();
|
||||
|
||||
// TODO(sprang): We _really_ need to make an overhaul of this class. :(
|
||||
// If we're dropping frames in order to meet a target framerate, adjust the
|
||||
// bitrate assigned to the encoder so the total average bitrate is correct.
|
||||
float encoder_config_bitrate_kbps = target_bitrate_kbps;
|
||||
if (target_framerate_ && capture_framerate_ &&
|
||||
*target_framerate_ < *capture_framerate_) {
|
||||
encoder_config_bitrate_kbps *=
|
||||
static_cast<float>(*capture_framerate_) / *target_framerate_;
|
||||
}
|
||||
|
||||
if (bitrate_updated_ ||
|
||||
encoder_config_.rc_target_bitrate !=
|
||||
absl::make_optional(encoder_config_bitrate_kbps)) {
|
||||
encoder_config_.rc_target_bitrate = encoder_config_bitrate_kbps;
|
||||
|
||||
// Don't reconfigure qp limits during quality boost frames.
|
||||
if (active_layer_ == -1 ||
|
||||
layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) {
|
||||
const int min_qp = min_qp_.value();
|
||||
const int max_qp = max_qp_.value();
|
||||
|
||||
// After a dropped frame, a frame with max qp will be encoded and the
|
||||
// quality will then ramp up from there. To boost the speed of recovery,
|
||||
// encode the next frame with lower max qp, if there is sufficient
|
||||
// bandwidth to do so without causing excessive delay.
|
||||
// TL0 is the most important to improve since the errors in this layer
|
||||
// will propagate to TL1.
|
||||
// Currently, reduce max qp by 20% for TL0 and 15% for TL1.
|
||||
if (layers_[1].target_rate_kbps_ >= kMinBitrateKbpsForQpBoost) {
|
||||
layers_[0].enhanced_max_qp = min_qp + (((max_qp - min_qp) * 80) / 100);
|
||||
layers_[1].enhanced_max_qp = min_qp + (((max_qp - min_qp) * 85) / 100);
|
||||
} else {
|
||||
layers_[0].enhanced_max_qp = -1;
|
||||
layers_[1].enhanced_max_qp = -1;
|
||||
}
|
||||
}
|
||||
|
||||
if (capture_framerate_) {
|
||||
int avg_frame_size =
|
||||
(target_bitrate_kbps * 1000) / (8 * *capture_framerate_);
|
||||
// Allow max debt to be the size of a single optimal frame.
|
||||
// TODO(sprang): Determine if this needs to be adjusted by some factor.
|
||||
// (Lower values may cause more frame drops, higher may lead to queuing
|
||||
// delays.)
|
||||
max_debt_bytes_ = avg_frame_size;
|
||||
}
|
||||
|
||||
bitrate_updated_ = false;
|
||||
}
|
||||
|
||||
// Don't try to update boosts state if not active yet.
|
||||
if (active_layer_ == -1)
|
||||
return encoder_config_;
|
||||
|
||||
if (number_of_temporal_layers_ <= 1)
|
||||
return encoder_config_;
|
||||
|
||||
// If layer is in the quality boost state (following a dropped frame), update
|
||||
// the configuration with the adjusted (lower) qp and set the state back to
|
||||
// normal.
|
||||
unsigned int adjusted_max_qp = max_qp_.value(); // Set the normal max qp.
|
||||
if (layers_[active_layer_].state == TemporalLayer::State::kQualityBoost) {
|
||||
if (layers_[active_layer_].enhanced_max_qp != -1) {
|
||||
// Bitrate is high enough for quality boost, update max qp.
|
||||
adjusted_max_qp = layers_[active_layer_].enhanced_max_qp;
|
||||
}
|
||||
// Regardless of qp, reset the boost state for the next frame.
|
||||
layers_[active_layer_].state = TemporalLayer::State::kNormal;
|
||||
}
|
||||
encoder_config_.rc_max_quantizer = adjusted_max_qp;
|
||||
|
||||
return encoder_config_;
|
||||
}
|
||||
|
||||
void ScreenshareLayers::TemporalLayer::UpdateDebt(int64_t delta_ms) {
|
||||
uint32_t debt_reduction_bytes = target_rate_kbps_ * delta_ms / 8;
|
||||
if (debt_reduction_bytes >= debt_bytes_) {
|
||||
debt_bytes_ = 0;
|
||||
} else {
|
||||
debt_bytes_ -= debt_reduction_bytes;
|
||||
}
|
||||
}
|
||||
|
||||
void ScreenshareLayers::UpdateHistograms() {
|
||||
if (stats_.first_frame_time_ms_ == -1)
|
||||
return;
|
||||
int64_t duration_sec =
|
||||
(rtc::TimeMillis() - stats_.first_frame_time_ms_ + 500) / 1000;
|
||||
if (duration_sec >= metrics::kMinRunTimeInSeconds) {
|
||||
RTC_HISTOGRAM_COUNTS_10000(
|
||||
"WebRTC.Video.Screenshare.Layer0.FrameRate",
|
||||
(stats_.num_tl0_frames_ + (duration_sec / 2)) / duration_sec);
|
||||
RTC_HISTOGRAM_COUNTS_10000(
|
||||
"WebRTC.Video.Screenshare.Layer1.FrameRate",
|
||||
(stats_.num_tl1_frames_ + (duration_sec / 2)) / duration_sec);
|
||||
int total_frames = stats_.num_tl0_frames_ + stats_.num_tl1_frames_;
|
||||
RTC_HISTOGRAM_COUNTS_10000(
|
||||
"WebRTC.Video.Screenshare.FramesPerDrop",
|
||||
(stats_.num_dropped_frames_ == 0
|
||||
? 0
|
||||
: total_frames / stats_.num_dropped_frames_));
|
||||
RTC_HISTOGRAM_COUNTS_10000(
|
||||
"WebRTC.Video.Screenshare.FramesPerOvershoot",
|
||||
(stats_.num_overshoots_ == 0 ? 0
|
||||
: total_frames / stats_.num_overshoots_));
|
||||
if (stats_.num_tl0_frames_ > 0) {
|
||||
RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.Layer0.Qp",
|
||||
stats_.tl0_qp_sum_ / stats_.num_tl0_frames_);
|
||||
RTC_HISTOGRAM_COUNTS_10000(
|
||||
"WebRTC.Video.Screenshare.Layer0.TargetBitrate",
|
||||
stats_.tl0_target_bitrate_sum_ / stats_.num_tl0_frames_);
|
||||
}
|
||||
if (stats_.num_tl1_frames_ > 0) {
|
||||
RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.Layer1.Qp",
|
||||
stats_.tl1_qp_sum_ / stats_.num_tl1_frames_);
|
||||
RTC_HISTOGRAM_COUNTS_10000(
|
||||
"WebRTC.Video.Screenshare.Layer1.TargetBitrate",
|
||||
stats_.tl1_target_bitrate_sum_ / stats_.num_tl1_frames_);
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,164 @@
|
|||
/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_
|
||||
#define MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "api/video_codecs/vp8_frame_config.h"
|
||||
#include "api/video_codecs/vp8_temporal_layers.h"
|
||||
#include "modules/video_coding/codecs/vp8/include/temporal_layers_checker.h"
|
||||
#include "modules/video_coding/include/video_codec_interface.h"
|
||||
#include "modules/video_coding/utility/frame_dropper.h"
|
||||
#include "rtc_base/numerics/sequence_number_unwrapper.h"
|
||||
#include "rtc_base/rate_statistics.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
struct CodecSpecificInfoVP8;
|
||||
class Clock;
|
||||
|
||||
class ScreenshareLayers final : public Vp8FrameBufferController {
|
||||
public:
|
||||
static const double kMaxTL0FpsReduction;
|
||||
static const double kAcceptableTargetOvershoot;
|
||||
static const int kMaxFrameIntervalMs;
|
||||
|
||||
explicit ScreenshareLayers(int num_temporal_layers);
|
||||
~ScreenshareLayers() override;
|
||||
|
||||
void SetQpLimits(size_t stream_index, int min_qp, int max_qp) override;
|
||||
|
||||
size_t StreamCount() const override;
|
||||
|
||||
bool SupportsEncoderFrameDropping(size_t stream_index) const override;
|
||||
|
||||
// Returns the recommended VP8 encode flags needed. May refresh the decoder
|
||||
// and/or update the reference buffers.
|
||||
Vp8FrameConfig NextFrameConfig(size_t stream_index,
|
||||
uint32_t rtp_timestamp) override;
|
||||
|
||||
// New target bitrate, per temporal layer.
|
||||
void OnRatesUpdated(size_t stream_index,
|
||||
const std::vector<uint32_t>& bitrates_bps,
|
||||
int framerate_fps) override;
|
||||
|
||||
Vp8EncoderConfig UpdateConfiguration(size_t stream_index) override;
|
||||
|
||||
void OnEncodeDone(size_t stream_index,
|
||||
uint32_t rtp_timestamp,
|
||||
size_t size_bytes,
|
||||
bool is_keyframe,
|
||||
int qp,
|
||||
CodecSpecificInfo* info) override;
|
||||
|
||||
void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) override;
|
||||
|
||||
void OnPacketLossRateUpdate(float packet_loss_rate) override;
|
||||
|
||||
void OnRttUpdate(int64_t rtt_ms) override;
|
||||
|
||||
void OnLossNotification(
|
||||
const VideoEncoder::LossNotification& loss_notification) override;
|
||||
|
||||
private:
|
||||
enum class TemporalLayerState : int { kDrop, kTl0, kTl1, kTl1Sync };
|
||||
|
||||
struct DependencyInfo {
|
||||
DependencyInfo() = default;
|
||||
DependencyInfo(absl::string_view indication_symbols,
|
||||
Vp8FrameConfig frame_config)
|
||||
: decode_target_indications(
|
||||
webrtc_impl::StringToDecodeTargetIndications(indication_symbols)),
|
||||
frame_config(frame_config) {}
|
||||
|
||||
absl::InlinedVector<DecodeTargetIndication, 10> decode_target_indications;
|
||||
Vp8FrameConfig frame_config;
|
||||
};
|
||||
|
||||
bool TimeToSync(int64_t timestamp) const;
|
||||
uint32_t GetCodecTargetBitrateKbps() const;
|
||||
|
||||
const int number_of_temporal_layers_;
|
||||
|
||||
// TODO(eladalon/sprang): These should be made into const-int set in the ctor.
|
||||
absl::optional<int> min_qp_;
|
||||
absl::optional<int> max_qp_;
|
||||
|
||||
int active_layer_;
|
||||
int64_t last_timestamp_;
|
||||
int64_t last_sync_timestamp_;
|
||||
int64_t last_emitted_tl0_timestamp_;
|
||||
int64_t last_frame_time_ms_;
|
||||
RtpTimestampUnwrapper time_wrap_handler_;
|
||||
uint32_t max_debt_bytes_;
|
||||
|
||||
std::map<uint32_t, DependencyInfo> pending_frame_configs_;
|
||||
|
||||
// Configured max framerate.
|
||||
absl::optional<uint32_t> target_framerate_;
|
||||
// Incoming framerate from capturer.
|
||||
absl::optional<uint32_t> capture_framerate_;
|
||||
|
||||
// Tracks what framerate we actually encode, and drops frames on overshoot.
|
||||
RateStatistics encode_framerate_;
|
||||
bool bitrate_updated_;
|
||||
|
||||
static constexpr int kMaxNumTemporalLayers = 2;
|
||||
struct TemporalLayer {
|
||||
TemporalLayer()
|
||||
: state(State::kNormal),
|
||||
enhanced_max_qp(-1),
|
||||
last_qp(-1),
|
||||
debt_bytes_(0),
|
||||
target_rate_kbps_(0) {}
|
||||
|
||||
enum class State {
|
||||
kNormal,
|
||||
kDropped,
|
||||
kReencoded,
|
||||
kQualityBoost,
|
||||
kKeyFrame
|
||||
} state;
|
||||
|
||||
int enhanced_max_qp;
|
||||
int last_qp;
|
||||
uint32_t debt_bytes_;
|
||||
uint32_t target_rate_kbps_;
|
||||
|
||||
void UpdateDebt(int64_t delta_ms);
|
||||
} layers_[kMaxNumTemporalLayers];
|
||||
|
||||
void UpdateHistograms();
|
||||
FrameDependencyStructure GetTemplateStructure(int num_layers) const;
|
||||
|
||||
// Data for histogram statistics.
|
||||
struct Stats {
|
||||
int64_t first_frame_time_ms_ = -1;
|
||||
int64_t num_tl0_frames_ = 0;
|
||||
int64_t num_tl1_frames_ = 0;
|
||||
int64_t num_dropped_frames_ = 0;
|
||||
int64_t num_overshoots_ = 0;
|
||||
int64_t tl0_qp_sum_ = 0;
|
||||
int64_t tl1_qp_sum_ = 0;
|
||||
int64_t tl0_target_bitrate_sum_ = 0;
|
||||
int64_t tl1_target_bitrate_sum_ = 0;
|
||||
} stats_;
|
||||
|
||||
Vp8EncoderConfig encoder_config_;
|
||||
|
||||
// Optional utility used to verify reference validity.
|
||||
std::unique_ptr<TemporalLayersChecker> checker_;
|
||||
};
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_
|
||||
|
|
@ -0,0 +1,788 @@
|
|||
/*
|
||||
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/video_coding/codecs/vp8/screenshare_layers.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "api/video_codecs/vp8_frame_config.h"
|
||||
#include "modules/video_coding/codecs/interface/common_constants.h"
|
||||
#include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h"
|
||||
#include "modules/video_coding/include/video_codec_interface.h"
|
||||
#include "rtc_base/checks.h"
|
||||
#include "rtc_base/fake_clock.h"
|
||||
#include "system_wrappers/include/metrics.h"
|
||||
#include "test/gmock.h"
|
||||
#include "test/gtest.h"
|
||||
#include "vpx/vp8cx.h"
|
||||
|
||||
using ::testing::_;
|
||||
using ::testing::ElementsAre;
|
||||
using ::testing::NiceMock;
|
||||
|
||||
namespace webrtc {
|
||||
namespace {
|
||||
// 5 frames per second at 90 kHz.
|
||||
const uint32_t kTimestampDelta5Fps = 90000 / 5;
|
||||
const int kDefaultQp = 54;
|
||||
const int kDefaultTl0BitrateKbps = 200;
|
||||
const int kDefaultTl1BitrateKbps = 2000;
|
||||
const int kFrameRate = 5;
|
||||
const int kSyncPeriodSeconds = 2;
|
||||
const int kMaxSyncPeriodSeconds = 4;
|
||||
|
||||
// Expected flags for corresponding temporal layers.
|
||||
const int kTl0Flags = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
|
||||
VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF;
|
||||
const int kTl1Flags =
|
||||
VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST;
|
||||
const int kTl1SyncFlags = VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF |
|
||||
VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST;
|
||||
const std::vector<uint32_t> kDefault2TlBitratesBps = {
|
||||
kDefaultTl0BitrateKbps * 1000,
|
||||
(kDefaultTl1BitrateKbps - kDefaultTl0BitrateKbps) * 1000};
|
||||
|
||||
} // namespace
|
||||
|
||||
class ScreenshareLayerTest : public ::testing::Test {
|
||||
protected:
|
||||
ScreenshareLayerTest()
|
||||
: min_qp_(2),
|
||||
max_qp_(kDefaultQp),
|
||||
frame_size_(-1),
|
||||
timestamp_(90),
|
||||
config_updated_(false) {}
|
||||
virtual ~ScreenshareLayerTest() {}
|
||||
|
||||
void SetUp() override {
|
||||
layers_.reset(new ScreenshareLayers(2));
|
||||
cfg_ = ConfigureBitrates();
|
||||
}
|
||||
|
||||
int EncodeFrame(bool base_sync, CodecSpecificInfo* info = nullptr) {
|
||||
CodecSpecificInfo ignored_info;
|
||||
if (!info) {
|
||||
info = &ignored_info;
|
||||
}
|
||||
|
||||
int flags = ConfigureFrame(base_sync);
|
||||
if (flags != -1)
|
||||
layers_->OnEncodeDone(0, timestamp_, frame_size_, base_sync, kDefaultQp,
|
||||
info);
|
||||
return flags;
|
||||
}
|
||||
|
||||
int ConfigureFrame(bool key_frame) {
|
||||
tl_config_ = NextFrameConfig(0, timestamp_);
|
||||
EXPECT_EQ(0, tl_config_.encoder_layer_id)
|
||||
<< "ScreenshareLayers always encodes using the bitrate allocator for "
|
||||
"layer 0, but may reference different buffers and packetize "
|
||||
"differently.";
|
||||
if (tl_config_.drop_frame) {
|
||||
return -1;
|
||||
}
|
||||
const uint32_t prev_rc_target_bitrate = cfg_.rc_target_bitrate.value_or(-1);
|
||||
const uint32_t prev_rc_max_quantizer = cfg_.rc_max_quantizer.value_or(-1);
|
||||
|
||||
cfg_ = layers_->UpdateConfiguration(0);
|
||||
|
||||
config_updated_ =
|
||||
cfg_.temporal_layer_config.has_value() ||
|
||||
(cfg_.rc_target_bitrate.has_value() &&
|
||||
cfg_.rc_target_bitrate.value() != prev_rc_target_bitrate) ||
|
||||
(cfg_.rc_max_quantizer.has_value() &&
|
||||
cfg_.rc_max_quantizer.value() != prev_rc_max_quantizer) ||
|
||||
cfg_.g_error_resilient.has_value();
|
||||
|
||||
int flags = LibvpxVp8Encoder::EncodeFlags(tl_config_);
|
||||
EXPECT_NE(-1, frame_size_);
|
||||
return flags;
|
||||
}
|
||||
|
||||
Vp8FrameConfig NextFrameConfig(size_t stream_index, uint32_t timestamp) {
|
||||
int64_t timestamp_ms = timestamp / 90;
|
||||
clock_.AdvanceTime(TimeDelta::Millis(timestamp_ms - rtc::TimeMillis()));
|
||||
return layers_->NextFrameConfig(stream_index, timestamp);
|
||||
}
|
||||
|
||||
int FrameSizeForBitrate(int bitrate_kbps) {
|
||||
return ((bitrate_kbps * 1000) / 8) / kFrameRate;
|
||||
}
|
||||
|
||||
Vp8EncoderConfig ConfigureBitrates() {
|
||||
layers_->SetQpLimits(0, min_qp_, max_qp_);
|
||||
layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, kFrameRate);
|
||||
const Vp8EncoderConfig vp8_cfg = layers_->UpdateConfiguration(0);
|
||||
EXPECT_TRUE(vp8_cfg.rc_target_bitrate.has_value());
|
||||
frame_size_ = FrameSizeForBitrate(vp8_cfg.rc_target_bitrate.value());
|
||||
return vp8_cfg;
|
||||
}
|
||||
|
||||
void WithQpLimits(int min_qp, int max_qp) {
|
||||
min_qp_ = min_qp;
|
||||
max_qp_ = max_qp;
|
||||
}
|
||||
|
||||
// Runs a few initial frames and makes sure we have seen frames on both
|
||||
// temporal layers, including sync and non-sync frames.
|
||||
bool RunGracePeriod() {
|
||||
bool got_tl0 = false;
|
||||
bool got_tl1 = false;
|
||||
bool got_tl1_sync = false;
|
||||
for (int i = 0; i < 10; ++i) {
|
||||
CodecSpecificInfo info;
|
||||
EXPECT_NE(-1, EncodeFrame(false, &info));
|
||||
timestamp_ += kTimestampDelta5Fps;
|
||||
if (info.codecSpecific.VP8.temporalIdx == 0) {
|
||||
got_tl0 = true;
|
||||
} else if (info.codecSpecific.VP8.layerSync) {
|
||||
got_tl1_sync = true;
|
||||
} else {
|
||||
got_tl1 = true;
|
||||
}
|
||||
if (got_tl0 && got_tl1 && got_tl1_sync)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Adds frames until we get one in the specified temporal layer. The last
|
||||
// FrameEncoded() call will be omitted and needs to be done by the caller.
|
||||
// Returns the flags for the last frame.
|
||||
int SkipUntilTl(int layer) {
|
||||
return SkipUntilTlAndSync(layer, absl::nullopt);
|
||||
}
|
||||
|
||||
// Same as SkipUntilTl, but also waits until the sync bit condition is met.
|
||||
int SkipUntilTlAndSync(int layer, absl::optional<bool> sync) {
|
||||
int flags = 0;
|
||||
const int kMaxFramesToSkip =
|
||||
1 + (sync.value_or(false) ? kMaxSyncPeriodSeconds : 1) * kFrameRate;
|
||||
for (int i = 0; i < kMaxFramesToSkip; ++i) {
|
||||
flags = ConfigureFrame(false);
|
||||
if (tl_config_.packetizer_temporal_idx != layer ||
|
||||
(sync && *sync != tl_config_.layer_sync)) {
|
||||
if (flags != -1) {
|
||||
// If flags do not request a frame drop, report some default values
|
||||
// for frame size etc.
|
||||
CodecSpecificInfo info;
|
||||
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
|
||||
&info);
|
||||
}
|
||||
timestamp_ += kTimestampDelta5Fps;
|
||||
} else {
|
||||
// Found frame from sought after layer.
|
||||
return flags;
|
||||
}
|
||||
}
|
||||
ADD_FAILURE() << "Did not get a frame of TL" << layer << " in time.";
|
||||
return -1;
|
||||
}
|
||||
|
||||
int min_qp_;
|
||||
uint32_t max_qp_;
|
||||
int frame_size_;
|
||||
rtc::ScopedFakeClock clock_;
|
||||
std::unique_ptr<ScreenshareLayers> layers_;
|
||||
|
||||
uint32_t timestamp_;
|
||||
Vp8FrameConfig tl_config_;
|
||||
Vp8EncoderConfig cfg_;
|
||||
bool config_updated_;
|
||||
|
||||
CodecSpecificInfo* IgnoredCodecSpecificInfo() {
|
||||
ignored_codec_specific_info_ = std::make_unique<CodecSpecificInfo>();
|
||||
return ignored_codec_specific_info_.get();
|
||||
}
|
||||
|
||||
private:
|
||||
std::unique_ptr<CodecSpecificInfo> ignored_codec_specific_info_;
|
||||
};
|
||||
|
||||
TEST_F(ScreenshareLayerTest, 1Layer) {
|
||||
layers_.reset(new ScreenshareLayers(1));
|
||||
ConfigureBitrates();
|
||||
// One layer screenshare should not use the frame dropper as all frames will
|
||||
// belong to the base layer.
|
||||
const int kSingleLayerFlags = 0;
|
||||
auto info = std::make_unique<CodecSpecificInfo>();
|
||||
int flags = EncodeFrame(/*base_sync=*/false, info.get());
|
||||
timestamp_ += kTimestampDelta5Fps;
|
||||
EXPECT_EQ(static_cast<uint8_t>(kNoTemporalIdx),
|
||||
info->codecSpecific.VP8.temporalIdx);
|
||||
EXPECT_FALSE(info->codecSpecific.VP8.layerSync);
|
||||
EXPECT_EQ(info->generic_frame_info->temporal_id, 0);
|
||||
|
||||
info = std::make_unique<CodecSpecificInfo>();
|
||||
flags = EncodeFrame(/*base_sync=*/false, info.get());
|
||||
EXPECT_EQ(kSingleLayerFlags, flags);
|
||||
EXPECT_EQ(static_cast<uint8_t>(kNoTemporalIdx),
|
||||
info->codecSpecific.VP8.temporalIdx);
|
||||
EXPECT_FALSE(info->codecSpecific.VP8.layerSync);
|
||||
EXPECT_EQ(info->generic_frame_info->temporal_id, 0);
|
||||
}
|
||||
|
||||
TEST_F(ScreenshareLayerTest, 2LayersPeriodicSync) {
|
||||
std::vector<int> sync_times;
|
||||
const int kNumFrames = kSyncPeriodSeconds * kFrameRate * 2 - 1;
|
||||
for (int i = 0; i < kNumFrames; ++i) {
|
||||
CodecSpecificInfo info;
|
||||
EncodeFrame(false, &info);
|
||||
timestamp_ += kTimestampDelta5Fps;
|
||||
if (info.codecSpecific.VP8.temporalIdx == 1 &&
|
||||
info.codecSpecific.VP8.layerSync) {
|
||||
sync_times.push_back(timestamp_);
|
||||
}
|
||||
}
|
||||
|
||||
ASSERT_EQ(2u, sync_times.size());
|
||||
EXPECT_GE(sync_times[1] - sync_times[0], 90000 * kSyncPeriodSeconds);
|
||||
}
|
||||
|
||||
TEST_F(ScreenshareLayerTest, 2LayersSyncAfterTimeout) {
|
||||
std::vector<int> sync_times;
|
||||
const int kNumFrames = kMaxSyncPeriodSeconds * kFrameRate * 2 - 1;
|
||||
for (int i = 0; i < kNumFrames; ++i) {
|
||||
CodecSpecificInfo info;
|
||||
|
||||
tl_config_ = NextFrameConfig(0, timestamp_);
|
||||
cfg_ = layers_->UpdateConfiguration(0);
|
||||
|
||||
// Simulate TL1 being at least 8 qp steps better.
|
||||
if (tl_config_.packetizer_temporal_idx == 0) {
|
||||
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
|
||||
&info);
|
||||
} else {
|
||||
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp - 8,
|
||||
&info);
|
||||
}
|
||||
|
||||
if (info.codecSpecific.VP8.temporalIdx == 1 &&
|
||||
info.codecSpecific.VP8.layerSync)
|
||||
sync_times.push_back(timestamp_);
|
||||
|
||||
timestamp_ += kTimestampDelta5Fps;
|
||||
}
|
||||
|
||||
ASSERT_EQ(2u, sync_times.size());
|
||||
EXPECT_GE(sync_times[1] - sync_times[0], 90000 * kMaxSyncPeriodSeconds);
|
||||
}
|
||||
|
||||
TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) {
|
||||
std::vector<int> sync_times;
|
||||
|
||||
const int kNumFrames = (kSyncPeriodSeconds +
|
||||
((kMaxSyncPeriodSeconds - kSyncPeriodSeconds) / 2)) *
|
||||
kFrameRate;
|
||||
for (int i = 0; i < kNumFrames; ++i) {
|
||||
CodecSpecificInfo info;
|
||||
|
||||
ConfigureFrame(false);
|
||||
|
||||
// Simulate TL1 being at least 8 qp steps better.
|
||||
if (tl_config_.packetizer_temporal_idx == 0) {
|
||||
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
|
||||
&info);
|
||||
} else {
|
||||
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp - 8,
|
||||
&info);
|
||||
}
|
||||
|
||||
if (info.codecSpecific.VP8.temporalIdx == 1 &&
|
||||
info.codecSpecific.VP8.layerSync)
|
||||
sync_times.push_back(timestamp_);
|
||||
|
||||
timestamp_ += kTimestampDelta5Fps;
|
||||
}
|
||||
|
||||
ASSERT_EQ(1u, sync_times.size());
|
||||
|
||||
bool bumped_tl0_quality = false;
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
CodecSpecificInfo info;
|
||||
|
||||
int flags = ConfigureFrame(false);
|
||||
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp - 8,
|
||||
&info);
|
||||
if (info.codecSpecific.VP8.temporalIdx == 0) {
|
||||
// Bump TL0 to same quality as TL1.
|
||||
bumped_tl0_quality = true;
|
||||
} else {
|
||||
if (bumped_tl0_quality) {
|
||||
EXPECT_TRUE(info.codecSpecific.VP8.layerSync);
|
||||
EXPECT_EQ(kTl1SyncFlags, flags);
|
||||
return;
|
||||
}
|
||||
}
|
||||
timestamp_ += kTimestampDelta5Fps;
|
||||
}
|
||||
ADD_FAILURE() << "No TL1 frame arrived within time limit.";
|
||||
}
|
||||
|
||||
TEST_F(ScreenshareLayerTest, 2LayersToggling) {
|
||||
EXPECT_TRUE(RunGracePeriod());
|
||||
|
||||
// Insert 50 frames. 2/5 should be TL0.
|
||||
int tl0_frames = 0;
|
||||
int tl1_frames = 0;
|
||||
for (int i = 0; i < 50; ++i) {
|
||||
CodecSpecificInfo info;
|
||||
EncodeFrame(/*base_sync=*/false, &info);
|
||||
EXPECT_EQ(info.codecSpecific.VP8.temporalIdx,
|
||||
info.generic_frame_info->temporal_id);
|
||||
timestamp_ += kTimestampDelta5Fps;
|
||||
switch (info.codecSpecific.VP8.temporalIdx) {
|
||||
case 0:
|
||||
++tl0_frames;
|
||||
break;
|
||||
case 1:
|
||||
++tl1_frames;
|
||||
break;
|
||||
default:
|
||||
abort();
|
||||
}
|
||||
}
|
||||
EXPECT_EQ(20, tl0_frames);
|
||||
EXPECT_EQ(30, tl1_frames);
|
||||
}
|
||||
|
||||
TEST_F(ScreenshareLayerTest, AllFitsLayer0) {
|
||||
frame_size_ = FrameSizeForBitrate(kDefaultTl0BitrateKbps);
|
||||
|
||||
// Insert 50 frames, small enough that all fits in TL0.
|
||||
for (int i = 0; i < 50; ++i) {
|
||||
CodecSpecificInfo info;
|
||||
int flags = EncodeFrame(false, &info);
|
||||
timestamp_ += kTimestampDelta5Fps;
|
||||
EXPECT_EQ(kTl0Flags, flags);
|
||||
EXPECT_EQ(0, info.codecSpecific.VP8.temporalIdx);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ScreenshareLayerTest, TooHighBitrate) {
|
||||
frame_size_ = 2 * FrameSizeForBitrate(kDefaultTl1BitrateKbps);
|
||||
|
||||
// Insert 100 frames. Half should be dropped.
|
||||
int tl0_frames = 0;
|
||||
int tl1_frames = 0;
|
||||
int dropped_frames = 0;
|
||||
for (int i = 0; i < 100; ++i) {
|
||||
CodecSpecificInfo info;
|
||||
int flags = EncodeFrame(false, &info);
|
||||
timestamp_ += kTimestampDelta5Fps;
|
||||
if (flags == -1) {
|
||||
++dropped_frames;
|
||||
} else {
|
||||
switch (info.codecSpecific.VP8.temporalIdx) {
|
||||
case 0:
|
||||
++tl0_frames;
|
||||
break;
|
||||
case 1:
|
||||
++tl1_frames;
|
||||
break;
|
||||
default:
|
||||
ADD_FAILURE() << "Unexpected temporal id";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_NEAR(50, tl0_frames + tl1_frames, 1);
|
||||
EXPECT_NEAR(50, dropped_frames, 1);
|
||||
}
|
||||
|
||||
TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL0) {
|
||||
const int kTl0_kbps = 100;
|
||||
const int kTl1_kbps = 1000;
|
||||
const std::vector<uint32_t> layer_rates = {kTl0_kbps * 1000,
|
||||
(kTl1_kbps - kTl0_kbps) * 1000};
|
||||
layers_->OnRatesUpdated(0, layer_rates, kFrameRate);
|
||||
cfg_ = layers_->UpdateConfiguration(0);
|
||||
|
||||
EXPECT_EQ(static_cast<unsigned int>(
|
||||
ScreenshareLayers::kMaxTL0FpsReduction * kTl0_kbps + 0.5),
|
||||
cfg_.rc_target_bitrate);
|
||||
}
|
||||
|
||||
TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL1) {
|
||||
const int kTl0_kbps = 100;
|
||||
const int kTl1_kbps = 450;
|
||||
const std::vector<uint32_t> layer_rates = {kTl0_kbps * 1000,
|
||||
(kTl1_kbps - kTl0_kbps) * 1000};
|
||||
layers_->OnRatesUpdated(0, layer_rates, kFrameRate);
|
||||
cfg_ = layers_->UpdateConfiguration(0);
|
||||
|
||||
EXPECT_EQ(static_cast<unsigned int>(
|
||||
kTl1_kbps / ScreenshareLayers::kAcceptableTargetOvershoot),
|
||||
cfg_.rc_target_bitrate);
|
||||
}
|
||||
|
||||
TEST_F(ScreenshareLayerTest, TargetBitrateBelowTL0) {
|
||||
const int kTl0_kbps = 100;
|
||||
const std::vector<uint32_t> layer_rates = {kTl0_kbps * 1000};
|
||||
layers_->OnRatesUpdated(0, layer_rates, kFrameRate);
|
||||
cfg_ = layers_->UpdateConfiguration(0);
|
||||
|
||||
EXPECT_EQ(static_cast<uint32_t>(kTl0_kbps), cfg_.rc_target_bitrate);
|
||||
}
|
||||
|
||||
TEST_F(ScreenshareLayerTest, EncoderDrop) {
|
||||
EXPECT_TRUE(RunGracePeriod());
|
||||
SkipUntilTl(0);
|
||||
|
||||
// Size 0 indicates dropped frame.
|
||||
layers_->OnEncodeDone(0, timestamp_, 0, false, 0, IgnoredCodecSpecificInfo());
|
||||
|
||||
// Re-encode frame (so don't advance timestamp).
|
||||
int flags = EncodeFrame(false);
|
||||
timestamp_ += kTimestampDelta5Fps;
|
||||
EXPECT_FALSE(config_updated_);
|
||||
EXPECT_EQ(kTl0Flags, flags);
|
||||
|
||||
// Next frame should have boosted quality...
|
||||
SkipUntilTl(0);
|
||||
EXPECT_TRUE(config_updated_);
|
||||
EXPECT_LT(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
|
||||
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
timestamp_ += kTimestampDelta5Fps;
|
||||
|
||||
// ...then back to standard setup.
|
||||
SkipUntilTl(0);
|
||||
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
timestamp_ += kTimestampDelta5Fps;
|
||||
EXPECT_EQ(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
|
||||
|
||||
// Next drop in TL1.
|
||||
SkipUntilTl(1);
|
||||
layers_->OnEncodeDone(0, timestamp_, 0, false, 0, IgnoredCodecSpecificInfo());
|
||||
|
||||
// Re-encode frame (so don't advance timestamp).
|
||||
flags = EncodeFrame(false);
|
||||
timestamp_ += kTimestampDelta5Fps;
|
||||
EXPECT_FALSE(config_updated_);
|
||||
EXPECT_EQ(kTl1Flags, flags);
|
||||
|
||||
// Next frame should have boosted QP.
|
||||
SkipUntilTl(1);
|
||||
EXPECT_TRUE(config_updated_);
|
||||
EXPECT_LT(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
|
||||
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
timestamp_ += kTimestampDelta5Fps;
|
||||
|
||||
// ...and back to normal.
|
||||
SkipUntilTl(1);
|
||||
EXPECT_EQ(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
|
||||
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
timestamp_ += kTimestampDelta5Fps;
|
||||
}
|
||||
|
||||
TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) {
|
||||
const int kLowBitrateKbps = 50;
|
||||
const int kLargeFrameSizeBytes = 100000;
|
||||
const uint32_t kStartTimestamp = 1234;
|
||||
|
||||
const std::vector<uint32_t> layer_rates = {kLowBitrateKbps * 1000};
|
||||
layers_->OnRatesUpdated(0, layer_rates, kFrameRate);
|
||||
cfg_ = layers_->UpdateConfiguration(0);
|
||||
|
||||
EXPECT_EQ(kTl0Flags,
|
||||
LibvpxVp8Encoder::EncodeFlags(NextFrameConfig(0, kStartTimestamp)));
|
||||
layers_->OnEncodeDone(0, kStartTimestamp, kLargeFrameSizeBytes, false,
|
||||
kDefaultQp, IgnoredCodecSpecificInfo());
|
||||
|
||||
const uint32_t kTwoSecondsLater =
|
||||
kStartTimestamp + (ScreenshareLayers::kMaxFrameIntervalMs * 90);
|
||||
|
||||
// Sanity check, repayment time should exceed kMaxFrameIntervalMs.
|
||||
ASSERT_GT(kStartTimestamp + 90 * (kLargeFrameSizeBytes * 8) / kLowBitrateKbps,
|
||||
kStartTimestamp + (ScreenshareLayers::kMaxFrameIntervalMs * 90));
|
||||
|
||||
// Expect drop one frame interval before the two second timeout. If we try
|
||||
// any later, the frame will be dropped anyway by the frame rate throttling
|
||||
// logic.
|
||||
EXPECT_TRUE(
|
||||
NextFrameConfig(0, kTwoSecondsLater - kTimestampDelta5Fps).drop_frame);
|
||||
|
||||
// More than two seconds has passed since last frame, one should be emitted
|
||||
// even if bitrate target is then exceeded.
|
||||
EXPECT_EQ(kTl0Flags, LibvpxVp8Encoder::EncodeFlags(
|
||||
NextFrameConfig(0, kTwoSecondsLater + 90)));
|
||||
}
|
||||
|
||||
TEST_F(ScreenshareLayerTest, UpdatesHistograms) {
|
||||
metrics::Reset();
|
||||
bool trigger_drop = false;
|
||||
bool dropped_frame = false;
|
||||
bool overshoot = false;
|
||||
const int kTl0Qp = 35;
|
||||
const int kTl1Qp = 30;
|
||||
for (int64_t timestamp = 0;
|
||||
timestamp < kTimestampDelta5Fps * 5 * metrics::kMinRunTimeInSeconds;
|
||||
timestamp += kTimestampDelta5Fps) {
|
||||
tl_config_ = NextFrameConfig(0, timestamp);
|
||||
if (tl_config_.drop_frame) {
|
||||
dropped_frame = true;
|
||||
continue;
|
||||
}
|
||||
int flags = LibvpxVp8Encoder::EncodeFlags(tl_config_);
|
||||
if (flags != -1)
|
||||
cfg_ = layers_->UpdateConfiguration(0);
|
||||
|
||||
if (timestamp >= kTimestampDelta5Fps * 5 && !overshoot && flags != -1) {
|
||||
// Simulate one overshoot.
|
||||
layers_->OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
|
||||
overshoot = true;
|
||||
}
|
||||
|
||||
if (flags == kTl0Flags) {
|
||||
if (timestamp >= kTimestampDelta5Fps * 20 && !trigger_drop) {
|
||||
// Simulate a too large frame, to cause frame drop.
|
||||
layers_->OnEncodeDone(0, timestamp, frame_size_ * 10, false, kTl0Qp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
trigger_drop = true;
|
||||
} else {
|
||||
layers_->OnEncodeDone(0, timestamp, frame_size_, false, kTl0Qp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
}
|
||||
} else if (flags == kTl1Flags || flags == kTl1SyncFlags) {
|
||||
layers_->OnEncodeDone(0, timestamp, frame_size_, false, kTl1Qp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
} else if (flags == -1) {
|
||||
dropped_frame = true;
|
||||
} else {
|
||||
RTC_DCHECK_NOTREACHED() << "Unexpected flags";
|
||||
}
|
||||
clock_.AdvanceTime(TimeDelta::Millis(1000 / 5));
|
||||
}
|
||||
|
||||
EXPECT_TRUE(overshoot);
|
||||
EXPECT_TRUE(dropped_frame);
|
||||
|
||||
layers_.reset(); // Histograms are reported on destruction.
|
||||
|
||||
EXPECT_METRIC_EQ(
|
||||
1, metrics::NumSamples("WebRTC.Video.Screenshare.Layer0.FrameRate"));
|
||||
EXPECT_METRIC_EQ(
|
||||
1, metrics::NumSamples("WebRTC.Video.Screenshare.Layer1.FrameRate"));
|
||||
EXPECT_METRIC_EQ(
|
||||
1, metrics::NumSamples("WebRTC.Video.Screenshare.FramesPerDrop"));
|
||||
EXPECT_METRIC_EQ(
|
||||
1, metrics::NumSamples("WebRTC.Video.Screenshare.FramesPerOvershoot"));
|
||||
EXPECT_METRIC_EQ(1,
|
||||
metrics::NumSamples("WebRTC.Video.Screenshare.Layer0.Qp"));
|
||||
EXPECT_METRIC_EQ(1,
|
||||
metrics::NumSamples("WebRTC.Video.Screenshare.Layer1.Qp"));
|
||||
EXPECT_METRIC_EQ(
|
||||
1, metrics::NumSamples("WebRTC.Video.Screenshare.Layer0.TargetBitrate"));
|
||||
EXPECT_METRIC_EQ(
|
||||
1, metrics::NumSamples("WebRTC.Video.Screenshare.Layer1.TargetBitrate"));
|
||||
|
||||
EXPECT_METRIC_GT(
|
||||
metrics::MinSample("WebRTC.Video.Screenshare.Layer0.FrameRate"), 1);
|
||||
EXPECT_METRIC_GT(
|
||||
metrics::MinSample("WebRTC.Video.Screenshare.Layer1.FrameRate"), 1);
|
||||
EXPECT_METRIC_GT(metrics::MinSample("WebRTC.Video.Screenshare.FramesPerDrop"),
|
||||
1);
|
||||
EXPECT_METRIC_GT(
|
||||
metrics::MinSample("WebRTC.Video.Screenshare.FramesPerOvershoot"), 1);
|
||||
EXPECT_METRIC_EQ(
|
||||
1, metrics::NumEvents("WebRTC.Video.Screenshare.Layer0.Qp", kTl0Qp));
|
||||
EXPECT_METRIC_EQ(
|
||||
1, metrics::NumEvents("WebRTC.Video.Screenshare.Layer1.Qp", kTl1Qp));
|
||||
EXPECT_METRIC_EQ(
|
||||
1, metrics::NumEvents("WebRTC.Video.Screenshare.Layer0.TargetBitrate",
|
||||
kDefaultTl0BitrateKbps));
|
||||
EXPECT_METRIC_EQ(
|
||||
1, metrics::NumEvents("WebRTC.Video.Screenshare.Layer1.TargetBitrate",
|
||||
kDefaultTl1BitrateKbps));
|
||||
}
|
||||
|
||||
TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) {
|
||||
int64_t kTestSpanMs = 2000;
|
||||
int64_t kFrameIntervalsMs = 1000 / kFrameRate;
|
||||
|
||||
uint32_t timestamp = 1234;
|
||||
int num_input_frames = 0;
|
||||
int num_discarded_frames = 0;
|
||||
|
||||
// Send at regular rate - no drops expected.
|
||||
for (int64_t i = 0; i < kTestSpanMs; i += kFrameIntervalsMs) {
|
||||
if (NextFrameConfig(0, timestamp).drop_frame) {
|
||||
++num_discarded_frames;
|
||||
} else {
|
||||
size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8;
|
||||
layers_->OnEncodeDone(0, timestamp, frame_size_bytes, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
}
|
||||
timestamp += kFrameIntervalsMs * 90;
|
||||
clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalsMs));
|
||||
|
||||
++num_input_frames;
|
||||
}
|
||||
EXPECT_EQ(0, num_discarded_frames);
|
||||
|
||||
// Send at twice the configured rate - drop every other frame.
|
||||
num_input_frames = 0;
|
||||
num_discarded_frames = 0;
|
||||
for (int64_t i = 0; i < kTestSpanMs; i += kFrameIntervalsMs / 2) {
|
||||
if (NextFrameConfig(0, timestamp).drop_frame) {
|
||||
++num_discarded_frames;
|
||||
} else {
|
||||
size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8;
|
||||
layers_->OnEncodeDone(0, timestamp, frame_size_bytes, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
}
|
||||
timestamp += kFrameIntervalsMs * 90 / 2;
|
||||
clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalsMs));
|
||||
++num_input_frames;
|
||||
}
|
||||
|
||||
// Allow for some rounding errors in the measurements.
|
||||
EXPECT_NEAR(num_discarded_frames, num_input_frames / 2, 2);
|
||||
}
|
||||
|
||||
TEST_F(ScreenshareLayerTest, 2LayersSyncAtOvershootDrop) {
|
||||
// Run grace period so we have existing frames in both TL0 and Tl1.
|
||||
EXPECT_TRUE(RunGracePeriod());
|
||||
|
||||
// Move ahead until we have a sync frame in TL1.
|
||||
EXPECT_EQ(kTl1SyncFlags, SkipUntilTlAndSync(1, true));
|
||||
ASSERT_TRUE(tl_config_.layer_sync);
|
||||
|
||||
// Simulate overshoot of this frame.
|
||||
layers_->OnEncodeDone(0, timestamp_, 0, false, 0, nullptr);
|
||||
|
||||
cfg_ = layers_->UpdateConfiguration(0);
|
||||
EXPECT_EQ(kTl1SyncFlags, LibvpxVp8Encoder::EncodeFlags(tl_config_));
|
||||
|
||||
CodecSpecificInfo new_info;
|
||||
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
|
||||
&new_info);
|
||||
EXPECT_TRUE(new_info.codecSpecific.VP8.layerSync);
|
||||
}
|
||||
|
||||
TEST_F(ScreenshareLayerTest, DropOnTooShortFrameInterval) {
|
||||
// Run grace period so we have existing frames in both TL0 and Tl1.
|
||||
EXPECT_TRUE(RunGracePeriod());
|
||||
|
||||
// Add a large gap, so there's plenty of room in the rate tracker.
|
||||
timestamp_ += kTimestampDelta5Fps * 3;
|
||||
EXPECT_FALSE(NextFrameConfig(0, timestamp_).drop_frame);
|
||||
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
|
||||
IgnoredCodecSpecificInfo());
|
||||
|
||||
// Frame interval below 90% if desired time is not allowed, try inserting
|
||||
// frame just before this limit.
|
||||
const int64_t kMinFrameInterval = (kTimestampDelta5Fps * 85) / 100;
|
||||
timestamp_ += kMinFrameInterval - 90;
|
||||
EXPECT_TRUE(NextFrameConfig(0, timestamp_).drop_frame);
|
||||
|
||||
// Try again at the limit, now it should pass.
|
||||
timestamp_ += 90;
|
||||
EXPECT_FALSE(NextFrameConfig(0, timestamp_).drop_frame);
|
||||
}
|
||||
|
||||
TEST_F(ScreenshareLayerTest, AdjustsBitrateWhenDroppingFrames) {
|
||||
const uint32_t kTimestampDelta10Fps = kTimestampDelta5Fps / 2;
|
||||
const int kNumFrames = 30;
|
||||
ASSERT_TRUE(cfg_.rc_target_bitrate.has_value());
|
||||
const uint32_t default_bitrate = cfg_.rc_target_bitrate.value();
|
||||
layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, 10);
|
||||
|
||||
int num_dropped_frames = 0;
|
||||
for (int i = 0; i < kNumFrames; ++i) {
|
||||
if (EncodeFrame(false) == -1)
|
||||
++num_dropped_frames;
|
||||
timestamp_ += kTimestampDelta10Fps;
|
||||
}
|
||||
cfg_ = layers_->UpdateConfiguration(0);
|
||||
|
||||
EXPECT_EQ(num_dropped_frames, kNumFrames / 2);
|
||||
EXPECT_EQ(cfg_.rc_target_bitrate, default_bitrate * 2);
|
||||
}
|
||||
|
||||
TEST_F(ScreenshareLayerTest, UpdatesConfigurationAfterRateChange) {
|
||||
// Set inital rate again, no need to update configuration.
|
||||
layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, kFrameRate);
|
||||
cfg_ = layers_->UpdateConfiguration(0);
|
||||
|
||||
// Rate changed, now update config.
|
||||
std::vector<uint32_t> bitrates = kDefault2TlBitratesBps;
|
||||
bitrates[1] -= 100000;
|
||||
layers_->OnRatesUpdated(0, bitrates, 5);
|
||||
cfg_ = layers_->UpdateConfiguration(0);
|
||||
|
||||
// Changed rate, but then set changed rate again before trying to update
|
||||
// configuration, update should still apply.
|
||||
bitrates[1] -= 100000;
|
||||
layers_->OnRatesUpdated(0, bitrates, 5);
|
||||
layers_->OnRatesUpdated(0, bitrates, 5);
|
||||
cfg_ = layers_->UpdateConfiguration(0);
|
||||
}
|
||||
|
||||
TEST_F(ScreenshareLayerTest, MaxQpRestoredAfterDoubleDrop) {
|
||||
// Run grace period so we have existing frames in both TL0 and Tl1.
|
||||
EXPECT_TRUE(RunGracePeriod());
|
||||
|
||||
// Move ahead until we have a sync frame in TL1.
|
||||
EXPECT_EQ(kTl1SyncFlags, SkipUntilTlAndSync(1, true));
|
||||
ASSERT_TRUE(tl_config_.layer_sync);
|
||||
|
||||
// Simulate overshoot of this frame.
|
||||
layers_->OnEncodeDone(0, timestamp_, 0, false, -1, nullptr);
|
||||
|
||||
// Simulate re-encoded frame.
|
||||
layers_->OnEncodeDone(0, timestamp_, 1, false, max_qp_,
|
||||
IgnoredCodecSpecificInfo());
|
||||
|
||||
// Next frame, expect boosted quality.
|
||||
// Slightly alter bitrate between each frame.
|
||||
std::vector<uint32_t> kDefault2TlBitratesBpsAlt = kDefault2TlBitratesBps;
|
||||
kDefault2TlBitratesBpsAlt[1] += 4000;
|
||||
layers_->OnRatesUpdated(0, kDefault2TlBitratesBpsAlt, kFrameRate);
|
||||
EXPECT_EQ(kTl1Flags, SkipUntilTlAndSync(1, false));
|
||||
EXPECT_TRUE(config_updated_);
|
||||
EXPECT_LT(cfg_.rc_max_quantizer, max_qp_);
|
||||
ASSERT_TRUE(cfg_.rc_max_quantizer.has_value());
|
||||
const uint32_t adjusted_qp = cfg_.rc_max_quantizer.value();
|
||||
|
||||
// Simulate overshoot of this frame.
|
||||
layers_->OnEncodeDone(0, timestamp_, 0, false, -1, nullptr);
|
||||
|
||||
// Simulate re-encoded frame.
|
||||
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, max_qp_,
|
||||
IgnoredCodecSpecificInfo());
|
||||
|
||||
// A third frame, expect boosted quality.
|
||||
layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, kFrameRate);
|
||||
EXPECT_EQ(kTl1Flags, SkipUntilTlAndSync(1, false));
|
||||
EXPECT_TRUE(config_updated_);
|
||||
EXPECT_LT(cfg_.rc_max_quantizer, max_qp_);
|
||||
EXPECT_EQ(adjusted_qp, cfg_.rc_max_quantizer);
|
||||
|
||||
// Frame encoded.
|
||||
layers_->OnEncodeDone(0, timestamp_, frame_size_, false, max_qp_,
|
||||
IgnoredCodecSpecificInfo());
|
||||
|
||||
// A fourth frame, max qp should be restored.
|
||||
layers_->OnRatesUpdated(0, kDefault2TlBitratesBpsAlt, kFrameRate);
|
||||
EXPECT_EQ(kTl1Flags, SkipUntilTlAndSync(1, false));
|
||||
EXPECT_EQ(cfg_.rc_max_quantizer, max_qp_);
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
/*
|
||||
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_
|
||||
#define MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_
|
||||
|
||||
// TODO(webrtc:9012) Remove this file when downstream projects have updated.
|
||||
#include "api/video_codecs/vp8_temporal_layers.h"
|
||||
|
||||
#endif // MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_
|
||||
|
|
@ -0,0 +1,146 @@
|
|||
/*
|
||||
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/video_coding/codecs/vp8/include/temporal_layers_checker.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "modules/video_coding/codecs/interface/common_constants.h"
|
||||
#include "modules/video_coding/codecs/vp8/default_temporal_layers.h"
|
||||
#include "rtc_base/logging.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
std::unique_ptr<TemporalLayersChecker>
|
||||
TemporalLayersChecker::CreateTemporalLayersChecker(Vp8TemporalLayersType type,
|
||||
int num_temporal_layers) {
|
||||
switch (type) {
|
||||
case Vp8TemporalLayersType::kFixedPattern:
|
||||
return std::make_unique<DefaultTemporalLayersChecker>(
|
||||
num_temporal_layers);
|
||||
case Vp8TemporalLayersType::kBitrateDynamic:
|
||||
// Conference mode temporal layering for screen content in base stream.
|
||||
return std::make_unique<TemporalLayersChecker>(num_temporal_layers);
|
||||
}
|
||||
RTC_CHECK_NOTREACHED();
|
||||
}
|
||||
|
||||
TemporalLayersChecker::TemporalLayersChecker(int num_temporal_layers)
|
||||
: num_temporal_layers_(num_temporal_layers),
|
||||
sequence_number_(0),
|
||||
last_sync_sequence_number_(0),
|
||||
last_tl0_sequence_number_(0) {}
|
||||
|
||||
bool TemporalLayersChecker::CheckAndUpdateBufferState(
|
||||
BufferState* state,
|
||||
bool* need_sync,
|
||||
bool frame_is_keyframe,
|
||||
uint8_t temporal_layer,
|
||||
Vp8FrameConfig::BufferFlags flags,
|
||||
uint32_t sequence_number,
|
||||
uint32_t* lowest_sequence_referenced) {
|
||||
if (flags & Vp8FrameConfig::BufferFlags::kReference) {
|
||||
if (state->temporal_layer > 0 && !state->is_keyframe) {
|
||||
*need_sync = false;
|
||||
}
|
||||
if (!state->is_keyframe && !frame_is_keyframe &&
|
||||
state->sequence_number < *lowest_sequence_referenced) {
|
||||
*lowest_sequence_referenced = state->sequence_number;
|
||||
}
|
||||
if (!frame_is_keyframe && !state->is_keyframe &&
|
||||
state->temporal_layer > temporal_layer) {
|
||||
RTC_LOG(LS_ERROR) << "Frame is referencing higher temporal layer.";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if ((flags & Vp8FrameConfig::BufferFlags::kUpdate)) {
|
||||
state->temporal_layer = temporal_layer;
|
||||
state->sequence_number = sequence_number;
|
||||
state->is_keyframe = frame_is_keyframe;
|
||||
}
|
||||
if (frame_is_keyframe)
|
||||
state->is_keyframe = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool TemporalLayersChecker::CheckTemporalConfig(
|
||||
bool frame_is_keyframe,
|
||||
const Vp8FrameConfig& frame_config) {
|
||||
if (frame_config.drop_frame ||
|
||||
frame_config.packetizer_temporal_idx == kNoTemporalIdx) {
|
||||
return true;
|
||||
}
|
||||
++sequence_number_;
|
||||
if (frame_config.packetizer_temporal_idx >= num_temporal_layers_ ||
|
||||
(frame_config.packetizer_temporal_idx == kNoTemporalIdx &&
|
||||
num_temporal_layers_ > 1)) {
|
||||
RTC_LOG(LS_ERROR) << "Incorrect temporal layer set for frame: "
|
||||
<< frame_config.packetizer_temporal_idx
|
||||
<< " num_temporal_layers: " << num_temporal_layers_;
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t lowest_sequence_referenced = sequence_number_;
|
||||
bool need_sync = frame_config.packetizer_temporal_idx > 0 &&
|
||||
frame_config.packetizer_temporal_idx != kNoTemporalIdx;
|
||||
|
||||
if (!CheckAndUpdateBufferState(
|
||||
&last_, &need_sync, frame_is_keyframe,
|
||||
frame_config.packetizer_temporal_idx, frame_config.last_buffer_flags,
|
||||
sequence_number_, &lowest_sequence_referenced)) {
|
||||
RTC_LOG(LS_ERROR) << "Error in the Last buffer";
|
||||
return false;
|
||||
}
|
||||
if (!CheckAndUpdateBufferState(&golden_, &need_sync, frame_is_keyframe,
|
||||
frame_config.packetizer_temporal_idx,
|
||||
frame_config.golden_buffer_flags,
|
||||
sequence_number_,
|
||||
&lowest_sequence_referenced)) {
|
||||
RTC_LOG(LS_ERROR) << "Error in the Golden buffer";
|
||||
return false;
|
||||
}
|
||||
if (!CheckAndUpdateBufferState(
|
||||
&arf_, &need_sync, frame_is_keyframe,
|
||||
frame_config.packetizer_temporal_idx, frame_config.arf_buffer_flags,
|
||||
sequence_number_, &lowest_sequence_referenced)) {
|
||||
RTC_LOG(LS_ERROR) << "Error in the Arf buffer";
|
||||
return false;
|
||||
}
|
||||
|
||||
if (lowest_sequence_referenced < last_sync_sequence_number_ &&
|
||||
!frame_is_keyframe) {
|
||||
RTC_LOG(LS_ERROR) << "Reference past the last sync frame. Referenced "
|
||||
<< lowest_sequence_referenced << ", but sync was at "
|
||||
<< last_sync_sequence_number_;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (frame_config.packetizer_temporal_idx == 0) {
|
||||
last_tl0_sequence_number_ = sequence_number_;
|
||||
}
|
||||
|
||||
if (frame_is_keyframe) {
|
||||
last_sync_sequence_number_ = sequence_number_;
|
||||
}
|
||||
|
||||
if (need_sync) {
|
||||
last_sync_sequence_number_ = last_tl0_sequence_number_;
|
||||
}
|
||||
|
||||
// Ignore sync flag on key-frames as it really doesn't matter.
|
||||
if (need_sync != frame_config.layer_sync && !frame_is_keyframe) {
|
||||
RTC_LOG(LS_ERROR) << "Sync bit is set incorrectly on a frame. Expected: "
|
||||
<< need_sync << " Actual: " << frame_config.layer_sync;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,24 @@
|
|||
/*
|
||||
* Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/video_coding/codecs/vp8/vp8_scalability.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
bool VP8SupportsScalabilityMode(ScalabilityMode scalability_mode) {
|
||||
for (const auto& entry : kVP8SupportedScalabilityModes) {
|
||||
if (entry == scalability_mode) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
/*
|
||||
* Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_VIDEO_CODING_CODECS_VP8_VP8_SCALABILITY_H_
|
||||
#define MODULES_VIDEO_CODING_CODECS_VP8_VP8_SCALABILITY_H_
|
||||
|
||||
#include "api/video_codecs/scalability_mode.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
inline constexpr ScalabilityMode kVP8SupportedScalabilityModes[] = {
|
||||
ScalabilityMode::kL1T1, ScalabilityMode::kL1T2, ScalabilityMode::kL1T3};
|
||||
bool VP8SupportsScalabilityMode(ScalabilityMode scalability_mode);
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_VIDEO_CODING_CODECS_VP8_VP8_SCALABILITY_H_
|
||||
Loading…
Add table
Add a link
Reference in a new issue