Repo created
This commit is contained in:
parent
81b91f4139
commit
f8c34fa5ee
22732 changed files with 4815320 additions and 2 deletions
|
|
@ -0,0 +1,61 @@
|
|||
/*
|
||||
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/video_coding/timing/decode_time_percentile_filter.h"
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
namespace {
|
||||
|
||||
// The first kIgnoredSampleCount samples will be ignored.
|
||||
const int kIgnoredSampleCount = 5;
|
||||
// Return the `kPercentile` value in RequiredDecodeTimeMs().
|
||||
const float kPercentile = 0.95f;
|
||||
// The window size in ms.
|
||||
const int64_t kTimeLimitMs = 10000;
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
DecodeTimePercentileFilter::DecodeTimePercentileFilter()
|
||||
: ignored_sample_count_(0), filter_(kPercentile) {}
|
||||
DecodeTimePercentileFilter::~DecodeTimePercentileFilter() = default;
|
||||
|
||||
void DecodeTimePercentileFilter::AddTiming(int64_t decode_time_ms,
|
||||
int64_t now_ms) {
|
||||
// Ignore the first `kIgnoredSampleCount` samples.
|
||||
if (ignored_sample_count_ < kIgnoredSampleCount) {
|
||||
++ignored_sample_count_;
|
||||
return;
|
||||
}
|
||||
|
||||
// Insert new decode time value.
|
||||
filter_.Insert(decode_time_ms);
|
||||
history_.emplace(decode_time_ms, now_ms);
|
||||
|
||||
// Pop old decode time values.
|
||||
while (!history_.empty() &&
|
||||
now_ms - history_.front().sample_time_ms > kTimeLimitMs) {
|
||||
filter_.Erase(history_.front().decode_time_ms);
|
||||
history_.pop();
|
||||
}
|
||||
}
|
||||
|
||||
// Get the 95th percentile observed decode time within a time window.
|
||||
int64_t DecodeTimePercentileFilter::RequiredDecodeTimeMs() const {
|
||||
return filter_.GetPercentileValue();
|
||||
}
|
||||
|
||||
DecodeTimePercentileFilter::Sample::Sample(int64_t decode_time_ms,
|
||||
int64_t sample_time_ms)
|
||||
: decode_time_ms(decode_time_ms), sample_time_ms(sample_time_ms) {}
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
/*
|
||||
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_VIDEO_CODING_TIMING_DECODE_TIME_PERCENTILE_FILTER_H_
|
||||
#define MODULES_VIDEO_CODING_TIMING_DECODE_TIME_PERCENTILE_FILTER_H_
|
||||
|
||||
#include <queue>
|
||||
|
||||
#include "rtc_base/numerics/percentile_filter.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
// The `DecodeTimePercentileFilter` filters the actual per-frame decode times
|
||||
// and provides an estimate for the 95th percentile of those decode times. This
|
||||
// estimate can be used to determine how large the "decode delay term" should be
|
||||
// when determining the render timestamp for a frame.
|
||||
class DecodeTimePercentileFilter {
|
||||
public:
|
||||
DecodeTimePercentileFilter();
|
||||
~DecodeTimePercentileFilter();
|
||||
|
||||
// Add a new decode time to the filter.
|
||||
void AddTiming(int64_t new_decode_time_ms, int64_t now_ms);
|
||||
|
||||
// Get the required decode time in ms. It is the 95th percentile observed
|
||||
// decode time within a time window.
|
||||
int64_t RequiredDecodeTimeMs() const;
|
||||
|
||||
private:
|
||||
struct Sample {
|
||||
Sample(int64_t decode_time_ms, int64_t sample_time_ms);
|
||||
int64_t decode_time_ms;
|
||||
int64_t sample_time_ms;
|
||||
};
|
||||
|
||||
// The number of samples ignored so far.
|
||||
int ignored_sample_count_;
|
||||
// Queue with history of latest decode time values.
|
||||
std::queue<Sample> history_;
|
||||
// `filter_` contains the same values as `history_`, but in a data structure
|
||||
// that allows efficient retrieval of the percentile value.
|
||||
PercentileFilter<int64_t> filter_;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_VIDEO_CODING_TIMING_DECODE_TIME_PERCENTILE_FILTER_H_
|
||||
|
|
@ -0,0 +1,148 @@
|
|||
/*
|
||||
* Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/video_coding/timing/frame_delay_variation_kalman_filter.h"
|
||||
|
||||
#include "api/units/data_size.h"
|
||||
#include "api/units/time_delta.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
namespace {
|
||||
// TODO(brandtr): The value below corresponds to 8 Gbps. Is that reasonable?
|
||||
constexpr double kMaxBandwidth = 0.000001; // Unit: [1 / bytes per ms].
|
||||
} // namespace
|
||||
|
||||
FrameDelayVariationKalmanFilter::FrameDelayVariationKalmanFilter() {
|
||||
// TODO(brandtr): Is there a factor 1000 missing here?
|
||||
estimate_[0] = 1 / (512e3 / 8); // Unit: [1 / bytes per ms]
|
||||
estimate_[1] = 0; // Unit: [ms]
|
||||
|
||||
// Initial estimate covariance.
|
||||
estimate_cov_[0][0] = 1e-4; // Unit: [(1 / bytes per ms)^2]
|
||||
estimate_cov_[1][1] = 1e2; // Unit: [ms^2]
|
||||
estimate_cov_[0][1] = estimate_cov_[1][0] = 0;
|
||||
|
||||
// Process noise covariance.
|
||||
process_noise_cov_diag_[0] = 2.5e-10; // Unit: [(1 / bytes per ms)^2]
|
||||
process_noise_cov_diag_[1] = 1e-10; // Unit: [ms^2]
|
||||
}
|
||||
|
||||
void FrameDelayVariationKalmanFilter::PredictAndUpdate(
|
||||
double frame_delay_variation_ms,
|
||||
double frame_size_variation_bytes,
|
||||
double max_frame_size_bytes,
|
||||
double var_noise) {
|
||||
// Sanity checks.
|
||||
if (max_frame_size_bytes < 1) {
|
||||
return;
|
||||
}
|
||||
if (var_noise <= 0.0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// This member function follows the data flow in
|
||||
// https://en.wikipedia.org/wiki/Kalman_filter#Details.
|
||||
|
||||
// 1) Estimate prediction: `x = F*x`.
|
||||
// For this model, there is no need to explicitly predict the estimate, since
|
||||
// the state transition matrix is the identity.
|
||||
|
||||
// 2) Estimate covariance prediction: `P = F*P*F' + Q`.
|
||||
// Again, since the state transition matrix is the identity, this update
|
||||
// is performed by simply adding the process noise covariance.
|
||||
estimate_cov_[0][0] += process_noise_cov_diag_[0];
|
||||
estimate_cov_[1][1] += process_noise_cov_diag_[1];
|
||||
|
||||
// 3) Innovation: `y = z - H*x`.
|
||||
// This is the part of the measurement that cannot be explained by the current
|
||||
// estimate.
|
||||
double innovation =
|
||||
frame_delay_variation_ms -
|
||||
GetFrameDelayVariationEstimateTotal(frame_size_variation_bytes);
|
||||
|
||||
// 4) Innovation variance: `s = H*P*H' + r`.
|
||||
double estim_cov_times_obs[2];
|
||||
estim_cov_times_obs[0] =
|
||||
estimate_cov_[0][0] * frame_size_variation_bytes + estimate_cov_[0][1];
|
||||
estim_cov_times_obs[1] =
|
||||
estimate_cov_[1][0] * frame_size_variation_bytes + estimate_cov_[1][1];
|
||||
double observation_noise_stddev =
|
||||
(300.0 * exp(-fabs(frame_size_variation_bytes) /
|
||||
(1e0 * max_frame_size_bytes)) +
|
||||
1) *
|
||||
sqrt(var_noise);
|
||||
if (observation_noise_stddev < 1.0) {
|
||||
observation_noise_stddev = 1.0;
|
||||
}
|
||||
// TODO(brandtr): Shouldn't we add observation_noise_stddev^2 here? Otherwise,
|
||||
// the dimensional analysis fails.
|
||||
double innovation_var = frame_size_variation_bytes * estim_cov_times_obs[0] +
|
||||
estim_cov_times_obs[1] + observation_noise_stddev;
|
||||
if ((innovation_var < 1e-9 && innovation_var >= 0) ||
|
||||
(innovation_var > -1e-9 && innovation_var <= 0)) {
|
||||
RTC_DCHECK_NOTREACHED();
|
||||
return;
|
||||
}
|
||||
|
||||
// 5) Optimal Kalman gain: `K = P*H'/s`.
|
||||
// How much to trust the model vs. how much to trust the measurement.
|
||||
double kalman_gain[2];
|
||||
kalman_gain[0] = estim_cov_times_obs[0] / innovation_var;
|
||||
kalman_gain[1] = estim_cov_times_obs[1] / innovation_var;
|
||||
|
||||
// 6) Estimate update: `x = x + K*y`.
|
||||
// Optimally weight the new information in the innovation and add it to the
|
||||
// old estimate.
|
||||
estimate_[0] += kalman_gain[0] * innovation;
|
||||
estimate_[1] += kalman_gain[1] * innovation;
|
||||
|
||||
// (This clamping is not part of the linear Kalman filter.)
|
||||
if (estimate_[0] < kMaxBandwidth) {
|
||||
estimate_[0] = kMaxBandwidth;
|
||||
}
|
||||
|
||||
// 7) Estimate covariance update: `P = (I - K*H)*P`
|
||||
double t00 = estimate_cov_[0][0];
|
||||
double t01 = estimate_cov_[0][1];
|
||||
estimate_cov_[0][0] =
|
||||
(1 - kalman_gain[0] * frame_size_variation_bytes) * t00 -
|
||||
kalman_gain[0] * estimate_cov_[1][0];
|
||||
estimate_cov_[0][1] =
|
||||
(1 - kalman_gain[0] * frame_size_variation_bytes) * t01 -
|
||||
kalman_gain[0] * estimate_cov_[1][1];
|
||||
estimate_cov_[1][0] = estimate_cov_[1][0] * (1 - kalman_gain[1]) -
|
||||
kalman_gain[1] * frame_size_variation_bytes * t00;
|
||||
estimate_cov_[1][1] = estimate_cov_[1][1] * (1 - kalman_gain[1]) -
|
||||
kalman_gain[1] * frame_size_variation_bytes * t01;
|
||||
|
||||
// Covariance matrix, must be positive semi-definite.
|
||||
RTC_DCHECK(estimate_cov_[0][0] + estimate_cov_[1][1] >= 0 &&
|
||||
estimate_cov_[0][0] * estimate_cov_[1][1] -
|
||||
estimate_cov_[0][1] * estimate_cov_[1][0] >=
|
||||
0 &&
|
||||
estimate_cov_[0][0] >= 0);
|
||||
}
|
||||
|
||||
double FrameDelayVariationKalmanFilter::GetFrameDelayVariationEstimateSizeBased(
|
||||
double frame_size_variation_bytes) const {
|
||||
// Unit: [1 / bytes per millisecond] * [bytes] = [milliseconds].
|
||||
return estimate_[0] * frame_size_variation_bytes;
|
||||
}
|
||||
|
||||
double FrameDelayVariationKalmanFilter::GetFrameDelayVariationEstimateTotal(
|
||||
double frame_size_variation_bytes) const {
|
||||
double frame_transmission_delay_ms =
|
||||
GetFrameDelayVariationEstimateSizeBased(frame_size_variation_bytes);
|
||||
double link_queuing_delay_ms = estimate_[1];
|
||||
return frame_transmission_delay_ms + link_queuing_delay_ms;
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
/*
|
||||
* Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_VIDEO_CODING_TIMING_FRAME_DELAY_VARIATION_KALMAN_FILTER_H_
|
||||
#define MODULES_VIDEO_CODING_TIMING_FRAME_DELAY_VARIATION_KALMAN_FILTER_H_
|
||||
|
||||
#include "api/units/data_size.h"
|
||||
#include "api/units/time_delta.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
// This class uses a linear Kalman filter (see
|
||||
// https://en.wikipedia.org/wiki/Kalman_filter) to estimate the frame delay
|
||||
// variation (i.e., the difference in transmission time between a frame and the
|
||||
// prior frame) for a frame, given its size variation in bytes (i.e., the
|
||||
// difference in size between a frame and the prior frame). The idea is that,
|
||||
// given a fixed link bandwidth, a larger frame (in bytes) would take
|
||||
// proportionally longer to arrive than a correspondingly smaller frame. Using
|
||||
// the variations of frame delay and frame size, the underlying bandwidth and
|
||||
// queuing delay variation of the network link can be estimated.
|
||||
//
|
||||
// The filter takes as input the frame delay variation, the difference between
|
||||
// the actual inter-frame arrival time and the expected inter-frame arrival time
|
||||
// (based on RTP timestamp), and frame size variation, the inter-frame size
|
||||
// delta for a single frame. The frame delay variation is seen as the
|
||||
// measurement and the frame size variation is used in the observation model.
|
||||
// The hidden state of the filter is the link bandwidth and queuing delay
|
||||
// buildup. The estimated state can be used to get the expected frame delay
|
||||
// variation for a frame, given its frame size variation. This information can
|
||||
// then be used to estimate the frame delay variation coming from network
|
||||
// jitter.
|
||||
//
|
||||
// Mathematical details:
|
||||
// * The state (`x` in Wikipedia notation) is a 2x1 vector comprising the
|
||||
// reciprocal of link bandwidth [1 / bytes per ms] and the
|
||||
// link queuing delay buildup [ms].
|
||||
// * The state transition matrix (`F`) is the 2x2 identity matrix, meaning that
|
||||
// link bandwidth and link queuing delay buildup are modeled as independent.
|
||||
// * The measurement (`z`) is the (scalar) frame delay variation [ms].
|
||||
// * The observation matrix (`H`) is a 1x2 vector set as
|
||||
// `{frame_size_variation [bytes], 1.0}`.
|
||||
// * The state estimate covariance (`P`) is a symmetric 2x2 matrix.
|
||||
// * The process noise covariance (`Q`) is a constant 2x2 diagonal matrix
|
||||
// [(1 / bytes per ms)^2, ms^2].
|
||||
// * The observation noise covariance (`r`) is a scalar [ms^2] that is
|
||||
// determined externally to this class.
|
||||
class FrameDelayVariationKalmanFilter {
|
||||
public:
|
||||
FrameDelayVariationKalmanFilter();
|
||||
~FrameDelayVariationKalmanFilter() = default;
|
||||
|
||||
// Predicts and updates the filter, given a new pair of frame delay variation
|
||||
// and frame size variation.
|
||||
//
|
||||
// Inputs:
|
||||
// `frame_delay_variation_ms`:
|
||||
// Frame delay variation as calculated by the `InterFrameDelay` estimator.
|
||||
//
|
||||
// `frame_size_variation_bytes`:
|
||||
// Frame size variation, i.e., the current frame size minus the previous
|
||||
// frame size (in bytes). Note that this quantity may be negative.
|
||||
//
|
||||
// `max_frame_size_bytes`:
|
||||
// Filtered largest frame size received since the last reset.
|
||||
//
|
||||
// `var_noise`:
|
||||
// Variance of the estimated random jitter.
|
||||
//
|
||||
// TODO(bugs.webrtc.org/14381): For now use doubles as input parameters as
|
||||
// units defined in api/units have insufficient underlying precision for
|
||||
// jitter estimation.
|
||||
void PredictAndUpdate(double frame_delay_variation_ms,
|
||||
double frame_size_variation_bytes,
|
||||
double max_frame_size_bytes,
|
||||
double var_noise);
|
||||
|
||||
// Given a frame size variation, returns the estimated frame delay variation
|
||||
// explained by the link bandwidth alone.
|
||||
double GetFrameDelayVariationEstimateSizeBased(
|
||||
double frame_size_variation_bytes) const;
|
||||
|
||||
// Given a frame size variation, returns the estimated frame delay variation
|
||||
// explained by both link bandwidth and link queuing delay buildup.
|
||||
double GetFrameDelayVariationEstimateTotal(
|
||||
double frame_size_variation_bytes) const;
|
||||
|
||||
private:
|
||||
// State estimate (bandwidth [1 / bytes per ms], queue buildup [ms]).
|
||||
double estimate_[2];
|
||||
double estimate_cov_[2][2]; // Estimate covariance.
|
||||
|
||||
// Process noise covariance. This is a diagonal matrix, so we only store the
|
||||
// diagonal entries.
|
||||
double process_noise_cov_diag_[2];
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_VIDEO_CODING_TIMING_FRAME_DELAY_VARIATION_KALMAN_FILTER_H_
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
/*
|
||||
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/video_coding/timing/inter_frame_delay_variation_calculator.h"
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "api/units/frequency.h"
|
||||
#include "api/units/time_delta.h"
|
||||
#include "modules/include/module_common_types_public.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
namespace {
|
||||
constexpr Frequency k90kHz = Frequency::KiloHertz(90);
|
||||
}
|
||||
|
||||
InterFrameDelayVariationCalculator::InterFrameDelayVariationCalculator() {
|
||||
Reset();
|
||||
}
|
||||
|
||||
void InterFrameDelayVariationCalculator::Reset() {
|
||||
prev_wall_clock_ = absl::nullopt;
|
||||
prev_rtp_timestamp_unwrapped_ = 0;
|
||||
}
|
||||
|
||||
absl::optional<TimeDelta> InterFrameDelayVariationCalculator::Calculate(
|
||||
uint32_t rtp_timestamp,
|
||||
Timestamp now) {
|
||||
int64_t rtp_timestamp_unwrapped = unwrapper_.Unwrap(rtp_timestamp);
|
||||
|
||||
if (!prev_wall_clock_) {
|
||||
prev_wall_clock_ = now;
|
||||
prev_rtp_timestamp_unwrapped_ = rtp_timestamp_unwrapped;
|
||||
// Inter-frame delay variation is undefined for a single frame.
|
||||
// TODO(brandtr): Should this return absl::nullopt instead?
|
||||
return TimeDelta::Zero();
|
||||
}
|
||||
|
||||
// Account for reordering in jitter variance estimate in the future?
|
||||
// Note that this also captures incomplete frames which are grabbed for
|
||||
// decoding after a later frame has been complete, i.e. real packet losses.
|
||||
uint32_t cropped_prev = static_cast<uint32_t>(prev_rtp_timestamp_unwrapped_);
|
||||
if (rtp_timestamp_unwrapped < prev_rtp_timestamp_unwrapped_ ||
|
||||
!IsNewerTimestamp(rtp_timestamp, cropped_prev)) {
|
||||
return absl::nullopt;
|
||||
}
|
||||
|
||||
// Compute the compensated timestamp difference.
|
||||
TimeDelta delta_wall = now - *prev_wall_clock_;
|
||||
int64_t d_rtp_ticks = rtp_timestamp_unwrapped - prev_rtp_timestamp_unwrapped_;
|
||||
TimeDelta delta_rtp = d_rtp_ticks / k90kHz;
|
||||
|
||||
// The inter-frame delay variation is the second order difference between the
|
||||
// RTP and wall clocks of the two frames, or in other words, the first order
|
||||
// difference between `delta_rtp` and `delta_wall`.
|
||||
TimeDelta inter_frame_delay_variation = delta_wall - delta_rtp;
|
||||
|
||||
prev_wall_clock_ = now;
|
||||
prev_rtp_timestamp_unwrapped_ = rtp_timestamp_unwrapped;
|
||||
|
||||
return inter_frame_delay_variation;
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
/*
|
||||
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_VIDEO_CODING_TIMING_INTER_FRAME_DELAY_VARIATION_CALCULATOR_H_
|
||||
#define MODULES_VIDEO_CODING_TIMING_INTER_FRAME_DELAY_VARIATION_CALCULATOR_H_
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "api/units/time_delta.h"
|
||||
#include "api/units/timestamp.h"
|
||||
#include "rtc_base/numerics/sequence_number_unwrapper.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
// This class calculates the inter-frame delay variation (see RFC5481) between
|
||||
// the current frame (as supplied through the current call to `Calculate`) and
|
||||
// the previous frame (as supplied through the previous call to `Calculate`).
|
||||
class InterFrameDelayVariationCalculator {
|
||||
public:
|
||||
InterFrameDelayVariationCalculator();
|
||||
|
||||
// Resets the calculator.
|
||||
void Reset();
|
||||
|
||||
// Calculates the inter-frame delay variation of a frame with the given
|
||||
// RTP timestamp. This method is called when the frame is complete.
|
||||
absl::optional<TimeDelta> Calculate(uint32_t rtp_timestamp, Timestamp now);
|
||||
|
||||
private:
|
||||
// The previous wall clock timestamp used in the calculation.
|
||||
absl::optional<Timestamp> prev_wall_clock_;
|
||||
// The previous RTP timestamp used in the calculation.
|
||||
int64_t prev_rtp_timestamp_unwrapped_;
|
||||
|
||||
RtpTimestampUnwrapper unwrapper_;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_VIDEO_CODING_TIMING_INTER_FRAME_DELAY_VARIATION_CALCULATOR_H_
|
||||
|
|
@ -0,0 +1,476 @@
|
|||
/*
|
||||
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/video_coding/timing/jitter_estimator.h"
|
||||
|
||||
#include <math.h>
|
||||
#include <string.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "api/field_trials_view.h"
|
||||
#include "api/units/data_size.h"
|
||||
#include "api/units/frequency.h"
|
||||
#include "api/units/time_delta.h"
|
||||
#include "api/units/timestamp.h"
|
||||
#include "modules/video_coding/timing/rtt_filter.h"
|
||||
#include "rtc_base/checks.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "rtc_base/numerics/safe_conversions.h"
|
||||
#include "system_wrappers/include/clock.h"
|
||||
|
||||
namespace webrtc {
|
||||
namespace {
|
||||
|
||||
// Number of frames to wait for before post processing estimate. Also used in
|
||||
// the frame rate estimator ramp-up.
|
||||
constexpr size_t kFrameProcessingStartupCount = 30;
|
||||
|
||||
// Number of frames to wait for before enabling the frame size filters.
|
||||
constexpr size_t kFramesUntilSizeFiltering = 5;
|
||||
|
||||
// Initial value for frame size filters.
|
||||
constexpr double kInitialAvgAndMaxFrameSizeBytes = 500.0;
|
||||
|
||||
// Time constant for average frame size filter.
|
||||
constexpr double kPhi = 0.97;
|
||||
// Time constant for max frame size filter.
|
||||
constexpr double kPsi = 0.9999;
|
||||
// Default constants for percentile frame size filter.
|
||||
constexpr double kDefaultMaxFrameSizePercentile = 0.95;
|
||||
constexpr int kDefaultFrameSizeWindow = 30 * 10;
|
||||
|
||||
// Outlier rejection constants.
|
||||
constexpr double kNumStdDevDelayClamp = 3.5;
|
||||
constexpr double kNumStdDevDelayOutlier = 15.0;
|
||||
constexpr double kNumStdDevSizeOutlier = 3.0;
|
||||
constexpr double kCongestionRejectionFactor = -0.25;
|
||||
|
||||
// Rampup constant for deviation noise filters.
|
||||
constexpr size_t kAlphaCountMax = 400;
|
||||
|
||||
// Noise threshold constants.
|
||||
// ~Less than 1% chance (look up in normal distribution table)...
|
||||
constexpr double kNoiseStdDevs = 2.33;
|
||||
// ...of getting 30 ms freezes
|
||||
constexpr double kNoiseStdDevOffset = 30.0;
|
||||
|
||||
// Jitter estimate clamping limits.
|
||||
constexpr TimeDelta kMinJitterEstimate = TimeDelta::Millis(1);
|
||||
constexpr TimeDelta kMaxJitterEstimate = TimeDelta::Seconds(10);
|
||||
|
||||
// A constant describing the delay from the jitter buffer to the delay on the
|
||||
// receiving side which is not accounted for by the jitter buffer nor the
|
||||
// decoding delay estimate.
|
||||
constexpr TimeDelta OPERATING_SYSTEM_JITTER = TimeDelta::Millis(10);
|
||||
|
||||
// Time constant for reseting the NACK count.
|
||||
constexpr TimeDelta kNackCountTimeout = TimeDelta::Seconds(60);
|
||||
|
||||
// RTT mult activation.
|
||||
constexpr size_t kNackLimit = 3;
|
||||
|
||||
// Frame rate estimate clamping limit.
|
||||
constexpr Frequency kMaxFramerateEstimate = Frequency::Hertz(200);
|
||||
|
||||
} // namespace
|
||||
|
||||
constexpr char JitterEstimator::Config::kFieldTrialsKey[];
|
||||
|
||||
JitterEstimator::Config JitterEstimator::Config::ParseAndValidate(
|
||||
absl::string_view field_trial) {
|
||||
Config config;
|
||||
config.Parser()->Parse(field_trial);
|
||||
|
||||
// The `MovingPercentileFilter` RTC_CHECKs on the validity of the
|
||||
// percentile and window length, so we'd better validate the field trial
|
||||
// provided values here.
|
||||
if (config.max_frame_size_percentile) {
|
||||
double original = *config.max_frame_size_percentile;
|
||||
config.max_frame_size_percentile = std::min(std::max(0.0, original), 1.0);
|
||||
if (config.max_frame_size_percentile != original) {
|
||||
RTC_LOG(LS_ERROR) << "Skipping invalid max_frame_size_percentile="
|
||||
<< original;
|
||||
}
|
||||
}
|
||||
if (config.frame_size_window && config.frame_size_window < 1) {
|
||||
RTC_LOG(LS_ERROR) << "Skipping invalid frame_size_window="
|
||||
<< *config.frame_size_window;
|
||||
config.frame_size_window = 1;
|
||||
}
|
||||
|
||||
// General sanity checks.
|
||||
if (config.num_stddev_delay_clamp && config.num_stddev_delay_clamp < 0.0) {
|
||||
RTC_LOG(LS_ERROR) << "Skipping invalid num_stddev_delay_clamp="
|
||||
<< *config.num_stddev_delay_clamp;
|
||||
config.num_stddev_delay_clamp = 0.0;
|
||||
}
|
||||
if (config.num_stddev_delay_outlier &&
|
||||
config.num_stddev_delay_outlier < 0.0) {
|
||||
RTC_LOG(LS_ERROR) << "Skipping invalid num_stddev_delay_outlier="
|
||||
<< *config.num_stddev_delay_outlier;
|
||||
config.num_stddev_delay_outlier = 0.0;
|
||||
}
|
||||
if (config.num_stddev_size_outlier && config.num_stddev_size_outlier < 0.0) {
|
||||
RTC_LOG(LS_ERROR) << "Skipping invalid num_stddev_size_outlier="
|
||||
<< *config.num_stddev_size_outlier;
|
||||
config.num_stddev_size_outlier = 0.0;
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
JitterEstimator::JitterEstimator(Clock* clock,
|
||||
const FieldTrialsView& field_trials)
|
||||
: config_(Config::ParseAndValidate(
|
||||
field_trials.Lookup(Config::kFieldTrialsKey))),
|
||||
avg_frame_size_median_bytes_(static_cast<size_t>(
|
||||
config_.frame_size_window.value_or(kDefaultFrameSizeWindow))),
|
||||
max_frame_size_bytes_percentile_(
|
||||
config_.max_frame_size_percentile.value_or(
|
||||
kDefaultMaxFrameSizePercentile),
|
||||
static_cast<size_t>(
|
||||
config_.frame_size_window.value_or(kDefaultFrameSizeWindow))),
|
||||
fps_counter_(30), // TODO(sprang): Use an estimator with limit based
|
||||
// on time, rather than number of samples.
|
||||
clock_(clock) {
|
||||
Reset();
|
||||
}
|
||||
|
||||
JitterEstimator::~JitterEstimator() = default;
|
||||
|
||||
// Resets the JitterEstimate.
|
||||
void JitterEstimator::Reset() {
|
||||
avg_frame_size_bytes_ = kInitialAvgAndMaxFrameSizeBytes;
|
||||
max_frame_size_bytes_ = kInitialAvgAndMaxFrameSizeBytes;
|
||||
var_frame_size_bytes2_ = 100;
|
||||
avg_frame_size_median_bytes_.Reset();
|
||||
max_frame_size_bytes_percentile_.Reset();
|
||||
last_update_time_ = absl::nullopt;
|
||||
prev_estimate_ = absl::nullopt;
|
||||
prev_frame_size_ = absl::nullopt;
|
||||
avg_noise_ms_ = 0.0;
|
||||
var_noise_ms2_ = 4.0;
|
||||
alpha_count_ = 1;
|
||||
filter_jitter_estimate_ = TimeDelta::Zero();
|
||||
latest_nack_ = Timestamp::Zero();
|
||||
nack_count_ = 0;
|
||||
startup_frame_size_sum_bytes_ = 0;
|
||||
startup_frame_size_count_ = 0;
|
||||
startup_count_ = 0;
|
||||
rtt_filter_.Reset();
|
||||
fps_counter_.Reset();
|
||||
|
||||
kalman_filter_ = FrameDelayVariationKalmanFilter();
|
||||
}
|
||||
|
||||
// Updates the estimates with the new measurements.
|
||||
void JitterEstimator::UpdateEstimate(TimeDelta frame_delay,
|
||||
DataSize frame_size) {
|
||||
if (frame_size.IsZero()) {
|
||||
return;
|
||||
}
|
||||
// Can't use DataSize since this can be negative.
|
||||
double delta_frame_bytes =
|
||||
frame_size.bytes() - prev_frame_size_.value_or(DataSize::Zero()).bytes();
|
||||
if (startup_frame_size_count_ < kFramesUntilSizeFiltering) {
|
||||
startup_frame_size_sum_bytes_ += frame_size.bytes();
|
||||
startup_frame_size_count_++;
|
||||
} else if (startup_frame_size_count_ == kFramesUntilSizeFiltering) {
|
||||
// Give the frame size filter.
|
||||
avg_frame_size_bytes_ = startup_frame_size_sum_bytes_ /
|
||||
static_cast<double>(startup_frame_size_count_);
|
||||
startup_frame_size_count_++;
|
||||
}
|
||||
|
||||
double avg_frame_size_bytes =
|
||||
kPhi * avg_frame_size_bytes_ + (1 - kPhi) * frame_size.bytes();
|
||||
double deviation_size_bytes = 2 * sqrt(var_frame_size_bytes2_);
|
||||
if (frame_size.bytes() < avg_frame_size_bytes_ + deviation_size_bytes) {
|
||||
// Only update the average frame size if this sample wasn't a key frame.
|
||||
avg_frame_size_bytes_ = avg_frame_size_bytes;
|
||||
}
|
||||
|
||||
double delta_bytes = frame_size.bytes() - avg_frame_size_bytes;
|
||||
var_frame_size_bytes2_ = std::max(
|
||||
kPhi * var_frame_size_bytes2_ + (1 - kPhi) * (delta_bytes * delta_bytes),
|
||||
1.0);
|
||||
|
||||
// Update non-linear IIR estimate of max frame size.
|
||||
max_frame_size_bytes_ =
|
||||
std::max<double>(kPsi * max_frame_size_bytes_, frame_size.bytes());
|
||||
|
||||
// Maybe update percentile estimates of frame sizes.
|
||||
if (config_.avg_frame_size_median) {
|
||||
avg_frame_size_median_bytes_.Insert(frame_size.bytes());
|
||||
}
|
||||
if (config_.MaxFrameSizePercentileEnabled()) {
|
||||
max_frame_size_bytes_percentile_.Insert(frame_size.bytes());
|
||||
}
|
||||
|
||||
if (!prev_frame_size_) {
|
||||
prev_frame_size_ = frame_size;
|
||||
return;
|
||||
}
|
||||
prev_frame_size_ = frame_size;
|
||||
|
||||
// Cap frame_delay based on the current time deviation noise.
|
||||
double num_stddev_delay_clamp =
|
||||
config_.num_stddev_delay_clamp.value_or(kNumStdDevDelayClamp);
|
||||
TimeDelta max_time_deviation =
|
||||
TimeDelta::Millis(num_stddev_delay_clamp * sqrt(var_noise_ms2_) + 0.5);
|
||||
frame_delay.Clamp(-max_time_deviation, max_time_deviation);
|
||||
|
||||
double delay_deviation_ms =
|
||||
frame_delay.ms() -
|
||||
kalman_filter_.GetFrameDelayVariationEstimateTotal(delta_frame_bytes);
|
||||
|
||||
// Outlier rejection: these conditions depend on filtered versions of the
|
||||
// delay and frame size _means_, respectively, together with a configurable
|
||||
// number of standard deviations. If a sample is large with respect to the
|
||||
// corresponding mean and dispersion (defined by the number of
|
||||
// standard deviations and the sample standard deviation), it is deemed an
|
||||
// outlier. This "empirical rule" is further described in
|
||||
// https://en.wikipedia.org/wiki/68-95-99.7_rule. Note that neither of the
|
||||
// estimated means are true sample means, which implies that they are possibly
|
||||
// not normally distributed. Hence, this rejection method is just a heuristic.
|
||||
double num_stddev_delay_outlier =
|
||||
config_.num_stddev_delay_outlier.value_or(kNumStdDevDelayOutlier);
|
||||
// Delay outlier rejection is two-sided.
|
||||
bool abs_delay_is_not_outlier =
|
||||
fabs(delay_deviation_ms) <
|
||||
num_stddev_delay_outlier * sqrt(var_noise_ms2_);
|
||||
// The reasoning above means, in particular, that we should use the sample
|
||||
// mean-style `avg_frame_size_bytes_` estimate, as opposed to the
|
||||
// median-filtered version, even if configured to use latter for the
|
||||
// calculation in `CalculateEstimate()`.
|
||||
// Size outlier rejection is one-sided.
|
||||
double num_stddev_size_outlier =
|
||||
config_.num_stddev_size_outlier.value_or(kNumStdDevSizeOutlier);
|
||||
bool size_is_positive_outlier =
|
||||
frame_size.bytes() >
|
||||
avg_frame_size_bytes_ +
|
||||
num_stddev_size_outlier * sqrt(var_frame_size_bytes2_);
|
||||
|
||||
// Only update the Kalman filter if the sample is not considered an extreme
|
||||
// outlier. Even if it is an extreme outlier from a delay point of view, if
|
||||
// the frame size also is large the deviation is probably due to an incorrect
|
||||
// line slope.
|
||||
if (abs_delay_is_not_outlier || size_is_positive_outlier) {
|
||||
// Prevent updating with frames which have been congested by a large frame,
|
||||
// and therefore arrives almost at the same time as that frame.
|
||||
// This can occur when we receive a large frame (key frame) which has been
|
||||
// delayed. The next frame is of normal size (delta frame), and thus deltaFS
|
||||
// will be << 0. This removes all frame samples which arrives after a key
|
||||
// frame.
|
||||
double congestion_rejection_factor =
|
||||
config_.congestion_rejection_factor.value_or(
|
||||
kCongestionRejectionFactor);
|
||||
double filtered_max_frame_size_bytes =
|
||||
config_.MaxFrameSizePercentileEnabled()
|
||||
? max_frame_size_bytes_percentile_.GetFilteredValue()
|
||||
: max_frame_size_bytes_;
|
||||
bool is_not_congested =
|
||||
delta_frame_bytes >
|
||||
congestion_rejection_factor * filtered_max_frame_size_bytes;
|
||||
|
||||
if (is_not_congested || config_.estimate_noise_when_congested) {
|
||||
// Update the variance of the deviation from the line given by the Kalman
|
||||
// filter.
|
||||
EstimateRandomJitter(delay_deviation_ms);
|
||||
}
|
||||
if (is_not_congested) {
|
||||
// Neither a delay outlier nor a congested frame, so we can safely update
|
||||
// the Kalman filter with the sample.
|
||||
kalman_filter_.PredictAndUpdate(frame_delay.ms(), delta_frame_bytes,
|
||||
filtered_max_frame_size_bytes,
|
||||
var_noise_ms2_);
|
||||
}
|
||||
} else {
|
||||
// Delay outliers affect the noise estimate through a value equal to the
|
||||
// outlier rejection threshold.
|
||||
double num_stddev = (delay_deviation_ms >= 0) ? num_stddev_delay_outlier
|
||||
: -num_stddev_delay_outlier;
|
||||
EstimateRandomJitter(num_stddev * sqrt(var_noise_ms2_));
|
||||
}
|
||||
// Post process the total estimated jitter
|
||||
if (startup_count_ >= kFrameProcessingStartupCount) {
|
||||
PostProcessEstimate();
|
||||
} else {
|
||||
startup_count_++;
|
||||
}
|
||||
}
|
||||
|
||||
// Updates the nack/packet ratio.
|
||||
void JitterEstimator::FrameNacked() {
|
||||
if (nack_count_ < kNackLimit) {
|
||||
nack_count_++;
|
||||
}
|
||||
latest_nack_ = clock_->CurrentTime();
|
||||
}
|
||||
|
||||
void JitterEstimator::UpdateRtt(TimeDelta rtt) {
|
||||
rtt_filter_.Update(rtt);
|
||||
}
|
||||
|
||||
JitterEstimator::Config JitterEstimator::GetConfigForTest() const {
|
||||
return config_;
|
||||
}
|
||||
|
||||
// Estimates the random jitter by calculating the variance of the sample
|
||||
// distance from the line given by the Kalman filter.
|
||||
void JitterEstimator::EstimateRandomJitter(double d_dT) {
|
||||
Timestamp now = clock_->CurrentTime();
|
||||
if (last_update_time_.has_value()) {
|
||||
fps_counter_.AddSample((now - *last_update_time_).us());
|
||||
}
|
||||
last_update_time_ = now;
|
||||
|
||||
if (alpha_count_ == 0) {
|
||||
RTC_DCHECK_NOTREACHED();
|
||||
return;
|
||||
}
|
||||
double alpha =
|
||||
static_cast<double>(alpha_count_ - 1) / static_cast<double>(alpha_count_);
|
||||
alpha_count_++;
|
||||
if (alpha_count_ > kAlphaCountMax)
|
||||
alpha_count_ = kAlphaCountMax;
|
||||
|
||||
// In order to avoid a low frame rate stream to react slower to changes,
|
||||
// scale the alpha weight relative a 30 fps stream.
|
||||
Frequency fps = GetFrameRate();
|
||||
if (fps > Frequency::Zero()) {
|
||||
constexpr Frequency k30Fps = Frequency::Hertz(30);
|
||||
double rate_scale = k30Fps / fps;
|
||||
// At startup, there can be a lot of noise in the fps estimate.
|
||||
// Interpolate rate_scale linearly, from 1.0 at sample #1, to 30.0 / fps
|
||||
// at sample #kFrameProcessingStartupCount.
|
||||
if (alpha_count_ < kFrameProcessingStartupCount) {
|
||||
rate_scale = (alpha_count_ * rate_scale +
|
||||
(kFrameProcessingStartupCount - alpha_count_)) /
|
||||
kFrameProcessingStartupCount;
|
||||
}
|
||||
alpha = pow(alpha, rate_scale);
|
||||
}
|
||||
|
||||
double avg_noise_ms = alpha * avg_noise_ms_ + (1 - alpha) * d_dT;
|
||||
double var_noise_ms2 = alpha * var_noise_ms2_ + (1 - alpha) *
|
||||
(d_dT - avg_noise_ms_) *
|
||||
(d_dT - avg_noise_ms_);
|
||||
avg_noise_ms_ = avg_noise_ms;
|
||||
var_noise_ms2_ = var_noise_ms2;
|
||||
if (var_noise_ms2_ < 1.0) {
|
||||
// The variance should never be zero, since we might get stuck and consider
|
||||
// all samples as outliers.
|
||||
var_noise_ms2_ = 1.0;
|
||||
}
|
||||
}
|
||||
|
||||
double JitterEstimator::NoiseThreshold() const {
|
||||
double noise_threshold_ms =
|
||||
kNoiseStdDevs * sqrt(var_noise_ms2_) - kNoiseStdDevOffset;
|
||||
if (noise_threshold_ms < 1.0) {
|
||||
noise_threshold_ms = 1.0;
|
||||
}
|
||||
return noise_threshold_ms;
|
||||
}
|
||||
|
||||
// Calculates the current jitter estimate from the filtered estimates.
|
||||
TimeDelta JitterEstimator::CalculateEstimate() {
|
||||
// Using median- and percentile-filtered versions of the frame sizes may be
|
||||
// more robust than using sample mean-style estimates.
|
||||
double filtered_avg_frame_size_bytes =
|
||||
config_.avg_frame_size_median
|
||||
? avg_frame_size_median_bytes_.GetFilteredValue()
|
||||
: avg_frame_size_bytes_;
|
||||
double filtered_max_frame_size_bytes =
|
||||
config_.MaxFrameSizePercentileEnabled()
|
||||
? max_frame_size_bytes_percentile_.GetFilteredValue()
|
||||
: max_frame_size_bytes_;
|
||||
double worst_case_frame_size_deviation_bytes =
|
||||
filtered_max_frame_size_bytes - filtered_avg_frame_size_bytes;
|
||||
double ret_ms = kalman_filter_.GetFrameDelayVariationEstimateSizeBased(
|
||||
worst_case_frame_size_deviation_bytes) +
|
||||
NoiseThreshold();
|
||||
TimeDelta ret = TimeDelta::Millis(ret_ms);
|
||||
|
||||
// A very low estimate (or negative) is neglected.
|
||||
if (ret < kMinJitterEstimate) {
|
||||
ret = prev_estimate_.value_or(kMinJitterEstimate);
|
||||
// Sanity check to make sure that no other method has set `prev_estimate_`
|
||||
// to a value lower than `kMinJitterEstimate`.
|
||||
RTC_DCHECK_GE(ret, kMinJitterEstimate);
|
||||
} else if (ret > kMaxJitterEstimate) { // Sanity
|
||||
ret = kMaxJitterEstimate;
|
||||
}
|
||||
prev_estimate_ = ret;
|
||||
return ret;
|
||||
}
|
||||
|
||||
void JitterEstimator::PostProcessEstimate() {
|
||||
filter_jitter_estimate_ = CalculateEstimate();
|
||||
}
|
||||
|
||||
// Returns the current filtered estimate if available,
|
||||
// otherwise tries to calculate an estimate.
|
||||
TimeDelta JitterEstimator::GetJitterEstimate(
|
||||
double rtt_multiplier,
|
||||
absl::optional<TimeDelta> rtt_mult_add_cap) {
|
||||
TimeDelta jitter = CalculateEstimate() + OPERATING_SYSTEM_JITTER;
|
||||
Timestamp now = clock_->CurrentTime();
|
||||
|
||||
if (now - latest_nack_ > kNackCountTimeout)
|
||||
nack_count_ = 0;
|
||||
|
||||
if (filter_jitter_estimate_ > jitter)
|
||||
jitter = filter_jitter_estimate_;
|
||||
if (nack_count_ >= kNackLimit) {
|
||||
if (rtt_mult_add_cap.has_value()) {
|
||||
jitter += std::min(rtt_filter_.Rtt() * rtt_multiplier,
|
||||
rtt_mult_add_cap.value());
|
||||
} else {
|
||||
jitter += rtt_filter_.Rtt() * rtt_multiplier;
|
||||
}
|
||||
}
|
||||
|
||||
static const Frequency kJitterScaleLowThreshold = Frequency::Hertz(5);
|
||||
static const Frequency kJitterScaleHighThreshold = Frequency::Hertz(10);
|
||||
Frequency fps = GetFrameRate();
|
||||
// Ignore jitter for very low fps streams.
|
||||
if (fps < kJitterScaleLowThreshold) {
|
||||
if (fps.IsZero()) {
|
||||
return std::max(TimeDelta::Zero(), jitter);
|
||||
}
|
||||
return TimeDelta::Zero();
|
||||
}
|
||||
|
||||
// Semi-low frame rate; scale by factor linearly interpolated from 0.0 at
|
||||
// kJitterScaleLowThreshold to 1.0 at kJitterScaleHighThreshold.
|
||||
if (fps < kJitterScaleHighThreshold) {
|
||||
jitter = (1.0 / (kJitterScaleHighThreshold - kJitterScaleLowThreshold)) *
|
||||
(fps - kJitterScaleLowThreshold) * jitter;
|
||||
}
|
||||
|
||||
return std::max(TimeDelta::Zero(), jitter);
|
||||
}
|
||||
|
||||
Frequency JitterEstimator::GetFrameRate() const {
|
||||
TimeDelta mean_frame_period = TimeDelta::Micros(fps_counter_.ComputeMean());
|
||||
if (mean_frame_period <= TimeDelta::Zero())
|
||||
return Frequency::Zero();
|
||||
|
||||
Frequency fps = 1 / mean_frame_period;
|
||||
// Sanity check.
|
||||
RTC_DCHECK_GE(fps, Frequency::Zero());
|
||||
return std::min(fps, kMaxFramerateEstimate);
|
||||
}
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,218 @@
|
|||
/*
|
||||
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_VIDEO_CODING_TIMING_JITTER_ESTIMATOR_H_
|
||||
#define MODULES_VIDEO_CODING_TIMING_JITTER_ESTIMATOR_H_
|
||||
|
||||
#include <algorithm>
|
||||
#include <memory>
|
||||
#include <queue>
|
||||
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "absl/types/optional.h"
|
||||
#include "api/field_trials_view.h"
|
||||
#include "api/units/data_size.h"
|
||||
#include "api/units/frequency.h"
|
||||
#include "api/units/time_delta.h"
|
||||
#include "api/units/timestamp.h"
|
||||
#include "modules/video_coding/timing/frame_delay_variation_kalman_filter.h"
|
||||
#include "modules/video_coding/timing/rtt_filter.h"
|
||||
#include "rtc_base/experiments/struct_parameters_parser.h"
|
||||
#include "rtc_base/numerics/moving_percentile_filter.h"
|
||||
#include "rtc_base/rolling_accumulator.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
class Clock;
|
||||
|
||||
class JitterEstimator {
|
||||
public:
|
||||
// Configuration struct for statically overriding some constants and
|
||||
// behaviour, configurable through field trials.
|
||||
struct Config {
|
||||
static constexpr char kFieldTrialsKey[] = "WebRTC-JitterEstimatorConfig";
|
||||
|
||||
// Parses a field trial string and validates the values.
|
||||
static Config ParseAndValidate(absl::string_view field_trial);
|
||||
|
||||
std::unique_ptr<StructParametersParser> Parser() {
|
||||
// clang-format off
|
||||
return StructParametersParser::Create(
|
||||
"avg_frame_size_median", &avg_frame_size_median,
|
||||
"max_frame_size_percentile", &max_frame_size_percentile,
|
||||
"frame_size_window", &frame_size_window,
|
||||
"num_stddev_delay_clamp", &num_stddev_delay_clamp,
|
||||
"num_stddev_delay_outlier", &num_stddev_delay_outlier,
|
||||
"num_stddev_size_outlier", &num_stddev_size_outlier,
|
||||
"congestion_rejection_factor", &congestion_rejection_factor,
|
||||
"estimate_noise_when_congested", &estimate_noise_when_congested);
|
||||
// clang-format on
|
||||
}
|
||||
|
||||
bool MaxFrameSizePercentileEnabled() const {
|
||||
return max_frame_size_percentile.has_value();
|
||||
}
|
||||
|
||||
// If true, the "avg" frame size is calculated as the median over a window
|
||||
// of recent frame sizes.
|
||||
bool avg_frame_size_median = false;
|
||||
|
||||
// If set, the "max" frame size is calculated as this percentile over a
|
||||
// window of recent frame sizes.
|
||||
absl::optional<double> max_frame_size_percentile = absl::nullopt;
|
||||
|
||||
// The length of the percentile filters' window, in number of frames.
|
||||
absl::optional<int> frame_size_window = absl::nullopt;
|
||||
|
||||
// The incoming frame delay variation samples are clamped to be at most
|
||||
// this number of standard deviations away from zero.
|
||||
//
|
||||
// Increasing this value clamps fewer samples.
|
||||
absl::optional<double> num_stddev_delay_clamp = absl::nullopt;
|
||||
|
||||
// A (relative) frame delay variation sample is an outlier if its absolute
|
||||
// deviation from the Kalman filter model falls outside this number of
|
||||
// sample standard deviations.
|
||||
//
|
||||
// Increasing this value rejects fewer samples.
|
||||
absl::optional<double> num_stddev_delay_outlier = absl::nullopt;
|
||||
|
||||
// An (absolute) frame size sample is an outlier if its positive deviation
|
||||
// from the estimated average frame size falls outside this number of sample
|
||||
// standard deviations.
|
||||
//
|
||||
// Increasing this value rejects fewer samples.
|
||||
absl::optional<double> num_stddev_size_outlier = absl::nullopt;
|
||||
|
||||
// A (relative) frame size variation sample is deemed "congested", and is
|
||||
// thus rejected, if its value is less than this factor times the estimated
|
||||
// max frame size.
|
||||
//
|
||||
// Decreasing this value rejects fewer samples.
|
||||
absl::optional<double> congestion_rejection_factor = absl::nullopt;
|
||||
|
||||
// If true, the noise estimate will be updated for congestion rejected
|
||||
// frames. This is currently enabled by default, but that may not be optimal
|
||||
// since congested frames typically are not spread around the line with
|
||||
// Gaussian noise. (This is the whole reason for the congestion rejection!)
|
||||
bool estimate_noise_when_congested = true;
|
||||
};
|
||||
|
||||
JitterEstimator(Clock* clock, const FieldTrialsView& field_trials);
|
||||
JitterEstimator(const JitterEstimator&) = delete;
|
||||
JitterEstimator& operator=(const JitterEstimator&) = delete;
|
||||
~JitterEstimator();
|
||||
|
||||
// Resets the estimate to the initial state.
|
||||
void Reset();
|
||||
|
||||
// Updates the jitter estimate with the new data.
|
||||
//
|
||||
// Input:
|
||||
// - frame_delay : Delay-delta calculated by UTILDelayEstimate.
|
||||
// - frame_size : Frame size of the current frame.
|
||||
void UpdateEstimate(TimeDelta frame_delay, DataSize frame_size);
|
||||
|
||||
// Returns the current jitter estimate and adds an RTT dependent term in cases
|
||||
// of retransmission.
|
||||
// Input:
|
||||
// - rtt_multiplier : RTT param multiplier (when applicable).
|
||||
// - rtt_mult_add_cap : Multiplier cap from the RTTMultExperiment.
|
||||
//
|
||||
// Return value : Jitter estimate.
|
||||
TimeDelta GetJitterEstimate(double rtt_multiplier,
|
||||
absl::optional<TimeDelta> rtt_mult_add_cap);
|
||||
|
||||
// Updates the nack counter.
|
||||
void FrameNacked();
|
||||
|
||||
// Updates the RTT filter.
|
||||
//
|
||||
// Input:
|
||||
// - rtt : Round trip time.
|
||||
void UpdateRtt(TimeDelta rtt);
|
||||
|
||||
// Returns the configuration. Only to be used by unit tests.
|
||||
Config GetConfigForTest() const;
|
||||
|
||||
private:
|
||||
// Updates the random jitter estimate, i.e. the variance of the time
|
||||
// deviations from the line given by the Kalman filter.
|
||||
//
|
||||
// Input:
|
||||
// - d_dT : The deviation from the kalman estimate.
|
||||
void EstimateRandomJitter(double d_dT);
|
||||
|
||||
double NoiseThreshold() const;
|
||||
|
||||
// Calculates the current jitter estimate.
|
||||
//
|
||||
// Return value : The current jitter estimate.
|
||||
TimeDelta CalculateEstimate();
|
||||
|
||||
// Post process the calculated estimate.
|
||||
void PostProcessEstimate();
|
||||
|
||||
// Returns the estimated incoming frame rate.
|
||||
Frequency GetFrameRate() const;
|
||||
|
||||
// Configuration that may override some internals.
|
||||
const Config config_;
|
||||
|
||||
// Filters the {frame_delay_delta, frame_size_delta} measurements through
|
||||
// a linear Kalman filter.
|
||||
FrameDelayVariationKalmanFilter kalman_filter_;
|
||||
|
||||
// TODO(bugs.webrtc.org/14381): Update `avg_frame_size_bytes_` to DataSize
|
||||
// when api/units have sufficient precision.
|
||||
double avg_frame_size_bytes_; // Average frame size
|
||||
double var_frame_size_bytes2_; // Frame size variance. Unit is bytes^2.
|
||||
// Largest frame size received (descending with a factor kPsi).
|
||||
// Used by default.
|
||||
// TODO(bugs.webrtc.org/14381): Update `max_frame_size_bytes_` to DataSize
|
||||
// when api/units have sufficient precision.
|
||||
double max_frame_size_bytes_;
|
||||
// Percentile frame sized received (over a window). Only used if configured.
|
||||
MovingMedianFilter<int64_t> avg_frame_size_median_bytes_;
|
||||
MovingPercentileFilter<int64_t> max_frame_size_bytes_percentile_;
|
||||
// TODO(bugs.webrtc.org/14381): Update `startup_frame_size_sum_bytes_` to
|
||||
// DataSize when api/units have sufficient precision.
|
||||
double startup_frame_size_sum_bytes_;
|
||||
size_t startup_frame_size_count_;
|
||||
|
||||
absl::optional<Timestamp> last_update_time_;
|
||||
// The previously returned jitter estimate
|
||||
absl::optional<TimeDelta> prev_estimate_;
|
||||
// Frame size of the previous frame
|
||||
absl::optional<DataSize> prev_frame_size_;
|
||||
// Average of the random jitter. Unit is milliseconds.
|
||||
double avg_noise_ms_;
|
||||
// Variance of the time-deviation from the line. Unit is milliseconds^2.
|
||||
double var_noise_ms2_;
|
||||
size_t alpha_count_;
|
||||
// The filtered sum of jitter estimates
|
||||
TimeDelta filter_jitter_estimate_ = TimeDelta::Zero();
|
||||
|
||||
size_t startup_count_;
|
||||
// Time when the latest nack was seen
|
||||
Timestamp latest_nack_ = Timestamp::Zero();
|
||||
// Keeps track of the number of nacks received, but never goes above
|
||||
// kNackLimit.
|
||||
size_t nack_count_;
|
||||
RttFilter rtt_filter_;
|
||||
|
||||
// Tracks frame rates in microseconds.
|
||||
rtc::RollingAccumulator<uint64_t> fps_counter_;
|
||||
Clock* clock_;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_VIDEO_CODING_TIMING_JITTER_ESTIMATOR_H_
|
||||
|
|
@ -0,0 +1,161 @@
|
|||
/*
|
||||
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/video_coding/timing/rtt_filter.h"
|
||||
|
||||
#include <math.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#include "absl/algorithm/container.h"
|
||||
#include "absl/container/inlined_vector.h"
|
||||
#include "api/units/time_delta.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr TimeDelta kMaxRtt = TimeDelta::Seconds(3);
|
||||
constexpr uint32_t kFilterFactorMax = 35;
|
||||
constexpr double kJumpStddev = 2.5;
|
||||
constexpr double kDriftStdDev = 3.5;
|
||||
|
||||
} // namespace
|
||||
|
||||
RttFilter::RttFilter()
|
||||
: avg_rtt_(TimeDelta::Zero()),
|
||||
var_rtt_(0),
|
||||
max_rtt_(TimeDelta::Zero()),
|
||||
jump_buf_(kMaxDriftJumpCount, TimeDelta::Zero()),
|
||||
drift_buf_(kMaxDriftJumpCount, TimeDelta::Zero()) {
|
||||
Reset();
|
||||
}
|
||||
|
||||
void RttFilter::Reset() {
|
||||
got_non_zero_update_ = false;
|
||||
avg_rtt_ = TimeDelta::Zero();
|
||||
var_rtt_ = 0;
|
||||
max_rtt_ = TimeDelta::Zero();
|
||||
filt_fact_count_ = 1;
|
||||
absl::c_fill(jump_buf_, TimeDelta::Zero());
|
||||
absl::c_fill(drift_buf_, TimeDelta::Zero());
|
||||
}
|
||||
|
||||
void RttFilter::Update(TimeDelta rtt) {
|
||||
if (!got_non_zero_update_) {
|
||||
if (rtt.IsZero()) {
|
||||
return;
|
||||
}
|
||||
got_non_zero_update_ = true;
|
||||
}
|
||||
|
||||
// Sanity check
|
||||
if (rtt > kMaxRtt) {
|
||||
rtt = kMaxRtt;
|
||||
}
|
||||
|
||||
double filt_factor = 0;
|
||||
if (filt_fact_count_ > 1) {
|
||||
filt_factor = static_cast<double>(filt_fact_count_ - 1) / filt_fact_count_;
|
||||
}
|
||||
filt_fact_count_++;
|
||||
if (filt_fact_count_ > kFilterFactorMax) {
|
||||
// This prevents filt_factor from going above
|
||||
// (_filt_fact_max - 1) / filt_fact_max_,
|
||||
// e.g., filt_fact_max_ = 50 => filt_factor = 49/50 = 0.98
|
||||
filt_fact_count_ = kFilterFactorMax;
|
||||
}
|
||||
TimeDelta old_avg = avg_rtt_;
|
||||
int64_t old_var = var_rtt_;
|
||||
avg_rtt_ = filt_factor * avg_rtt_ + (1 - filt_factor) * rtt;
|
||||
int64_t delta_ms = (rtt - avg_rtt_).ms();
|
||||
var_rtt_ = filt_factor * var_rtt_ + (1 - filt_factor) * (delta_ms * delta_ms);
|
||||
max_rtt_ = std::max(rtt, max_rtt_);
|
||||
if (!JumpDetection(rtt) || !DriftDetection(rtt)) {
|
||||
// In some cases we don't want to update the statistics
|
||||
avg_rtt_ = old_avg;
|
||||
var_rtt_ = old_var;
|
||||
}
|
||||
}
|
||||
|
||||
bool RttFilter::JumpDetection(TimeDelta rtt) {
|
||||
TimeDelta diff_from_avg = avg_rtt_ - rtt;
|
||||
// Unit of var_rtt_ is ms^2.
|
||||
TimeDelta jump_threshold = TimeDelta::Millis(kJumpStddev * sqrt(var_rtt_));
|
||||
if (diff_from_avg.Abs() > jump_threshold) {
|
||||
bool positive_diff = diff_from_avg >= TimeDelta::Zero();
|
||||
if (!jump_buf_.empty() && positive_diff != last_jump_positive_) {
|
||||
// Since the signs differ the samples currently
|
||||
// in the buffer is useless as they represent a
|
||||
// jump in a different direction.
|
||||
jump_buf_.clear();
|
||||
}
|
||||
if (jump_buf_.size() < kMaxDriftJumpCount) {
|
||||
// Update the buffer used for the short time statistics.
|
||||
// The sign of the diff is used for updating the counter since
|
||||
// we want to use the same buffer for keeping track of when
|
||||
// the RTT jumps down and up.
|
||||
jump_buf_.push_back(rtt);
|
||||
last_jump_positive_ = positive_diff;
|
||||
}
|
||||
if (jump_buf_.size() >= kMaxDriftJumpCount) {
|
||||
// Detected an RTT jump
|
||||
ShortRttFilter(jump_buf_);
|
||||
filt_fact_count_ = kMaxDriftJumpCount + 1;
|
||||
jump_buf_.clear();
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
jump_buf_.clear();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool RttFilter::DriftDetection(TimeDelta rtt) {
|
||||
// Unit of sqrt of var_rtt_ is ms.
|
||||
TimeDelta drift_threshold = TimeDelta::Millis(kDriftStdDev * sqrt(var_rtt_));
|
||||
if (max_rtt_ - avg_rtt_ > drift_threshold) {
|
||||
if (drift_buf_.size() < kMaxDriftJumpCount) {
|
||||
// Update the buffer used for the short time statistics.
|
||||
drift_buf_.push_back(rtt);
|
||||
}
|
||||
if (drift_buf_.size() >= kMaxDriftJumpCount) {
|
||||
// Detected an RTT drift
|
||||
ShortRttFilter(drift_buf_);
|
||||
filt_fact_count_ = kMaxDriftJumpCount + 1;
|
||||
drift_buf_.clear();
|
||||
}
|
||||
} else {
|
||||
drift_buf_.clear();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void RttFilter::ShortRttFilter(const BufferList& buf) {
|
||||
RTC_DCHECK_EQ(buf.size(), kMaxDriftJumpCount);
|
||||
max_rtt_ = TimeDelta::Zero();
|
||||
avg_rtt_ = TimeDelta::Zero();
|
||||
for (const TimeDelta& rtt : buf) {
|
||||
if (rtt > max_rtt_) {
|
||||
max_rtt_ = rtt;
|
||||
}
|
||||
avg_rtt_ += rtt;
|
||||
}
|
||||
avg_rtt_ = avg_rtt_ / static_cast<double>(buf.size());
|
||||
}
|
||||
|
||||
TimeDelta RttFilter::Rtt() const {
|
||||
return max_rtt_;
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
/*
|
||||
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_VIDEO_CODING_TIMING_RTT_FILTER_H_
|
||||
#define MODULES_VIDEO_CODING_TIMING_RTT_FILTER_H_
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "absl/container/inlined_vector.h"
|
||||
#include "api/units/time_delta.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
class RttFilter {
|
||||
public:
|
||||
RttFilter();
|
||||
RttFilter(const RttFilter&) = delete;
|
||||
RttFilter& operator=(const RttFilter&) = delete;
|
||||
|
||||
// Resets the filter.
|
||||
void Reset();
|
||||
// Updates the filter with a new sample.
|
||||
void Update(TimeDelta rtt);
|
||||
// A getter function for the current RTT level.
|
||||
TimeDelta Rtt() const;
|
||||
|
||||
private:
|
||||
// The size of the drift and jump memory buffers
|
||||
// and thus also the detection threshold for these
|
||||
// detectors in number of samples.
|
||||
static constexpr int kMaxDriftJumpCount = 5;
|
||||
using BufferList = absl::InlinedVector<TimeDelta, kMaxDriftJumpCount>;
|
||||
|
||||
// Detects RTT jumps by comparing the difference between
|
||||
// samples and average to the standard deviation.
|
||||
// Returns true if the long time statistics should be updated
|
||||
// and false otherwise
|
||||
bool JumpDetection(TimeDelta rtt);
|
||||
|
||||
// Detects RTT drifts by comparing the difference between
|
||||
// max and average to the standard deviation.
|
||||
// Returns true if the long time statistics should be updated
|
||||
// and false otherwise
|
||||
bool DriftDetection(TimeDelta rtt);
|
||||
|
||||
// Computes the short time average and maximum of the vector buf.
|
||||
void ShortRttFilter(const BufferList& buf);
|
||||
|
||||
bool got_non_zero_update_;
|
||||
TimeDelta avg_rtt_;
|
||||
// Variance units are TimeDelta^2. Store as ms^2.
|
||||
int64_t var_rtt_;
|
||||
TimeDelta max_rtt_;
|
||||
uint32_t filt_fact_count_;
|
||||
bool last_jump_positive_ = false;
|
||||
BufferList jump_buf_;
|
||||
BufferList drift_buf_;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_VIDEO_CODING_TIMING_RTT_FILTER_H_
|
||||
|
|
@ -0,0 +1,173 @@
|
|||
/*
|
||||
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/video_coding/timing/timestamp_extrapolator.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "rtc_base/numerics/sequence_number_unwrapper.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr double kLambda = 1;
|
||||
constexpr uint32_t kStartUpFilterDelayInPackets = 2;
|
||||
constexpr double kAlarmThreshold = 60e3;
|
||||
// in timestamp ticks, i.e. 15 ms
|
||||
constexpr double kAccDrift = 6600;
|
||||
constexpr double kAccMaxError = 7000;
|
||||
constexpr double kP11 = 1e10;
|
||||
|
||||
} // namespace
|
||||
|
||||
TimestampExtrapolator::TimestampExtrapolator(Timestamp start)
|
||||
: start_(Timestamp::Zero()),
|
||||
prev_(Timestamp::Zero()),
|
||||
packet_count_(0),
|
||||
detector_accumulator_pos_(0),
|
||||
detector_accumulator_neg_(0) {
|
||||
Reset(start);
|
||||
}
|
||||
|
||||
void TimestampExtrapolator::Reset(Timestamp start) {
|
||||
start_ = start;
|
||||
prev_ = start_;
|
||||
first_unwrapped_timestamp_ = absl::nullopt;
|
||||
w_[0] = 90.0;
|
||||
w_[1] = 0;
|
||||
p_[0][0] = 1;
|
||||
p_[1][1] = kP11;
|
||||
p_[0][1] = p_[1][0] = 0;
|
||||
unwrapper_ = RtpTimestampUnwrapper();
|
||||
packet_count_ = 0;
|
||||
detector_accumulator_pos_ = 0;
|
||||
detector_accumulator_neg_ = 0;
|
||||
}
|
||||
|
||||
void TimestampExtrapolator::Update(Timestamp now, uint32_t ts90khz) {
|
||||
if (now - prev_ > TimeDelta::Seconds(10)) {
|
||||
// Ten seconds without a complete frame.
|
||||
// Reset the extrapolator
|
||||
Reset(now);
|
||||
} else {
|
||||
prev_ = now;
|
||||
}
|
||||
|
||||
// Remove offset to prevent badly scaled matrices
|
||||
const TimeDelta offset = now - start_;
|
||||
double t_ms = offset.ms();
|
||||
|
||||
int64_t unwrapped_ts90khz = unwrapper_.Unwrap(ts90khz);
|
||||
|
||||
if (!first_unwrapped_timestamp_) {
|
||||
// Make an initial guess of the offset,
|
||||
// should be almost correct since t_ms - start
|
||||
// should about zero at this time.
|
||||
w_[1] = -w_[0] * t_ms;
|
||||
first_unwrapped_timestamp_ = unwrapped_ts90khz;
|
||||
}
|
||||
|
||||
double residual =
|
||||
(static_cast<double>(unwrapped_ts90khz) - *first_unwrapped_timestamp_) -
|
||||
t_ms * w_[0] - w_[1];
|
||||
if (DelayChangeDetection(residual) &&
|
||||
packet_count_ >= kStartUpFilterDelayInPackets) {
|
||||
// A sudden change of average network delay has been detected.
|
||||
// Force the filter to adjust its offset parameter by changing
|
||||
// the offset uncertainty. Don't do this during startup.
|
||||
p_[1][1] = kP11;
|
||||
}
|
||||
|
||||
if (prev_unwrapped_timestamp_ &&
|
||||
unwrapped_ts90khz < prev_unwrapped_timestamp_) {
|
||||
// Drop reordered frames.
|
||||
return;
|
||||
}
|
||||
|
||||
// T = [t(k) 1]';
|
||||
// that = T'*w;
|
||||
// K = P*T/(lambda + T'*P*T);
|
||||
double K[2];
|
||||
K[0] = p_[0][0] * t_ms + p_[0][1];
|
||||
K[1] = p_[1][0] * t_ms + p_[1][1];
|
||||
double TPT = kLambda + t_ms * K[0] + K[1];
|
||||
K[0] /= TPT;
|
||||
K[1] /= TPT;
|
||||
// w = w + K*(ts(k) - that);
|
||||
w_[0] = w_[0] + K[0] * residual;
|
||||
w_[1] = w_[1] + K[1] * residual;
|
||||
// P = 1/lambda*(P - K*T'*P);
|
||||
double p00 =
|
||||
1 / kLambda * (p_[0][0] - (K[0] * t_ms * p_[0][0] + K[0] * p_[1][0]));
|
||||
double p01 =
|
||||
1 / kLambda * (p_[0][1] - (K[0] * t_ms * p_[0][1] + K[0] * p_[1][1]));
|
||||
p_[1][0] =
|
||||
1 / kLambda * (p_[1][0] - (K[1] * t_ms * p_[0][0] + K[1] * p_[1][0]));
|
||||
p_[1][1] =
|
||||
1 / kLambda * (p_[1][1] - (K[1] * t_ms * p_[0][1] + K[1] * p_[1][1]));
|
||||
p_[0][0] = p00;
|
||||
p_[0][1] = p01;
|
||||
prev_unwrapped_timestamp_ = unwrapped_ts90khz;
|
||||
if (packet_count_ < kStartUpFilterDelayInPackets) {
|
||||
packet_count_++;
|
||||
}
|
||||
}
|
||||
|
||||
absl::optional<Timestamp> TimestampExtrapolator::ExtrapolateLocalTime(
|
||||
uint32_t timestamp90khz) const {
|
||||
int64_t unwrapped_ts90khz = unwrapper_.PeekUnwrap(timestamp90khz);
|
||||
|
||||
if (!first_unwrapped_timestamp_) {
|
||||
return absl::nullopt;
|
||||
} else if (packet_count_ < kStartUpFilterDelayInPackets) {
|
||||
constexpr double kRtpTicksPerMs = 90;
|
||||
TimeDelta diff = TimeDelta::Millis(
|
||||
(unwrapped_ts90khz - *prev_unwrapped_timestamp_) / kRtpTicksPerMs);
|
||||
if (prev_.us() + diff.us() < 0) {
|
||||
// Prevent the construction of a negative Timestamp.
|
||||
// This scenario can occur when the RTP timestamp wraps around.
|
||||
return absl::nullopt;
|
||||
}
|
||||
return prev_ + diff;
|
||||
} else if (w_[0] < 1e-3) {
|
||||
return start_;
|
||||
} else {
|
||||
double timestampDiff = unwrapped_ts90khz - *first_unwrapped_timestamp_;
|
||||
TimeDelta diff = TimeDelta::Millis(
|
||||
static_cast<int64_t>((timestampDiff - w_[1]) / w_[0] + 0.5));
|
||||
if (start_.us() + diff.us() < 0) {
|
||||
// Prevent the construction of a negative Timestamp.
|
||||
// This scenario can occur when the RTP timestamp wraps around.
|
||||
return absl::nullopt;
|
||||
}
|
||||
return start_ + diff;
|
||||
}
|
||||
}
|
||||
|
||||
bool TimestampExtrapolator::DelayChangeDetection(double error) {
|
||||
// CUSUM detection of sudden delay changes
|
||||
error = (error > 0) ? std::min(error, kAccMaxError)
|
||||
: std::max(error, -kAccMaxError);
|
||||
detector_accumulator_pos_ =
|
||||
std::max(detector_accumulator_pos_ + error - kAccDrift, double{0});
|
||||
detector_accumulator_neg_ =
|
||||
std::min(detector_accumulator_neg_ + error + kAccDrift, double{0});
|
||||
if (detector_accumulator_pos_ > kAlarmThreshold ||
|
||||
detector_accumulator_neg_ < -kAlarmThreshold) {
|
||||
// Alarm
|
||||
detector_accumulator_pos_ = detector_accumulator_neg_ = 0;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
/*
|
||||
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_VIDEO_CODING_TIMING_TIMESTAMP_EXTRAPOLATOR_H_
|
||||
#define MODULES_VIDEO_CODING_TIMING_TIMESTAMP_EXTRAPOLATOR_H_
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "api/units/timestamp.h"
|
||||
#include "rtc_base/numerics/sequence_number_unwrapper.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
// Not thread safe.
|
||||
class TimestampExtrapolator {
|
||||
public:
|
||||
explicit TimestampExtrapolator(Timestamp start);
|
||||
void Update(Timestamp now, uint32_t ts90khz);
|
||||
absl::optional<Timestamp> ExtrapolateLocalTime(uint32_t timestamp90khz) const;
|
||||
void Reset(Timestamp start);
|
||||
|
||||
private:
|
||||
void CheckForWrapArounds(uint32_t ts90khz);
|
||||
bool DelayChangeDetection(double error);
|
||||
|
||||
double w_[2];
|
||||
double p_[2][2];
|
||||
Timestamp start_;
|
||||
Timestamp prev_;
|
||||
absl::optional<int64_t> first_unwrapped_timestamp_;
|
||||
RtpTimestampUnwrapper unwrapper_;
|
||||
absl::optional<int64_t> prev_unwrapped_timestamp_;
|
||||
uint32_t packet_count_;
|
||||
double detector_accumulator_pos_;
|
||||
double detector_accumulator_neg_;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_VIDEO_CODING_TIMING_TIMESTAMP_EXTRAPOLATOR_H_
|
||||
|
|
@ -0,0 +1,306 @@
|
|||
/*
|
||||
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/video_coding/timing/timing.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#include "api/units/time_delta.h"
|
||||
#include "modules/video_coding/timing/decode_time_percentile_filter.h"
|
||||
#include "modules/video_coding/timing/timestamp_extrapolator.h"
|
||||
#include "rtc_base/experiments/field_trial_parser.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "system_wrappers/include/clock.h"
|
||||
|
||||
namespace webrtc {
|
||||
namespace {
|
||||
|
||||
// Default pacing that is used for the low-latency renderer path.
|
||||
constexpr TimeDelta kZeroPlayoutDelayDefaultMinPacing = TimeDelta::Millis(8);
|
||||
constexpr TimeDelta kLowLatencyStreamMaxPlayoutDelayThreshold =
|
||||
TimeDelta::Millis(500);
|
||||
|
||||
void CheckDelaysValid(TimeDelta min_delay, TimeDelta max_delay) {
|
||||
if (min_delay > max_delay) {
|
||||
RTC_LOG(LS_ERROR)
|
||||
<< "Playout delays set incorrectly: min playout delay (" << min_delay
|
||||
<< ") > max playout delay (" << max_delay
|
||||
<< "). This is undefined behaviour. Application writers should "
|
||||
"ensure that the min delay is always less than or equals max "
|
||||
"delay. If trying to use the playout delay header extensions "
|
||||
"described in "
|
||||
"https://webrtc.googlesource.com/src/+/refs/heads/main/docs/"
|
||||
"native-code/rtp-hdrext/playout-delay/, be careful that a playout "
|
||||
"delay hint or A/V sync settings may have caused this conflict.";
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
VCMTiming::VCMTiming(Clock* clock, const FieldTrialsView& field_trials)
|
||||
: clock_(clock),
|
||||
ts_extrapolator_(
|
||||
std::make_unique<TimestampExtrapolator>(clock_->CurrentTime())),
|
||||
decode_time_filter_(std::make_unique<DecodeTimePercentileFilter>()),
|
||||
render_delay_(kDefaultRenderDelay),
|
||||
min_playout_delay_(TimeDelta::Zero()),
|
||||
max_playout_delay_(TimeDelta::Seconds(10)),
|
||||
jitter_delay_(TimeDelta::Zero()),
|
||||
current_delay_(TimeDelta::Zero()),
|
||||
prev_frame_timestamp_(0),
|
||||
num_decoded_frames_(0),
|
||||
zero_playout_delay_min_pacing_("min_pacing",
|
||||
kZeroPlayoutDelayDefaultMinPacing),
|
||||
last_decode_scheduled_(Timestamp::Zero()) {
|
||||
ParseFieldTrial({&zero_playout_delay_min_pacing_},
|
||||
field_trials.Lookup("WebRTC-ZeroPlayoutDelay"));
|
||||
}
|
||||
|
||||
void VCMTiming::Reset() {
|
||||
MutexLock lock(&mutex_);
|
||||
ts_extrapolator_->Reset(clock_->CurrentTime());
|
||||
decode_time_filter_ = std::make_unique<DecodeTimePercentileFilter>();
|
||||
render_delay_ = kDefaultRenderDelay;
|
||||
min_playout_delay_ = TimeDelta::Zero();
|
||||
jitter_delay_ = TimeDelta::Zero();
|
||||
current_delay_ = TimeDelta::Zero();
|
||||
prev_frame_timestamp_ = 0;
|
||||
}
|
||||
|
||||
void VCMTiming::set_render_delay(TimeDelta render_delay) {
|
||||
MutexLock lock(&mutex_);
|
||||
render_delay_ = render_delay;
|
||||
}
|
||||
|
||||
TimeDelta VCMTiming::min_playout_delay() const {
|
||||
MutexLock lock(&mutex_);
|
||||
return min_playout_delay_;
|
||||
}
|
||||
|
||||
void VCMTiming::set_min_playout_delay(TimeDelta min_playout_delay) {
|
||||
MutexLock lock(&mutex_);
|
||||
if (min_playout_delay_ != min_playout_delay) {
|
||||
CheckDelaysValid(min_playout_delay, max_playout_delay_);
|
||||
min_playout_delay_ = min_playout_delay;
|
||||
}
|
||||
}
|
||||
|
||||
void VCMTiming::set_max_playout_delay(TimeDelta max_playout_delay) {
|
||||
MutexLock lock(&mutex_);
|
||||
if (max_playout_delay_ != max_playout_delay) {
|
||||
CheckDelaysValid(min_playout_delay_, max_playout_delay);
|
||||
max_playout_delay_ = max_playout_delay;
|
||||
}
|
||||
}
|
||||
|
||||
void VCMTiming::SetJitterDelay(TimeDelta jitter_delay) {
|
||||
MutexLock lock(&mutex_);
|
||||
if (jitter_delay != jitter_delay_) {
|
||||
jitter_delay_ = jitter_delay;
|
||||
// When in initial state, set current delay to minimum delay.
|
||||
if (current_delay_.IsZero()) {
|
||||
current_delay_ = jitter_delay_;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void VCMTiming::UpdateCurrentDelay(uint32_t frame_timestamp) {
|
||||
MutexLock lock(&mutex_);
|
||||
TimeDelta target_delay = TargetDelayInternal();
|
||||
|
||||
if (current_delay_.IsZero()) {
|
||||
// Not initialized, set current delay to target.
|
||||
current_delay_ = target_delay;
|
||||
} else if (target_delay != current_delay_) {
|
||||
TimeDelta delay_diff = target_delay - current_delay_;
|
||||
// Never change the delay with more than 100 ms every second. If we're
|
||||
// changing the delay in too large steps we will get noticeable freezes. By
|
||||
// limiting the change we can increase the delay in smaller steps, which
|
||||
// will be experienced as the video is played in slow motion. When lowering
|
||||
// the delay the video will be played at a faster pace.
|
||||
TimeDelta max_change = TimeDelta::Zero();
|
||||
if (frame_timestamp < 0x0000ffff && prev_frame_timestamp_ > 0xffff0000) {
|
||||
// wrap
|
||||
max_change =
|
||||
TimeDelta::Millis(kDelayMaxChangeMsPerS *
|
||||
(frame_timestamp + (static_cast<int64_t>(1) << 32) -
|
||||
prev_frame_timestamp_) /
|
||||
90000);
|
||||
} else {
|
||||
max_change =
|
||||
TimeDelta::Millis(kDelayMaxChangeMsPerS *
|
||||
(frame_timestamp - prev_frame_timestamp_) / 90000);
|
||||
}
|
||||
|
||||
if (max_change <= TimeDelta::Zero()) {
|
||||
// Any changes less than 1 ms are truncated and will be postponed.
|
||||
// Negative change will be due to reordering and should be ignored.
|
||||
return;
|
||||
}
|
||||
delay_diff = std::max(delay_diff, -max_change);
|
||||
delay_diff = std::min(delay_diff, max_change);
|
||||
|
||||
current_delay_ = current_delay_ + delay_diff;
|
||||
}
|
||||
prev_frame_timestamp_ = frame_timestamp;
|
||||
}
|
||||
|
||||
void VCMTiming::UpdateCurrentDelay(Timestamp render_time,
|
||||
Timestamp actual_decode_time) {
|
||||
MutexLock lock(&mutex_);
|
||||
TimeDelta target_delay = TargetDelayInternal();
|
||||
TimeDelta delayed = (actual_decode_time - render_time) +
|
||||
EstimatedMaxDecodeTime() + render_delay_;
|
||||
|
||||
// Only consider `delayed` as negative by more than a few microseconds.
|
||||
if (delayed.ms() < 0) {
|
||||
return;
|
||||
}
|
||||
if (current_delay_ + delayed <= target_delay) {
|
||||
current_delay_ += delayed;
|
||||
} else {
|
||||
current_delay_ = target_delay;
|
||||
}
|
||||
}
|
||||
|
||||
void VCMTiming::StopDecodeTimer(TimeDelta decode_time, Timestamp now) {
|
||||
MutexLock lock(&mutex_);
|
||||
decode_time_filter_->AddTiming(decode_time.ms(), now.ms());
|
||||
RTC_DCHECK_GE(decode_time, TimeDelta::Zero());
|
||||
++num_decoded_frames_;
|
||||
}
|
||||
|
||||
void VCMTiming::IncomingTimestamp(uint32_t rtp_timestamp, Timestamp now) {
|
||||
MutexLock lock(&mutex_);
|
||||
ts_extrapolator_->Update(now, rtp_timestamp);
|
||||
}
|
||||
|
||||
Timestamp VCMTiming::RenderTime(uint32_t frame_timestamp, Timestamp now) const {
|
||||
MutexLock lock(&mutex_);
|
||||
return RenderTimeInternal(frame_timestamp, now);
|
||||
}
|
||||
|
||||
void VCMTiming::SetLastDecodeScheduledTimestamp(
|
||||
Timestamp last_decode_scheduled) {
|
||||
MutexLock lock(&mutex_);
|
||||
last_decode_scheduled_ = last_decode_scheduled;
|
||||
}
|
||||
|
||||
Timestamp VCMTiming::RenderTimeInternal(uint32_t frame_timestamp,
|
||||
Timestamp now) const {
|
||||
if (UseLowLatencyRendering()) {
|
||||
// Render as soon as possible or with low-latency renderer algorithm.
|
||||
return Timestamp::Zero();
|
||||
}
|
||||
// Note that TimestampExtrapolator::ExtrapolateLocalTime is not a const
|
||||
// method; it mutates the object's wraparound state.
|
||||
Timestamp estimated_complete_time =
|
||||
ts_extrapolator_->ExtrapolateLocalTime(frame_timestamp).value_or(now);
|
||||
|
||||
// Make sure the actual delay stays in the range of `min_playout_delay_`
|
||||
// and `max_playout_delay_`.
|
||||
TimeDelta actual_delay =
|
||||
current_delay_.Clamped(min_playout_delay_, max_playout_delay_);
|
||||
return estimated_complete_time + actual_delay;
|
||||
}
|
||||
|
||||
TimeDelta VCMTiming::EstimatedMaxDecodeTime() const {
|
||||
const int decode_time_ms = decode_time_filter_->RequiredDecodeTimeMs();
|
||||
RTC_DCHECK_GE(decode_time_ms, 0);
|
||||
return TimeDelta::Millis(decode_time_ms);
|
||||
}
|
||||
|
||||
TimeDelta VCMTiming::MaxWaitingTime(Timestamp render_time,
|
||||
Timestamp now,
|
||||
bool too_many_frames_queued) const {
|
||||
MutexLock lock(&mutex_);
|
||||
|
||||
if (render_time.IsZero() && zero_playout_delay_min_pacing_->us() > 0 &&
|
||||
min_playout_delay_.IsZero() && max_playout_delay_ > TimeDelta::Zero()) {
|
||||
// `render_time` == 0 indicates that the frame should be decoded and
|
||||
// rendered as soon as possible. However, the decoder can be choked if too
|
||||
// many frames are sent at once. Therefore, limit the interframe delay to
|
||||
// |zero_playout_delay_min_pacing_| unless too many frames are queued in
|
||||
// which case the frames are sent to the decoder at once.
|
||||
if (too_many_frames_queued) {
|
||||
return TimeDelta::Zero();
|
||||
}
|
||||
Timestamp earliest_next_decode_start_time =
|
||||
last_decode_scheduled_ + zero_playout_delay_min_pacing_;
|
||||
TimeDelta max_wait_time = now >= earliest_next_decode_start_time
|
||||
? TimeDelta::Zero()
|
||||
: earliest_next_decode_start_time - now;
|
||||
return max_wait_time;
|
||||
}
|
||||
return render_time - now - EstimatedMaxDecodeTime() - render_delay_;
|
||||
}
|
||||
|
||||
TimeDelta VCMTiming::TargetVideoDelay() const {
|
||||
MutexLock lock(&mutex_);
|
||||
return TargetDelayInternal();
|
||||
}
|
||||
|
||||
TimeDelta VCMTiming::TargetDelayInternal() const {
|
||||
return std::max(min_playout_delay_,
|
||||
jitter_delay_ + EstimatedMaxDecodeTime() + render_delay_);
|
||||
}
|
||||
|
||||
// TODO(crbug.com/webrtc/15197): Centralize delay arithmetic.
|
||||
TimeDelta VCMTiming::StatsTargetDelayInternal() const {
|
||||
TimeDelta stats_target_delay =
|
||||
TargetDelayInternal() - (EstimatedMaxDecodeTime() + render_delay_);
|
||||
return std::max(TimeDelta::Zero(), stats_target_delay);
|
||||
}
|
||||
|
||||
VideoFrame::RenderParameters VCMTiming::RenderParameters() const {
|
||||
MutexLock lock(&mutex_);
|
||||
return {.use_low_latency_rendering = UseLowLatencyRendering(),
|
||||
.max_composition_delay_in_frames = max_composition_delay_in_frames_};
|
||||
}
|
||||
|
||||
bool VCMTiming::UseLowLatencyRendering() const {
|
||||
// min_playout_delay_==0,
|
||||
// max_playout_delay_<=kLowLatencyStreamMaxPlayoutDelayThreshold indicates
|
||||
// that the low-latency path should be used, which means that frames should be
|
||||
// decoded and rendered as soon as possible.
|
||||
return min_playout_delay_.IsZero() &&
|
||||
max_playout_delay_ <= kLowLatencyStreamMaxPlayoutDelayThreshold;
|
||||
}
|
||||
|
||||
VCMTiming::VideoDelayTimings VCMTiming::GetTimings() const {
|
||||
MutexLock lock(&mutex_);
|
||||
return VideoDelayTimings{
|
||||
.num_decoded_frames = num_decoded_frames_,
|
||||
.minimum_delay = jitter_delay_,
|
||||
.estimated_max_decode_time = EstimatedMaxDecodeTime(),
|
||||
.render_delay = render_delay_,
|
||||
.min_playout_delay = min_playout_delay_,
|
||||
.max_playout_delay = max_playout_delay_,
|
||||
.target_delay = StatsTargetDelayInternal(),
|
||||
.current_delay = current_delay_};
|
||||
}
|
||||
|
||||
void VCMTiming::SetTimingFrameInfo(const TimingFrameInfo& info) {
|
||||
MutexLock lock(&mutex_);
|
||||
timing_frame_info_.emplace(info);
|
||||
}
|
||||
|
||||
absl::optional<TimingFrameInfo> VCMTiming::GetTimingFrameInfo() {
|
||||
MutexLock lock(&mutex_);
|
||||
return timing_frame_info_;
|
||||
}
|
||||
|
||||
void VCMTiming::SetMaxCompositionDelayInFrames(
|
||||
absl::optional<int> max_composition_delay_in_frames) {
|
||||
MutexLock lock(&mutex_);
|
||||
max_composition_delay_in_frames_ = max_composition_delay_in_frames;
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,174 @@
|
|||
/*
|
||||
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_VIDEO_CODING_TIMING_TIMING_H_
|
||||
#define MODULES_VIDEO_CODING_TIMING_TIMING_H_
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "api/field_trials_view.h"
|
||||
#include "api/units/time_delta.h"
|
||||
#include "api/video/video_frame.h"
|
||||
#include "api/video/video_timing.h"
|
||||
#include "modules/video_coding/timing/decode_time_percentile_filter.h"
|
||||
#include "modules/video_coding/timing/timestamp_extrapolator.h"
|
||||
#include "rtc_base/experiments/field_trial_parser.h"
|
||||
#include "rtc_base/synchronization/mutex.h"
|
||||
#include "rtc_base/thread_annotations.h"
|
||||
#include "system_wrappers/include/clock.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
class VCMTiming {
|
||||
public:
|
||||
struct VideoDelayTimings {
|
||||
size_t num_decoded_frames;
|
||||
// Pre-decode delay added to smooth out frame delay variation ("jitter")
|
||||
// caused by the network. The target delay will be no smaller than this
|
||||
// delay, thus it is called `minimum_delay`.
|
||||
TimeDelta minimum_delay;
|
||||
// Estimated time needed to decode a video frame. Obtained as the 95th
|
||||
// percentile decode time over a recent time window.
|
||||
TimeDelta estimated_max_decode_time;
|
||||
// Post-decode delay added to smooth out frame delay variation caused by
|
||||
// decoding and rendering. Set to a constant.
|
||||
TimeDelta render_delay;
|
||||
// Minimum total delay used when determining render time for a frame.
|
||||
// Obtained from API, `playout-delay` RTP header extension, or A/V sync.
|
||||
TimeDelta min_playout_delay;
|
||||
// Maximum total delay used when determining render time for a frame.
|
||||
// Obtained from `playout-delay` RTP header extension.
|
||||
TimeDelta max_playout_delay;
|
||||
// Target total delay. Obtained from all the elements above.
|
||||
TimeDelta target_delay;
|
||||
// Current total delay. Obtained by smoothening the `target_delay`.
|
||||
TimeDelta current_delay;
|
||||
};
|
||||
|
||||
static constexpr TimeDelta kDefaultRenderDelay = TimeDelta::Millis(10);
|
||||
static constexpr int kDelayMaxChangeMsPerS = 100;
|
||||
|
||||
VCMTiming(Clock* clock, const FieldTrialsView& field_trials);
|
||||
virtual ~VCMTiming() = default;
|
||||
|
||||
// Resets the timing to the initial state.
|
||||
void Reset();
|
||||
|
||||
// Set the amount of time needed to render an image. Defaults to 10 ms.
|
||||
void set_render_delay(TimeDelta render_delay);
|
||||
|
||||
// Set the minimum time the video must be delayed on the receiver to
|
||||
// get the desired jitter buffer level.
|
||||
void SetJitterDelay(TimeDelta required_delay);
|
||||
|
||||
// Set/get the minimum playout delay from capture to render.
|
||||
TimeDelta min_playout_delay() const;
|
||||
void set_min_playout_delay(TimeDelta min_playout_delay);
|
||||
|
||||
// Set/get the maximum playout delay from capture to render in ms.
|
||||
void set_max_playout_delay(TimeDelta max_playout_delay);
|
||||
|
||||
// Increases or decreases the current delay to get closer to the target delay.
|
||||
// Calculates how long it has been since the previous call to this function,
|
||||
// and increases/decreases the delay in proportion to the time difference.
|
||||
void UpdateCurrentDelay(uint32_t frame_timestamp);
|
||||
|
||||
// Increases or decreases the current delay to get closer to the target delay.
|
||||
// Given the actual decode time in ms and the render time in ms for a frame,
|
||||
// this function calculates how late the frame is and increases the delay
|
||||
// accordingly.
|
||||
void UpdateCurrentDelay(Timestamp render_time, Timestamp actual_decode_time);
|
||||
|
||||
// Stops the decoder timer, should be called when the decoder returns a frame
|
||||
// or when the decoded frame callback is called.
|
||||
void StopDecodeTimer(TimeDelta decode_time, Timestamp now);
|
||||
|
||||
// Used to report that a frame is passed to decoding. Updates the timestamp
|
||||
// filter which is used to map between timestamps and receiver system time.
|
||||
virtual void IncomingTimestamp(uint32_t rtp_timestamp,
|
||||
Timestamp last_packet_time);
|
||||
|
||||
// Returns the receiver system time when the frame with timestamp
|
||||
// `frame_timestamp` should be rendered, assuming that the system time
|
||||
// currently is `now`.
|
||||
virtual Timestamp RenderTime(uint32_t frame_timestamp, Timestamp now) const;
|
||||
|
||||
// Returns the maximum time in ms that we can wait for a frame to become
|
||||
// complete before we must pass it to the decoder. render_time==0 indicates
|
||||
// that the frames should be processed as quickly as possible, with possibly
|
||||
// only a small delay added to make sure that the decoder is not overloaded.
|
||||
// In this case, the parameter too_many_frames_queued is used to signal that
|
||||
// the decode queue is full and that the frame should be decoded as soon as
|
||||
// possible.
|
||||
virtual TimeDelta MaxWaitingTime(Timestamp render_time,
|
||||
Timestamp now,
|
||||
bool too_many_frames_queued) const;
|
||||
|
||||
// Returns the current target delay which is required delay + decode time +
|
||||
// render delay.
|
||||
TimeDelta TargetVideoDelay() const;
|
||||
|
||||
// Return current timing information.
|
||||
VideoDelayTimings GetTimings() const;
|
||||
|
||||
void SetTimingFrameInfo(const TimingFrameInfo& info);
|
||||
absl::optional<TimingFrameInfo> GetTimingFrameInfo();
|
||||
|
||||
void SetMaxCompositionDelayInFrames(
|
||||
absl::optional<int> max_composition_delay_in_frames);
|
||||
|
||||
VideoFrame::RenderParameters RenderParameters() const;
|
||||
|
||||
// Updates the last time a frame was scheduled for decoding.
|
||||
void SetLastDecodeScheduledTimestamp(Timestamp last_decode_scheduled);
|
||||
|
||||
private:
|
||||
TimeDelta EstimatedMaxDecodeTime() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
Timestamp RenderTimeInternal(uint32_t frame_timestamp, Timestamp now) const
|
||||
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
TimeDelta TargetDelayInternal() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
TimeDelta StatsTargetDelayInternal() const
|
||||
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
bool UseLowLatencyRendering() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
|
||||
mutable Mutex mutex_;
|
||||
Clock* const clock_;
|
||||
const std::unique_ptr<TimestampExtrapolator> ts_extrapolator_
|
||||
RTC_PT_GUARDED_BY(mutex_);
|
||||
std::unique_ptr<DecodeTimePercentileFilter> decode_time_filter_
|
||||
RTC_GUARDED_BY(mutex_) RTC_PT_GUARDED_BY(mutex_);
|
||||
TimeDelta render_delay_ RTC_GUARDED_BY(mutex_);
|
||||
// Best-effort playout delay range for frames from capture to render.
|
||||
// The receiver tries to keep the delay between `min_playout_delay_ms_`
|
||||
// and `max_playout_delay_ms_` taking the network jitter into account.
|
||||
// A special case is where min_playout_delay_ms_ = max_playout_delay_ms_ = 0,
|
||||
// in which case the receiver tries to play the frames as they arrive.
|
||||
TimeDelta min_playout_delay_ RTC_GUARDED_BY(mutex_);
|
||||
TimeDelta max_playout_delay_ RTC_GUARDED_BY(mutex_);
|
||||
TimeDelta jitter_delay_ RTC_GUARDED_BY(mutex_);
|
||||
TimeDelta current_delay_ RTC_GUARDED_BY(mutex_);
|
||||
uint32_t prev_frame_timestamp_ RTC_GUARDED_BY(mutex_);
|
||||
absl::optional<TimingFrameInfo> timing_frame_info_ RTC_GUARDED_BY(mutex_);
|
||||
size_t num_decoded_frames_ RTC_GUARDED_BY(mutex_);
|
||||
absl::optional<int> max_composition_delay_in_frames_ RTC_GUARDED_BY(mutex_);
|
||||
// Set by the field trial WebRTC-ZeroPlayoutDelay. The parameter min_pacing
|
||||
// determines the minimum delay between frames scheduled for decoding that is
|
||||
// used when min playout delay=0 and max playout delay>=0.
|
||||
FieldTrialParameter<TimeDelta> zero_playout_delay_min_pacing_
|
||||
RTC_GUARDED_BY(mutex_);
|
||||
// Timestamp at which the last frame was scheduled to be sent to the decoder.
|
||||
// Used only when the RTP header extension playout delay is set to min=0 ms
|
||||
// which is indicated by a render time set to 0.
|
||||
Timestamp last_decode_scheduled_ RTC_GUARDED_BY(mutex_);
|
||||
};
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_VIDEO_CODING_TIMING_TIMING_H_
|
||||
Loading…
Add table
Add a link
Reference in a new issue