Repo created
This commit is contained in:
parent
81b91f4139
commit
f8c34fa5ee
22732 changed files with 4815320 additions and 2 deletions
|
|
@ -0,0 +1,39 @@
|
|||
#ifndef TGCALLS_AUDIO_DEVICE_MODULE_IOS
|
||||
#define TGCALLS_AUDIO_DEVICE_MODULE_IOS
|
||||
|
||||
#include "platform/PlatformInterface.h"
|
||||
|
||||
namespace tgcalls {
|
||||
|
||||
class AudioDeviceModuleIOS : public DefaultWrappedAudioDeviceModule {
|
||||
public:
|
||||
AudioDeviceModuleIOS(webrtc::scoped_refptr<webrtc::AudioDeviceModule> impl) :
|
||||
DefaultWrappedAudioDeviceModule(impl) {
|
||||
}
|
||||
|
||||
virtual ~AudioDeviceModuleIOS() {
|
||||
}
|
||||
|
||||
virtual int32_t StopPlayout() override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
virtual int32_t StopRecording() override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
virtual int32_t Terminate() override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
virtual void Stop() override {
|
||||
WrappedInstance()->StopPlayout();
|
||||
WrappedInstance()->StopRecording();
|
||||
WrappedInstance()->Terminate();
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
|
||||
#ifndef TGCALLS_AUDIO_DEVICE_MODULE_MACOS
|
||||
#define TGCALLS_AUDIO_DEVICE_MODULE_MACOS
|
||||
|
||||
#include "platform/PlatformInterface.h"
|
||||
|
||||
namespace tgcalls {
|
||||
|
||||
class AudioDeviceModuleMacos : public DefaultWrappedAudioDeviceModule {
|
||||
public:
|
||||
AudioDeviceModuleMacos(webrtc::scoped_refptr<webrtc::AudioDeviceModule> impl) :
|
||||
DefaultWrappedAudioDeviceModule(impl) {
|
||||
}
|
||||
|
||||
virtual ~AudioDeviceModuleMacos() {
|
||||
}
|
||||
virtual int32_t SetStereoPlayout(bool enable) override {
|
||||
return WrappedInstance()->SetStereoPlayout(enable);
|
||||
}
|
||||
|
||||
|
||||
virtual void Stop() override {
|
||||
WrappedInstance()->StopPlayout();
|
||||
WrappedInstance()->StopRecording();
|
||||
WrappedInstance()->Terminate();
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
#ifndef TGCALLS_CUSTOM_EXTERNAL_CAPTURER_H
|
||||
#define TGCALLS_CUSTOM_EXTERNAL_CAPTURER_H
|
||||
#ifdef WEBRTC_IOS
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
|
||||
#include <memory>
|
||||
#include "api/scoped_refptr.h"
|
||||
#include "api/media_stream_interface.h"
|
||||
#import "base/RTCVideoFrame.h"
|
||||
#include "Instance.h"
|
||||
|
||||
@interface CustomExternalCapturer : NSObject
|
||||
|
||||
- (instancetype)initWithSource:(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source;
|
||||
|
||||
+ (void)passPixelBuffer:(CVPixelBufferRef)pixelBuffer sampleBufferReference:(CMSampleBufferRef)sampleBufferReference rotation:(RTCVideoRotation)rotation toSource:(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source croppingBuffer:(std::vector<uint8_t> &)croppingBuffer;
|
||||
|
||||
@end
|
||||
#endif // WEBRTC_IOS
|
||||
#endif
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
#include "CustomExternalCapturer.h"
|
||||
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
|
||||
#include "rtc_base/logging.h"
|
||||
#import "base/RTCLogging.h"
|
||||
#import "base/RTCVideoFrameBuffer.h"
|
||||
#import "TGRTCCVPixelBuffer.h"
|
||||
#import "sdk/objc/native/src/objc_video_track_source.h"
|
||||
#import "sdk/objc/native/src/objc_frame_buffer.h"
|
||||
//#import "api/video_track_source_proxy.h"
|
||||
|
||||
#import "helpers/UIDevice+RTCDevice.h"
|
||||
|
||||
#import "helpers/AVCaptureSession+DevicePosition.h"
|
||||
#import "helpers/RTCDispatcher+Private.h"
|
||||
#import "base/RTCVideoFrame.h"
|
||||
|
||||
#include "common_video/libyuv/include/webrtc_libyuv.h"
|
||||
#include "rtc_base/checks.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "third_party/libyuv/include/libyuv.h"
|
||||
#include "pc/video_track_source_proxy.h"
|
||||
|
||||
#include "DarwinVideoSource.h"
|
||||
|
||||
static const int64_t kNanosecondsPerSecond = 1000000000;
|
||||
|
||||
static tgcalls::DarwinVideoTrackSource *getObjCVideoSource(const webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> nativeSource) {
|
||||
webrtc::VideoTrackSourceProxy *proxy_source =
|
||||
static_cast<webrtc::VideoTrackSourceProxy *>(nativeSource.get());
|
||||
return static_cast<tgcalls::DarwinVideoTrackSource *>(proxy_source->internal());
|
||||
}
|
||||
|
||||
@interface CustomExternalCapturer () {
|
||||
webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> _source;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@implementation CustomExternalCapturer
|
||||
|
||||
- (instancetype)initWithSource:(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source {
|
||||
self = [super init];
|
||||
if (self != nil) {
|
||||
_source = source;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
}
|
||||
|
||||
+ (void)passPixelBuffer:(CVPixelBufferRef)pixelBuffer sampleBufferReference:(CMSampleBufferRef)sampleBufferReference rotation:(RTCVideoRotation)rotation toSource:(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source croppingBuffer:(std::vector<uint8_t> &)croppingBuffer {
|
||||
TGRTCCVPixelBuffer *rtcPixelBuffer = [[TGRTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer];
|
||||
if (sampleBufferReference) {
|
||||
[rtcPixelBuffer storeSampleBufferReference:sampleBufferReference];
|
||||
}
|
||||
rtcPixelBuffer.deviceRelativeVideoRotation = -1;
|
||||
|
||||
int width = rtcPixelBuffer.width;
|
||||
int height = rtcPixelBuffer.height;
|
||||
|
||||
width -= width % 4;
|
||||
height -= height % 4;
|
||||
|
||||
if (width != rtcPixelBuffer.width || height != rtcPixelBuffer.height) {
|
||||
CVPixelBufferRef outputPixelBufferRef = NULL;
|
||||
OSType pixelFormat = CVPixelBufferGetPixelFormatType(rtcPixelBuffer.pixelBuffer);
|
||||
CVPixelBufferCreate(NULL, width, height, pixelFormat, NULL, &outputPixelBufferRef);
|
||||
if (outputPixelBufferRef) {
|
||||
int bufferSize = [rtcPixelBuffer bufferSizeForCroppingAndScalingToWidth:width height:height];
|
||||
if (croppingBuffer.size() < bufferSize) {
|
||||
croppingBuffer.resize(bufferSize);
|
||||
}
|
||||
if ([rtcPixelBuffer cropAndScaleTo:outputPixelBufferRef withTempBuffer:croppingBuffer.data()]) {
|
||||
rtcPixelBuffer = [[TGRTCCVPixelBuffer alloc] initWithPixelBuffer:outputPixelBufferRef];
|
||||
rtcPixelBuffer.deviceRelativeVideoRotation = -1;
|
||||
}
|
||||
CVPixelBufferRelease(outputPixelBufferRef);
|
||||
}
|
||||
}
|
||||
|
||||
int64_t timeStampNs = CACurrentMediaTime() * kNanosecondsPerSecond;
|
||||
RTCVideoFrame *videoFrame = [[RTCVideoFrame alloc] initWithBuffer:(id<RTCVideoFrameBuffer>)[rtcPixelBuffer toI420] rotation:rotation timeStampNs:timeStampNs];
|
||||
|
||||
getObjCVideoSource(source)->OnCapturedFrame(videoFrame);
|
||||
}
|
||||
|
||||
@end
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,200 @@
|
|||
/*
|
||||
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*
|
||||
*/
|
||||
|
||||
#ifndef TGCALLS_CUSTOM_SIMULCAST_ENCODER_ADAPTER_H_
|
||||
#define TGCALLS_CUSTOM_SIMULCAST_ENCODER_ADAPTER_H_
|
||||
|
||||
#include <atomic>
|
||||
#include <list>
|
||||
#include <memory>
|
||||
#include <stack>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "api/fec_controller_override.h"
|
||||
#include "api/sequence_checker.h"
|
||||
#include "api/video_codecs/sdp_video_format.h"
|
||||
#include "api/video_codecs/video_encoder.h"
|
||||
#include "api/video_codecs/video_encoder_factory.h"
|
||||
#include "common_video/framerate_controller.h"
|
||||
#include "modules/video_coding/include/video_codec_interface.h"
|
||||
#include "rtc_base/experiments/encoder_info_settings.h"
|
||||
#include "rtc_base/system/no_unique_address.h"
|
||||
#include "rtc_base/system/rtc_export.h"
|
||||
#include "api/field_trials_view.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
// CustomSimulcastEncoderAdapter implements simulcast support by creating multiple
|
||||
// webrtc::VideoEncoder instances with the given VideoEncoderFactory.
|
||||
// The object is created and destroyed on the worker thread, but all public
|
||||
// interfaces should be called from the encoder task queue.
|
||||
class RTC_EXPORT CustomSimulcastEncoderAdapter : public VideoEncoder {
|
||||
public:
|
||||
// TODO(bugs.webrtc.org/11000): Remove when downstream usage is gone.
|
||||
CustomSimulcastEncoderAdapter(VideoEncoderFactory* primarty_factory,
|
||||
const SdpVideoFormat& format);
|
||||
// `primary_factory` produces the first-choice encoders to use.
|
||||
// `fallback_factory`, if non-null, is used to create fallback encoder that
|
||||
// will be used if InitEncode() fails for the primary encoder.
|
||||
CustomSimulcastEncoderAdapter(VideoEncoderFactory* primary_factory,
|
||||
VideoEncoderFactory* hardware_factory,
|
||||
const SdpVideoFormat& format,
|
||||
const FieldTrialsView& field_trials);
|
||||
~CustomSimulcastEncoderAdapter() override;
|
||||
|
||||
// Implements VideoEncoder.
|
||||
void SetFecControllerOverride(
|
||||
FecControllerOverride* fec_controller_override) override;
|
||||
int Release() override;
|
||||
int InitEncode(const VideoCodec* codec_settings,
|
||||
const VideoEncoder::Settings& settings) override;
|
||||
int Encode(const VideoFrame& input_image,
|
||||
const std::vector<VideoFrameType>* frame_types) override;
|
||||
int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override;
|
||||
void SetRates(const RateControlParameters& parameters) override;
|
||||
void OnPacketLossRateUpdate(float packet_loss_rate) override;
|
||||
void OnRttUpdate(int64_t rtt_ms) override;
|
||||
void OnLossNotification(const LossNotification& loss_notification) override;
|
||||
|
||||
EncoderInfo GetEncoderInfo() const override;
|
||||
|
||||
private:
|
||||
class EncoderContext {
|
||||
public:
|
||||
EncoderContext(std::unique_ptr<VideoEncoder> encoder,
|
||||
bool prefer_temporal_support,
|
||||
VideoEncoder::EncoderInfo primary_info,
|
||||
VideoEncoder::EncoderInfo fallback_info);
|
||||
EncoderContext& operator=(EncoderContext&&) = delete;
|
||||
|
||||
VideoEncoder& encoder() { return *encoder_; }
|
||||
bool prefer_temporal_support() { return prefer_temporal_support_; }
|
||||
void Release();
|
||||
|
||||
const VideoEncoder::EncoderInfo& PrimaryInfo() { return primary_info_; }
|
||||
|
||||
const VideoEncoder::EncoderInfo& FallbackInfo() { return fallback_info_; }
|
||||
|
||||
private:
|
||||
std::unique_ptr<VideoEncoder> encoder_;
|
||||
bool prefer_temporal_support_;
|
||||
const VideoEncoder::EncoderInfo primary_info_;
|
||||
const VideoEncoder::EncoderInfo fallback_info_;
|
||||
};
|
||||
|
||||
class StreamContext : public EncodedImageCallback {
|
||||
public:
|
||||
StreamContext(CustomSimulcastEncoderAdapter* parent,
|
||||
std::unique_ptr<EncoderContext> encoder_context,
|
||||
std::unique_ptr<FramerateController> framerate_controller,
|
||||
int stream_idx,
|
||||
uint16_t width,
|
||||
uint16_t height,
|
||||
bool send_stream);
|
||||
StreamContext(StreamContext&& rhs);
|
||||
StreamContext& operator=(StreamContext&&) = delete;
|
||||
~StreamContext() override;
|
||||
|
||||
Result OnEncodedImage(
|
||||
const EncodedImage& encoded_image,
|
||||
const CodecSpecificInfo* codec_specific_info) override;
|
||||
void OnDroppedFrame(DropReason reason) override;
|
||||
|
||||
VideoEncoder& encoder() { return encoder_context_->encoder(); }
|
||||
const VideoEncoder& encoder() const { return encoder_context_->encoder(); }
|
||||
int stream_idx() const { return stream_idx_; }
|
||||
uint16_t width() const { return width_; }
|
||||
uint16_t height() const { return height_; }
|
||||
bool is_keyframe_needed() const {
|
||||
return !is_paused_ && is_keyframe_needed_;
|
||||
}
|
||||
void set_is_keyframe_needed() { is_keyframe_needed_ = true; }
|
||||
bool is_paused() const { return is_paused_; }
|
||||
void set_is_paused(bool is_paused) { is_paused_ = is_paused; }
|
||||
absl::optional<double> target_fps() const {
|
||||
return framerate_controller_ == nullptr
|
||||
? absl::nullopt
|
||||
: absl::optional<double>(
|
||||
framerate_controller_->GetMaxFramerate());
|
||||
}
|
||||
|
||||
std::unique_ptr<EncoderContext> ReleaseEncoderContext() &&;
|
||||
void OnKeyframe(Timestamp timestamp);
|
||||
bool ShouldDropFrame(Timestamp timestamp);
|
||||
|
||||
private:
|
||||
CustomSimulcastEncoderAdapter* const parent_;
|
||||
std::unique_ptr<EncoderContext> encoder_context_;
|
||||
std::unique_ptr<FramerateController> framerate_controller_;
|
||||
const int stream_idx_;
|
||||
const uint16_t width_;
|
||||
const uint16_t height_;
|
||||
bool is_keyframe_needed_;
|
||||
bool is_paused_;
|
||||
};
|
||||
|
||||
bool Initialized() const;
|
||||
|
||||
void DestroyStoredEncoders();
|
||||
|
||||
// This method creates encoder. May reuse previously created encoders from
|
||||
// `cached_encoder_contexts_`. It's const because it's used from
|
||||
// const GetEncoderInfo().
|
||||
std::unique_ptr<EncoderContext> FetchOrCreateEncoderContext(
|
||||
bool is_lowest_quality_stream, bool is_highest_quality_stream) const;
|
||||
|
||||
webrtc::VideoCodec MakeStreamCodec(const webrtc::VideoCodec& codec,
|
||||
int stream_idx,
|
||||
uint32_t start_bitrate_kbps,
|
||||
bool is_lowest_quality_stream,
|
||||
bool is_highest_quality_stream);
|
||||
|
||||
EncodedImageCallback::Result OnEncodedImage(
|
||||
size_t stream_idx,
|
||||
const EncodedImage& encoded_image,
|
||||
const CodecSpecificInfo* codec_specific_info);
|
||||
|
||||
void OnDroppedFrame(size_t stream_idx);
|
||||
|
||||
void OverrideFromFieldTrial(VideoEncoder::EncoderInfo* info) const;
|
||||
|
||||
std::atomic<int> inited_;
|
||||
VideoEncoderFactory* const primary_encoder_factory_;
|
||||
VideoEncoderFactory* const hardware_encoder_factory_;
|
||||
const SdpVideoFormat video_format_;
|
||||
VideoCodec codec_;
|
||||
int total_streams_count_;
|
||||
bool bypass_mode_;
|
||||
std::vector<StreamContext> stream_contexts_;
|
||||
EncodedImageCallback* encoded_complete_callback_;
|
||||
|
||||
// Used for checking the single-threaded access of the encoder interface.
|
||||
RTC_NO_UNIQUE_ADDRESS SequenceChecker encoder_queue_;
|
||||
|
||||
// Store previously created and released encoders , so they don't have to be
|
||||
// recreated. Remaining encoders are destroyed by the destructor.
|
||||
// Marked as `mutable` becuase we may need to temporarily create encoder in
|
||||
// GetEncoderInfo(), which is const.
|
||||
mutable std::list<std::unique_ptr<EncoderContext>> cached_encoder_contexts_;
|
||||
|
||||
const absl::optional<unsigned int> experimental_boosted_screenshare_qp_;
|
||||
const bool boost_base_layer_quality_;
|
||||
const bool prefer_temporal_support_on_base_layer_;
|
||||
|
||||
const SimulcastEncoderAdapterEncoderInfoSettings encoder_info_override_;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // TGCALLS_CUSTOM_SIMULCAST_ENCODER_ADAPTER_H_
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
#ifndef TGCALLS_DARWIN_FFMPEG_H
|
||||
#define TGCALLS_DARWIN_FFMPEG_H
|
||||
|
||||
#include "platform/PlatformInterface.h"
|
||||
|
||||
namespace tgcalls {
|
||||
|
||||
void setupDarwinVideoDecoding(AVCodecContext *codecContext);
|
||||
webrtc::scoped_refptr<webrtc::VideoFrameBuffer> createDarwinPlatformFrameFromData(AVFrame const *frame);
|
||||
|
||||
} // namespace tgcalls
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
#include "DarwinFFMpeg.h"
|
||||
|
||||
extern "C" {
|
||||
#include <libavutil/frame.h>
|
||||
#include <libavutil/pixfmt.h>
|
||||
#include <libavcodec/avcodec.h>
|
||||
}
|
||||
|
||||
#import "ExtractCVPixelBuffer.h"
|
||||
|
||||
namespace tgcalls {
|
||||
|
||||
static enum AVPixelFormat getDarwinPreferredPixelFormat(__unused AVCodecContext *ctx, __unused const enum AVPixelFormat *pix_fmts) {
|
||||
return AV_PIX_FMT_VIDEOTOOLBOX;
|
||||
}
|
||||
|
||||
void setupDarwinVideoDecoding(AVCodecContext *codecContext) {
|
||||
return;
|
||||
|
||||
#if TARGET_IPHONE_SIMULATOR
|
||||
#else
|
||||
if (!codecContext) {
|
||||
return;
|
||||
}
|
||||
av_hwdevice_ctx_create(&codecContext->hw_device_ctx, AV_HWDEVICE_TYPE_VIDEOTOOLBOX, nullptr, nullptr, 0);
|
||||
codecContext->get_format = getDarwinPreferredPixelFormat;
|
||||
#endif
|
||||
}
|
||||
|
||||
webrtc::scoped_refptr<webrtc::VideoFrameBuffer> createDarwinPlatformFrameFromData(AVFrame const *frame) {
|
||||
if (!frame) {
|
||||
return nullptr;
|
||||
}
|
||||
if (frame->format == AV_PIX_FMT_VIDEOTOOLBOX && frame->data[3]) {
|
||||
return extractCVPixelBuffer((void *)frame->data[3]);
|
||||
} else {
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
#ifndef TGCALLS_DARWIN_INTERFACE_H
|
||||
#define TGCALLS_DARWIN_INTERFACE_H
|
||||
|
||||
#include "platform/PlatformInterface.h"
|
||||
|
||||
#import <CoreVideo/CoreVideo.h>
|
||||
|
||||
namespace tgcalls {
|
||||
|
||||
class DarwinVideoFrame : public PlatformVideoFrame {
|
||||
public:
|
||||
DarwinVideoFrame(CVPixelBufferRef pixelBuffer);
|
||||
virtual ~DarwinVideoFrame();
|
||||
|
||||
CVPixelBufferRef pixelBuffer() const {
|
||||
return _pixelBuffer;
|
||||
}
|
||||
|
||||
private:
|
||||
CVPixelBufferRef _pixelBuffer = nullptr;
|
||||
};
|
||||
|
||||
class DarwinInterface : public PlatformInterface {
|
||||
public:
|
||||
std::unique_ptr<rtc::NetworkMonitorFactory> createNetworkMonitorFactory() override;
|
||||
void configurePlatformAudio(int numChannels) override;
|
||||
std::unique_ptr<webrtc::VideoEncoderFactory> makeVideoEncoderFactory(bool preferHardwareEncoding, bool isScreencast) override;
|
||||
std::unique_ptr<webrtc::VideoDecoderFactory> makeVideoDecoderFactory() override;
|
||||
bool supportsEncoding(const std::string &codecName) override;
|
||||
webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> makeVideoSource(rtc::Thread *signalingThread, rtc::Thread *workerThread) override;
|
||||
virtual void adaptVideoSource(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> videoSource, int width, int height, int fps) override;
|
||||
std::unique_ptr<VideoCapturerInterface> makeVideoCapturer(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> source, std::string deviceId, std::function<void(VideoState)> stateUpdated, std::function<void(PlatformCaptureInfo)> captureInfoUpdated, std::shared_ptr<PlatformContext> platformContext, std::pair<int, int> &outResolution) override;
|
||||
virtual webrtc::scoped_refptr<WrappedAudioDeviceModule> wrapAudioDeviceModule(webrtc::scoped_refptr<webrtc::AudioDeviceModule> module) override;
|
||||
virtual void setupVideoDecoding(AVCodecContext *codecContext) override;
|
||||
virtual webrtc::scoped_refptr<webrtc::VideoFrameBuffer> createPlatformFrameFromData(AVFrame const *frame) override;
|
||||
};
|
||||
|
||||
} // namespace tgcalls
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,158 @@
|
|||
#include "DarwinInterface.h"
|
||||
|
||||
#include "VideoCapturerInterfaceImpl.h"
|
||||
#include "sdk/objc/native/src/objc_video_track_source.h"
|
||||
#include "sdk/objc/native/api/network_monitor_factory.h"
|
||||
|
||||
#include "media/base/media_constants.h"
|
||||
#include "TGRTCDefaultVideoEncoderFactory.h"
|
||||
#include "TGRTCDefaultVideoDecoderFactory.h"
|
||||
#include "sdk/objc/native/api/video_encoder_factory.h"
|
||||
#include "sdk/objc/native/api/video_decoder_factory.h"
|
||||
#include "pc/video_track_source_proxy.h"
|
||||
#import "base/RTCLogging.h"
|
||||
#include "AudioDeviceModuleIOS.h"
|
||||
#include "AudioDeviceModuleMacos.h"
|
||||
#include "DarwinVideoSource.h"
|
||||
#include "objc_video_encoder_factory.h"
|
||||
#include "objc_video_decoder_factory.h"
|
||||
#include "sdk/objc/native/src/objc_frame_buffer.h"
|
||||
#import "sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.h"
|
||||
|
||||
#import "DarwinFFMpeg.h"
|
||||
|
||||
#ifdef WEBRTC_IOS
|
||||
#include "platform/darwin/iOS/RTCAudioSession.h"
|
||||
#include "platform/darwin/iOS/RTCAudioSessionConfiguration.h"
|
||||
#import <UIKit/UIKit.h>
|
||||
#endif // WEBRTC_IOS
|
||||
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
|
||||
#include <sys/sysctl.h>
|
||||
|
||||
namespace tgcalls {
|
||||
|
||||
std::unique_ptr<webrtc::VideoDecoderFactory> CustomObjCToNativeVideoDecoderFactory(
|
||||
id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> objc_video_decoder_factory) {
|
||||
return std::make_unique<webrtc::CustomObjCVideoDecoderFactory>(objc_video_decoder_factory);
|
||||
}
|
||||
|
||||
static DarwinVideoTrackSource *getObjCVideoSource(const webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> nativeSource) {
|
||||
webrtc::VideoTrackSourceProxy *proxy_source =
|
||||
static_cast<webrtc::VideoTrackSourceProxy *>(nativeSource.get());
|
||||
return static_cast<DarwinVideoTrackSource *>(proxy_source->internal());
|
||||
}
|
||||
|
||||
[[maybe_unused]] static NSString *getPlatformInfo() {
|
||||
const char *typeSpecifier = "hw.machine";
|
||||
|
||||
size_t size;
|
||||
sysctlbyname(typeSpecifier, NULL, &size, NULL, 0);
|
||||
|
||||
char *answer = (char *)malloc(size);
|
||||
sysctlbyname(typeSpecifier, answer, &size, NULL, 0);
|
||||
|
||||
NSString *results = [NSString stringWithCString:answer encoding:NSUTF8StringEncoding];
|
||||
|
||||
free(answer);
|
||||
return results;
|
||||
}
|
||||
|
||||
std::unique_ptr<rtc::NetworkMonitorFactory> DarwinInterface::createNetworkMonitorFactory() {
|
||||
return webrtc::CreateNetworkMonitorFactory();
|
||||
}
|
||||
|
||||
void DarwinInterface::configurePlatformAudio(int numChanels) {
|
||||
}
|
||||
|
||||
std::unique_ptr<webrtc::VideoEncoderFactory> DarwinInterface::makeVideoEncoderFactory(bool preferHardwareEncoding, bool isScreencast) {
|
||||
auto nativeFactory = std::make_unique<webrtc::CustomObjCVideoEncoderFactory>([[TGRTCDefaultVideoEncoderFactory alloc] initWithPreferHardwareH264:preferHardwareEncoding preferX264:false]);
|
||||
if (!preferHardwareEncoding) {
|
||||
auto nativeHardwareFactory = std::make_unique<webrtc::CustomObjCVideoEncoderFactory>([[TGRTCDefaultVideoEncoderFactory alloc] initWithPreferHardwareH264:true preferX264:false]);
|
||||
return std::make_unique<webrtc::SimulcastVideoEncoderFactory>(std::move(nativeFactory), std::move(nativeHardwareFactory));
|
||||
}
|
||||
return nativeFactory;
|
||||
}
|
||||
|
||||
std::unique_ptr<webrtc::VideoDecoderFactory> DarwinInterface::makeVideoDecoderFactory() {
|
||||
return CustomObjCToNativeVideoDecoderFactory([[TGRTCDefaultVideoDecoderFactory alloc] init]);
|
||||
}
|
||||
|
||||
bool DarwinInterface::supportsEncoding(const std::string &codecName) {
|
||||
if (false) {
|
||||
}
|
||||
#ifndef WEBRTC_DISABLE_H265
|
||||
else if (codecName == cricket::kH265CodecName) {
|
||||
#ifdef WEBRTC_IOS
|
||||
if (@available(iOS 11.0, *)) {
|
||||
return [[AVAssetExportSession allExportPresets] containsObject:AVAssetExportPresetHEVCHighestQuality];
|
||||
}
|
||||
#elif defined WEBRTC_MAC // WEBRTC_IOS
|
||||
|
||||
#ifdef __x86_64__
|
||||
return NO;
|
||||
#else
|
||||
return YES;
|
||||
#endif
|
||||
#endif // WEBRTC_IOS || WEBRTC_MAC
|
||||
}
|
||||
#endif
|
||||
else if (codecName == cricket::kH264CodecName) {
|
||||
#ifdef __x86_64__
|
||||
return YES;
|
||||
#else
|
||||
return NO;
|
||||
#endif
|
||||
} else if (codecName == cricket::kVp8CodecName) {
|
||||
return true;
|
||||
} else if (codecName == cricket::kVp9CodecName) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> DarwinInterface::makeVideoSource(rtc::Thread *signalingThread, rtc::Thread *workerThread) {
|
||||
webrtc::scoped_refptr<tgcalls::DarwinVideoTrackSource> objCVideoTrackSource(new rtc::RefCountedObject<tgcalls::DarwinVideoTrackSource>());
|
||||
return webrtc::VideoTrackSourceProxy::Create(signalingThread, workerThread, objCVideoTrackSource);
|
||||
}
|
||||
|
||||
void DarwinInterface::adaptVideoSource(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> videoSource, int width, int height, int fps) {
|
||||
getObjCVideoSource(videoSource)->OnOutputFormatRequest(width, height, fps);
|
||||
}
|
||||
|
||||
std::unique_ptr<VideoCapturerInterface> DarwinInterface::makeVideoCapturer(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> source, std::string deviceId, std::function<void(VideoState)> stateUpdated, std::function<void(PlatformCaptureInfo)> captureInfoUpdated, std::shared_ptr<PlatformContext> platformContext, std::pair<int, int> &outResolution) {
|
||||
return std::make_unique<VideoCapturerInterfaceImpl>(source, deviceId, stateUpdated, captureInfoUpdated, outResolution);
|
||||
}
|
||||
|
||||
webrtc::scoped_refptr<WrappedAudioDeviceModule> DarwinInterface::wrapAudioDeviceModule(webrtc::scoped_refptr<webrtc::AudioDeviceModule> module) {
|
||||
#ifdef WEBRTC_IOS
|
||||
return rtc::make_ref_counted<AudioDeviceModuleIOS>(module);
|
||||
#else
|
||||
return rtc::make_ref_counted<AudioDeviceModuleMacos>(module);
|
||||
#endif
|
||||
}
|
||||
|
||||
void DarwinInterface::setupVideoDecoding(AVCodecContext *codecContext) {
|
||||
return setupDarwinVideoDecoding(codecContext);
|
||||
}
|
||||
|
||||
webrtc::scoped_refptr<webrtc::VideoFrameBuffer> DarwinInterface::createPlatformFrameFromData(AVFrame const *frame) {
|
||||
return createDarwinPlatformFrameFromData(frame);
|
||||
}
|
||||
|
||||
std::unique_ptr<PlatformInterface> CreatePlatformInterface() {
|
||||
return std::make_unique<DarwinInterface>();
|
||||
}
|
||||
|
||||
DarwinVideoFrame::DarwinVideoFrame(CVPixelBufferRef pixelBuffer) {
|
||||
_pixelBuffer = CVPixelBufferRetain(pixelBuffer);
|
||||
}
|
||||
|
||||
DarwinVideoFrame::~DarwinVideoFrame() {
|
||||
if (_pixelBuffer) {
|
||||
CVPixelBufferRelease(_pixelBuffer);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace tgcalls
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
/*
|
||||
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef TGCALLS_DARWIN_VIDEO_SOURCE_H
|
||||
#define TGCALLS_DARWIN_VIDEO_SOURCE_H
|
||||
//#ifdef WEBRTC_IOS
|
||||
|
||||
#import "base/RTCVideoCapturer.h"
|
||||
|
||||
#include "base/RTCMacros.h"
|
||||
#include "media/base/adapted_video_track_source.h"
|
||||
#include "rtc_base/timestamp_aligner.h"
|
||||
|
||||
RTC_FWD_DECL_OBJC_CLASS(RTC_OBJC_TYPE(RTCVideoFrame));
|
||||
|
||||
namespace tgcalls {
|
||||
|
||||
class DarwinVideoTrackSource : public rtc::AdaptedVideoTrackSource {
|
||||
public:
|
||||
DarwinVideoTrackSource();
|
||||
|
||||
// This class can not be used for implementing screen casting. Hopefully, this
|
||||
// function will be removed before we add that to iOS/Mac.
|
||||
bool is_screencast() const override;
|
||||
|
||||
// Indicates that the encoder should denoise video before encoding it.
|
||||
// If it is not set, the default configuration is used which is different
|
||||
// depending on video codec.
|
||||
absl::optional<bool> needs_denoising() const override;
|
||||
|
||||
SourceState state() const override;
|
||||
|
||||
bool remote() const override;
|
||||
|
||||
void OnCapturedFrame(RTC_OBJC_TYPE(RTCVideoFrame) * frame);
|
||||
bool OnCapturedFrame(const webrtc::VideoFrame& frame);
|
||||
|
||||
// Called by RTCVideoSource.
|
||||
void OnOutputFormatRequest(int width, int height, int fps);
|
||||
|
||||
private:
|
||||
rtc::VideoBroadcaster broadcaster_;
|
||||
rtc::TimestampAligner timestamp_aligner_;
|
||||
|
||||
};
|
||||
|
||||
} // namespace tgcalls
|
||||
|
||||
//#endif //WEBRTC_IOS
|
||||
#endif
|
||||
|
|
@ -0,0 +1,165 @@
|
|||
/*
|
||||
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "DarwinVideoSource.h"
|
||||
|
||||
#import "base/RTCVideoFrame.h"
|
||||
#import "base/RTCVideoFrameBuffer.h"
|
||||
#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
|
||||
|
||||
#include "api/video/i420_buffer.h"
|
||||
#include "sdk/objc/native/src/objc_frame_buffer.h"
|
||||
|
||||
namespace tgcalls {
|
||||
|
||||
DarwinVideoTrackSource::DarwinVideoTrackSource()
|
||||
: AdaptedVideoTrackSource(/* required resolution alignment */ 2) {}
|
||||
|
||||
bool DarwinVideoTrackSource::is_screencast() const {
|
||||
return false;
|
||||
}
|
||||
|
||||
absl::optional<bool> DarwinVideoTrackSource::needs_denoising() const {
|
||||
return false;
|
||||
}
|
||||
|
||||
webrtc::MediaSourceInterface::SourceState DarwinVideoTrackSource::state() const {
|
||||
return SourceState::kLive;
|
||||
}
|
||||
|
||||
bool DarwinVideoTrackSource::remote() const {
|
||||
return false;
|
||||
}
|
||||
|
||||
void DarwinVideoTrackSource::OnOutputFormatRequest(int width, int height, int fps) {
|
||||
cricket::VideoFormat format(width, height, cricket::VideoFormat::FpsToInterval(fps), 0);
|
||||
video_adapter()->OnOutputFormatRequest(format);
|
||||
}
|
||||
|
||||
void DarwinVideoTrackSource::OnCapturedFrame(RTC_OBJC_TYPE(RTCVideoFrame) * frame) {
|
||||
const int64_t timestamp_us = frame.timeStampNs / rtc::kNumNanosecsPerMicrosec;
|
||||
const int64_t translated_timestamp_us =
|
||||
timestamp_aligner_.TranslateTimestamp(timestamp_us, rtc::TimeMicros());
|
||||
|
||||
int adapted_width;
|
||||
int adapted_height;
|
||||
int crop_width;
|
||||
int crop_height;
|
||||
int crop_x;
|
||||
int crop_y;
|
||||
if (!AdaptFrame(frame.width,
|
||||
frame.height,
|
||||
timestamp_us,
|
||||
&adapted_width,
|
||||
&adapted_height,
|
||||
&crop_width,
|
||||
&crop_height,
|
||||
&crop_x,
|
||||
&crop_y)) {
|
||||
return;
|
||||
}
|
||||
|
||||
webrtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer;
|
||||
if (adapted_width == frame.width && adapted_height == frame.height) {
|
||||
// No adaption - optimized path.
|
||||
@autoreleasepool {
|
||||
buffer = new rtc::RefCountedObject<webrtc::ObjCFrameBuffer>(frame.buffer);
|
||||
}
|
||||
} else if ([frame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) {
|
||||
// Adapted CVPixelBuffer frame.
|
||||
@autoreleasepool {
|
||||
RTC_OBJC_TYPE(RTCCVPixelBuffer) *rtcPixelBuffer =
|
||||
(RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer;
|
||||
buffer = new rtc::RefCountedObject<webrtc::ObjCFrameBuffer>([[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc]
|
||||
initWithPixelBuffer:rtcPixelBuffer.pixelBuffer
|
||||
adaptedWidth:adapted_width
|
||||
adaptedHeight:adapted_height
|
||||
cropWidth:crop_width
|
||||
cropHeight:crop_height
|
||||
cropX:crop_x + rtcPixelBuffer.cropX
|
||||
cropY:crop_y + rtcPixelBuffer.cropY]);
|
||||
}
|
||||
} else {
|
||||
@autoreleasepool {
|
||||
// Adapted I420 frame.
|
||||
// TODO(magjed): Optimize this I420 path.
|
||||
webrtc::scoped_refptr<webrtc::I420Buffer> i420_buffer = webrtc::I420Buffer::Create(adapted_width, adapted_height);
|
||||
buffer = new rtc::RefCountedObject<webrtc::ObjCFrameBuffer>(frame.buffer);
|
||||
i420_buffer->CropAndScaleFrom(*buffer->ToI420(), crop_x, crop_y, crop_width, crop_height);
|
||||
buffer = i420_buffer;
|
||||
}
|
||||
}
|
||||
|
||||
// Applying rotation is only supported for legacy reasons and performance is
|
||||
// not critical here.
|
||||
webrtc::VideoRotation rotation = static_cast<webrtc::VideoRotation>(frame.rotation);
|
||||
if (apply_rotation() && rotation != webrtc::kVideoRotation_0) {
|
||||
buffer = webrtc::I420Buffer::Rotate(*buffer->ToI420(), rotation);
|
||||
rotation = webrtc::kVideoRotation_0;
|
||||
}
|
||||
|
||||
OnFrame(webrtc::VideoFrame::Builder()
|
||||
.set_video_frame_buffer(buffer)
|
||||
.set_rotation(rotation)
|
||||
.set_timestamp_us(translated_timestamp_us)
|
||||
.build());
|
||||
}
|
||||
|
||||
bool DarwinVideoTrackSource::OnCapturedFrame(const webrtc::VideoFrame& frame) {
|
||||
const int64_t timestamp_us = frame.timestamp_us() / rtc::kNumNanosecsPerMicrosec;
|
||||
const int64_t translated_timestamp_us =
|
||||
timestamp_aligner_.TranslateTimestamp(timestamp_us, rtc::TimeMicros());
|
||||
|
||||
int adapted_width;
|
||||
int adapted_height;
|
||||
int crop_width;
|
||||
int crop_height;
|
||||
int crop_x;
|
||||
int crop_y;
|
||||
if (!AdaptFrame(frame.width(),
|
||||
frame.height(),
|
||||
timestamp_us,
|
||||
&adapted_width,
|
||||
&adapted_height,
|
||||
&crop_width,
|
||||
&crop_height,
|
||||
&crop_x,
|
||||
&crop_y)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
webrtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer;
|
||||
if (adapted_width == frame.width() && adapted_height == frame.height()) {
|
||||
buffer = frame.video_frame_buffer();
|
||||
} else {
|
||||
webrtc::scoped_refptr<webrtc::I420Buffer> i420_buffer = webrtc::I420Buffer::Create(adapted_width, adapted_height);
|
||||
buffer = frame.video_frame_buffer();
|
||||
i420_buffer->CropAndScaleFrom(*buffer->ToI420(), crop_x, crop_y, crop_width, crop_height);
|
||||
buffer = i420_buffer;
|
||||
}
|
||||
|
||||
// Applying rotation is only supported for legacy reasons and performance is
|
||||
// not critical here.
|
||||
webrtc::VideoRotation rotation = frame.rotation();
|
||||
if (apply_rotation() && rotation != webrtc::kVideoRotation_0) {
|
||||
buffer = webrtc::I420Buffer::Rotate(*buffer->ToI420(), rotation);
|
||||
rotation = webrtc::kVideoRotation_0;
|
||||
}
|
||||
|
||||
OnFrame(webrtc::VideoFrame::Builder()
|
||||
.set_video_frame_buffer(buffer)
|
||||
.set_rotation(rotation)
|
||||
.set_timestamp_us(translated_timestamp_us)
|
||||
.build());
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
//
|
||||
// DesktopCaptureSourceView.h
|
||||
// TgVoipWebrtc
|
||||
//
|
||||
// Created by Mikhail Filimonov on 28.12.2020.
|
||||
// Copyright © 2020 Mikhail Filimonov. All rights reserved.
|
||||
//
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <AppKit/AppKit.h>
|
||||
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
@protocol VideoSourceMac
|
||||
-(NSString *)deviceIdKey;
|
||||
-(NSString *)title;
|
||||
-(NSString *)uniqueKey;
|
||||
-(BOOL)isEqual:(id)another;
|
||||
@end
|
||||
|
||||
@interface DesktopCaptureSourceDataMac : NSObject
|
||||
@property CGSize aspectSize;
|
||||
@property double fps;
|
||||
@property bool captureMouse;
|
||||
-(id)initWithSize:(CGSize)size fps:(double)fps captureMouse:(bool)captureMouse;
|
||||
|
||||
-(NSString *)cachedKey;
|
||||
@end
|
||||
|
||||
@interface DesktopCaptureSourceMac : NSObject <VideoSourceMac>
|
||||
-(long)uniqueId;
|
||||
-(BOOL)isWindow;
|
||||
@end
|
||||
|
||||
@interface DesktopCaptureSourceScopeMac : NSObject
|
||||
@property(nonatomic, strong, readonly) DesktopCaptureSourceDataMac *data;
|
||||
@property(nonatomic, strong, readonly) DesktopCaptureSourceMac *source;
|
||||
-(id)initWithSource:(DesktopCaptureSourceMac *)source data:(DesktopCaptureSourceDataMac *)data;
|
||||
-(NSString *)cachedKey;
|
||||
@end
|
||||
|
||||
@interface DesktopCaptureSourceManagerMac : NSObject
|
||||
|
||||
-(instancetype)init_s;
|
||||
-(instancetype)init_w;
|
||||
-(NSArray<DesktopCaptureSourceMac *> *)list;
|
||||
|
||||
-(NSView *)createForScope:(DesktopCaptureSourceScopeMac *)scope;
|
||||
-(void)start:(DesktopCaptureSourceScopeMac *)scope;
|
||||
-(void)stop:(DesktopCaptureSourceScopeMac *)scope;
|
||||
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
|
|
@ -0,0 +1,207 @@
|
|||
//
|
||||
// DesktopCaptureSourceView.m
|
||||
// TgVoipWebrtc
|
||||
//
|
||||
// Created by Mikhail Filimonov on 28.12.2020.
|
||||
// Copyright © 2020 Mikhail Filimonov. All rights reserved.
|
||||
//
|
||||
#import <Cocoa/Cocoa.h>
|
||||
#import "DesktopCaptureSourceViewMac.h"
|
||||
#import "platform/darwin/VideoMetalViewMac.h"
|
||||
#import "tgcalls/desktop_capturer/DesktopCaptureSource.h"
|
||||
#import "tgcalls/desktop_capturer/DesktopCaptureSourceHelper.h"
|
||||
#import "tgcalls/desktop_capturer/DesktopCaptureSourceManager.h"
|
||||
#import "platform/darwin/VideoMetalViewMac.h"
|
||||
|
||||
|
||||
|
||||
@interface DesktopCaptureSourceViewMetal : VideoMetalView
|
||||
@end
|
||||
|
||||
|
||||
@implementation DesktopCaptureSourceViewMetal
|
||||
|
||||
-(id)initWithHelper:(tgcalls::DesktopCaptureSourceHelper)helper {
|
||||
if (self = [super initWithFrame:CGRectZero]) {
|
||||
std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink = [self getSink];
|
||||
helper.setOutput(sink);
|
||||
[self setVideoContentMode:kCAGravityResizeAspectFill];
|
||||
}
|
||||
return self;
|
||||
}
|
||||
@end
|
||||
|
||||
|
||||
@implementation DesktopCaptureSourceDataMac
|
||||
-(id)initWithSize:(CGSize)size fps:(double)fps captureMouse:(bool)captureMouse {
|
||||
if (self = [super init]) {
|
||||
self.aspectSize = size;
|
||||
self.fps = fps;
|
||||
self.captureMouse = captureMouse;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
-(NSString *)cachedKey {
|
||||
return [[NSString alloc] initWithFormat:@"%@:%f:%d", NSStringFromSize(self.aspectSize), self.fps, self.captureMouse];
|
||||
}
|
||||
@end
|
||||
|
||||
@interface DesktopCaptureSourceMac ()
|
||||
{
|
||||
absl::optional<tgcalls::DesktopCaptureSource> _source;
|
||||
BOOL _isWindow;
|
||||
}
|
||||
-(id)initWithSource:(tgcalls::DesktopCaptureSource)source isWindow:(BOOL)isWindow;
|
||||
-(tgcalls::DesktopCaptureSource)getSource;
|
||||
|
||||
@end
|
||||
|
||||
@implementation DesktopCaptureSourceMac
|
||||
|
||||
-(tgcalls::DesktopCaptureSource)getSource {
|
||||
return _source.value();
|
||||
}
|
||||
|
||||
-(NSString *)title {
|
||||
if (_isWindow) {
|
||||
const tgcalls::DesktopCaptureSource source = _source.value();
|
||||
return [[NSString alloc] initWithCString:source.title().c_str() encoding:NSUTF8StringEncoding];
|
||||
}
|
||||
else
|
||||
return [[NSString alloc] initWithFormat:@"Screen"];
|
||||
}
|
||||
|
||||
-(long)uniqueId {
|
||||
return _source.value().uniqueId();
|
||||
}
|
||||
-(BOOL)isWindow {
|
||||
return _isWindow;
|
||||
}
|
||||
-(NSString *)uniqueKey {
|
||||
return [[NSString alloc] initWithFormat:@"%ld:%@", self.uniqueId, _isWindow ? @"Window" : @"Screen"];
|
||||
}
|
||||
|
||||
-(NSString *)deviceIdKey {
|
||||
return [[NSString alloc] initWithFormat:@"desktop_capturer_%@_%ld", _isWindow ? @"window" : @"screen", self.uniqueId];
|
||||
}
|
||||
|
||||
-(BOOL)isEqual:(id)object {
|
||||
return [[((DesktopCaptureSourceMac *)object) uniqueKey] isEqualToString:[self uniqueKey]];
|
||||
}
|
||||
- (BOOL)isEqualTo:(id)object {
|
||||
return [[((DesktopCaptureSourceMac *)object) uniqueKey] isEqualToString:[self uniqueKey]];
|
||||
}
|
||||
|
||||
-(id)initWithSource:(tgcalls::DesktopCaptureSource)source isWindow:(BOOL)isWindow {
|
||||
if (self = [super init]) {
|
||||
_source = source;
|
||||
_isWindow = isWindow;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
|
||||
@interface DesktopCaptureSourceScopeMac ()
|
||||
-(tgcalls::DesktopCaptureSourceData)getData;
|
||||
-(tgcalls::DesktopCaptureSource)getSource;
|
||||
@end
|
||||
|
||||
@implementation DesktopCaptureSourceScopeMac
|
||||
|
||||
-(id)initWithSource:(DesktopCaptureSourceMac *)source data:(DesktopCaptureSourceDataMac *)data {
|
||||
if (self = [super init]) {
|
||||
_data = data;
|
||||
_source = source;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
-(NSString *)cachedKey {
|
||||
return [[NSString alloc] initWithFormat:@"%@:%@", _source.uniqueKey, _data.cachedKey];
|
||||
}
|
||||
|
||||
-(tgcalls::DesktopCaptureSourceData)getData {
|
||||
tgcalls::DesktopCaptureSourceData data{
|
||||
/*.aspectSize = */{ (int)_data.aspectSize.width, (int)_data.aspectSize.height},
|
||||
/*.fps = */_data.fps,
|
||||
/*.captureMouse = */_data.captureMouse,
|
||||
};
|
||||
return data;
|
||||
}
|
||||
-(tgcalls::DesktopCaptureSource)getSource {
|
||||
return [_source getSource];
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@implementation DesktopCaptureSourceManagerMac
|
||||
{
|
||||
std::map<std::string, tgcalls::DesktopCaptureSourceHelper> _cached;
|
||||
std::unique_ptr<tgcalls::DesktopCaptureSourceManager> _manager;
|
||||
BOOL _isWindow;
|
||||
}
|
||||
|
||||
-(instancetype)init_s {
|
||||
if (self = [super init]) {
|
||||
_manager = std::make_unique<tgcalls::DesktopCaptureSourceManager>(tgcalls::DesktopCaptureType::Screen);
|
||||
_isWindow = NO;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
-(instancetype)init_w {
|
||||
if (self = [super init]) {
|
||||
_manager = std::make_unique<tgcalls::DesktopCaptureSourceManager>(tgcalls::DesktopCaptureType::Window);
|
||||
_isWindow = YES;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
-(NSArray<DesktopCaptureSourceMac *> *)list {
|
||||
std::vector<tgcalls::DesktopCaptureSource> sources = _manager->sources();
|
||||
NSMutableArray<DesktopCaptureSourceMac *> *macSources = [[NSMutableArray alloc] init];
|
||||
for (auto i = sources.begin(); i != sources.end(); ++i) {
|
||||
[macSources addObject:[[DesktopCaptureSourceMac alloc] initWithSource:*i isWindow:_isWindow]];
|
||||
}
|
||||
return macSources;
|
||||
}
|
||||
|
||||
-(NSView *)createForScope:(DesktopCaptureSourceScopeMac*)scope {
|
||||
auto i = _cached.find(std::string([scope.cachedKey UTF8String]));
|
||||
if (i == _cached.end()) {
|
||||
i = _cached.emplace(
|
||||
std::string([scope.cachedKey UTF8String]),
|
||||
tgcalls::DesktopCaptureSourceHelper([scope getSource], [scope getData])).first;
|
||||
}
|
||||
|
||||
DesktopCaptureSourceViewMetal *view = [[DesktopCaptureSourceViewMetal alloc] initWithHelper:i->second];
|
||||
if (scope.data.captureMouse) {
|
||||
[view setVideoContentMode:kCAGravityResizeAspect];
|
||||
}
|
||||
return view;
|
||||
}
|
||||
|
||||
-(void)start:(DesktopCaptureSourceScopeMac *)scope {
|
||||
const auto i = _cached.find(std::string([scope.cachedKey UTF8String]));
|
||||
if (i != _cached.end()) {
|
||||
i->second.start();
|
||||
}
|
||||
}
|
||||
|
||||
-(void)stop:(DesktopCaptureSourceScopeMac *)scope {
|
||||
const auto i = _cached.find(std::string([scope.cachedKey UTF8String]));
|
||||
if (i != _cached.end()) {
|
||||
i->second.stop();
|
||||
}
|
||||
}
|
||||
|
||||
-(void)dealloc {
|
||||
for (auto &[key, helper] : _cached) {
|
||||
helper.stop();
|
||||
}
|
||||
_manager.reset();
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
#ifndef SCREEN_CAPTURER_H
|
||||
#define SCREEN_CAPTURER_H
|
||||
#ifndef WEBRTC_IOS
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
|
||||
#import "api/video/video_sink_interface.h"
|
||||
#import "api/media_stream_interface.h"
|
||||
#import "rtc_base/time_utils.h"
|
||||
|
||||
#import "api/video/video_sink_interface.h"
|
||||
#import "api/media_stream_interface.h"
|
||||
|
||||
#import "sdk/objc/native/src/objc_video_track_source.h"
|
||||
#import "sdk/objc/native/src/objc_frame_buffer.h"
|
||||
#import "pc/video_track_source_proxy.h"
|
||||
#import "tgcalls/platform/darwin/VideoCameraCapturerMac.h"
|
||||
#import "tgcalls/desktop_capturer/DesktopCaptureSource.h"
|
||||
|
||||
@interface DesktopSharingCapturer : NSObject<CapturerInterface>
|
||||
- (instancetype)initWithSource:(rtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)trackSource captureSource:(tgcalls::DesktopCaptureSource)captureSource;
|
||||
|
||||
@end
|
||||
|
||||
|
||||
#endif //WEBRTC_IOS
|
||||
#endif
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
#import "DesktopSharingCapturer.h"
|
||||
|
||||
#include "modules/desktop_capture/mac/screen_capturer_mac.h"
|
||||
#include "modules/desktop_capture/desktop_and_cursor_composer.h"
|
||||
#include "modules/desktop_capture/desktop_capturer_differ_wrapper.h"
|
||||
#include "third_party/libyuv/include/libyuv.h"
|
||||
#include "api/video/i420_buffer.h"
|
||||
#include "common_video/libyuv/include/webrtc_libyuv.h"
|
||||
#include "rtc_base/checks.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "third_party/libyuv/include/libyuv.h"
|
||||
|
||||
#include "tgcalls/desktop_capturer/DesktopCaptureSource.h"
|
||||
#include "tgcalls/desktop_capturer/DesktopCaptureSourceHelper.h"
|
||||
#include "tgcalls/desktop_capturer/DesktopCaptureSourceManager.h"
|
||||
#include "DarwinVideoSource.h"
|
||||
|
||||
#import "helpers/RTCDispatcher+Private.h"
|
||||
#import <QuartzCore/QuartzCore.h>
|
||||
|
||||
static RTCVideoFrame *customToObjCVideoFrame(const webrtc::VideoFrame &frame, RTCVideoRotation &rotation) {
|
||||
rotation = RTCVideoRotation(frame.rotation());
|
||||
RTCVideoFrame *videoFrame =
|
||||
[[RTCVideoFrame alloc] initWithBuffer:webrtc::ToObjCVideoFrameBuffer(frame.video_frame_buffer())
|
||||
rotation:rotation
|
||||
timeStampNs:frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec];
|
||||
videoFrame.timeStamp = frame.timestamp();
|
||||
|
||||
return videoFrame;
|
||||
}
|
||||
|
||||
static tgcalls::DarwinVideoTrackSource *getObjCVideoSource(const rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> nativeSource) {
|
||||
webrtc::VideoTrackSourceProxy *proxy_source =
|
||||
static_cast<webrtc::VideoTrackSourceProxy *>(nativeSource.get());
|
||||
return static_cast<tgcalls::DarwinVideoTrackSource *>(proxy_source->internal());
|
||||
}
|
||||
|
||||
class RendererAdapterImpl : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
|
||||
public:
|
||||
RendererAdapterImpl(void (^frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation)) {
|
||||
_frameReceived = [frameReceived copy];
|
||||
}
|
||||
|
||||
void OnFrame(const webrtc::VideoFrame& nativeVideoFrame) override {
|
||||
RTCVideoRotation rotation = RTCVideoRotation_0;
|
||||
RTCVideoFrame* videoFrame = customToObjCVideoFrame(nativeVideoFrame, rotation);
|
||||
|
||||
CGSize currentSize = (videoFrame.rotation % 180 == 0) ? CGSizeMake(videoFrame.width, videoFrame.height) : CGSizeMake(videoFrame.height, videoFrame.width);
|
||||
|
||||
if (_frameReceived) {
|
||||
_frameReceived(currentSize, videoFrame, rotation);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
void (^_frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation);
|
||||
|
||||
};
|
||||
|
||||
@implementation DesktopSharingCapturer {
|
||||
absl::optional<tgcalls::DesktopCaptureSourceHelper> renderer;
|
||||
std::shared_ptr<RendererAdapterImpl> _sink;
|
||||
BOOL _isPaused;
|
||||
}
|
||||
- (instancetype)initWithSource:(rtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)trackSource captureSource:(tgcalls::DesktopCaptureSource)captureSource {
|
||||
self = [super init];
|
||||
if (self != nil) {
|
||||
_sink.reset(new RendererAdapterImpl(^(CGSize size, RTCVideoFrame *videoFrame, RTCVideoRotation rotation) {
|
||||
getObjCVideoSource(trackSource)->OnCapturedFrame(videoFrame);
|
||||
}));
|
||||
|
||||
tgcalls::DesktopCaptureSourceData data{
|
||||
/*.aspectSize = */{ 1920, 1080 },
|
||||
/*.fps = */25,
|
||||
/*.captureMouse = */true,
|
||||
};
|
||||
renderer.emplace(captureSource, data);
|
||||
renderer->setOutput(_sink);
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
-(void)setOnFatalError:(std::function<void ()>)error {
|
||||
renderer->setOnFatalError(error);
|
||||
}
|
||||
-(void)setOnPause:(std::function<void (bool)>)pause {
|
||||
renderer->setOnPause(pause);
|
||||
}
|
||||
|
||||
-(void)start {
|
||||
renderer->start();
|
||||
}
|
||||
|
||||
-(void)stop {
|
||||
renderer->stop();
|
||||
}
|
||||
|
||||
- (void)setIsEnabled:(bool)isEnabled {
|
||||
BOOL updated = _isPaused != !isEnabled;
|
||||
_isPaused = !isEnabled;
|
||||
if (updated) {
|
||||
if (isEnabled) {
|
||||
renderer->start();
|
||||
} else {
|
||||
renderer->stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
- (void)setPreferredCaptureAspectRatio:(float)aspectRatio {
|
||||
}
|
||||
|
||||
- (void)setUncroppedSink:(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame> >)sink {
|
||||
renderer->setSecondaryOutput(sink);
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
#ifndef TGCALLS_EXTRACT_CV_PIXEL_BUFFER_H
|
||||
#define TGCALLS_EXTRACT_CV_PIXEL_BUFFER_H
|
||||
|
||||
#include "platform/PlatformInterface.h"
|
||||
|
||||
namespace tgcalls {
|
||||
|
||||
webrtc::scoped_refptr<webrtc::VideoFrameBuffer> extractCVPixelBuffer(void *data);
|
||||
|
||||
} // namespace tgcalls
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
#include "ExtractCVPixelBuffer.h"
|
||||
|
||||
#include "sdk/objc/native/src/objc_frame_buffer.h"
|
||||
#import "sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.h"
|
||||
|
||||
namespace tgcalls {
|
||||
|
||||
|
||||
webrtc::scoped_refptr<webrtc::VideoFrameBuffer> extractCVPixelBuffer(void *data) {
|
||||
CVPixelBufferRef pixelBuffer = (CVPixelBufferRef)(void *)data;
|
||||
return rtc::make_ref_counted<webrtc::ObjCFrameBuffer>([[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer]);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
/* This file is borrowed from sdk/objc/components/video_codec/RTCCodecSpecificInfoH264+Private.h */
|
||||
|
||||
#import "RTCCodecSpecificInfoH265.h"
|
||||
|
||||
#include "modules/video_coding/include/video_codec_interface.h"
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
/* Interfaces for converting to/from internal C++ formats. */
|
||||
@interface RTCCodecSpecificInfoH265 ()
|
||||
|
||||
- (webrtc::CodecSpecificInfo)nativeCodecSpecificInfo;
|
||||
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
/* This file is borrowed from sdk/objc/components/video_codec/RTCCodecSpecificInfoH264.h. */
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
#import "RTCCodecSpecificInfo.h"
|
||||
#import "RTCMacros.h"
|
||||
|
||||
/** Class for H265 specific config. */
|
||||
typedef NS_ENUM(NSUInteger, RTCH265PacketizationMode) {
|
||||
RTCH265PacketizationModeNonInterleaved = 0, // Mode 1 - STAP-A, FU-A is allowed
|
||||
RTCH265PacketizationModeSingleNalUnit // Mode 0 - only single NALU allowed
|
||||
};
|
||||
|
||||
RTC_OBJC_EXPORT
|
||||
@interface RTCCodecSpecificInfoH265 : NSObject <RTCCodecSpecificInfo>
|
||||
|
||||
@property(nonatomic, assign) RTCH265PacketizationMode packetizationMode;
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
/* This file is borrowed from sdk/objc/components/video_codec/RTCCodecSpecificInfoH264.mm */
|
||||
|
||||
#import "RTCCodecSpecificInfoH265+Private.h"
|
||||
|
||||
// H265 specific settings.
|
||||
@implementation RTCCodecSpecificInfoH265
|
||||
|
||||
@synthesize packetizationMode = _packetizationMode;
|
||||
|
||||
- (webrtc::CodecSpecificInfo)nativeCodecSpecificInfo {
|
||||
webrtc::CodecSpecificInfo codecSpecificInfo;
|
||||
codecSpecificInfo.codecType = webrtc::kVideoCodecH265;
|
||||
//codecSpecificInfo.codecSpecific.H265.packetization_mode = (webrtc::H265PacketizationMode)_packetizationMode;
|
||||
|
||||
return codecSpecificInfo;
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
#import "RTCMacros.h"
|
||||
|
||||
RTC_OBJC_EXPORT extern NSString *const kRTCVideoCodecH265Name;
|
||||
RTC_OBJC_EXPORT extern NSString *const kRTCLevel31Main;
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
/*
|
||||
* Copyright 2018 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*
|
||||
*/
|
||||
|
||||
#import "RTCH265ProfileLevelId.h"
|
||||
|
||||
#include "media/base/media_constants.h"
|
||||
|
||||
NSString *const kRTCVideoCodecH265Name = @(cricket::kH265CodecName);
|
||||
// TODO(jianjunz): This is value is not correct.
|
||||
NSString *const kRTCLevel31Main = @"4d001f";
|
||||
19
TMessagesProj/jni/voip/tgcalls/platform/darwin/SQueue.h
Normal file
19
TMessagesProj/jni/voip/tgcalls/platform/darwin/SQueue.h
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
#import <Foundation/Foundation.h>
|
||||
|
||||
@interface SQueue : NSObject
|
||||
|
||||
+ (SQueue * _Nonnull)mainQueue;
|
||||
+ (SQueue * _Nonnull)concurrentDefaultQueue;
|
||||
+ (SQueue * _Nonnull)concurrentBackgroundQueue;
|
||||
|
||||
+ (SQueue * _Nonnull)wrapConcurrentNativeQueue:(dispatch_queue_t _Nonnull)nativeQueue;
|
||||
|
||||
- (void)dispatch:(dispatch_block_t _Nonnull)block;
|
||||
- (void)dispatchSync:(dispatch_block_t _Nonnull)block;
|
||||
- (void)dispatch:(dispatch_block_t _Nonnull)block synchronous:(bool)synchronous;
|
||||
|
||||
- (dispatch_queue_t _Nonnull)_dispatch_queue;
|
||||
|
||||
- (bool)isCurrentQueue;
|
||||
|
||||
@end
|
||||
124
TMessagesProj/jni/voip/tgcalls/platform/darwin/SQueue.m
Normal file
124
TMessagesProj/jni/voip/tgcalls/platform/darwin/SQueue.m
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
#import "SQueue.h"
|
||||
|
||||
static const void *SQueueSpecificKey = &SQueueSpecificKey;
|
||||
|
||||
@interface SQueue ()
|
||||
{
|
||||
dispatch_queue_t _queue;
|
||||
void *_queueSpecific;
|
||||
bool _specialIsMainQueue;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@implementation SQueue
|
||||
|
||||
+ (SQueue *)mainQueue
|
||||
{
|
||||
static SQueue *queue = nil;
|
||||
static dispatch_once_t onceToken;
|
||||
dispatch_once(&onceToken, ^
|
||||
{
|
||||
queue = [[SQueue alloc] initWithNativeQueue:dispatch_get_main_queue() queueSpecific:NULL];
|
||||
queue->_specialIsMainQueue = true;
|
||||
});
|
||||
|
||||
return queue;
|
||||
}
|
||||
|
||||
+ (SQueue *)concurrentDefaultQueue
|
||||
{
|
||||
static SQueue *queue = nil;
|
||||
static dispatch_once_t onceToken;
|
||||
dispatch_once(&onceToken, ^
|
||||
{
|
||||
queue = [[SQueue alloc] initWithNativeQueue:dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0) queueSpecific:NULL];
|
||||
});
|
||||
|
||||
return queue;
|
||||
}
|
||||
|
||||
+ (SQueue *)concurrentBackgroundQueue
|
||||
{
|
||||
static SQueue *queue = nil;
|
||||
static dispatch_once_t onceToken;
|
||||
dispatch_once(&onceToken, ^
|
||||
{
|
||||
queue = [[SQueue alloc] initWithNativeQueue:dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0) queueSpecific:NULL];
|
||||
});
|
||||
|
||||
return queue;
|
||||
}
|
||||
|
||||
+ (SQueue *)wrapConcurrentNativeQueue:(dispatch_queue_t)nativeQueue
|
||||
{
|
||||
return [[SQueue alloc] initWithNativeQueue:nativeQueue queueSpecific:NULL];
|
||||
}
|
||||
|
||||
- (instancetype)init
|
||||
{
|
||||
dispatch_queue_t queue = dispatch_queue_create(NULL, NULL);
|
||||
dispatch_queue_set_specific(queue, SQueueSpecificKey, (__bridge void *)self, NULL);
|
||||
return [self initWithNativeQueue:queue queueSpecific:(__bridge void *)self];
|
||||
}
|
||||
|
||||
- (instancetype)initWithNativeQueue:(dispatch_queue_t)queue queueSpecific:(void *)queueSpecific
|
||||
{
|
||||
self = [super init];
|
||||
if (self != nil)
|
||||
{
|
||||
_queue = queue;
|
||||
_queueSpecific = queueSpecific;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (dispatch_queue_t)_dispatch_queue
|
||||
{
|
||||
return _queue;
|
||||
}
|
||||
|
||||
- (void)dispatch:(dispatch_block_t)block
|
||||
{
|
||||
if (_queueSpecific != NULL && dispatch_get_specific(SQueueSpecificKey) == _queueSpecific)
|
||||
block();
|
||||
else if (_specialIsMainQueue && [NSThread isMainThread])
|
||||
block();
|
||||
else
|
||||
dispatch_async(_queue, block);
|
||||
}
|
||||
|
||||
- (void)dispatchSync:(dispatch_block_t)block
|
||||
{
|
||||
if (_queueSpecific != NULL && dispatch_get_specific(SQueueSpecificKey) == _queueSpecific)
|
||||
block();
|
||||
else if (_specialIsMainQueue && [NSThread isMainThread])
|
||||
block();
|
||||
else
|
||||
dispatch_sync(_queue, block);
|
||||
}
|
||||
|
||||
- (void)dispatch:(dispatch_block_t)block synchronous:(bool)synchronous {
|
||||
if (_queueSpecific != NULL && dispatch_get_specific(SQueueSpecificKey) == _queueSpecific)
|
||||
block();
|
||||
else if (_specialIsMainQueue && [NSThread isMainThread])
|
||||
block();
|
||||
else {
|
||||
if (synchronous) {
|
||||
dispatch_sync(_queue, block);
|
||||
} else {
|
||||
dispatch_async(_queue, block);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
- (bool)isCurrentQueue
|
||||
{
|
||||
if (_queueSpecific != NULL && dispatch_get_specific(SQueueSpecificKey) == _queueSpecific)
|
||||
return true;
|
||||
else if (_specialIsMainQueue && [NSThread isMainThread])
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
//
|
||||
// SQueueLocalObject.h
|
||||
// SSignalKit
|
||||
//
|
||||
// Created by Mikhail Filimonov on 13.01.2021.
|
||||
// Copyright © 2021 Telegram. All rights reserved.
|
||||
//
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
#import "SQueue.h"
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
@interface SQueueLocalObject : NSObject
|
||||
-(id)initWithQueue:(SQueue *)queue generate:(id (^)(void))next;
|
||||
-(void)with:(void (^)(id object))f;
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
//
|
||||
// SQueueLocalObject.m
|
||||
// SSignalKit
|
||||
//
|
||||
// Created by Mikhail Filimonov on 13.01.2021.
|
||||
// Copyright © 2021 Telegram. All rights reserved.
|
||||
//
|
||||
|
||||
#import "SQueueLocalObject.h"
|
||||
|
||||
@implementation SQueueLocalObject {
|
||||
SQueue *_queue;
|
||||
id valueRef;
|
||||
}
|
||||
-(id)initWithQueue:(SQueue *)queue generate:(id _Nonnull (^)(void))next {
|
||||
if (self = [super init]) {
|
||||
self->_queue = queue;
|
||||
[queue dispatch:^{
|
||||
self->valueRef = next();
|
||||
}];
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
-(void)with:(void (^)(id object))f {
|
||||
[self->_queue dispatch:^{
|
||||
f(self->valueRef);
|
||||
}];
|
||||
}
|
||||
|
||||
-(void)dealloc {
|
||||
__block id value = self->valueRef;
|
||||
self->valueRef = nil;
|
||||
[_queue dispatch:^{
|
||||
value = nil;
|
||||
}];
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
//
|
||||
// Capturer.h
|
||||
// CoreMediaMacCapture
|
||||
//
|
||||
// Created by Mikhail Filimonov on 21.06.2021.
|
||||
//
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <CoreMedia/CoreMedia.h>
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
|
||||
typedef void(^renderBlock)(CMSampleBufferRef);
|
||||
|
||||
|
||||
@interface TGCMIOCapturer : NSObject
|
||||
|
||||
|
||||
-(id)initWithDeviceId:(AVCaptureDevice *)device;
|
||||
|
||||
-(void)start:(renderBlock)renderBlock;
|
||||
-(void)stop;
|
||||
|
||||
|
||||
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
//
|
||||
// Capturer.m
|
||||
// CoreMediaMacCapture
|
||||
//
|
||||
// Created by Mikhail Filimonov on 21.06.2021.
|
||||
//
|
||||
|
||||
#import "TGCMIOCapturer.h"
|
||||
#import "TGCMIODevice.h"
|
||||
|
||||
@interface TGCMIOCapturer ()
|
||||
|
||||
@end
|
||||
|
||||
@implementation TGCMIOCapturer
|
||||
{
|
||||
AVCaptureDevice * _captureDevice;
|
||||
TGCMIODevice * _device;
|
||||
}
|
||||
-(id)initWithDeviceId:(AVCaptureDevice *)device {
|
||||
if (self = [super init]) {
|
||||
_captureDevice = device;
|
||||
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
-(void)start:(renderBlock)renderBlock {
|
||||
_device = [TGCMIODevice FindDeviceByUniqueId:_captureDevice];
|
||||
|
||||
[_device run:^(CMSampleBufferRef sampleBuffer) {
|
||||
renderBlock(sampleBuffer);
|
||||
}];
|
||||
|
||||
|
||||
}
|
||||
-(void)stop {
|
||||
[_device stop];
|
||||
}
|
||||
|
||||
-(void)dealloc {
|
||||
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
//
|
||||
// CoreMediaVideoHAL.h
|
||||
// CoreMediaMacCapture
|
||||
//
|
||||
// Created by Mikhail Filimonov on 21.06.2021.
|
||||
//
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <CoreMedia/CoreMedia.h>
|
||||
#import <CoreMediaIO/CMIOHardware.h>
|
||||
#import <CoreMedia/CoreMedia.h>
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
typedef void(^RenderBlock)(CMSampleBufferRef);
|
||||
|
||||
@interface TGCMIODevice : NSObject
|
||||
|
||||
+(TGCMIODevice * __nullable)FindDeviceByUniqueId:(AVCaptureDevice *)device;
|
||||
|
||||
-(void)run:(RenderBlock)render;
|
||||
-(void)stop;
|
||||
|
||||
-(CMIODeviceID)cmioDevice;
|
||||
@end
|
||||
|
||||
|
||||
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
193
TMessagesProj/jni/voip/tgcalls/platform/darwin/TGCMIODevice.mm
Normal file
193
TMessagesProj/jni/voip/tgcalls/platform/darwin/TGCMIODevice.mm
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
//
|
||||
// CoreMediaVideoHAL.m
|
||||
// CoreMediaMacCapture
|
||||
//
|
||||
// Created by Mikhail Filimonov on 21.06.2021.
|
||||
//
|
||||
|
||||
#import "TGCMIODevice.h"
|
||||
|
||||
#import <CoreMediaIO/CMIOHardware.h>
|
||||
#import <CoreMediaIO/CMIOHardwareStream.h>
|
||||
#import <CoreMedia/CoreMedia.h>
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
@interface TGCMIODevice ()
|
||||
{
|
||||
CMIODeviceID _deviceId;
|
||||
CMIOStreamID _streamId;
|
||||
CMSimpleQueueRef _queueRef;
|
||||
RenderBlock _renderBlock;
|
||||
}
|
||||
|
||||
-(CMSimpleQueueRef)queue;
|
||||
-(RenderBlock)block;
|
||||
|
||||
@end
|
||||
|
||||
static void handleStreamQueueAltered(CMIOStreamID streamID, void* token, void* refCon) {
|
||||
CMSampleBufferRef sb = 0;
|
||||
|
||||
TGCMIODevice *renderBlock = (__bridge TGCMIODevice *)refCon;
|
||||
CMSimpleQueueRef queueRef = [renderBlock queue];
|
||||
|
||||
while(0 != (sb = (CMSampleBufferRef)CMSimpleQueueDequeue(queueRef))) {
|
||||
renderBlock.block(sb);
|
||||
CFRelease(sb);
|
||||
}
|
||||
}
|
||||
|
||||
OSStatus GetPropertyData(CMIOObjectID objID, int32_t sel, CMIOObjectPropertyScope scope,
|
||||
UInt32 qualifierDataSize, const void* qualifierData, UInt32 dataSize,
|
||||
UInt32& dataUsed, void* data) {
|
||||
CMIOObjectPropertyAddress addr={ (CMIOObjectPropertySelector)sel, scope,
|
||||
kCMIOObjectPropertyElementMaster };
|
||||
return CMIOObjectGetPropertyData(objID, &addr, qualifierDataSize, qualifierData,
|
||||
dataSize, &dataUsed, data);
|
||||
}
|
||||
OSStatus GetPropertyData(CMIOObjectID objID, int32_t selector, UInt32 qualifierDataSize,
|
||||
const void* qualifierData, UInt32 dataSize, UInt32& dataUsed,
|
||||
void* data) {
|
||||
return GetPropertyData(objID, selector, 0, qualifierDataSize,
|
||||
qualifierData, dataSize, dataUsed, data);
|
||||
}
|
||||
|
||||
OSStatus GetPropertyDataSize(CMIOObjectID objID, int32_t sel,
|
||||
CMIOObjectPropertyScope scope, uint32_t& size) {
|
||||
CMIOObjectPropertyAddress addr={ (CMIOObjectPropertySelector)sel, scope,
|
||||
kCMIOObjectPropertyElementMaster };
|
||||
return CMIOObjectGetPropertyDataSize(objID, &addr, 0, 0, &size);
|
||||
}
|
||||
|
||||
OSStatus GetPropertyDataSize(CMIOObjectID objID, int32_t selector, uint32_t& size) {
|
||||
return GetPropertyDataSize(objID, selector, 0, size);
|
||||
}
|
||||
|
||||
OSStatus GetNumberDevices(uint32_t& cnt) {
|
||||
if(0 != GetPropertyDataSize(kCMIOObjectSystemObject, kCMIOHardwarePropertyDevices, cnt))
|
||||
return -1;
|
||||
cnt /= sizeof(CMIODeviceID);
|
||||
return 0;
|
||||
}
|
||||
|
||||
OSStatus GetDevices(uint32_t& cnt, CMIODeviceID* pDevs) {
|
||||
OSStatus status;
|
||||
uint32_t numberDevices = 0, used = 0;
|
||||
if((status = GetNumberDevices(numberDevices)) < 0)
|
||||
return status;
|
||||
if(numberDevices > (cnt = numberDevices))
|
||||
return -1;
|
||||
uint32_t size = numberDevices * sizeof(CMIODeviceID);
|
||||
return GetPropertyData(kCMIOObjectSystemObject, kCMIOHardwarePropertyDevices,
|
||||
0, NULL, size, used, pDevs);
|
||||
}
|
||||
|
||||
template< const int C_Size >
|
||||
OSStatus GetDeviceStrProp(CMIOObjectID objID, CMIOObjectPropertySelector sel,
|
||||
char (&pValue)[C_Size]) {
|
||||
CFStringRef answer = NULL;
|
||||
UInt32 dataUsed= 0;
|
||||
OSStatus status = GetPropertyData(objID, sel, 0, NULL, sizeof(answer),
|
||||
dataUsed, &answer);
|
||||
if(0 == status)// SUCCESS
|
||||
CFStringCopyUTF8String(answer, pValue);
|
||||
return status;
|
||||
}
|
||||
|
||||
template< const int C_Size >
|
||||
Boolean CFStringCopyUTF8String(CFStringRef aString, char (&pText)[C_Size]) {
|
||||
CFIndex length = CFStringGetLength(aString);
|
||||
if(sizeof(pText) < (length + 1))
|
||||
return false;
|
||||
CFIndex maxSize = CFStringGetMaximumSizeForEncoding(length, kCFStringEncodingUTF8);
|
||||
return CFStringGetCString(aString, pText, maxSize, kCFStringEncodingUTF8);
|
||||
}
|
||||
|
||||
|
||||
|
||||
uint32_t GetNumberInputStreams(CMIODeviceID devID)
|
||||
{
|
||||
uint32 size = 0;
|
||||
GetPropertyDataSize(devID, kCMIODevicePropertyStreams,
|
||||
kCMIODevicePropertyScopeInput, size);
|
||||
return size / sizeof(CMIOStreamID);
|
||||
}
|
||||
OSStatus GetInputStreams(CMIODeviceID devID, uint32_t&
|
||||
ioNumberStreams, CMIOStreamID* streamList)
|
||||
{
|
||||
ioNumberStreams = MIN(GetNumberInputStreams(devID), ioNumberStreams);
|
||||
uint32_t size = ioNumberStreams * sizeof(CMIOStreamID);
|
||||
uint32_t dataUsed = 0;
|
||||
OSStatus err = GetPropertyData(devID, kCMIODevicePropertyStreams,
|
||||
kCMIODevicePropertyScopeInput, 0,
|
||||
NULL, size, dataUsed, streamList);
|
||||
if(0 != err)
|
||||
return err;
|
||||
ioNumberStreams = size / sizeof(CMIOStreamID);
|
||||
CMIOStreamID* firstItem = &(streamList[0]);
|
||||
CMIOStreamID* lastItem = firstItem + ioNumberStreams;
|
||||
|
||||
//std::sort(firstItem, lastItem);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@implementation TGCMIODevice
|
||||
|
||||
-(id)initWithDeviceId:(CMIODeviceID)deviceId streamId:(CMIOStreamID)streamId {
|
||||
if (self = [super init]) {
|
||||
_deviceId = deviceId;
|
||||
_streamId = streamId;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
-(CMIODeviceID)cmioDevice {
|
||||
return _deviceId;
|
||||
}
|
||||
|
||||
+(TGCMIODevice *)FindDeviceByUniqueId:(AVCaptureDevice *)device {
|
||||
NSNumber *_connectionID = ((NSNumber *)[device valueForKey:@"_connectionID"]);
|
||||
CMIODeviceID deviceId = (CMIODeviceID)[_connectionID intValue];
|
||||
|
||||
uint32_t numStreams = GetNumberInputStreams(deviceId);
|
||||
CMIOStreamID* pStreams = (CMIOStreamID*)alloca(numStreams * sizeof(CMIOStreamID));
|
||||
GetInputStreams(deviceId, numStreams, pStreams);
|
||||
if (numStreams <= 0)
|
||||
return nil;
|
||||
|
||||
CMIOStreamID streamId = pStreams[0];
|
||||
return [[TGCMIODevice alloc] initWithDeviceId:deviceId streamId:streamId];
|
||||
|
||||
}
|
||||
|
||||
-(CMSimpleQueueRef)queue {
|
||||
return _queueRef;
|
||||
}
|
||||
-(RenderBlock)block {
|
||||
return _renderBlock;
|
||||
}
|
||||
|
||||
-(void)run:(RenderBlock)render {
|
||||
_renderBlock = render;
|
||||
|
||||
|
||||
|
||||
CMIOStreamCopyBufferQueue(_streamId, handleStreamQueueAltered, (void*)CFBridgingRetain(self), &_queueRef);
|
||||
|
||||
CMIODeviceStartStream(_deviceId, _streamId);
|
||||
|
||||
}
|
||||
|
||||
-(void)stop {
|
||||
CMIODeviceStopStream(_deviceId, _streamId);
|
||||
CMIOStreamCopyBufferQueue(_streamId, nil, nil, &_queueRef);
|
||||
if (_queueRef)
|
||||
CFRelease(_queueRef);
|
||||
}
|
||||
|
||||
-(void)dealloc {
|
||||
[self stop];
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
#ifndef TGRTCCVPIXELBUFFER_H
|
||||
#define TGRTCCVPIXELBUFFER_H
|
||||
|
||||
#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
|
||||
|
||||
@interface TGRTCCVPixelBuffer : RTCCVPixelBuffer
|
||||
|
||||
@property (nonatomic) bool shouldBeMirrored;
|
||||
@property (nonatomic) int deviceRelativeVideoRotation;
|
||||
|
||||
- (void)storeSampleBufferReference:(CMSampleBufferRef _Nonnull)sampleBuffer;
|
||||
|
||||
@end
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
#import "TGRTCCVPixelBuffer.h"
|
||||
|
||||
@interface TGRTCCVPixelBuffer () {
|
||||
CMSampleBufferRef _sampleBuffer;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@implementation TGRTCCVPixelBuffer
|
||||
|
||||
- (void)dealloc {
|
||||
if (_sampleBuffer) {
|
||||
CFRelease(_sampleBuffer);
|
||||
}
|
||||
}
|
||||
|
||||
- (void)storeSampleBufferReference:(CMSampleBufferRef _Nonnull)sampleBuffer {
|
||||
if (_sampleBuffer) {
|
||||
CFRelease(_sampleBuffer);
|
||||
_sampleBuffer = nil;
|
||||
}
|
||||
|
||||
if (sampleBuffer) {
|
||||
_sampleBuffer = (CMSampleBufferRef)CFRetain(sampleBuffer);
|
||||
}
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
#import "RTCMacros.h"
|
||||
#import "RTCVideoDecoderFactory.h"
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
/** This decoder factory include support for all codecs bundled with WebRTC. If using custom
|
||||
* codecs, create custom implementations of RTCVideoEncoderFactory and RTCVideoDecoderFactory.
|
||||
*/
|
||||
RTC_OBJC_EXPORT
|
||||
@interface TGRTCDefaultVideoDecoderFactory : NSObject <RTCVideoDecoderFactory>
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC Project Authors. All rights reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import "TGRTCDefaultVideoDecoderFactory.h"
|
||||
|
||||
#import "RTCH264ProfileLevelId.h"
|
||||
#import "TGRTCVideoDecoderH264.h"
|
||||
#import "api/video_codec/RTCVideoCodecConstants.h"
|
||||
#import "api/video_codec/RTCVideoDecoderVP8.h"
|
||||
#import "base/RTCVideoCodecInfo.h"
|
||||
#if defined(RTC_ENABLE_VP9)
|
||||
#import "api/video_codec/RTCVideoDecoderVP9.h"
|
||||
#endif
|
||||
#if !defined(WEBRTC_DISABLE_H265)
|
||||
#import "RTCH265ProfileLevelId.h"
|
||||
#import "TGRTCVideoDecoderH265.h"
|
||||
#endif
|
||||
|
||||
#import "sdk/objc/api/video_codec/RTCWrappedNativeVideoDecoder.h"
|
||||
#include "modules/video_coding/codecs/h264/include/h264.h"
|
||||
|
||||
@implementation TGRTCDefaultVideoDecoderFactory
|
||||
|
||||
- (NSArray<RTCVideoCodecInfo *> *)supportedCodecs {
|
||||
NSDictionary<NSString *, NSString *> *constrainedHighParams = @{
|
||||
@"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedHigh,
|
||||
@"level-asymmetry-allowed" : @"1",
|
||||
@"packetization-mode" : @"1",
|
||||
};
|
||||
RTCVideoCodecInfo *constrainedHighInfo =
|
||||
[[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH264Name
|
||||
parameters:constrainedHighParams];
|
||||
|
||||
NSDictionary<NSString *, NSString *> *constrainedBaselineParams = @{
|
||||
@"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedBaseline,
|
||||
@"level-asymmetry-allowed" : @"1",
|
||||
@"packetization-mode" : @"1",
|
||||
};
|
||||
RTCVideoCodecInfo *constrainedBaselineInfo =
|
||||
[[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH264Name
|
||||
parameters:constrainedBaselineParams];
|
||||
|
||||
RTCVideoCodecInfo *vp8Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecVp8Name];
|
||||
|
||||
#if defined(RTC_ENABLE_VP9)
|
||||
RTCVideoCodecInfo *vp9Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecVp9Name];
|
||||
#endif
|
||||
|
||||
#if !defined(WEBRTC_DISABLE_H265)
|
||||
RTCVideoCodecInfo *h265Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH265Name];
|
||||
#endif
|
||||
|
||||
NSMutableArray<RTCVideoCodecInfo *> *result = [[NSMutableArray alloc] initWithArray:@[
|
||||
constrainedHighInfo,
|
||||
constrainedBaselineInfo,
|
||||
vp8Info,
|
||||
#if defined(RTC_ENABLE_VP9)
|
||||
vp9Info,
|
||||
#endif
|
||||
]];
|
||||
|
||||
#if !defined(WEBRTC_DISABLE_H265)
|
||||
#ifdef WEBRTC_IOS
|
||||
if (@available(iOS 11.0, *)) {
|
||||
[result addObject:h265Info];
|
||||
}
|
||||
#else // WEBRTC_IOS
|
||||
if (@available(macOS 10.13, *)) {
|
||||
[result addObject:h265Info];
|
||||
}
|
||||
#endif // WEBRTC_IOS
|
||||
#endif
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
- (id<RTCVideoDecoder>)createDecoder:(RTCVideoCodecInfo *)info {
|
||||
if ([info.name isEqualToString:kRTCVideoCodecH264Name]) {
|
||||
return [[TGRTCVideoDecoderH264 alloc] init];
|
||||
} else if ([info.name isEqualToString:kRTCVideoCodecVp8Name]) {
|
||||
return [RTCVideoDecoderVP8 vp8Decoder];
|
||||
}
|
||||
|
||||
#if defined(RTC_ENABLE_VP9)
|
||||
if ([info.name isEqualToString:kRTCVideoCodecVp9Name]) {
|
||||
return [RTCVideoDecoderVP9 vp9Decoder];
|
||||
}
|
||||
#endif
|
||||
|
||||
#if !defined(WEBRTC_DISABLE_H265)
|
||||
#ifdef WEBRTC_IOS
|
||||
if (@available(iOS 11.0, *)) {
|
||||
if ([info.name isEqualToString:kRTCVideoCodecH265Name]) {
|
||||
return [[TGRTCVideoDecoderH265 alloc] init];
|
||||
}
|
||||
}
|
||||
#else // WEBRTC_IOS
|
||||
if (@available(macOS 10.13, *)) {
|
||||
if ([info.name isEqualToString:kRTCVideoCodecH265Name]) {
|
||||
return [[TGRTCVideoDecoderH265 alloc] init];
|
||||
}
|
||||
}
|
||||
#endif // WEBRTC_IOS
|
||||
#endif // !DISABLE_H265
|
||||
|
||||
return nil;
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
#import "RTCMacros.h"
|
||||
#import "RTCVideoEncoderFactory.h"
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
/** This encoder factory include support for all codecs bundled with WebRTC. If using custom
|
||||
* codecs, create custom implementations of RTCVideoEncoderFactory and RTCVideoDecoderFactory.
|
||||
*/
|
||||
RTC_OBJC_EXPORT
|
||||
@interface TGRTCDefaultVideoEncoderFactory : NSObject <RTCVideoEncoderFactory>
|
||||
|
||||
@property(nonatomic, retain) RTCVideoCodecInfo *preferredCodec;
|
||||
|
||||
- (instancetype)initWithPreferHardwareH264:(bool)preferHardwareH264 preferX264:(bool)preferX264;
|
||||
|
||||
+ (NSArray<RTCVideoCodecInfo *> *)supportedCodecs;
|
||||
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
|
|
@ -0,0 +1,172 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC Project Authors. All rights reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import "TGRTCDefaultVideoEncoderFactory.h"
|
||||
|
||||
#import "RTCH264ProfileLevelId.h"
|
||||
#import "TGRTCVideoEncoderH264.h"
|
||||
#import "api/video_codec/RTCVideoCodecConstants.h"
|
||||
#import "api/video_codec/RTCVideoEncoderVP8.h"
|
||||
#import "base/RTCVideoCodecInfo.h"
|
||||
#if defined(RTC_ENABLE_VP9)
|
||||
#import "api/video_codec/RTCVideoEncoderVP9.h"
|
||||
#endif
|
||||
#ifndef WEBRTC_DISABLE_H265
|
||||
#import "RTCH265ProfileLevelId.h"
|
||||
#import "TGRTCVideoEncoderH265.h"
|
||||
#endif
|
||||
|
||||
#import "sdk/objc/api/video_codec/RTCWrappedNativeVideoEncoder.h"
|
||||
#import "modules/video_coding/codecs/h264/include/h264.h"
|
||||
#import "h264_encoder_impl.h"
|
||||
|
||||
@interface TGRTCDefaultVideoEncoderFactory () {
|
||||
bool _preferHardwareH264;
|
||||
bool _preferX264;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@implementation TGRTCDefaultVideoEncoderFactory
|
||||
|
||||
@synthesize preferredCodec;
|
||||
|
||||
- (instancetype)initWithPreferHardwareH264:(bool)preferHardwareH264 preferX264:(bool)preferX264 {
|
||||
self = [super init];
|
||||
if (self != nil) {
|
||||
_preferHardwareH264 = preferHardwareH264;
|
||||
_preferX264 = preferX264;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
+ (NSArray<RTCVideoCodecInfo *> *)supportedCodecs {
|
||||
NSDictionary<NSString *, NSString *> *constrainedHighParams = @{
|
||||
@"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedHigh,
|
||||
@"level-asymmetry-allowed" : @"1",
|
||||
@"packetization-mode" : @"1",
|
||||
};
|
||||
RTCVideoCodecInfo *constrainedHighInfo =
|
||||
[[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH264Name
|
||||
parameters:constrainedHighParams];
|
||||
|
||||
NSDictionary<NSString *, NSString *> *constrainedBaselineParams = @{
|
||||
@"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedBaseline,
|
||||
@"level-asymmetry-allowed" : @"1",
|
||||
@"packetization-mode" : @"1",
|
||||
};
|
||||
RTCVideoCodecInfo *constrainedBaselineInfo =
|
||||
[[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH264Name
|
||||
parameters:constrainedBaselineParams];
|
||||
|
||||
RTCVideoCodecInfo *vp8Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecVp8Name];
|
||||
|
||||
#if defined(RTC_ENABLE_VP9)
|
||||
RTCVideoCodecInfo *vp9Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecVp9Name];
|
||||
#endif
|
||||
|
||||
#if !defined(WEBRTC_DISABLE_H265)
|
||||
RTCVideoCodecInfo *h265Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH265Name];
|
||||
#endif
|
||||
|
||||
NSMutableArray *result = [[NSMutableArray alloc] initWithArray:@[
|
||||
constrainedHighInfo,
|
||||
constrainedBaselineInfo,
|
||||
vp8Info,
|
||||
#if defined(RTC_ENABLE_VP9)
|
||||
vp9Info,
|
||||
#endif
|
||||
]];
|
||||
|
||||
#if !defined(WEBRTC_DISABLE_H265)
|
||||
#ifdef WEBRTC_IOS
|
||||
if (@available(iOS 11.0, *)) {
|
||||
if ([[AVAssetExportSession allExportPresets] containsObject:AVAssetExportPresetHEVCHighestQuality]) {
|
||||
[result addObject:h265Info];
|
||||
}
|
||||
}
|
||||
#else // WEBRTC_IOS
|
||||
if (@available(macOS 10.13, *)) {
|
||||
if ([[AVAssetExportSession allExportPresets] containsObject:AVAssetExportPresetHEVCHighestQuality]) {
|
||||
[result addObject:h265Info];
|
||||
}
|
||||
}
|
||||
#endif // WEBRTC_IOS
|
||||
#endif
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
- (id<RTCVideoEncoder>)createEncoder:(RTCVideoCodecInfo *)info {
|
||||
if ([info.name isEqualToString:kRTCVideoCodecH264Name]) {
|
||||
if (_preferHardwareH264) {
|
||||
return [[TGRTCVideoEncoderH264 alloc] initWithCodecInfo:info];
|
||||
} else {
|
||||
|
||||
webrtc::SdpVideoFormat videoFormat(info.name.UTF8String);
|
||||
for (NSString *key in info.parameters) {
|
||||
videoFormat.parameters.insert(std::make_pair(key.UTF8String, info.parameters[key].UTF8String));
|
||||
}
|
||||
|
||||
cricket::VideoCodec videoCodec = cricket::CreateVideoCodec(videoFormat);
|
||||
|
||||
#ifdef TGCALLS_ENABLE_X264
|
||||
if (_preferX264) {
|
||||
return [[RTC_OBJC_TYPE(RTCWrappedNativeVideoEncoder) alloc] initWithNativeEncoder:std::make_unique<webrtc::H264EncoderX264Impl>(videoCodec)];
|
||||
} else {
|
||||
#endif
|
||||
return [[RTC_OBJC_TYPE(RTCWrappedNativeVideoEncoder) alloc] initWithNativeEncoder:std::unique_ptr<webrtc::VideoEncoder>(webrtc::H264Encoder::Create(videoCodec))];
|
||||
#ifdef TGCALLS_ENABLE_X264
|
||||
}
|
||||
#endif
|
||||
}
|
||||
} else if ([info.name isEqualToString:kRTCVideoCodecVp8Name]) {
|
||||
return [RTCVideoEncoderVP8 vp8Encoder];
|
||||
}
|
||||
#if defined(RTC_ENABLE_VP9)
|
||||
if ([info.name isEqualToString:kRTCVideoCodecVp9Name]) {
|
||||
return [RTCVideoEncoderVP9 vp9Encoder];
|
||||
}
|
||||
#endif
|
||||
|
||||
#if !defined(WEBRTC_DISABLE_H265)
|
||||
#ifdef WEBRTC_IOS
|
||||
if (@available(iOS 11, *)) {
|
||||
if ([info.name isEqualToString:kRTCVideoCodecH265Name]) {
|
||||
return [[RTCVideoEncoderH265 alloc] initWithCodecInfo:info];
|
||||
}
|
||||
}
|
||||
#else // WEBRTC_IOS
|
||||
if (@available(macOS 10.13, *)) {
|
||||
if ([info.name isEqualToString:kRTCVideoCodecH265Name]) {
|
||||
return [[RTCVideoEncoderH265 alloc] initWithCodecInfo:info];
|
||||
}
|
||||
}
|
||||
#endif // WEBRTC_IOS
|
||||
#endif // !DISABLE_H265
|
||||
|
||||
return nil;
|
||||
}
|
||||
|
||||
- (NSArray<RTCVideoCodecInfo *> *)supportedCodecs {
|
||||
NSMutableArray<RTCVideoCodecInfo *> *codecs = [[[self class] supportedCodecs] mutableCopy];
|
||||
|
||||
NSMutableArray<RTCVideoCodecInfo *> *orderedCodecs = [NSMutableArray array];
|
||||
NSUInteger index = [codecs indexOfObject:self.preferredCodec];
|
||||
if (index != NSNotFound) {
|
||||
[orderedCodecs addObject:[codecs objectAtIndex:index]];
|
||||
[codecs removeObjectAtIndex:index];
|
||||
}
|
||||
[orderedCodecs addObjectsFromArray:codecs];
|
||||
|
||||
return [orderedCodecs copy];
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
#import "RTCMacros.h"
|
||||
#import "RTCVideoDecoder.h"
|
||||
|
||||
RTC_OBJC_EXPORT
|
||||
@interface RTC_OBJC_TYPE (TGRTCVideoDecoderH264) : NSObject <RTC_OBJC_TYPE(RTCVideoDecoder)>
|
||||
@end
|
||||
|
|
@ -0,0 +1,292 @@
|
|||
/*
|
||||
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*
|
||||
*/
|
||||
|
||||
#import "TGRTCVideoDecoderH264.h"
|
||||
|
||||
#import <VideoToolbox/VideoToolbox.h>
|
||||
|
||||
#import "base/RTCVideoFrame.h"
|
||||
#import "base/RTCVideoFrameBuffer.h"
|
||||
#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
|
||||
#import "helpers.h"
|
||||
#import "helpers/scoped_cftyperef.h"
|
||||
|
||||
#if defined(WEBRTC_IOS)
|
||||
#import "helpers/UIDevice+RTCDevice.h"
|
||||
#endif
|
||||
|
||||
#include "modules/video_coding/include/video_error_codes.h"
|
||||
#include "rtc_base/checks.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "rtc_base/time_utils.h"
|
||||
#include "sdk/objc/components/video_codec/nalu_rewriter.h"
|
||||
|
||||
// Struct that we pass to the decoder per frame to decode. We receive it again
|
||||
// in the decoder callback.
|
||||
struct RTCFrameDecodeParams {
|
||||
RTCFrameDecodeParams(RTCVideoDecoderCallback cb, int64_t ts) : callback(cb), timestamp(ts) {}
|
||||
RTCVideoDecoderCallback callback;
|
||||
int64_t timestamp;
|
||||
};
|
||||
|
||||
@interface RTC_OBJC_TYPE (TGRTCVideoDecoderH264)
|
||||
() - (void)setError : (OSStatus)error;
|
||||
@end
|
||||
|
||||
// This is the callback function that VideoToolbox calls when decode is
|
||||
// complete.
|
||||
static void decompressionOutputCallback(void *decoderRef,
|
||||
void *params,
|
||||
OSStatus status,
|
||||
VTDecodeInfoFlags infoFlags,
|
||||
CVImageBufferRef imageBuffer,
|
||||
CMTime timestamp,
|
||||
CMTime duration) {
|
||||
std::unique_ptr<RTCFrameDecodeParams> decodeParams(
|
||||
reinterpret_cast<RTCFrameDecodeParams *>(params));
|
||||
if (status != noErr) {
|
||||
RTC_OBJC_TYPE(TGRTCVideoDecoderH264) *decoder =
|
||||
(__bridge RTC_OBJC_TYPE(TGRTCVideoDecoderH264) *)decoderRef;
|
||||
[decoder setError:status];
|
||||
RTC_LOG(LS_ERROR) << "Failed to decode frame. Status: " << status;
|
||||
return;
|
||||
}
|
||||
// TODO(tkchin): Handle CVO properly.
|
||||
@autoreleasepool {
|
||||
RTC_OBJC_TYPE(RTCCVPixelBuffer) *frameBuffer =
|
||||
[[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:imageBuffer];
|
||||
RTC_OBJC_TYPE(RTCVideoFrame) *decodedFrame = [[RTC_OBJC_TYPE(RTCVideoFrame) alloc]
|
||||
initWithBuffer:frameBuffer
|
||||
rotation:RTCVideoRotation_0
|
||||
timeStampNs:CMTimeGetSeconds(timestamp) * rtc::kNumNanosecsPerSec];
|
||||
decodedFrame.timeStamp = decodeParams->timestamp;
|
||||
decodeParams->callback(decodedFrame);
|
||||
}
|
||||
}
|
||||
|
||||
// Decoder.
|
||||
@implementation RTC_OBJC_TYPE (TGRTCVideoDecoderH264) {
|
||||
CMVideoFormatDescriptionRef _videoFormat;
|
||||
CMMemoryPoolRef _memoryPool;
|
||||
VTDecompressionSessionRef _decompressionSession;
|
||||
RTCVideoDecoderCallback _callback;
|
||||
OSStatus _error;
|
||||
}
|
||||
|
||||
- (instancetype)init {
|
||||
self = [super init];
|
||||
if (self) {
|
||||
_memoryPool = CMMemoryPoolCreate(nil);
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
CMMemoryPoolInvalidate(_memoryPool);
|
||||
CFRelease(_memoryPool);
|
||||
[self destroyDecompressionSession];
|
||||
[self setVideoFormat:nullptr];
|
||||
}
|
||||
|
||||
- (NSInteger)startDecodeWithNumberOfCores:(int)numberOfCores {
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
- (NSInteger)decode:(RTC_OBJC_TYPE(RTCEncodedImage) *)inputImage
|
||||
missingFrames:(BOOL)missingFrames
|
||||
codecSpecificInfo:(nullable id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)>)info
|
||||
renderTimeMs:(int64_t)renderTimeMs {
|
||||
RTC_DCHECK(inputImage.buffer);
|
||||
|
||||
if (_error != noErr) {
|
||||
RTC_LOG(LS_WARNING) << "Last frame decode failed.";
|
||||
_error = noErr;
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
|
||||
rtc::ScopedCFTypeRef<CMVideoFormatDescriptionRef> inputFormat =
|
||||
rtc::ScopedCF(webrtc::CreateVideoFormatDescription((uint8_t *)inputImage.buffer.bytes,
|
||||
inputImage.buffer.length));
|
||||
if (inputFormat) {
|
||||
// Check if the video format has changed, and reinitialize decoder if
|
||||
// needed.
|
||||
if (!CMFormatDescriptionEqual(inputFormat.get(), _videoFormat)) {
|
||||
[self setVideoFormat:inputFormat.get()];
|
||||
int resetDecompressionSessionError = [self resetDecompressionSession];
|
||||
if (resetDecompressionSessionError != WEBRTC_VIDEO_CODEC_OK) {
|
||||
return resetDecompressionSessionError;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!_videoFormat) {
|
||||
// We received a frame but we don't have format information so we can't
|
||||
// decode it.
|
||||
// This can happen after backgrounding. We need to wait for the next
|
||||
// sps/pps before we can resume so we request a keyframe by returning an
|
||||
// error.
|
||||
RTC_LOG(LS_WARNING) << "Missing video format. Frame with sps/pps required.";
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
CMSampleBufferRef sampleBuffer = nullptr;
|
||||
if (!webrtc::H264AnnexBBufferToCMSampleBuffer((uint8_t *)inputImage.buffer.bytes,
|
||||
inputImage.buffer.length,
|
||||
_videoFormat,
|
||||
&sampleBuffer,
|
||||
_memoryPool)) {
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
RTC_DCHECK(sampleBuffer);
|
||||
VTDecodeFrameFlags decodeFlags = kVTDecodeFrame_EnableAsynchronousDecompression;
|
||||
std::unique_ptr<RTCFrameDecodeParams> frameDecodeParams;
|
||||
frameDecodeParams.reset(new RTCFrameDecodeParams(_callback, inputImage.timeStamp));
|
||||
OSStatus status = VTDecompressionSessionDecodeFrame(
|
||||
_decompressionSession, sampleBuffer, decodeFlags, frameDecodeParams.release(), nullptr);
|
||||
#if defined(WEBRTC_IOS)
|
||||
// Re-initialize the decoder if we have an invalid session while the app is
|
||||
// active or decoder malfunctions and retry the decode request.
|
||||
if ((status == kVTInvalidSessionErr || status == kVTVideoDecoderMalfunctionErr) &&
|
||||
[self resetDecompressionSession] == WEBRTC_VIDEO_CODEC_OK) {
|
||||
RTC_LOG(LS_INFO) << "Failed to decode frame with code: " << status
|
||||
<< " retrying decode after decompression session reset";
|
||||
frameDecodeParams.reset(new RTCFrameDecodeParams(_callback, inputImage.timeStamp));
|
||||
status = VTDecompressionSessionDecodeFrame(
|
||||
_decompressionSession, sampleBuffer, decodeFlags, frameDecodeParams.release(), nullptr);
|
||||
}
|
||||
#endif
|
||||
CFRelease(sampleBuffer);
|
||||
if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to decode frame with code: " << status;
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
- (void)setCallback:(RTCVideoDecoderCallback)callback {
|
||||
_callback = callback;
|
||||
}
|
||||
|
||||
- (void)setError:(OSStatus)error {
|
||||
_error = error;
|
||||
}
|
||||
|
||||
- (NSInteger)releaseDecoder {
|
||||
// Need to invalidate the session so that callbacks no longer occur and it
|
||||
// is safe to null out the callback.
|
||||
[self destroyDecompressionSession];
|
||||
[self setVideoFormat:nullptr];
|
||||
_callback = nullptr;
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
#pragma mark - Private
|
||||
|
||||
- (int)resetDecompressionSession {
|
||||
[self destroyDecompressionSession];
|
||||
|
||||
// Need to wait for the first SPS to initialize decoder.
|
||||
if (!_videoFormat) {
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
// Set keys for OpenGL and IOSurface compatibilty, which makes the encoder
|
||||
// create pixel buffers with GPU backed memory. The intent here is to pass
|
||||
// the pixel buffers directly so we avoid a texture upload later during
|
||||
// rendering. This currently is moot because we are converting back to an
|
||||
// I420 frame after decode, but eventually we will be able to plumb
|
||||
// CVPixelBuffers directly to the renderer.
|
||||
// TODO(tkchin): Maybe only set OpenGL/IOSurface keys if we know that that
|
||||
// we can pass CVPixelBuffers as native handles in decoder output.
|
||||
#if TARGET_OS_SIMULATOR
|
||||
static size_t const attributesSize = 2;
|
||||
#else
|
||||
static size_t const attributesSize = 3;
|
||||
#endif
|
||||
|
||||
CFTypeRef keys[attributesSize] = {
|
||||
#if defined(WEBRTC_IOS)
|
||||
kCVPixelBufferMetalCompatibilityKey,
|
||||
#elif defined(WEBRTC_MAC)
|
||||
kCVPixelBufferOpenGLCompatibilityKey,
|
||||
#endif
|
||||
#if !(TARGET_OS_SIMULATOR)
|
||||
kCVPixelBufferIOSurfacePropertiesKey,
|
||||
#endif
|
||||
kCVPixelBufferPixelFormatTypeKey};
|
||||
CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0);
|
||||
int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
|
||||
CFNumberRef pixelFormat = CFNumberCreate(nullptr, kCFNumberLongType, &nv12type);
|
||||
#if TARGET_OS_SIMULATOR
|
||||
CFTypeRef values[attributesSize] = {kCFBooleanTrue, pixelFormat};
|
||||
#else
|
||||
CFTypeRef values[attributesSize] = {kCFBooleanTrue, ioSurfaceValue, pixelFormat};
|
||||
#endif
|
||||
|
||||
CFDictionaryRef attributes = CreateCFTypeDictionary(keys, values, attributesSize);
|
||||
if (ioSurfaceValue) {
|
||||
CFRelease(ioSurfaceValue);
|
||||
ioSurfaceValue = nullptr;
|
||||
}
|
||||
if (pixelFormat) {
|
||||
CFRelease(pixelFormat);
|
||||
pixelFormat = nullptr;
|
||||
}
|
||||
VTDecompressionOutputCallbackRecord record = {
|
||||
decompressionOutputCallback, (__bridge void *)self,
|
||||
};
|
||||
OSStatus status = VTDecompressionSessionCreate(
|
||||
nullptr, _videoFormat, nullptr, attributes, &record, &_decompressionSession);
|
||||
CFRelease(attributes);
|
||||
if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to create decompression session: " << status;
|
||||
[self destroyDecompressionSession];
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
[self configureDecompressionSession];
|
||||
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
- (void)configureDecompressionSession {
|
||||
RTC_DCHECK(_decompressionSession);
|
||||
#if defined(WEBRTC_IOS)
|
||||
VTSessionSetProperty(_decompressionSession, kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue);
|
||||
#endif
|
||||
}
|
||||
|
||||
- (void)destroyDecompressionSession {
|
||||
if (_decompressionSession) {
|
||||
#if defined(WEBRTC_IOS)
|
||||
VTDecompressionSessionWaitForAsynchronousFrames(_decompressionSession);
|
||||
#endif
|
||||
VTDecompressionSessionInvalidate(_decompressionSession);
|
||||
CFRelease(_decompressionSession);
|
||||
_decompressionSession = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)setVideoFormat:(CMVideoFormatDescriptionRef)videoFormat {
|
||||
if (_videoFormat == videoFormat) {
|
||||
return;
|
||||
}
|
||||
if (_videoFormat) {
|
||||
CFRelease(_videoFormat);
|
||||
}
|
||||
_videoFormat = videoFormat;
|
||||
if (_videoFormat) {
|
||||
CFRetain(_videoFormat);
|
||||
}
|
||||
}
|
||||
|
||||
- (NSString *)implementationName {
|
||||
return @"VideoToolbox";
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef WEBRTC_DISABLE_H265
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
#import "RTCMacros.h"
|
||||
#import "RTCVideoDecoder.h"
|
||||
|
||||
RTC_OBJC_EXPORT
|
||||
API_AVAILABLE(ios(11.0))
|
||||
@interface TGRTCVideoDecoderH265 : NSObject <RTCVideoDecoder>
|
||||
@end
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,359 @@
|
|||
/*
|
||||
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*
|
||||
*/
|
||||
|
||||
#ifndef WEBRTC_DISABLE_H265
|
||||
|
||||
#import "TGRTCVideoDecoderH265.h"
|
||||
|
||||
#import <VideoToolbox/VideoToolbox.h>
|
||||
|
||||
#import "base/RTCVideoFrame.h"
|
||||
#import "base/RTCVideoFrameBuffer.h"
|
||||
#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
|
||||
#import "helpers.h"
|
||||
#import "helpers/scoped_cftyperef.h"
|
||||
|
||||
#if defined(WEBRTC_IOS)
|
||||
#import "helpers/UIDevice+RTCDevice.h"
|
||||
#endif
|
||||
|
||||
#include "modules/video_coding/include/video_error_codes.h"
|
||||
#include "rtc_base/checks.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "rtc_base/time_utils.h"
|
||||
#include "rtc_base/thread.h"
|
||||
#include "sdk/objc/components/video_codec/nalu_rewriter.h"
|
||||
#include "h265_nalu_rewriter.h"
|
||||
|
||||
#include "StaticThreads.h"
|
||||
|
||||
@interface MarkedDecodedH2651RTCCVPixelBuffer : RTCCVPixelBuffer
|
||||
|
||||
@end
|
||||
|
||||
@implementation MarkedDecodedH2651RTCCVPixelBuffer
|
||||
|
||||
@end
|
||||
|
||||
typedef void (^TGRTCVideoDecoderRequestKeyframeCallback)();
|
||||
|
||||
// Struct that we pass to the decoder per frame to decode. We receive it again
|
||||
// in the decoder callback.
|
||||
struct RTCH265FrameDecodeParams {
|
||||
RTCH265FrameDecodeParams(RTCVideoDecoderCallback cb, int64_t ts, TGRTCVideoDecoderRequestKeyframeCallback requestFrame)
|
||||
: callback(cb), timestamp(ts), requestFrame(requestFrame) {}
|
||||
RTCVideoDecoderCallback callback;
|
||||
int64_t timestamp;
|
||||
TGRTCVideoDecoderRequestKeyframeCallback requestFrame;
|
||||
};
|
||||
|
||||
// This is the callback function that VideoToolbox calls when decode is
|
||||
// complete.
|
||||
static void tg_h265DecompressionOutputCallback(void* decoder,
|
||||
void* params,
|
||||
OSStatus status,
|
||||
VTDecodeInfoFlags infoFlags,
|
||||
CVImageBufferRef imageBuffer,
|
||||
CMTime timestamp,
|
||||
CMTime duration) {
|
||||
std::unique_ptr<RTCH265FrameDecodeParams> decodeParams(
|
||||
reinterpret_cast<RTCH265FrameDecodeParams*>(params));
|
||||
if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to decode frame. Status: " << status;
|
||||
if (status == -12909) {
|
||||
decodeParams->requestFrame();
|
||||
}
|
||||
return;
|
||||
}
|
||||
// TODO(tkchin): Handle CVO properly.
|
||||
RTCCVPixelBuffer* frameBuffer =
|
||||
[[MarkedDecodedH2651RTCCVPixelBuffer alloc] initWithPixelBuffer:imageBuffer];
|
||||
RTCVideoFrame* decodedFrame = [[RTCVideoFrame alloc]
|
||||
initWithBuffer:frameBuffer
|
||||
rotation:RTCVideoRotation_0
|
||||
timeStampNs:CMTimeGetSeconds(timestamp) * rtc::kNumNanosecsPerSec];
|
||||
decodedFrame.timeStamp = (int32_t)decodeParams->timestamp;
|
||||
decodeParams->callback(decodedFrame);
|
||||
}
|
||||
|
||||
@interface TGRTCVideoDecoderH265RequestKeyframeHolder : NSObject
|
||||
|
||||
@property (nonatomic, strong) NSLock *lock;
|
||||
@property (nonatomic) bool shouldRequestKeyframe;
|
||||
|
||||
@end
|
||||
|
||||
@implementation TGRTCVideoDecoderH265RequestKeyframeHolder
|
||||
|
||||
- (instancetype)init {
|
||||
self = [super init];
|
||||
if (self != nil) {
|
||||
_lock = [[NSLock alloc] init];
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
// Decoder.
|
||||
@implementation TGRTCVideoDecoderH265 {
|
||||
CMVideoFormatDescriptionRef _videoFormat;
|
||||
VTDecompressionSessionRef _decompressionSession;
|
||||
RTCVideoDecoderCallback _callback;
|
||||
TGRTCVideoDecoderH265RequestKeyframeHolder *_requestKeyframeHolder;
|
||||
TGRTCVideoDecoderRequestKeyframeCallback _requestFrame;
|
||||
OSStatus _error;
|
||||
}
|
||||
|
||||
- (instancetype)init {
|
||||
if (self = [super init]) {
|
||||
_requestKeyframeHolder = [[TGRTCVideoDecoderH265RequestKeyframeHolder alloc] init];
|
||||
TGRTCVideoDecoderH265RequestKeyframeHolder *requestKeyframeHolder = _requestKeyframeHolder;
|
||||
_requestFrame = ^{
|
||||
[requestKeyframeHolder.lock lock];
|
||||
requestKeyframeHolder.shouldRequestKeyframe = true;
|
||||
[requestKeyframeHolder.lock unlock];
|
||||
};
|
||||
NSNotificationCenter *center = [NSNotificationCenter defaultCenter];
|
||||
#ifdef WEBRTC_IOS
|
||||
[center addObserver:self
|
||||
selector:@selector(handleApplicationDidBecomeActive:)
|
||||
name:UIApplicationWillEnterForegroundNotification
|
||||
object:[UIApplication sharedApplication]];
|
||||
#endif
|
||||
}
|
||||
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
[self destroyDecompressionSession];
|
||||
[self setVideoFormat:nullptr];
|
||||
[[NSNotificationCenter defaultCenter] removeObserver:self];
|
||||
}
|
||||
|
||||
- (NSInteger)startDecodeWithNumberOfCores:(int)numberOfCores {
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
- (void)handleApplicationDidBecomeActive:(NSNotification *)notification {
|
||||
__weak TGRTCVideoDecoderH265 *weakSelf = self;
|
||||
tgcalls::StaticThreads::getMediaThread()->PostTask([weakSelf]() {
|
||||
__strong TGRTCVideoDecoderH265 *strongSelf = weakSelf;
|
||||
if (strongSelf == nil) {
|
||||
return;
|
||||
}
|
||||
strongSelf->_videoFormat = nil;
|
||||
});
|
||||
}
|
||||
|
||||
- (NSInteger)decode:(RTCEncodedImage*)inputImage
|
||||
missingFrames:(BOOL)missingFrames
|
||||
codecSpecificInfo:(__nullable id<RTCCodecSpecificInfo>)info
|
||||
renderTimeMs:(int64_t)renderTimeMs {
|
||||
RTC_DCHECK(inputImage.buffer);
|
||||
|
||||
if (_error != noErr) {
|
||||
RTC_LOG(LS_WARNING) << "Last frame decode failed.";
|
||||
_error = noErr;
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
|
||||
rtc::ScopedCFTypeRef<CMVideoFormatDescriptionRef> inputFormat =
|
||||
rtc::ScopedCF(webrtc::CreateH265VideoFormatDescription(
|
||||
(uint8_t*)inputImage.buffer.bytes, inputImage.buffer.length));
|
||||
if (inputFormat) {
|
||||
CMVideoDimensions dimensions =
|
||||
CMVideoFormatDescriptionGetDimensions(inputFormat.get());
|
||||
RTC_LOG(LS_INFO) << "Resolution: " << dimensions.width << " x "
|
||||
<< dimensions.height;
|
||||
// Check if the video format has changed, and reinitialize decoder if
|
||||
// needed.
|
||||
if (!CMFormatDescriptionEqual(inputFormat.get(), _videoFormat)) {
|
||||
[self setVideoFormat:inputFormat.get()];
|
||||
int resetDecompressionSessionError = [self resetDecompressionSession];
|
||||
if (resetDecompressionSessionError != WEBRTC_VIDEO_CODEC_OK) {
|
||||
return resetDecompressionSessionError;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!_videoFormat) {
|
||||
// We received a frame but we don't have format information so we can't
|
||||
// decode it.
|
||||
// This can happen after backgrounding. We need to wait for the next
|
||||
// sps/pps before we can resume so we request a keyframe by returning an
|
||||
// error.
|
||||
RTC_LOG(LS_WARNING) << "Missing video format. Frame with sps/pps required.";
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
CMSampleBufferRef sampleBuffer = nullptr;
|
||||
if (!webrtc::H265AnnexBBufferToCMSampleBuffer(
|
||||
(uint8_t*)inputImage.buffer.bytes, inputImage.buffer.length,
|
||||
_videoFormat, &sampleBuffer)) {
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
RTC_DCHECK(sampleBuffer);
|
||||
VTDecodeFrameFlags decodeFlags =
|
||||
kVTDecodeFrame_EnableAsynchronousDecompression;
|
||||
std::unique_ptr<RTCH265FrameDecodeParams> frameDecodeParams;
|
||||
frameDecodeParams.reset(
|
||||
new RTCH265FrameDecodeParams(_callback, inputImage.timeStamp, _requestFrame));
|
||||
OSStatus status = VTDecompressionSessionDecodeFrame(
|
||||
_decompressionSession, sampleBuffer, decodeFlags,
|
||||
frameDecodeParams.release(), nullptr);
|
||||
#if defined(WEBRTC_IOS)
|
||||
// Re-initialize the decoder if we have an invalid session while the app is
|
||||
// active and retry the decode request.
|
||||
if (status == kVTInvalidSessionErr &&
|
||||
[self resetDecompressionSession] == WEBRTC_VIDEO_CODEC_OK) {
|
||||
frameDecodeParams.reset(
|
||||
new RTCH265FrameDecodeParams(_callback, inputImage.timeStamp, _requestFrame));
|
||||
status = VTDecompressionSessionDecodeFrame(
|
||||
_decompressionSession, sampleBuffer, decodeFlags,
|
||||
frameDecodeParams.release(), nullptr);
|
||||
}
|
||||
#endif
|
||||
CFRelease(sampleBuffer);
|
||||
if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to decode frame with code: " << status;
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
|
||||
bool requestKeyframe = false;
|
||||
|
||||
[_requestKeyframeHolder.lock lock];
|
||||
if (_requestKeyframeHolder.shouldRequestKeyframe) {
|
||||
_requestKeyframeHolder.shouldRequestKeyframe = false;
|
||||
requestKeyframe = true;
|
||||
}
|
||||
[_requestKeyframeHolder.lock unlock];
|
||||
|
||||
if (requestKeyframe) {
|
||||
RTC_LOG(LS_ERROR) << "Decoder asynchronously asked to request keyframe";
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
- (void)setCallback:(RTCVideoDecoderCallback)callback {
|
||||
_callback = callback;
|
||||
}
|
||||
|
||||
- (NSInteger)releaseDecoder {
|
||||
// Need to invalidate the session so that callbacks no longer occur and it
|
||||
// is safe to null out the callback.
|
||||
[self destroyDecompressionSession];
|
||||
[self setVideoFormat:nullptr];
|
||||
_callback = nullptr;
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
#pragma mark - Private
|
||||
|
||||
- (int)resetDecompressionSession {
|
||||
[self destroyDecompressionSession];
|
||||
|
||||
// Need to wait for the first SPS to initialize decoder.
|
||||
if (!_videoFormat) {
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
// Set keys for OpenGL and IOSurface compatibilty, which makes the encoder
|
||||
// create pixel buffers with GPU backed memory. The intent here is to pass
|
||||
// the pixel buffers directly so we avoid a texture upload later during
|
||||
// rendering. This currently is moot because we are converting back to an
|
||||
// I420 frame after decode, but eventually we will be able to plumb
|
||||
// CVPixelBuffers directly to the renderer.
|
||||
// TODO(tkchin): Maybe only set OpenGL/IOSurface keys if we know that that
|
||||
// we can pass CVPixelBuffers as native handles in decoder output.
|
||||
static size_t const attributesSize = 3;
|
||||
CFTypeRef keys[attributesSize] = {
|
||||
#if defined(WEBRTC_IOS)
|
||||
kCVPixelBufferOpenGLESCompatibilityKey,
|
||||
#elif defined(WEBRTC_MAC)
|
||||
kCVPixelBufferOpenGLCompatibilityKey,
|
||||
#endif
|
||||
kCVPixelBufferIOSurfacePropertiesKey,
|
||||
kCVPixelBufferPixelFormatTypeKey
|
||||
};
|
||||
CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0);
|
||||
int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
|
||||
CFNumberRef pixelFormat =
|
||||
CFNumberCreate(nullptr, kCFNumberLongType, &nv12type);
|
||||
CFTypeRef values[attributesSize] = {kCFBooleanTrue, ioSurfaceValue,
|
||||
pixelFormat};
|
||||
CFDictionaryRef attributes =
|
||||
CreateCFTypeDictionary(keys, values, attributesSize);
|
||||
if (ioSurfaceValue) {
|
||||
CFRelease(ioSurfaceValue);
|
||||
ioSurfaceValue = nullptr;
|
||||
}
|
||||
if (pixelFormat) {
|
||||
CFRelease(pixelFormat);
|
||||
pixelFormat = nullptr;
|
||||
}
|
||||
VTDecompressionOutputCallbackRecord record = {
|
||||
tg_h265DecompressionOutputCallback,
|
||||
nullptr,
|
||||
};
|
||||
OSStatus status =
|
||||
VTDecompressionSessionCreate(nullptr, _videoFormat, nullptr, attributes,
|
||||
&record, &_decompressionSession);
|
||||
CFRelease(attributes);
|
||||
if (status != noErr) {
|
||||
[self destroyDecompressionSession];
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
[self configureDecompressionSession];
|
||||
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
- (void)configureDecompressionSession {
|
||||
RTC_DCHECK(_decompressionSession);
|
||||
#if defined(WEBRTC_IOS)
|
||||
// VTSessionSetProperty(_decompressionSession,
|
||||
// kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue);
|
||||
#endif
|
||||
}
|
||||
|
||||
- (void)destroyDecompressionSession {
|
||||
if (_decompressionSession) {
|
||||
#if defined(WEBRTC_IOS)
|
||||
VTDecompressionSessionWaitForAsynchronousFrames(_decompressionSession);
|
||||
#endif
|
||||
VTDecompressionSessionInvalidate(_decompressionSession);
|
||||
CFRelease(_decompressionSession);
|
||||
_decompressionSession = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)setVideoFormat:(CMVideoFormatDescriptionRef)videoFormat {
|
||||
if (_videoFormat == videoFormat) {
|
||||
return;
|
||||
}
|
||||
if (_videoFormat) {
|
||||
CFRelease(_videoFormat);
|
||||
}
|
||||
_videoFormat = videoFormat;
|
||||
if (_videoFormat) {
|
||||
CFRetain(_videoFormat);
|
||||
}
|
||||
}
|
||||
|
||||
- (NSString*)implementationName {
|
||||
return @"VideoToolbox";
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
#import "RTCMacros.h"
|
||||
#import "RTCVideoCodecInfo.h"
|
||||
#import "RTCVideoEncoder.h"
|
||||
|
||||
RTC_OBJC_EXPORT
|
||||
@interface TGRTCVideoEncoderH264 : NSObject <RTCVideoEncoder>
|
||||
|
||||
- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo *)codecInfo;
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,851 @@
|
|||
/*
|
||||
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*
|
||||
*/
|
||||
|
||||
#import "TGRTCVideoEncoderH264.h"
|
||||
|
||||
#import <VideoToolbox/VideoToolbox.h>
|
||||
#include <vector>
|
||||
|
||||
#if defined(WEBRTC_IOS)
|
||||
#import "helpers/UIDevice+RTCDevice.h"
|
||||
#endif
|
||||
#import "RTCCodecSpecificInfoH264.h"
|
||||
#import "RTCH264ProfileLevelId.h"
|
||||
#import "api/peerconnection/RTCVideoCodecInfo+Private.h"
|
||||
#import "base/RTCCodecSpecificInfo.h"
|
||||
#import "base/RTCI420Buffer.h"
|
||||
#import "base/RTCVideoEncoder.h"
|
||||
#import "base/RTCVideoFrame.h"
|
||||
#import "base/RTCVideoFrameBuffer.h"
|
||||
#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
|
||||
#import "helpers.h"
|
||||
|
||||
#include "common_video/h264/h264_bitstream_parser.h"
|
||||
|
||||
#include "api/video_codecs/h264_profile_level_id.h"
|
||||
|
||||
#include "common_video/include/bitrate_adjuster.h"
|
||||
#include "modules/include/module_common_types.h"
|
||||
#include "modules/video_coding/include/video_error_codes.h"
|
||||
#include "rtc_base/buffer.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "rtc_base/time_utils.h"
|
||||
#include "sdk/objc/components/video_codec/nalu_rewriter.h"
|
||||
#include "third_party/libyuv/include/libyuv/convert_from.h"
|
||||
|
||||
@interface TGRTCVideoEncoderH264 ()
|
||||
|
||||
- (void)frameWasEncoded:(OSStatus)status
|
||||
flags:(VTEncodeInfoFlags)infoFlags
|
||||
sampleBuffer:(CMSampleBufferRef)sampleBuffer
|
||||
codecSpecificInfo:(id<RTCCodecSpecificInfo>)codecSpecificInfo
|
||||
width:(int32_t)width
|
||||
height:(int32_t)height
|
||||
renderTimeMs:(int64_t)renderTimeMs
|
||||
timestamp:(uint32_t)timestamp
|
||||
rotation:(RTCVideoRotation)rotation;
|
||||
|
||||
@end
|
||||
|
||||
namespace { // anonymous namespace
|
||||
|
||||
// The ratio between kVTCompressionPropertyKey_DataRateLimits and
|
||||
// kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher
|
||||
// than the average bit rate to avoid undershooting the target.
|
||||
const float kLimitToAverageBitRateFactor = 1.5f;
|
||||
// These thresholds deviate from the default h264 QP thresholds, as they
|
||||
// have been found to work better on devices that support VideoToolbox
|
||||
const int kLowH264QpThreshold = 28;
|
||||
const int kHighH264QpThreshold = 39;
|
||||
|
||||
const OSType kNV12PixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
|
||||
|
||||
// Struct that we pass to the encoder per frame to encode. We receive it again
|
||||
// in the encoder callback.
|
||||
struct RTCFrameEncodeParams {
|
||||
RTCFrameEncodeParams(TGRTCVideoEncoderH264 *e,
|
||||
RTCCodecSpecificInfoH264 *csi,
|
||||
int32_t w,
|
||||
int32_t h,
|
||||
int64_t rtms,
|
||||
uint32_t ts,
|
||||
RTCVideoRotation r)
|
||||
: encoder(e), width(w), height(h), render_time_ms(rtms), timestamp(ts), rotation(r) {
|
||||
if (csi) {
|
||||
codecSpecificInfo = csi;
|
||||
} else {
|
||||
codecSpecificInfo = [[RTCCodecSpecificInfoH264 alloc] init];
|
||||
}
|
||||
}
|
||||
|
||||
TGRTCVideoEncoderH264 *encoder;
|
||||
RTCCodecSpecificInfoH264 *codecSpecificInfo;
|
||||
int32_t width;
|
||||
int32_t height;
|
||||
int64_t render_time_ms;
|
||||
uint32_t timestamp;
|
||||
RTCVideoRotation rotation;
|
||||
};
|
||||
|
||||
// We receive I420Frames as input, but we need to feed CVPixelBuffers into the
|
||||
// encoder. This performs the copy and format conversion.
|
||||
// TODO(tkchin): See if encoder will accept i420 frames and compare performance.
|
||||
static bool CopyVideoFrameToNV12PixelBuffer(id<RTCI420Buffer> frameBuffer, CVPixelBufferRef pixelBuffer) {
|
||||
RTC_DCHECK(pixelBuffer);
|
||||
RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixelBuffer), kNV12PixelFormat);
|
||||
RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixelBuffer, 0), frameBuffer.height);
|
||||
RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixelBuffer, 0), frameBuffer.width);
|
||||
|
||||
CVReturn cvRet = CVPixelBufferLockBaseAddress(pixelBuffer, 0);
|
||||
if (cvRet != kCVReturnSuccess) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
|
||||
return false;
|
||||
}
|
||||
uint8_t *dstY = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
|
||||
int dstStrideY = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
|
||||
uint8_t *dstUV = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
|
||||
int dstStrideUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
|
||||
// Convert I420 to NV12.
|
||||
int ret = libyuv::I420ToNV12(frameBuffer.dataY,
|
||||
frameBuffer.strideY,
|
||||
frameBuffer.dataU,
|
||||
frameBuffer.strideU,
|
||||
frameBuffer.dataV,
|
||||
frameBuffer.strideV,
|
||||
dstY,
|
||||
dstStrideY,
|
||||
dstUV,
|
||||
dstStrideUV,
|
||||
frameBuffer.width,
|
||||
frameBuffer.height);
|
||||
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
|
||||
if (ret) {
|
||||
RTC_LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) {
|
||||
if (!pixel_buffer_pool) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to get pixel buffer pool.";
|
||||
return nullptr;
|
||||
}
|
||||
CVPixelBufferRef pixel_buffer;
|
||||
CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool, &pixel_buffer);
|
||||
if (ret != kCVReturnSuccess) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret;
|
||||
// We probably want to drop frames here, since failure probably means
|
||||
// that the pool is empty.
|
||||
return nullptr;
|
||||
}
|
||||
return pixel_buffer;
|
||||
}
|
||||
|
||||
// This is the callback function that VideoToolbox calls when encode is
|
||||
// complete. From inspection this happens on its own queue.
|
||||
static void compressionOutputCallback(void *encoder,
|
||||
void *params,
|
||||
OSStatus status,
|
||||
VTEncodeInfoFlags infoFlags,
|
||||
CMSampleBufferRef sampleBuffer) {
|
||||
if (!params) {
|
||||
// If there are pending callbacks when the encoder is destroyed, this can happen.
|
||||
return;
|
||||
}
|
||||
std::unique_ptr<RTCFrameEncodeParams> encodeParams(
|
||||
reinterpret_cast<RTCFrameEncodeParams *>(params));
|
||||
[encodeParams->encoder frameWasEncoded:status
|
||||
flags:infoFlags
|
||||
sampleBuffer:sampleBuffer
|
||||
codecSpecificInfo:encodeParams->codecSpecificInfo
|
||||
width:encodeParams->width
|
||||
height:encodeParams->height
|
||||
renderTimeMs:encodeParams->render_time_ms
|
||||
timestamp:encodeParams->timestamp
|
||||
rotation:encodeParams->rotation];
|
||||
}
|
||||
|
||||
// Extract VideoToolbox profile out of the webrtc::SdpVideoFormat. If there is
|
||||
// no specific VideoToolbox profile for the specified level, AutoLevel will be
|
||||
// returned. The user must initialize the encoder with a resolution and
|
||||
// framerate conforming to the selected H264 level regardless.
|
||||
CFStringRef ExtractProfile(const webrtc::H264ProfileLevelId &profile_level_id) {
|
||||
switch (profile_level_id.profile) {
|
||||
case webrtc::H264Profile::kProfileConstrainedBaseline:
|
||||
case webrtc::H264Profile::kProfileBaseline:
|
||||
switch (profile_level_id.level) {
|
||||
case webrtc::H264Level::kLevel3:
|
||||
return kVTProfileLevel_H264_Baseline_3_0;
|
||||
case webrtc::H264Level::kLevel3_1:
|
||||
return kVTProfileLevel_H264_Baseline_3_1;
|
||||
case webrtc::H264Level::kLevel3_2:
|
||||
return kVTProfileLevel_H264_Baseline_3_2;
|
||||
case webrtc::H264Level::kLevel4:
|
||||
return kVTProfileLevel_H264_Baseline_4_0;
|
||||
case webrtc::H264Level::kLevel4_1:
|
||||
return kVTProfileLevel_H264_Baseline_4_1;
|
||||
case webrtc::H264Level::kLevel4_2:
|
||||
return kVTProfileLevel_H264_Baseline_4_2;
|
||||
case webrtc::H264Level::kLevel5:
|
||||
return kVTProfileLevel_H264_Baseline_5_0;
|
||||
case webrtc::H264Level::kLevel5_1:
|
||||
return kVTProfileLevel_H264_Baseline_5_1;
|
||||
case webrtc::H264Level::kLevel5_2:
|
||||
return kVTProfileLevel_H264_Baseline_5_2;
|
||||
case webrtc::H264Level::kLevel1:
|
||||
case webrtc::H264Level::kLevel1_b:
|
||||
case webrtc::H264Level::kLevel1_1:
|
||||
case webrtc::H264Level::kLevel1_2:
|
||||
case webrtc::H264Level::kLevel1_3:
|
||||
case webrtc::H264Level::kLevel2:
|
||||
case webrtc::H264Level::kLevel2_1:
|
||||
case webrtc::H264Level::kLevel2_2:
|
||||
return kVTProfileLevel_H264_Baseline_AutoLevel;
|
||||
}
|
||||
|
||||
case webrtc::H264Profile::kProfileMain:
|
||||
switch (profile_level_id.level) {
|
||||
case webrtc::H264Level::kLevel3:
|
||||
return kVTProfileLevel_H264_Main_3_0;
|
||||
case webrtc::H264Level::kLevel3_1:
|
||||
return kVTProfileLevel_H264_Main_3_1;
|
||||
case webrtc::H264Level::kLevel3_2:
|
||||
return kVTProfileLevel_H264_Main_3_2;
|
||||
case webrtc::H264Level::kLevel4:
|
||||
return kVTProfileLevel_H264_Main_4_0;
|
||||
case webrtc::H264Level::kLevel4_1:
|
||||
return kVTProfileLevel_H264_Main_4_1;
|
||||
case webrtc::H264Level::kLevel4_2:
|
||||
return kVTProfileLevel_H264_Main_4_2;
|
||||
case webrtc::H264Level::kLevel5:
|
||||
return kVTProfileLevel_H264_Main_5_0;
|
||||
case webrtc::H264Level::kLevel5_1:
|
||||
return kVTProfileLevel_H264_Main_5_1;
|
||||
case webrtc::H264Level::kLevel5_2:
|
||||
return kVTProfileLevel_H264_Main_5_2;
|
||||
case webrtc::H264Level::kLevel1:
|
||||
case webrtc::H264Level::kLevel1_b:
|
||||
case webrtc::H264Level::kLevel1_1:
|
||||
case webrtc::H264Level::kLevel1_2:
|
||||
case webrtc::H264Level::kLevel1_3:
|
||||
case webrtc::H264Level::kLevel2:
|
||||
case webrtc::H264Level::kLevel2_1:
|
||||
case webrtc::H264Level::kLevel2_2:
|
||||
return kVTProfileLevel_H264_Main_AutoLevel;
|
||||
}
|
||||
|
||||
case webrtc::H264Profile::kProfileConstrainedHigh:
|
||||
case webrtc::H264Profile::kProfileHigh:
|
||||
#ifdef __aarch64__
|
||||
if (profile_level_id.level < webrtc::H264Level::kLevel4) {
|
||||
return kVTProfileLevel_H264_Main_4_0;
|
||||
}
|
||||
#endif
|
||||
|
||||
switch (profile_level_id.level) {
|
||||
case webrtc::H264Level::kLevel3:
|
||||
return kVTProfileLevel_H264_High_3_0;
|
||||
case webrtc::H264Level::kLevel3_1:
|
||||
return kVTProfileLevel_H264_High_3_1;
|
||||
case webrtc::H264Level::kLevel3_2:
|
||||
return kVTProfileLevel_H264_High_3_2;
|
||||
case webrtc::H264Level::kLevel4:
|
||||
return kVTProfileLevel_H264_High_4_0;
|
||||
case webrtc::H264Level::kLevel4_1:
|
||||
return kVTProfileLevel_H264_High_4_1;
|
||||
case webrtc::H264Level::kLevel4_2:
|
||||
return kVTProfileLevel_H264_High_4_2;
|
||||
case webrtc::H264Level::kLevel5:
|
||||
return kVTProfileLevel_H264_High_5_0;
|
||||
case webrtc::H264Level::kLevel5_1:
|
||||
return kVTProfileLevel_H264_High_5_1;
|
||||
case webrtc::H264Level::kLevel5_2:
|
||||
return kVTProfileLevel_H264_High_5_2;
|
||||
case webrtc::H264Level::kLevel1:
|
||||
case webrtc::H264Level::kLevel1_b:
|
||||
case webrtc::H264Level::kLevel1_1:
|
||||
case webrtc::H264Level::kLevel1_2:
|
||||
case webrtc::H264Level::kLevel1_3:
|
||||
case webrtc::H264Level::kLevel2:
|
||||
case webrtc::H264Level::kLevel2_1:
|
||||
case webrtc::H264Level::kLevel2_2:
|
||||
return kVTProfileLevel_H264_High_AutoLevel;
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
// The function returns the max allowed sample rate (pixels per second) that
|
||||
// can be processed by given encoder with |profile_level_id|.
|
||||
// See https://www.itu.int/rec/dologin_pub.asp?lang=e&id=T-REC-H.264-201610-S!!PDF-E&type=items
|
||||
// for details.
|
||||
NSUInteger GetMaxSampleRate(const webrtc::H264ProfileLevelId &profile_level_id) {
|
||||
switch (profile_level_id.level) {
|
||||
case webrtc::H264Level::kLevel3:
|
||||
return 10368000;
|
||||
case webrtc::H264Level::kLevel3_1:
|
||||
return 27648000;
|
||||
case webrtc::H264Level::kLevel3_2:
|
||||
return 55296000;
|
||||
case webrtc::H264Level::kLevel4:
|
||||
case webrtc::H264Level::kLevel4_1:
|
||||
return 62914560;
|
||||
case webrtc::H264Level::kLevel4_2:
|
||||
return 133693440;
|
||||
case webrtc::H264Level::kLevel5:
|
||||
return 150994944;
|
||||
case webrtc::H264Level::kLevel5_1:
|
||||
return 251658240;
|
||||
case webrtc::H264Level::kLevel5_2:
|
||||
return 530841600;
|
||||
case webrtc::H264Level::kLevel1:
|
||||
case webrtc::H264Level::kLevel1_b:
|
||||
case webrtc::H264Level::kLevel1_1:
|
||||
case webrtc::H264Level::kLevel1_2:
|
||||
case webrtc::H264Level::kLevel1_3:
|
||||
case webrtc::H264Level::kLevel2:
|
||||
case webrtc::H264Level::kLevel2_1:
|
||||
case webrtc::H264Level::kLevel2_2:
|
||||
// Zero means auto rate setting.
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
@implementation TGRTCVideoEncoderH264 {
|
||||
RTCVideoCodecInfo *_codecInfo;
|
||||
std::unique_ptr<webrtc::BitrateAdjuster> _bitrateAdjuster;
|
||||
uint32_t _targetBitrateBps;
|
||||
uint32_t _encoderBitrateBps;
|
||||
uint32_t _encoderFrameRate;
|
||||
uint32_t _maxAllowedFrameRate;
|
||||
RTCH264PacketizationMode _packetizationMode;
|
||||
absl::optional<webrtc::H264ProfileLevelId> _profile_level_id;
|
||||
RTCVideoEncoderCallback _callback;
|
||||
int32_t _width;
|
||||
int32_t _height;
|
||||
VTCompressionSessionRef _compressionSession;
|
||||
CVPixelBufferPoolRef _pixelBufferPool;
|
||||
RTCVideoCodecMode _mode;
|
||||
|
||||
webrtc::H264BitstreamParser _h264BitstreamParser;
|
||||
std::vector<uint8_t> _frameScaleBuffer;
|
||||
}
|
||||
|
||||
// .5 is set as a mininum to prevent overcompensating for large temporary
|
||||
// overshoots. We don't want to degrade video quality too badly.
|
||||
// .95 is set to prevent oscillations. When a lower bitrate is set on the
|
||||
// encoder than previously set, its output seems to have a brief period of
|
||||
// drastically reduced bitrate, so we want to avoid that. In steady state
|
||||
// conditions, 0.95 seems to give us better overall bitrate over long periods
|
||||
// of time.
|
||||
- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo *)codecInfo {
|
||||
if (self = [super init]) {
|
||||
_codecInfo = codecInfo;
|
||||
_bitrateAdjuster.reset(new webrtc::BitrateAdjuster(.5, .95));
|
||||
_packetizationMode = RTCH264PacketizationModeNonInterleaved;
|
||||
_profile_level_id =
|
||||
webrtc::ParseSdpForH264ProfileLevelId([codecInfo nativeSdpVideoFormat].parameters);
|
||||
RTC_DCHECK(_profile_level_id);
|
||||
RTC_LOG(LS_INFO) << "Using profile " << CFStringToString(ExtractProfile(*_profile_level_id));
|
||||
RTC_CHECK([codecInfo.name isEqualToString:kRTCVideoCodecH264Name]);
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
[self destroyCompressionSession];
|
||||
}
|
||||
|
||||
- (NSInteger)startEncodeWithSettings:(RTCVideoEncoderSettings *)settings
|
||||
numberOfCores:(int)numberOfCores {
|
||||
RTC_DCHECK(settings);
|
||||
RTC_DCHECK([settings.name isEqualToString:kRTCVideoCodecH264Name]);
|
||||
|
||||
_width = settings.width;
|
||||
_height = settings.height;
|
||||
_mode = settings.mode;
|
||||
|
||||
uint32_t aligned_width = (((_width + 15) >> 4) << 4);
|
||||
uint32_t aligned_height = (((_height + 15) >> 4) << 4);
|
||||
_maxAllowedFrameRate = static_cast<uint32_t>(GetMaxSampleRate(*_profile_level_id) /
|
||||
(aligned_width * aligned_height));
|
||||
|
||||
// We can only set average bitrate on the HW encoder.
|
||||
_targetBitrateBps = settings.startBitrate * 1000; // startBitrate is in kbps.
|
||||
_bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps);
|
||||
_encoderFrameRate = MIN(settings.maxFramerate, _maxAllowedFrameRate);
|
||||
if (settings.maxFramerate > _maxAllowedFrameRate && _maxAllowedFrameRate > 0) {
|
||||
RTC_LOG(LS_WARNING) << "Initial encoder frame rate setting " << settings.maxFramerate
|
||||
<< " is larger than the "
|
||||
<< "maximal allowed frame rate " << _maxAllowedFrameRate << ".";
|
||||
}
|
||||
|
||||
// TODO(tkchin): Try setting payload size via
|
||||
// kVTCompressionPropertyKey_MaxH264SliceBytes.
|
||||
|
||||
return [self resetCompressionSessionWithPixelFormat:kNV12PixelFormat];
|
||||
}
|
||||
|
||||
- (NSInteger)encode:(RTCVideoFrame *)frame
|
||||
codecSpecificInfo:(nullable id<RTCCodecSpecificInfo>)codecSpecificInfo
|
||||
frameTypes:(NSArray<NSNumber *> *)frameTypes {
|
||||
//RTC_DCHECK_EQ(frame.width, _width);
|
||||
//RTC_DCHECK_EQ(frame.height, _height);
|
||||
if (!_callback || !_compressionSession) {
|
||||
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
|
||||
}
|
||||
BOOL isKeyframeRequired = NO;
|
||||
|
||||
// Get a pixel buffer from the pool and copy frame data over.
|
||||
if ([self resetCompressionSessionIfNeededWithFrame:frame]) {
|
||||
isKeyframeRequired = YES;
|
||||
}
|
||||
|
||||
CVPixelBufferRef pixelBuffer = nullptr;
|
||||
if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
|
||||
// Native frame buffer
|
||||
RTCCVPixelBuffer *rtcPixelBuffer = (RTCCVPixelBuffer *)frame.buffer;
|
||||
if (![rtcPixelBuffer requiresCropping]) {
|
||||
// This pixel buffer might have a higher resolution than what the
|
||||
// compression session is configured to. The compression session can
|
||||
// handle that and will output encoded frames in the configured
|
||||
// resolution regardless of the input pixel buffer resolution.
|
||||
pixelBuffer = rtcPixelBuffer.pixelBuffer;
|
||||
CVBufferRetain(pixelBuffer);
|
||||
} else {
|
||||
// Cropping required, we need to crop and scale to a new pixel buffer.
|
||||
pixelBuffer = CreatePixelBuffer(_pixelBufferPool);
|
||||
if (!pixelBuffer) {
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
int dstWidth = CVPixelBufferGetWidth(pixelBuffer);
|
||||
int dstHeight = CVPixelBufferGetHeight(pixelBuffer);
|
||||
if ([rtcPixelBuffer requiresScalingToWidth:dstWidth height:dstHeight]) {
|
||||
int size =
|
||||
[rtcPixelBuffer bufferSizeForCroppingAndScalingToWidth:dstWidth height:dstHeight];
|
||||
_frameScaleBuffer.resize(size);
|
||||
} else {
|
||||
_frameScaleBuffer.clear();
|
||||
}
|
||||
_frameScaleBuffer.shrink_to_fit();
|
||||
if (![rtcPixelBuffer cropAndScaleTo:pixelBuffer withTempBuffer:_frameScaleBuffer.data()]) {
|
||||
CVBufferRelease(pixelBuffer);
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!pixelBuffer) {
|
||||
// We did not have a native frame buffer
|
||||
pixelBuffer = CreatePixelBuffer(_pixelBufferPool);
|
||||
if (!pixelBuffer) {
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
RTC_DCHECK(pixelBuffer);
|
||||
if (!CopyVideoFrameToNV12PixelBuffer([frame.buffer toI420], pixelBuffer)) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to copy frame data.";
|
||||
CVBufferRelease(pixelBuffer);
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
// Check if we need a keyframe.
|
||||
if (!isKeyframeRequired && frameTypes) {
|
||||
for (NSNumber *frameType in frameTypes) {
|
||||
if ((RTCFrameType)frameType.intValue == RTCFrameTypeVideoFrameKey) {
|
||||
isKeyframeRequired = YES;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
CMTime presentationTimeStamp = CMTimeMake(frame.timeStampNs / rtc::kNumNanosecsPerMillisec, 1000);
|
||||
CFDictionaryRef frameProperties = nullptr;
|
||||
if (isKeyframeRequired) {
|
||||
CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame};
|
||||
CFTypeRef values[] = {kCFBooleanTrue};
|
||||
frameProperties = CreateCFTypeDictionary(keys, values, 1);
|
||||
}
|
||||
|
||||
std::unique_ptr<RTCFrameEncodeParams> encodeParams;
|
||||
encodeParams.reset(new RTCFrameEncodeParams(self,
|
||||
codecSpecificInfo,
|
||||
_width,
|
||||
_height,
|
||||
frame.timeStampNs / rtc::kNumNanosecsPerMillisec,
|
||||
frame.timeStamp,
|
||||
frame.rotation));
|
||||
encodeParams->codecSpecificInfo.packetizationMode = _packetizationMode;
|
||||
|
||||
// Update the bitrate if needed.
|
||||
[self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps() frameRate:_encoderFrameRate];
|
||||
|
||||
OSStatus status = VTCompressionSessionEncodeFrame(_compressionSession,
|
||||
pixelBuffer,
|
||||
presentationTimeStamp,
|
||||
kCMTimeInvalid,
|
||||
frameProperties,
|
||||
encodeParams.release(),
|
||||
nullptr);
|
||||
if (frameProperties) {
|
||||
CFRelease(frameProperties);
|
||||
}
|
||||
if (pixelBuffer) {
|
||||
CVBufferRelease(pixelBuffer);
|
||||
}
|
||||
|
||||
if (status == kVTInvalidSessionErr) {
|
||||
// This error occurs when entering foreground after backgrounding the app.
|
||||
RTC_LOG(LS_ERROR) << "Invalid compression session, resetting.";
|
||||
[self resetCompressionSessionWithPixelFormat:[self pixelFormatOfFrame:frame]];
|
||||
|
||||
return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
|
||||
} else if (status == kVTVideoEncoderMalfunctionErr) {
|
||||
// Sometimes the encoder malfunctions and needs to be restarted.
|
||||
RTC_LOG(LS_ERROR)
|
||||
<< "Encountered video encoder malfunction error. Resetting compression session.";
|
||||
[self resetCompressionSessionWithPixelFormat:[self pixelFormatOfFrame:frame]];
|
||||
|
||||
return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
|
||||
} else if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to encode frame with code: " << status;
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
- (void)setCallback:(RTCVideoEncoderCallback)callback {
|
||||
_callback = callback;
|
||||
}
|
||||
|
||||
- (int)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate {
|
||||
_targetBitrateBps = 1000 * bitrateKbit;
|
||||
_bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps);
|
||||
if (framerate > _maxAllowedFrameRate && _maxAllowedFrameRate > 0) {
|
||||
RTC_LOG(LS_WARNING) << "Encoder frame rate setting " << framerate << " is larger than the "
|
||||
<< "maximal allowed frame rate " << _maxAllowedFrameRate << ".";
|
||||
}
|
||||
framerate = MIN(framerate, _maxAllowedFrameRate);
|
||||
[self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps() frameRate:framerate];
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
- (NSInteger)resolutionAlignment {
|
||||
return 1;
|
||||
}
|
||||
|
||||
- (BOOL)applyAlignmentToAllSimulcastLayers {
|
||||
return NO;
|
||||
}
|
||||
|
||||
- (BOOL)supportsNativeHandle {
|
||||
return YES;
|
||||
}
|
||||
|
||||
#pragma mark - Private
|
||||
|
||||
- (NSInteger)releaseEncoder {
|
||||
// Need to destroy so that the session is invalidated and won't use the
|
||||
// callback anymore. Do not remove callback until the session is invalidated
|
||||
// since async encoder callbacks can occur until invalidation.
|
||||
[self destroyCompressionSession];
|
||||
_callback = nullptr;
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
- (OSType)pixelFormatOfFrame:(RTCVideoFrame *)frame {
|
||||
// Use NV12 for non-native frames.
|
||||
if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
|
||||
RTCCVPixelBuffer *rtcPixelBuffer = (RTCCVPixelBuffer *)frame.buffer;
|
||||
return CVPixelBufferGetPixelFormatType(rtcPixelBuffer.pixelBuffer);
|
||||
}
|
||||
|
||||
return kNV12PixelFormat;
|
||||
}
|
||||
|
||||
- (BOOL)resetCompressionSessionIfNeededWithFrame:(RTCVideoFrame *)frame {
|
||||
BOOL resetCompressionSession = NO;
|
||||
|
||||
// If we're capturing native frames in another pixel format than the compression session is
|
||||
// configured with, make sure the compression session is reset using the correct pixel format.
|
||||
OSType framePixelFormat = [self pixelFormatOfFrame:frame];
|
||||
|
||||
if (_compressionSession) {
|
||||
_pixelBufferPool = VTCompressionSessionGetPixelBufferPool(_compressionSession);
|
||||
// The pool attribute `kCVPixelBufferPixelFormatTypeKey` can contain either an array of pixel
|
||||
// formats or a single pixel format.
|
||||
NSDictionary *poolAttributes =
|
||||
(__bridge NSDictionary *)CVPixelBufferPoolGetPixelBufferAttributes(_pixelBufferPool);
|
||||
id pixelFormats =
|
||||
[poolAttributes objectForKey:(__bridge NSString *)kCVPixelBufferPixelFormatTypeKey];
|
||||
NSArray<NSNumber *> *compressionSessionPixelFormats = nil;
|
||||
if ([pixelFormats isKindOfClass:[NSArray class]]) {
|
||||
compressionSessionPixelFormats = (NSArray *)pixelFormats;
|
||||
} else if ([pixelFormats isKindOfClass:[NSNumber class]]) {
|
||||
compressionSessionPixelFormats = @[ (NSNumber *)pixelFormats ];
|
||||
}
|
||||
|
||||
if (![compressionSessionPixelFormats
|
||||
containsObject:[NSNumber numberWithLong:framePixelFormat]]) {
|
||||
resetCompressionSession = YES;
|
||||
RTC_LOG(LS_INFO) << "Resetting compression session due to non-matching pixel format.";
|
||||
}
|
||||
} else {
|
||||
resetCompressionSession = YES;
|
||||
}
|
||||
|
||||
if (resetCompressionSession) {
|
||||
[self resetCompressionSessionWithPixelFormat:framePixelFormat];
|
||||
}
|
||||
return resetCompressionSession;
|
||||
}
|
||||
|
||||
- (int)resetCompressionSessionWithPixelFormat:(OSType)framePixelFormat {
|
||||
[self destroyCompressionSession];
|
||||
|
||||
// Set source image buffer attributes. These attributes will be present on
|
||||
// buffers retrieved from the encoder's pixel buffer pool.
|
||||
const size_t attributesSize = 3;
|
||||
CFTypeRef keys[attributesSize] = {
|
||||
#if defined(WEBRTC_IOS)
|
||||
kCVPixelBufferOpenGLESCompatibilityKey,
|
||||
#elif defined(WEBRTC_MAC)
|
||||
kCVPixelBufferOpenGLCompatibilityKey,
|
||||
#endif
|
||||
kCVPixelBufferIOSurfacePropertiesKey,
|
||||
kCVPixelBufferPixelFormatTypeKey
|
||||
};
|
||||
CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0);
|
||||
int64_t pixelFormatType = framePixelFormat;
|
||||
CFNumberRef pixelFormat = CFNumberCreate(nullptr, kCFNumberLongType, &pixelFormatType);
|
||||
CFTypeRef values[attributesSize] = {kCFBooleanTrue, ioSurfaceValue, pixelFormat};
|
||||
CFDictionaryRef sourceAttributes = CreateCFTypeDictionary(keys, values, attributesSize);
|
||||
if (ioSurfaceValue) {
|
||||
CFRelease(ioSurfaceValue);
|
||||
ioSurfaceValue = nullptr;
|
||||
}
|
||||
if (pixelFormat) {
|
||||
CFRelease(pixelFormat);
|
||||
pixelFormat = nullptr;
|
||||
}
|
||||
CFMutableDictionaryRef encoder_specs = nullptr;
|
||||
#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
|
||||
// Currently hw accl is supported above 360p on mac, below 360p
|
||||
// the compression session will be created with hw accl disabled.
|
||||
encoder_specs = CFDictionaryCreateMutable(
|
||||
nullptr, 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
|
||||
CFDictionarySetValue(encoder_specs,
|
||||
kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
|
||||
kCFBooleanTrue);
|
||||
#endif
|
||||
OSStatus status =
|
||||
VTCompressionSessionCreate(nullptr, // use default allocator
|
||||
_width,
|
||||
_height,
|
||||
kCMVideoCodecType_H264,
|
||||
encoder_specs, // use hardware accelerated encoder if available
|
||||
sourceAttributes,
|
||||
nullptr, // use default compressed data allocator
|
||||
compressionOutputCallback,
|
||||
nullptr,
|
||||
&_compressionSession);
|
||||
if (sourceAttributes) {
|
||||
CFRelease(sourceAttributes);
|
||||
sourceAttributes = nullptr;
|
||||
}
|
||||
if (encoder_specs) {
|
||||
CFRelease(encoder_specs);
|
||||
encoder_specs = nullptr;
|
||||
}
|
||||
if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to create compression session: " << status;
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
|
||||
CFBooleanRef hwaccl_enabled = nullptr;
|
||||
status = VTSessionCopyProperty(_compressionSession,
|
||||
kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder,
|
||||
nullptr,
|
||||
&hwaccl_enabled);
|
||||
if (status == noErr && (CFBooleanGetValue(hwaccl_enabled))) {
|
||||
RTC_LOG(LS_INFO) << "Compression session created with hw accl enabled";
|
||||
} else {
|
||||
RTC_LOG(LS_INFO) << "Compression session created with hw accl disabled";
|
||||
}
|
||||
#endif
|
||||
[self configureCompressionSession];
|
||||
|
||||
// The pixel buffer pool is dependent on the compression session so if the session is reset, the
|
||||
// pool should be reset as well.
|
||||
_pixelBufferPool = VTCompressionSessionGetPixelBufferPool(_compressionSession);
|
||||
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
- (void)configureCompressionSession {
|
||||
RTC_DCHECK(_compressionSession);
|
||||
SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_RealTime, false);
|
||||
SetVTSessionProperty(_compressionSession,
|
||||
kVTCompressionPropertyKey_ProfileLevel,
|
||||
ExtractProfile(*_profile_level_id));
|
||||
SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AllowFrameReordering, false);
|
||||
[self setEncoderBitrateBps:_targetBitrateBps frameRate:_encoderFrameRate];
|
||||
// TODO(tkchin): Look at entropy mode and colorspace matrices.
|
||||
// TODO(tkchin): Investigate to see if there's any way to make this work.
|
||||
// May need it to interop with Android. Currently this call just fails.
|
||||
// On inspecting encoder output on iOS8, this value is set to 6.
|
||||
// internal::SetVTSessionProperty(compression_session_,
|
||||
// kVTCompressionPropertyKey_MaxFrameDelayCount,
|
||||
// 1);
|
||||
|
||||
// Set a relatively large value for keyframe emission (7200 frames or 4 minutes).
|
||||
SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200);
|
||||
SetVTSessionProperty(
|
||||
_compressionSession, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240);
|
||||
}
|
||||
|
||||
- (void)destroyCompressionSession {
|
||||
if (_compressionSession) {
|
||||
VTCompressionSessionInvalidate(_compressionSession);
|
||||
CFRelease(_compressionSession);
|
||||
_compressionSession = nullptr;
|
||||
_pixelBufferPool = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
- (NSString *)implementationName {
|
||||
return @"VideoToolbox";
|
||||
}
|
||||
|
||||
- (void)setBitrateBps:(uint32_t)bitrateBps frameRate:(uint32_t)frameRate {
|
||||
if (_encoderBitrateBps != bitrateBps || _encoderFrameRate != frameRate) {
|
||||
[self setEncoderBitrateBps:bitrateBps frameRate:frameRate];
|
||||
}
|
||||
}
|
||||
|
||||
- (void)setEncoderBitrateBps:(uint32_t)bitrateBps frameRate:(uint32_t)frameRate {
|
||||
if (_compressionSession) {
|
||||
RTC_LOG(LS_ERROR) << "TGRTCVideoEncoderH264: setting bitrate to " << bitrateBps / 1024 << " kbit/s";
|
||||
|
||||
SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AverageBitRate, bitrateBps);
|
||||
|
||||
// With zero |_maxAllowedFrameRate|, we fall back to automatic frame rate detection.
|
||||
if (_maxAllowedFrameRate > 0) {
|
||||
SetVTSessionProperty(
|
||||
_compressionSession, kVTCompressionPropertyKey_ExpectedFrameRate, frameRate);
|
||||
}
|
||||
|
||||
// TODO(tkchin): Add a helper method to set array value.
|
||||
int64_t dataLimitBytesPerSecondValue =
|
||||
static_cast<int64_t>(bitrateBps * kLimitToAverageBitRateFactor / 8);
|
||||
CFNumberRef bytesPerSecond =
|
||||
CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &dataLimitBytesPerSecondValue);
|
||||
int64_t oneSecondValue = 1;
|
||||
CFNumberRef oneSecond =
|
||||
CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &oneSecondValue);
|
||||
const void *nums[2] = {bytesPerSecond, oneSecond};
|
||||
CFArrayRef dataRateLimits = CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks);
|
||||
OSStatus status = VTSessionSetProperty(
|
||||
_compressionSession, kVTCompressionPropertyKey_DataRateLimits, dataRateLimits);
|
||||
if (bytesPerSecond) {
|
||||
CFRelease(bytesPerSecond);
|
||||
}
|
||||
if (oneSecond) {
|
||||
CFRelease(oneSecond);
|
||||
}
|
||||
if (dataRateLimits) {
|
||||
CFRelease(dataRateLimits);
|
||||
}
|
||||
if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to set data rate limit with code: " << status;
|
||||
}
|
||||
|
||||
_encoderBitrateBps = bitrateBps;
|
||||
_encoderFrameRate = frameRate;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)frameWasEncoded:(OSStatus)status
|
||||
flags:(VTEncodeInfoFlags)infoFlags
|
||||
sampleBuffer:(CMSampleBufferRef)sampleBuffer
|
||||
codecSpecificInfo:(id<RTCCodecSpecificInfo>)codecSpecificInfo
|
||||
width:(int32_t)width
|
||||
height:(int32_t)height
|
||||
renderTimeMs:(int64_t)renderTimeMs
|
||||
timestamp:(uint32_t)timestamp
|
||||
rotation:(RTCVideoRotation)rotation {
|
||||
if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "H264 encode failed with code: " << status;
|
||||
return;
|
||||
}
|
||||
if (infoFlags & kVTEncodeInfo_FrameDropped) {
|
||||
RTC_LOG(LS_INFO) << "H264 encode dropped frame.";
|
||||
return;
|
||||
}
|
||||
|
||||
BOOL isKeyframe = NO;
|
||||
CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, 0);
|
||||
if (attachments != nullptr && CFArrayGetCount(attachments)) {
|
||||
CFDictionaryRef attachment =
|
||||
static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
|
||||
isKeyframe = !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync);
|
||||
}
|
||||
|
||||
if (isKeyframe) {
|
||||
RTC_LOG(LS_INFO) << "Generated keyframe";
|
||||
}
|
||||
|
||||
__block std::unique_ptr<rtc::Buffer> buffer = std::make_unique<rtc::Buffer>();
|
||||
{
|
||||
bool result = webrtc::H264CMSampleBufferToAnnexBBuffer(sampleBuffer, isKeyframe, buffer.get());
|
||||
if (!result) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
RTCEncodedImage *frame = [[RTCEncodedImage alloc] init];
|
||||
// This assumes ownership of `buffer` and is responsible for freeing it when done.
|
||||
frame.buffer = [[NSData alloc] initWithBytesNoCopy:buffer->data()
|
||||
length:buffer->size()
|
||||
deallocator:^(void *bytes, NSUInteger size) {
|
||||
buffer.reset();
|
||||
}];
|
||||
frame.encodedWidth = width;
|
||||
frame.encodedHeight = height;
|
||||
//frame.completeFrame = YES;
|
||||
frame.frameType = isKeyframe ? RTCFrameTypeVideoFrameKey : RTCFrameTypeVideoFrameDelta;
|
||||
frame.captureTimeMs = renderTimeMs;
|
||||
frame.timeStamp = timestamp;
|
||||
frame.rotation = rotation;
|
||||
frame.contentType = (_mode == RTCVideoCodecModeScreensharing) ? RTCVideoContentTypeScreenshare :
|
||||
RTCVideoContentTypeUnspecified;
|
||||
frame.flags = webrtc::VideoSendTiming::kInvalid;
|
||||
|
||||
int qp;
|
||||
_h264BitstreamParser.ParseBitstream(*buffer);
|
||||
frame.qp = @(_h264BitstreamParser.GetLastSliceQp().value_or(-1));
|
||||
|
||||
BOOL res = _callback(frame, codecSpecificInfo);
|
||||
if (!res) {
|
||||
RTC_LOG(LS_ERROR) << "Encode callback failed";
|
||||
return;
|
||||
}
|
||||
_bitrateAdjuster->Update(frame.buffer.length);
|
||||
}
|
||||
|
||||
- (nullable RTCVideoEncoderQpThresholds *)scalingSettings {
|
||||
return [[RTCVideoEncoderQpThresholds alloc] initWithThresholdsLow:kLowH264QpThreshold
|
||||
high:kHighH264QpThreshold];
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef WEBRTC_DISABLE_H265
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
#import "RTCMacros.h"
|
||||
#import "RTCVideoCodecInfo.h"
|
||||
#import "RTCVideoEncoder.h"
|
||||
|
||||
RTC_OBJC_EXPORT
|
||||
API_AVAILABLE(ios(11.0))
|
||||
@interface RTCVideoEncoderH265 : NSObject <RTCVideoEncoder>
|
||||
|
||||
- (instancetype _Nonnull)initWithCodecInfo:(RTCVideoCodecInfo * _Nonnull)codecInfo;
|
||||
|
||||
- (nullable RTC_OBJC_TYPE(RTCVideoEncoderQpThresholds) *)scalingSettings;
|
||||
|
||||
/** Resolutions should be aligned to this value. */
|
||||
@property(nonatomic, readonly) NSInteger resolutionAlignment;
|
||||
|
||||
/** If enabled, resolution alignment is applied to all simulcast layers simultaneously so that when
|
||||
scaled, all resolutions comply with 'resolutionAlignment'. */
|
||||
@property(nonatomic, readonly) BOOL applyAlignmentToAllSimulcastLayers;
|
||||
|
||||
/** If YES, the reciever is expected to resample/scale the source texture to the expected output
|
||||
size. */
|
||||
@property(nonatomic, readonly) BOOL supportsNativeHandle;
|
||||
|
||||
@end
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,624 @@
|
|||
/*
|
||||
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*
|
||||
*/
|
||||
|
||||
#import "TGRTCVideoEncoderH265.h"
|
||||
|
||||
#import <VideoToolbox/VideoToolbox.h>
|
||||
#include <vector>
|
||||
|
||||
#import "RTCCodecSpecificInfoH265.h"
|
||||
#import "api/peerconnection/RTCVideoCodecInfo+Private.h"
|
||||
#import "base/RTCI420Buffer.h"
|
||||
#import "base/RTCVideoFrame.h"
|
||||
#import "base/RTCVideoFrameBuffer.h"
|
||||
#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
|
||||
#import "helpers.h"
|
||||
#if defined(WEBRTC_IOS)
|
||||
#import "helpers/UIDevice+RTCDevice.h"
|
||||
#endif
|
||||
|
||||
#include "api/video_codecs/h264_profile_level_id.h"
|
||||
#include "common_video/h265/h265_bitstream_parser.h"
|
||||
#include "common_video/include/bitrate_adjuster.h"
|
||||
#include "libyuv/convert_from.h"
|
||||
#include "modules/include/module_common_types.h"
|
||||
#include "modules/video_coding/include/video_error_codes.h"
|
||||
#include "rtc_base/buffer.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "rtc_base/time_utils.h"
|
||||
#include "sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h"
|
||||
#include "system_wrappers/include/clock.h"
|
||||
#include "h265_nalu_rewriter.h"
|
||||
|
||||
@interface RTC_OBJC_TYPE (RTCVideoEncoderH265)
|
||||
()
|
||||
|
||||
- (void)frameWasEncoded : (OSStatus)status flags : (VTEncodeInfoFlags)infoFlags sampleBuffer
|
||||
: (CMSampleBufferRef)sampleBuffer codecSpecificInfo
|
||||
: (id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)>)codecSpecificInfo width : (int32_t)width height
|
||||
: (int32_t)height renderTimeMs : (int64_t)renderTimeMs timestamp : (uint32_t)timestamp rotation
|
||||
: (RTCVideoRotation)rotation;
|
||||
|
||||
@end
|
||||
|
||||
namespace { // anonymous namespace
|
||||
|
||||
// The ratio between kVTCompressionPropertyKey_DataRateLimits and
|
||||
// kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher
|
||||
// than the average bit rate to avoid undershooting the target.
|
||||
const float kLimitToAverageBitRateFactor = 1.5f;
|
||||
// These thresholds deviate from the default h265 QP thresholds, as they
|
||||
// have been found to work better on devices that support VideoToolbox
|
||||
const int kLowh265QpThreshold = 28;
|
||||
const int kHighh265QpThreshold = 39;
|
||||
|
||||
// Struct that we pass to the encoder per frame to encode. We receive it again
|
||||
// in the encoder callback.
|
||||
struct API_AVAILABLE(ios(11.0)) RTCFrameEncodeParams {
|
||||
RTCFrameEncodeParams(RTC_OBJC_TYPE(RTCVideoEncoderH265) * e,
|
||||
RTC_OBJC_TYPE(RTCCodecSpecificInfoH265) * csi,
|
||||
int32_t w,
|
||||
int32_t h,
|
||||
int64_t rtms,
|
||||
uint32_t ts,
|
||||
RTCVideoRotation r)
|
||||
: encoder(e),
|
||||
width(w),
|
||||
height(h),
|
||||
render_time_ms(rtms),
|
||||
timestamp(ts),
|
||||
rotation(r) {
|
||||
if (csi) {
|
||||
codecSpecificInfo = csi;
|
||||
} else {
|
||||
codecSpecificInfo = [[RTC_OBJC_TYPE(RTCCodecSpecificInfoH265) alloc] init];
|
||||
}
|
||||
}
|
||||
|
||||
RTC_OBJC_TYPE(RTCVideoEncoderH265) * encoder;
|
||||
RTC_OBJC_TYPE(RTCCodecSpecificInfoH265) * codecSpecificInfo;
|
||||
int32_t width;
|
||||
int32_t height;
|
||||
int64_t render_time_ms;
|
||||
uint32_t timestamp;
|
||||
RTCVideoRotation rotation;
|
||||
};
|
||||
|
||||
// We receive I420Frames as input, but we need to feed CVPixelBuffers into the
|
||||
// encoder. This performs the copy and format conversion.
|
||||
// TODO(tkchin): See if encoder will accept i420 frames and compare performance.
|
||||
bool CopyVideoFrameToPixelBuffer(id<RTCI420Buffer> frameBuffer,
|
||||
CVPixelBufferRef pixelBuffer) {
|
||||
RTC_DCHECK(pixelBuffer);
|
||||
RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixelBuffer),
|
||||
kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
|
||||
RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixelBuffer, 0),
|
||||
frameBuffer.height);
|
||||
RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixelBuffer, 0),
|
||||
frameBuffer.width);
|
||||
|
||||
CVReturn cvRet = CVPixelBufferLockBaseAddress(pixelBuffer, 0);
|
||||
if (cvRet != kCVReturnSuccess) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
|
||||
return false;
|
||||
}
|
||||
|
||||
uint8_t* dstY = reinterpret_cast<uint8_t*>(
|
||||
CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
|
||||
int dstStrideY = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
|
||||
uint8_t* dstUV = reinterpret_cast<uint8_t*>(
|
||||
CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
|
||||
int dstStrideUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
|
||||
// Convert I420 to NV12.
|
||||
int ret = libyuv::I420ToNV12(
|
||||
frameBuffer.dataY, frameBuffer.strideY, frameBuffer.dataU,
|
||||
frameBuffer.strideU, frameBuffer.dataV, frameBuffer.strideV, dstY,
|
||||
dstStrideY, dstUV, dstStrideUV, frameBuffer.width, frameBuffer.height);
|
||||
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
|
||||
if (ret) {
|
||||
RTC_LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) {
|
||||
if (!pixel_buffer_pool) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to get pixel buffer pool.";
|
||||
return nullptr;
|
||||
}
|
||||
CVPixelBufferRef pixel_buffer;
|
||||
CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool,
|
||||
&pixel_buffer);
|
||||
if (ret != kCVReturnSuccess) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret;
|
||||
// We probably want to drop frames here, since failure probably means
|
||||
// that the pool is empty.
|
||||
return nullptr;
|
||||
}
|
||||
return pixel_buffer;
|
||||
}
|
||||
|
||||
// This is the callback function that VideoToolbox calls when encode is
|
||||
// complete. From inspection this happens on its own queue.
|
||||
void compressionOutputCallback(void* encoder,
|
||||
void* params,
|
||||
OSStatus status,
|
||||
VTEncodeInfoFlags infoFlags,
|
||||
CMSampleBufferRef sampleBuffer)
|
||||
API_AVAILABLE(ios(11.0)) {
|
||||
if (!params) {
|
||||
// If there are pending callbacks when the encoder is destroyed, this can happen.
|
||||
return;
|
||||
}
|
||||
std::unique_ptr<RTCFrameEncodeParams> encodeParams(
|
||||
reinterpret_cast<RTCFrameEncodeParams*>(params));
|
||||
RTC_CHECK(encodeParams->encoder);
|
||||
[encodeParams->encoder frameWasEncoded:status
|
||||
flags:infoFlags
|
||||
sampleBuffer:sampleBuffer
|
||||
codecSpecificInfo:encodeParams->codecSpecificInfo
|
||||
width:encodeParams->width
|
||||
height:encodeParams->height
|
||||
renderTimeMs:encodeParams->render_time_ms
|
||||
timestamp:encodeParams->timestamp
|
||||
rotation:encodeParams->rotation];
|
||||
}
|
||||
} // namespace
|
||||
|
||||
@implementation RTCVideoEncoderH265 {
|
||||
RTCVideoCodecInfo* _codecInfo;
|
||||
std::unique_ptr<webrtc::BitrateAdjuster> _bitrateAdjuster;
|
||||
uint32_t _targetBitrateBps;
|
||||
uint32_t _encoderBitrateBps;
|
||||
CFStringRef _profile;
|
||||
RTCVideoEncoderCallback _callback;
|
||||
int32_t _width;
|
||||
int32_t _height;
|
||||
VTCompressionSessionRef _compressionSession;
|
||||
RTCVideoCodecMode _mode;
|
||||
int framesLeft;
|
||||
|
||||
webrtc::H265BitstreamParser _h265BitstreamParser;
|
||||
std::vector<uint8_t> _nv12ScaleBuffer;
|
||||
}
|
||||
|
||||
// .5 is set as a mininum to prevent overcompensating for large temporary
|
||||
// overshoots. We don't want to degrade video quality too badly.
|
||||
// .95 is set to prevent oscillations. When a lower bitrate is set on the
|
||||
// encoder than previously set, its output seems to have a brief period of
|
||||
// drastically reduced bitrate, so we want to avoid that. In steady state
|
||||
// conditions, 0.95 seems to give us better overall bitrate over long periods
|
||||
// of time.
|
||||
- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo*)codecInfo {
|
||||
if (self = [super init]) {
|
||||
_codecInfo = codecInfo;
|
||||
_bitrateAdjuster.reset(new webrtc::BitrateAdjuster(.5, .95));
|
||||
RTC_CHECK([codecInfo.name isEqualToString:@"H265"]);
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
[self destroyCompressionSession];
|
||||
}
|
||||
|
||||
- (NSInteger)startEncodeWithSettings:(RTCVideoEncoderSettings*)settings
|
||||
numberOfCores:(int)numberOfCores {
|
||||
RTC_DCHECK(settings);
|
||||
RTC_DCHECK([settings.name isEqualToString:@"H265"]);
|
||||
|
||||
_width = settings.width;
|
||||
_height = settings.height;
|
||||
_mode = settings.mode;
|
||||
|
||||
// We can only set average bitrate on the HW encoder.
|
||||
_targetBitrateBps = settings.startBitrate;
|
||||
_bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps);
|
||||
|
||||
// TODO(tkchin): Try setting payload size via
|
||||
// kVTCompressionPropertyKey_Maxh265SliceBytes.
|
||||
|
||||
return [self resetCompressionSession];
|
||||
}
|
||||
|
||||
- (NSInteger)encode:(RTCVideoFrame*)frame
|
||||
codecSpecificInfo:(id<RTCCodecSpecificInfo>)codecSpecificInfo
|
||||
frameTypes:(NSArray<NSNumber*>*)frameTypes {
|
||||
RTC_DCHECK_EQ(frame.width, _width);
|
||||
RTC_DCHECK_EQ(frame.height, _height);
|
||||
if (!_callback || !_compressionSession) {
|
||||
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
|
||||
}
|
||||
BOOL isKeyframeRequired = NO;
|
||||
|
||||
// Get a pixel buffer from the pool and copy frame data over.
|
||||
CVPixelBufferPoolRef pixelBufferPool =
|
||||
VTCompressionSessionGetPixelBufferPool(_compressionSession);
|
||||
|
||||
#if defined(WEBRTC_IOS)
|
||||
if (!pixelBufferPool) {
|
||||
// Kind of a hack. On backgrounding, the compression session seems to get
|
||||
// invalidated, which causes this pool call to fail when the application
|
||||
// is foregrounded and frames are being sent for encoding again.
|
||||
// Resetting the session when this happens fixes the issue.
|
||||
// In addition we request a keyframe so video can recover quickly.
|
||||
[self resetCompressionSession];
|
||||
pixelBufferPool =
|
||||
VTCompressionSessionGetPixelBufferPool(_compressionSession);
|
||||
isKeyframeRequired = YES;
|
||||
RTC_LOG(LS_INFO) << "Resetting compression session due to invalid pool.";
|
||||
}
|
||||
#endif
|
||||
|
||||
CVPixelBufferRef pixelBuffer = nullptr;
|
||||
if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
|
||||
// Native frame buffer
|
||||
RTCCVPixelBuffer* rtcPixelBuffer = (RTCCVPixelBuffer*)frame.buffer;
|
||||
if (![rtcPixelBuffer requiresCropping]) {
|
||||
// This pixel buffer might have a higher resolution than what the
|
||||
// compression session is configured to. The compression session can
|
||||
// handle that and will output encoded frames in the configured
|
||||
// resolution regardless of the input pixel buffer resolution.
|
||||
pixelBuffer = rtcPixelBuffer.pixelBuffer;
|
||||
CVBufferRetain(pixelBuffer);
|
||||
} else {
|
||||
// Cropping required, we need to crop and scale to a new pixel buffer.
|
||||
pixelBuffer = CreatePixelBuffer(pixelBufferPool);
|
||||
if (!pixelBuffer) {
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
int dstWidth = CVPixelBufferGetWidth(pixelBuffer);
|
||||
int dstHeight = CVPixelBufferGetHeight(pixelBuffer);
|
||||
if ([rtcPixelBuffer requiresScalingToWidth:dstWidth height:dstHeight]) {
|
||||
int size =
|
||||
[rtcPixelBuffer bufferSizeForCroppingAndScalingToWidth:dstWidth
|
||||
height:dstHeight];
|
||||
_nv12ScaleBuffer.resize(size);
|
||||
} else {
|
||||
_nv12ScaleBuffer.clear();
|
||||
}
|
||||
_nv12ScaleBuffer.shrink_to_fit();
|
||||
if (![rtcPixelBuffer cropAndScaleTo:pixelBuffer
|
||||
withTempBuffer:_nv12ScaleBuffer.data()]) {
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!pixelBuffer) {
|
||||
// We did not have a native frame buffer
|
||||
pixelBuffer = CreatePixelBuffer(pixelBufferPool);
|
||||
if (!pixelBuffer) {
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
RTC_DCHECK(pixelBuffer);
|
||||
if (!CopyVideoFrameToPixelBuffer([frame.buffer toI420], pixelBuffer)) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to copy frame data.";
|
||||
CVBufferRelease(pixelBuffer);
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
// Check if we need a keyframe.
|
||||
if (!isKeyframeRequired && frameTypes) {
|
||||
for (NSNumber* frameType in frameTypes) {
|
||||
if ((RTCFrameType)frameType.intValue == RTCFrameTypeVideoFrameKey) {
|
||||
isKeyframeRequired = YES;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
CMTime presentationTimeStamp =
|
||||
CMTimeMake(frame.timeStampNs / rtc::kNumNanosecsPerMillisec, 1000);
|
||||
CFDictionaryRef frameProperties = nullptr;
|
||||
if (isKeyframeRequired) {
|
||||
CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame};
|
||||
CFTypeRef values[] = {kCFBooleanTrue};
|
||||
frameProperties = CreateCFTypeDictionary(keys, values, 1);
|
||||
}
|
||||
|
||||
std::unique_ptr<RTCFrameEncodeParams> encodeParams;
|
||||
encodeParams.reset(new RTCFrameEncodeParams(
|
||||
self, codecSpecificInfo,_width, _height,
|
||||
frame.timeStampNs / rtc::kNumNanosecsPerMillisec, frame.timeStamp,
|
||||
frame.rotation));
|
||||
|
||||
// Update the bitrate if needed.
|
||||
[self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps()];
|
||||
|
||||
OSStatus status = VTCompressionSessionEncodeFrame(
|
||||
_compressionSession, pixelBuffer, presentationTimeStamp, kCMTimeInvalid,
|
||||
frameProperties, encodeParams.release(), nullptr);
|
||||
if (frameProperties) {
|
||||
CFRelease(frameProperties);
|
||||
}
|
||||
if (pixelBuffer) {
|
||||
CVBufferRelease(pixelBuffer);
|
||||
}
|
||||
if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to encode frame with code: " << status;
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
- (void)setCallback:(RTCVideoEncoderCallback)callback {
|
||||
_callback = callback;
|
||||
}
|
||||
|
||||
- (int)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate {
|
||||
_targetBitrateBps = 1000 * bitrateKbit;
|
||||
_bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps);
|
||||
[self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps()];
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
#pragma mark - Private
|
||||
|
||||
- (NSInteger)releaseEncoder {
|
||||
// Need to destroy so that the session is invalidated and won't use the
|
||||
// callback anymore. Do not remove callback until the session is invalidated
|
||||
// since async encoder callbacks can occur until invalidation.
|
||||
[self destroyCompressionSession];
|
||||
_callback = nullptr;
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
- (int)resetCompressionSession {
|
||||
[self destroyCompressionSession];
|
||||
|
||||
// Set source image buffer attributes. These attributes will be present on
|
||||
// buffers retrieved from the encoder's pixel buffer pool.
|
||||
const size_t attributesSize = 3;
|
||||
CFTypeRef keys[attributesSize] = {
|
||||
#if defined(WEBRTC_IOS)
|
||||
kCVPixelBufferOpenGLESCompatibilityKey,
|
||||
#elif defined(WEBRTC_MAC)
|
||||
kCVPixelBufferOpenGLCompatibilityKey,
|
||||
#endif
|
||||
kCVPixelBufferIOSurfacePropertiesKey,
|
||||
kCVPixelBufferPixelFormatTypeKey
|
||||
};
|
||||
CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0);
|
||||
int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
|
||||
CFNumberRef pixelFormat =
|
||||
CFNumberCreate(nullptr, kCFNumberLongType, &nv12type);
|
||||
CFTypeRef values[attributesSize] = {kCFBooleanTrue, ioSurfaceValue,
|
||||
pixelFormat};
|
||||
CFDictionaryRef sourceAttributes =
|
||||
CreateCFTypeDictionary(keys, values, attributesSize);
|
||||
if (ioSurfaceValue) {
|
||||
CFRelease(ioSurfaceValue);
|
||||
ioSurfaceValue = nullptr;
|
||||
}
|
||||
if (pixelFormat) {
|
||||
CFRelease(pixelFormat);
|
||||
pixelFormat = nullptr;
|
||||
}
|
||||
CFMutableDictionaryRef encoder_specs = nullptr;
|
||||
#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
|
||||
// Currently hw accl is supported above 360p on mac, below 360p
|
||||
// the compression session will be created with hw accl disabled.
|
||||
encoder_specs =
|
||||
CFDictionaryCreateMutable(nullptr, 1, &kCFTypeDictionaryKeyCallBacks,
|
||||
&kCFTypeDictionaryValueCallBacks);
|
||||
CFDictionarySetValue(
|
||||
encoder_specs,
|
||||
kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
|
||||
kCFBooleanTrue);
|
||||
#endif
|
||||
OSStatus status = VTCompressionSessionCreate(
|
||||
nullptr, // use default allocator
|
||||
_width, _height, kCMVideoCodecType_HEVC,
|
||||
encoder_specs, // use hardware accelerated encoder if available
|
||||
sourceAttributes,
|
||||
nullptr, // use default compressed data allocator
|
||||
compressionOutputCallback, nullptr, &_compressionSession);
|
||||
if (sourceAttributes) {
|
||||
CFRelease(sourceAttributes);
|
||||
sourceAttributes = nullptr;
|
||||
}
|
||||
if (encoder_specs) {
|
||||
CFRelease(encoder_specs);
|
||||
encoder_specs = nullptr;
|
||||
}
|
||||
if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to create compression session: " << status;
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
|
||||
CFBooleanRef hwaccl_enabled = nullptr;
|
||||
status = VTSessionCopyProperty(
|
||||
_compressionSession,
|
||||
kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder, nullptr,
|
||||
&hwaccl_enabled);
|
||||
if (status == noErr && (CFBooleanGetValue(hwaccl_enabled))) {
|
||||
RTC_LOG(LS_INFO) << "Compression session created with hw accl enabled";
|
||||
} else {
|
||||
RTC_LOG(LS_INFO) << "Compression session created with hw accl disabled";
|
||||
}
|
||||
#endif
|
||||
[self configureCompressionSession];
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
- (void)configureCompressionSession {
|
||||
RTC_DCHECK(_compressionSession);
|
||||
SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_RealTime,
|
||||
false);
|
||||
// SetVTSessionProperty(_compressionSession,
|
||||
// kVTCompressionPropertyKey_ProfileLevel, _profile);
|
||||
SetVTSessionProperty(_compressionSession,
|
||||
kVTCompressionPropertyKey_AllowFrameReordering, false);
|
||||
[self setEncoderBitrateBps:_targetBitrateBps];
|
||||
// TODO(tkchin): Look at entropy mode and colorspace matrices.
|
||||
// TODO(tkchin): Investigate to see if there's any way to make this work.
|
||||
// May need it to interop with Android. Currently this call just fails.
|
||||
// On inspecting encoder output on iOS8, this value is set to 6.
|
||||
// internal::SetVTSessionProperty(compression_session_,
|
||||
// kVTCompressionPropertyKey_MaxFrameDelayCount,
|
||||
// 1);
|
||||
|
||||
// Set a relatively large value for keyframe emission (7200 frames or 4
|
||||
// minutes).
|
||||
SetVTSessionProperty(_compressionSession,
|
||||
kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200);
|
||||
SetVTSessionProperty(_compressionSession,
|
||||
kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration,
|
||||
240);
|
||||
OSStatus status =
|
||||
VTCompressionSessionPrepareToEncodeFrames(_compressionSession);
|
||||
if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "Compression session failed to prepare encode frames.";
|
||||
}
|
||||
}
|
||||
|
||||
- (void)destroyCompressionSession {
|
||||
if (_compressionSession) {
|
||||
VTCompressionSessionInvalidate(_compressionSession);
|
||||
CFRelease(_compressionSession);
|
||||
_compressionSession = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
- (NSString*)implementationName {
|
||||
return @"VideoToolbox";
|
||||
}
|
||||
|
||||
- (void)setBitrateBps:(uint32_t)bitrateBps {
|
||||
if (_encoderBitrateBps != bitrateBps) {
|
||||
[self setEncoderBitrateBps:bitrateBps];
|
||||
}
|
||||
}
|
||||
|
||||
- (void)setEncoderBitrateBps:(uint32_t)bitrateBps {
|
||||
if (_compressionSession) {
|
||||
SetVTSessionProperty(_compressionSession,
|
||||
kVTCompressionPropertyKey_AverageBitRate, bitrateBps);
|
||||
|
||||
// TODO(tkchin): Add a helper method to set array value.
|
||||
int64_t dataLimitBytesPerSecondValue =
|
||||
static_cast<int64_t>(bitrateBps * kLimitToAverageBitRateFactor / 8);
|
||||
CFNumberRef bytesPerSecond =
|
||||
CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type,
|
||||
&dataLimitBytesPerSecondValue);
|
||||
int64_t oneSecondValue = 1;
|
||||
CFNumberRef oneSecond = CFNumberCreate(
|
||||
kCFAllocatorDefault, kCFNumberSInt64Type, &oneSecondValue);
|
||||
const void* nums[2] = {bytesPerSecond, oneSecond};
|
||||
CFArrayRef dataRateLimits =
|
||||
CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks);
|
||||
OSStatus status = VTSessionSetProperty(
|
||||
_compressionSession, kVTCompressionPropertyKey_DataRateLimits,
|
||||
dataRateLimits);
|
||||
if (bytesPerSecond) {
|
||||
CFRelease(bytesPerSecond);
|
||||
}
|
||||
if (oneSecond) {
|
||||
CFRelease(oneSecond);
|
||||
}
|
||||
if (dataRateLimits) {
|
||||
CFRelease(dataRateLimits);
|
||||
}
|
||||
if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to set data rate limit";
|
||||
}
|
||||
|
||||
_encoderBitrateBps = bitrateBps;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)frameWasEncoded:(OSStatus)status
|
||||
flags:(VTEncodeInfoFlags)infoFlags
|
||||
sampleBuffer:(CMSampleBufferRef)sampleBuffer
|
||||
codecSpecificInfo:(id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)>)codecSpecificInfo
|
||||
width:(int32_t)width
|
||||
height:(int32_t)height
|
||||
renderTimeMs:(int64_t)renderTimeMs
|
||||
timestamp:(uint32_t)timestamp
|
||||
rotation:(RTCVideoRotation)rotation {
|
||||
if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "h265 encode failed.";
|
||||
return;
|
||||
}
|
||||
if (infoFlags & kVTEncodeInfo_FrameDropped) {
|
||||
RTC_LOG(LS_INFO) << "h265 encoder dropped a frame.";
|
||||
return;
|
||||
}
|
||||
|
||||
BOOL isKeyframe = NO;
|
||||
CFArrayRef attachments =
|
||||
CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, 0);
|
||||
if (attachments != nullptr && CFArrayGetCount(attachments)) {
|
||||
CFDictionaryRef attachment =
|
||||
static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
|
||||
isKeyframe =
|
||||
!CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync);
|
||||
}
|
||||
|
||||
if (isKeyframe) {
|
||||
RTC_LOG(LS_INFO) << "Generated keyframe";
|
||||
}
|
||||
|
||||
__block std::unique_ptr<rtc::Buffer> buffer = std::make_unique<rtc::Buffer>();
|
||||
if (!webrtc::H265CMSampleBufferToAnnexBBuffer(sampleBuffer, isKeyframe, buffer.get())) {
|
||||
return;
|
||||
}
|
||||
|
||||
RTCEncodedImage* frame = [[RTCEncodedImage alloc] init];
|
||||
frame.buffer = [NSData dataWithBytesNoCopy:buffer->data()
|
||||
length:buffer->size()
|
||||
freeWhenDone:NO];
|
||||
frame.encodedWidth = width;
|
||||
frame.encodedHeight = height;
|
||||
frame.frameType =
|
||||
isKeyframe ? RTCFrameTypeVideoFrameKey : RTCFrameTypeVideoFrameDelta;
|
||||
frame.captureTimeMs = renderTimeMs;
|
||||
frame.timeStamp = timestamp;
|
||||
frame.rotation = rotation;
|
||||
frame.contentType = (_mode == RTCVideoCodecModeScreensharing)
|
||||
? RTCVideoContentTypeScreenshare
|
||||
: RTCVideoContentTypeUnspecified;
|
||||
frame.flags = webrtc::VideoSendTiming::kInvalid;
|
||||
|
||||
_h265BitstreamParser.ParseBitstream(rtc::ArrayView<const uint8_t>(buffer->data(), buffer->size()));
|
||||
absl::optional<int> qp = _h265BitstreamParser.GetLastSliceQp();
|
||||
frame.qp = @(qp.value_or(1));
|
||||
|
||||
BOOL res = _callback(frame, codecSpecificInfo);
|
||||
if (!res) {
|
||||
RTC_LOG(LS_ERROR) << "Encode callback failed.";
|
||||
return;
|
||||
}
|
||||
_bitrateAdjuster->Update(frame.buffer.length);
|
||||
}
|
||||
|
||||
- (RTCVideoEncoderQpThresholds*)scalingSettings {
|
||||
return [[RTCVideoEncoderQpThresholds alloc]
|
||||
initWithThresholdsLow:kLowh265QpThreshold
|
||||
high:kHighh265QpThreshold];
|
||||
}
|
||||
|
||||
- (NSInteger)resolutionAlignment {
|
||||
return 1;
|
||||
}
|
||||
|
||||
- (BOOL)applyAlignmentToAllSimulcastLayers {
|
||||
return NO;
|
||||
}
|
||||
|
||||
- (BOOL)supportsNativeHandle {
|
||||
return YES;
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
//
|
||||
// VideoCMIOCapture.h
|
||||
// TgVoipWebrtc
|
||||
//
|
||||
// Created by Mikhail Filimonov on 21.06.2021.
|
||||
// Copyright © 2021 Mikhail Filimonov. All rights reserved.
|
||||
//
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
#include "VideoCameraCapturerMac.h"
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
@interface VideoCMIOCapture : NSObject<CapturerInterface>
|
||||
- (instancetype)initWithSource:(rtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source;
|
||||
- (void)setupCaptureWithDevice:(AVCaptureDevice *)device;
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
|
|
@ -0,0 +1,237 @@
|
|||
//
|
||||
// VideoCMIOCapture.m
|
||||
// TgVoipWebrtc
|
||||
//
|
||||
// Created by Mikhail Filimonov on 21.06.2021.
|
||||
// Copyright © 2021 Mikhail Filimonov. All rights reserved.
|
||||
//
|
||||
|
||||
#import "VideoCMIOCapture.h"
|
||||
#import "TGCMIODevice.h"
|
||||
#import "TGCMIOCapturer.h"
|
||||
#import <VideoToolbox/VideoToolbox.h>
|
||||
#import "TGRTCCVPixelBuffer.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#import "base/RTCLogging.h"
|
||||
#import "base/RTCVideoFrameBuffer.h"
|
||||
#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
|
||||
#import "sdk/objc/native/src/objc_video_track_source.h"
|
||||
#import "sdk/objc/native/src/objc_frame_buffer.h"
|
||||
|
||||
#import <CoreMediaIO/CMIOHardware.h>
|
||||
|
||||
#import "helpers/AVCaptureSession+DevicePosition.h"
|
||||
#import "helpers/RTCDispatcher+Private.h"
|
||||
#import "base/RTCVideoFrame.h"
|
||||
|
||||
#include "common_video/libyuv/include/webrtc_libyuv.h"
|
||||
#include "pc/video_track_source_proxy.h"
|
||||
#include "rtc_base/checks.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "third_party/libyuv/include/libyuv.h"
|
||||
#include "DarwinVideoSource.h"
|
||||
|
||||
struct MTLFrameSize {
|
||||
int width = 0;
|
||||
int height = 0;
|
||||
};
|
||||
|
||||
MTLFrameSize AspectFitted(MTLFrameSize from, MTLFrameSize to) {
|
||||
double scale = std::min(
|
||||
from.width / std::max(1., double(to.width)),
|
||||
from.height / std::max(1., double(to.height)));
|
||||
return {
|
||||
int(std::ceil(to.width * scale)),
|
||||
int(std::ceil(to.height * scale))
|
||||
};
|
||||
}
|
||||
|
||||
static const int64_t kNanosecondsPerSecond = 1000000000;
|
||||
|
||||
@interface VideoCMIOCapture ()
|
||||
-(void)applyPixelBuffer:(CVPixelBufferRef)pixelBuffer timeStampNs:(int64_t)timeStampNs;
|
||||
@end
|
||||
|
||||
|
||||
void decompressionSessionDecodeFrameCallback(void *decompressionOutputRefCon,
|
||||
void *sourceFrameRefCon,
|
||||
OSStatus status,
|
||||
VTDecodeInfoFlags infoFlags,
|
||||
CVImageBufferRef imageBuffer,
|
||||
CMTime presentationTimeStamp,
|
||||
CMTime presentationDuration)
|
||||
{
|
||||
VideoCMIOCapture *manager = (__bridge VideoCMIOCapture *)decompressionOutputRefCon;
|
||||
|
||||
if (status == noErr)
|
||||
{
|
||||
[manager applyPixelBuffer:imageBuffer timeStampNs: CMTimeGetSeconds(presentationTimeStamp) * kNanosecondsPerSecond];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static tgcalls::DarwinVideoTrackSource *getObjCVideoSource(const rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> nativeSource) {
|
||||
webrtc::VideoTrackSourceProxy *proxy_source =
|
||||
static_cast<webrtc::VideoTrackSourceProxy *>(nativeSource.get());
|
||||
return static_cast<tgcalls::DarwinVideoTrackSource *>(proxy_source->internal());
|
||||
}
|
||||
|
||||
@implementation VideoCMIOCapture
|
||||
{
|
||||
TGCMIOCapturer *_capturer;
|
||||
rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> _source;
|
||||
std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> _uncroppedSink;
|
||||
std::function<void ()> _onFatalError;
|
||||
|
||||
|
||||
BOOL _hadFatalError;
|
||||
BOOL _isRunning;
|
||||
BOOL _shouldBeMirrored;
|
||||
VTDecompressionSessionRef _decompressionSession;
|
||||
|
||||
}
|
||||
|
||||
|
||||
- (void)start {
|
||||
|
||||
__weak VideoCMIOCapture *weakSelf = self;
|
||||
|
||||
[_capturer start:^(CMSampleBufferRef sampleBuffer) {
|
||||
[weakSelf apply:sampleBuffer];
|
||||
}];
|
||||
}
|
||||
|
||||
|
||||
-(void)apply:(CMSampleBufferRef)sampleBuffer {
|
||||
if (CMSampleBufferGetNumSamples(sampleBuffer) != 1 || !CMSampleBufferIsValid(sampleBuffer) ||
|
||||
!CMSampleBufferDataIsReady(sampleBuffer)) {
|
||||
return;
|
||||
}
|
||||
|
||||
CMVideoFormatDescriptionRef formatDesc = CMSampleBufferGetFormatDescription(sampleBuffer);
|
||||
|
||||
CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
|
||||
if (pixelBuffer == nil) {
|
||||
if (_decompressionSession == nil) {
|
||||
[self createDecompSession:formatDesc];
|
||||
}
|
||||
[self render:sampleBuffer];
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
int64_t timeStampNs = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) *
|
||||
kNanosecondsPerSecond;
|
||||
|
||||
|
||||
[self applyPixelBuffer:pixelBuffer timeStampNs: timeStampNs];
|
||||
|
||||
}
|
||||
|
||||
-(void)applyPixelBuffer:(CVPixelBufferRef)pixelBuffer timeStampNs:(int64_t)timeStampNs {
|
||||
|
||||
int width = (int)CVPixelBufferGetWidth(pixelBuffer);
|
||||
int height = (int)CVPixelBufferGetHeight(pixelBuffer);
|
||||
|
||||
MTLFrameSize fittedSize = AspectFitted({ 1280, 720 }, { width, height });
|
||||
|
||||
fittedSize.width -= (fittedSize.width % 4);
|
||||
fittedSize.height -= (fittedSize.height % 4);
|
||||
|
||||
TGRTCCVPixelBuffer *rtcPixelBuffer = [[TGRTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer adaptedWidth:fittedSize.width adaptedHeight:fittedSize.height cropWidth:width cropHeight:height cropX:0 cropY:0];
|
||||
|
||||
rtcPixelBuffer.shouldBeMirrored = _shouldBeMirrored;
|
||||
|
||||
RTCVideoFrame *videoFrame = [[RTCVideoFrame alloc] initWithBuffer:rtcPixelBuffer
|
||||
rotation:RTCVideoRotation_0
|
||||
timeStampNs:timeStampNs];
|
||||
|
||||
if (_uncroppedSink) {
|
||||
const int64_t timestamp_us = timeStampNs / rtc::kNumNanosecsPerMicrosec;
|
||||
|
||||
rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer;
|
||||
buffer = new rtc::RefCountedObject<webrtc::ObjCFrameBuffer>(videoFrame.buffer);
|
||||
|
||||
webrtc::VideoRotation rotation = static_cast<webrtc::VideoRotation>(videoFrame.rotation);
|
||||
|
||||
_uncroppedSink->OnFrame(webrtc::VideoFrame::Builder()
|
||||
.set_video_frame_buffer(buffer)
|
||||
.set_rotation(rotation)
|
||||
.set_timestamp_us(timestamp_us)
|
||||
.build());
|
||||
}
|
||||
|
||||
getObjCVideoSource(_source)->OnCapturedFrame(videoFrame);
|
||||
}
|
||||
|
||||
- (void)stop {
|
||||
[_capturer stop];
|
||||
}
|
||||
- (void)setIsEnabled:(bool)isEnabled {
|
||||
|
||||
}
|
||||
- (void)setUncroppedSink:(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)sink {
|
||||
self->_uncroppedSink = sink;
|
||||
}
|
||||
- (void)setPreferredCaptureAspectRatio:(float)aspectRatio {
|
||||
|
||||
}
|
||||
- (void)setOnFatalError:(std::function<void()>)error {
|
||||
if (!self->_hadFatalError) {
|
||||
_onFatalError = std::move(error);
|
||||
} else if (error) {
|
||||
error();
|
||||
}
|
||||
}
|
||||
- (void)setOnPause:(std::function<void(bool)>)pause {
|
||||
|
||||
}
|
||||
|
||||
- (instancetype)initWithSource:(rtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source {
|
||||
self = [super init];
|
||||
if (self != nil) {
|
||||
_source = source;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
- (void)setupCaptureWithDevice:(AVCaptureDevice *)device {
|
||||
|
||||
_shouldBeMirrored = NO;
|
||||
_capturer = [[TGCMIOCapturer alloc] initWithDeviceId:device];
|
||||
}
|
||||
|
||||
- (void) render:(CMSampleBufferRef)sampleBuffer
|
||||
{
|
||||
VTDecodeFrameFlags flags = kVTDecodeFrame_EnableAsynchronousDecompression | kVTDecodeFrame_1xRealTimePlayback;
|
||||
VTDecodeInfoFlags flagOut;
|
||||
NSDate* currentTime = [NSDate date];
|
||||
VTDecompressionSessionDecodeFrame(_decompressionSession, sampleBuffer, flags,
|
||||
(void*)CFBridgingRetain(currentTime), &flagOut);
|
||||
}
|
||||
|
||||
-(void) createDecompSession:(CMVideoFormatDescriptionRef)formatDesc
|
||||
{
|
||||
if (_decompressionSession) {
|
||||
CFRelease(_decompressionSession);
|
||||
}
|
||||
_decompressionSession = NULL;
|
||||
VTDecompressionOutputCallbackRecord callBackRecord;
|
||||
callBackRecord.decompressionOutputCallback = decompressionSessionDecodeFrameCallback;
|
||||
|
||||
callBackRecord.decompressionOutputRefCon = (__bridge void *)self;
|
||||
|
||||
|
||||
|
||||
VTDecompressionSessionCreate(NULL, formatDesc, NULL,
|
||||
NULL,
|
||||
&callBackRecord, &_decompressionSession);
|
||||
}
|
||||
|
||||
-(void)dealloc {
|
||||
if (_decompressionSession) {
|
||||
CFRelease(_decompressionSession);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
#ifndef TGCALLS_VIDEO_CAMERA_CAPTURER_H
|
||||
#define TGCALLS_VIDEO_CAMERA_CAPTURER_H
|
||||
#ifdef WEBRTC_IOS
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
|
||||
#include <memory>
|
||||
#include "api/scoped_refptr.h"
|
||||
#include "api/media_stream_interface.h"
|
||||
#include "Instance.h"
|
||||
|
||||
@class VideoCaptureView;
|
||||
|
||||
@interface VideoCameraCapturer : NSObject
|
||||
|
||||
+ (NSArray<AVCaptureDevice *> *)captureDevices;
|
||||
+ (NSArray<AVCaptureDeviceFormat *> *)supportedFormatsForDevice:(AVCaptureDevice *)device;
|
||||
|
||||
- (instancetype)initWithSource:(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source useFrontCamera:(bool)useFrontCamera keepLandscape:(bool)keepLandscape isActiveUpdated:(void (^)(bool))isActiveUpdated rotationUpdated:(void (^)(int))rotationUpdated;
|
||||
|
||||
- (void)startCaptureWithDevice:(AVCaptureDevice *)device format:(AVCaptureDeviceFormat *)format fps:(NSInteger)fps;
|
||||
- (void)stopCapture;
|
||||
- (void)setIsEnabled:(bool)isEnabled;
|
||||
- (void)setPreferredCaptureAspectRatio:(float)aspectRatio;
|
||||
- (void)setUncroppedSink:(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)sink;
|
||||
- (int)getRotation;
|
||||
|
||||
- (void)addPreviewView:(VideoCaptureView *)previewView;
|
||||
- (void)addDirectSink:(std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)directSink;
|
||||
|
||||
@end
|
||||
#endif // WEBRTC_IOS
|
||||
#endif
|
||||
|
|
@ -0,0 +1,935 @@
|
|||
#include "VideoCameraCapturer.h"
|
||||
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
|
||||
#include "rtc_base/logging.h"
|
||||
#import "base/RTCLogging.h"
|
||||
#import "base/RTCVideoFrameBuffer.h"
|
||||
#import "TGRTCCVPixelBuffer.h"
|
||||
#import "sdk/objc/native/src/objc_video_track_source.h"
|
||||
#import "sdk/objc/native/src/objc_frame_buffer.h"
|
||||
#import "pc/video_track_source_proxy.h"
|
||||
|
||||
#import "helpers/UIDevice+RTCDevice.h"
|
||||
|
||||
#import "helpers/AVCaptureSession+DevicePosition.h"
|
||||
#import "helpers/RTCDispatcher+Private.h"
|
||||
#import "base/RTCVideoFrame.h"
|
||||
#include "DarwinVideoSource.h"
|
||||
|
||||
#include "common_video/libyuv/include/webrtc_libyuv.h"
|
||||
#include "rtc_base/checks.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "third_party/libyuv/include/libyuv.h"
|
||||
#include "api/video/i420_buffer.h"
|
||||
#include "api/video/nv12_buffer.h"
|
||||
|
||||
#include "VideoCaptureView.h"
|
||||
|
||||
namespace {
|
||||
|
||||
static const int64_t kNanosecondsPerSecond = 1000000000;
|
||||
|
||||
static tgcalls::DarwinVideoTrackSource *getObjCVideoSource(const webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> nativeSource) {
|
||||
webrtc::VideoTrackSourceProxy *proxy_source =
|
||||
static_cast<webrtc::VideoTrackSourceProxy *>(nativeSource.get());
|
||||
return static_cast<tgcalls::DarwinVideoTrackSource *>(proxy_source->internal());
|
||||
}
|
||||
|
||||
static UIDeviceOrientation deviceOrientation(UIInterfaceOrientation orientation) {
|
||||
switch (orientation) {
|
||||
case UIInterfaceOrientationPortrait:
|
||||
return UIDeviceOrientationPortrait;
|
||||
case UIInterfaceOrientationPortraitUpsideDown:
|
||||
return UIDeviceOrientationPortraitUpsideDown;
|
||||
case UIInterfaceOrientationLandscapeLeft:
|
||||
return UIDeviceOrientationLandscapeRight;
|
||||
case UIInterfaceOrientationLandscapeRight:
|
||||
return UIDeviceOrientationLandscapeLeft;
|
||||
default:
|
||||
return UIDeviceOrientationPortrait;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@interface VideoCameraCapturerPreviewRecord : NSObject
|
||||
|
||||
@property (nonatomic, weak) VideoCaptureView *view;
|
||||
|
||||
@end
|
||||
|
||||
@implementation VideoCameraCapturerPreviewRecord
|
||||
|
||||
- (instancetype)initWithCaptureView:(VideoCaptureView *)view {
|
||||
self = [super init];
|
||||
if (self != nil) {
|
||||
self.view = view;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@interface VideoCameraCapturer () <AVCaptureVideoDataOutputSampleBufferDelegate> {
|
||||
webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> _source;
|
||||
|
||||
// Live on main thread.
|
||||
bool _isFrontCamera;
|
||||
bool _keepLandscape;
|
||||
|
||||
dispatch_queue_t _frameQueue;
|
||||
|
||||
// Live on RTCDispatcherTypeCaptureSession.
|
||||
AVCaptureDevice *_currentDevice;
|
||||
BOOL _hasRetriedOnFatalError;
|
||||
BOOL _isRunning;
|
||||
|
||||
// Live on RTCDispatcherTypeCaptureSession and main thread.
|
||||
std::atomic<bool> _willBeRunning;
|
||||
|
||||
AVCaptureVideoDataOutput *_videoDataOutput;
|
||||
AVCaptureSession *_captureSession;
|
||||
FourCharCode _preferredOutputPixelFormat;
|
||||
FourCharCode _outputPixelFormat;
|
||||
RTCVideoRotation _rotation;
|
||||
UIDeviceOrientation _orientation;
|
||||
bool _didReceiveOrientationUpdate;
|
||||
bool _rotationLock;
|
||||
|
||||
// Live on mainThread.
|
||||
void (^_isActiveUpdated)(bool);
|
||||
bool _isActiveValue;
|
||||
bool _inForegroundValue;
|
||||
|
||||
void (^_rotationUpdated)(int);
|
||||
|
||||
// Live on frameQueue and main thread.
|
||||
std::atomic<bool> _isPaused;
|
||||
|
||||
// Live on frameQueue.
|
||||
float _aspectRatio;
|
||||
std::vector<uint8_t> _croppingBuffer;
|
||||
std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> _uncroppedSink;
|
||||
|
||||
// Live on frameQueue and RTCDispatcherTypeCaptureSession.
|
||||
std::atomic<int> _warmupFrameCount;
|
||||
|
||||
webrtc::NV12ToI420Scaler _nv12ToI420Scaler;
|
||||
|
||||
NSMutableArray<VideoCameraCapturerPreviewRecord *> *_previews;
|
||||
std::vector<std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>> _directSinks;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@implementation VideoCameraCapturer
|
||||
|
||||
- (instancetype)initWithSource:(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source useFrontCamera:(bool)useFrontCamera keepLandscape:(bool)keepLandscape isActiveUpdated:(void (^)(bool))isActiveUpdated rotationUpdated:(void (^)(int))rotationUpdated {
|
||||
self = [super init];
|
||||
if (self != nil) {
|
||||
_source = source;
|
||||
_isFrontCamera = useFrontCamera;
|
||||
_keepLandscape = keepLandscape;
|
||||
_isActiveValue = true;
|
||||
_inForegroundValue = true;
|
||||
_isPaused = false;
|
||||
_isActiveUpdated = [isActiveUpdated copy];
|
||||
_rotationUpdated = [rotationUpdated copy];
|
||||
|
||||
_warmupFrameCount = 100;
|
||||
|
||||
_previews = [[NSMutableArray alloc] init];
|
||||
|
||||
if (![self setupCaptureSession:[[AVCaptureSession alloc] init]]) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
NSNotificationCenter *center = [NSNotificationCenter defaultCenter];
|
||||
_orientation = deviceOrientation([[UIApplication sharedApplication] statusBarOrientation]);
|
||||
_rotation = RTCVideoRotation_90;
|
||||
|
||||
switch (_orientation) {
|
||||
case UIDeviceOrientationPortrait:
|
||||
_rotation = RTCVideoRotation_90;
|
||||
break;
|
||||
case UIDeviceOrientationPortraitUpsideDown:
|
||||
_rotation = RTCVideoRotation_270;
|
||||
break;
|
||||
case UIDeviceOrientationLandscapeLeft:
|
||||
_rotation = useFrontCamera ? RTCVideoRotation_180 : RTCVideoRotation_0;
|
||||
break;
|
||||
case UIDeviceOrientationLandscapeRight:
|
||||
_rotation = useFrontCamera ? RTCVideoRotation_0 : RTCVideoRotation_180;
|
||||
break;
|
||||
case UIDeviceOrientationFaceUp:
|
||||
case UIDeviceOrientationFaceDown:
|
||||
case UIDeviceOrientationUnknown:
|
||||
// Ignore.
|
||||
break;
|
||||
}
|
||||
|
||||
if (_rotationUpdated) {
|
||||
int angle = 0;
|
||||
switch (_rotation) {
|
||||
case RTCVideoRotation_0: {
|
||||
angle = 0;
|
||||
break;
|
||||
}
|
||||
case RTCVideoRotation_90: {
|
||||
angle = 90;
|
||||
break;
|
||||
}
|
||||
case RTCVideoRotation_180: {
|
||||
angle = 180;
|
||||
break;
|
||||
}
|
||||
case RTCVideoRotation_270: {
|
||||
angle = 270;
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
break;
|
||||
}
|
||||
}
|
||||
_rotationUpdated(angle);
|
||||
}
|
||||
[center addObserver:self
|
||||
selector:@selector(deviceOrientationDidChange:)
|
||||
name:UIDeviceOrientationDidChangeNotification
|
||||
object:nil];
|
||||
[center addObserver:self
|
||||
selector:@selector(handleCaptureSessionInterruption:)
|
||||
name:AVCaptureSessionWasInterruptedNotification
|
||||
object:_captureSession];
|
||||
[center addObserver:self
|
||||
selector:@selector(handleCaptureSessionInterruptionEnded:)
|
||||
name:AVCaptureSessionInterruptionEndedNotification
|
||||
object:_captureSession];
|
||||
[center addObserver:self
|
||||
selector:@selector(handleApplicationDidBecomeActive:)
|
||||
name:UIApplicationDidBecomeActiveNotification
|
||||
object:[UIApplication sharedApplication]];
|
||||
[center addObserver:self
|
||||
selector:@selector(handleApplicationWillEnterForeground:)
|
||||
name:UIApplicationWillEnterForegroundNotification
|
||||
object:[UIApplication sharedApplication]];
|
||||
[center addObserver:self
|
||||
selector:@selector(handleCaptureSessionRuntimeError:)
|
||||
name:AVCaptureSessionRuntimeErrorNotification
|
||||
object:_captureSession];
|
||||
[center addObserver:self
|
||||
selector:@selector(handleCaptureSessionDidStartRunning:)
|
||||
name:AVCaptureSessionDidStartRunningNotification
|
||||
object:_captureSession];
|
||||
[center addObserver:self
|
||||
selector:@selector(handleCaptureSessionDidStopRunning:)
|
||||
name:AVCaptureSessionDidStopRunningNotification
|
||||
object:_captureSession];
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
NSAssert(!_willBeRunning, @"Session was still running in RTCCameraVideoCapturer dealloc. Forgot to call stopCapture?");
|
||||
[[NSNotificationCenter defaultCenter] removeObserver:self];
|
||||
}
|
||||
|
||||
+ (NSArray<AVCaptureDevice *> *)captureDevices {
|
||||
if (@available(iOS 10.0, *)) {
|
||||
AVCaptureDeviceDiscoverySession *session = [AVCaptureDeviceDiscoverySession discoverySessionWithDeviceTypes:@[AVCaptureDeviceTypeBuiltInWideAngleCamera] mediaType:AVMediaTypeVideo position:AVCaptureDevicePositionUnspecified];
|
||||
return session.devices;
|
||||
} else {
|
||||
NSMutableArray<AVCaptureDevice *> *result = [[NSMutableArray alloc] init];
|
||||
for (AVCaptureDevice *device in [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]) {
|
||||
if (device.position == AVCaptureDevicePositionFront || device.position == AVCaptureDevicePositionBack) {
|
||||
[result addObject:device];
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
+ (NSArray<AVCaptureDeviceFormat *> *)supportedFormatsForDevice:(AVCaptureDevice *)device {
|
||||
// Support opening the device in any format. We make sure it's converted to a format we
|
||||
// can handle, if needed, in the method `-setupVideoDataOutput`.
|
||||
return device.formats;
|
||||
}
|
||||
|
||||
- (FourCharCode)preferredOutputPixelFormat {
|
||||
return _preferredOutputPixelFormat;
|
||||
}
|
||||
|
||||
- (void)startCaptureWithDevice:(AVCaptureDevice *)device
|
||||
format:(AVCaptureDeviceFormat *)format
|
||||
fps:(NSInteger)fps {
|
||||
[self startCaptureWithDevice:device format:format fps:fps completionHandler:nil];
|
||||
}
|
||||
|
||||
- (void)stopCapture {
|
||||
_isActiveUpdated = nil;
|
||||
[self stopCaptureWithCompletionHandler:nil];
|
||||
}
|
||||
|
||||
- (void)setIsEnabled:(bool)isEnabled {
|
||||
_isPaused = !isEnabled;
|
||||
[self updateIsActiveValue];
|
||||
}
|
||||
|
||||
- (void)setUncroppedSink:(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)sink {
|
||||
dispatch_async(self.frameQueue, ^{
|
||||
_uncroppedSink = sink;
|
||||
});
|
||||
}
|
||||
|
||||
- (int)getRotation {
|
||||
switch (_rotation) {
|
||||
case RTCVideoRotation_0:
|
||||
return 0;
|
||||
case RTCVideoRotation_90:
|
||||
return 90;
|
||||
case RTCVideoRotation_180:
|
||||
return 180;
|
||||
case RTCVideoRotation_270:
|
||||
return 270;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)addPreviewView:(VideoCaptureView *)previewView {
|
||||
[_previews addObject:[[VideoCameraCapturerPreviewRecord alloc] initWithCaptureView:previewView]];
|
||||
[previewView previewLayer].session = _captureSession;
|
||||
}
|
||||
|
||||
- (void)addDirectSink:(std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)directSink {
|
||||
_directSinks.push_back(directSink);
|
||||
}
|
||||
|
||||
- (void)setPreferredCaptureAspectRatio:(float)aspectRatio {
|
||||
dispatch_async(self.frameQueue, ^{
|
||||
_aspectRatio = aspectRatio;
|
||||
});
|
||||
}
|
||||
|
||||
- (void)startCaptureWithDevice:(AVCaptureDevice *)device
|
||||
format:(AVCaptureDeviceFormat *)format
|
||||
fps:(NSInteger)fps
|
||||
completionHandler:(nullable void (^)(NSError *))completionHandler {
|
||||
_willBeRunning = true;
|
||||
[RTCDispatcher
|
||||
dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
|
||||
block:^{
|
||||
RTCLogInfo("startCaptureWithDevice %@ @ %ld fps", format, (long)fps);
|
||||
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
[[UIDevice currentDevice] beginGeneratingDeviceOrientationNotifications];
|
||||
});
|
||||
|
||||
_currentDevice = device;
|
||||
|
||||
NSError *error = nil;
|
||||
if (![_currentDevice lockForConfiguration:&error]) {
|
||||
RTCLogError(@"Failed to lock device %@. Error: %@",
|
||||
_currentDevice,
|
||||
error.userInfo);
|
||||
if (completionHandler) {
|
||||
completionHandler(error);
|
||||
}
|
||||
_willBeRunning = false;
|
||||
return;
|
||||
}
|
||||
[self reconfigureCaptureSessionInput];
|
||||
[self updateDeviceCaptureFormat:format fps:fps];
|
||||
[self updateVideoDataOutputPixelFormat:format];
|
||||
[_captureSession startRunning];
|
||||
[_currentDevice unlockForConfiguration];
|
||||
_isRunning = YES;
|
||||
if (completionHandler) {
|
||||
completionHandler(nil);
|
||||
}
|
||||
}];
|
||||
}
|
||||
|
||||
- (void)stopCaptureWithCompletionHandler:(nullable void (^)(void))completionHandler {
|
||||
_willBeRunning = false;
|
||||
[RTCDispatcher
|
||||
dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
|
||||
block:^{
|
||||
RTCLogInfo("Stop");
|
||||
_currentDevice = nil;
|
||||
for (AVCaptureDeviceInput *oldInput in [_captureSession.inputs copy]) {
|
||||
[_captureSession removeInput:oldInput];
|
||||
}
|
||||
[_captureSession stopRunning];
|
||||
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
[[UIDevice currentDevice] endGeneratingDeviceOrientationNotifications];
|
||||
});
|
||||
_isRunning = NO;
|
||||
if (completionHandler) {
|
||||
completionHandler();
|
||||
}
|
||||
}];
|
||||
}
|
||||
|
||||
#pragma mark iOS notifications
|
||||
|
||||
#if TARGET_OS_IPHONE
|
||||
- (void)deviceOrientationDidChange:(NSNotification *)notification {
|
||||
[RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession block:^{
|
||||
_didReceiveOrientationUpdate = true;
|
||||
[self updateOrientation];
|
||||
}];
|
||||
}
|
||||
#endif
|
||||
|
||||
#pragma mark AVCaptureVideoDataOutputSampleBufferDelegate
|
||||
|
||||
- (webrtc::scoped_refptr<webrtc::VideoFrameBuffer>)prepareI420Buffer:(CVPixelBufferRef)pixelBuffer {
|
||||
if (!pixelBuffer) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
const OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer);
|
||||
|
||||
CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
|
||||
|
||||
auto resultBuffer = rtc::make_ref_counted<webrtc::I420Buffer>(CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer));
|
||||
|
||||
switch (pixelFormat) {
|
||||
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
|
||||
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: {
|
||||
const uint8_t* srcY =
|
||||
static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
|
||||
const int srcYStride = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
|
||||
const uint8_t* srcUV =
|
||||
static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
|
||||
const int srcUVStride = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
|
||||
|
||||
// TODO(magjed): Use a frame buffer pool.
|
||||
_nv12ToI420Scaler.NV12ToI420Scale(srcY,
|
||||
srcYStride,
|
||||
srcUV,
|
||||
srcUVStride,
|
||||
resultBuffer->width(),
|
||||
resultBuffer->height(),
|
||||
resultBuffer->MutableDataY(),
|
||||
resultBuffer->StrideY(),
|
||||
resultBuffer->MutableDataU(),
|
||||
resultBuffer->StrideU(),
|
||||
resultBuffer->MutableDataV(),
|
||||
resultBuffer->StrideV(),
|
||||
resultBuffer->width(),
|
||||
resultBuffer->height());
|
||||
break;
|
||||
}
|
||||
case kCVPixelFormatType_32BGRA:
|
||||
case kCVPixelFormatType_32ARGB: {
|
||||
return nullptr;
|
||||
}
|
||||
default: { RTC_DCHECK_NOTREACHED() << "Unsupported pixel format."; }
|
||||
}
|
||||
|
||||
CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
|
||||
|
||||
return resultBuffer;
|
||||
}
|
||||
|
||||
- (webrtc::scoped_refptr<webrtc::VideoFrameBuffer>)prepareNV12Buffer:(CVPixelBufferRef)pixelBuffer {
|
||||
if (!pixelBuffer) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
const OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer);
|
||||
|
||||
CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
|
||||
|
||||
switch (pixelFormat) {
|
||||
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
|
||||
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: {
|
||||
const uint8_t* srcY =
|
||||
static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
|
||||
const int srcYStride = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
|
||||
const uint8_t* srcUV =
|
||||
static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
|
||||
const int srcUVStride = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
|
||||
|
||||
const int srcWidth = (int)CVPixelBufferGetWidth(pixelBuffer);
|
||||
const int srcHeight = (int)CVPixelBufferGetHeight(pixelBuffer);
|
||||
|
||||
int resultWidth = (int)(srcWidth * 0.8f);
|
||||
resultWidth &= ~1;
|
||||
int resultHeight = (int)(srcHeight * 0.8f);
|
||||
resultHeight &= ~1;
|
||||
|
||||
webrtc::scoped_refptr<webrtc::NV12Buffer> resultBuffer = rtc::make_ref_counted<webrtc::NV12Buffer>(resultWidth, resultHeight, srcYStride, srcUVStride);
|
||||
|
||||
libyuv::NV12Scale(srcY, srcYStride, srcUV, srcUVStride,
|
||||
resultWidth, resultHeight, resultBuffer->MutableDataY(),
|
||||
resultBuffer->StrideY(), resultBuffer->MutableDataUV(), resultBuffer->StrideUV(), resultBuffer->width(),
|
||||
resultBuffer->height(), libyuv::kFilterBilinear);
|
||||
|
||||
CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
|
||||
|
||||
return resultBuffer;
|
||||
}
|
||||
case kCVPixelFormatType_32BGRA:
|
||||
case kCVPixelFormatType_32ARGB: {
|
||||
CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
|
||||
return nullptr;
|
||||
}
|
||||
default: { RTC_DCHECK_NOTREACHED() << "Unsupported pixel format."; }
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
- (void)captureOutput:(AVCaptureOutput *)captureOutput
|
||||
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
|
||||
fromConnection:(AVCaptureConnection *)connection {
|
||||
NSParameterAssert(captureOutput == _videoDataOutput);
|
||||
|
||||
int minWarmupFrameCount = 12;
|
||||
_warmupFrameCount++;
|
||||
if (_warmupFrameCount < minWarmupFrameCount) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (CMSampleBufferGetNumSamples(sampleBuffer) != 1 || !CMSampleBufferIsValid(sampleBuffer) ||
|
||||
!CMSampleBufferDataIsReady(sampleBuffer)) {
|
||||
return;
|
||||
}
|
||||
|
||||
CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
|
||||
if (pixelBuffer == nil) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Default to portrait orientation on iPhone.
|
||||
BOOL usingFrontCamera = NO;
|
||||
// Check the image's EXIF for the camera the image came from as the image could have been
|
||||
// delayed as we set alwaysDiscardsLateVideoFrames to NO.
|
||||
AVCaptureDevicePosition cameraPosition =
|
||||
[AVCaptureSession devicePositionForSampleBuffer:sampleBuffer];
|
||||
if (cameraPosition != AVCaptureDevicePositionUnspecified) {
|
||||
usingFrontCamera = AVCaptureDevicePositionFront == cameraPosition;
|
||||
} else {
|
||||
AVCaptureDeviceInput *deviceInput =
|
||||
(AVCaptureDeviceInput *)((AVCaptureInputPort *)connection.inputPorts.firstObject).input;
|
||||
usingFrontCamera = AVCaptureDevicePositionFront == deviceInput.device.position;
|
||||
}
|
||||
int deviceRelativeVideoRotation = usingFrontCamera ? RTCVideoRotation_90 : RTCVideoRotation_90;
|
||||
if (!_rotationLock) {
|
||||
RTCVideoRotation updatedRotation = _rotation;
|
||||
switch (_orientation) {
|
||||
case UIDeviceOrientationPortrait:
|
||||
updatedRotation = RTCVideoRotation_90;
|
||||
break;
|
||||
case UIDeviceOrientationPortraitUpsideDown:
|
||||
updatedRotation = RTCVideoRotation_270;
|
||||
break;
|
||||
case UIDeviceOrientationLandscapeLeft:
|
||||
updatedRotation = usingFrontCamera ? RTCVideoRotation_180 : RTCVideoRotation_0;
|
||||
break;
|
||||
case UIDeviceOrientationLandscapeRight:
|
||||
updatedRotation = usingFrontCamera ? RTCVideoRotation_0 : RTCVideoRotation_180;
|
||||
break;
|
||||
case UIDeviceOrientationFaceUp:
|
||||
case UIDeviceOrientationFaceDown:
|
||||
case UIDeviceOrientationUnknown:
|
||||
// Ignore.
|
||||
break;
|
||||
}
|
||||
if (_rotation != updatedRotation) {
|
||||
_rotation = updatedRotation;
|
||||
if (_rotationUpdated) {
|
||||
int angle = 0;
|
||||
switch (_rotation) {
|
||||
case RTCVideoRotation_0: {
|
||||
angle = 0;
|
||||
break;
|
||||
}
|
||||
case RTCVideoRotation_90: {
|
||||
angle = 90;
|
||||
break;
|
||||
}
|
||||
case RTCVideoRotation_180: {
|
||||
angle = 180;
|
||||
break;
|
||||
}
|
||||
case RTCVideoRotation_270: {
|
||||
angle = 270;
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
break;
|
||||
}
|
||||
}
|
||||
_rotationUpdated(angle);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TGRTCCVPixelBuffer *rtcPixelBuffer = [[TGRTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer];
|
||||
rtcPixelBuffer.shouldBeMirrored = usingFrontCamera;
|
||||
rtcPixelBuffer.deviceRelativeVideoRotation = deviceRelativeVideoRotation;
|
||||
|
||||
TGRTCCVPixelBuffer *uncroppedRtcPixelBuffer = rtcPixelBuffer;
|
||||
|
||||
CGSize initialSize = CGSizeMake(uncroppedRtcPixelBuffer.width, uncroppedRtcPixelBuffer.height);
|
||||
|
||||
if (_aspectRatio > FLT_EPSILON) {
|
||||
float aspect = 1.0f / _aspectRatio;
|
||||
|
||||
int width = rtcPixelBuffer.width;
|
||||
int height = rtcPixelBuffer.height;
|
||||
|
||||
int cropX = 0;
|
||||
int cropY = 0;
|
||||
|
||||
if (_keepLandscape && width > height) {
|
||||
float aspectWidth = 404.0f;
|
||||
float aspectHeight = 720.0f;
|
||||
cropX = (int)((width - aspectWidth) / 2.0f);
|
||||
cropY = (int)((height - aspectHeight) / 2.0f);
|
||||
width = aspectWidth;
|
||||
height = aspectHeight;
|
||||
} else {
|
||||
float aspectWidth = width;
|
||||
float aspectHeight = ((float)(width)) / aspect;
|
||||
cropX = (int)((width - aspectWidth) / 2.0f);
|
||||
cropY = (int)((height - aspectHeight) / 2.0f);
|
||||
width = (int)aspectWidth;
|
||||
width &= ~1;
|
||||
height = (int)aspectHeight;
|
||||
height &= ~1;
|
||||
}
|
||||
|
||||
height = MIN(rtcPixelBuffer.height, height + 16);
|
||||
|
||||
if (width < rtcPixelBuffer.width || height < rtcPixelBuffer.height) {
|
||||
rtcPixelBuffer = [[TGRTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer adaptedWidth:width adaptedHeight:height cropWidth:width cropHeight:height cropX:cropX cropY:cropY];
|
||||
rtcPixelBuffer.shouldBeMirrored = usingFrontCamera;
|
||||
rtcPixelBuffer.deviceRelativeVideoRotation = deviceRelativeVideoRotation;
|
||||
|
||||
CVPixelBufferRef outputPixelBufferRef = NULL;
|
||||
OSType pixelFormat = CVPixelBufferGetPixelFormatType(rtcPixelBuffer.pixelBuffer);
|
||||
CVPixelBufferCreate(NULL, width, height, pixelFormat, NULL, &outputPixelBufferRef);
|
||||
if (outputPixelBufferRef) {
|
||||
int bufferSize = [rtcPixelBuffer bufferSizeForCroppingAndScalingToWidth:width height:height];
|
||||
if (_croppingBuffer.size() < bufferSize) {
|
||||
_croppingBuffer.resize(bufferSize);
|
||||
}
|
||||
if ([rtcPixelBuffer cropAndScaleTo:outputPixelBufferRef withTempBuffer:_croppingBuffer.data()]) {
|
||||
rtcPixelBuffer = [[TGRTCCVPixelBuffer alloc] initWithPixelBuffer:outputPixelBufferRef];
|
||||
rtcPixelBuffer.shouldBeMirrored = usingFrontCamera;
|
||||
rtcPixelBuffer.deviceRelativeVideoRotation = deviceRelativeVideoRotation;
|
||||
}
|
||||
CVPixelBufferRelease(outputPixelBufferRef);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int64_t timeStampNs = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) * kNanosecondsPerSecond;
|
||||
|
||||
//RTCVideoFrame *videoFrame = [[RTCVideoFrame alloc] initWithBuffer:rtcPixelBuffer rotation:_rotation timeStampNs:timeStampNs];
|
||||
|
||||
//RTCVideoFrame *videoFrame = [[RTCVideoFrame alloc] initWithBuffer:(id<RTCVideoFrameBuffer>)[rtcPixelBuffer toI420] rotation:_rotation timeStampNs:timeStampNs];
|
||||
|
||||
webrtc::VideoRotation rotation = static_cast<webrtc::VideoRotation>(_rotation);
|
||||
|
||||
int previewRotation = 0;
|
||||
CGSize previewSize = initialSize;
|
||||
if (rotation == 90 || rotation == 270) {
|
||||
previewSize = CGSizeMake(previewSize.height, previewSize.width);
|
||||
}
|
||||
|
||||
for (VideoCameraCapturerPreviewRecord *record in _previews) {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
VideoCaptureView *captureView = record.view;
|
||||
[captureView onFrameGenerated:previewSize isMirrored:true rotation:previewRotation];
|
||||
});
|
||||
}
|
||||
|
||||
auto i420Buffer = [self prepareI420Buffer:[rtcPixelBuffer pixelBuffer]];
|
||||
|
||||
if (!_isPaused && i420Buffer) {
|
||||
auto videoFrame = webrtc::VideoFrame::Builder()
|
||||
.set_video_frame_buffer(i420Buffer)
|
||||
.set_rotation(rotation)
|
||||
.set_timestamp_us(timeStampNs)
|
||||
.build();
|
||||
|
||||
if (getObjCVideoSource(_source)->OnCapturedFrame(videoFrame)) {
|
||||
if (!_directSinks.empty()) {
|
||||
for (const auto &it : _directSinks) {
|
||||
if (const auto value = it.lock()) {
|
||||
value->OnFrame(videoFrame);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (uncroppedRtcPixelBuffer) {
|
||||
const auto uncroppedSink = _uncroppedSink.lock();
|
||||
if (uncroppedSink) {
|
||||
int64_t timeStampNs = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) *
|
||||
kNanosecondsPerSecond;
|
||||
RTCVideoFrame *frame = [[RTCVideoFrame alloc] initWithBuffer:uncroppedRtcPixelBuffer rotation:_rotation timeStampNs:timeStampNs];
|
||||
|
||||
const int64_t timestamp_us = frame.timeStampNs / rtc::kNumNanosecsPerMicrosec;
|
||||
|
||||
webrtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer;
|
||||
buffer = new rtc::RefCountedObject<webrtc::ObjCFrameBuffer>(frame.buffer);
|
||||
|
||||
webrtc::VideoRotation rotation = static_cast<webrtc::VideoRotation>(frame.rotation);
|
||||
|
||||
uncroppedSink->OnFrame(webrtc::VideoFrame::Builder()
|
||||
.set_video_frame_buffer(buffer)
|
||||
.set_rotation(rotation)
|
||||
.set_timestamp_us(timestamp_us)
|
||||
.build());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
- (void)captureOutput:(AVCaptureOutput *)captureOutput
|
||||
didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer
|
||||
fromConnection:(AVCaptureConnection *)connection {
|
||||
NSString *droppedReason =
|
||||
(__bridge NSString *)CMGetAttachment(sampleBuffer, kCMSampleBufferAttachmentKey_DroppedFrameReason, nil);
|
||||
RTCLogError(@"Dropped sample buffer. Reason: %@", droppedReason);
|
||||
}
|
||||
|
||||
#pragma mark - AVCaptureSession notifications
|
||||
|
||||
- (void)handleCaptureSessionInterruption:(NSNotification *)notification {
|
||||
NSString *reasonString = nil;
|
||||
NSNumber *reason = notification.userInfo[AVCaptureSessionInterruptionReasonKey];
|
||||
if (reason) {
|
||||
switch (reason.intValue) {
|
||||
case AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableInBackground:
|
||||
reasonString = @"VideoDeviceNotAvailableInBackground";
|
||||
break;
|
||||
case AVCaptureSessionInterruptionReasonAudioDeviceInUseByAnotherClient:
|
||||
reasonString = @"AudioDeviceInUseByAnotherClient";
|
||||
break;
|
||||
case AVCaptureSessionInterruptionReasonVideoDeviceInUseByAnotherClient:
|
||||
reasonString = @"VideoDeviceInUseByAnotherClient";
|
||||
break;
|
||||
case AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableWithMultipleForegroundApps:
|
||||
reasonString = @"VideoDeviceNotAvailableWithMultipleForegroundApps";
|
||||
break;
|
||||
}
|
||||
}
|
||||
RTCLog(@"Capture session interrupted: %@", reasonString);
|
||||
}
|
||||
|
||||
- (void)handleCaptureSessionInterruptionEnded:(NSNotification *)notification {
|
||||
RTCLog(@"Capture session interruption ended.");
|
||||
}
|
||||
|
||||
- (void)handleCaptureSessionRuntimeError:(NSNotification *)notification {
|
||||
NSError *error = [notification.userInfo objectForKey:AVCaptureSessionErrorKey];
|
||||
RTCLogError(@"Capture session runtime error: %@", error);
|
||||
|
||||
[RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
|
||||
block:^{
|
||||
if (error.code == AVErrorMediaServicesWereReset) {
|
||||
[self handleNonFatalError];
|
||||
} else {
|
||||
[self handleFatalError];
|
||||
}
|
||||
}];
|
||||
}
|
||||
|
||||
- (void)handleCaptureSessionDidStartRunning:(NSNotification *)notification {
|
||||
RTCLog(@"Capture session started.");
|
||||
|
||||
[RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
|
||||
block:^{
|
||||
// If we successfully restarted after an unknown error,
|
||||
// allow future retries on fatal errors.
|
||||
_hasRetriedOnFatalError = NO;
|
||||
}];
|
||||
|
||||
_inForegroundValue = true;
|
||||
[self updateIsActiveValue];
|
||||
}
|
||||
|
||||
- (void)handleCaptureSessionDidStopRunning:(NSNotification *)notification {
|
||||
RTCLog(@"Capture session stopped.");
|
||||
_inForegroundValue = false;
|
||||
[self updateIsActiveValue];
|
||||
}
|
||||
|
||||
- (void)updateIsActiveValue {
|
||||
bool isActive = _inForegroundValue && !_isPaused;
|
||||
if (isActive != _isActiveValue) {
|
||||
_isActiveValue = isActive;
|
||||
if (_isActiveUpdated) {
|
||||
_isActiveUpdated(_isActiveValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
- (void)handleFatalError {
|
||||
[RTCDispatcher
|
||||
dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
|
||||
block:^{
|
||||
if (!_hasRetriedOnFatalError) {
|
||||
RTCLogWarning(@"Attempting to recover from fatal capture error.");
|
||||
[self handleNonFatalError];
|
||||
_hasRetriedOnFatalError = YES;
|
||||
} else {
|
||||
RTCLogError(@"Previous fatal error recovery failed.");
|
||||
}
|
||||
}];
|
||||
}
|
||||
|
||||
- (void)handleNonFatalError {
|
||||
[RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
|
||||
block:^{
|
||||
RTCLog(@"Restarting capture session after error.");
|
||||
if (_isRunning) {
|
||||
[_captureSession startRunning];
|
||||
}
|
||||
}];
|
||||
}
|
||||
|
||||
#pragma mark - UIApplication notifications
|
||||
|
||||
- (void)handleApplicationDidBecomeActive:(NSNotification *)notification {
|
||||
[RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
|
||||
block:^{
|
||||
if (_isRunning && !_captureSession.isRunning) {
|
||||
RTCLog(@"Restarting capture session on active.");
|
||||
_warmupFrameCount = 0;
|
||||
[_captureSession startRunning];
|
||||
}
|
||||
}];
|
||||
}
|
||||
|
||||
- (void)handleApplicationWillEnterForeground:(NSNotification *)notification {
|
||||
[RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
|
||||
block:^{
|
||||
RTCLog(@"Resetting warmup due to backgrounding.");
|
||||
_warmupFrameCount = 0;
|
||||
}];
|
||||
}
|
||||
|
||||
#pragma mark - Private
|
||||
|
||||
- (dispatch_queue_t)frameQueue {
|
||||
if (!_frameQueue) {
|
||||
_frameQueue =
|
||||
dispatch_queue_create("org.webrtc.cameravideocapturer.video", DISPATCH_QUEUE_SERIAL);
|
||||
dispatch_set_target_queue(_frameQueue,
|
||||
dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0));
|
||||
}
|
||||
return _frameQueue;
|
||||
}
|
||||
|
||||
- (BOOL)setupCaptureSession:(AVCaptureSession *)captureSession {
|
||||
NSAssert(_captureSession == nil, @"Setup capture session called twice.");
|
||||
_captureSession = captureSession;
|
||||
_captureSession.sessionPreset = AVCaptureSessionPresetInputPriority;
|
||||
_captureSession.usesApplicationAudioSession = true;
|
||||
if (@available(iOS 16.0, *)) {
|
||||
if (_captureSession.isMultitaskingCameraAccessSupported) {
|
||||
_captureSession.multitaskingCameraAccessEnabled = true;
|
||||
}
|
||||
}
|
||||
[self setupVideoDataOutput];
|
||||
// Add the output.
|
||||
if (![_captureSession canAddOutput:_videoDataOutput]) {
|
||||
RTCLogError(@"Video data output unsupported.");
|
||||
return NO;
|
||||
}
|
||||
[_captureSession addOutput:_videoDataOutput];
|
||||
|
||||
return YES;
|
||||
}
|
||||
|
||||
- (void)setupVideoDataOutput {
|
||||
NSAssert(_videoDataOutput == nil, @"Setup video data output called twice.");
|
||||
AVCaptureVideoDataOutput *videoDataOutput = [[AVCaptureVideoDataOutput alloc] init];
|
||||
|
||||
// `videoDataOutput.availableVideoCVPixelFormatTypes` returns the pixel formats supported by the
|
||||
// device with the most efficient output format first. Find the first format that we support.
|
||||
NSSet<NSNumber *> *supportedPixelFormats = [TGRTCCVPixelBuffer supportedPixelFormats];
|
||||
NSMutableOrderedSet *availablePixelFormats =
|
||||
[NSMutableOrderedSet orderedSetWithArray:videoDataOutput.availableVideoCVPixelFormatTypes];
|
||||
[availablePixelFormats intersectSet:supportedPixelFormats];
|
||||
NSNumber *pixelFormat = availablePixelFormats.firstObject;
|
||||
NSAssert(pixelFormat, @"Output device has no supported formats.");
|
||||
|
||||
_preferredOutputPixelFormat = [pixelFormat unsignedIntValue];
|
||||
_outputPixelFormat = _preferredOutputPixelFormat;
|
||||
videoDataOutput.videoSettings = @{(NSString *)kCVPixelBufferPixelFormatTypeKey : pixelFormat};
|
||||
videoDataOutput.alwaysDiscardsLateVideoFrames = NO;
|
||||
[videoDataOutput setSampleBufferDelegate:self queue:self.frameQueue];
|
||||
_videoDataOutput = videoDataOutput;
|
||||
}
|
||||
|
||||
- (void)updateVideoDataOutputPixelFormat:(AVCaptureDeviceFormat *)format {
|
||||
FourCharCode mediaSubType = CMFormatDescriptionGetMediaSubType(format.formatDescription);
|
||||
if (![[TGRTCCVPixelBuffer supportedPixelFormats] containsObject:@(mediaSubType)]) {
|
||||
mediaSubType = _preferredOutputPixelFormat;
|
||||
}
|
||||
|
||||
if (mediaSubType != _outputPixelFormat) {
|
||||
_outputPixelFormat = mediaSubType;
|
||||
_videoDataOutput.videoSettings =
|
||||
@{ (NSString *)kCVPixelBufferPixelFormatTypeKey : @(mediaSubType) };
|
||||
}
|
||||
}
|
||||
|
||||
#pragma mark - Private, called inside capture queue
|
||||
|
||||
- (void)updateDeviceCaptureFormat:(AVCaptureDeviceFormat *)format fps:(NSInteger)fps {
|
||||
NSAssert([RTCDispatcher isOnQueueForType:RTCDispatcherTypeCaptureSession],
|
||||
@"updateDeviceCaptureFormat must be called on the capture queue.");
|
||||
@try {
|
||||
_currentDevice.activeFormat = format;
|
||||
_currentDevice.activeVideoMinFrameDuration = CMTimeMake(1, (int32_t)fps);
|
||||
} @catch (NSException *exception) {
|
||||
RTCLogError(@"Failed to set active format!\n User info:%@", exception.userInfo);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)reconfigureCaptureSessionInput {
|
||||
NSAssert([RTCDispatcher isOnQueueForType:RTCDispatcherTypeCaptureSession],
|
||||
@"reconfigureCaptureSessionInput must be called on the capture queue.");
|
||||
NSError *error = nil;
|
||||
AVCaptureDeviceInput *input =
|
||||
[AVCaptureDeviceInput deviceInputWithDevice:_currentDevice error:&error];
|
||||
if (!input) {
|
||||
RTCLogError(@"Failed to create front camera input: %@", error.localizedDescription);
|
||||
return;
|
||||
}
|
||||
[_captureSession beginConfiguration];
|
||||
for (AVCaptureDeviceInput *oldInput in [_captureSession.inputs copy]) {
|
||||
[_captureSession removeInput:oldInput];
|
||||
}
|
||||
if ([_captureSession canAddInput:input]) {
|
||||
[_captureSession addInput:input];
|
||||
} else {
|
||||
RTCLogError(@"Cannot add camera as an input to the session.");
|
||||
}
|
||||
[_captureSession commitConfiguration];
|
||||
}
|
||||
|
||||
- (void)updateOrientation {
|
||||
NSAssert([RTCDispatcher isOnQueueForType:RTCDispatcherTypeCaptureSession],
|
||||
@"updateOrientation must be called on the capture queue.");
|
||||
if (_didReceiveOrientationUpdate) {
|
||||
_orientation = [UIDevice currentDevice].orientation;
|
||||
}
|
||||
}
|
||||
|
||||
@end
|
||||
//
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
#ifndef TGCALLS_VIDEO_CAMERA_CAPTURER_MAC_H
|
||||
#define TGCALLS_VIDEO_CAMERA_CAPTURER_MAC_H
|
||||
#ifndef WEBRTC_IOS
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
|
||||
#include <memory>
|
||||
#include "api/scoped_refptr.h"
|
||||
#include "api/media_stream_interface.h"
|
||||
|
||||
|
||||
@protocol CapturerInterface
|
||||
- (void)start;
|
||||
- (void)stop;
|
||||
- (void)setIsEnabled:(bool)isEnabled;
|
||||
- (void)setUncroppedSink:(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)sink;
|
||||
- (void)setPreferredCaptureAspectRatio:(float)aspectRatio;
|
||||
- (void)setOnFatalError:(std::function<void()>)error;
|
||||
- (void)setOnPause:(std::function<void(bool)>)pause;
|
||||
@end
|
||||
|
||||
@interface VideoCameraCapturer : NSObject<CapturerInterface>
|
||||
|
||||
+ (NSArray<AVCaptureDevice *> *)captureDevices;
|
||||
+ (NSArray<AVCaptureDeviceFormat *> *)supportedFormatsForDevice:(AVCaptureDevice *)device;
|
||||
|
||||
- (instancetype)initWithSource:(rtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source isActiveUpdated:(void (^)(bool))isActiveUpdated;
|
||||
|
||||
- (void)setupCaptureWithDevice:(AVCaptureDevice *)device format:(AVCaptureDeviceFormat *)format fps:(NSInteger)fps;
|
||||
- (void)setUncroppedSink:(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)sink;
|
||||
- (BOOL)deviceIsCaptureCompitable:(AVCaptureDevice *)device;
|
||||
|
||||
@end
|
||||
#endif //WEBRTC_MAC
|
||||
#endif
|
||||
|
|
@ -0,0 +1,725 @@
|
|||
#include "VideoCameraCapturerMac.h"
|
||||
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
#import "TGRTCCVPixelBuffer.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#import "base/RTCLogging.h"
|
||||
#import "base/RTCVideoFrameBuffer.h"
|
||||
#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
|
||||
#import "sdk/objc/native/src/objc_video_track_source.h"
|
||||
#import "sdk/objc/native/src/objc_frame_buffer.h"
|
||||
#import <CoreMediaIO/CMIOHardware.h>
|
||||
#import "TGCMIODevice.h"
|
||||
#import "helpers/AVCaptureSession+DevicePosition.h"
|
||||
#import "helpers/RTCDispatcher+Private.h"
|
||||
#import "base/RTCVideoFrame.h"
|
||||
|
||||
#include "common_video/libyuv/include/webrtc_libyuv.h"
|
||||
#include "rtc_base/checks.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "pc/video_track_source_proxy.h"
|
||||
#include "third_party/libyuv/include/libyuv.h"
|
||||
#include "DarwinVideoSource.h"
|
||||
|
||||
struct CameraFrameSize {
|
||||
int width = 0;
|
||||
int height = 0;
|
||||
};
|
||||
|
||||
CameraFrameSize AspectFitted(CameraFrameSize from, CameraFrameSize to) {
|
||||
double scale = std::min(
|
||||
from.width / std::max(1., double(to.width)),
|
||||
from.height / std::max(1., double(to.height)));
|
||||
return {
|
||||
int(std::ceil(to.width * scale)),
|
||||
int(std::ceil(to.height * scale))
|
||||
};
|
||||
}
|
||||
|
||||
static const int64_t kNanosecondsPerSecond = 1000000000;
|
||||
|
||||
static tgcalls::DarwinVideoTrackSource *getObjCVideoSource(const rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> nativeSource) {
|
||||
webrtc::VideoTrackSourceProxy *proxy_source =
|
||||
static_cast<webrtc::VideoTrackSourceProxy *>(nativeSource.get());
|
||||
return static_cast<tgcalls::DarwinVideoTrackSource *>(proxy_source->internal());
|
||||
}
|
||||
|
||||
|
||||
@interface RTCCVPixelBuffer (CustomCropping)
|
||||
|
||||
@end
|
||||
|
||||
@implementation RTCCVPixelBuffer (CustomCropping)
|
||||
|
||||
- (BOOL)custom_cropAndScaleTo:(CVPixelBufferRef)outputPixelBuffer
|
||||
withTempBuffer:(nullable uint8_t*)tmpBuffer {
|
||||
const OSType srcPixelFormat = CVPixelBufferGetPixelFormatType(self.pixelBuffer);
|
||||
const OSType dstPixelFormat = CVPixelBufferGetPixelFormatType(outputPixelBuffer);
|
||||
|
||||
switch (srcPixelFormat) {
|
||||
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
|
||||
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: {
|
||||
size_t dstWidth = CVPixelBufferGetWidth(outputPixelBuffer);
|
||||
size_t dstHeight = CVPixelBufferGetHeight(outputPixelBuffer);
|
||||
if (dstWidth > 0 && dstHeight > 0) {
|
||||
RTC_DCHECK(dstPixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ||
|
||||
dstPixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange);
|
||||
if ([self requiresScalingToWidth:(int)dstWidth height:(int)dstHeight]) {
|
||||
RTC_DCHECK(tmpBuffer);
|
||||
}
|
||||
[self custom_cropAndScaleNV12To:outputPixelBuffer withTempBuffer:tmpBuffer];
|
||||
}
|
||||
break;
|
||||
}
|
||||
case kCVPixelFormatType_32BGRA:
|
||||
case kCVPixelFormatType_32ARGB: {
|
||||
RTC_DCHECK(srcPixelFormat == dstPixelFormat);
|
||||
[self custom_cropAndScaleARGBTo:outputPixelBuffer];
|
||||
break;
|
||||
}
|
||||
default: { RTC_DCHECK_NOTREACHED() << "Unsupported pixel format."; }
|
||||
}
|
||||
|
||||
return YES;
|
||||
}
|
||||
|
||||
- (void)custom_cropAndScaleNV12To:(CVPixelBufferRef)outputPixelBuffer withTempBuffer:(uint8_t*)tmpBuffer {
|
||||
// Prepare output pointers.
|
||||
CVReturn cvRet = CVPixelBufferLockBaseAddress(outputPixelBuffer, 0);
|
||||
if (cvRet != kCVReturnSuccess) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
|
||||
}
|
||||
const int dstWidth = (int)CVPixelBufferGetWidth(outputPixelBuffer);
|
||||
const int dstHeight = (int)CVPixelBufferGetHeight(outputPixelBuffer);
|
||||
uint8_t* dstY =
|
||||
reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 0));
|
||||
const int dstYStride = (int)CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 0);
|
||||
uint8_t* dstUV =
|
||||
reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 1));
|
||||
const int dstUVStride = (int)CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 1);
|
||||
|
||||
// Prepare source pointers.
|
||||
CVPixelBufferLockBaseAddress(self.pixelBuffer, kCVPixelBufferLock_ReadOnly);
|
||||
const uint8_t* srcY = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(self.pixelBuffer, 0));
|
||||
const int srcYStride = (int)CVPixelBufferGetBytesPerRowOfPlane(self.pixelBuffer, 0);
|
||||
const uint8_t* srcUV = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(self.pixelBuffer, 1));
|
||||
const int srcUVStride = (int)CVPixelBufferGetBytesPerRowOfPlane(self.pixelBuffer, 1);
|
||||
|
||||
// Crop just by modifying pointers.
|
||||
srcY += srcYStride * self.cropY + self.cropX;
|
||||
srcUV += srcUVStride * (self.cropY / 2) + self.cropX;
|
||||
|
||||
webrtc::NV12Scale(tmpBuffer,
|
||||
srcY,
|
||||
srcYStride,
|
||||
srcUV,
|
||||
srcUVStride,
|
||||
self.cropWidth,
|
||||
self.cropHeight,
|
||||
dstY,
|
||||
dstYStride,
|
||||
dstUV,
|
||||
dstUVStride,
|
||||
dstWidth,
|
||||
dstHeight);
|
||||
|
||||
CVPixelBufferUnlockBaseAddress(self.pixelBuffer, kCVPixelBufferLock_ReadOnly);
|
||||
CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0);
|
||||
}
|
||||
|
||||
- (void)custom_cropAndScaleARGBTo:(CVPixelBufferRef)outputPixelBuffer {
|
||||
// Prepare output pointers.
|
||||
CVReturn cvRet = CVPixelBufferLockBaseAddress(outputPixelBuffer, 0);
|
||||
if (cvRet != kCVReturnSuccess) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
|
||||
}
|
||||
const int dstWidth = (int)CVPixelBufferGetWidth(outputPixelBuffer);
|
||||
const int dstHeight = (int)CVPixelBufferGetHeight(outputPixelBuffer);
|
||||
|
||||
uint8_t* dst = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddress(outputPixelBuffer));
|
||||
const int dstStride = (int)CVPixelBufferGetBytesPerRow(outputPixelBuffer);
|
||||
|
||||
// Prepare source pointers.
|
||||
CVPixelBufferLockBaseAddress(self.pixelBuffer, kCVPixelBufferLock_ReadOnly);
|
||||
const uint8_t* src = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(self.pixelBuffer));
|
||||
const int srcStride = (int)CVPixelBufferGetBytesPerRow(self.pixelBuffer);
|
||||
|
||||
// Crop just by modifying pointers. Need to ensure that src pointer points to a byte corresponding
|
||||
// to the start of a new pixel (byte with B for BGRA) so that libyuv scales correctly.
|
||||
const int bytesPerPixel = 4;
|
||||
src += srcStride * self.cropY + (self.cropX * bytesPerPixel);
|
||||
|
||||
// kCVPixelFormatType_32BGRA corresponds to libyuv::FOURCC_ARGB
|
||||
libyuv::ARGBScale(src,
|
||||
srcStride,
|
||||
self.cropWidth,
|
||||
self.cropHeight,
|
||||
dst,
|
||||
dstStride,
|
||||
dstWidth,
|
||||
dstHeight,
|
||||
libyuv::kFilterBox);
|
||||
|
||||
CVPixelBufferUnlockBaseAddress(self.pixelBuffer, kCVPixelBufferLock_ReadOnly);
|
||||
CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0);
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
|
||||
|
||||
@interface VideoCameraCapturer () <AVCaptureVideoDataOutputSampleBufferDelegate> {
|
||||
rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> _source;
|
||||
|
||||
dispatch_queue_t _frameQueue;
|
||||
AVCaptureDevice *_currentDevice;
|
||||
AVCaptureInput *_currentInput;
|
||||
|
||||
// Live on RTCDispatcherTypeCaptureSession.
|
||||
BOOL _hasRetriedOnFatalError;
|
||||
BOOL _hadFatalError;
|
||||
BOOL _isRunning;
|
||||
|
||||
BOOL _shouldBeMirrored;
|
||||
|
||||
// Live on RTCDispatcherTypeCaptureSession and main thread.
|
||||
std::atomic<bool> _willBeRunning;
|
||||
|
||||
AVCaptureVideoDataOutput *_videoDataOutput;
|
||||
AVCaptureSession *_captureSession;
|
||||
|
||||
AVCaptureConnection *_videoConnection;
|
||||
AVCaptureDevice *_videoDevice;
|
||||
AVCaptureDeviceInput *_videoInputDevice;
|
||||
FourCharCode _preferredOutputPixelFormat;
|
||||
FourCharCode _outputPixelFormat;
|
||||
RTCVideoRotation _rotation;
|
||||
|
||||
// Live on mainThread.
|
||||
void (^_isActiveUpdated)(bool);
|
||||
bool _isActiveValue;
|
||||
bool _inForegroundValue;
|
||||
|
||||
// Live on frameQueue and main thread.
|
||||
std::atomic<bool> _isPaused;
|
||||
std::atomic<int> _skippedFrame;
|
||||
|
||||
// Live on frameQueue;
|
||||
float _aspectRatio;
|
||||
std::vector<uint8_t> _croppingBuffer;
|
||||
std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> _uncroppedSink;
|
||||
std::function<void ()> _onFatalError;
|
||||
int _warmupFrameCount;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@implementation VideoCameraCapturer
|
||||
|
||||
- (instancetype)initWithSource:(rtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source isActiveUpdated:(void (^)(bool))isActiveUpdated {
|
||||
self = [super init];
|
||||
if (self != nil) {
|
||||
_source = source;
|
||||
_isActiveUpdated = [isActiveUpdated copy];
|
||||
_isActiveValue = true;
|
||||
_inForegroundValue = true;
|
||||
_isPaused = false;
|
||||
_skippedFrame = 0;
|
||||
_rotation = RTCVideoRotation_0;
|
||||
|
||||
_warmupFrameCount = 100;
|
||||
|
||||
if (![self setupCaptureSession:[[AVCaptureSession alloc] init]]) {
|
||||
return nil;
|
||||
}
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
|
||||
|
||||
NSAssert(!_willBeRunning, @"Session was still running in RTCCameraVideoCapturer dealloc. Forgot to call stopCapture?");
|
||||
[[NSNotificationCenter defaultCenter] removeObserver:self];
|
||||
}
|
||||
|
||||
+ (NSArray<AVCaptureDevice *> *)captureDevices {
|
||||
AVCaptureDevice * defaultDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
|
||||
NSMutableArray<AVCaptureDevice *> * devices = [[AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo] mutableCopy];
|
||||
|
||||
[devices addObjectsFromArray:[AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed]];
|
||||
|
||||
if ([devices count] > 0) {
|
||||
[devices insertObject:defaultDevice atIndex:0];
|
||||
}
|
||||
|
||||
return devices;
|
||||
}
|
||||
|
||||
- (BOOL)deviceIsCaptureCompitable:(AVCaptureDevice *)device {
|
||||
if (![device isConnected] || [device isSuspended]) {
|
||||
return NO;
|
||||
}
|
||||
AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:device error:nil];
|
||||
|
||||
return [_captureSession canAddInput:input];
|
||||
}
|
||||
|
||||
+ (NSArray<AVCaptureDeviceFormat *> *)supportedFormatsForDevice:(AVCaptureDevice *)device {
|
||||
// Support opening the device in any format. We make sure it's converted to a format we
|
||||
// can handle, if needed, in the method `-setupVideoDataOutput`.
|
||||
return device.formats;
|
||||
}
|
||||
|
||||
- (FourCharCode)preferredOutputPixelFormat {
|
||||
return _preferredOutputPixelFormat;
|
||||
}
|
||||
|
||||
- (void)setupCaptureWithDevice:(AVCaptureDevice *)device
|
||||
format:(AVCaptureDeviceFormat *)format
|
||||
fps:(NSInteger)fps {
|
||||
[self setupCaptureWithDevice:device format:format fps:fps completionHandler:nil];
|
||||
}
|
||||
|
||||
-(void)setOnFatalError:(std::function<void ()>)error {
|
||||
if (!self->_hadFatalError) {
|
||||
_onFatalError = std::move(error);
|
||||
} else if (error) {
|
||||
error();
|
||||
}
|
||||
}
|
||||
|
||||
-(void)setOnPause:(std::function<void (bool)>)pause {
|
||||
|
||||
}
|
||||
|
||||
- (void)stop {
|
||||
_isActiveUpdated = nil;
|
||||
[self stopCaptureWithCompletionHandler:nil];
|
||||
}
|
||||
|
||||
|
||||
- (void)setIsEnabled:(bool)isEnabled {
|
||||
BOOL updated = _isPaused != !isEnabled;
|
||||
_isPaused = !isEnabled;
|
||||
_skippedFrame = 0;
|
||||
if (updated) {
|
||||
if (_isPaused) {
|
||||
// [RTCDispatcher
|
||||
// dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
|
||||
// block:^{
|
||||
[self->_captureSession stopRunning];
|
||||
self->_isRunning = NO;
|
||||
// }];
|
||||
} else {
|
||||
// [RTCDispatcher
|
||||
// dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
|
||||
// block:^{
|
||||
[self->_captureSession startRunning];
|
||||
self->_isRunning = YES;
|
||||
// }];
|
||||
}
|
||||
}
|
||||
|
||||
[self updateIsActiveValue];
|
||||
}
|
||||
|
||||
|
||||
- (void)setUncroppedSink:(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)sink {
|
||||
dispatch_async(self.frameQueue, ^{
|
||||
self->_uncroppedSink = sink;
|
||||
});
|
||||
}
|
||||
|
||||
- (void)setPreferredCaptureAspectRatio:(float)aspectRatio {
|
||||
dispatch_async(self.frameQueue, ^{
|
||||
self->_aspectRatio = aspectRatio;
|
||||
});
|
||||
}
|
||||
|
||||
- (void)updateIsActiveValue {
|
||||
bool isActive = _inForegroundValue && !_isPaused;
|
||||
if (isActive != _isActiveValue) {
|
||||
_isActiveValue = isActive;
|
||||
if (_isActiveUpdated) {
|
||||
_isActiveUpdated(_isActiveValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
- (void)setupCaptureWithDevice:(AVCaptureDevice *)device
|
||||
format:(AVCaptureDeviceFormat *)format
|
||||
fps:(NSInteger)fps
|
||||
completionHandler:(nullable void (^)(NSError *))completionHandler {
|
||||
|
||||
|
||||
CMIOObjectPropertyAddress latency_pa = {
|
||||
kCMIODevicePropertyLatency,
|
||||
kCMIOObjectPropertyScopeWildcard,
|
||||
kCMIOObjectPropertyElementWildcard
|
||||
};
|
||||
UInt32 dataSize = 0;
|
||||
|
||||
NSNumber *_connectionID = ((NSNumber *)[device valueForKey:@"_connectionID"]);
|
||||
|
||||
CMIODeviceID deviceId = (CMIODeviceID)[_connectionID intValue];
|
||||
|
||||
if (device) {
|
||||
if (CMIOObjectGetPropertyDataSize(deviceId, &latency_pa, 0, nil, &dataSize) == noErr) {
|
||||
_shouldBeMirrored = NO;
|
||||
} else {
|
||||
_shouldBeMirrored = YES;
|
||||
}
|
||||
} else {
|
||||
_shouldBeMirrored = [device hasMediaType:AVMediaTypeVideo];
|
||||
}
|
||||
// [RTCDispatcher
|
||||
// dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
|
||||
// block:^{
|
||||
RTCLogInfo("startCaptureWithDevice %@ @ %ld fps", format, (long)fps);
|
||||
NSError *error = nil;
|
||||
|
||||
self->_currentDevice = device;
|
||||
|
||||
|
||||
self->_currentInput = [[AVCaptureDeviceInput alloc] initWithDevice:device error:&error];
|
||||
if (![self->_currentDevice lockForConfiguration:&error]) {
|
||||
RTCLogError(@"Failed to lock device %@. Error: %@",
|
||||
self->_currentDevice,
|
||||
error.userInfo);
|
||||
if (completionHandler) {
|
||||
completionHandler(error);
|
||||
}
|
||||
self->_willBeRunning = false;
|
||||
return;
|
||||
}
|
||||
[self updateDeviceCaptureFormat:format fps:fps];
|
||||
[self updateVideoDataOutputPixelFormat:format];
|
||||
[self->_currentDevice unlockForConfiguration];
|
||||
[self reconfigureCaptureSessionInput];
|
||||
if (completionHandler) {
|
||||
completionHandler(nil);
|
||||
}
|
||||
// }];
|
||||
}
|
||||
|
||||
-(void)start {
|
||||
_willBeRunning = true;
|
||||
// [RTCDispatcher
|
||||
// dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
|
||||
// block:^{
|
||||
[self->_captureSession startRunning];
|
||||
self->_isRunning = YES;
|
||||
// }];
|
||||
}
|
||||
|
||||
|
||||
|
||||
- (void)stopCaptureWithCompletionHandler:(nullable void (^)(void))completionHandler {
|
||||
_willBeRunning = false;
|
||||
// [RTCDispatcher
|
||||
// dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
|
||||
// block:^{
|
||||
RTCLogInfo("Stop");
|
||||
self->_currentDevice = nil;
|
||||
for (AVCaptureDeviceInput *oldInput in [self->_captureSession.inputs copy]) {
|
||||
[self->_captureSession removeInput:oldInput];
|
||||
}
|
||||
[self->_captureSession stopRunning];
|
||||
|
||||
self->_isRunning = NO;
|
||||
if (completionHandler) {
|
||||
completionHandler();
|
||||
}
|
||||
// }];
|
||||
}
|
||||
|
||||
|
||||
#pragma mark AVCaptureVideoDataOutputSampleBufferDelegate
|
||||
|
||||
- (void)captureOutput:(AVCaptureOutput *)captureOutput
|
||||
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
|
||||
fromConnection:(AVCaptureConnection *)connection {
|
||||
NSParameterAssert(captureOutput == _videoDataOutput);
|
||||
|
||||
int minWarmupFrameCount = 12;
|
||||
_warmupFrameCount++;
|
||||
if (_warmupFrameCount < minWarmupFrameCount) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (CMSampleBufferGetNumSamples(sampleBuffer) != 1 || !CMSampleBufferIsValid(sampleBuffer) ||
|
||||
!CMSampleBufferDataIsReady(sampleBuffer)) {
|
||||
return;
|
||||
}
|
||||
|
||||
CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
|
||||
if (pixelBuffer == nil) {
|
||||
return;
|
||||
}
|
||||
|
||||
int width = (int)CVPixelBufferGetWidth(pixelBuffer);
|
||||
int height = (int)CVPixelBufferGetHeight(pixelBuffer);
|
||||
|
||||
CameraFrameSize fittedSize = { width, height };
|
||||
|
||||
fittedSize.width -= (fittedSize.width % 32);
|
||||
fittedSize.height -= (fittedSize.height % 4);
|
||||
|
||||
TGRTCCVPixelBuffer *rtcPixelBuffer = [[TGRTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer adaptedWidth:fittedSize.width adaptedHeight:fittedSize.height cropWidth:width cropHeight:height cropX:0 cropY:0];
|
||||
|
||||
rtcPixelBuffer.shouldBeMirrored = _shouldBeMirrored;
|
||||
|
||||
if (!_isPaused && _uncroppedSink) {
|
||||
int64_t timeStampNs = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) *
|
||||
kNanosecondsPerSecond;
|
||||
RTCVideoFrame *frame = [[RTCVideoFrame alloc] initWithBuffer:rtcPixelBuffer
|
||||
rotation:_rotation
|
||||
timeStampNs:timeStampNs];
|
||||
|
||||
const int64_t timestamp_us = frame.timeStampNs / rtc::kNumNanosecsPerMicrosec;
|
||||
|
||||
rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer;
|
||||
buffer = new rtc::RefCountedObject<webrtc::ObjCFrameBuffer>(frame.buffer);
|
||||
|
||||
webrtc::VideoRotation rotation = static_cast<webrtc::VideoRotation>(frame.rotation);
|
||||
|
||||
_uncroppedSink->OnFrame(webrtc::VideoFrame::Builder()
|
||||
.set_video_frame_buffer(buffer)
|
||||
.set_rotation(rotation)
|
||||
.set_timestamp_us(timestamp_us)
|
||||
.build());
|
||||
}
|
||||
|
||||
|
||||
int64_t timeStampNs = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) *
|
||||
kNanosecondsPerSecond;
|
||||
RTCVideoFrame *videoFrame = [[RTCVideoFrame alloc] initWithBuffer:rtcPixelBuffer
|
||||
rotation:_rotation
|
||||
timeStampNs:timeStampNs];
|
||||
if (!_isPaused) {
|
||||
getObjCVideoSource(_source)->OnCapturedFrame(videoFrame);
|
||||
}
|
||||
_skippedFrame = MIN(_skippedFrame + 1, 16);
|
||||
}
|
||||
|
||||
- (void)captureOutput:(AVCaptureOutput *)captureOutput
|
||||
didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer
|
||||
fromConnection:(AVCaptureConnection *)connection {
|
||||
NSString *droppedReason =
|
||||
(__bridge NSString *)CMGetAttachment(sampleBuffer, kCMSampleBufferAttachmentKey_DroppedFrameReason, nil);
|
||||
RTCLogError(@"Dropped sample buffer. Reason: %@", droppedReason);
|
||||
}
|
||||
|
||||
#pragma mark - AVCaptureSession notifications
|
||||
|
||||
- (void)handleCaptureSessionInterruption:(NSNotification *)notification {
|
||||
|
||||
}
|
||||
|
||||
- (void)handleCaptureSessionInterruptionEnded:(NSNotification *)notification {
|
||||
RTCLog(@"Capture session interruption ended.");
|
||||
}
|
||||
|
||||
- (void)handleCaptureSessionRuntimeError:(NSNotification *)notification {
|
||||
NSError *error = [notification.userInfo objectForKey:AVCaptureSessionErrorKey];
|
||||
RTCLogError(@"Capture session runtime error: %@", error);
|
||||
|
||||
// [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
|
||||
// block:^{
|
||||
[self handleFatalError];
|
||||
// }];
|
||||
}
|
||||
|
||||
- (void)handleCaptureSessionDidStartRunning:(NSNotification *)notification {
|
||||
RTCLog(@"Capture session started.");
|
||||
|
||||
// [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
|
||||
// block:^{
|
||||
// If we successfully restarted after an unknown error,
|
||||
// allow future retries on fatal errors.
|
||||
self->_hasRetriedOnFatalError = NO;
|
||||
self->_hadFatalError = NO;
|
||||
// }];
|
||||
|
||||
|
||||
_inForegroundValue = true;
|
||||
[self updateIsActiveValue];
|
||||
}
|
||||
|
||||
- (void)handleCaptureSessionDidStopRunning:(NSNotification *)notification {
|
||||
RTCLog(@"Capture session stopped.");
|
||||
_inForegroundValue = false;
|
||||
[self updateIsActiveValue];
|
||||
|
||||
}
|
||||
|
||||
- (void)handleFatalError {
|
||||
// [RTCDispatcher
|
||||
// dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
|
||||
// block:^{
|
||||
if (!self->_hasRetriedOnFatalError) {
|
||||
RTCLogWarning(@"Attempting to recover from fatal capture error.");
|
||||
[self handleNonFatalError];
|
||||
self->_warmupFrameCount = 0;
|
||||
self->_hasRetriedOnFatalError = YES;
|
||||
} else {
|
||||
RTCLogError(@"Previous fatal error recovery failed.");
|
||||
if (_onFatalError) {
|
||||
_onFatalError();
|
||||
} else {
|
||||
self->_hadFatalError = YES;
|
||||
}
|
||||
}
|
||||
// }];
|
||||
}
|
||||
|
||||
- (void)handleNonFatalError {
|
||||
// [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
|
||||
// block:^{
|
||||
RTCLog(@"Restarting capture session after error.");
|
||||
if (self->_isRunning) {
|
||||
self->_warmupFrameCount = 0;
|
||||
[self->_captureSession startRunning];
|
||||
}
|
||||
// }];
|
||||
}
|
||||
|
||||
#pragma mark - UIApplication notifications
|
||||
|
||||
- (void)handleApplicationDidBecomeActive:(NSNotification *)notification {
|
||||
// [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
|
||||
// block:^{
|
||||
if (self->_isRunning && !self->_captureSession.isRunning) {
|
||||
RTCLog(@"Restarting capture session on active.");
|
||||
self->_warmupFrameCount = 0;
|
||||
[self->_captureSession startRunning];
|
||||
}
|
||||
// }];
|
||||
}
|
||||
|
||||
#pragma mark - Private
|
||||
|
||||
- (dispatch_queue_t)frameQueue {
|
||||
if (!_frameQueue) {
|
||||
_frameQueue =
|
||||
dispatch_queue_create("org.webrtc.cameravideocapturer.video", DISPATCH_QUEUE_SERIAL);
|
||||
dispatch_set_target_queue(_frameQueue,
|
||||
dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0));
|
||||
}
|
||||
return _frameQueue;
|
||||
}
|
||||
|
||||
- (BOOL)setupCaptureSession:(AVCaptureSession *)captureSession {
|
||||
NSAssert(_captureSession == nil, @"Setup capture session called twice.");
|
||||
_captureSession = captureSession;
|
||||
|
||||
[self setupVideoDataOutput];
|
||||
// Add the output.
|
||||
if (![_captureSession canAddOutput:_videoDataOutput]) {
|
||||
RTCLogError(@"Video data output unsupported.");
|
||||
return NO;
|
||||
}
|
||||
[_captureSession addOutput:_videoDataOutput];
|
||||
|
||||
|
||||
return YES;
|
||||
}
|
||||
|
||||
|
||||
|
||||
- (void)setupVideoDataOutput {
|
||||
NSAssert(_videoDataOutput == nil, @"Setup video data output called twice.");
|
||||
AVCaptureVideoDataOutput *videoDataOutput = [[AVCaptureVideoDataOutput alloc] init];
|
||||
|
||||
// `videoDataOutput.availableVideoCVPixelFormatTypes` returns the pixel formats supported by the
|
||||
// device with the most efficient output format first. Find the first format that we support.
|
||||
NSSet<NSNumber *> *supportedPixelFormats = [RTCCVPixelBuffer supportedPixelFormats];
|
||||
NSMutableOrderedSet *availablePixelFormats =
|
||||
[NSMutableOrderedSet orderedSetWithArray:videoDataOutput.availableVideoCVPixelFormatTypes];
|
||||
[availablePixelFormats intersectSet:supportedPixelFormats];
|
||||
NSNumber *pixelFormat = availablePixelFormats.firstObject;
|
||||
NSAssert(pixelFormat, @"Output device has no supported formats.");
|
||||
|
||||
_preferredOutputPixelFormat = [pixelFormat unsignedIntValue];
|
||||
_outputPixelFormat = _preferredOutputPixelFormat;
|
||||
videoDataOutput.videoSettings = @{(NSString *)kCVPixelBufferPixelFormatTypeKey : pixelFormat};
|
||||
videoDataOutput.alwaysDiscardsLateVideoFrames = YES;
|
||||
|
||||
[videoDataOutput setSampleBufferDelegate:self queue:self.frameQueue];
|
||||
_videoDataOutput = videoDataOutput;
|
||||
}
|
||||
|
||||
- (void)updateVideoDataOutputPixelFormat:(AVCaptureDeviceFormat *)format {
|
||||
FourCharCode mediaSubType = CMFormatDescriptionGetMediaSubType(format.formatDescription);
|
||||
if (![[RTCCVPixelBuffer supportedPixelFormats] containsObject:@(mediaSubType)]) {
|
||||
mediaSubType = _preferredOutputPixelFormat;
|
||||
}
|
||||
CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(format.formatDescription);
|
||||
|
||||
if (mediaSubType != _outputPixelFormat) {
|
||||
_outputPixelFormat = mediaSubType;
|
||||
_videoDataOutput.videoSettings =
|
||||
@{ (NSString *)kCVPixelBufferPixelFormatTypeKey : @(mediaSubType) };
|
||||
} else {
|
||||
// _videoDataOutput.videoSettings =
|
||||
// @{ (NSString *)kCVPixelBufferWidthKey: @(dimensions.width), (NSString *)kCVPixelBufferHeightKey: @(dimensions.height) };
|
||||
}
|
||||
AVCaptureConnection *connection = [_videoDataOutput connectionWithMediaType:AVMediaTypeVideo];
|
||||
|
||||
}
|
||||
|
||||
#pragma mark - Private, called inside capture queue
|
||||
|
||||
- (void)updateDeviceCaptureFormat:(AVCaptureDeviceFormat *)format fps:(NSInteger)fps {
|
||||
// NSAssert([RTCDispatcher isOnQueueForType:RTCDispatcherTypeCaptureSession],
|
||||
// @"updateDeviceCaptureFormat must be called on the capture queue.");
|
||||
@try {
|
||||
_currentDevice.activeFormat = format;
|
||||
if (format.videoSupportedFrameRateRanges.count > 0) {
|
||||
int target = 24;
|
||||
int closest = -1;
|
||||
CMTime result;
|
||||
for (int i = 0; i < format.videoSupportedFrameRateRanges.count; i++) {
|
||||
const auto rateRange = format.videoSupportedFrameRateRanges[i];
|
||||
int gap = abs(rateRange.minFrameRate - target);
|
||||
if (gap <= closest || closest == -1) {
|
||||
closest = gap;
|
||||
result = rateRange.maxFrameDuration;
|
||||
}
|
||||
}
|
||||
if (closest >= 0) {
|
||||
_currentDevice.activeVideoMaxFrameDuration = result;
|
||||
}
|
||||
}
|
||||
} @catch (NSException *exception) {
|
||||
RTCLogError(@"Failed to set active format!\n User info:%@", exception.userInfo);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)reconfigureCaptureSessionInput {
|
||||
// NSAssert([RTCDispatcher isOnQueueForType:RTCDispatcherTypeCaptureSession],
|
||||
// @"reconfigureCaptureSessionInput must be called on the capture queue.");
|
||||
NSError *error = nil;
|
||||
AVCaptureInput *input = _currentInput;
|
||||
if (!input) {
|
||||
RTCLogError(@"Failed to create front camera input: %@", error.localizedDescription);
|
||||
return;
|
||||
}
|
||||
[_captureSession beginConfiguration];
|
||||
for (AVCaptureInput *oldInput in [_captureSession.inputs copy]) {
|
||||
[_captureSession removeInput:oldInput];
|
||||
}
|
||||
if ([_captureSession canAddInput:input]) {
|
||||
[_captureSession addInput:input];
|
||||
} else {
|
||||
RTCLogError(@"Cannot add camera as an input to the session.");
|
||||
}
|
||||
[_captureSession commitConfiguration];
|
||||
}
|
||||
|
||||
|
||||
@end
|
||||
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
#ifndef TGCALLS_VIDEO_CAPTURE_VIEW_H
|
||||
#define TGCALLS_VIDEO_CAPTURE_VIEW_H
|
||||
|
||||
#ifdef WEBRTC_IOS
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <UIKit/UIKit.h>
|
||||
|
||||
#include <memory>
|
||||
|
||||
@class AVCaptureVideoPreviewLayer;
|
||||
|
||||
@interface VideoCaptureView : UIView
|
||||
|
||||
@property(nonatomic) UIViewContentMode videoContentMode;
|
||||
@property(nonatomic, getter=isEnabled) BOOL enabled;
|
||||
@property(nonatomic, nullable) NSValue* rotationOverride;
|
||||
|
||||
@property (nonatomic, readwrite) int internalOrientation;
|
||||
@property (nonatomic, readwrite) CGFloat internalAspect;
|
||||
|
||||
- (void)setOnFirstFrameReceived:(void (^ _Nullable)())onFirstFrameReceived;
|
||||
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated;
|
||||
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated;
|
||||
|
||||
- (void)onFrameGenerated:(CGSize)size isMirrored:(bool)isMirrored rotation:(int)rotation;
|
||||
|
||||
- (AVCaptureVideoPreviewLayer * _Nonnull)previewLayer;
|
||||
|
||||
@end
|
||||
|
||||
#endif //WEBRTC_IOS
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,264 @@
|
|||
#import "VideoCaptureView.h"
|
||||
|
||||
#import <CoreVideo/CoreVideo.h>
|
||||
|
||||
#import "base/RTCLogging.h"
|
||||
#import "base/RTCVideoFrame.h"
|
||||
#import "base/RTCVideoFrameBuffer.h"
|
||||
#import "TGRTCCVPixelBuffer.h"
|
||||
#include "sdk/objc/native/api/video_frame.h"
|
||||
#include "sdk/objc/native/src/objc_frame_buffer.h"
|
||||
#include "sdk/objc/base/RTCI420Buffer.h"
|
||||
|
||||
#import "api/video/video_sink_interface.h"
|
||||
#import "api/media_stream_interface.h"
|
||||
#import "rtc_base/time_utils.h"
|
||||
|
||||
@interface VideoCaptureContentView : UIView
|
||||
|
||||
@end
|
||||
|
||||
@implementation VideoCaptureContentView
|
||||
|
||||
+ (Class)layerClass {
|
||||
return [AVCaptureVideoPreviewLayer class];
|
||||
}
|
||||
|
||||
- (AVCaptureVideoPreviewLayer * _Nonnull)videoLayer {
|
||||
return (AVCaptureVideoPreviewLayer *)self.layer;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@interface VideoCaptureFrame : NSObject
|
||||
|
||||
@property (nonatomic, readonly) CGSize size;
|
||||
@property (nonatomic, readonly) bool isMirrored;
|
||||
@property (nonatomic, readonly) int rotation;
|
||||
|
||||
@end
|
||||
|
||||
@implementation VideoCaptureFrame
|
||||
|
||||
- (instancetype)initWithSize:(CGSize)size isMirrored:(bool)isMirrored rotation:(int)rotation {
|
||||
self = [super init];
|
||||
if (self != nil) {
|
||||
_size = size;
|
||||
_isMirrored = isMirrored;
|
||||
_rotation = rotation;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@interface VideoCaptureView () {
|
||||
VideoCaptureContentView *_captureView;
|
||||
|
||||
VideoCaptureFrame *_videoFrame;
|
||||
VideoCaptureFrame *_stashedVideoFrame;
|
||||
|
||||
int _isWaitingForLayoutFrameCount;
|
||||
bool _didStartWaitingForLayout;
|
||||
CGSize _videoFrameSize;
|
||||
int64_t _lastFrameTimeNs;
|
||||
|
||||
CGSize _currentSize;
|
||||
|
||||
void (^_onFirstFrameReceived)();
|
||||
bool _firstFrameReceivedReported;
|
||||
|
||||
void (^_onOrientationUpdated)(int, CGFloat);
|
||||
|
||||
void (^_onIsMirroredUpdated)(bool);
|
||||
|
||||
bool _didSetShouldBeMirrored;
|
||||
bool _shouldBeMirrored;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@implementation VideoCaptureView
|
||||
|
||||
- (instancetype)initWithFrame:(CGRect)frameRect {
|
||||
self = [super initWithFrame:frameRect];
|
||||
if (self) {
|
||||
[self configure];
|
||||
|
||||
_enabled = true;
|
||||
|
||||
_currentSize = CGSizeZero;
|
||||
_rotationOverride = @(RTCVideoRotation_0);
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
}
|
||||
|
||||
- (AVCaptureVideoPreviewLayer * _Nonnull)previewLayer {
|
||||
return _captureView.videoLayer;
|
||||
}
|
||||
|
||||
- (void)setEnabled:(BOOL)enabled {
|
||||
if (_enabled != enabled) {
|
||||
_enabled = enabled;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)setVideoContentMode:(UIViewContentMode)mode {
|
||||
_videoContentMode = mode;
|
||||
}
|
||||
|
||||
- (void)configure {
|
||||
_captureView = [[VideoCaptureContentView alloc] init];
|
||||
[_captureView videoLayer].videoGravity = AVLayerVideoGravityResizeAspectFill;
|
||||
[self addSubview:_captureView];
|
||||
|
||||
_videoFrameSize = CGSizeZero;
|
||||
}
|
||||
|
||||
- (void)layoutSubviews {
|
||||
[super layoutSubviews];
|
||||
|
||||
CGRect bounds = self.bounds;
|
||||
|
||||
_captureView.frame = bounds;
|
||||
|
||||
if (_didStartWaitingForLayout) {
|
||||
_didStartWaitingForLayout = false;
|
||||
_isWaitingForLayoutFrameCount = 0;
|
||||
if (_stashedVideoFrame != nil) {
|
||||
_videoFrame = _stashedVideoFrame;
|
||||
_stashedVideoFrame = nil;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#pragma mark -
|
||||
|
||||
- (void)setRotationOverride:(NSValue *)rotationOverride {
|
||||
_rotationOverride = rotationOverride;
|
||||
|
||||
[self setNeedsLayout];
|
||||
}
|
||||
|
||||
- (RTCVideoRotation)frameRotation {
|
||||
if (_rotationOverride) {
|
||||
RTCVideoRotation rotation;
|
||||
if (@available(iOS 11, *)) {
|
||||
[_rotationOverride getValue:&rotation size:sizeof(rotation)];
|
||||
} else {
|
||||
[_rotationOverride getValue:&rotation];
|
||||
}
|
||||
return rotation;
|
||||
}
|
||||
|
||||
if (_videoFrame) {
|
||||
switch (_videoFrame.rotation) {
|
||||
case webrtc::kVideoRotation_0:
|
||||
return RTCVideoRotation_0;
|
||||
case webrtc::kVideoRotation_90:
|
||||
return RTCVideoRotation_90;
|
||||
case webrtc::kVideoRotation_180:
|
||||
return RTCVideoRotation_180;
|
||||
case webrtc::kVideoRotation_270:
|
||||
return RTCVideoRotation_270;
|
||||
default:
|
||||
return RTCVideoRotation_0;
|
||||
}
|
||||
} else {
|
||||
return RTCVideoRotation_0;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)onFrameGenerated:(CGSize)size isMirrored:(bool)isMirrored rotation:(int)rotation {
|
||||
assert([NSThread isMainThread]);
|
||||
|
||||
if (!CGSizeEqualToSize(size, _currentSize)) {
|
||||
_currentSize = size;
|
||||
}
|
||||
|
||||
int mappedValue = 0;
|
||||
switch (RTCVideoRotation(rotation)) {
|
||||
case RTCVideoRotation_90: {
|
||||
mappedValue = 1;
|
||||
break;
|
||||
}
|
||||
case RTCVideoRotation_180: {
|
||||
mappedValue = 2;
|
||||
break;
|
||||
}
|
||||
case RTCVideoRotation_270: {
|
||||
mappedValue = 3;
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
mappedValue = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
[self setInternalOrientationAndSize:mappedValue size:size];
|
||||
|
||||
if (!_firstFrameReceivedReported && _onFirstFrameReceived) {
|
||||
_firstFrameReceivedReported = true;
|
||||
_onFirstFrameReceived();
|
||||
}
|
||||
|
||||
if (!self.isEnabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
VideoCaptureFrame *frame = [[VideoCaptureFrame alloc] initWithSize:size isMirrored:isMirrored rotation:rotation];
|
||||
|
||||
if (_isWaitingForLayoutFrameCount > 0) {
|
||||
_stashedVideoFrame = frame;
|
||||
_isWaitingForLayoutFrameCount--;
|
||||
return;
|
||||
}
|
||||
if (!_didStartWaitingForLayout) {
|
||||
if (_videoFrame && _videoFrame.size.width > 0 && _videoFrame.size.height > 0 && frame.size.width > 0 && frame.size.height > 0) {
|
||||
float previousAspect = ((float)_videoFrame.size.width) / ((float)_videoFrame.size.height);
|
||||
float updatedAspect = ((float)frame.size.width) / ((float)frame.size.height);
|
||||
if ((previousAspect < 1.0f) != (updatedAspect < 1.0f)) {
|
||||
_stashedVideoFrame = frame;
|
||||
_didStartWaitingForLayout = true;
|
||||
_isWaitingForLayoutFrameCount = 5;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_videoFrame = frame;
|
||||
}
|
||||
|
||||
- (void)setOnFirstFrameReceived:(void (^ _Nullable)())onFirstFrameReceived {
|
||||
_onFirstFrameReceived = [onFirstFrameReceived copy];
|
||||
_firstFrameReceivedReported = false;
|
||||
}
|
||||
|
||||
- (void)setInternalOrientationAndSize:(int)internalOrientation size:(CGSize)size {
|
||||
CGFloat aspect = 1.0f;
|
||||
if (size.width > 1.0f && size.height > 1.0f) {
|
||||
aspect = size.width / size.height;
|
||||
}
|
||||
if (_internalOrientation != internalOrientation || ABS(_internalAspect - aspect) > 0.001) {
|
||||
RTCLogInfo(@"VideoCaptureView@%lx orientation: %d, aspect: %f", (intptr_t)self, internalOrientation, (float)aspect);
|
||||
|
||||
_internalOrientation = internalOrientation;
|
||||
_internalAspect = aspect;
|
||||
if (_onOrientationUpdated) {
|
||||
_onOrientationUpdated(internalOrientation, aspect);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated {
|
||||
_onOrientationUpdated = [onOrientationUpdated copy];
|
||||
}
|
||||
|
||||
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated {
|
||||
_onIsMirroredUpdated = [onIsMirroredUpdated copy];
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
#ifndef TGCALLS_VIDEO_CAPTURER_INTERFACE_IMPL_H
|
||||
#define TGCALLS_VIDEO_CAPTURER_INTERFACE_IMPL_H
|
||||
|
||||
#include "VideoCapturerInterface.h"
|
||||
|
||||
#include "sdk/objc/native/src/objc_video_track_source.h"
|
||||
//#include "api/video_track_source_proxy.h"
|
||||
|
||||
@interface VideoCapturerInterfaceImplHolder : NSObject
|
||||
|
||||
@property (nonatomic) void *reference;
|
||||
|
||||
@end
|
||||
|
||||
namespace tgcalls {
|
||||
|
||||
struct PlatformCaptureInfo;
|
||||
|
||||
class VideoCapturerInterfaceImpl : public VideoCapturerInterface {
|
||||
public:
|
||||
VideoCapturerInterfaceImpl(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> source, std::string deviceId, std::function<void(VideoState)> stateUpdated, std::function<void(PlatformCaptureInfo)> captureInfoUpdated, std::pair<int, int> &outResolution);
|
||||
~VideoCapturerInterfaceImpl() override;
|
||||
|
||||
void setState(VideoState state) override;
|
||||
void setPreferredCaptureAspectRatio(float aspectRatio) override;
|
||||
void withNativeImplementation(std::function<void(void *)> completion) override;
|
||||
void setUncroppedOutput(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink) override;
|
||||
void setOnFatalError(std::function<void()> error) override;
|
||||
void setOnPause(std::function<void(bool)> pause) override;
|
||||
int getRotation() override;
|
||||
|
||||
id getInternalReference();
|
||||
|
||||
private:
|
||||
webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> _source;
|
||||
VideoCapturerInterfaceImplHolder *_implReference;
|
||||
};
|
||||
|
||||
} // namespace tgcalls
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,480 @@
|
|||
#include "VideoCapturerInterfaceImpl.h"
|
||||
|
||||
#include "absl/strings/match.h"
|
||||
#include "api/audio_codecs/audio_decoder_factory_template.h"
|
||||
#include "api/audio_codecs/audio_encoder_factory_template.h"
|
||||
#include "api/audio_codecs/opus/audio_decoder_opus.h"
|
||||
#include "api/audio_codecs/opus/audio_encoder_opus.h"
|
||||
#include "api/rtp_parameters.h"
|
||||
#include "api/task_queue/default_task_queue_factory.h"
|
||||
#include "media/base/codec.h"
|
||||
#include "media/base/media_constants.h"
|
||||
#include "media/engine/webrtc_media_engine.h"
|
||||
#include "modules/audio_device/include/audio_device_default.h"
|
||||
#include "rtc_base/task_utils/repeating_task.h"
|
||||
#include "system_wrappers/include/field_trial.h"
|
||||
#include "api/video/builtin_video_bitrate_allocator_factory.h"
|
||||
#include "api/video/video_bitrate_allocation.h"
|
||||
|
||||
#include "sdk/objc/native/api/video_encoder_factory.h"
|
||||
#include "sdk/objc/native/api/video_decoder_factory.h"
|
||||
|
||||
#include "sdk/objc/api/RTCVideoRendererAdapter.h"
|
||||
#include "sdk/objc/native/api/video_frame.h"
|
||||
#include "api/media_types.h"
|
||||
|
||||
#ifndef WEBRTC_IOS
|
||||
#import "VideoCameraCapturerMac.h"
|
||||
#import "tgcalls/platform/darwin/DesktopSharingCapturer.h"
|
||||
#import "tgcalls/desktop_capturer/DesktopCaptureSourceHelper.h"
|
||||
#import "CustomExternalCapturer.h"
|
||||
#import "VideoCMIOCapture.h"
|
||||
#else
|
||||
#import "VideoCameraCapturer.h"
|
||||
#import "CustomExternalCapturer.h"
|
||||
#endif
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
|
||||
#import "VideoCaptureInterface.h"
|
||||
#import "platform/PlatformInterface.h"
|
||||
|
||||
@interface VideoCapturerInterfaceImplSourceDescription : NSObject
|
||||
|
||||
@property (nonatomic, readonly) bool isFrontCamera;
|
||||
@property (nonatomic, readonly) bool keepLandscape;
|
||||
@property (nonatomic, readonly) NSString *deviceId;
|
||||
@property (nonatomic, strong, readonly, nullable) AVCaptureDevice *device;
|
||||
@property (nonatomic, strong, readonly, nullable) AVCaptureDeviceFormat *format;
|
||||
|
||||
@end
|
||||
|
||||
@implementation VideoCapturerInterfaceImplSourceDescription
|
||||
|
||||
- (instancetype)initWithIsFrontCamera:(bool)isFrontCamera keepLandscape:(bool)keepLandscape deviceId:(NSString *)deviceId device:(AVCaptureDevice * _Nullable)device format:(AVCaptureDeviceFormat * _Nullable)format {
|
||||
self = [super init];
|
||||
if (self != nil) {
|
||||
_isFrontCamera = isFrontCamera;
|
||||
_keepLandscape = keepLandscape;
|
||||
_deviceId = deviceId;
|
||||
_device = device;
|
||||
_format = format;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@interface VideoCapturerInterfaceImplReference : NSObject {
|
||||
#ifdef WEBRTC_IOS
|
||||
CustomExternalCapturer *_customExternalCapturer;
|
||||
VideoCameraCapturer *_videoCameraCapturer;
|
||||
#else
|
||||
id<CapturerInterface> _videoCapturer;
|
||||
#endif
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@implementation VideoCapturerInterfaceImplReference
|
||||
|
||||
- (id)videoCameraCapturer {
|
||||
#ifdef WEBRTC_IOS
|
||||
return _videoCameraCapturer;
|
||||
#else
|
||||
return _videoCapturer;
|
||||
#endif
|
||||
}
|
||||
|
||||
+ (AVCaptureDevice *)selectCapturerDeviceWithDeviceId:(NSString *)deviceId {
|
||||
AVCaptureDevice *selectedCamera = nil;
|
||||
|
||||
#ifdef WEBRTC_IOS
|
||||
bool useFrontCamera = ![deviceId hasPrefix:@"back"];
|
||||
AVCaptureDevice *frontCamera = nil;
|
||||
AVCaptureDevice *backCamera = nil;
|
||||
for (AVCaptureDevice *device in [VideoCameraCapturer captureDevices]) {
|
||||
if (device.position == AVCaptureDevicePositionFront) {
|
||||
frontCamera = device;
|
||||
} else if (device.position == AVCaptureDevicePositionBack) {
|
||||
backCamera = device;
|
||||
}
|
||||
}
|
||||
if (useFrontCamera && frontCamera != nil) {
|
||||
selectedCamera = frontCamera;
|
||||
} else {
|
||||
selectedCamera = backCamera;
|
||||
}
|
||||
#else
|
||||
|
||||
NSArray *deviceComponents = [deviceId componentsSeparatedByString:@":"];
|
||||
if (deviceComponents.count == 2) {
|
||||
deviceId = deviceComponents[0];
|
||||
}
|
||||
//&& [devices[i] hasMediaType:AVMediaTypeVideo]
|
||||
NSArray<AVCaptureDevice *> *devices = [VideoCameraCapturer captureDevices];
|
||||
for (int i = 0; i < devices.count; i++) {
|
||||
if (devices[i].isConnected && !devices[i].isSuspended ) {
|
||||
if ([deviceId isEqualToString:@""] || [deviceId isEqualToString:devices[i].uniqueID]) {
|
||||
selectedCamera = devices[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (selectedCamera == nil) {
|
||||
for (int i = 0; i < devices.count; i++) {
|
||||
if (devices[i].isConnected && !devices[i].isSuspended) {
|
||||
selectedCamera = devices[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
return selectedCamera;
|
||||
}
|
||||
|
||||
+ (AVCaptureDeviceFormat *)selectCaptureDeviceFormatForDevice:(AVCaptureDevice *)selectedCamera {
|
||||
NSMutableArray<AVCaptureDeviceFormat *> *sortedFormats = [NSMutableArray arrayWithArray:[[VideoCameraCapturer supportedFormatsForDevice:selectedCamera] sortedArrayUsingComparator:^NSComparisonResult(AVCaptureDeviceFormat* lhs, AVCaptureDeviceFormat *rhs) {
|
||||
int32_t width1 = CMVideoFormatDescriptionGetDimensions(lhs.formatDescription).width;
|
||||
int32_t width2 = CMVideoFormatDescriptionGetDimensions(rhs.formatDescription).width;
|
||||
return width1 < width2 ? NSOrderedAscending : NSOrderedDescending;
|
||||
}]];
|
||||
for (int i = (int)[sortedFormats count] - 1; i >= 0; i--) {
|
||||
if ([[sortedFormats[i] description] containsString:@"x420"]) {
|
||||
[sortedFormats removeObjectAtIndex:i];
|
||||
}
|
||||
}
|
||||
|
||||
AVCaptureDeviceFormat *bestFormat = sortedFormats.firstObject;
|
||||
|
||||
bool didSelectPreferredFormat = false;
|
||||
#ifdef WEBRTC_IOS
|
||||
for (AVCaptureDeviceFormat *format in sortedFormats) {
|
||||
CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(format.formatDescription);
|
||||
if (dimensions.width == 1280 && dimensions.height == 720) {
|
||||
if (format.videoFieldOfView > 60.0f && format.videoSupportedFrameRateRanges.lastObject.maxFrameRate == 30) {
|
||||
didSelectPreferredFormat = true;
|
||||
bestFormat = format;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
if (!didSelectPreferredFormat) {
|
||||
for (AVCaptureDeviceFormat *format in sortedFormats) {
|
||||
CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(format.formatDescription);
|
||||
if (dimensions.width >= 720 && dimensions.height >= 720) {
|
||||
didSelectPreferredFormat = true;
|
||||
bestFormat = format;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!didSelectPreferredFormat) {
|
||||
for (AVCaptureDeviceFormat *format in sortedFormats) {
|
||||
CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(format.formatDescription);
|
||||
if (dimensions.width >= 640 && dimensions.height >= 640) {
|
||||
didSelectPreferredFormat = true;
|
||||
bestFormat = format;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (bestFormat == nil) {
|
||||
assert(false);
|
||||
return nil;
|
||||
}
|
||||
|
||||
|
||||
return bestFormat;
|
||||
}
|
||||
|
||||
+ (VideoCapturerInterfaceImplSourceDescription *)selectCapturerDescriptionWithDeviceId:(NSString *)deviceId {
|
||||
if ([deviceId isEqualToString:@":ios_custom"]) {
|
||||
return [[VideoCapturerInterfaceImplSourceDescription alloc] initWithIsFrontCamera:false keepLandscape:false deviceId:deviceId device:nil format:nil];
|
||||
}
|
||||
|
||||
if ([deviceId hasPrefix:@"desktop_capturer_"]) {
|
||||
return [[VideoCapturerInterfaceImplSourceDescription alloc] initWithIsFrontCamera:false keepLandscape:true deviceId: deviceId device: nil format: nil];
|
||||
}
|
||||
AVCaptureDevice *selectedCamera = [VideoCapturerInterfaceImplReference selectCapturerDeviceWithDeviceId:deviceId];
|
||||
|
||||
if (selectedCamera == nil) {
|
||||
return [[VideoCapturerInterfaceImplSourceDescription alloc] initWithIsFrontCamera:![deviceId hasPrefix:@"back"] keepLandscape:[deviceId containsString:@"landscape"] deviceId: deviceId device: nil format: nil];
|
||||
}
|
||||
|
||||
AVCaptureDeviceFormat *bestFormat = [VideoCapturerInterfaceImplReference selectCaptureDeviceFormatForDevice:selectedCamera];
|
||||
|
||||
return [[VideoCapturerInterfaceImplSourceDescription alloc] initWithIsFrontCamera:![deviceId hasPrefix:@"back"] keepLandscape:[deviceId containsString:@"landscape"] deviceId:deviceId device:selectedCamera format:bestFormat];
|
||||
}
|
||||
|
||||
- (instancetype)initWithSource:(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source sourceDescription:(VideoCapturerInterfaceImplSourceDescription *)sourceDescription isActiveUpdated:(void (^)(bool))isActiveUpdated rotationUpdated:(void (^)(int))rotationUpdated {
|
||||
self = [super init];
|
||||
if (self != nil) {
|
||||
assert([NSThread isMainThread]);
|
||||
|
||||
#ifdef WEBRTC_IOS
|
||||
if ([sourceDescription.deviceId isEqualToString:@":ios_custom"]) {
|
||||
_customExternalCapturer = [[CustomExternalCapturer alloc] initWithSource:source];
|
||||
} else {
|
||||
_videoCameraCapturer = [[VideoCameraCapturer alloc] initWithSource:source useFrontCamera:sourceDescription.isFrontCamera keepLandscape:sourceDescription.keepLandscape isActiveUpdated:isActiveUpdated rotationUpdated:rotationUpdated];
|
||||
[_videoCameraCapturer startCaptureWithDevice:sourceDescription.device format:sourceDescription.format fps:30];
|
||||
}
|
||||
#else
|
||||
if (const auto desktopCaptureSource = tgcalls::DesktopCaptureSourceForKey([sourceDescription.deviceId UTF8String])) {
|
||||
DesktopSharingCapturer *sharing = [[DesktopSharingCapturer alloc] initWithSource:source captureSource:desktopCaptureSource];
|
||||
_videoCapturer = sharing;
|
||||
} else if (!tgcalls::ShouldBeDesktopCapture([sourceDescription.deviceId UTF8String])) {
|
||||
id<CapturerInterface> camera;
|
||||
if ([sourceDescription.device hasMediaType:AVMediaTypeMuxed]) {
|
||||
VideoCMIOCapture *value = [[VideoCMIOCapture alloc] initWithSource:source];
|
||||
[value setupCaptureWithDevice:sourceDescription.device];
|
||||
camera = value;
|
||||
} else {
|
||||
VideoCameraCapturer *value = [[VideoCameraCapturer alloc] initWithSource:source isActiveUpdated:isActiveUpdated];
|
||||
[value setupCaptureWithDevice:sourceDescription.device format:sourceDescription.format fps:30];
|
||||
camera = value;
|
||||
}
|
||||
_videoCapturer = camera;
|
||||
} else {
|
||||
_videoCapturer = nil;
|
||||
}
|
||||
if (_videoCapturer) {
|
||||
[_videoCapturer start];
|
||||
}
|
||||
#endif
|
||||
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
assert([NSThread isMainThread]);
|
||||
|
||||
#ifdef WEBRTC_IOS
|
||||
[_videoCameraCapturer stopCapture];
|
||||
#elif TARGET_OS_OSX
|
||||
if (_videoCapturer) {
|
||||
[_videoCapturer stop];
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
-(void)setOnFatalError:(std::function<void()>)error {
|
||||
#ifdef WEBRTC_IOS
|
||||
#else
|
||||
if (_videoCapturer) {
|
||||
[_videoCapturer setOnFatalError:error];
|
||||
} else if (error) {
|
||||
error();
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
-(void)setOnPause:(std::function<void(bool)>)pause {
|
||||
#ifdef WEBRTC_IOS
|
||||
#else
|
||||
if (_videoCapturer) {
|
||||
[_videoCapturer setOnPause:pause];
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
- (void)setIsEnabled:(bool)isEnabled {
|
||||
#ifdef WEBRTC_IOS
|
||||
if (_videoCameraCapturer) {
|
||||
[_videoCameraCapturer setIsEnabled:isEnabled];
|
||||
}
|
||||
#else
|
||||
if (_videoCapturer) {
|
||||
[_videoCapturer setIsEnabled:isEnabled];
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
- (void)setUncroppedSink:(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)sink {
|
||||
#ifdef WEBRTC_IOS
|
||||
if (_videoCameraCapturer) {
|
||||
[_videoCameraCapturer setUncroppedSink:sink];
|
||||
}
|
||||
#else
|
||||
if (_videoCapturer) {
|
||||
[_videoCapturer setUncroppedSink:sink];
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
- (void)setPreferredCaptureAspectRatio:(float)aspectRatio {
|
||||
#ifdef WEBRTC_IOS
|
||||
if (_videoCameraCapturer) {
|
||||
[_videoCameraCapturer setPreferredCaptureAspectRatio:aspectRatio];
|
||||
}
|
||||
#else
|
||||
if (_videoCapturer) {
|
||||
[_videoCapturer setPreferredCaptureAspectRatio:aspectRatio];
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
- (int)getRotation {
|
||||
#ifdef WEBRTC_IOS
|
||||
if (_videoCameraCapturer) {
|
||||
return [_videoCameraCapturer getRotation];
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
#elif TARGET_OS_OSX
|
||||
return 0;
|
||||
#else
|
||||
#error "Unsupported platform"
|
||||
#endif
|
||||
}
|
||||
|
||||
- (id)getInternalReference {
|
||||
#ifdef WEBRTC_IOS
|
||||
if (_videoCameraCapturer) {
|
||||
return _videoCameraCapturer;
|
||||
} else if (_customExternalCapturer) {
|
||||
return _customExternalCapturer;
|
||||
} else {
|
||||
return nil;
|
||||
}
|
||||
#endif
|
||||
return nil;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@implementation VideoCapturerInterfaceImplHolder
|
||||
|
||||
@end
|
||||
|
||||
namespace tgcalls {
|
||||
|
||||
VideoCapturerInterfaceImpl::VideoCapturerInterfaceImpl(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> source, std::string deviceId, std::function<void(VideoState)> stateUpdated, std::function<void(PlatformCaptureInfo)> captureInfoUpdated, std::pair<int, int> &outResolution) :
|
||||
_source(source) {
|
||||
VideoCapturerInterfaceImplSourceDescription *sourceDescription = [VideoCapturerInterfaceImplReference selectCapturerDescriptionWithDeviceId:[NSString stringWithUTF8String:deviceId.c_str()]];
|
||||
|
||||
CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(sourceDescription.format.formatDescription);
|
||||
#ifdef WEBRTC_IOS
|
||||
outResolution.first = dimensions.height;
|
||||
outResolution.second = dimensions.width;
|
||||
#else
|
||||
outResolution.first = dimensions.width;
|
||||
outResolution.second = dimensions.height;
|
||||
#endif
|
||||
|
||||
_implReference = [[VideoCapturerInterfaceImplHolder alloc] init];
|
||||
VideoCapturerInterfaceImplHolder *implReference = _implReference;
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
VideoCapturerInterfaceImplReference *value = [[VideoCapturerInterfaceImplReference alloc] initWithSource:source sourceDescription:sourceDescription isActiveUpdated:^(bool isActive) {
|
||||
stateUpdated(isActive ? VideoState::Active : VideoState::Paused);
|
||||
} rotationUpdated:^(int angle) {
|
||||
PlatformCaptureInfo info;
|
||||
bool isLandscape = angle == 180 || angle == 0;
|
||||
info.shouldBeAdaptedToReceiverAspectRate = !isLandscape;
|
||||
info.rotation = angle;
|
||||
captureInfoUpdated(info);
|
||||
}];
|
||||
if (value != nil) {
|
||||
implReference.reference = (void *)CFBridgingRetain(value);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
VideoCapturerInterfaceImpl::~VideoCapturerInterfaceImpl() {
|
||||
VideoCapturerInterfaceImplHolder *implReference = _implReference;
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (implReference.reference != nil) {
|
||||
CFBridgingRelease(implReference.reference);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void VideoCapturerInterfaceImpl::setState(VideoState state) {
|
||||
VideoCapturerInterfaceImplHolder *implReference = _implReference;
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (implReference.reference != nil) {
|
||||
VideoCapturerInterfaceImplReference *reference = (__bridge VideoCapturerInterfaceImplReference *)implReference.reference;
|
||||
[reference setIsEnabled:(state == VideoState::Active)];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void VideoCapturerInterfaceImpl::setPreferredCaptureAspectRatio(float aspectRatio) {
|
||||
VideoCapturerInterfaceImplHolder *implReference = _implReference;
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (implReference.reference != nil) {
|
||||
VideoCapturerInterfaceImplReference *reference = (__bridge VideoCapturerInterfaceImplReference *)implReference.reference;
|
||||
[reference setPreferredCaptureAspectRatio:aspectRatio];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void VideoCapturerInterfaceImpl::withNativeImplementation(std::function<void(void *)> completion) {
|
||||
VideoCapturerInterfaceImplHolder *implReference = _implReference;
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (implReference.reference != nil) {
|
||||
VideoCapturerInterfaceImplReference *reference = (__bridge VideoCapturerInterfaceImplReference *)implReference.reference;
|
||||
completion((__bridge void *)[reference videoCameraCapturer]);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void VideoCapturerInterfaceImpl::setUncroppedOutput(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink) {
|
||||
VideoCapturerInterfaceImplHolder *implReference = _implReference;
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (implReference.reference != nil) {
|
||||
VideoCapturerInterfaceImplReference *reference = (__bridge VideoCapturerInterfaceImplReference *)implReference.reference;
|
||||
[reference setUncroppedSink:sink];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void VideoCapturerInterfaceImpl::setOnFatalError(std::function<void()> error) {
|
||||
VideoCapturerInterfaceImplHolder *implReference = _implReference;
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (implReference.reference != nil) {
|
||||
VideoCapturerInterfaceImplReference *reference = (__bridge VideoCapturerInterfaceImplReference *)implReference.reference;
|
||||
[reference setOnFatalError:error];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void VideoCapturerInterfaceImpl::setOnPause(std::function<void(bool)> pause) {
|
||||
VideoCapturerInterfaceImplHolder *implReference = _implReference;
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (implReference.reference != nil) {
|
||||
VideoCapturerInterfaceImplReference *reference = (__bridge VideoCapturerInterfaceImplReference *)implReference.reference;
|
||||
[reference setOnPause: pause];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
int VideoCapturerInterfaceImpl::getRotation() {
|
||||
__block int value = 0;
|
||||
VideoCapturerInterfaceImplHolder *implReference = _implReference;
|
||||
dispatch_sync(dispatch_get_main_queue(), ^{
|
||||
if (implReference.reference != nil) {
|
||||
VideoCapturerInterfaceImplReference *reference = (__bridge VideoCapturerInterfaceImplReference *)implReference.reference;
|
||||
value = [reference getRotation];
|
||||
}
|
||||
});
|
||||
return value;
|
||||
}
|
||||
|
||||
id VideoCapturerInterfaceImpl::getInternalReference() {
|
||||
__block id value = nil;
|
||||
VideoCapturerInterfaceImplHolder *implReference = _implReference;
|
||||
dispatch_sync(dispatch_get_main_queue(), ^{
|
||||
if (implReference.reference != nil) {
|
||||
VideoCapturerInterfaceImplReference *reference = (__bridge VideoCapturerInterfaceImplReference *)implReference.reference;
|
||||
value = [reference getInternalReference];
|
||||
}
|
||||
});
|
||||
return value;
|
||||
}
|
||||
|
||||
} // namespace tgcalls
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
#ifndef TGCALLS_VIDEO_METAL_VIEW_H
|
||||
#define TGCALLS_VIDEO_METAL_VIEW_H
|
||||
#ifdef WEBRTC_IOS
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <UIKit/UIKit.h>
|
||||
|
||||
#import "api/media_stream_interface.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
@class RTCVideoFrame;
|
||||
|
||||
@interface VideoMetalView : UIView
|
||||
|
||||
+ (bool)isSupported;
|
||||
|
||||
@property(nonatomic) UIViewContentMode videoContentMode;
|
||||
@property(nonatomic, getter=isEnabled) BOOL enabled;
|
||||
@property(nonatomic, nullable) NSValue* rotationOverride;
|
||||
|
||||
@property (nonatomic, readwrite) int internalOrientation;
|
||||
@property (nonatomic, readwrite) CGFloat internalAspect;
|
||||
|
||||
- (void)setSize:(CGSize)size;
|
||||
- (void)renderFrame:(nullable RTCVideoFrame *)frame;
|
||||
|
||||
- (std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)getSink;
|
||||
- (void)setOnFirstFrameReceived:(void (^ _Nullable)())onFirstFrameReceived;
|
||||
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated;
|
||||
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated;
|
||||
|
||||
- (void)setClone:(VideoMetalView * _Nullable)clone;
|
||||
|
||||
@end
|
||||
|
||||
#endif //WEBRTC_IOS
|
||||
#endif
|
||||
459
TMessagesProj/jni/voip/tgcalls/platform/darwin/VideoMetalView.mm
Normal file
459
TMessagesProj/jni/voip/tgcalls/platform/darwin/VideoMetalView.mm
Normal file
|
|
@ -0,0 +1,459 @@
|
|||
#import "VideoMetalView.h"
|
||||
|
||||
#import <Metal/Metal.h>
|
||||
#import <MetalKit/MetalKit.h>
|
||||
|
||||
#import "base/RTCLogging.h"
|
||||
#import "base/RTCVideoFrame.h"
|
||||
#import "base/RTCVideoFrameBuffer.h"
|
||||
#import "TGRTCCVPixelBuffer.h"
|
||||
#include "sdk/objc/native/api/video_frame.h"
|
||||
#include "sdk/objc/native/src/objc_frame_buffer.h"
|
||||
|
||||
#import "api/video/video_sink_interface.h"
|
||||
#import "api/media_stream_interface.h"
|
||||
#import "rtc_base/time_utils.h"
|
||||
|
||||
#import "RTCMTLI420Renderer.h"
|
||||
#import "RTCMTLNV12Renderer.h"
|
||||
#import "RTCMTLRGBRenderer.h"
|
||||
|
||||
#define MTKViewClass NSClassFromString(@"MTKView")
|
||||
#define RTCMTLNV12RendererClass NSClassFromString(@"RTCMTLNV12Renderer")
|
||||
#define RTCMTLI420RendererClass NSClassFromString(@"RTCMTLI420Renderer")
|
||||
#define RTCMTLRGBRendererClass NSClassFromString(@"RTCMTLRGBRenderer")
|
||||
|
||||
namespace {
|
||||
|
||||
static RTCVideoFrame *customToObjCVideoFrame(const webrtc::VideoFrame &frame, RTCVideoRotation &rotation) {
|
||||
rotation = RTCVideoRotation(frame.rotation());
|
||||
RTCVideoFrame *videoFrame =
|
||||
[[RTCVideoFrame alloc] initWithBuffer:webrtc::ToObjCVideoFrameBuffer(frame.video_frame_buffer())
|
||||
rotation:rotation
|
||||
timeStampNs:frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec];
|
||||
videoFrame.timeStamp = frame.timestamp();
|
||||
|
||||
return videoFrame;
|
||||
}
|
||||
|
||||
class VideoRendererAdapterImpl : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
|
||||
public:
|
||||
VideoRendererAdapterImpl(void (^frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation)) {
|
||||
_frameReceived = [frameReceived copy];
|
||||
}
|
||||
|
||||
void OnFrame(const webrtc::VideoFrame& nativeVideoFrame) override {
|
||||
@autoreleasepool {
|
||||
RTCVideoRotation rotation = RTCVideoRotation_90;
|
||||
RTCVideoFrame* videoFrame = customToObjCVideoFrame(nativeVideoFrame, rotation);
|
||||
|
||||
//CGSize currentSize = (videoFrame.rotation % 180 == 0) ? CGSizeMake(videoFrame.width, videoFrame.height) : CGSizeMake(videoFrame.height, videoFrame.width);
|
||||
CGSize currentSize = CGSizeMake(videoFrame.width, videoFrame.height);
|
||||
|
||||
if (_frameReceived) {
|
||||
_frameReceived(currentSize, videoFrame, rotation);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
void (^_frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@interface VideoMetalView () <MTKViewDelegate> {
|
||||
RTCMTLI420Renderer *_rendererI420;
|
||||
RTCMTLNV12Renderer *_rendererNV12;
|
||||
RTCMTLRGBRenderer *_rendererRGB;
|
||||
MTKView *_metalView;
|
||||
RTCVideoFrame *_videoFrame;
|
||||
RTCVideoFrame *_stashedVideoFrame;
|
||||
int _isWaitingForLayoutFrameCount;
|
||||
bool _didStartWaitingForLayout;
|
||||
CGSize _videoFrameSize;
|
||||
int64_t _lastFrameTimeNs;
|
||||
|
||||
CGSize _currentSize;
|
||||
std::shared_ptr<VideoRendererAdapterImpl> _sink;
|
||||
|
||||
void (^_onFirstFrameReceived)();
|
||||
bool _firstFrameReceivedReported;
|
||||
|
||||
void (^_onOrientationUpdated)(int, CGFloat);
|
||||
|
||||
void (^_onIsMirroredUpdated)(bool);
|
||||
|
||||
bool _didSetShouldBeMirrored;
|
||||
bool _shouldBeMirrored;
|
||||
bool _shouldBeMirroredVertically;
|
||||
|
||||
__weak VideoMetalView *_cloneView;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@implementation VideoMetalView
|
||||
|
||||
+ (bool)isSupported {
|
||||
static bool value;
|
||||
static dispatch_once_t onceToken;
|
||||
dispatch_once(&onceToken, ^{
|
||||
id<MTLDevice> device = MTLCreateSystemDefaultDevice();
|
||||
value = device != nil;
|
||||
});
|
||||
return value;
|
||||
}
|
||||
|
||||
- (instancetype)initWithFrame:(CGRect)frameRect {
|
||||
self = [super initWithFrame:frameRect];
|
||||
if (self) {
|
||||
[self configure];
|
||||
|
||||
_currentSize = CGSizeZero;
|
||||
_rotationOverride = @(RTCVideoRotation_0);
|
||||
|
||||
__weak VideoMetalView *weakSelf = self;
|
||||
_sink.reset(new VideoRendererAdapterImpl(^(CGSize size, RTCVideoFrame *videoFrame, RTCVideoRotation rotation) {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
__strong VideoMetalView *strongSelf = weakSelf;
|
||||
if (strongSelf == nil) {
|
||||
return;
|
||||
}
|
||||
if (!CGSizeEqualToSize(size, strongSelf->_currentSize)) {
|
||||
strongSelf->_currentSize = size;
|
||||
[strongSelf setSize:size];
|
||||
}
|
||||
|
||||
int mappedValue = 0;
|
||||
switch (rotation) {
|
||||
case RTCVideoRotation_90:
|
||||
mappedValue = 1;
|
||||
break;
|
||||
case RTCVideoRotation_180:
|
||||
mappedValue = 2;
|
||||
break;
|
||||
case RTCVideoRotation_270:
|
||||
mappedValue = 3;
|
||||
break;
|
||||
default:
|
||||
mappedValue = 0;
|
||||
break;
|
||||
}
|
||||
[strongSelf setInternalOrientationAndSize:mappedValue size:size];
|
||||
|
||||
[strongSelf renderFrame:videoFrame];
|
||||
|
||||
VideoMetalView *cloneView = strongSelf->_cloneView;
|
||||
if (cloneView) {
|
||||
if (!CGSizeEqualToSize(size, cloneView->_currentSize)) {
|
||||
cloneView->_currentSize = size;
|
||||
[cloneView setSize:size];
|
||||
}
|
||||
|
||||
[cloneView setInternalOrientationAndSize:mappedValue size:size];
|
||||
[cloneView renderFrame:videoFrame];
|
||||
}
|
||||
});
|
||||
}));
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
_sink.reset();
|
||||
}
|
||||
|
||||
- (BOOL)isEnabled {
|
||||
return !_metalView.paused;
|
||||
}
|
||||
|
||||
- (void)setEnabled:(BOOL)enabled {
|
||||
_metalView.paused = !enabled;
|
||||
}
|
||||
|
||||
- (UIViewContentMode)videoContentMode {
|
||||
return _metalView.contentMode;
|
||||
}
|
||||
|
||||
- (void)setVideoContentMode:(UIViewContentMode)mode {
|
||||
_metalView.contentMode = mode;
|
||||
}
|
||||
|
||||
#pragma mark - Private
|
||||
|
||||
+ (BOOL)isMetalAvailable {
|
||||
return MTLCreateSystemDefaultDevice() != nil;
|
||||
}
|
||||
|
||||
+ (MTKView *)createMetalView:(CGRect)frame {
|
||||
return [[MTKViewClass alloc] initWithFrame:frame];
|
||||
}
|
||||
|
||||
+ (RTCMTLNV12Renderer *)createNV12Renderer {
|
||||
return [[RTCMTLNV12RendererClass alloc] init];
|
||||
}
|
||||
|
||||
+ (RTCMTLI420Renderer *)createI420Renderer {
|
||||
return [[RTCMTLI420RendererClass alloc] init];
|
||||
}
|
||||
|
||||
+ (RTCMTLRGBRenderer *)createRGBRenderer {
|
||||
return [[RTCMTLRGBRenderer alloc] init];
|
||||
}
|
||||
|
||||
- (void)configure {
|
||||
NSAssert([VideoMetalView isMetalAvailable], @"Metal not availiable on this device");
|
||||
|
||||
_metalView = [VideoMetalView createMetalView:self.bounds];
|
||||
_metalView.delegate = self;
|
||||
_metalView.contentMode = UIViewContentModeScaleToFill;
|
||||
_metalView.preferredFramesPerSecond = 30;
|
||||
[self addSubview:_metalView];
|
||||
_videoFrameSize = CGSizeZero;
|
||||
}
|
||||
|
||||
- (void)setMultipleTouchEnabled:(BOOL)multipleTouchEnabled {
|
||||
[super setMultipleTouchEnabled:multipleTouchEnabled];
|
||||
_metalView.multipleTouchEnabled = multipleTouchEnabled;
|
||||
}
|
||||
|
||||
- (void)layoutSubviews {
|
||||
[super layoutSubviews];
|
||||
|
||||
CGRect bounds = self.bounds;
|
||||
_metalView.frame = bounds;
|
||||
if (!CGSizeEqualToSize(_videoFrameSize, CGSizeZero)) {
|
||||
_metalView.drawableSize = [self drawableSize];
|
||||
} else {
|
||||
_metalView.drawableSize = bounds.size;
|
||||
}
|
||||
|
||||
if (_didStartWaitingForLayout) {
|
||||
_didStartWaitingForLayout = false;
|
||||
_isWaitingForLayoutFrameCount = 0;
|
||||
if (_stashedVideoFrame != nil) {
|
||||
_videoFrame = _stashedVideoFrame;
|
||||
_stashedVideoFrame = nil;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#pragma mark - MTKViewDelegate methods
|
||||
|
||||
- (void)drawInMTKView:(nonnull MTKView *)view {
|
||||
NSAssert(view == _metalView, @"Receiving draw callbacks from foreign instance.");
|
||||
RTCVideoFrame *videoFrame = _videoFrame;
|
||||
// Skip rendering if we've already rendered this frame.
|
||||
if (!videoFrame || videoFrame.timeStampNs == _lastFrameTimeNs) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (CGRectIsEmpty(view.bounds)) {
|
||||
return;
|
||||
}
|
||||
|
||||
RTCMTLRenderer *renderer;
|
||||
if ([videoFrame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
|
||||
RTCCVPixelBuffer *buffer = (RTCCVPixelBuffer*)videoFrame.buffer;
|
||||
|
||||
if ([buffer isKindOfClass:[TGRTCCVPixelBuffer class]]) {
|
||||
bool shouldBeMirrored = ((TGRTCCVPixelBuffer *)buffer).shouldBeMirrored;
|
||||
bool shouldBeMirroredVertically = _internalOrientation == 1 || _internalOrientation == 3;
|
||||
if (shouldBeMirrored != _shouldBeMirrored || shouldBeMirroredVertically != _shouldBeMirroredVertically) {
|
||||
_shouldBeMirrored = shouldBeMirrored;
|
||||
_shouldBeMirroredVertically = shouldBeMirroredVertically;
|
||||
if (_shouldBeMirrored) {
|
||||
if (_shouldBeMirroredVertically) {
|
||||
_metalView.transform = CGAffineTransformMakeScale(1.0f, -1.0f);
|
||||
} else {
|
||||
_metalView.transform = CGAffineTransformMakeScale(-1.0f, 1.0f);
|
||||
}
|
||||
} else {
|
||||
_metalView.transform = CGAffineTransformIdentity;
|
||||
}
|
||||
|
||||
if (_didSetShouldBeMirrored) {
|
||||
if (_onIsMirroredUpdated) {
|
||||
_onIsMirroredUpdated(_shouldBeMirrored);
|
||||
}
|
||||
} else {
|
||||
_didSetShouldBeMirrored = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const OSType pixelFormat = CVPixelBufferGetPixelFormatType(buffer.pixelBuffer);
|
||||
if (pixelFormat == kCVPixelFormatType_32BGRA || pixelFormat == kCVPixelFormatType_32ARGB) {
|
||||
if (!_rendererRGB) {
|
||||
_rendererRGB = [VideoMetalView createRGBRenderer];
|
||||
if (![_rendererRGB addRenderingDestination:_metalView]) {
|
||||
_rendererRGB = nil;
|
||||
RTCLogError(@"Failed to create RGB renderer");
|
||||
return;
|
||||
}
|
||||
}
|
||||
renderer = _rendererRGB;
|
||||
} else {
|
||||
if (!_rendererNV12) {
|
||||
_rendererNV12 = [VideoMetalView createNV12Renderer];
|
||||
if (![_rendererNV12 addRenderingDestination:_metalView]) {
|
||||
_rendererNV12 = nil;
|
||||
RTCLogError(@"Failed to create NV12 renderer");
|
||||
return;
|
||||
}
|
||||
}
|
||||
renderer = _rendererNV12;
|
||||
}
|
||||
} else {
|
||||
if (!_rendererI420) {
|
||||
_rendererI420 = [VideoMetalView createI420Renderer];
|
||||
if (![_rendererI420 addRenderingDestination:_metalView]) {
|
||||
_rendererI420 = nil;
|
||||
RTCLogError(@"Failed to create I420 renderer");
|
||||
return;
|
||||
}
|
||||
}
|
||||
renderer = _rendererI420;
|
||||
}
|
||||
|
||||
renderer.rotationOverride = _rotationOverride;
|
||||
|
||||
[renderer drawFrame:videoFrame];
|
||||
_lastFrameTimeNs = videoFrame.timeStampNs;
|
||||
}
|
||||
|
||||
- (void)mtkView:(MTKView *)view drawableSizeWillChange:(CGSize)size {
|
||||
}
|
||||
|
||||
#pragma mark -
|
||||
|
||||
- (void)setRotationOverride:(NSValue *)rotationOverride {
|
||||
_rotationOverride = rotationOverride;
|
||||
|
||||
_metalView.drawableSize = [self drawableSize];
|
||||
[self setNeedsLayout];
|
||||
}
|
||||
|
||||
- (RTCVideoRotation)frameRotation {
|
||||
if (_rotationOverride) {
|
||||
RTCVideoRotation rotation;
|
||||
if (@available(iOS 11, *)) {
|
||||
[_rotationOverride getValue:&rotation size:sizeof(rotation)];
|
||||
} else {
|
||||
[_rotationOverride getValue:&rotation];
|
||||
}
|
||||
return rotation;
|
||||
}
|
||||
|
||||
return _videoFrame.rotation;
|
||||
}
|
||||
|
||||
- (CGSize)drawableSize {
|
||||
// Flip width/height if the rotations are not the same.
|
||||
CGSize videoFrameSize = _videoFrameSize;
|
||||
return videoFrameSize;
|
||||
|
||||
/*RTCVideoRotation frameRotation = [self frameRotation];
|
||||
|
||||
BOOL useLandscape =
|
||||
(frameRotation == RTCVideoRotation_0) || (frameRotation == RTCVideoRotation_180);
|
||||
BOOL sizeIsLandscape = (_videoFrame.rotation == RTCVideoRotation_0) ||
|
||||
(_videoFrame.rotation == RTCVideoRotation_180);
|
||||
|
||||
if (useLandscape == sizeIsLandscape) {
|
||||
return videoFrameSize;
|
||||
} else {
|
||||
return CGSizeMake(videoFrameSize.height, videoFrameSize.width);
|
||||
}*/
|
||||
}
|
||||
|
||||
#pragma mark - RTCVideoRenderer
|
||||
|
||||
- (void)setSize:(CGSize)size {
|
||||
assert([NSThread isMainThread]);
|
||||
|
||||
_videoFrameSize = size;
|
||||
_metalView.drawableSize = [self drawableSize];
|
||||
|
||||
//_metalView.drawableSize = drawableSize;
|
||||
//[self setNeedsLayout];
|
||||
//[strongSelf.delegate videoView:self didChangeVideoSize:size];
|
||||
}
|
||||
|
||||
- (void)renderFrame:(nullable RTCVideoFrame *)frame {
|
||||
assert([NSThread isMainThread]);
|
||||
|
||||
if (!_firstFrameReceivedReported && _onFirstFrameReceived) {
|
||||
_firstFrameReceivedReported = true;
|
||||
_onFirstFrameReceived();
|
||||
}
|
||||
|
||||
if (!self.isEnabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (frame == nil) {
|
||||
RTCLogInfo(@"Incoming frame is nil. Exiting render callback.");
|
||||
return;
|
||||
}
|
||||
if (_isWaitingForLayoutFrameCount > 0) {
|
||||
_stashedVideoFrame = frame;
|
||||
_isWaitingForLayoutFrameCount--;
|
||||
return;
|
||||
}
|
||||
if (!_didStartWaitingForLayout) {
|
||||
if (_videoFrame != nil && _videoFrame.width > 0 && _videoFrame.height > 0 && frame.width > 0 && frame.height > 0) {
|
||||
float previousAspect = ((float)_videoFrame.width) / ((float)_videoFrame.height);
|
||||
float updatedAspect = ((float)frame.width) / ((float)frame.height);
|
||||
if ((previousAspect < 1.0f) != (updatedAspect < 1.0f)) {
|
||||
_stashedVideoFrame = frame;
|
||||
_didStartWaitingForLayout = true;
|
||||
_isWaitingForLayoutFrameCount = 5;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
_videoFrame = frame;
|
||||
}
|
||||
|
||||
- (std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)getSink {
|
||||
assert([NSThread isMainThread]);
|
||||
|
||||
return _sink;
|
||||
}
|
||||
|
||||
- (void)setOnFirstFrameReceived:(void (^ _Nullable)())onFirstFrameReceived {
|
||||
_onFirstFrameReceived = [onFirstFrameReceived copy];
|
||||
_firstFrameReceivedReported = false;
|
||||
}
|
||||
|
||||
- (void)setInternalOrientationAndSize:(int)internalOrientation size:(CGSize)size {
|
||||
CGFloat aspect = 1.0f;
|
||||
if (size.width > 1.0f && size.height > 1.0f) {
|
||||
aspect = size.width / size.height;
|
||||
}
|
||||
if (_internalOrientation != internalOrientation || ABS(_internalAspect - aspect) > 0.001) {
|
||||
RTCLogInfo(@"VideoMetalView@%lx orientation: %d, aspect: %f", (intptr_t)self, internalOrientation, (float)aspect);
|
||||
|
||||
_internalOrientation = internalOrientation;
|
||||
_internalAspect = aspect;
|
||||
if (_onOrientationUpdated) {
|
||||
_onOrientationUpdated(internalOrientation, aspect);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated {
|
||||
_onOrientationUpdated = [onOrientationUpdated copy];
|
||||
}
|
||||
|
||||
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated {
|
||||
_onIsMirroredUpdated = [onIsMirroredUpdated copy];
|
||||
}
|
||||
|
||||
- (void)setClone:(VideoMetalView * _Nullable)clone {
|
||||
_cloneView = clone;
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
#ifndef TGCALLS_VIDEO_METAL_VIEW_MAC_H
|
||||
#define TGCALLS_VIDEO_METAL_VIEW_MAC_H
|
||||
#ifndef WEBRTC_IOS
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <AppKit/AppKit.h>
|
||||
|
||||
#import "api/media_stream_interface.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
@class RTCVideoFrame;
|
||||
|
||||
@interface VideoMetalView : NSView
|
||||
|
||||
+ (bool)isSupported;
|
||||
|
||||
@property(nonatomic) CALayerContentsGravity _Nullable videoContentMode;
|
||||
@property(nonatomic, getter=isEnabled) BOOL enabled;
|
||||
@property(nonatomic, nullable) NSValue* rotationOverride;
|
||||
|
||||
@property (nonatomic, readwrite) int internalOrientation;
|
||||
@property (nonatomic, readwrite) CGFloat internalAspect;
|
||||
|
||||
|
||||
- (void)setSize:(CGSize)size;
|
||||
- (void)renderFrame:(nullable RTCVideoFrame *)frame;
|
||||
|
||||
|
||||
- (std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)getSink;
|
||||
- (void)setOnFirstFrameReceived:(void (^ _Nullable)(float))onFirstFrameReceived;
|
||||
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated;
|
||||
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated;
|
||||
- (void)setForceMirrored:(BOOL)forceMirrored;
|
||||
|
||||
-(void)setIsPaused:(bool)paused;
|
||||
-(void)renderToSize:(NSSize)size animated: (bool)animated;
|
||||
|
||||
@end
|
||||
|
||||
#endif // WEBRTC_MAC
|
||||
#endif
|
||||
|
|
@ -0,0 +1,503 @@
|
|||
#import "VideoMetalViewMac.h"
|
||||
#import <Metal/Metal.h>
|
||||
#import <MetalKit/MetalKit.h>
|
||||
#import "TGRTCCVPixelBuffer.h"
|
||||
#import "base/RTCLogging.h"
|
||||
#import "base/RTCVideoFrame.h"
|
||||
#import "base/RTCVideoFrameBuffer.h"
|
||||
#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
|
||||
#include "sdk/objc/native/api/video_frame.h"
|
||||
#include "sdk/objc/native/src/objc_frame_buffer.h"
|
||||
|
||||
#import "api/video/video_sink_interface.h"
|
||||
#import "api/media_stream_interface.h"
|
||||
#import "rtc_base/time_utils.h"
|
||||
|
||||
#import "SQueueLocalObject.h"
|
||||
|
||||
|
||||
#import "api/video/video_sink_interface.h"
|
||||
#import "api/media_stream_interface.h"
|
||||
|
||||
#import "tgcalls/platform/darwin/macOS/TGRTCMTLI420Renderer.h"
|
||||
|
||||
#define MTKViewClass NSClassFromString(@"MTKView")
|
||||
#define TGRTCMTLI420RendererClass NSClassFromString(@"TGRTCMTLI420Renderer")
|
||||
|
||||
SQueue *renderQueue = [[SQueue alloc] init];
|
||||
|
||||
namespace {
|
||||
|
||||
static RTCVideoFrame *customToObjCVideoFrame(const webrtc::VideoFrame &frame, RTCVideoRotation &rotation) {
|
||||
rotation = RTCVideoRotation(frame.rotation());
|
||||
RTCVideoFrame *videoFrame =
|
||||
[[RTCVideoFrame alloc] initWithBuffer:webrtc::ToObjCVideoFrameBuffer(frame.video_frame_buffer())
|
||||
rotation:rotation
|
||||
timeStampNs:frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec];
|
||||
videoFrame.timeStamp = frame.timestamp();
|
||||
|
||||
return videoFrame;
|
||||
}
|
||||
|
||||
class VideoRendererAdapterImpl : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
|
||||
public:
|
||||
VideoRendererAdapterImpl(void (^frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation)) {
|
||||
_frameReceived = [frameReceived copy];
|
||||
}
|
||||
|
||||
void OnFrame(const webrtc::VideoFrame& nativeVideoFrame) override {
|
||||
RTCVideoRotation rotation = RTCVideoRotation_0;
|
||||
RTCVideoFrame* videoFrame = customToObjCVideoFrame(nativeVideoFrame, rotation);
|
||||
|
||||
CGSize currentSize = (videoFrame.rotation % 180 == 0) ? CGSizeMake(videoFrame.width, videoFrame.height) : CGSizeMake(videoFrame.height, videoFrame.width);
|
||||
|
||||
if (_frameReceived) {
|
||||
_frameReceived(currentSize, videoFrame, rotation);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
void (^_frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@interface TGCAMetalLayer : CAMetalLayer
|
||||
|
||||
@end
|
||||
|
||||
@implementation TGCAMetalLayer
|
||||
|
||||
|
||||
-(void)dealloc {
|
||||
}
|
||||
@end
|
||||
|
||||
@interface VideoMetalView () {
|
||||
SQueueLocalObject *_rendererI420;
|
||||
|
||||
CAMetalLayer *_metalView;
|
||||
NSView *_foregroundView;
|
||||
|
||||
CGSize _videoFrameSize;
|
||||
RTCVideoRotation _rotation;
|
||||
int64_t _lastFrameTimeNs;
|
||||
|
||||
CGSize _currentSize;
|
||||
std::shared_ptr<VideoRendererAdapterImpl> _sink;
|
||||
|
||||
void (^_onFirstFrameReceived)(float);
|
||||
bool _firstFrameReceivedReported;
|
||||
void (^_onOrientationUpdated)(int, CGFloat);
|
||||
void (^_onIsMirroredUpdated)(bool);
|
||||
|
||||
bool _didSetShouldBeMirrored;
|
||||
bool _shouldBeMirrored;
|
||||
bool _forceMirrored;
|
||||
|
||||
bool _isPaused;
|
||||
|
||||
NSMutableArray<RTCVideoFrame *> *_frames;
|
||||
RTCVideoFrame *_videoFrame;
|
||||
BOOL _drawing;
|
||||
|
||||
BOOL _deleteForegroundOnNextDrawing;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@implementation VideoMetalView
|
||||
|
||||
+ (bool)isSupported {
|
||||
return [VideoMetalView isMetalAvailable];
|
||||
}
|
||||
|
||||
- (instancetype)initWithFrame:(CGRect)frameRect {
|
||||
self = [super initWithFrame:frameRect];
|
||||
if (self) {
|
||||
[self configure];
|
||||
_lastFrameTimeNs = INT32_MAX;
|
||||
_currentSize = CGSizeZero;
|
||||
_frames = [[NSMutableArray alloc] init];
|
||||
_drawing = false;
|
||||
_isPaused = false;
|
||||
_deleteForegroundOnNextDrawing = false;
|
||||
__weak VideoMetalView *weakSelf = self;
|
||||
_sink.reset(new VideoRendererAdapterImpl(^(CGSize size, RTCVideoFrame *videoFrame, RTCVideoRotation rotation) {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
__strong VideoMetalView *strongSelf = weakSelf;
|
||||
if (strongSelf == nil) {
|
||||
return;
|
||||
}
|
||||
strongSelf->_rotation = videoFrame.rotation;
|
||||
if (!CGSizeEqualToSize(size, strongSelf->_currentSize)) {
|
||||
strongSelf->_currentSize = size;
|
||||
[strongSelf setSize:size];
|
||||
}
|
||||
|
||||
int mappedValue = 0;
|
||||
switch (rotation) {
|
||||
case RTCVideoRotation_90:
|
||||
mappedValue = 0;
|
||||
break;
|
||||
case RTCVideoRotation_180:
|
||||
mappedValue = 1;
|
||||
break;
|
||||
case RTCVideoRotation_270:
|
||||
mappedValue = 2;
|
||||
break;
|
||||
default:
|
||||
mappedValue = 0;
|
||||
break;
|
||||
}
|
||||
[strongSelf setInternalOrientation:mappedValue];
|
||||
|
||||
[strongSelf renderFrame:videoFrame];
|
||||
|
||||
if ([videoFrame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
|
||||
RTCCVPixelBuffer *buffer = (RTCCVPixelBuffer*)videoFrame.buffer;
|
||||
|
||||
if ([buffer isKindOfClass:[TGRTCCVPixelBuffer class]]) {
|
||||
bool shouldBeMirrored = ((TGRTCCVPixelBuffer *)buffer).shouldBeMirrored;
|
||||
if (shouldBeMirrored != strongSelf->_shouldBeMirrored) {
|
||||
strongSelf->_shouldBeMirrored = shouldBeMirrored;
|
||||
if (strongSelf->_onIsMirroredUpdated) {
|
||||
strongSelf->_onIsMirroredUpdated(strongSelf->_shouldBeMirrored);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!strongSelf->_firstFrameReceivedReported && strongSelf->_onFirstFrameReceived) {
|
||||
strongSelf->_firstFrameReceivedReported = true;
|
||||
strongSelf->_onFirstFrameReceived((float)videoFrame.width / (float)videoFrame.height);
|
||||
}
|
||||
});
|
||||
}));
|
||||
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (BOOL)isEnabled {
|
||||
return !_isPaused;
|
||||
}
|
||||
|
||||
- (void)setEnabled:(BOOL)enabled {
|
||||
_isPaused = enabled;
|
||||
}
|
||||
-(void)setIsPaused:(bool)paused {
|
||||
_isPaused = paused;
|
||||
[self updateDrawingSize:self.frame.size];
|
||||
}
|
||||
-(void)renderToSize:(NSSize)size animated: (bool)animated {
|
||||
[CATransaction begin];
|
||||
[CATransaction setDisableActions:YES];
|
||||
if (animated) {
|
||||
if (!_foregroundView) {
|
||||
_foregroundView = [[NSView alloc] initWithFrame:self.bounds];
|
||||
_foregroundView.wantsLayer = YES;
|
||||
_foregroundView.autoresizingMask = 0;
|
||||
_foregroundView.layer = [VideoMetalView createMetalView:self.bounds];
|
||||
_foregroundView.layer.contentsGravity = kCAGravityResizeAspect;
|
||||
[self addSubview:_foregroundView];
|
||||
}
|
||||
|
||||
CAMetalLayer *layer = _metalView;
|
||||
CAMetalLayer *foreground = (CAMetalLayer *)_foregroundView.layer;
|
||||
|
||||
[_rendererI420 with:^(TGRTCMTLI420Renderer * renderer) {
|
||||
[renderer setDoubleRendering:layer foreground:foreground];
|
||||
}];
|
||||
_deleteForegroundOnNextDrawing = false;
|
||||
} else {
|
||||
_deleteForegroundOnNextDrawing = true;
|
||||
CAMetalLayer *layer = _metalView;
|
||||
|
||||
[_rendererI420 with:^(TGRTCMTLI420Renderer * renderer) {
|
||||
[renderer setSingleRendering:layer];
|
||||
}];
|
||||
}
|
||||
[self updateDrawingSize:size];
|
||||
[CATransaction commit];
|
||||
}
|
||||
|
||||
- (CALayerContentsGravity)videoContentMode {
|
||||
return _metalView.contentsGravity;
|
||||
}
|
||||
|
||||
- (void)setVideoContentMode:(CALayerContentsGravity)mode {
|
||||
// _metalView.contentsGravity = mode;
|
||||
}
|
||||
|
||||
#pragma mark - Private
|
||||
|
||||
+ (BOOL)isMetalAvailable {
|
||||
return initMetal();
|
||||
}
|
||||
|
||||
+ (CAMetalLayer *)createMetalView:(CGRect)frame {
|
||||
CAMetalLayer *layer = [[TGCAMetalLayer alloc] init];
|
||||
[CATransaction begin];
|
||||
[CATransaction setDisableActions:true];
|
||||
layer.framebufferOnly = true;
|
||||
layer.opaque = false;
|
||||
// layer.cornerRadius = 4;
|
||||
if (@available(macOS 10.13, *)) {
|
||||
layer.displaySyncEnabled = NO;
|
||||
}
|
||||
// layer.presentsWithTransaction = YES;
|
||||
layer.backgroundColor = [NSColor clearColor].CGColor;
|
||||
layer.contentsGravity = kCAGravityResizeAspectFill;
|
||||
layer.frame = frame;
|
||||
|
||||
[CATransaction commit];
|
||||
return layer;
|
||||
}
|
||||
|
||||
+ (TGRTCMTLI420Renderer *)createI420Renderer {
|
||||
return [[TGRTCMTLI420RendererClass alloc] init];
|
||||
}
|
||||
|
||||
|
||||
- (void)configure {
|
||||
NSAssert([VideoMetalView isMetalAvailable], @"Metal not availiable on this device");
|
||||
self.wantsLayer = YES;
|
||||
self.layerContentsRedrawPolicy = NSViewLayerContentsRedrawOnSetNeedsDisplay;
|
||||
_metalView = [VideoMetalView createMetalView:self.bounds];
|
||||
|
||||
|
||||
self.layer = _metalView;
|
||||
_videoFrameSize = CGSizeZero;
|
||||
|
||||
CAMetalLayer *layer = _metalView;
|
||||
|
||||
_rendererI420 = [[SQueueLocalObject alloc] initWithQueue:renderQueue generate: ^{
|
||||
TGRTCMTLI420Renderer *renderer = [VideoMetalView createI420Renderer];
|
||||
[renderer setSingleRendering:layer];
|
||||
return renderer;
|
||||
}];
|
||||
}
|
||||
|
||||
|
||||
-(void)setFrameSize:(NSSize)newSize {
|
||||
[super setFrameSize:newSize];
|
||||
|
||||
[self updateDrawingSize: newSize];
|
||||
|
||||
}
|
||||
- (void)layout {
|
||||
[super layout];
|
||||
}
|
||||
|
||||
-(void)updateDrawingSize:(NSSize)size {
|
||||
if (_isPaused) {
|
||||
return;
|
||||
}
|
||||
_metalView.frame = CGRectMake(0, 0, size.width, size.height);
|
||||
_foregroundView.frame = self.bounds;
|
||||
if (!CGSizeEqualToSize(_videoFrameSize, CGSizeZero)) {
|
||||
_metalView.drawableSize = [self drawableSize:size];
|
||||
((CAMetalLayer *)_foregroundView.layer).drawableSize = _videoFrameSize;
|
||||
} else {
|
||||
_metalView.drawableSize = size;
|
||||
((CAMetalLayer *)_foregroundView.layer).drawableSize = size;
|
||||
}
|
||||
|
||||
if(!_isPaused) {
|
||||
RTCVideoFrame *frame = [_frames lastObject];
|
||||
if (frame == nil) {
|
||||
frame = _videoFrame;
|
||||
}
|
||||
if (frame) {
|
||||
[self renderFrame:frame];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
-(void)dealloc {
|
||||
int bp = 0;
|
||||
bp += 1;
|
||||
}
|
||||
|
||||
#pragma mark -
|
||||
|
||||
- (void)setRotationOverride:(NSValue *)rotationOverride {
|
||||
_rotationOverride = rotationOverride;
|
||||
|
||||
[self setNeedsLayout:YES];
|
||||
}
|
||||
|
||||
- (RTCVideoRotation)rtcFrameRotation {
|
||||
if (_rotationOverride) {
|
||||
RTCVideoRotation rotation;
|
||||
if (@available(macOS 10.13, *)) {
|
||||
[_rotationOverride getValue:&rotation size:sizeof(rotation)];
|
||||
} else {
|
||||
[_rotationOverride getValue:&rotation];
|
||||
}
|
||||
return rotation;
|
||||
}
|
||||
|
||||
return _rotation;
|
||||
}
|
||||
|
||||
- (CGSize)drawableSize:(NSSize)forSize {
|
||||
|
||||
MTLFrameSize from;
|
||||
MTLFrameSize to;
|
||||
|
||||
from.width = _videoFrameSize.width;
|
||||
from.height = _videoFrameSize.height;
|
||||
|
||||
if (CGSizeEqualToSize(CGSizeZero, forSize)) {
|
||||
to.width = _videoFrameSize.width;
|
||||
to.height = _videoFrameSize.height;
|
||||
} else {
|
||||
to.width = forSize.width;
|
||||
to.height = forSize.height;
|
||||
}
|
||||
|
||||
|
||||
MTLFrameSize size = MTLAspectFilled(to, from);
|
||||
|
||||
return CGSizeMake(size.width, size.height);
|
||||
}
|
||||
|
||||
#pragma mark - RTCVideoRenderer
|
||||
|
||||
- (void)setSize:(CGSize)size {
|
||||
assert([NSThread isMainThread]);
|
||||
|
||||
_videoFrameSize = size;
|
||||
[self updateDrawingSize:self.frame.size];
|
||||
|
||||
_internalAspect = _videoFrameSize.width / _videoFrameSize.height;
|
||||
}
|
||||
|
||||
- (void)renderFrame:(nullable RTCVideoFrame *)frame {
|
||||
assert([NSThread isMainThread]);
|
||||
|
||||
if (!self.isEnabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (frame == nil) {
|
||||
RTCLogInfo(@"Incoming frame is nil. Exiting render callback.");
|
||||
return;
|
||||
}
|
||||
|
||||
RTCVideoFrame *videoFrame = frame;
|
||||
// Skip rendering if we've already rendered this frame.
|
||||
if (!videoFrame) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (CGRectIsEmpty(self.bounds)) {
|
||||
return;
|
||||
}
|
||||
if (CGRectIsEmpty(self.visibleRect)) {
|
||||
return;
|
||||
}
|
||||
if (self.window == nil || self.superview == nil) {
|
||||
return;
|
||||
}
|
||||
if ((self.window.occlusionState & NSWindowOcclusionStateVisible) == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
if (_frames.count >= 5) {
|
||||
[_frames removeAllObjects];
|
||||
[_frames addObject:videoFrame];
|
||||
[self enqueue];
|
||||
return;
|
||||
}
|
||||
|
||||
[_frames addObject:videoFrame];
|
||||
|
||||
[self enqueue];
|
||||
|
||||
}
|
||||
|
||||
-(void)enqueue {
|
||||
if(_frames.count > 0 && !_drawing) {
|
||||
RTCVideoFrame *videoFrame = [_frames firstObject];
|
||||
|
||||
NSValue * rotationOverride = _rotationOverride;
|
||||
|
||||
|
||||
int64_t timeStampNs = videoFrame.timeStampNs;
|
||||
|
||||
__weak VideoMetalView *weakSelf = self;
|
||||
dispatch_block_t completion = ^{
|
||||
__strong VideoMetalView *strongSelf = weakSelf;
|
||||
if (strongSelf && strongSelf->_frames.count > 0) {
|
||||
[strongSelf->_frames removeObjectAtIndex:0];
|
||||
strongSelf->_drawing = false;
|
||||
strongSelf->_lastFrameTimeNs = timeStampNs;
|
||||
if (strongSelf->_deleteForegroundOnNextDrawing) {
|
||||
[strongSelf->_foregroundView removeFromSuperview];
|
||||
strongSelf->_foregroundView = nil;
|
||||
strongSelf->_deleteForegroundOnNextDrawing = false;
|
||||
}
|
||||
[strongSelf enqueue];
|
||||
}
|
||||
};
|
||||
|
||||
_videoFrame = videoFrame;
|
||||
|
||||
self->_drawing = true;
|
||||
|
||||
[_rendererI420 with:^(TGRTCMTLI420Renderer * object) {
|
||||
object.rotationOverride = rotationOverride;
|
||||
[object drawFrame:videoFrame];
|
||||
dispatch_async(dispatch_get_main_queue(), completion);
|
||||
}];
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
- (std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)getSink {
|
||||
assert([NSThread isMainThread]);
|
||||
|
||||
return _sink;
|
||||
}
|
||||
|
||||
- (void)setOnFirstFrameReceived:(void (^ _Nullable)(float))onFirstFrameReceived {
|
||||
_onFirstFrameReceived = [onFirstFrameReceived copy];
|
||||
_firstFrameReceivedReported = false;
|
||||
}
|
||||
|
||||
- (void)setInternalOrientationAndSize:(int)internalOrientation size:(CGSize)size {
|
||||
CGFloat aspect = 1.0f;
|
||||
if (size.width > 1.0f && size.height > 1.0f) {
|
||||
aspect = size.width / size.height;
|
||||
}
|
||||
if (_internalOrientation != internalOrientation || ABS(_internalAspect - aspect) > 0.001) {
|
||||
RTCLogInfo(@"VideoMetalView@%lx orientation: %d, aspect: %f", (intptr_t)self, internalOrientation, (float)aspect);
|
||||
|
||||
_internalOrientation = internalOrientation;
|
||||
_internalAspect = aspect;
|
||||
if (_onOrientationUpdated) {
|
||||
_onOrientationUpdated(internalOrientation, aspect);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated {
|
||||
_onOrientationUpdated = [onOrientationUpdated copy];
|
||||
}
|
||||
|
||||
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated {
|
||||
_onIsMirroredUpdated = [onIsMirroredUpdated copy];
|
||||
}
|
||||
|
||||
- (void)setForceMirrored:(BOOL)forceMirrored {
|
||||
_forceMirrored = forceMirrored;
|
||||
[self setNeedsLayout:YES];
|
||||
}
|
||||
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
#ifndef TGCALLS_VIDEO_SAMPLE_BUFFER_VIEW_H
|
||||
#define TGCALLS_VIDEO_SAMPLE_BUFFER_VIEW_H
|
||||
|
||||
#ifdef WEBRTC_IOS
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <UIKit/UIKit.h>
|
||||
|
||||
#import "api/media_stream_interface.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
@interface VideoSampleBufferView : UIView
|
||||
|
||||
@property(nonatomic) UIViewContentMode videoContentMode;
|
||||
@property(nonatomic, getter=isEnabled) BOOL enabled;
|
||||
@property(nonatomic, nullable) NSValue* rotationOverride;
|
||||
|
||||
@property (nonatomic, readwrite) int internalOrientation;
|
||||
@property (nonatomic, readwrite) CGFloat internalAspect;
|
||||
|
||||
- (void)setSize:(CGSize)size;
|
||||
|
||||
- (std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)getSink;
|
||||
- (void)setOnFirstFrameReceived:(void (^ _Nullable)())onFirstFrameReceived;
|
||||
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated;
|
||||
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated;
|
||||
- (void)addFrame:(const webrtc::VideoFrame&)frame;
|
||||
|
||||
- (void)setCloneTarget:(VideoSampleBufferView * _Nullable)cloneTarget;
|
||||
|
||||
@end
|
||||
|
||||
#endif //WEBRTC_IOS
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,798 @@
|
|||
#import "VideoSampleBufferView.h"
|
||||
|
||||
#import <CoreVideo/CoreVideo.h>
|
||||
|
||||
#import "base/RTCLogging.h"
|
||||
#import "base/RTCVideoFrame.h"
|
||||
#import "base/RTCVideoFrameBuffer.h"
|
||||
#import "TGRTCCVPixelBuffer.h"
|
||||
#include "sdk/objc/native/api/video_frame.h"
|
||||
#include "sdk/objc/native/src/objc_frame_buffer.h"
|
||||
#include "sdk/objc/base/RTCI420Buffer.h"
|
||||
|
||||
#import "api/video/video_sink_interface.h"
|
||||
#import "api/media_stream_interface.h"
|
||||
#import "rtc_base/time_utils.h"
|
||||
|
||||
#import "RTCMTLI420Renderer.h"
|
||||
#import "RTCMTLNV12Renderer.h"
|
||||
#import "RTCMTLRGBRenderer.h"
|
||||
|
||||
#include "libyuv.h"
|
||||
|
||||
#define MTKViewClass NSClassFromString(@"MTKView")
|
||||
#define RTCMTLNV12RendererClass NSClassFromString(@"RTCMTLNV12Renderer")
|
||||
#define RTCMTLI420RendererClass NSClassFromString(@"RTCMTLI420Renderer")
|
||||
#define RTCMTLRGBRendererClass NSClassFromString(@"RTCMTLRGBRenderer")
|
||||
|
||||
namespace {
|
||||
|
||||
class VideoRendererAdapterImpl : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
|
||||
public:
|
||||
VideoRendererAdapterImpl(void (^frameReceived)(std::shared_ptr<webrtc::VideoFrame>)) {
|
||||
_frameReceived = [frameReceived copy];
|
||||
}
|
||||
|
||||
void OnFrame(const webrtc::VideoFrame& nativeVideoFrame) override {
|
||||
@autoreleasepool {
|
||||
if (_frameReceived) {
|
||||
std::shared_ptr<webrtc::VideoFrame> videoFrame = std::make_shared<webrtc::VideoFrame>(nativeVideoFrame);
|
||||
_frameReceived(videoFrame);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
void (^_frameReceived)(std::shared_ptr<webrtc::VideoFrame>);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@interface VideoSampleBufferContentView : UIView
|
||||
|
||||
@property (nonatomic) bool isPaused;
|
||||
|
||||
@end
|
||||
|
||||
@implementation VideoSampleBufferContentView
|
||||
|
||||
+ (Class)layerClass {
|
||||
return [AVSampleBufferDisplayLayer class];
|
||||
}
|
||||
|
||||
- (AVSampleBufferDisplayLayer * _Nonnull)videoLayer {
|
||||
return (AVSampleBufferDisplayLayer *)self.layer;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@interface VideoSampleBufferViewRenderingContext : NSObject {
|
||||
__weak VideoSampleBufferContentView *_sampleBufferView;
|
||||
__weak VideoSampleBufferContentView *_cloneTarget;
|
||||
|
||||
CVPixelBufferPoolRef _pixelBufferPool;
|
||||
int _pixelBufferPoolWidth;
|
||||
int _pixelBufferPoolHeight;
|
||||
|
||||
bool _isBusy;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@implementation VideoSampleBufferViewRenderingContext
|
||||
|
||||
+ (dispatch_queue_t)sharedQueue {
|
||||
static dispatch_queue_t queue;
|
||||
static dispatch_once_t onceToken;
|
||||
dispatch_once(&onceToken, ^{
|
||||
queue = dispatch_queue_create("VideoSampleBufferViewRenderingContext", 0);
|
||||
});
|
||||
return queue;
|
||||
}
|
||||
|
||||
- (instancetype)initWithView:(VideoSampleBufferContentView *)view {
|
||||
self = [super init];
|
||||
if (self != nil) {
|
||||
_sampleBufferView = view;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
_isBusy = true;
|
||||
if (_pixelBufferPool) {
|
||||
CFRelease(_pixelBufferPool);
|
||||
}
|
||||
|
||||
void *opaqueReference = (__bridge_retained void *)_sampleBufferView;
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
__strong VideoSampleBufferContentView *object = (__bridge_transfer VideoSampleBufferContentView *)opaqueReference;
|
||||
[object description];
|
||||
});
|
||||
}
|
||||
|
||||
static bool CopyVideoFrameToNV12PixelBuffer(const webrtc::I420BufferInterface *frameBuffer, CVPixelBufferRef pixelBuffer) {
|
||||
if (!frameBuffer) {
|
||||
return false;
|
||||
}
|
||||
RTC_DCHECK(pixelBuffer);
|
||||
RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixelBuffer), kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
|
||||
RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixelBuffer, 0), frameBuffer->height());
|
||||
RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixelBuffer, 0), frameBuffer->width());
|
||||
|
||||
CVReturn cvRet = CVPixelBufferLockBaseAddress(pixelBuffer, 0);
|
||||
if (cvRet != kCVReturnSuccess) {
|
||||
return false;
|
||||
}
|
||||
uint8_t *dstY = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
|
||||
int dstStrideY = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
|
||||
uint8_t *dstUV = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
|
||||
int dstStrideUV = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
|
||||
// Convert I420 to NV12.
|
||||
int ret = libyuv::I420ToNV12(frameBuffer->DataY(),
|
||||
frameBuffer->StrideY(),
|
||||
frameBuffer->DataU(),
|
||||
frameBuffer->StrideU(),
|
||||
frameBuffer->DataV(),
|
||||
frameBuffer->StrideV(),
|
||||
dstY,
|
||||
dstStrideY,
|
||||
dstUV,
|
||||
dstStrideUV,
|
||||
frameBuffer->width(),
|
||||
frameBuffer->height());
|
||||
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
|
||||
if (ret) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool CopyNV12VideoFrameToNV12PixelBuffer(const webrtc::NV12BufferInterface *frameBuffer, CVPixelBufferRef pixelBuffer) {
|
||||
if (!frameBuffer) {
|
||||
return false;
|
||||
}
|
||||
RTC_DCHECK(pixelBuffer);
|
||||
RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixelBuffer), kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
|
||||
RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixelBuffer, 0), frameBuffer->height());
|
||||
RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixelBuffer, 0), frameBuffer->width());
|
||||
|
||||
CVReturn cvRet = CVPixelBufferLockBaseAddress(pixelBuffer, 0);
|
||||
if (cvRet != kCVReturnSuccess) {
|
||||
return false;
|
||||
}
|
||||
uint8_t *dstY = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
|
||||
int dstStrideY = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
|
||||
uint8_t *dstUV = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
|
||||
int dstStrideUV = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
|
||||
// Convert I420 to NV12.
|
||||
int ret = libyuv::NV12Copy(frameBuffer->DataY(), frameBuffer->StrideY(), frameBuffer->DataUV(), frameBuffer->StrideUV(), dstY, dstStrideY, dstUV, dstStrideUV, frameBuffer->width(), frameBuffer->height());
|
||||
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
|
||||
if (ret) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
- (CVPixelBufferPoolRef)createPixelBufferPoolWithWidth:(int32_t)width height:(int32_t)height pixelFormat:(FourCharCode)pixelFormat maxBufferCount:(int32_t) maxBufferCount {
|
||||
CVPixelBufferPoolRef outputPool = NULL;
|
||||
|
||||
NSDictionary *sourcePixelBufferOptions = @{
|
||||
(id)kCVPixelBufferPixelFormatTypeKey : @(pixelFormat),
|
||||
(id)kCVPixelBufferWidthKey : @(width),
|
||||
(id)kCVPixelBufferHeightKey : @(height),
|
||||
(id)kCVPixelBufferIOSurfacePropertiesKey : @{}
|
||||
};
|
||||
|
||||
NSDictionary *pixelBufferPoolOptions = @{ (id)kCVPixelBufferPoolMinimumBufferCountKey : @(maxBufferCount) };
|
||||
CVPixelBufferPoolCreate(kCFAllocatorDefault, (__bridge CFDictionaryRef)pixelBufferPoolOptions, (__bridge CFDictionaryRef)sourcePixelBufferOptions, &outputPool);
|
||||
|
||||
return outputPool;
|
||||
}
|
||||
|
||||
- (CMSampleBufferRef)createSampleBufferFromI420Buffer:(const webrtc::I420BufferInterface *)buffer {
|
||||
if (!buffer) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
NSMutableDictionary *ioSurfaceProperties = [[NSMutableDictionary alloc] init];
|
||||
//ioSurfaceProperties[@"IOSurfaceIsGlobal"] = @(true);
|
||||
|
||||
NSMutableDictionary *options = [[NSMutableDictionary alloc] init];
|
||||
//options[(__bridge NSString *)kCVPixelBufferBytesPerRowAlignmentKey] = @(buffer.strideY);
|
||||
options[(__bridge NSString *)kCVPixelBufferIOSurfacePropertiesKey] = ioSurfaceProperties;
|
||||
|
||||
CVPixelBufferRef pixelBufferRef = nil;
|
||||
|
||||
if (!(_pixelBufferPool != nil && _pixelBufferPoolWidth == buffer->width() && _pixelBufferPoolHeight == buffer->height())) {
|
||||
if (_pixelBufferPool) {
|
||||
CFRelease(_pixelBufferPool);
|
||||
_pixelBufferPool = nil;
|
||||
}
|
||||
_pixelBufferPool = [self createPixelBufferPoolWithWidth:buffer->width() height:buffer->height() pixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange maxBufferCount:10];
|
||||
_pixelBufferPoolWidth = buffer->width();
|
||||
_pixelBufferPoolHeight = buffer->height();
|
||||
}
|
||||
|
||||
if (_pixelBufferPool != nil && _pixelBufferPoolWidth == buffer->width() && _pixelBufferPoolHeight == buffer->height()) {
|
||||
CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(kCFAllocatorDefault, _pixelBufferPool, nil, &pixelBufferRef);
|
||||
} else {
|
||||
CVPixelBufferCreate(
|
||||
kCFAllocatorDefault,
|
||||
buffer->width(),
|
||||
buffer->height(),
|
||||
kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
|
||||
(__bridge CFDictionaryRef)options,
|
||||
&pixelBufferRef
|
||||
);
|
||||
}
|
||||
|
||||
if (pixelBufferRef == nil) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
CopyVideoFrameToNV12PixelBuffer(buffer, pixelBufferRef);
|
||||
|
||||
CMVideoFormatDescriptionRef formatRef = nil;
|
||||
OSStatus status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBufferRef, &formatRef);
|
||||
if (status != 0) {
|
||||
return nil;
|
||||
}
|
||||
if (formatRef == nil) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
CMSampleTimingInfo timingInfo;
|
||||
timingInfo.duration = CMTimeMake(1, 30);
|
||||
timingInfo.presentationTimeStamp = CMTimeMake(0, 30);
|
||||
timingInfo.decodeTimeStamp = CMTimeMake(0, 30);
|
||||
|
||||
CMSampleBufferRef sampleBuffer = nil;
|
||||
OSStatus bufferStatus = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, pixelBufferRef, formatRef, &timingInfo, &sampleBuffer);
|
||||
|
||||
CFRelease(formatRef);
|
||||
CFRelease(pixelBufferRef);
|
||||
|
||||
if (bufferStatus != noErr) {
|
||||
return nil;
|
||||
}
|
||||
if (sampleBuffer == nil) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
NSArray *attachments = (__bridge NSArray *)CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true);
|
||||
NSMutableDictionary *dict = (NSMutableDictionary *)attachments[0];
|
||||
|
||||
dict[(__bridge NSString *)kCMSampleAttachmentKey_DisplayImmediately] = @(true);
|
||||
|
||||
return sampleBuffer;
|
||||
}
|
||||
|
||||
- (CMSampleBufferRef)createSampleBufferFromNV12Buffer:(const webrtc::NV12BufferInterface *)buffer {
|
||||
if (!buffer) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
NSMutableDictionary *ioSurfaceProperties = [[NSMutableDictionary alloc] init];
|
||||
//ioSurfaceProperties[@"IOSurfaceIsGlobal"] = @(true);
|
||||
|
||||
NSMutableDictionary *options = [[NSMutableDictionary alloc] init];
|
||||
//options[(__bridge NSString *)kCVPixelBufferBytesPerRowAlignmentKey] = @(buffer.strideY);
|
||||
options[(__bridge NSString *)kCVPixelBufferIOSurfacePropertiesKey] = ioSurfaceProperties;
|
||||
|
||||
CVPixelBufferRef pixelBufferRef = nil;
|
||||
|
||||
if (!(_pixelBufferPool != nil && _pixelBufferPoolWidth == buffer->width() && _pixelBufferPoolHeight == buffer->height())) {
|
||||
if (_pixelBufferPool) {
|
||||
CFRelease(_pixelBufferPool);
|
||||
_pixelBufferPool = nil;
|
||||
}
|
||||
_pixelBufferPool = [self createPixelBufferPoolWithWidth:buffer->width() height:buffer->height() pixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange maxBufferCount:10];
|
||||
_pixelBufferPoolWidth = buffer->width();
|
||||
_pixelBufferPoolHeight = buffer->height();
|
||||
}
|
||||
|
||||
if (_pixelBufferPool != nil && _pixelBufferPoolWidth == buffer->width() && _pixelBufferPoolHeight == buffer->height()) {
|
||||
CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(kCFAllocatorDefault, _pixelBufferPool, nil, &pixelBufferRef);
|
||||
} else {
|
||||
CVPixelBufferCreate(
|
||||
kCFAllocatorDefault,
|
||||
buffer->width(),
|
||||
buffer->height(),
|
||||
kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
|
||||
(__bridge CFDictionaryRef)options,
|
||||
&pixelBufferRef
|
||||
);
|
||||
}
|
||||
|
||||
if (pixelBufferRef == nil) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
CopyNV12VideoFrameToNV12PixelBuffer(buffer, pixelBufferRef);
|
||||
|
||||
CMVideoFormatDescriptionRef formatRef = nil;
|
||||
OSStatus status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBufferRef, &formatRef);
|
||||
if (status != 0) {
|
||||
return nil;
|
||||
}
|
||||
if (formatRef == nil) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
CMSampleTimingInfo timingInfo;
|
||||
timingInfo.duration = CMTimeMake(1, 30);
|
||||
timingInfo.presentationTimeStamp = CMTimeMake(0, 30);
|
||||
timingInfo.decodeTimeStamp = CMTimeMake(0, 30);
|
||||
|
||||
CMSampleBufferRef sampleBuffer = nil;
|
||||
OSStatus bufferStatus = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, pixelBufferRef, formatRef, &timingInfo, &sampleBuffer);
|
||||
|
||||
CFRelease(formatRef);
|
||||
CFRelease(pixelBufferRef);
|
||||
|
||||
if (bufferStatus != noErr) {
|
||||
return nil;
|
||||
}
|
||||
if (sampleBuffer == nil) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
NSArray *attachments = (__bridge NSArray *)CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true);
|
||||
NSMutableDictionary *dict = (NSMutableDictionary *)attachments[0];
|
||||
|
||||
dict[(__bridge NSString *)kCMSampleAttachmentKey_DisplayImmediately] = @(true);
|
||||
|
||||
return sampleBuffer;
|
||||
}
|
||||
|
||||
- (CMSampleBufferRef)createSampleBufferFromPixelBuffer:(CVPixelBufferRef)pixelBufferRef {
|
||||
CMVideoFormatDescriptionRef formatRef = nil;
|
||||
OSStatus status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBufferRef, &formatRef);
|
||||
if (status != 0) {
|
||||
return nil;
|
||||
}
|
||||
if (formatRef == nil) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
CMSampleTimingInfo timingInfo;
|
||||
timingInfo.duration = CMTimeMake(1, 30);
|
||||
timingInfo.presentationTimeStamp = CMTimeMake(0, 30);
|
||||
timingInfo.decodeTimeStamp = CMTimeMake(0, 30);
|
||||
|
||||
CMSampleBufferRef sampleBuffer = nil;
|
||||
OSStatus bufferStatus = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, pixelBufferRef, formatRef, &timingInfo, &sampleBuffer);
|
||||
|
||||
CFRelease(formatRef);
|
||||
|
||||
if (bufferStatus != noErr) {
|
||||
return nil;
|
||||
}
|
||||
if (sampleBuffer == nil) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
NSArray *attachments = (__bridge NSArray *)CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true);
|
||||
NSMutableDictionary *dict = (NSMutableDictionary *)attachments[0];
|
||||
|
||||
dict[(__bridge NSString *)kCMSampleAttachmentKey_DisplayImmediately] = @(true);
|
||||
|
||||
return sampleBuffer;
|
||||
}
|
||||
|
||||
- (void)renderFrameIfReady:(std::shared_ptr<webrtc::VideoFrame>)frame {
|
||||
if (_isBusy) {
|
||||
return;
|
||||
}
|
||||
VideoSampleBufferContentView *sampleBufferView = _sampleBufferView;
|
||||
if (!sampleBufferView) {
|
||||
return;
|
||||
}
|
||||
|
||||
AVSampleBufferDisplayLayer *layer = [sampleBufferView videoLayer];
|
||||
|
||||
VideoSampleBufferContentView *cloneTarget = _cloneTarget;
|
||||
__weak AVSampleBufferDisplayLayer *cloneLayer = nil;
|
||||
if (cloneTarget) {
|
||||
cloneLayer = [cloneTarget videoLayer];
|
||||
}
|
||||
|
||||
_isBusy = true;
|
||||
dispatch_async([VideoSampleBufferViewRenderingContext sharedQueue], ^{
|
||||
__strong AVSampleBufferDisplayLayer *strongCloneLayer = cloneLayer;
|
||||
|
||||
switch (frame->video_frame_buffer()->type()) {
|
||||
case webrtc::VideoFrameBuffer::Type::kI420:
|
||||
case webrtc::VideoFrameBuffer::Type::kI420A: {
|
||||
CMSampleBufferRef sampleBuffer = [self createSampleBufferFromI420Buffer:frame->video_frame_buffer()->GetI420()];
|
||||
if (sampleBuffer) {
|
||||
[layer enqueueSampleBuffer:sampleBuffer];
|
||||
[cloneLayer enqueueSampleBuffer:sampleBuffer];
|
||||
|
||||
if ([layer status] == AVQueuedSampleBufferRenderingStatusFailed) {
|
||||
[layer flush];
|
||||
}
|
||||
if ([cloneLayer status] == AVQueuedSampleBufferRenderingStatusFailed) {
|
||||
[cloneLayer flush];
|
||||
}
|
||||
|
||||
CFRelease(sampleBuffer);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case webrtc::VideoFrameBuffer::Type::kNV12: {
|
||||
CMSampleBufferRef sampleBuffer = [self createSampleBufferFromNV12Buffer:(webrtc::NV12BufferInterface *)frame->video_frame_buffer().get()];
|
||||
if (sampleBuffer) {
|
||||
[layer enqueueSampleBuffer:sampleBuffer];
|
||||
[cloneLayer enqueueSampleBuffer:sampleBuffer];
|
||||
|
||||
CFRelease(sampleBuffer);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case webrtc::VideoFrameBuffer::Type::kNative: {
|
||||
id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> nativeBuffer = static_cast<webrtc::ObjCFrameBuffer *>(frame->video_frame_buffer().get())->wrapped_frame_buffer();
|
||||
if ([nativeBuffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) {
|
||||
RTCCVPixelBuffer *pixelBuffer = (RTCCVPixelBuffer *)nativeBuffer;
|
||||
CMSampleBufferRef sampleBuffer = [self createSampleBufferFromPixelBuffer:pixelBuffer.pixelBuffer];
|
||||
if (sampleBuffer) {
|
||||
[layer enqueueSampleBuffer:sampleBuffer];
|
||||
[cloneLayer enqueueSampleBuffer:sampleBuffer];
|
||||
|
||||
CFRelease(sampleBuffer);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
_isBusy = false;
|
||||
|
||||
void *opaqueReference = (__bridge_retained void *)layer;
|
||||
void *cloneLayerReference = (__bridge_retained void *)strongCloneLayer;
|
||||
strongCloneLayer = nil;
|
||||
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
__strong AVSampleBufferDisplayLayer *object = (__bridge_transfer AVSampleBufferDisplayLayer *)opaqueReference;
|
||||
object = nil;
|
||||
|
||||
__strong AVSampleBufferDisplayLayer *cloneObject = (__bridge_transfer AVSampleBufferDisplayLayer *)cloneLayerReference;
|
||||
cloneObject = nil;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
- (void)setCloneTarget:(VideoSampleBufferContentView * _Nullable)cloneTarget {
|
||||
_cloneTarget = cloneTarget;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@protocol ClonePortalView
|
||||
|
||||
- (void)setSourceView:(UIView * _Nullable)sourceView;
|
||||
- (void)setHidesSourceView:(bool)arg1;
|
||||
- (void)setMatchesAlpha:(bool)arg1;
|
||||
- (void)setMatchesPosition:(bool)arg1;
|
||||
- (void)setMatchesTransform:(bool)arg1;
|
||||
|
||||
@end
|
||||
|
||||
@interface VideoSampleBufferView () {
|
||||
VideoSampleBufferContentView *_sampleBufferView;
|
||||
VideoSampleBufferViewRenderingContext *_renderingContext;
|
||||
|
||||
std::shared_ptr<webrtc::VideoFrame> _videoFrame;
|
||||
std::shared_ptr<webrtc::VideoFrame> _stashedVideoFrame;
|
||||
|
||||
int _isWaitingForLayoutFrameCount;
|
||||
bool _didStartWaitingForLayout;
|
||||
CGSize _videoFrameSize;
|
||||
int64_t _lastFrameTimeNs;
|
||||
|
||||
CGSize _currentSize;
|
||||
std::shared_ptr<VideoRendererAdapterImpl> _sink;
|
||||
|
||||
void (^_onFirstFrameReceived)();
|
||||
bool _firstFrameReceivedReported;
|
||||
|
||||
void (^_onOrientationUpdated)(int, CGFloat);
|
||||
|
||||
void (^_onIsMirroredUpdated)(bool);
|
||||
|
||||
bool _didSetShouldBeMirrored;
|
||||
bool _shouldBeMirrored;
|
||||
|
||||
__weak VideoSampleBufferView *_cloneTarget;
|
||||
UIView<ClonePortalView> *_portalView;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@implementation VideoSampleBufferView
|
||||
|
||||
- (instancetype)initWithFrame:(CGRect)frameRect {
|
||||
self = [super initWithFrame:frameRect];
|
||||
if (self) {
|
||||
[self configure];
|
||||
|
||||
_enabled = true;
|
||||
|
||||
_currentSize = CGSizeZero;
|
||||
_rotationOverride = @(RTCVideoRotation_0);
|
||||
|
||||
__weak VideoSampleBufferView *weakSelf = self;
|
||||
_sink.reset(new VideoRendererAdapterImpl(^(std::shared_ptr<webrtc::VideoFrame> videoFrame) {
|
||||
if (!videoFrame) {
|
||||
return;
|
||||
}
|
||||
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
__strong VideoSampleBufferView *strongSelf = weakSelf;
|
||||
if (strongSelf == nil) {
|
||||
return;
|
||||
}
|
||||
|
||||
[strongSelf renderFrame:videoFrame];
|
||||
});
|
||||
}));
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
}
|
||||
|
||||
- (void)setEnabled:(BOOL)enabled {
|
||||
if (_enabled != enabled) {
|
||||
_enabled = enabled;
|
||||
_sampleBufferView.isPaused = !enabled;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)setVideoContentMode:(UIViewContentMode)mode {
|
||||
_videoContentMode = mode;
|
||||
}
|
||||
|
||||
#pragma mark - Private
|
||||
|
||||
- (void)configure {
|
||||
_sampleBufferView = [[VideoSampleBufferContentView alloc] init];
|
||||
[self addSubview:_sampleBufferView];
|
||||
|
||||
_renderingContext = [[VideoSampleBufferViewRenderingContext alloc] initWithView:_sampleBufferView];
|
||||
|
||||
_videoFrameSize = CGSizeZero;
|
||||
}
|
||||
|
||||
- (void)layoutSubviews {
|
||||
[super layoutSubviews];
|
||||
|
||||
CGRect bounds = self.bounds;
|
||||
|
||||
if (!CGRectEqualToRect(_sampleBufferView.frame, bounds)) {
|
||||
_sampleBufferView.frame = bounds;
|
||||
_portalView.frame = bounds;
|
||||
}
|
||||
|
||||
if (_didStartWaitingForLayout) {
|
||||
_didStartWaitingForLayout = false;
|
||||
_isWaitingForLayoutFrameCount = 0;
|
||||
if (_stashedVideoFrame != nil) {
|
||||
_videoFrame = _stashedVideoFrame;
|
||||
_stashedVideoFrame = nil;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#pragma mark -
|
||||
|
||||
- (void)setRotationOverride:(NSValue *)rotationOverride {
|
||||
_rotationOverride = rotationOverride;
|
||||
|
||||
[self setNeedsLayout];
|
||||
}
|
||||
|
||||
- (RTCVideoRotation)frameRotation {
|
||||
if (_rotationOverride) {
|
||||
RTCVideoRotation rotation;
|
||||
if (@available(iOS 11, *)) {
|
||||
[_rotationOverride getValue:&rotation size:sizeof(rotation)];
|
||||
} else {
|
||||
[_rotationOverride getValue:&rotation];
|
||||
}
|
||||
return rotation;
|
||||
}
|
||||
|
||||
if (_videoFrame) {
|
||||
switch (_videoFrame->rotation()) {
|
||||
case webrtc::kVideoRotation_0:
|
||||
return RTCVideoRotation_0;
|
||||
case webrtc::kVideoRotation_90:
|
||||
return RTCVideoRotation_90;
|
||||
case webrtc::kVideoRotation_180:
|
||||
return RTCVideoRotation_180;
|
||||
case webrtc::kVideoRotation_270:
|
||||
return RTCVideoRotation_270;
|
||||
default:
|
||||
return RTCVideoRotation_0;
|
||||
}
|
||||
} else {
|
||||
return RTCVideoRotation_0;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)setSize:(CGSize)size {
|
||||
assert([NSThread isMainThread]);
|
||||
}
|
||||
|
||||
- (void)renderFrame:(std::shared_ptr<webrtc::VideoFrame>)frame {
|
||||
[self renderFrameInternal:frame skipRendering:false];
|
||||
VideoSampleBufferView *cloneTarget = _cloneTarget;
|
||||
if (cloneTarget) {
|
||||
[cloneTarget renderFrameInternal:frame skipRendering:true];
|
||||
}
|
||||
}
|
||||
|
||||
- (void)renderFrameInternal:(std::shared_ptr<webrtc::VideoFrame>)frame skipRendering:(bool)skipRendering {
|
||||
assert([NSThread isMainThread]);
|
||||
|
||||
CGSize size = CGSizeMake(frame->width(), frame->height());
|
||||
if (!CGSizeEqualToSize(size, _currentSize)) {
|
||||
_currentSize = size;
|
||||
[self setSize:size];
|
||||
}
|
||||
|
||||
int mappedValue = 0;
|
||||
switch (RTCVideoRotation(frame->rotation())) {
|
||||
case RTCVideoRotation_90: {
|
||||
mappedValue = 1;
|
||||
break;
|
||||
}
|
||||
case RTCVideoRotation_180: {
|
||||
mappedValue = 2;
|
||||
break;
|
||||
}
|
||||
case RTCVideoRotation_270: {
|
||||
mappedValue = 3;
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
mappedValue = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
[self setInternalOrientationAndSize:mappedValue size:size];
|
||||
|
||||
if (!_firstFrameReceivedReported && _onFirstFrameReceived) {
|
||||
_firstFrameReceivedReported = true;
|
||||
_onFirstFrameReceived();
|
||||
}
|
||||
|
||||
if (!self.isEnabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!frame) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (_isWaitingForLayoutFrameCount > 0) {
|
||||
_stashedVideoFrame = frame;
|
||||
_isWaitingForLayoutFrameCount--;
|
||||
return;
|
||||
}
|
||||
if (!_didStartWaitingForLayout) {
|
||||
if (_videoFrame && _videoFrame->width() > 0 && _videoFrame->height() > 0 && frame->width() > 0 && frame->height() > 0) {
|
||||
float previousAspect = ((float)_videoFrame->width()) / ((float)_videoFrame->height());
|
||||
float updatedAspect = ((float)frame->width()) / ((float)frame->height());
|
||||
if ((previousAspect < 1.0f) != (updatedAspect < 1.0f)) {
|
||||
_stashedVideoFrame = frame;
|
||||
_didStartWaitingForLayout = true;
|
||||
_isWaitingForLayoutFrameCount = 5;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_videoFrame = frame;
|
||||
|
||||
if (!skipRendering) {
|
||||
[_renderingContext renderFrameIfReady:frame];
|
||||
}
|
||||
}
|
||||
|
||||
- (std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)getSink {
|
||||
assert([NSThread isMainThread]);
|
||||
|
||||
return _sink;
|
||||
}
|
||||
|
||||
- (void)addFrame:(const webrtc::VideoFrame&)frame {
|
||||
std::shared_ptr<webrtc::VideoFrame> videoFrame = std::make_shared<webrtc::VideoFrame>(frame);
|
||||
[self renderFrame:videoFrame];
|
||||
}
|
||||
|
||||
static NSString * _Nonnull shiftString(NSString *string, int key) {
|
||||
NSMutableString *result = [[NSMutableString alloc] init];
|
||||
|
||||
for (int i = 0; i < (int)[string length]; i++) {
|
||||
unichar c = [string characterAtIndex:i];
|
||||
c += key;
|
||||
[result appendString:[NSString stringWithCharacters:&c length:1]];
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/*- (void)addAsCloneTarget:(VideoSampleBufferView *)sourceView {
|
||||
if (_portalView) {
|
||||
[_portalView setSourceView:nil];
|
||||
[_portalView removeFromSuperview];
|
||||
}
|
||||
|
||||
if (!sourceView) {
|
||||
return;
|
||||
}
|
||||
static Class portalViewClass = nil;
|
||||
static dispatch_once_t onceToken;
|
||||
dispatch_once(&onceToken, ^{
|
||||
portalViewClass = NSClassFromString(@"_UIPortalView");
|
||||
});
|
||||
if (portalViewClass) {
|
||||
_portalView = (UIView<ClonePortalView> *)[[portalViewClass alloc] init];
|
||||
_portalView.frame = sourceView->_sampleBufferView.frame;
|
||||
_portalView.backgroundColor = [UIColor redColor];
|
||||
[_portalView setSourceView:sourceView->_sampleBufferView];
|
||||
[_portalView setHidesSourceView:true];
|
||||
[_portalView setMatchesAlpha:false];
|
||||
[_portalView setMatchesPosition:false];
|
||||
[_portalView setMatchesTransform:false];
|
||||
|
||||
[self addSubview:_portalView];
|
||||
}
|
||||
}*/
|
||||
|
||||
- (void)setCloneTarget:(VideoSampleBufferView * _Nullable)cloneTarget {
|
||||
_cloneTarget = cloneTarget;
|
||||
if (cloneTarget) {
|
||||
[_renderingContext setCloneTarget:cloneTarget->_sampleBufferView];
|
||||
//[cloneTarget addAsCloneTarget:self];
|
||||
}
|
||||
}
|
||||
|
||||
- (void)setOnFirstFrameReceived:(void (^ _Nullable)())onFirstFrameReceived {
|
||||
_onFirstFrameReceived = [onFirstFrameReceived copy];
|
||||
_firstFrameReceivedReported = false;
|
||||
}
|
||||
|
||||
- (void)setInternalOrientationAndSize:(int)internalOrientation size:(CGSize)size {
|
||||
CGFloat aspect = 1.0f;
|
||||
if (size.width > 1.0f && size.height > 1.0f) {
|
||||
aspect = size.width / size.height;
|
||||
}
|
||||
if (_internalOrientation != internalOrientation || ABS(_internalAspect - aspect) > 0.001) {
|
||||
RTCLogInfo(@"VideoSampleBufferView@%lx orientation: %d, aspect: %f", (intptr_t)self, internalOrientation, (float)aspect);
|
||||
|
||||
_internalOrientation = internalOrientation;
|
||||
_internalAspect = aspect;
|
||||
if (_onOrientationUpdated) {
|
||||
_onOrientationUpdated(internalOrientation, aspect);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated {
|
||||
_onOrientationUpdated = [onOrientationUpdated copy];
|
||||
}
|
||||
|
||||
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated {
|
||||
_onIsMirroredUpdated = [onIsMirroredUpdated copy];
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
#ifndef TGCALLS_VIDEO_SAMPLE_BUFFER_VIEW_H
|
||||
#define TGCALLS_VIDEO_SAMPLE_BUFFER_VIEW_H
|
||||
|
||||
#ifdef WEBRTC_MAC
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <Cocoa/Cocoa.h>
|
||||
|
||||
#import "api/media_stream_interface.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
@interface VideoSampleBufferView : NSView
|
||||
|
||||
|
||||
|
||||
@property(nonatomic) CALayerContentsGravity _Nullable videoContentMode;
|
||||
@property(nonatomic, getter=isEnabled) BOOL enabled;
|
||||
@property(nonatomic, nullable) NSValue* rotationOverride;
|
||||
|
||||
@property (nonatomic, readwrite) int internalOrientation;
|
||||
@property (nonatomic, readwrite) CGFloat internalAspect;
|
||||
|
||||
- (void)setSize:(CGSize)size;
|
||||
|
||||
- (std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)getSink;
|
||||
- (void)setOnFirstFrameReceived:(void (^ _Nullable)())onFirstFrameReceived;
|
||||
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated;
|
||||
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated;
|
||||
|
||||
+(BOOL)isAvailable;
|
||||
|
||||
@end
|
||||
|
||||
#endif //WEBRTC_MAC
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,427 @@
|
|||
#import "VideoSampleBufferViewMac.h"
|
||||
|
||||
#import <CoreVideo/CoreVideo.h>
|
||||
|
||||
#import "base/RTCLogging.h"
|
||||
#import "base/RTCVideoFrame.h"
|
||||
#import "base/RTCVideoFrameBuffer.h"
|
||||
#import "TGRTCCVPixelBuffer.h"
|
||||
#include "sdk/objc/native/api/video_frame.h"
|
||||
#include "sdk/objc/native/src/objc_frame_buffer.h"
|
||||
#include "sdk/objc/base/RTCI420Buffer.h"
|
||||
|
||||
#import "api/video/video_sink_interface.h"
|
||||
#import "api/media_stream_interface.h"
|
||||
#import "rtc_base/time_utils.h"
|
||||
|
||||
#import "RTCMTLI420Renderer.h"
|
||||
#import "RTCMTLNV12Renderer.h"
|
||||
#import "RTCMTLRGBRenderer.h"
|
||||
|
||||
#include "libyuv.h"
|
||||
|
||||
#define MTKViewClass NSClassFromString(@"MTKView")
|
||||
#define RTCMTLNV12RendererClass NSClassFromString(@"RTCMTLNV12Renderer")
|
||||
#define RTCMTLI420RendererClass NSClassFromString(@"RTCMTLI420Renderer")
|
||||
#define RTCMTLRGBRendererClass NSClassFromString(@"RTCMTLRGBRenderer")
|
||||
|
||||
namespace {
|
||||
|
||||
static RTCVideoFrame *customToObjCVideoFrame(const webrtc::VideoFrame &frame, RTCVideoRotation &rotation) {
|
||||
rotation = RTCVideoRotation(frame.rotation());
|
||||
RTCVideoFrame *videoFrame =
|
||||
[[RTCVideoFrame alloc] initWithBuffer:webrtc::ToObjCVideoFrameBuffer(frame.video_frame_buffer())
|
||||
rotation:rotation
|
||||
timeStampNs:frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec];
|
||||
videoFrame.timeStamp = frame.timestamp();
|
||||
|
||||
return videoFrame;
|
||||
}
|
||||
|
||||
class VideoRendererAdapterImpl : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
|
||||
public:
|
||||
VideoRendererAdapterImpl(void (^frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation)) {
|
||||
_frameReceived = [frameReceived copy];
|
||||
}
|
||||
|
||||
void OnFrame(const webrtc::VideoFrame& nativeVideoFrame) override {
|
||||
@autoreleasepool {
|
||||
RTCVideoRotation rotation = RTCVideoRotation_90;
|
||||
RTCVideoFrame* videoFrame = customToObjCVideoFrame(nativeVideoFrame, rotation);
|
||||
|
||||
//CGSize currentSize = (videoFrame.rotation % 180 == 0) ? CGSizeMake(videoFrame.width, videoFrame.height) : CGSizeMake(videoFrame.height, videoFrame.width);
|
||||
CGSize currentSize = CGSizeMake(videoFrame.width, videoFrame.height);
|
||||
|
||||
if (_frameReceived) {
|
||||
_frameReceived(currentSize, videoFrame, rotation);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
void (^_frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@interface VideoSampleBufferView () {
|
||||
AVSampleBufferDisplayLayer *_sampleBufferLayer;
|
||||
|
||||
RTCVideoFrame *_videoFrame;
|
||||
RTCVideoFrame *_stashedVideoFrame;
|
||||
|
||||
int _isWaitingForLayoutFrameCount;
|
||||
bool _didStartWaitingForLayout;
|
||||
CGSize _videoFrameSize;
|
||||
int64_t _lastFrameTimeNs;
|
||||
|
||||
CGSize _currentSize;
|
||||
std::shared_ptr<VideoRendererAdapterImpl> _sink;
|
||||
|
||||
void (^_onFirstFrameReceived)();
|
||||
bool _firstFrameReceivedReported;
|
||||
|
||||
void (^_onOrientationUpdated)(int, CGFloat);
|
||||
|
||||
void (^_onIsMirroredUpdated)(bool);
|
||||
|
||||
bool _didSetShouldBeMirrored;
|
||||
bool _shouldBeMirrored;
|
||||
|
||||
CVPixelBufferPoolRef _pixelBufferPool;
|
||||
int _pixelBufferPoolWidth;
|
||||
int _pixelBufferPoolHeight;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@implementation VideoSampleBufferView
|
||||
|
||||
- (instancetype)initWithFrame:(CGRect)frameRect {
|
||||
self = [super initWithFrame:frameRect];
|
||||
if (self) {
|
||||
[self configure];
|
||||
|
||||
_enabled = true;
|
||||
_currentSize = CGSizeZero;
|
||||
_rotationOverride = @(RTCVideoRotation_0);
|
||||
|
||||
__weak VideoSampleBufferView *weakSelf = self;
|
||||
_sink.reset(new VideoRendererAdapterImpl(^(CGSize size, RTCVideoFrame *videoFrame, RTCVideoRotation rotation) {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
__strong VideoSampleBufferView *strongSelf = weakSelf;
|
||||
if (strongSelf == nil) {
|
||||
return;
|
||||
}
|
||||
if (!CGSizeEqualToSize(size, strongSelf->_currentSize)) {
|
||||
strongSelf->_currentSize = size;
|
||||
[strongSelf setSize:size];
|
||||
}
|
||||
|
||||
int mappedValue = 0;
|
||||
switch (rotation) {
|
||||
case RTCVideoRotation_90:
|
||||
mappedValue = 1;
|
||||
break;
|
||||
case RTCVideoRotation_180:
|
||||
mappedValue = 2;
|
||||
break;
|
||||
case RTCVideoRotation_270:
|
||||
mappedValue = 3;
|
||||
break;
|
||||
default:
|
||||
mappedValue = 0;
|
||||
break;
|
||||
}
|
||||
[strongSelf setInternalOrientationAndSize:mappedValue size:size];
|
||||
|
||||
[strongSelf renderFrame:videoFrame];
|
||||
});
|
||||
}));
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
if (_pixelBufferPool) {
|
||||
CFRelease(_pixelBufferPool);
|
||||
}
|
||||
}
|
||||
|
||||
- (void)setEnabled:(BOOL)enabled {
|
||||
_enabled = enabled;
|
||||
}
|
||||
|
||||
- (void)setVideoContentMode:(CALayerContentsGravity)mode {
|
||||
_videoContentMode = mode;
|
||||
}
|
||||
|
||||
#pragma mark - Private
|
||||
|
||||
- (void)configure {
|
||||
self.wantsLayer = YES;
|
||||
_sampleBufferLayer = [[AVSampleBufferDisplayLayer alloc] init];
|
||||
self.layer = _sampleBufferLayer;
|
||||
// [self.layer addSublayer:_sampleBufferLayer];
|
||||
|
||||
|
||||
_videoFrameSize = CGSizeZero;
|
||||
}
|
||||
|
||||
- (void)layout {
|
||||
[super layout];
|
||||
|
||||
CGRect bounds = self.bounds;
|
||||
[CATransaction begin];
|
||||
[CATransaction setAnimationDuration:0];
|
||||
[CATransaction setDisableActions:true];
|
||||
_sampleBufferLayer.frame = bounds;
|
||||
[CATransaction commit];
|
||||
|
||||
if (_didStartWaitingForLayout) {
|
||||
_didStartWaitingForLayout = false;
|
||||
_isWaitingForLayoutFrameCount = 0;
|
||||
if (_stashedVideoFrame != nil) {
|
||||
_videoFrame = _stashedVideoFrame;
|
||||
_stashedVideoFrame = nil;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#pragma mark -
|
||||
|
||||
- (void)setRotationOverride:(NSValue *)rotationOverride {
|
||||
_rotationOverride = rotationOverride;
|
||||
|
||||
[self setNeedsLayout:YES];
|
||||
}
|
||||
|
||||
- (RTCVideoRotation)frameRotation {
|
||||
if (_rotationOverride) {
|
||||
RTCVideoRotation rotation;
|
||||
if (@available(iOS 11, *)) {
|
||||
[_rotationOverride getValue:&rotation size:sizeof(rotation)];
|
||||
} else {
|
||||
[_rotationOverride getValue:&rotation];
|
||||
}
|
||||
return rotation;
|
||||
}
|
||||
|
||||
return _videoFrame.rotation;
|
||||
}
|
||||
|
||||
#pragma mark - RTCVideoRenderer
|
||||
|
||||
bool CopyVideoFrameToNV12PixelBuffer(id<RTC_OBJC_TYPE(RTCI420Buffer)> frameBuffer,
|
||||
CVPixelBufferRef pixelBuffer) {
|
||||
RTC_DCHECK(pixelBuffer);
|
||||
RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixelBuffer), kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
|
||||
RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixelBuffer, 0), frameBuffer.height);
|
||||
RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixelBuffer, 0), frameBuffer.width);
|
||||
|
||||
CVReturn cvRet = CVPixelBufferLockBaseAddress(pixelBuffer, 0);
|
||||
if (cvRet != kCVReturnSuccess) {
|
||||
return false;
|
||||
}
|
||||
uint8_t *dstY = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
|
||||
size_t dstStrideY = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
|
||||
uint8_t *dstUV = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
|
||||
size_t dstStrideUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
|
||||
|
||||
int ret = libyuv::I420ToNV12(frameBuffer.dataY,
|
||||
frameBuffer.strideY,
|
||||
frameBuffer.dataU,
|
||||
frameBuffer.strideU,
|
||||
frameBuffer.dataV,
|
||||
frameBuffer.strideV,
|
||||
dstY,
|
||||
(int)dstStrideY,
|
||||
dstUV,
|
||||
(int)dstStrideUV,
|
||||
frameBuffer.width,
|
||||
frameBuffer.height);
|
||||
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
|
||||
if (ret) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
+ (CVPixelBufferPoolRef)createPixelBufferPoolWithWidth:(int32_t)width height:(int32_t)height pixelFormat:(FourCharCode)pixelFormat maxBufferCount:(int32_t) maxBufferCount {
|
||||
CVPixelBufferPoolRef outputPool = NULL;
|
||||
|
||||
NSDictionary *sourcePixelBufferOptions = @{
|
||||
(id)kCVPixelBufferPixelFormatTypeKey : @(pixelFormat),
|
||||
(id)kCVPixelBufferWidthKey : @(width),
|
||||
(id)kCVPixelBufferHeightKey : @(height),
|
||||
(id)kCVPixelBufferIOSurfacePropertiesKey : @{}
|
||||
};
|
||||
|
||||
NSDictionary *pixelBufferPoolOptions = @{ (id)kCVPixelBufferPoolMinimumBufferCountKey : @(maxBufferCount) };
|
||||
CVPixelBufferPoolCreate(kCFAllocatorDefault, (__bridge CFDictionaryRef)pixelBufferPoolOptions, (__bridge CFDictionaryRef)sourcePixelBufferOptions, &outputPool);
|
||||
|
||||
return outputPool;
|
||||
}
|
||||
|
||||
- (CMSampleBufferRef)createSampleBufferFromBuffer:(id<RTC_OBJC_TYPE(RTCI420Buffer)>)buffer {
|
||||
NSMutableDictionary *ioSurfaceProperties = [[NSMutableDictionary alloc] init];
|
||||
//ioSurfaceProperties[@"IOSurfaceIsGlobal"] = @(true);
|
||||
|
||||
NSMutableDictionary *options = [[NSMutableDictionary alloc] init];
|
||||
//options[(__bridge NSString *)kCVPixelBufferBytesPerRowAlignmentKey] = @(buffer.strideY);
|
||||
options[(__bridge NSString *)kCVPixelBufferIOSurfacePropertiesKey] = ioSurfaceProperties;
|
||||
|
||||
CVPixelBufferRef pixelBufferRef = nil;
|
||||
|
||||
if (!(_pixelBufferPool != nil && _pixelBufferPoolWidth == buffer.width && _pixelBufferPoolHeight == buffer.height)) {
|
||||
if (_pixelBufferPool) {
|
||||
CFRelease(_pixelBufferPool);
|
||||
_pixelBufferPool = nil;
|
||||
}
|
||||
_pixelBufferPool = [VideoSampleBufferView createPixelBufferPoolWithWidth:buffer.width height:buffer.height pixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange maxBufferCount:10];
|
||||
_pixelBufferPoolWidth = buffer.width;
|
||||
_pixelBufferPoolHeight = buffer.height;
|
||||
}
|
||||
|
||||
if (_pixelBufferPool != nil && _pixelBufferPoolWidth == buffer.width && _pixelBufferPoolHeight == buffer.height) {
|
||||
CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(kCFAllocatorDefault, _pixelBufferPool, nil, &pixelBufferRef);
|
||||
} else {
|
||||
CVPixelBufferCreate(
|
||||
kCFAllocatorDefault,
|
||||
buffer.width,
|
||||
buffer.height,
|
||||
kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
|
||||
(__bridge CFDictionaryRef)options,
|
||||
&pixelBufferRef
|
||||
);
|
||||
}
|
||||
|
||||
if (pixelBufferRef == nil) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
CopyVideoFrameToNV12PixelBuffer([buffer toI420], pixelBufferRef);
|
||||
|
||||
CMVideoFormatDescriptionRef formatRef = nil;
|
||||
OSStatus status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBufferRef, &formatRef);
|
||||
if (status != 0) {
|
||||
return nil;
|
||||
}
|
||||
if (formatRef == nil) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
CMSampleTimingInfo timingInfo;
|
||||
timingInfo.duration = CMTimeMake(1, 30);
|
||||
timingInfo.presentationTimeStamp = CMTimeMake(0, 30);
|
||||
timingInfo.decodeTimeStamp = CMTimeMake(0, 30);
|
||||
|
||||
CMSampleBufferRef sampleBuffer = nil;
|
||||
OSStatus bufferStatus = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, pixelBufferRef, formatRef, &timingInfo, &sampleBuffer);
|
||||
|
||||
CFRelease(pixelBufferRef);
|
||||
|
||||
if (bufferStatus != noErr) {
|
||||
return nil;
|
||||
}
|
||||
if (sampleBuffer == nil) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
NSArray *attachments = (__bridge NSArray *)CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true);
|
||||
NSMutableDictionary *dict = (NSMutableDictionary *)attachments[0];
|
||||
|
||||
dict[(__bridge NSString *)kCMSampleAttachmentKey_DisplayImmediately] = @(true);
|
||||
|
||||
return sampleBuffer;
|
||||
}
|
||||
|
||||
- (void)setSize:(CGSize)size {
|
||||
assert([NSThread isMainThread]);
|
||||
}
|
||||
|
||||
- (void)renderFrame:(nullable RTCVideoFrame *)frame {
|
||||
assert([NSThread isMainThread]);
|
||||
|
||||
if (!_firstFrameReceivedReported && _onFirstFrameReceived) {
|
||||
_firstFrameReceivedReported = true;
|
||||
_onFirstFrameReceived();
|
||||
}
|
||||
|
||||
if (!self.isEnabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (frame == nil) {
|
||||
RTCLogInfo(@"Incoming frame is nil. Exiting render callback.");
|
||||
return;
|
||||
}
|
||||
if (_isWaitingForLayoutFrameCount > 0) {
|
||||
_stashedVideoFrame = frame;
|
||||
_isWaitingForLayoutFrameCount--;
|
||||
return;
|
||||
}
|
||||
if (!_didStartWaitingForLayout) {
|
||||
if (_videoFrame != nil && _videoFrame.width > 0 && _videoFrame.height > 0 && frame.width > 0 && frame.height > 0) {
|
||||
float previousAspect = ((float)_videoFrame.width) / ((float)_videoFrame.height);
|
||||
float updatedAspect = ((float)frame.width) / ((float)frame.height);
|
||||
if ((previousAspect < 1.0f) != (updatedAspect < 1.0f)) {
|
||||
_stashedVideoFrame = frame;
|
||||
_didStartWaitingForLayout = true;
|
||||
_isWaitingForLayoutFrameCount = 5;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
_videoFrame = frame;
|
||||
|
||||
id<RTC_OBJC_TYPE(RTCI420Buffer)> buffer = [frame.buffer toI420];
|
||||
CMSampleBufferRef sampleBuffer = [self createSampleBufferFromBuffer:buffer];
|
||||
if (sampleBuffer) {
|
||||
[_sampleBufferLayer enqueueSampleBuffer:sampleBuffer];
|
||||
|
||||
CFRelease(sampleBuffer);
|
||||
}
|
||||
}
|
||||
|
||||
- (std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)getSink {
|
||||
assert([NSThread isMainThread]);
|
||||
|
||||
return _sink;
|
||||
}
|
||||
|
||||
- (void)setOnFirstFrameReceived:(void (^ _Nullable)())onFirstFrameReceived {
|
||||
_onFirstFrameReceived = [onFirstFrameReceived copy];
|
||||
_firstFrameReceivedReported = false;
|
||||
}
|
||||
|
||||
- (void)setInternalOrientationAndSize:(int)internalOrientation size:(CGSize)size {
|
||||
CGFloat aspect = 1.0f;
|
||||
if (size.width > 1.0f && size.height > 1.0f) {
|
||||
aspect = size.width / size.height;
|
||||
}
|
||||
if (_internalOrientation != internalOrientation || ABS(_internalAspect - aspect) > 0.001) {
|
||||
RTCLogInfo(@"VideoSampleBufferView@%lx orientation: %d, aspect: %f", (intptr_t)self, internalOrientation, (float)aspect);
|
||||
|
||||
_internalOrientation = internalOrientation;
|
||||
_internalAspect = aspect;
|
||||
if (_onOrientationUpdated) {
|
||||
_onOrientationUpdated(internalOrientation, aspect);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated {
|
||||
_onOrientationUpdated = [onOrientationUpdated copy];
|
||||
}
|
||||
|
||||
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated {
|
||||
_onIsMirroredUpdated = [onIsMirroredUpdated copy];
|
||||
}
|
||||
|
||||
+(BOOL)isAvailable {
|
||||
return YES;
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,349 @@
|
|||
/*
|
||||
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*
|
||||
*/
|
||||
|
||||
#ifdef TGCALLS_ENABLE_X264
|
||||
|
||||
#include "h264_encoder_impl.h"
|
||||
|
||||
#include <limits>
|
||||
|
||||
#include "rtc_base/checks.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "common_video/libyuv/include/webrtc_libyuv.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
namespace {
|
||||
|
||||
int NumberOfThreads(int width, int height, int number_of_cores) {
|
||||
if (width * height >= 1920 * 1080 && number_of_cores > 8) {
|
||||
return 8; // 8 threads for 1080p on high perf machines.
|
||||
} else if (width * height > 1280 * 960 && number_of_cores >= 6) {
|
||||
return 3; // 3 threads for 1080p.
|
||||
} else if (width * height > 640 * 480 && number_of_cores >= 3) {
|
||||
return 2; // 2 threads for qHD/HD.
|
||||
} else {
|
||||
return 1; // 1 thread for VGA or less.
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
H264EncoderX264Impl::H264EncoderX264Impl(cricket::VideoCodec const &videoCodec) :
|
||||
packetization_mode_(H264PacketizationMode::SingleNalUnit),
|
||||
encoded_image_callback_(nullptr),
|
||||
inited_(false),
|
||||
encoder_(nullptr) {
|
||||
}
|
||||
|
||||
H264EncoderX264Impl::~H264EncoderX264Impl() {
|
||||
Release();
|
||||
}
|
||||
|
||||
int32_t H264EncoderX264Impl::InitEncode(const VideoCodec* inst,
|
||||
int32_t number_of_cores,
|
||||
size_t max_payload_size) {
|
||||
|
||||
if (inst == NULL) {
|
||||
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
|
||||
}
|
||||
if (inst->maxFramerate < 1) {
|
||||
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
|
||||
}
|
||||
// allow zero to represent an unspecified maxBitRate
|
||||
if (inst->maxBitrate > 0 && inst->startBitrate > inst->maxBitrate) {
|
||||
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
|
||||
}
|
||||
if (inst->width < 1 || inst->height < 1) {
|
||||
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
|
||||
}
|
||||
if (number_of_cores < 1) {
|
||||
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
|
||||
}
|
||||
|
||||
int ret_val = Release();
|
||||
if (ret_val < 0) {
|
||||
return ret_val;
|
||||
}
|
||||
codec_settings_ = *inst;
|
||||
/* Get default params for preset/tuning */
|
||||
x264_param_t param;
|
||||
memset(¶m, 0, sizeof(param));
|
||||
x264_param_default(¶m);
|
||||
ret_val = x264_param_default_preset(¶m, "fast", "zerolatency");
|
||||
if (ret_val != 0) {
|
||||
RTC_LOG(LS_ERROR) << "H264EncoderX264Impl::InitEncode() fails to initialize encoder ret_val " << ret_val;
|
||||
x264_encoder_close(encoder_);
|
||||
encoder_ = NULL;
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
|
||||
param.i_log_level = X264_LOG_WARNING;
|
||||
param.i_threads = 1;
|
||||
param.i_width = inst->width;
|
||||
param.i_height = inst->height;
|
||||
param.i_frame_total = 0;
|
||||
|
||||
param.b_sliced_threads = 0;
|
||||
//param.i_keyint_min = 5000;
|
||||
param.i_keyint_max = 60;
|
||||
|
||||
/*param.i_bframe = 0;
|
||||
param.b_open_gop = 0;
|
||||
param.i_bframe_pyramid = 0;
|
||||
param.i_bframe_adaptive = X264_B_ADAPT_TRELLIS;
|
||||
param.i_nal_hrd = 0;
|
||||
|
||||
param.i_fps_den = 1;
|
||||
param.i_fps_num = inst->maxFramerate;
|
||||
param.i_timebase_den = 1;
|
||||
param.i_timebase_num = inst->maxFramerate;
|
||||
param.b_intra_refresh = 0;
|
||||
param.i_frame_reference = 1;
|
||||
param.b_annexb = 1;
|
||||
param.i_csp = X264_CSP_I420;
|
||||
param.b_aud = 1;
|
||||
|
||||
param.b_vfr_input = 0;
|
||||
param.b_repeat_headers = 1;*/
|
||||
|
||||
param.rc.i_rc_method = X264_RC_ABR;
|
||||
param.rc.i_bitrate = codec_settings_.maxBitrate;
|
||||
param.rc.i_vbv_max_bitrate = param.rc.i_bitrate;
|
||||
param.rc.i_vbv_buffer_size = param.rc.i_bitrate * 2;
|
||||
param.rc.f_vbv_buffer_init = 1.0;
|
||||
//param.rc.i_qp_max = codec_settings_.qpMax;
|
||||
|
||||
/*param.rc.i_rc_method = X264_RC_CRF;
|
||||
param.rc.f_rf_constant_max = 20;
|
||||
param.rc.f_rf_constant_max = 25;*/
|
||||
|
||||
//param.rc.b_filler = 0;
|
||||
|
||||
param.i_slice_max_size = (int)max_payload_size;
|
||||
param.i_sps_id = sps_id_;
|
||||
|
||||
ret_val = x264_param_apply_profile(¶m, "high");
|
||||
if (ret_val != 0) {
|
||||
RTC_LOG(LS_ERROR) << "H264EncoderX264Impl::InitEncode() fails to initialize encoder ret_val " << ret_val;
|
||||
x264_encoder_close(encoder_);
|
||||
encoder_ = NULL;
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
|
||||
param.analyse.i_weighted_pred = X264_WEIGHTP_NONE;
|
||||
|
||||
ret_val = x264_picture_alloc(&pic_, param.i_csp, param.i_width, param.i_height);
|
||||
if (ret_val != 0) {
|
||||
RTC_LOG(LS_ERROR) << "H264EncoderX264Impl::InitEncode() fails to initialize encoder ret_val " << ret_val;
|
||||
x264_encoder_close(encoder_);
|
||||
encoder_ = NULL;
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
|
||||
encoder_ = x264_encoder_open(¶m);
|
||||
if (!encoder_){
|
||||
RTC_LOG(LS_ERROR) << "H264EncoderX264Impl::InitEncode() fails to initialize encoder ret_val " << ret_val;
|
||||
x264_encoder_close(encoder_);
|
||||
x264_picture_clean(&pic_);
|
||||
encoder_ = NULL;
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
|
||||
const size_t new_capacity = CalcBufferSize(VideoType::kI420, codec_settings_.width, codec_settings_.height);
|
||||
encoded_image_.SetEncodedData(EncodedImageBuffer::Create(new_capacity));
|
||||
encoded_image_._encodedWidth = codec_settings_.width;
|
||||
encoded_image_._encodedHeight = codec_settings_.height;
|
||||
encoded_image_.ClearEncodedData();
|
||||
|
||||
inited_ = true;
|
||||
RTC_LOG(LS_INFO) << "H264EncoderX264Impl::InitEncode(width: " << inst->width << ", height: " << inst->height << ", framerate: " << inst->maxFramerate << ", start_bitrate: " << inst->startBitrate << ", max_bitrate: " << inst->maxBitrate << ")";
|
||||
|
||||
sps_id_++;
|
||||
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
|
||||
int32_t H264EncoderX264Impl::Release() {
|
||||
if (encoder_) {
|
||||
encoder_ = nullptr;
|
||||
}
|
||||
encoded_image_.ClearEncodedData();
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
int32_t H264EncoderX264Impl::RegisterEncodeCompleteCallback(EncodedImageCallback* callback) {
|
||||
encoded_image_callback_ = callback;
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
void H264EncoderX264Impl::SetRates(const RateControlParameters& parameters) {
|
||||
codec_settings_.maxBitrate = parameters.bitrate.GetSpatialLayerSum(0);
|
||||
codec_settings_.maxFramerate = (uint32_t)parameters.framerate_fps;
|
||||
}
|
||||
|
||||
int32_t H264EncoderX264Impl::Encode(const VideoFrame& frame, const std::vector<VideoFrameType>* frame_types) {
|
||||
if (!inited_) {
|
||||
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
|
||||
}
|
||||
if (frame.size() == 0) {
|
||||
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
|
||||
}
|
||||
if (encoded_image_callback_ == NULL) {
|
||||
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
|
||||
}
|
||||
|
||||
VideoFrameType frame_type = VideoFrameType::kVideoFrameDelta;
|
||||
if (frame_types && frame_types->size() > 0) {
|
||||
frame_type = (*frame_types)[0];
|
||||
}
|
||||
|
||||
pic_.img.i_csp = X264_CSP_I420;
|
||||
pic_.img.i_plane = 3;
|
||||
switch (frame_type) {
|
||||
case VideoFrameType::kEmptyFrame: {
|
||||
pic_.i_type = X264_TYPE_AUTO;
|
||||
break;
|
||||
}
|
||||
case VideoFrameType::kVideoFrameDelta: {
|
||||
pic_.i_type = X264_TYPE_AUTO;
|
||||
break;
|
||||
}
|
||||
case VideoFrameType::kVideoFrameKey: {
|
||||
pic_.i_type = X264_TYPE_AUTO;
|
||||
x264_encoder_intra_refresh(encoder_);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
pic_.i_type = X264_TYPE_AUTO;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
webrtc::scoped_refptr<I420BufferInterface> frame_buffer = frame.video_frame_buffer()->ToI420();
|
||||
|
||||
pic_.img.plane[0] = (uint8_t *)frame_buffer->DataY();
|
||||
pic_.img.i_stride[0] = frame_buffer->StrideY();
|
||||
pic_.img.plane[1] = (uint8_t *)frame_buffer->DataU();
|
||||
pic_.img.i_stride[1] = frame_buffer->StrideU();
|
||||
pic_.img.plane[2] = (uint8_t *)frame_buffer->DataV();
|
||||
pic_.img.i_stride[2] = frame_buffer->StrideV();
|
||||
|
||||
pic_.i_pts++;
|
||||
|
||||
int n_nal = 0;
|
||||
int i_frame_size = x264_encoder_encode(encoder_, &nal_t_, &n_nal, &pic_, &pic_out_);
|
||||
if (i_frame_size < 0) {
|
||||
RTC_LOG(LS_ERROR) << "H264EncoderX264Impl::Encode() fails to encode " << i_frame_size;
|
||||
x264_encoder_close(encoder_);
|
||||
x264_picture_clean(&pic_);
|
||||
encoder_ = NULL;
|
||||
return WEBRTC_VIDEO_CODEC_ERROR;
|
||||
}
|
||||
|
||||
if (i_frame_size) {
|
||||
if (n_nal == 0) {
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
// Encoder can skip frames to save bandwidth in which case
|
||||
// `encoded_images_[i]._length` == 0.
|
||||
/*if (encoded_images_[i].size() > 0) {
|
||||
// Parse QP.
|
||||
h264_bitstream_parser_.ParseBitstream(encoded_images_[i]);
|
||||
encoded_images_[i].qp_ =
|
||||
h264_bitstream_parser_.GetLastSliceQp().value_or(-1);
|
||||
|
||||
// Deliver encoded image.
|
||||
CodecSpecificInfo codec_specific;
|
||||
codec_specific.codecType = kVideoCodecH264;
|
||||
codec_specific.codecSpecific.H264.packetization_mode =
|
||||
packetization_mode_;
|
||||
codec_specific.codecSpecific.H264.temporal_idx = kNoTemporalIdx;
|
||||
codec_specific.codecSpecific.H264.idr_frame =
|
||||
info.eFrameType == videoFrameTypeIDR;
|
||||
codec_specific.codecSpecific.H264.base_layer_sync = false;
|
||||
if (configurations_[i].num_temporal_layers > 1) {
|
||||
const uint8_t tid = info.sLayerInfo[0].uiTemporalId;
|
||||
codec_specific.codecSpecific.H264.temporal_idx = tid;
|
||||
codec_specific.codecSpecific.H264.base_layer_sync =
|
||||
tid > 0 && tid < tl0sync_limit_[i];
|
||||
if (codec_specific.codecSpecific.H264.base_layer_sync) {
|
||||
tl0sync_limit_[i] = tid;
|
||||
}
|
||||
if (tid == 0) {
|
||||
tl0sync_limit_[i] = configurations_[i].num_temporal_layers;
|
||||
}
|
||||
}
|
||||
encoded_image_callback_->OnEncodedImage(encoded_images_[i],
|
||||
&codec_specific);
|
||||
}
|
||||
}*/
|
||||
|
||||
size_t required_capacity = 0;
|
||||
size_t fragments_count = 0;
|
||||
for (int layer = 0; layer < 1; ++layer) {
|
||||
for (int nal = 0; nal < n_nal; ++nal, ++fragments_count) {
|
||||
int currentNaluSize = nal_t_[nal].i_payload;
|
||||
RTC_CHECK_GE(currentNaluSize, 0);
|
||||
// Ensure `required_capacity` will not overflow.
|
||||
RTC_CHECK_LE(currentNaluSize,
|
||||
std::numeric_limits<size_t>::max() - required_capacity);
|
||||
required_capacity += currentNaluSize;
|
||||
}
|
||||
}
|
||||
// TODO(nisse): Use a cache or buffer pool to avoid allocation?
|
||||
auto buffer = EncodedImageBuffer::Create(required_capacity);
|
||||
encoded_image_.SetEncodedData(buffer);
|
||||
|
||||
int currentBufferLength = 0;
|
||||
uint32_t totalNaluIndex = 0;
|
||||
for (int nal_index = 0; nal_index < n_nal; nal_index++) {
|
||||
uint32_t currentNaluSize = 0;
|
||||
currentNaluSize = nal_t_[nal_index].i_payload;
|
||||
memcpy(buffer->data() + currentBufferLength, nal_t_[nal_index].p_payload, currentNaluSize);
|
||||
currentBufferLength += currentNaluSize;
|
||||
|
||||
totalNaluIndex++;
|
||||
}
|
||||
}
|
||||
i_frame++;
|
||||
if (encoded_image_.size() > 0) {
|
||||
encoded_image_._encodedWidth = codec_settings_.width;
|
||||
encoded_image_._encodedHeight = codec_settings_.height;
|
||||
encoded_image_.SetTimestamp(frame.timestamp());
|
||||
encoded_image_._frameType = frame_type;
|
||||
encoded_image_.SetSpatialIndex(0);
|
||||
|
||||
CodecSpecificInfo codec_specific;
|
||||
codec_specific.codecType = kVideoCodecH264;
|
||||
codec_specific.codecSpecific.H264.packetization_mode = packetization_mode_;
|
||||
codec_specific.codecSpecific.H264.temporal_idx = kNoTemporalIdx;
|
||||
|
||||
if (pic_out_.i_type == X264_TYPE_IDR) {
|
||||
codec_specific.codecSpecific.H264.idr_frame = true;
|
||||
} else {
|
||||
codec_specific.codecSpecific.H264.idr_frame = false;
|
||||
}
|
||||
codec_specific.codecSpecific.H264.base_layer_sync = false;
|
||||
encoded_image_callback_->OnEncodedImage(encoded_image_, &codec_specific);
|
||||
}
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
bool H264EncoderX264Impl::IsInitialized() const {
|
||||
return encoder_ != nullptr;
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
/*
|
||||
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*
|
||||
*/
|
||||
|
||||
#ifdef TGCALLS_ENABLE_x264
|
||||
|
||||
#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_H264_H264_ENCODER_IMPL_H_
|
||||
#define WEBRTC_MODULES_VIDEO_CODING_CODECS_H264_H264_ENCODER_IMPL_H_
|
||||
|
||||
#include "modules/video_coding/codecs/h264/include/h264.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "api/scoped_refptr.h"
|
||||
|
||||
#include <libx264/x264.h>
|
||||
|
||||
class ISVCEncoder;
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
class H264EncoderX264Impl : public H264Encoder {
|
||||
public:
|
||||
H264EncoderX264Impl(cricket::VideoCodec const &videoCodec);
|
||||
~H264EncoderX264Impl() override;
|
||||
|
||||
// |max_payload_size| is ignored.
|
||||
// The following members of |codec_settings| are used. The rest are ignored.;
|
||||
// - codecType (must be kVideoCodecH264)
|
||||
// - targetBitrate
|
||||
// - maxFramerate
|
||||
// - width
|
||||
// - height
|
||||
int32_t InitEncode(const VideoCodec* codec_settings,
|
||||
int32_t number_of_cores,
|
||||
size_t /*max_payload_size*/) override;
|
||||
int32_t Release() override;
|
||||
|
||||
int32_t RegisterEncodeCompleteCallback(
|
||||
EncodedImageCallback* callback) override;
|
||||
void SetRates(const RateControlParameters& parameters) override;
|
||||
|
||||
// The result of encoding - an EncodedImage and RTPFragmentationHeader - are
|
||||
// passed to the encode complete callback.
|
||||
int32_t Encode(const VideoFrame& frame,
|
||||
const std::vector<VideoFrameType>* frame_types) override;
|
||||
|
||||
private:
|
||||
bool IsInitialized() const;
|
||||
|
||||
// ISVCEncoder* openh264_encoder_;
|
||||
VideoCodec codec_settings_;
|
||||
|
||||
H264PacketizationMode packetization_mode_;
|
||||
|
||||
EncodedImage encoded_image_;
|
||||
EncodedImageCallback* encoded_image_callback_;
|
||||
|
||||
bool inited_;
|
||||
x264_picture_t pic_;
|
||||
x264_picture_t pic_out_;
|
||||
x264_t *encoder_;
|
||||
int i_frame = 0;//frame index
|
||||
x264_nal_t *nal_t_;
|
||||
int sps_id_ = 1;
|
||||
// x264_param_t x264_parameter;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_H264_H264_ENCODER_IMPL_H_
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,406 @@
|
|||
/*
|
||||
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*
|
||||
*/
|
||||
|
||||
#include "h265_nalu_rewriter.h"
|
||||
|
||||
#include <CoreFoundation/CoreFoundation.h>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "rtc_base/checks.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "common_video/h265/h265_common.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
namespace {
|
||||
|
||||
const char kAnnexBHeaderBytes[4] = {0, 0, 0, 1};
|
||||
const size_t kAvccHeaderByteSize = sizeof(uint32_t);
|
||||
|
||||
// Helper class for reading NALUs from an RTP Annex B buffer.
|
||||
class H265AnnexBBufferReader final {
|
||||
public:
|
||||
H265AnnexBBufferReader(const uint8_t* annexb_buffer, size_t length);
|
||||
~H265AnnexBBufferReader();
|
||||
H265AnnexBBufferReader(const H265AnnexBBufferReader& other) = delete;
|
||||
void operator=(const H265AnnexBBufferReader& other) = delete;
|
||||
|
||||
// Returns a pointer to the beginning of the next NALU slice without the
|
||||
// header bytes and its length. Returns false if no more slices remain.
|
||||
bool ReadNalu(const uint8_t** out_nalu, size_t* out_length);
|
||||
|
||||
// Returns the number of unread NALU bytes, including the size of the header.
|
||||
// If the buffer has no remaining NALUs this will return zero.
|
||||
size_t BytesRemaining() const;
|
||||
|
||||
// Reset the reader to start reading from the first NALU
|
||||
void SeekToStart();
|
||||
|
||||
// Seek to the next position that holds a NALU of the desired type,
|
||||
// or the end if no such NALU is found.
|
||||
// Return true if a NALU of the desired type is found, false if we
|
||||
// reached the end instead
|
||||
bool SeekToNextNaluOfType(H265::NaluType type);
|
||||
|
||||
private:
|
||||
// Returns the the next offset that contains NALU data.
|
||||
size_t FindNextNaluHeader(const uint8_t* start,
|
||||
size_t length,
|
||||
size_t offset) const;
|
||||
|
||||
const uint8_t* const start_;
|
||||
std::vector<NaluIndex> offsets_;
|
||||
std::vector<NaluIndex>::iterator offset_;
|
||||
const size_t length_;
|
||||
};
|
||||
|
||||
// Helper class for writing NALUs using avcc format into a buffer.
|
||||
class H265AvccBufferWriter final {
|
||||
public:
|
||||
H265AvccBufferWriter(uint8_t* const avcc_buffer, size_t length);
|
||||
~H265AvccBufferWriter() {}
|
||||
H265AvccBufferWriter(const H265AvccBufferWriter& other) = delete;
|
||||
void operator=(const H265AvccBufferWriter& other) = delete;
|
||||
|
||||
// Writes the data slice into the buffer. Returns false if there isn't
|
||||
// enough space left.
|
||||
bool WriteNalu(const uint8_t* data, size_t data_size);
|
||||
|
||||
// Returns the unused bytes in the buffer.
|
||||
size_t BytesRemaining() const;
|
||||
|
||||
private:
|
||||
uint8_t* const start_;
|
||||
size_t offset_;
|
||||
const size_t length_;
|
||||
};
|
||||
|
||||
H265AnnexBBufferReader::H265AnnexBBufferReader(const uint8_t* annexb_buffer,
|
||||
size_t length)
|
||||
: start_(annexb_buffer), length_(length) {
|
||||
RTC_DCHECK(annexb_buffer);
|
||||
offsets_ = H264::FindNaluIndices(annexb_buffer, length);
|
||||
offset_ = offsets_.begin();
|
||||
}
|
||||
|
||||
H265AnnexBBufferReader::~H265AnnexBBufferReader() = default;
|
||||
|
||||
bool H265AnnexBBufferReader::ReadNalu(const uint8_t** out_nalu,
|
||||
size_t* out_length) {
|
||||
RTC_DCHECK(out_nalu);
|
||||
RTC_DCHECK(out_length);
|
||||
*out_nalu = nullptr;
|
||||
*out_length = 0;
|
||||
|
||||
if (offset_ == offsets_.end()) {
|
||||
return false;
|
||||
}
|
||||
*out_nalu = start_ + offset_->payload_start_offset;
|
||||
*out_length = offset_->payload_size;
|
||||
++offset_;
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t H265AnnexBBufferReader::BytesRemaining() const {
|
||||
if (offset_ == offsets_.end()) {
|
||||
return 0;
|
||||
}
|
||||
return length_ - offset_->start_offset;
|
||||
}
|
||||
|
||||
void H265AnnexBBufferReader::SeekToStart() {
|
||||
offset_ = offsets_.begin();
|
||||
}
|
||||
|
||||
bool H265AnnexBBufferReader::SeekToNextNaluOfType(H265::NaluType type) {
|
||||
for (; offset_ != offsets_.end(); ++offset_) {
|
||||
if (offset_->payload_size < 1)
|
||||
continue;
|
||||
if (H265::ParseNaluType(*(start_ + offset_->payload_start_offset)) == type)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
H265AvccBufferWriter::H265AvccBufferWriter(uint8_t* const avcc_buffer, size_t length)
|
||||
: start_(avcc_buffer), offset_(0), length_(length) {
|
||||
RTC_DCHECK(avcc_buffer);
|
||||
}
|
||||
|
||||
bool H265AvccBufferWriter::WriteNalu(const uint8_t* data, size_t data_size) {
|
||||
// Check if we can write this length of data.
|
||||
if (data_size + kAvccHeaderByteSize > BytesRemaining()) {
|
||||
return false;
|
||||
}
|
||||
// Write length header, which needs to be big endian.
|
||||
uint32_t big_endian_length = CFSwapInt32HostToBig(data_size);
|
||||
memcpy(start_ + offset_, &big_endian_length, sizeof(big_endian_length));
|
||||
offset_ += sizeof(big_endian_length);
|
||||
// Write data.
|
||||
memcpy(start_ + offset_, data, data_size);
|
||||
offset_ += data_size;
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t H265AvccBufferWriter::BytesRemaining() const {
|
||||
return length_ - offset_;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
bool H265CMSampleBufferToAnnexBBuffer(
|
||||
CMSampleBufferRef hvcc_sample_buffer,
|
||||
bool is_keyframe,
|
||||
rtc::Buffer* annexb_buffer) {
|
||||
RTC_DCHECK(hvcc_sample_buffer);
|
||||
|
||||
// Get format description from the sample buffer.
|
||||
CMVideoFormatDescriptionRef description =
|
||||
CMSampleBufferGetFormatDescription(hvcc_sample_buffer);
|
||||
if (description == nullptr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to get sample buffer's description.";
|
||||
return false;
|
||||
}
|
||||
|
||||
// Get parameter set information.
|
||||
int nalu_header_size = 0;
|
||||
size_t param_set_count = 0;
|
||||
OSStatus status = CMVideoFormatDescriptionGetHEVCParameterSetAtIndex(
|
||||
description, 0, nullptr, nullptr, ¶m_set_count, &nalu_header_size);
|
||||
if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to get parameter set.";
|
||||
return false;
|
||||
}
|
||||
RTC_CHECK_EQ(nalu_header_size, kAvccHeaderByteSize);
|
||||
RTC_DCHECK_EQ(param_set_count, 3);
|
||||
|
||||
// Truncate any previous data in the buffer without changing its capacity.
|
||||
annexb_buffer->SetSize(0);
|
||||
|
||||
size_t nalu_offset = 0;
|
||||
std::vector<size_t> frag_offsets;
|
||||
std::vector<size_t> frag_lengths;
|
||||
|
||||
// Place all parameter sets at the front of buffer.
|
||||
if (is_keyframe) {
|
||||
size_t param_set_size = 0;
|
||||
const uint8_t* param_set = nullptr;
|
||||
for (size_t i = 0; i < param_set_count; ++i) {
|
||||
status = CMVideoFormatDescriptionGetHEVCParameterSetAtIndex(
|
||||
description, i, ¶m_set, ¶m_set_size, nullptr, nullptr);
|
||||
if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to get parameter set.";
|
||||
return false;
|
||||
}
|
||||
// Update buffer.
|
||||
annexb_buffer->AppendData(kAnnexBHeaderBytes, sizeof(kAnnexBHeaderBytes));
|
||||
annexb_buffer->AppendData(reinterpret_cast<const char*>(param_set),
|
||||
param_set_size);
|
||||
// Update fragmentation.
|
||||
frag_offsets.push_back(nalu_offset + sizeof(kAnnexBHeaderBytes));
|
||||
frag_lengths.push_back(param_set_size);
|
||||
nalu_offset += sizeof(kAnnexBHeaderBytes) + param_set_size;
|
||||
}
|
||||
}
|
||||
|
||||
// Get block buffer from the sample buffer.
|
||||
CMBlockBufferRef block_buffer =
|
||||
CMSampleBufferGetDataBuffer(hvcc_sample_buffer);
|
||||
if (block_buffer == nullptr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to get sample buffer's block buffer.";
|
||||
return false;
|
||||
}
|
||||
CMBlockBufferRef contiguous_buffer = nullptr;
|
||||
// Make sure block buffer is contiguous.
|
||||
if (!CMBlockBufferIsRangeContiguous(block_buffer, 0, 0)) {
|
||||
status = CMBlockBufferCreateContiguous(
|
||||
nullptr, block_buffer, nullptr, nullptr, 0, 0, 0, &contiguous_buffer);
|
||||
if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to flatten non-contiguous block buffer: "
|
||||
<< status;
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
contiguous_buffer = block_buffer;
|
||||
// Retain to make cleanup easier.
|
||||
CFRetain(contiguous_buffer);
|
||||
block_buffer = nullptr;
|
||||
}
|
||||
|
||||
// Now copy the actual data.
|
||||
char* data_ptr = nullptr;
|
||||
size_t block_buffer_size = CMBlockBufferGetDataLength(contiguous_buffer);
|
||||
status = CMBlockBufferGetDataPointer(contiguous_buffer, 0, nullptr, nullptr,
|
||||
&data_ptr);
|
||||
if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to get block buffer data.";
|
||||
CFRelease(contiguous_buffer);
|
||||
return false;
|
||||
}
|
||||
size_t bytes_remaining = block_buffer_size;
|
||||
while (bytes_remaining > 0) {
|
||||
// The size type here must match |nalu_header_size|, we expect 4 bytes.
|
||||
// Read the length of the next packet of data. Must convert from big endian
|
||||
// to host endian.
|
||||
RTC_DCHECK_GE(bytes_remaining, (size_t)nalu_header_size);
|
||||
uint32_t* uint32_data_ptr = reinterpret_cast<uint32_t*>(data_ptr);
|
||||
uint32_t packet_size = CFSwapInt32BigToHost(*uint32_data_ptr);
|
||||
// Update buffer.
|
||||
annexb_buffer->AppendData(kAnnexBHeaderBytes, sizeof(kAnnexBHeaderBytes));
|
||||
annexb_buffer->AppendData(data_ptr + nalu_header_size, packet_size);
|
||||
// Update fragmentation.
|
||||
frag_offsets.push_back(nalu_offset + sizeof(kAnnexBHeaderBytes));
|
||||
frag_lengths.push_back(packet_size);
|
||||
nalu_offset += sizeof(kAnnexBHeaderBytes) + packet_size;
|
||||
|
||||
size_t bytes_written = packet_size + sizeof(kAnnexBHeaderBytes);
|
||||
bytes_remaining -= bytes_written;
|
||||
data_ptr += bytes_written;
|
||||
}
|
||||
RTC_DCHECK_EQ(bytes_remaining, (size_t)0);
|
||||
|
||||
CFRelease(contiguous_buffer);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool H265AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer,
|
||||
size_t annexb_buffer_size,
|
||||
CMVideoFormatDescriptionRef video_format,
|
||||
CMSampleBufferRef* out_sample_buffer) {
|
||||
RTC_DCHECK(annexb_buffer);
|
||||
RTC_DCHECK(out_sample_buffer);
|
||||
RTC_DCHECK(video_format);
|
||||
*out_sample_buffer = nullptr;
|
||||
|
||||
H265AnnexBBufferReader reader(annexb_buffer, annexb_buffer_size);
|
||||
if (reader.SeekToNextNaluOfType(H265::kVps)) {
|
||||
// Buffer contains an SPS NALU - skip it and the following PPS
|
||||
const uint8_t* data;
|
||||
size_t data_len;
|
||||
if (!reader.ReadNalu(&data, &data_len)) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to read VPS";
|
||||
return false;
|
||||
}
|
||||
if (!reader.ReadNalu(&data, &data_len)) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to read SPS";
|
||||
return false;
|
||||
}
|
||||
if (!reader.ReadNalu(&data, &data_len)) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to read PPS";
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
// No SPS NALU - start reading from the first NALU in the buffer
|
||||
reader.SeekToStart();
|
||||
}
|
||||
|
||||
// Allocate memory as a block buffer.
|
||||
// TODO(tkchin): figure out how to use a pool.
|
||||
CMBlockBufferRef block_buffer = nullptr;
|
||||
OSStatus status = CMBlockBufferCreateWithMemoryBlock(
|
||||
nullptr, nullptr, reader.BytesRemaining(), nullptr, nullptr, 0,
|
||||
reader.BytesRemaining(), kCMBlockBufferAssureMemoryNowFlag,
|
||||
&block_buffer);
|
||||
if (status != kCMBlockBufferNoErr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to create block buffer.";
|
||||
return false;
|
||||
}
|
||||
|
||||
// Make sure block buffer is contiguous.
|
||||
CMBlockBufferRef contiguous_buffer = nullptr;
|
||||
if (!CMBlockBufferIsRangeContiguous(block_buffer, 0, 0)) {
|
||||
status = CMBlockBufferCreateContiguous(
|
||||
nullptr, block_buffer, nullptr, nullptr, 0, 0, 0, &contiguous_buffer);
|
||||
if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to flatten non-contiguous block buffer: "
|
||||
<< status;
|
||||
CFRelease(block_buffer);
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
contiguous_buffer = block_buffer;
|
||||
block_buffer = nullptr;
|
||||
}
|
||||
|
||||
// Get a raw pointer into allocated memory.
|
||||
size_t block_buffer_size = 0;
|
||||
char* data_ptr = nullptr;
|
||||
status = CMBlockBufferGetDataPointer(contiguous_buffer, 0, nullptr,
|
||||
&block_buffer_size, &data_ptr);
|
||||
if (status != kCMBlockBufferNoErr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to get block buffer data pointer.";
|
||||
CFRelease(contiguous_buffer);
|
||||
return false;
|
||||
}
|
||||
RTC_DCHECK(block_buffer_size == reader.BytesRemaining());
|
||||
|
||||
// Write Avcc NALUs into block buffer memory.
|
||||
H265AvccBufferWriter writer(reinterpret_cast<uint8_t*>(data_ptr),
|
||||
block_buffer_size);
|
||||
while (reader.BytesRemaining() > 0) {
|
||||
const uint8_t* nalu_data_ptr = nullptr;
|
||||
size_t nalu_data_size = 0;
|
||||
if (reader.ReadNalu(&nalu_data_ptr, &nalu_data_size)) {
|
||||
writer.WriteNalu(nalu_data_ptr, nalu_data_size);
|
||||
}
|
||||
}
|
||||
|
||||
// Create sample buffer.
|
||||
status = CMSampleBufferCreate(nullptr, contiguous_buffer, true, nullptr,
|
||||
nullptr, video_format, 1, 0, nullptr, 0,
|
||||
nullptr, out_sample_buffer);
|
||||
if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to create sample buffer.";
|
||||
CFRelease(contiguous_buffer);
|
||||
return false;
|
||||
}
|
||||
CFRelease(contiguous_buffer);
|
||||
return true;
|
||||
}
|
||||
|
||||
CMVideoFormatDescriptionRef CreateH265VideoFormatDescription(
|
||||
const uint8_t* annexb_buffer,
|
||||
size_t annexb_buffer_size) {
|
||||
const uint8_t* param_set_ptrs[3] = {};
|
||||
size_t param_set_sizes[3] = {};
|
||||
H265AnnexBBufferReader reader(annexb_buffer, annexb_buffer_size);
|
||||
// Skip everyting before the VPS, then read the VPS, SPS and PPS
|
||||
if (!reader.SeekToNextNaluOfType(H265::kVps)) {
|
||||
return nullptr;
|
||||
}
|
||||
if (!reader.ReadNalu(¶m_set_ptrs[0], ¶m_set_sizes[0])) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to read VPS";
|
||||
return nullptr;
|
||||
}
|
||||
if (!reader.ReadNalu(¶m_set_ptrs[1], ¶m_set_sizes[1])) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to read SPS";
|
||||
return nullptr;
|
||||
}
|
||||
if (!reader.ReadNalu(¶m_set_ptrs[2], ¶m_set_sizes[2])) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to read PPS";
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Parse the SPS and PPS into a CMVideoFormatDescription.
|
||||
CMVideoFormatDescriptionRef description = nullptr;
|
||||
OSStatus status = CMVideoFormatDescriptionCreateFromHEVCParameterSets(
|
||||
kCFAllocatorDefault, 3, param_set_ptrs, param_set_sizes, 4, nullptr,
|
||||
&description);
|
||||
if (status != noErr) {
|
||||
RTC_LOG(LS_ERROR) << "Failed to create video format description.";
|
||||
return nullptr;
|
||||
}
|
||||
return description;
|
||||
}
|
||||
|
||||
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
/*
|
||||
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*
|
||||
*/
|
||||
|
||||
#ifndef TGCALLS_PLATFORM_DARWIN_H265_NALU_REWRITER_H_
|
||||
#define TGCALLS_PLATFORM_DARWIN_H265_NALU_REWRITER_H_
|
||||
|
||||
#include "modules/video_coding/codecs/h264/include/h264.h"
|
||||
|
||||
#include <CoreMedia/CoreMedia.h>
|
||||
#include <vector>
|
||||
|
||||
#include "common_video/h264/h264_common.h"
|
||||
#include "common_video/h265/h265_common.h"
|
||||
#include "rtc_base/buffer.h"
|
||||
|
||||
using webrtc::H264::NaluIndex;
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
bool H265CMSampleBufferToAnnexBBuffer(
|
||||
CMSampleBufferRef hvcc_sample_buffer,
|
||||
bool is_keyframe,
|
||||
rtc::Buffer* annexb_buffer)
|
||||
__OSX_AVAILABLE_STARTING(__MAC_10_12, __IPHONE_11_0);
|
||||
|
||||
// Converts a buffer received from RTP into a sample buffer suitable for the
|
||||
// VideoToolbox decoder. The RTP buffer is in annex b format whereas the sample
|
||||
// buffer is in hvcc format.
|
||||
// If |is_keyframe| is true then |video_format| is ignored since the format will
|
||||
// be read from the buffer. Otherwise |video_format| must be provided.
|
||||
// Caller is responsible for releasing the created sample buffer.
|
||||
bool H265AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer,
|
||||
size_t annexb_buffer_size,
|
||||
CMVideoFormatDescriptionRef video_format,
|
||||
CMSampleBufferRef* out_sample_buffer)
|
||||
__OSX_AVAILABLE_STARTING(__MAC_10_12, __IPHONE_11_0);
|
||||
|
||||
CMVideoFormatDescriptionRef CreateH265VideoFormatDescription(
|
||||
const uint8_t* annexb_buffer,
|
||||
size_t annexb_buffer_size)
|
||||
__OSX_AVAILABLE_STARTING(__MAC_10_12, __IPHONE_11_0);
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // TGCALLS_PLATFORM_DARWIN_H265_NALU_REWRITER_H_
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
#ifndef TGCALLS_CALL_AUDIO_TONE_H_
|
||||
#define TGCALLS_CALL_AUDIO_TONE_H_
|
||||
|
||||
#include <vector>
|
||||
|
||||
namespace tgcalls {
|
||||
|
||||
class CallAudioTone {
|
||||
public:
|
||||
CallAudioTone(std::vector<int16_t> &&samples, int sampleRate, int loopCount) :
|
||||
_samples(std::move(samples)), _sampleRate(sampleRate), _loopCount(loopCount) {
|
||||
}
|
||||
|
||||
public:
|
||||
std::vector<int16_t> const samples() const {
|
||||
return _samples;
|
||||
}
|
||||
|
||||
int sampleRate() const {
|
||||
return _sampleRate;
|
||||
}
|
||||
|
||||
int loopCount() const {
|
||||
return _loopCount;
|
||||
}
|
||||
|
||||
void setLoopCount(int loopCount) {
|
||||
_loopCount = loopCount;
|
||||
}
|
||||
|
||||
size_t offset() const {
|
||||
return _offset;
|
||||
}
|
||||
|
||||
void setOffset(size_t offset) {
|
||||
_offset = offset;
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<int16_t> _samples;
|
||||
int _sampleRate = 48000;
|
||||
int _loopCount = 1;
|
||||
size_t _offset = 0;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif // TGCALLS_CALL_AUDIO_TONE_H_
|
||||
|
|
@ -0,0 +1,180 @@
|
|||
/*
|
||||
* Copyright 2016 The WebRTC Project Authors. All rights reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import "RTCAudioSession+Private.h"
|
||||
#import "RTCAudioSessionConfiguration.h"
|
||||
|
||||
#import "base/RTCLogging.h"
|
||||
|
||||
@implementation RTC_OBJC_TYPE (RTCAudioSession)
|
||||
(Configuration)
|
||||
|
||||
- (BOOL)setConfiguration : (RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration error
|
||||
: (NSError **)outError disableRecording:(BOOL)disableRecording {
|
||||
return [self setConfiguration:configuration
|
||||
active:NO
|
||||
shouldSetActive:NO
|
||||
error:outError
|
||||
disableRecording:disableRecording];
|
||||
}
|
||||
|
||||
- (BOOL)setConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration
|
||||
active:(BOOL)active
|
||||
error:(NSError **)outError disableRecording:(BOOL)disableRecording {
|
||||
return [self setConfiguration:configuration
|
||||
active:active
|
||||
shouldSetActive:YES
|
||||
error:outError
|
||||
disableRecording:disableRecording];
|
||||
}
|
||||
|
||||
#pragma mark - Private
|
||||
|
||||
- (BOOL)setConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration
|
||||
active:(BOOL)active
|
||||
shouldSetActive:(BOOL)shouldSetActive
|
||||
error:(NSError **)outError disableRecording:(BOOL)disableRecording {
|
||||
NSParameterAssert(configuration);
|
||||
if (outError) {
|
||||
*outError = nil;
|
||||
}
|
||||
|
||||
// Provide an error even if there isn't one so we can log it. We will not
|
||||
// return immediately on error in this function and instead try to set
|
||||
// everything we can.
|
||||
NSError *error = nil;
|
||||
|
||||
if (!disableRecording) {
|
||||
if (self.category != configuration.category ||
|
||||
self.categoryOptions != configuration.categoryOptions) {
|
||||
NSError *categoryError = nil;
|
||||
if (![self setCategory:configuration.category
|
||||
withOptions:configuration.categoryOptions
|
||||
error:&categoryError]) {
|
||||
RTCLogError(@"Failed to set category: %@",
|
||||
categoryError.localizedDescription);
|
||||
error = categoryError;
|
||||
} else {
|
||||
RTCLog(@"Set category to: %@", configuration.category);
|
||||
}
|
||||
}
|
||||
|
||||
if (self.mode != configuration.mode) {
|
||||
NSError *modeError = nil;
|
||||
if (![self setMode:configuration.mode error:&modeError]) {
|
||||
RTCLogError(@"Failed to set mode: %@",
|
||||
modeError.localizedDescription);
|
||||
error = modeError;
|
||||
} else {
|
||||
RTCLog(@"Set mode to: %@", configuration.mode);
|
||||
}
|
||||
}
|
||||
|
||||
// Sometimes category options don't stick after setting mode.
|
||||
if (self.categoryOptions != configuration.categoryOptions) {
|
||||
NSError *categoryError = nil;
|
||||
if (![self setCategory:configuration.category
|
||||
withOptions:configuration.categoryOptions
|
||||
error:&categoryError]) {
|
||||
RTCLogError(@"Failed to set category options: %@",
|
||||
categoryError.localizedDescription);
|
||||
error = categoryError;
|
||||
} else {
|
||||
RTCLog(@"Set category options to: %ld",
|
||||
(long)configuration.categoryOptions);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (self.preferredSampleRate != configuration.sampleRate) {
|
||||
NSError *sampleRateError = nil;
|
||||
if (![self setPreferredSampleRate:configuration.sampleRate
|
||||
error:&sampleRateError]) {
|
||||
RTCLogError(@"Failed to set preferred sample rate: %@",
|
||||
sampleRateError.localizedDescription);
|
||||
if (!self.ignoresPreferredAttributeConfigurationErrors) {
|
||||
error = sampleRateError;
|
||||
}
|
||||
} else {
|
||||
RTCLog(@"Set preferred sample rate to: %.2f",
|
||||
configuration.sampleRate);
|
||||
}
|
||||
}
|
||||
|
||||
if (self.preferredIOBufferDuration != configuration.ioBufferDuration) {
|
||||
NSError *bufferDurationError = nil;
|
||||
if (![self setPreferredIOBufferDuration:configuration.ioBufferDuration
|
||||
error:&bufferDurationError]) {
|
||||
RTCLogError(@"Failed to set preferred IO buffer duration: %@",
|
||||
bufferDurationError.localizedDescription);
|
||||
if (!self.ignoresPreferredAttributeConfigurationErrors) {
|
||||
error = bufferDurationError;
|
||||
}
|
||||
} else {
|
||||
RTCLog(@"Set preferred IO buffer duration to: %f",
|
||||
configuration.ioBufferDuration);
|
||||
}
|
||||
}
|
||||
|
||||
if (shouldSetActive) {
|
||||
NSError *activeError = nil;
|
||||
if (![self setActive:active error:&activeError]) {
|
||||
RTCLogError(@"Failed to setActive to %d: %@",
|
||||
active, activeError.localizedDescription);
|
||||
error = activeError;
|
||||
}
|
||||
}
|
||||
|
||||
if (self.isActive &&
|
||||
// TODO(tkchin): Figure out which category/mode numChannels is valid for.
|
||||
[self.mode isEqualToString:AVAudioSessionModeVoiceChat]) {
|
||||
// Try to set the preferred number of hardware audio channels. These calls
|
||||
// must be done after setting the audio session’s category and mode and
|
||||
// activating the session.
|
||||
NSInteger inputNumberOfChannels = configuration.inputNumberOfChannels;
|
||||
if (self.inputNumberOfChannels != inputNumberOfChannels) {
|
||||
NSError *inputChannelsError = nil;
|
||||
if (![self setPreferredInputNumberOfChannels:inputNumberOfChannels
|
||||
error:&inputChannelsError]) {
|
||||
RTCLogError(@"Failed to set preferred input number of channels: %@",
|
||||
inputChannelsError.localizedDescription);
|
||||
if (!self.ignoresPreferredAttributeConfigurationErrors) {
|
||||
error = inputChannelsError;
|
||||
}
|
||||
} else {
|
||||
RTCLog(@"Set input number of channels to: %ld",
|
||||
(long)inputNumberOfChannels);
|
||||
}
|
||||
}
|
||||
NSInteger outputNumberOfChannels = configuration.outputNumberOfChannels;
|
||||
if (self.outputNumberOfChannels != outputNumberOfChannels) {
|
||||
NSError *outputChannelsError = nil;
|
||||
if (![self setPreferredOutputNumberOfChannels:outputNumberOfChannels
|
||||
error:&outputChannelsError]) {
|
||||
RTCLogError(@"Failed to set preferred output number of channels: %@",
|
||||
outputChannelsError.localizedDescription);
|
||||
if (!self.ignoresPreferredAttributeConfigurationErrors) {
|
||||
error = outputChannelsError;
|
||||
}
|
||||
} else {
|
||||
RTCLog(@"Set output number of channels to: %ld",
|
||||
(long)outputNumberOfChannels);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (outError) {
|
||||
*outError = error;
|
||||
}
|
||||
|
||||
return error == nil;
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
/*
|
||||
* Copyright 2016 The WebRTC Project Authors. All rights reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import "RTCAudioSession.h"
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
@class RTC_OBJC_TYPE(RTCAudioSessionConfiguration);
|
||||
|
||||
@interface RTC_OBJC_TYPE (RTCAudioSession)
|
||||
()
|
||||
|
||||
/** Number of times setActive:YES has succeeded without a balanced call to
|
||||
* setActive:NO.
|
||||
*/
|
||||
@property(nonatomic, readonly) int activationCount;
|
||||
|
||||
/** The number of times `beginWebRTCSession` was called without a balanced call
|
||||
* to `endWebRTCSession`.
|
||||
*/
|
||||
@property(nonatomic, readonly) int webRTCSessionCount;
|
||||
|
||||
/** Convenience BOOL that checks useManualAudio and isAudioEnebled. */
|
||||
@property(readonly) BOOL canPlayOrRecord;
|
||||
|
||||
/** Tracks whether we have been sent an interruption event that hasn't been matched by either an
|
||||
* interrupted end event or a foreground event.
|
||||
*/
|
||||
@property(nonatomic, assign) BOOL isInterrupted;
|
||||
|
||||
/** Adds the delegate to the list of delegates, and places it at the front of
|
||||
* the list. This delegate will be notified before other delegates of
|
||||
* audio events.
|
||||
*/
|
||||
- (void)pushDelegate:(id<RTC_OBJC_TYPE(RTCAudioSessionDelegate)>)delegate;
|
||||
|
||||
/** Signals RTCAudioSession that a WebRTC session is about to begin and
|
||||
* audio configuration is needed. Will configure the audio session for WebRTC
|
||||
* if not already configured and if configuration is not delayed.
|
||||
* Successful calls must be balanced by a call to endWebRTCSession.
|
||||
*/
|
||||
- (BOOL)beginWebRTCSession:(NSError **)outError;
|
||||
|
||||
/** Signals RTCAudioSession that a WebRTC session is about to end and audio
|
||||
* unconfiguration is needed. Will unconfigure the audio session for WebRTC
|
||||
* if this is the last unmatched call and if configuration is not delayed.
|
||||
*/
|
||||
- (BOOL)endWebRTCSession:(NSError **)outError;
|
||||
|
||||
/** Configure the audio session for WebRTC. This call will fail if the session
|
||||
* is already configured. On other failures, we will attempt to restore the
|
||||
* previously used audio session configuration.
|
||||
* `lockForConfiguration` must be called first.
|
||||
* Successful calls to configureWebRTCSession must be matched by calls to
|
||||
* `unconfigureWebRTCSession`.
|
||||
*/
|
||||
- (BOOL)configureWebRTCSession:(NSError **)outError disableRecording:(BOOL)disableRecording;
|
||||
|
||||
/** Unconfigures the session for WebRTC. This will attempt to restore the
|
||||
* audio session to the settings used before `configureWebRTCSession` was
|
||||
* called.
|
||||
* `lockForConfiguration` must be called first.
|
||||
*/
|
||||
- (BOOL)unconfigureWebRTCSession:(NSError **)outError;
|
||||
|
||||
/** Returns a configuration error with the given description. */
|
||||
- (NSError *)configurationErrorWithDescription:(NSString *)description;
|
||||
|
||||
/** Notifies the receiver that a playout glitch was detected. */
|
||||
- (void)notifyDidDetectPlayoutGlitch:(int64_t)totalNumberOfGlitches;
|
||||
|
||||
/** Notifies the receiver that there was an error when starting an audio unit. */
|
||||
- (void)notifyAudioUnitStartFailedWithError:(OSStatus)error;
|
||||
|
||||
// Properties and methods for tests.
|
||||
- (void)notifyDidBeginInterruption;
|
||||
- (void)notifyDidEndInterruptionWithShouldResumeSession:(BOOL)shouldResumeSession;
|
||||
- (void)notifyDidChangeRouteWithReason:(AVAudioSessionRouteChangeReason)reason
|
||||
previousRoute:(AVAudioSessionRouteDescription *)previousRoute;
|
||||
- (void)notifyMediaServicesWereLost;
|
||||
- (void)notifyMediaServicesWereReset;
|
||||
- (void)notifyDidChangeCanPlayOrRecord:(BOOL)canPlayOrRecord;
|
||||
- (void)notifyDidStartPlayOrRecord;
|
||||
- (void)notifyDidStopPlayOrRecord;
|
||||
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
|
|
@ -0,0 +1,266 @@
|
|||
/*
|
||||
* Copyright 2016 The WebRTC Project Authors. All rights reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
#import "RTCMacros.h"
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
extern NSString *const kRTCAudioSessionErrorDomain;
|
||||
/** Method that requires lock was called without lock. */
|
||||
extern NSInteger const kRTCAudioSessionErrorLockRequired;
|
||||
/** Unknown configuration error occurred. */
|
||||
extern NSInteger const kRTCAudioSessionErrorConfiguration;
|
||||
|
||||
@class RTC_OBJC_TYPE(RTCAudioSession);
|
||||
@class RTC_OBJC_TYPE(RTCAudioSessionConfiguration);
|
||||
|
||||
// Surfaces AVAudioSession events. WebRTC will listen directly for notifications
|
||||
// from AVAudioSession and handle them before calling these delegate methods,
|
||||
// at which point applications can perform additional processing if required.
|
||||
RTC_OBJC_EXPORT
|
||||
@protocol RTC_OBJC_TYPE
|
||||
(RTCAudioSessionDelegate)<NSObject>
|
||||
|
||||
@optional
|
||||
/** Called on a system notification thread when AVAudioSession starts an
|
||||
* interruption event.
|
||||
*/
|
||||
- (void)audioSessionDidBeginInterruption:(RTC_OBJC_TYPE(RTCAudioSession) *)session;
|
||||
|
||||
/** Called on a system notification thread when AVAudioSession ends an
|
||||
* interruption event.
|
||||
*/
|
||||
- (void)audioSessionDidEndInterruption:(RTC_OBJC_TYPE(RTCAudioSession) *)session
|
||||
shouldResumeSession:(BOOL)shouldResumeSession;
|
||||
|
||||
/** Called on a system notification thread when AVAudioSession changes the
|
||||
* route.
|
||||
*/
|
||||
- (void)audioSessionDidChangeRoute:(RTC_OBJC_TYPE(RTCAudioSession) *)session
|
||||
reason:(AVAudioSessionRouteChangeReason)reason
|
||||
previousRoute:(AVAudioSessionRouteDescription *)previousRoute;
|
||||
|
||||
/** Called on a system notification thread when AVAudioSession media server
|
||||
* terminates.
|
||||
*/
|
||||
- (void)audioSessionMediaServerTerminated:(RTC_OBJC_TYPE(RTCAudioSession) *)session;
|
||||
|
||||
/** Called on a system notification thread when AVAudioSession media server
|
||||
* restarts.
|
||||
*/
|
||||
- (void)audioSessionMediaServerReset:(RTC_OBJC_TYPE(RTCAudioSession) *)session;
|
||||
|
||||
// TODO(tkchin): Maybe handle SilenceSecondaryAudioHintNotification.
|
||||
|
||||
- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)session
|
||||
didChangeCanPlayOrRecord:(BOOL)canPlayOrRecord;
|
||||
|
||||
/** Called on a WebRTC thread when the audio device is notified to begin
|
||||
* playback or recording.
|
||||
*/
|
||||
- (void)audioSessionDidStartPlayOrRecord:(RTC_OBJC_TYPE(RTCAudioSession) *)session;
|
||||
|
||||
/** Called on a WebRTC thread when the audio device is notified to stop
|
||||
* playback or recording.
|
||||
*/
|
||||
- (void)audioSessionDidStopPlayOrRecord:(RTC_OBJC_TYPE(RTCAudioSession) *)session;
|
||||
|
||||
/** Called when the AVAudioSession output volume value changes. */
|
||||
- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession
|
||||
didChangeOutputVolume:(float)outputVolume;
|
||||
|
||||
/** Called when the audio device detects a playout glitch. The argument is the
|
||||
* number of glitches detected so far in the current audio playout session.
|
||||
*/
|
||||
- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession
|
||||
didDetectPlayoutGlitch:(int64_t)totalNumberOfGlitches;
|
||||
|
||||
/** Called when the audio session is about to change the active state.
|
||||
*/
|
||||
- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession willSetActive:(BOOL)active;
|
||||
|
||||
/** Called after the audio session sucessfully changed the active state.
|
||||
*/
|
||||
- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession didSetActive:(BOOL)active;
|
||||
|
||||
/** Called after the audio session failed to change the active state.
|
||||
*/
|
||||
- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession
|
||||
failedToSetActive:(BOOL)active
|
||||
error:(NSError *)error;
|
||||
|
||||
- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession
|
||||
audioUnitStartFailedWithError:(NSError *)error;
|
||||
|
||||
@end
|
||||
|
||||
/** This is a protocol used to inform RTCAudioSession when the audio session
|
||||
* activation state has changed outside of RTCAudioSession. The current known use
|
||||
* case of this is when CallKit activates the audio session for the application
|
||||
*/
|
||||
RTC_OBJC_EXPORT
|
||||
@protocol RTC_OBJC_TYPE
|
||||
(RTCAudioSessionActivationDelegate)<NSObject>
|
||||
|
||||
/** Called when the audio session is activated outside of the app by iOS. */
|
||||
- (void)audioSessionDidActivate : (AVAudioSession *)session;
|
||||
|
||||
/** Called when the audio session is deactivated outside of the app by iOS. */
|
||||
- (void)audioSessionDidDeactivate:(AVAudioSession *)session;
|
||||
|
||||
@end
|
||||
|
||||
/** Proxy class for AVAudioSession that adds a locking mechanism similar to
|
||||
* AVCaptureDevice. This is used to that interleaving configurations between
|
||||
* WebRTC and the application layer are avoided.
|
||||
*
|
||||
* RTCAudioSession also coordinates activation so that the audio session is
|
||||
* activated only once. See `setActive:error:`.
|
||||
*/
|
||||
RTC_OBJC_EXPORT
|
||||
@interface RTC_OBJC_TYPE (RTCAudioSession) : NSObject <RTC_OBJC_TYPE(RTCAudioSessionActivationDelegate)>
|
||||
|
||||
/** Convenience property to access the AVAudioSession singleton. Callers should
|
||||
* not call setters on AVAudioSession directly, but other method invocations
|
||||
* are fine.
|
||||
*/
|
||||
@property(nonatomic, readonly) AVAudioSession *session;
|
||||
|
||||
/** Our best guess at whether the session is active based on results of calls to
|
||||
* AVAudioSession.
|
||||
*/
|
||||
@property(nonatomic, readonly) BOOL isActive;
|
||||
|
||||
/** If YES, WebRTC will not initialize the audio unit automatically when an
|
||||
* audio track is ready for playout or recording. Instead, applications should
|
||||
* call setIsAudioEnabled. If NO, WebRTC will initialize the audio unit
|
||||
* as soon as an audio track is ready for playout or recording.
|
||||
*/
|
||||
@property(nonatomic, assign) BOOL useManualAudio;
|
||||
|
||||
/** This property is only effective if useManualAudio is YES.
|
||||
* Represents permission for WebRTC to initialize the VoIP audio unit.
|
||||
* When set to NO, if the VoIP audio unit used by WebRTC is active, it will be
|
||||
* stopped and uninitialized. This will stop incoming and outgoing audio.
|
||||
* When set to YES, WebRTC will initialize and start the audio unit when it is
|
||||
* needed (e.g. due to establishing an audio connection).
|
||||
* This property was introduced to work around an issue where if an AVPlayer is
|
||||
* playing audio while the VoIP audio unit is initialized, its audio would be
|
||||
* either cut off completely or played at a reduced volume. By preventing
|
||||
* the audio unit from being initialized until after the audio has completed,
|
||||
* we are able to prevent the abrupt cutoff.
|
||||
*/
|
||||
@property(nonatomic, assign) BOOL isAudioEnabled;
|
||||
|
||||
// Proxy properties.
|
||||
@property(readonly) NSString *category;
|
||||
@property(readonly) AVAudioSessionCategoryOptions categoryOptions;
|
||||
@property(readonly) NSString *mode;
|
||||
@property(readonly) BOOL secondaryAudioShouldBeSilencedHint;
|
||||
@property(readonly) AVAudioSessionRouteDescription *currentRoute;
|
||||
@property(readonly) NSInteger maximumInputNumberOfChannels;
|
||||
@property(readonly) NSInteger maximumOutputNumberOfChannels;
|
||||
@property(readonly) float inputGain;
|
||||
@property(readonly) BOOL inputGainSettable;
|
||||
@property(readonly) BOOL inputAvailable;
|
||||
@property(readonly, nullable) NSArray<AVAudioSessionDataSourceDescription *> *inputDataSources;
|
||||
@property(readonly, nullable) AVAudioSessionDataSourceDescription *inputDataSource;
|
||||
@property(readonly, nullable) NSArray<AVAudioSessionDataSourceDescription *> *outputDataSources;
|
||||
@property(readonly, nullable) AVAudioSessionDataSourceDescription *outputDataSource;
|
||||
@property(readonly) double sampleRate;
|
||||
@property(readonly) double preferredSampleRate;
|
||||
@property(readonly) NSInteger inputNumberOfChannels;
|
||||
@property(readonly) NSInteger outputNumberOfChannels;
|
||||
@property(readonly) float outputVolume;
|
||||
@property(readonly) NSTimeInterval inputLatency;
|
||||
@property(readonly) NSTimeInterval outputLatency;
|
||||
@property(readonly) NSTimeInterval IOBufferDuration;
|
||||
@property(readonly) NSTimeInterval preferredIOBufferDuration;
|
||||
|
||||
/**
|
||||
When YES, calls to -setConfiguration:error: and -setConfiguration:active:error: ignore errors in
|
||||
configuring the audio session's "preferred" attributes (e.g. preferredInputNumberOfChannels).
|
||||
Typically, configurations to preferred attributes are optimizations, and ignoring this type of
|
||||
configuration error allows code flow to continue along the happy path when these optimization are
|
||||
not available. The default value of this property is NO.
|
||||
*/
|
||||
@property(nonatomic) BOOL ignoresPreferredAttributeConfigurationErrors;
|
||||
|
||||
/** Default constructor. */
|
||||
+ (instancetype)sharedInstance;
|
||||
- (instancetype)init NS_UNAVAILABLE;
|
||||
|
||||
/** Adds a delegate, which is held weakly. */
|
||||
- (void)addDelegate:(id<RTC_OBJC_TYPE(RTCAudioSessionDelegate)>)delegate;
|
||||
/** Removes an added delegate. */
|
||||
- (void)removeDelegate:(id<RTC_OBJC_TYPE(RTCAudioSessionDelegate)>)delegate;
|
||||
|
||||
/** Request exclusive access to the audio session for configuration. This call
|
||||
* will block if the lock is held by another object.
|
||||
*/
|
||||
- (void)lockForConfiguration;
|
||||
/** Relinquishes exclusive access to the audio session. */
|
||||
- (void)unlockForConfiguration;
|
||||
|
||||
/** If `active`, activates the audio session if it isn't already active.
|
||||
* Successful calls must be balanced with a setActive:NO when activation is no
|
||||
* longer required. If not `active`, deactivates the audio session if one is
|
||||
* active and this is the last balanced call. When deactivating, the
|
||||
* AVAudioSessionSetActiveOptionNotifyOthersOnDeactivation option is passed to
|
||||
* AVAudioSession.
|
||||
*/
|
||||
- (BOOL)setActive:(BOOL)active error:(NSError **)outError;
|
||||
|
||||
// The following methods are proxies for the associated methods on
|
||||
// AVAudioSession. `lockForConfiguration` must be called before using them
|
||||
// otherwise they will fail with kRTCAudioSessionErrorLockRequired.
|
||||
|
||||
- (BOOL)setCategory:(NSString *)category
|
||||
withOptions:(AVAudioSessionCategoryOptions)options
|
||||
error:(NSError **)outError;
|
||||
- (BOOL)setMode:(NSString *)mode error:(NSError **)outError;
|
||||
- (BOOL)setInputGain:(float)gain error:(NSError **)outError;
|
||||
- (BOOL)setPreferredSampleRate:(double)sampleRate error:(NSError **)outError;
|
||||
- (BOOL)setPreferredIOBufferDuration:(NSTimeInterval)duration error:(NSError **)outError;
|
||||
- (BOOL)setPreferredInputNumberOfChannels:(NSInteger)count error:(NSError **)outError;
|
||||
- (BOOL)setPreferredOutputNumberOfChannels:(NSInteger)count error:(NSError **)outError;
|
||||
- (BOOL)overrideOutputAudioPort:(AVAudioSessionPortOverride)portOverride error:(NSError **)outError;
|
||||
- (BOOL)setPreferredInput:(AVAudioSessionPortDescription *)inPort error:(NSError **)outError;
|
||||
- (BOOL)setInputDataSource:(AVAudioSessionDataSourceDescription *)dataSource
|
||||
error:(NSError **)outError;
|
||||
- (BOOL)setOutputDataSource:(AVAudioSessionDataSourceDescription *)dataSource
|
||||
error:(NSError **)outError;
|
||||
@end
|
||||
|
||||
@interface RTC_OBJC_TYPE (RTCAudioSession)
|
||||
(Configuration)
|
||||
|
||||
/** Applies the configuration to the current session. Attempts to set all
|
||||
* properties even if previous ones fail. Only the last error will be
|
||||
* returned.
|
||||
* `lockForConfiguration` must be called first.
|
||||
*/
|
||||
- (BOOL)setConfiguration : (RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration error
|
||||
: (NSError **)outError;
|
||||
|
||||
/** Convenience method that calls both setConfiguration and setActive.
|
||||
* `lockForConfiguration` must be called first.
|
||||
*/
|
||||
- (BOOL)setConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration
|
||||
active:(BOOL)active
|
||||
error:(NSError **)outError
|
||||
disableRecording:(BOOL)disableRecording;
|
||||
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,47 @@
|
|||
/*
|
||||
* Copyright 2016 The WebRTC Project Authors. All rights reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
#import "RTCMacros.h"
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
RTC_EXTERN const double kRTCAudioSessionHighPerformanceSampleRate;
|
||||
RTC_EXTERN const double kRTCAudioSessionLowComplexitySampleRate;
|
||||
RTC_EXTERN const double kRTCAudioSessionHighPerformanceIOBufferDuration;
|
||||
RTC_EXTERN const double kRTCAudioSessionLowComplexityIOBufferDuration;
|
||||
|
||||
// Struct to hold configuration values.
|
||||
RTC_OBJC_EXPORT
|
||||
@interface RTC_OBJC_TYPE (RTCAudioSessionConfiguration) : NSObject
|
||||
|
||||
@property(nonatomic, strong) NSString *category;
|
||||
@property(nonatomic, assign) AVAudioSessionCategoryOptions categoryOptions;
|
||||
@property(nonatomic, strong) NSString *mode;
|
||||
@property(nonatomic, assign) double sampleRate;
|
||||
@property(nonatomic, assign) NSTimeInterval ioBufferDuration;
|
||||
@property(nonatomic, assign) NSInteger inputNumberOfChannels;
|
||||
@property(nonatomic, assign) NSInteger outputNumberOfChannels;
|
||||
|
||||
/** Initializes configuration to defaults. */
|
||||
- (instancetype)init NS_DESIGNATED_INITIALIZER;
|
||||
|
||||
/** Returns the current configuration of the audio session. */
|
||||
+ (instancetype)currentConfiguration;
|
||||
/** Returns the configuration that WebRTC needs. */
|
||||
+ (instancetype)webRTCConfiguration;
|
||||
/** Provide a way to override the default configuration. */
|
||||
+ (void)setWebRTCConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration;
|
||||
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
|
|
@ -0,0 +1,133 @@
|
|||
/*
|
||||
* Copyright 2016 The WebRTC Project Authors. All rights reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import "RTCAudioSessionConfiguration.h"
|
||||
#import "RTCAudioSession.h"
|
||||
|
||||
#import "helpers/RTCDispatcher.h"
|
||||
#import "helpers/UIDevice+RTCDevice.h"
|
||||
|
||||
// Try to use mono to save resources. Also avoids channel format conversion
|
||||
// in the I/O audio unit. Initial tests have shown that it is possible to use
|
||||
// mono natively for built-in microphones and for BT headsets but not for
|
||||
// wired headsets. Wired headsets only support stereo as native channel format
|
||||
// but it is a low cost operation to do a format conversion to mono in the
|
||||
// audio unit. Hence, we will not hit a RTC_CHECK in
|
||||
// VerifyAudioParametersForActiveAudioSession() for a mismatch between the
|
||||
// preferred number of channels and the actual number of channels.
|
||||
const int kRTCAudioSessionPreferredNumberOfChannels = 1;
|
||||
|
||||
// Preferred hardware sample rate (unit is in Hertz). The client sample rate
|
||||
// will be set to this value as well to avoid resampling the the audio unit's
|
||||
// format converter. Note that, some devices, e.g. BT headsets, only supports
|
||||
// 8000Hz as native sample rate.
|
||||
const double kRTCAudioSessionHighPerformanceSampleRate = 48000.0;
|
||||
|
||||
// A lower sample rate will be used for devices with only one core
|
||||
// (e.g. iPhone 4). The goal is to reduce the CPU load of the application.
|
||||
const double kRTCAudioSessionLowComplexitySampleRate = 16000.0;
|
||||
|
||||
// Use a hardware I/O buffer size (unit is in seconds) that matches the 10ms
|
||||
// size used by WebRTC. The exact actual size will differ between devices.
|
||||
// Example: using 48kHz on iPhone 6 results in a native buffer size of
|
||||
// ~10.6667ms or 512 audio frames per buffer. The FineAudioBuffer instance will
|
||||
// take care of any buffering required to convert between native buffers and
|
||||
// buffers used by WebRTC. It is beneficial for the performance if the native
|
||||
// size is as an even multiple of 10ms as possible since it results in "clean"
|
||||
// callback sequence without bursts of callbacks back to back.
|
||||
const double kRTCAudioSessionHighPerformanceIOBufferDuration = 0.02;
|
||||
|
||||
// Use a larger buffer size on devices with only one core (e.g. iPhone 4).
|
||||
// It will result in a lower CPU consumption at the cost of a larger latency.
|
||||
// The size of 60ms is based on instrumentation that shows a significant
|
||||
// reduction in CPU load compared with 10ms on low-end devices.
|
||||
// TODO(henrika): monitor this size and determine if it should be modified.
|
||||
const double kRTCAudioSessionLowComplexityIOBufferDuration = 0.06;
|
||||
|
||||
static RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *gWebRTCConfiguration = nil;
|
||||
|
||||
@implementation RTC_OBJC_TYPE (RTCAudioSessionConfiguration)
|
||||
|
||||
@synthesize category = _category;
|
||||
@synthesize categoryOptions = _categoryOptions;
|
||||
@synthesize mode = _mode;
|
||||
@synthesize sampleRate = _sampleRate;
|
||||
@synthesize ioBufferDuration = _ioBufferDuration;
|
||||
@synthesize inputNumberOfChannels = _inputNumberOfChannels;
|
||||
@synthesize outputNumberOfChannels = _outputNumberOfChannels;
|
||||
|
||||
- (instancetype)init {
|
||||
if (self = [super init]) {
|
||||
// Use a category which supports simultaneous recording and playback.
|
||||
// By default, using this category implies that our app’s audio is
|
||||
// nonmixable, hence activating the session will interrupt any other
|
||||
// audio sessions which are also nonmixable.
|
||||
_category = AVAudioSessionCategoryPlayAndRecord;
|
||||
_categoryOptions = AVAudioSessionCategoryOptionAllowBluetooth;
|
||||
|
||||
// Specify mode for two-way voice communication (e.g. VoIP).
|
||||
_mode = AVAudioSessionModeVoiceChat;
|
||||
|
||||
// Set the session's sample rate or the hardware sample rate.
|
||||
// It is essential that we use the same sample rate as stream format
|
||||
// to ensure that the I/O unit does not have to do sample rate conversion.
|
||||
// Set the preferred audio I/O buffer duration, in seconds.
|
||||
NSUInteger processorCount = [NSProcessInfo processInfo].processorCount;
|
||||
// Use best sample rate and buffer duration if the CPU has more than one
|
||||
// core.
|
||||
if (processorCount > 1) {
|
||||
_sampleRate = kRTCAudioSessionHighPerformanceSampleRate;
|
||||
_ioBufferDuration = kRTCAudioSessionHighPerformanceIOBufferDuration;
|
||||
} else {
|
||||
_sampleRate = kRTCAudioSessionLowComplexitySampleRate;
|
||||
_ioBufferDuration = kRTCAudioSessionLowComplexityIOBufferDuration;
|
||||
}
|
||||
|
||||
// We try to use mono in both directions to save resources and format
|
||||
// conversions in the audio unit. Some devices does only support stereo;
|
||||
// e.g. wired headset on iPhone 6.
|
||||
// TODO(henrika): add support for stereo if needed.
|
||||
_inputNumberOfChannels = kRTCAudioSessionPreferredNumberOfChannels;
|
||||
_outputNumberOfChannels = kRTCAudioSessionPreferredNumberOfChannels;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
+ (void)initialize {
|
||||
gWebRTCConfiguration = [[self alloc] init];
|
||||
}
|
||||
|
||||
+ (instancetype)currentConfiguration {
|
||||
RTC_OBJC_TYPE(RTCAudioSession) *session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance];
|
||||
RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *config =
|
||||
[[RTC_OBJC_TYPE(RTCAudioSessionConfiguration) alloc] init];
|
||||
config.category = session.category;
|
||||
config.categoryOptions = session.categoryOptions;
|
||||
config.mode = session.mode;
|
||||
config.sampleRate = session.sampleRate;
|
||||
config.ioBufferDuration = session.IOBufferDuration;
|
||||
config.inputNumberOfChannels = session.inputNumberOfChannels;
|
||||
config.outputNumberOfChannels = session.outputNumberOfChannels;
|
||||
return config;
|
||||
}
|
||||
|
||||
+ (instancetype)webRTCConfiguration {
|
||||
@synchronized(self) {
|
||||
return (RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)gWebRTCConfiguration;
|
||||
}
|
||||
}
|
||||
|
||||
+ (void)setWebRTCConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration {
|
||||
@synchronized(self) {
|
||||
gWebRTCConfiguration = configuration;
|
||||
}
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
/*
|
||||
* Copyright 2018 The WebRTC Project Authors. All rights reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import "RTCAudioSession.h"
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
namespace webrtc {
|
||||
class AudioSessionObserver;
|
||||
}
|
||||
|
||||
/** Adapter that forwards RTCAudioSessionDelegate calls to the appropriate
|
||||
* methods on the AudioSessionObserver.
|
||||
*/
|
||||
@interface RTCNativeAudioSessionDelegateAdapter : NSObject <RTC_OBJC_TYPE (RTCAudioSessionDelegate)>
|
||||
|
||||
- (instancetype)init NS_UNAVAILABLE;
|
||||
|
||||
/** `observer` is a raw pointer and should be kept alive
|
||||
* for this object's lifetime.
|
||||
*/
|
||||
- (instancetype)initWithObserver:(webrtc::AudioSessionObserver *)observer NS_DESIGNATED_INITIALIZER;
|
||||
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
/*
|
||||
* Copyright 2018 The WebRTC Project Authors. All rights reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import "RTCNativeAudioSessionDelegateAdapter.h"
|
||||
|
||||
#include "sdk/objc/native/src/audio/audio_session_observer.h"
|
||||
|
||||
#import "base/RTCLogging.h"
|
||||
|
||||
@implementation RTCNativeAudioSessionDelegateAdapter {
|
||||
webrtc::AudioSessionObserver *_observer;
|
||||
}
|
||||
|
||||
- (instancetype)initWithObserver:(webrtc::AudioSessionObserver *)observer {
|
||||
RTC_DCHECK(observer);
|
||||
if (self = [super init]) {
|
||||
_observer = observer;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
#pragma mark - RTC_OBJC_TYPE(RTCAudioSessionDelegate)
|
||||
|
||||
- (void)audioSessionDidBeginInterruption:(RTC_OBJC_TYPE(RTCAudioSession) *)session {
|
||||
_observer->OnInterruptionBegin();
|
||||
}
|
||||
|
||||
- (void)audioSessionDidEndInterruption:(RTC_OBJC_TYPE(RTCAudioSession) *)session
|
||||
shouldResumeSession:(BOOL)shouldResumeSession {
|
||||
_observer->OnInterruptionEnd();
|
||||
}
|
||||
|
||||
- (void)audioSessionDidChangeRoute:(RTC_OBJC_TYPE(RTCAudioSession) *)session
|
||||
reason:(AVAudioSessionRouteChangeReason)reason
|
||||
previousRoute:(AVAudioSessionRouteDescription *)previousRoute {
|
||||
switch (reason) {
|
||||
case AVAudioSessionRouteChangeReasonUnknown:
|
||||
case AVAudioSessionRouteChangeReasonNewDeviceAvailable:
|
||||
case AVAudioSessionRouteChangeReasonOldDeviceUnavailable:
|
||||
case AVAudioSessionRouteChangeReasonCategoryChange:
|
||||
// It turns out that we see a category change (at least in iOS 9.2)
|
||||
// when making a switch from a BT device to e.g. Speaker using the
|
||||
// iOS Control Center and that we therefore must check if the sample
|
||||
// rate has changed. And if so is the case, restart the audio unit.
|
||||
case AVAudioSessionRouteChangeReasonOverride:
|
||||
case AVAudioSessionRouteChangeReasonWakeFromSleep:
|
||||
case AVAudioSessionRouteChangeReasonNoSuitableRouteForCategory:
|
||||
_observer->OnValidRouteChange();
|
||||
break;
|
||||
case AVAudioSessionRouteChangeReasonRouteConfigurationChange:
|
||||
// The set of input and output ports has not changed, but their
|
||||
// configuration has, e.g., a port’s selected data source has
|
||||
// changed. Ignore this type of route change since we are focusing
|
||||
// on detecting headset changes.
|
||||
RTCLog(@"Ignoring RouteConfigurationChange");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)audioSessionMediaServerTerminated:(RTC_OBJC_TYPE(RTCAudioSession) *)session {
|
||||
}
|
||||
|
||||
- (void)audioSessionMediaServerReset:(RTC_OBJC_TYPE(RTCAudioSession) *)session {
|
||||
}
|
||||
|
||||
- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)session
|
||||
didChangeCanPlayOrRecord:(BOOL)canPlayOrRecord {
|
||||
_observer->OnCanPlayOrRecordChange(canPlayOrRecord);
|
||||
}
|
||||
|
||||
- (void)audioSessionDidStartPlayOrRecord:(RTC_OBJC_TYPE(RTCAudioSession) *)session {
|
||||
}
|
||||
|
||||
- (void)audioSessionDidStopPlayOrRecord:(RTC_OBJC_TYPE(RTCAudioSession) *)session {
|
||||
}
|
||||
|
||||
- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession
|
||||
didChangeOutputVolume:(float)outputVolume {
|
||||
_observer->OnChangedOutputVolume();
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,329 @@
|
|||
/*
|
||||
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef TGCALLS_AUDIO_AUDIO_DEVICE_IOS_H_
|
||||
#define TGCALLS_AUDIO_AUDIO_DEVICE_IOS_H_
|
||||
|
||||
#include <atomic>
|
||||
#include <memory>
|
||||
|
||||
#include "api/scoped_refptr.h"
|
||||
#include "api/sequence_checker.h"
|
||||
#include "sdk/objc/native/src/audio/audio_session_observer.h"
|
||||
#include "api/task_queue/pending_task_safety_flag.h"
|
||||
#include "modules/audio_device/audio_device_generic.h"
|
||||
#include "rtc_base/buffer.h"
|
||||
#include "rtc_base/thread.h"
|
||||
#include "rtc_base/thread_annotations.h"
|
||||
#include "sdk/objc/base/RTCMacros.h"
|
||||
#include "tgcalls_voice_processing_audio_unit.h"
|
||||
|
||||
#include "platform/darwin/iOS/CallAudioTone.h"
|
||||
|
||||
RTC_FWD_DECL_OBJC_CLASS(RTCNativeAudioSessionDelegateAdapter);
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
class FineAudioBuffer;
|
||||
|
||||
namespace tgcalls_ios_adm {
|
||||
|
||||
// Implements full duplex 16-bit mono PCM audio support for iOS using a
|
||||
// Voice-Processing (VP) I/O audio unit in Core Audio. The VP I/O audio unit
|
||||
// supports audio echo cancellation. It also adds automatic gain control,
|
||||
// adjustment of voice-processing quality and muting.
|
||||
//
|
||||
// An instance must be created and destroyed on one and the same thread.
|
||||
// All supported public methods must also be called on the same thread.
|
||||
// A thread checker will RTC_DCHECK if any supported method is called on an
|
||||
// invalid thread.
|
||||
//
|
||||
// Recorded audio will be delivered on a real-time internal I/O thread in the
|
||||
// audio unit. The audio unit will also ask for audio data to play out on this
|
||||
// same thread.
|
||||
class AudioDeviceIOS : public AudioDeviceGeneric,
|
||||
public AudioSessionObserver,
|
||||
public VoiceProcessingAudioUnitObserver {
|
||||
public:
|
||||
explicit AudioDeviceIOS(bool bypass_voice_processing, bool disable_recording, bool enableSystemMute, int numChannels);
|
||||
~AudioDeviceIOS() override;
|
||||
|
||||
void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) override;
|
||||
|
||||
InitStatus Init() override;
|
||||
int32_t Terminate() override;
|
||||
bool Initialized() const override;
|
||||
|
||||
int32_t InitPlayout() override;
|
||||
bool PlayoutIsInitialized() const override;
|
||||
|
||||
int32_t InitRecording() override;
|
||||
bool RecordingIsInitialized() const override;
|
||||
|
||||
int32_t StartPlayout() override;
|
||||
int32_t StopPlayout() override;
|
||||
bool Playing() const override;
|
||||
|
||||
int32_t StartRecording() override;
|
||||
int32_t StopRecording() override;
|
||||
bool Recording() const override;
|
||||
|
||||
void setIsBufferPlaying(bool isBufferPlaying);
|
||||
void setIsBufferRecording(bool isBufferRecording);
|
||||
|
||||
// These methods returns hard-coded delay values and not dynamic delay
|
||||
// estimates. The reason is that iOS supports a built-in AEC and the WebRTC
|
||||
// AEC will always be disabled in the Libjingle layer to avoid running two
|
||||
// AEC implementations at the same time. And, it saves resources to avoid
|
||||
// updating these delay values continuously.
|
||||
// TODO(henrika): it would be possible to mark these two methods as not
|
||||
// implemented since they are only called for A/V-sync purposes today and
|
||||
// A/V-sync is not supported on iOS. However, we avoid adding error messages
|
||||
// the log by using these dummy implementations instead.
|
||||
int32_t PlayoutDelay(uint16_t& delayMS) const override;
|
||||
|
||||
// No implementation for playout underrun on iOS. We override it to avoid a
|
||||
// periodic log that it isn't available from the base class.
|
||||
int32_t GetPlayoutUnderrunCount() const override { return -1; }
|
||||
|
||||
// Native audio parameters stored during construction.
|
||||
// These methods are unique for the iOS implementation.
|
||||
int GetPlayoutAudioParameters(AudioParameters* params) const override;
|
||||
int GetRecordAudioParameters(AudioParameters* params) const override;
|
||||
|
||||
// These methods are currently not fully implemented on iOS:
|
||||
|
||||
// See audio_device_not_implemented.cc for trivial implementations.
|
||||
int32_t ActiveAudioLayer(
|
||||
AudioDeviceModule::AudioLayer& audioLayer) const override;
|
||||
int32_t PlayoutIsAvailable(bool& available) override;
|
||||
int32_t RecordingIsAvailable(bool& available) override;
|
||||
int16_t PlayoutDevices() override;
|
||||
int16_t RecordingDevices() override;
|
||||
int32_t PlayoutDeviceName(uint16_t index,
|
||||
char name[kAdmMaxDeviceNameSize],
|
||||
char guid[kAdmMaxGuidSize]) override;
|
||||
int32_t RecordingDeviceName(uint16_t index,
|
||||
char name[kAdmMaxDeviceNameSize],
|
||||
char guid[kAdmMaxGuidSize]) override;
|
||||
int32_t SetPlayoutDevice(uint16_t index) override;
|
||||
int32_t SetPlayoutDevice(
|
||||
AudioDeviceModule::WindowsDeviceType device) override;
|
||||
int32_t SetRecordingDevice(uint16_t index) override;
|
||||
int32_t SetRecordingDevice(
|
||||
AudioDeviceModule::WindowsDeviceType device) override;
|
||||
int32_t InitSpeaker() override;
|
||||
bool SpeakerIsInitialized() const override;
|
||||
int32_t InitMicrophone() override;
|
||||
bool MicrophoneIsInitialized() const override;
|
||||
int32_t SpeakerVolumeIsAvailable(bool& available) override;
|
||||
int32_t SetSpeakerVolume(uint32_t volume) override;
|
||||
int32_t SpeakerVolume(uint32_t& volume) const override;
|
||||
int32_t MaxSpeakerVolume(uint32_t& maxVolume) const override;
|
||||
int32_t MinSpeakerVolume(uint32_t& minVolume) const override;
|
||||
int32_t MicrophoneVolumeIsAvailable(bool& available) override;
|
||||
int32_t SetMicrophoneVolume(uint32_t volume) override;
|
||||
int32_t MicrophoneVolume(uint32_t& volume) const override;
|
||||
int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const override;
|
||||
int32_t MinMicrophoneVolume(uint32_t& minVolume) const override;
|
||||
int32_t MicrophoneMuteIsAvailable(bool& available) override;
|
||||
int32_t SetMicrophoneMute(bool enable) override;
|
||||
int32_t MicrophoneMute(bool& enabled) const override;
|
||||
int32_t SpeakerMuteIsAvailable(bool& available) override;
|
||||
int32_t SetSpeakerMute(bool enable) override;
|
||||
int32_t SpeakerMute(bool& enabled) const override;
|
||||
int32_t StereoPlayoutIsAvailable(bool& available) override;
|
||||
int32_t SetStereoPlayout(bool enable) override;
|
||||
int32_t StereoPlayout(bool& enabled) const override;
|
||||
int32_t StereoRecordingIsAvailable(bool& available) override;
|
||||
int32_t SetStereoRecording(bool enable) override;
|
||||
int32_t StereoRecording(bool& enabled) const override;
|
||||
|
||||
// AudioSessionObserver methods. May be called from any thread.
|
||||
void OnInterruptionBegin() override;
|
||||
void OnInterruptionEnd() override;
|
||||
void OnValidRouteChange() override;
|
||||
void OnCanPlayOrRecordChange(bool can_play_or_record) override;
|
||||
void OnChangedOutputVolume() override;
|
||||
|
||||
void setTone(std::shared_ptr<tgcalls::CallAudioTone> tone);
|
||||
|
||||
// VoiceProcessingAudioUnitObserver methods.
|
||||
OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags,
|
||||
const AudioTimeStamp* time_stamp,
|
||||
UInt32 bus_number,
|
||||
UInt32 num_frames,
|
||||
AudioBufferList* io_data) override;
|
||||
OSStatus OnGetPlayoutData(AudioUnitRenderActionFlags* flags,
|
||||
const AudioTimeStamp* time_stamp,
|
||||
UInt32 bus_number,
|
||||
UInt32 num_frames,
|
||||
AudioBufferList* io_data) override;
|
||||
void OnMutedSpeechStatusChanged(bool isDetectingSpeech) override;
|
||||
|
||||
bool IsInterrupted();
|
||||
|
||||
public:
|
||||
void (^mutedSpeechDetectionChanged)(bool);
|
||||
|
||||
private:
|
||||
// Called by the relevant AudioSessionObserver methods on `thread_`.
|
||||
void HandleInterruptionBegin();
|
||||
void HandleInterruptionEnd();
|
||||
void HandleValidRouteChange();
|
||||
void HandleCanPlayOrRecordChange(bool can_play_or_record);
|
||||
void HandleSampleRateChange();
|
||||
void HandlePlayoutGlitchDetected();
|
||||
void HandleOutputVolumeChange();
|
||||
|
||||
// Uses current `playout_parameters_` and `record_parameters_` to inform the
|
||||
// audio device buffer (ADB) about our internal audio parameters.
|
||||
void UpdateAudioDeviceBuffer();
|
||||
|
||||
// Since the preferred audio parameters are only hints to the OS, the actual
|
||||
// values may be different once the AVAudioSession has been activated.
|
||||
// This method asks for the current hardware parameters and takes actions
|
||||
// if they should differ from what we have asked for initially. It also
|
||||
// defines `playout_parameters_` and `record_parameters_`.
|
||||
void SetupAudioBuffersForActiveAudioSession();
|
||||
|
||||
// Creates the audio unit.
|
||||
bool CreateAudioUnit();
|
||||
|
||||
// Updates the audio unit state based on current state.
|
||||
void UpdateAudioUnit(bool can_play_or_record);
|
||||
|
||||
// Configures the audio session for WebRTC.
|
||||
bool ConfigureAudioSession();
|
||||
|
||||
// Like above, but requires caller to already hold session lock.
|
||||
bool ConfigureAudioSessionLocked();
|
||||
|
||||
// Unconfigures the audio session.
|
||||
void UnconfigureAudioSession();
|
||||
|
||||
// Activates our audio session, creates and initializes the voice-processing
|
||||
// audio unit and verifies that we got the preferred native audio parameters.
|
||||
bool InitPlayOrRecord();
|
||||
|
||||
// Closes and deletes the voice-processing I/O unit.
|
||||
void ShutdownPlayOrRecord();
|
||||
|
||||
// Resets thread-checkers before a call is restarted.
|
||||
void PrepareForNewStart();
|
||||
|
||||
// Determines whether voice processing should be enabled or disabled.
|
||||
const bool bypass_voice_processing_;
|
||||
|
||||
const bool disable_recording_;
|
||||
const bool enableSystemMute_ = false;
|
||||
const int numChannels_;
|
||||
|
||||
// Native I/O audio thread checker.
|
||||
SequenceChecker io_thread_checker_;
|
||||
|
||||
// Thread that this object is created on.
|
||||
rtc::Thread* thread_;
|
||||
|
||||
// Raw pointer handle provided to us in AttachAudioBuffer(). Owned by the
|
||||
// AudioDeviceModuleImpl class and called by AudioDeviceModule::Create().
|
||||
// The AudioDeviceBuffer is a member of the AudioDeviceModuleImpl instance
|
||||
// and therefore outlives this object.
|
||||
AudioDeviceBuffer* audio_device_buffer_;
|
||||
|
||||
// Contains audio parameters (sample rate, #channels, buffer size etc.) for
|
||||
// the playout and recording sides. These structure is set in two steps:
|
||||
// first, native sample rate and #channels are defined in Init(). Next, the
|
||||
// audio session is activated and we verify that the preferred parameters
|
||||
// were granted by the OS. At this stage it is also possible to add a third
|
||||
// component to the parameters; the native I/O buffer duration.
|
||||
// A RTC_CHECK will be hit if we for some reason fail to open an audio session
|
||||
// using the specified parameters.
|
||||
AudioParameters playout_parameters_;
|
||||
AudioParameters record_parameters_;
|
||||
|
||||
// The AudioUnit used to play and record audio.
|
||||
std::unique_ptr<VoiceProcessingAudioUnit> audio_unit_;
|
||||
|
||||
// FineAudioBuffer takes an AudioDeviceBuffer which delivers audio data
|
||||
// in chunks of 10ms. It then allows for this data to be pulled in
|
||||
// a finer or coarser granularity. I.e. interacting with this class instead
|
||||
// of directly with the AudioDeviceBuffer one can ask for any number of
|
||||
// audio data samples. Is also supports a similar scheme for the recording
|
||||
// side.
|
||||
// Example: native buffer size can be 128 audio frames at 16kHz sample rate.
|
||||
// WebRTC will provide 480 audio frames per 10ms but iOS asks for 128
|
||||
// in each callback (one every 8ms). This class can then ask for 128 and the
|
||||
// FineAudioBuffer will ask WebRTC for new data only when needed and also
|
||||
// cache non-utilized audio between callbacks. On the recording side, iOS
|
||||
// can provide audio data frames of size 128 and these are accumulated until
|
||||
// enough data to supply one 10ms call exists. This 10ms chunk is then sent
|
||||
// to WebRTC and the remaining part is stored.
|
||||
std::unique_ptr<FineAudioBuffer> fine_audio_buffer_;
|
||||
|
||||
// Temporary storage for recorded data. AudioUnitRender() renders into this
|
||||
// array as soon as a frame of the desired buffer size has been recorded.
|
||||
// On real iOS devices, the size will be fixed and set once. For iOS
|
||||
// simulators, the size can vary from callback to callback and the size
|
||||
// will be changed dynamically to account for this behavior.
|
||||
rtc::BufferT<int16_t> record_audio_buffer_;
|
||||
|
||||
// Set to 1 when recording is active and 0 otherwise.
|
||||
std::atomic<int> recording_;
|
||||
|
||||
// Set to 1 when playout is active and 0 otherwise.
|
||||
std::atomic<int> playing_;
|
||||
|
||||
// Set to true after successful call to Init(), false otherwise.
|
||||
bool initialized_ RTC_GUARDED_BY(thread_);
|
||||
|
||||
// Set to true after successful call to InitRecording() or InitPlayout(),
|
||||
// false otherwise.
|
||||
bool audio_is_initialized_;
|
||||
|
||||
// Set to true if audio session is interrupted, false otherwise.
|
||||
bool is_interrupted_;
|
||||
|
||||
// Audio interruption observer instance.
|
||||
RTCNativeAudioSessionDelegateAdapter* audio_session_observer_
|
||||
RTC_GUARDED_BY(thread_);
|
||||
|
||||
// Set to true if we've activated the audio session.
|
||||
bool has_configured_session_ RTC_GUARDED_BY(thread_);
|
||||
|
||||
// Counts number of detected audio glitches on the playout side.
|
||||
int64_t num_detected_playout_glitches_ RTC_GUARDED_BY(thread_);
|
||||
int64_t last_playout_time_ RTC_GUARDED_BY(io_thread_checker_);
|
||||
|
||||
// Counts number of playout callbacks per call.
|
||||
// The value is updated on the native I/O thread and later read on the
|
||||
// creating `thread_` but at this stage no audio is active.
|
||||
// Hence, it is a "thread safe" design and no lock is needed.
|
||||
int64_t num_playout_callbacks_;
|
||||
|
||||
// Contains the time for when the last output volume change was detected.
|
||||
int64_t last_output_volume_change_time_ RTC_GUARDED_BY(thread_);
|
||||
|
||||
// Avoids running pending task after `this` is Terminated.
|
||||
webrtc::scoped_refptr<PendingTaskSafetyFlag> safety_ =
|
||||
PendingTaskSafetyFlag::Create();
|
||||
|
||||
std::atomic<bool> _hasTone;
|
||||
std::shared_ptr<tgcalls::CallAudioTone> _tone;
|
||||
|
||||
bool isBufferPlaying_ = false;
|
||||
bool isBufferRecording_ = false;
|
||||
|
||||
bool isMicrophoneMuted_ = false;
|
||||
};
|
||||
} // namespace tgcalls_ios_adm
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_IOS_H_
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,161 @@
|
|||
/*
|
||||
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef TGCALLS_AUDIO_AUDIO_DEVICE_MODULE_IOS_H_
|
||||
#define TGCALLS_AUDIO_AUDIO_DEVICE_MODULE_IOS_H_
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "tgcalls_audio_device_ios.h"
|
||||
|
||||
#include "api/task_queue/task_queue_factory.h"
|
||||
#include "modules/audio_device/audio_device_buffer.h"
|
||||
#include "modules/audio_device/include/audio_device.h"
|
||||
#include "rtc_base/checks.h"
|
||||
|
||||
#include "platform/darwin/iOS/CallAudioTone.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
class AudioDeviceGeneric;
|
||||
|
||||
namespace tgcalls_ios_adm {
|
||||
|
||||
class AudioDeviceModuleIOS : public AudioDeviceModule {
|
||||
public:
|
||||
int32_t AttachAudioBuffer();
|
||||
|
||||
explicit AudioDeviceModuleIOS(bool bypass_voice_processing, bool disable_recording, bool enableSystemMute, int numChannels);
|
||||
~AudioDeviceModuleIOS() override;
|
||||
|
||||
// Retrieve the currently utilized audio layer
|
||||
int32_t ActiveAudioLayer(AudioLayer* audioLayer) const override;
|
||||
|
||||
// Full-duplex transportation of PCM audio
|
||||
int32_t RegisterAudioCallback(AudioTransport* audioCallback) override;
|
||||
|
||||
// Main initializaton and termination
|
||||
int32_t Init() override;
|
||||
int32_t Terminate() override;
|
||||
bool Initialized() const override;
|
||||
|
||||
// Device enumeration
|
||||
int16_t PlayoutDevices() override;
|
||||
int16_t RecordingDevices() override;
|
||||
int32_t PlayoutDeviceName(uint16_t index,
|
||||
char name[kAdmMaxDeviceNameSize],
|
||||
char guid[kAdmMaxGuidSize]) override;
|
||||
int32_t RecordingDeviceName(uint16_t index,
|
||||
char name[kAdmMaxDeviceNameSize],
|
||||
char guid[kAdmMaxGuidSize]) override;
|
||||
|
||||
// Device selection
|
||||
int32_t SetPlayoutDevice(uint16_t index) override;
|
||||
int32_t SetPlayoutDevice(WindowsDeviceType device) override;
|
||||
int32_t SetRecordingDevice(uint16_t index) override;
|
||||
int32_t SetRecordingDevice(WindowsDeviceType device) override;
|
||||
|
||||
// Audio transport initialization
|
||||
int32_t PlayoutIsAvailable(bool* available) override;
|
||||
int32_t InitPlayout() override;
|
||||
bool PlayoutIsInitialized() const override;
|
||||
int32_t RecordingIsAvailable(bool* available) override;
|
||||
int32_t InitRecording() override;
|
||||
bool RecordingIsInitialized() const override;
|
||||
|
||||
// Audio transport control
|
||||
int32_t InternalStartPlayout();
|
||||
int32_t StartPlayout() override;
|
||||
int32_t StopPlayout() override;
|
||||
bool Playing() const override;
|
||||
int32_t InternalStartRecording();
|
||||
int32_t StartRecording() override;
|
||||
int32_t StopRecording() override;
|
||||
bool Recording() const override;
|
||||
|
||||
// Audio mixer initialization
|
||||
int32_t InitSpeaker() override;
|
||||
bool SpeakerIsInitialized() const override;
|
||||
int32_t InitMicrophone() override;
|
||||
bool MicrophoneIsInitialized() const override;
|
||||
|
||||
// Speaker volume controls
|
||||
int32_t SpeakerVolumeIsAvailable(bool* available) override;
|
||||
int32_t SetSpeakerVolume(uint32_t volume) override;
|
||||
int32_t SpeakerVolume(uint32_t* volume) const override;
|
||||
int32_t MaxSpeakerVolume(uint32_t* maxVolume) const override;
|
||||
int32_t MinSpeakerVolume(uint32_t* minVolume) const override;
|
||||
|
||||
// Microphone volume controls
|
||||
int32_t MicrophoneVolumeIsAvailable(bool* available) override;
|
||||
int32_t SetMicrophoneVolume(uint32_t volume) override;
|
||||
int32_t MicrophoneVolume(uint32_t* volume) const override;
|
||||
int32_t MaxMicrophoneVolume(uint32_t* maxVolume) const override;
|
||||
int32_t MinMicrophoneVolume(uint32_t* minVolume) const override;
|
||||
|
||||
// Speaker mute control
|
||||
int32_t SpeakerMuteIsAvailable(bool* available) override;
|
||||
int32_t SetSpeakerMute(bool enable) override;
|
||||
int32_t SpeakerMute(bool* enabled) const override;
|
||||
|
||||
// Microphone mute control
|
||||
int32_t MicrophoneMuteIsAvailable(bool* available) override;
|
||||
int32_t SetMicrophoneMute(bool enable) override;
|
||||
int32_t MicrophoneMute(bool* enabled) const override;
|
||||
|
||||
// Stereo support
|
||||
int32_t StereoPlayoutIsAvailable(bool* available) const override;
|
||||
int32_t SetStereoPlayout(bool enable) override;
|
||||
int32_t StereoPlayout(bool* enabled) const override;
|
||||
int32_t StereoRecordingIsAvailable(bool* available) const override;
|
||||
int32_t SetStereoRecording(bool enable) override;
|
||||
int32_t StereoRecording(bool* enabled) const override;
|
||||
|
||||
// Delay information and control
|
||||
int32_t PlayoutDelay(uint16_t* delayMS) const override;
|
||||
|
||||
bool BuiltInAECIsAvailable() const override;
|
||||
int32_t EnableBuiltInAEC(bool enable) override;
|
||||
bool BuiltInAGCIsAvailable() const override;
|
||||
int32_t EnableBuiltInAGC(bool enable) override;
|
||||
bool BuiltInNSIsAvailable() const override;
|
||||
int32_t EnableBuiltInNS(bool enable) override;
|
||||
|
||||
int32_t GetPlayoutUnderrunCount() const override;
|
||||
|
||||
void setTone(std::shared_ptr<tgcalls::CallAudioTone> tone);
|
||||
|
||||
#if defined(WEBRTC_IOS)
|
||||
int GetPlayoutAudioParameters(AudioParameters* params) const override;
|
||||
int GetRecordAudioParameters(AudioParameters* params) const override;
|
||||
#endif // WEBRTC_IOS
|
||||
|
||||
public:
|
||||
void (^mutedSpeechDetectionChanged)(bool);
|
||||
|
||||
private:
|
||||
const bool bypass_voice_processing_;
|
||||
const bool disable_recording_;
|
||||
const bool enableSystemMute_;
|
||||
const int numChannels_;
|
||||
bool initialized_ = false;
|
||||
bool internalIsPlaying_ = false;
|
||||
bool audioBufferPlayoutStarted_ = false;
|
||||
bool audioBufferRecordingStarted_ = false;
|
||||
const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
|
||||
std::unique_ptr<AudioDeviceIOS> audio_device_;
|
||||
std::unique_ptr<AudioDeviceBuffer> audio_device_buffer_;
|
||||
|
||||
std::shared_ptr<tgcalls::CallAudioTone> pendingAudioTone_;
|
||||
};
|
||||
} // namespace tgcalls_ios_adm
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_MODULE_IOS_H_
|
||||
|
|
@ -0,0 +1,756 @@
|
|||
/*
|
||||
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "tgcalls_audio_device_module_ios.h"
|
||||
|
||||
#include "api/task_queue/default_task_queue_factory.h"
|
||||
#include "modules/audio_device/audio_device_config.h"
|
||||
#include "modules/audio_device/audio_device_generic.h"
|
||||
#include "rtc_base/checks.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "rtc_base/ref_count.h"
|
||||
#include "system_wrappers/include/metrics.h"
|
||||
#include "api/make_ref_counted.h"
|
||||
|
||||
#if defined(WEBRTC_IOS)
|
||||
#include "tgcalls_audio_device_ios.h"
|
||||
#endif
|
||||
|
||||
#define CHECKinitialized_() \
|
||||
{ \
|
||||
if (!initialized_) { \
|
||||
return -1; \
|
||||
}; \
|
||||
}
|
||||
|
||||
#define CHECKinitialized__BOOL() \
|
||||
{ \
|
||||
if (!initialized_) { \
|
||||
return false; \
|
||||
}; \
|
||||
}
|
||||
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
webrtc::scoped_refptr<AudioDeviceModule> CreateAudioDeviceModule(bool bypass_voice_processing) {
|
||||
return rtc::make_ref_counted<webrtc::tgcalls_ios_adm::AudioDeviceModuleIOS>(bypass_voice_processing, false, false, 1);
|
||||
}
|
||||
|
||||
webrtc::scoped_refptr<AudioDeviceModule> AudioDeviceModule::Create(
|
||||
AudioLayer audio_layer,
|
||||
TaskQueueFactory* task_queue_factory) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
return CreateAudioDeviceModule(false);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
namespace webrtc {
|
||||
namespace tgcalls_ios_adm {
|
||||
|
||||
AudioDeviceModuleIOS::AudioDeviceModuleIOS(bool bypass_voice_processing, bool disable_recording, bool enableSystemMute, int numChannels)
|
||||
: bypass_voice_processing_(bypass_voice_processing),
|
||||
disable_recording_(disable_recording),
|
||||
enableSystemMute_(enableSystemMute),
|
||||
numChannels_(numChannels),
|
||||
task_queue_factory_(CreateDefaultTaskQueueFactory()) {
|
||||
RTC_LOG(LS_INFO) << "current platform is IOS";
|
||||
RTC_LOG(LS_INFO) << "iPhone Audio APIs will be utilized.";
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::AttachAudioBuffer() {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
audio_device_->AttachAudioBuffer(audio_device_buffer_.get());
|
||||
return 0;
|
||||
}
|
||||
|
||||
AudioDeviceModuleIOS::~AudioDeviceModuleIOS() {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::ActiveAudioLayer(AudioLayer* audioLayer) const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
AudioLayer activeAudio;
|
||||
if (audio_device_->ActiveAudioLayer(activeAudio) == -1) {
|
||||
return -1;
|
||||
}
|
||||
*audioLayer = activeAudio;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::Init() {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
if (initialized_)
|
||||
return 0;
|
||||
|
||||
audio_device_buffer_.reset(new webrtc::AudioDeviceBuffer(task_queue_factory_.get()));
|
||||
audio_device_.reset(new tgcalls_ios_adm::AudioDeviceIOS(bypass_voice_processing_, disable_recording_, enableSystemMute_, numChannels_));
|
||||
RTC_CHECK(audio_device_);
|
||||
|
||||
if (audio_device_) {
|
||||
audio_device_->mutedSpeechDetectionChanged = [mutedSpeechDetectionChanged copy];
|
||||
}
|
||||
|
||||
this->AttachAudioBuffer();
|
||||
|
||||
AudioDeviceGeneric::InitStatus status = audio_device_->Init();
|
||||
RTC_HISTOGRAM_ENUMERATION(
|
||||
"WebRTC.Audio.InitializationResult", static_cast<int>(status),
|
||||
static_cast<int>(AudioDeviceGeneric::InitStatus::NUM_STATUSES));
|
||||
if (status != AudioDeviceGeneric::InitStatus::OK) {
|
||||
RTC_LOG(LS_ERROR) << "Audio device initialization failed.";
|
||||
return -1;
|
||||
}
|
||||
initialized_ = true;
|
||||
|
||||
if (pendingAudioTone_) {
|
||||
audio_device_->setTone(pendingAudioTone_);
|
||||
pendingAudioTone_ = nullptr;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::Terminate() {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
if (!initialized_)
|
||||
return 0;
|
||||
if (audio_device_->Terminate() == -1) {
|
||||
return -1;
|
||||
}
|
||||
initialized_ = false;
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool AudioDeviceModuleIOS::Initialized() const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__ << ": " << initialized_;
|
||||
return initialized_;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::InitSpeaker() {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
return audio_device_->InitSpeaker();
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::InitMicrophone() {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
return audio_device_->InitMicrophone();
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::SpeakerVolumeIsAvailable(bool* available) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
bool isAvailable = false;
|
||||
if (audio_device_->SpeakerVolumeIsAvailable(isAvailable) == -1) {
|
||||
return -1;
|
||||
}
|
||||
*available = isAvailable;
|
||||
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::SetSpeakerVolume(uint32_t volume) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << volume << ")";
|
||||
CHECKinitialized_();
|
||||
return audio_device_->SetSpeakerVolume(volume);
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::SpeakerVolume(uint32_t* volume) const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
uint32_t level = 0;
|
||||
if (audio_device_->SpeakerVolume(level) == -1) {
|
||||
return -1;
|
||||
}
|
||||
*volume = level;
|
||||
RTC_DLOG(LS_INFO) << "output: " << *volume;
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool AudioDeviceModuleIOS::SpeakerIsInitialized() const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized__BOOL();
|
||||
bool isInitialized = audio_device_->SpeakerIsInitialized();
|
||||
RTC_DLOG(LS_INFO) << "output: " << isInitialized;
|
||||
return isInitialized;
|
||||
}
|
||||
|
||||
bool AudioDeviceModuleIOS::MicrophoneIsInitialized() const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized__BOOL();
|
||||
bool isInitialized = audio_device_->MicrophoneIsInitialized();
|
||||
RTC_DLOG(LS_INFO) << "output: " << isInitialized;
|
||||
return isInitialized;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::MaxSpeakerVolume(uint32_t* maxVolume) const {
|
||||
CHECKinitialized_();
|
||||
uint32_t maxVol = 0;
|
||||
if (audio_device_->MaxSpeakerVolume(maxVol) == -1) {
|
||||
return -1;
|
||||
}
|
||||
*maxVolume = maxVol;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::MinSpeakerVolume(uint32_t* minVolume) const {
|
||||
CHECKinitialized_();
|
||||
uint32_t minVol = 0;
|
||||
if (audio_device_->MinSpeakerVolume(minVol) == -1) {
|
||||
return -1;
|
||||
}
|
||||
*minVolume = minVol;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::SpeakerMuteIsAvailable(bool* available) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
bool isAvailable = false;
|
||||
if (audio_device_->SpeakerMuteIsAvailable(isAvailable) == -1) {
|
||||
return -1;
|
||||
}
|
||||
*available = isAvailable;
|
||||
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::SetSpeakerMute(bool enable) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")";
|
||||
CHECKinitialized_();
|
||||
return audio_device_->SetSpeakerMute(enable);
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::SpeakerMute(bool* enabled) const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
bool muted = false;
|
||||
if (audio_device_->SpeakerMute(muted) == -1) {
|
||||
return -1;
|
||||
}
|
||||
*enabled = muted;
|
||||
RTC_DLOG(LS_INFO) << "output: " << muted;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::MicrophoneMuteIsAvailable(bool* available) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
bool isAvailable = false;
|
||||
if (audio_device_->MicrophoneMuteIsAvailable(isAvailable) == -1) {
|
||||
return -1;
|
||||
}
|
||||
*available = isAvailable;
|
||||
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::SetMicrophoneMute(bool enable) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")";
|
||||
CHECKinitialized_();
|
||||
return (audio_device_->SetMicrophoneMute(enable));
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::MicrophoneMute(bool* enabled) const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
bool muted = false;
|
||||
if (audio_device_->MicrophoneMute(muted) == -1) {
|
||||
return -1;
|
||||
}
|
||||
*enabled = muted;
|
||||
RTC_DLOG(LS_INFO) << "output: " << muted;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::MicrophoneVolumeIsAvailable(bool* available) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
bool isAvailable = false;
|
||||
if (audio_device_->MicrophoneVolumeIsAvailable(isAvailable) == -1) {
|
||||
return -1;
|
||||
}
|
||||
*available = isAvailable;
|
||||
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::SetMicrophoneVolume(uint32_t volume) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << volume << ")";
|
||||
CHECKinitialized_();
|
||||
return (audio_device_->SetMicrophoneVolume(volume));
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::MicrophoneVolume(uint32_t* volume) const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
uint32_t level = 0;
|
||||
if (audio_device_->MicrophoneVolume(level) == -1) {
|
||||
return -1;
|
||||
}
|
||||
*volume = level;
|
||||
RTC_DLOG(LS_INFO) << "output: " << *volume;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::StereoRecordingIsAvailable(
|
||||
bool* available) const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
bool isAvailable = false;
|
||||
if (audio_device_->StereoRecordingIsAvailable(isAvailable) == -1) {
|
||||
return -1;
|
||||
}
|
||||
*available = isAvailable;
|
||||
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::SetStereoRecording(bool enable) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")";
|
||||
CHECKinitialized_();
|
||||
if (enable) {
|
||||
RTC_LOG(LS_WARNING) << "recording in stereo is not supported";
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::StereoRecording(bool* enabled) const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
bool stereo = false;
|
||||
if (audio_device_->StereoRecording(stereo) == -1) {
|
||||
return -1;
|
||||
}
|
||||
*enabled = stereo;
|
||||
RTC_DLOG(LS_INFO) << "output: " << stereo;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::StereoPlayoutIsAvailable(bool* available) const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
bool isAvailable = false;
|
||||
if (audio_device_->StereoPlayoutIsAvailable(isAvailable) == -1) {
|
||||
return -1;
|
||||
}
|
||||
*available = isAvailable;
|
||||
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::SetStereoPlayout(bool enable) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")";
|
||||
CHECKinitialized_();
|
||||
if (audio_device_->PlayoutIsInitialized()) {
|
||||
RTC_LOG(LS_ERROR) << "unable to set stereo mode while playing side is initialized";
|
||||
return -1;
|
||||
}
|
||||
if (audio_device_->SetStereoPlayout(enable)) {
|
||||
RTC_LOG(LS_WARNING) << "stereo playout is not supported";
|
||||
return -1;
|
||||
}
|
||||
int8_t nChannels(1);
|
||||
if (enable) {
|
||||
nChannels = 2;
|
||||
}
|
||||
audio_device_buffer_.get()->SetPlayoutChannels(nChannels);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::StereoPlayout(bool* enabled) const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
bool stereo = false;
|
||||
if (audio_device_->StereoPlayout(stereo) == -1) {
|
||||
return -1;
|
||||
}
|
||||
*enabled = stereo;
|
||||
RTC_DLOG(LS_INFO) << "output: " << stereo;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::PlayoutIsAvailable(bool* available) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
bool isAvailable = false;
|
||||
if (audio_device_->PlayoutIsAvailable(isAvailable) == -1) {
|
||||
return -1;
|
||||
}
|
||||
*available = isAvailable;
|
||||
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::RecordingIsAvailable(bool* available) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
bool isAvailable = false;
|
||||
if (audio_device_->RecordingIsAvailable(isAvailable) == -1) {
|
||||
return -1;
|
||||
}
|
||||
*available = isAvailable;
|
||||
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::MaxMicrophoneVolume(uint32_t* maxVolume) const {
|
||||
CHECKinitialized_();
|
||||
uint32_t maxVol(0);
|
||||
if (audio_device_->MaxMicrophoneVolume(maxVol) == -1) {
|
||||
return -1;
|
||||
}
|
||||
*maxVolume = maxVol;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::MinMicrophoneVolume(uint32_t* minVolume) const {
|
||||
CHECKinitialized_();
|
||||
uint32_t minVol(0);
|
||||
if (audio_device_->MinMicrophoneVolume(minVol) == -1) {
|
||||
return -1;
|
||||
}
|
||||
*minVolume = minVol;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int16_t AudioDeviceModuleIOS::PlayoutDevices() {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
uint16_t nPlayoutDevices = audio_device_->PlayoutDevices();
|
||||
RTC_DLOG(LS_INFO) << "output: " << nPlayoutDevices;
|
||||
return (int16_t)(nPlayoutDevices);
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::SetPlayoutDevice(uint16_t index) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << index << ")";
|
||||
CHECKinitialized_();
|
||||
return audio_device_->SetPlayoutDevice(index);
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::SetPlayoutDevice(WindowsDeviceType device) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
return audio_device_->SetPlayoutDevice(device);
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::PlayoutDeviceName(
|
||||
uint16_t index,
|
||||
char name[kAdmMaxDeviceNameSize],
|
||||
char guid[kAdmMaxGuidSize]) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << index << ", ...)";
|
||||
CHECKinitialized_();
|
||||
if (name == NULL) {
|
||||
return -1;
|
||||
}
|
||||
if (audio_device_->PlayoutDeviceName(index, name, guid) == -1) {
|
||||
return -1;
|
||||
}
|
||||
if (name != NULL) {
|
||||
RTC_DLOG(LS_INFO) << "output: name = " << name;
|
||||
}
|
||||
if (guid != NULL) {
|
||||
RTC_DLOG(LS_INFO) << "output: guid = " << guid;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::RecordingDeviceName(
|
||||
uint16_t index,
|
||||
char name[kAdmMaxDeviceNameSize],
|
||||
char guid[kAdmMaxGuidSize]) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << index << ", ...)";
|
||||
CHECKinitialized_();
|
||||
if (name == NULL) {
|
||||
return -1;
|
||||
}
|
||||
if (audio_device_->RecordingDeviceName(index, name, guid) == -1) {
|
||||
return -1;
|
||||
}
|
||||
if (name != NULL) {
|
||||
RTC_DLOG(LS_INFO) << "output: name = " << name;
|
||||
}
|
||||
if (guid != NULL) {
|
||||
RTC_DLOG(LS_INFO) << "output: guid = " << guid;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int16_t AudioDeviceModuleIOS::RecordingDevices() {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
uint16_t nRecordingDevices = audio_device_->RecordingDevices();
|
||||
RTC_DLOG(LS_INFO) << "output: " << nRecordingDevices;
|
||||
return (int16_t)nRecordingDevices;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::SetRecordingDevice(uint16_t index) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << index << ")";
|
||||
CHECKinitialized_();
|
||||
return audio_device_->SetRecordingDevice(index);
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::SetRecordingDevice(WindowsDeviceType device) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
return audio_device_->SetRecordingDevice(device);
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::InitPlayout() {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
if (PlayoutIsInitialized()) {
|
||||
return 0;
|
||||
}
|
||||
int32_t result = audio_device_->InitPlayout();
|
||||
RTC_DLOG(LS_INFO) << "output: " << result;
|
||||
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.InitPlayoutSuccess",
|
||||
static_cast<int>(result == 0));
|
||||
return result;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::InitRecording() {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
if (RecordingIsInitialized()) {
|
||||
return 0;
|
||||
}
|
||||
int32_t result = audio_device_->InitRecording();
|
||||
RTC_DLOG(LS_INFO) << "output: " << result;
|
||||
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.InitRecordingSuccess",
|
||||
static_cast<int>(result == 0));
|
||||
return result;
|
||||
}
|
||||
|
||||
bool AudioDeviceModuleIOS::PlayoutIsInitialized() const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized__BOOL();
|
||||
return audio_device_->PlayoutIsInitialized();
|
||||
}
|
||||
|
||||
bool AudioDeviceModuleIOS::RecordingIsInitialized() const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized__BOOL();
|
||||
return audio_device_->RecordingIsInitialized();
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::InternalStartPlayout() {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
if (audio_device_->Playing()) {
|
||||
return 0;
|
||||
}
|
||||
int32_t result = audio_device_->StartPlayout();
|
||||
RTC_DLOG(LS_INFO) << "output: " << result;
|
||||
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StartPlayoutSuccess",
|
||||
static_cast<int>(result == 0));
|
||||
return result;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::StartPlayout() {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
if (audio_device_->Playing()) {
|
||||
if (!audioBufferPlayoutStarted_) {
|
||||
audioBufferPlayoutStarted_ = true;
|
||||
audio_device_buffer_.get()->StartPlayout();
|
||||
audio_device_->setIsBufferPlaying(true);
|
||||
}
|
||||
internalIsPlaying_ = true;
|
||||
return 0;
|
||||
}
|
||||
audio_device_buffer_.get()->StartPlayout();
|
||||
audio_device_->setIsBufferPlaying(true);
|
||||
audioBufferPlayoutStarted_ = true;
|
||||
int32_t result = audio_device_->StartPlayout();
|
||||
RTC_DLOG(LS_INFO) << "output: " << result;
|
||||
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StartPlayoutSuccess",
|
||||
static_cast<int>(result == 0));
|
||||
internalIsPlaying_ = true;
|
||||
return result;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::StopPlayout() {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
int32_t result = audio_device_->StopPlayout();
|
||||
if (audioBufferPlayoutStarted_) {
|
||||
audio_device_buffer_.get()->StopPlayout();
|
||||
audioBufferPlayoutStarted_ = false;
|
||||
audio_device_->setIsBufferPlaying(false);
|
||||
}
|
||||
RTC_DLOG(LS_INFO) << "output: " << result;
|
||||
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StopPlayoutSuccess",
|
||||
static_cast<int>(result == 0));
|
||||
return result;
|
||||
}
|
||||
|
||||
bool AudioDeviceModuleIOS::Playing() const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized__BOOL();
|
||||
return internalIsPlaying_;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::InternalStartRecording() {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
if (Recording()) {
|
||||
return 0;
|
||||
}
|
||||
int32_t result = audio_device_->StartRecording();
|
||||
RTC_DLOG(LS_INFO) << "output: " << result;
|
||||
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StartRecordingSuccess",
|
||||
static_cast<int>(result == 0));
|
||||
return result;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::StartRecording() {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
if (Recording()) {
|
||||
if (!audioBufferRecordingStarted_) {
|
||||
audioBufferRecordingStarted_ = true;
|
||||
audio_device_buffer_.get()->StartRecording();
|
||||
audio_device_->setIsBufferRecording(true);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
audio_device_buffer_.get()->StartRecording();
|
||||
audioBufferRecordingStarted_ = true;
|
||||
audio_device_->setIsBufferRecording(true);
|
||||
int32_t result = audio_device_->StartRecording();
|
||||
RTC_DLOG(LS_INFO) << "output: " << result;
|
||||
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StartRecordingSuccess",
|
||||
static_cast<int>(result == 0));
|
||||
return result;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::StopRecording() {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized_();
|
||||
int32_t result = audio_device_->StopRecording();
|
||||
if (audioBufferRecordingStarted_) {
|
||||
audioBufferRecordingStarted_ = false;
|
||||
audio_device_buffer_.get()->StopRecording();
|
||||
audio_device_->setIsBufferRecording(false);
|
||||
}
|
||||
RTC_DLOG(LS_INFO) << "output: " << result;
|
||||
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StopRecordingSuccess",
|
||||
static_cast<int>(result == 0));
|
||||
return result;
|
||||
}
|
||||
|
||||
bool AudioDeviceModuleIOS::Recording() const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized__BOOL();
|
||||
return audio_device_->Recording();
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::RegisterAudioCallback(
|
||||
AudioTransport* audioCallback) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
return audio_device_buffer_.get()->RegisterAudioCallback(audioCallback);
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::PlayoutDelay(uint16_t* delayMS) const {
|
||||
CHECKinitialized_();
|
||||
uint16_t delay = 0;
|
||||
if (audio_device_->PlayoutDelay(delay) == -1) {
|
||||
RTC_LOG(LS_ERROR) << "failed to retrieve the playout delay";
|
||||
return -1;
|
||||
}
|
||||
*delayMS = delay;
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool AudioDeviceModuleIOS::BuiltInAECIsAvailable() const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized__BOOL();
|
||||
bool isAvailable = audio_device_->BuiltInAECIsAvailable();
|
||||
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
|
||||
return isAvailable;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::EnableBuiltInAEC(bool enable) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")";
|
||||
CHECKinitialized_();
|
||||
int32_t ok = audio_device_->EnableBuiltInAEC(enable);
|
||||
RTC_DLOG(LS_INFO) << "output: " << ok;
|
||||
return ok;
|
||||
}
|
||||
|
||||
bool AudioDeviceModuleIOS::BuiltInAGCIsAvailable() const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized__BOOL();
|
||||
bool isAvailable = audio_device_->BuiltInAGCIsAvailable();
|
||||
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
|
||||
return isAvailable;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::EnableBuiltInAGC(bool enable) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")";
|
||||
CHECKinitialized_();
|
||||
int32_t ok = audio_device_->EnableBuiltInAGC(enable);
|
||||
RTC_DLOG(LS_INFO) << "output: " << ok;
|
||||
return ok;
|
||||
}
|
||||
|
||||
bool AudioDeviceModuleIOS::BuiltInNSIsAvailable() const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
CHECKinitialized__BOOL();
|
||||
bool isAvailable = audio_device_->BuiltInNSIsAvailable();
|
||||
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
|
||||
return isAvailable;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::EnableBuiltInNS(bool enable) {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")";
|
||||
CHECKinitialized_();
|
||||
int32_t ok = audio_device_->EnableBuiltInNS(enable);
|
||||
RTC_DLOG(LS_INFO) << "output: " << ok;
|
||||
return ok;
|
||||
}
|
||||
|
||||
int32_t AudioDeviceModuleIOS::GetPlayoutUnderrunCount() const {
|
||||
// Don't log here, as this method can be called very often.
|
||||
CHECKinitialized_();
|
||||
int32_t ok = audio_device_->GetPlayoutUnderrunCount();
|
||||
return ok;
|
||||
}
|
||||
|
||||
void AudioDeviceModuleIOS::setTone(std::shared_ptr<tgcalls::CallAudioTone> tone) {
|
||||
if (audio_device_) {
|
||||
audio_device_->setTone(tone);
|
||||
} else {
|
||||
pendingAudioTone_ = tone;
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(WEBRTC_IOS)
|
||||
int AudioDeviceModuleIOS::GetPlayoutAudioParameters(
|
||||
AudioParameters* params) const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
int r = audio_device_->GetPlayoutAudioParameters(params);
|
||||
RTC_DLOG(LS_INFO) << "output: " << r;
|
||||
return r;
|
||||
}
|
||||
|
||||
int AudioDeviceModuleIOS::GetRecordAudioParameters(
|
||||
AudioParameters* params) const {
|
||||
RTC_DLOG(LS_INFO) << __FUNCTION__;
|
||||
int r = audio_device_->GetRecordAudioParameters(params);
|
||||
RTC_DLOG(LS_INFO) << "output: " << r;
|
||||
return r;
|
||||
}
|
||||
#endif // WEBRTC_IOS
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,149 @@
|
|||
/*
|
||||
* Copyright 2016 The WebRTC Project Authors. All rights reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef TGCALLS_AUDIO_VOICE_PROCESSING_AUDIO_UNIT_H_
|
||||
#define TGCALLS_AUDIO_VOICE_PROCESSING_AUDIO_UNIT_H_
|
||||
|
||||
#include <AudioUnit/AudioUnit.h>
|
||||
|
||||
namespace webrtc {
|
||||
namespace tgcalls_ios_adm {
|
||||
|
||||
class VoiceProcessingAudioUnitObserver {
|
||||
public:
|
||||
// Callback function called on a real-time priority I/O thread from the audio
|
||||
// unit. This method is used to signal that recorded audio is available.
|
||||
virtual OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags,
|
||||
const AudioTimeStamp* time_stamp,
|
||||
UInt32 bus_number,
|
||||
UInt32 num_frames,
|
||||
AudioBufferList* io_data) = 0;
|
||||
|
||||
// Callback function called on a real-time priority I/O thread from the audio
|
||||
// unit. This method is used to provide audio samples to the audio unit.
|
||||
virtual OSStatus OnGetPlayoutData(AudioUnitRenderActionFlags* io_action_flags,
|
||||
const AudioTimeStamp* time_stamp,
|
||||
UInt32 bus_number,
|
||||
UInt32 num_frames,
|
||||
AudioBufferList* io_data) = 0;
|
||||
|
||||
virtual void OnMutedSpeechStatusChanged(bool isDetectingSpeech) {}
|
||||
|
||||
protected:
|
||||
~VoiceProcessingAudioUnitObserver() {}
|
||||
};
|
||||
|
||||
// Convenience class to abstract away the management of a Voice Processing
|
||||
// I/O Audio Unit. The Voice Processing I/O unit has the same characteristics
|
||||
// as the Remote I/O unit (supports full duplex low-latency audio input and
|
||||
// output) and adds AEC for for two-way duplex communication. It also adds AGC,
|
||||
// adjustment of voice-processing quality, and muting. Hence, ideal for
|
||||
// VoIP applications.
|
||||
class VoiceProcessingAudioUnit {
|
||||
public:
|
||||
VoiceProcessingAudioUnit(bool bypass_voice_processing, bool disable_recording, bool enableSystemMute, int numChannels,
|
||||
VoiceProcessingAudioUnitObserver* observer);
|
||||
~VoiceProcessingAudioUnit();
|
||||
|
||||
// TODO(tkchin): enum for state and state checking.
|
||||
enum State : int32_t {
|
||||
// Init() should be called.
|
||||
kInitRequired,
|
||||
// Audio unit created but not initialized.
|
||||
kUninitialized,
|
||||
// Initialized but not started. Equivalent to stopped.
|
||||
kInitialized,
|
||||
// Initialized and started.
|
||||
kStarted,
|
||||
};
|
||||
|
||||
// Number of bytes per audio sample for 16-bit signed integer representation.
|
||||
static const UInt32 kBytesPerSample;
|
||||
|
||||
// Initializes this class by creating the underlying audio unit instance.
|
||||
// Creates a Voice-Processing I/O unit and configures it for full-duplex
|
||||
// audio. The selected stream format is selected to avoid internal resampling
|
||||
// and to match the 10ms callback rate for WebRTC as well as possible.
|
||||
// Does not intialize the audio unit.
|
||||
bool Init();
|
||||
|
||||
VoiceProcessingAudioUnit::State GetState() const;
|
||||
|
||||
// Initializes the underlying audio unit with the given sample rate.
|
||||
bool Initialize(Float64 sample_rate);
|
||||
|
||||
// Starts the underlying audio unit.
|
||||
OSStatus Start();
|
||||
|
||||
// Stops the underlying audio unit.
|
||||
bool Stop();
|
||||
|
||||
// Uninitializes the underlying audio unit.
|
||||
bool Uninitialize();
|
||||
|
||||
void setIsMicrophoneMuted(bool isMuted);
|
||||
|
||||
// Calls render on the underlying audio unit.
|
||||
OSStatus Render(AudioUnitRenderActionFlags* flags,
|
||||
const AudioTimeStamp* time_stamp,
|
||||
UInt32 output_bus_number,
|
||||
UInt32 num_frames,
|
||||
AudioBufferList* io_data);
|
||||
|
||||
private:
|
||||
// The C API used to set callbacks requires static functions. When these are
|
||||
// called, they will invoke the relevant instance method by casting
|
||||
// in_ref_con to VoiceProcessingAudioUnit*.
|
||||
static OSStatus OnGetPlayoutData(void* in_ref_con,
|
||||
AudioUnitRenderActionFlags* flags,
|
||||
const AudioTimeStamp* time_stamp,
|
||||
UInt32 bus_number,
|
||||
UInt32 num_frames,
|
||||
AudioBufferList* io_data);
|
||||
static OSStatus OnDeliverRecordedData(void* in_ref_con,
|
||||
AudioUnitRenderActionFlags* flags,
|
||||
const AudioTimeStamp* time_stamp,
|
||||
UInt32 bus_number,
|
||||
UInt32 num_frames,
|
||||
AudioBufferList* io_data);
|
||||
|
||||
// Notifies observer that samples are needed for playback.
|
||||
OSStatus NotifyGetPlayoutData(AudioUnitRenderActionFlags* flags,
|
||||
const AudioTimeStamp* time_stamp,
|
||||
UInt32 bus_number,
|
||||
UInt32 num_frames,
|
||||
AudioBufferList* io_data);
|
||||
// Notifies observer that recorded samples are available for render.
|
||||
OSStatus NotifyDeliverRecordedData(AudioUnitRenderActionFlags* flags,
|
||||
const AudioTimeStamp* time_stamp,
|
||||
UInt32 bus_number,
|
||||
UInt32 num_frames,
|
||||
AudioBufferList* io_data);
|
||||
|
||||
// Returns the predetermined format with a specific sample rate. See
|
||||
// implementation file for details on format.
|
||||
AudioStreamBasicDescription GetFormat(Float64 sample_rate, int numChannels) const;
|
||||
|
||||
// Deletes the underlying audio unit.
|
||||
void DisposeAudioUnit();
|
||||
|
||||
const bool bypass_voice_processing_;
|
||||
const bool disable_recording_;
|
||||
const int numChannels_;
|
||||
bool enableSystemMute_ = false;
|
||||
bool isMicrophoneMuted_ = true;
|
||||
VoiceProcessingAudioUnitObserver* observer_;
|
||||
AudioUnit vpio_unit_;
|
||||
VoiceProcessingAudioUnit::State state_;
|
||||
};
|
||||
} // namespace tgcalls_ios_adm
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // SDK_OBJC_NATIVE_SRC_AUDIO_VOICE_PROCESSING_AUDIO_UNIT_H_
|
||||
|
|
@ -0,0 +1,544 @@
|
|||
/*
|
||||
* Copyright 2016 The WebRTC Project Authors. All rights reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import "tgcalls_voice_processing_audio_unit.h"
|
||||
|
||||
#include "rtc_base/checks.h"
|
||||
#include "system_wrappers/include/metrics.h"
|
||||
|
||||
#import "base/RTCLogging.h"
|
||||
#import "platform/darwin/iOS/RTCAudioSessionConfiguration.h"
|
||||
|
||||
#if !defined(NDEBUG)
|
||||
static void LogStreamDescription(AudioStreamBasicDescription description) {
|
||||
char formatIdString[5];
|
||||
UInt32 formatId = CFSwapInt32HostToBig(description.mFormatID);
|
||||
bcopy(&formatId, formatIdString, 4);
|
||||
formatIdString[4] = '\0';
|
||||
RTCLog(@"AudioStreamBasicDescription: {\n"
|
||||
" mSampleRate: %.2f\n"
|
||||
" formatIDString: %s\n"
|
||||
" mFormatFlags: 0x%X\n"
|
||||
" mBytesPerPacket: %u\n"
|
||||
" mFramesPerPacket: %u\n"
|
||||
" mBytesPerFrame: %u\n"
|
||||
" mChannelsPerFrame: %u\n"
|
||||
" mBitsPerChannel: %u\n"
|
||||
" mReserved: %u\n}",
|
||||
description.mSampleRate, formatIdString,
|
||||
static_cast<unsigned int>(description.mFormatFlags),
|
||||
static_cast<unsigned int>(description.mBytesPerPacket),
|
||||
static_cast<unsigned int>(description.mFramesPerPacket),
|
||||
static_cast<unsigned int>(description.mBytesPerFrame),
|
||||
static_cast<unsigned int>(description.mChannelsPerFrame),
|
||||
static_cast<unsigned int>(description.mBitsPerChannel),
|
||||
static_cast<unsigned int>(description.mReserved));
|
||||
}
|
||||
#endif
|
||||
|
||||
namespace webrtc {
|
||||
namespace tgcalls_ios_adm {
|
||||
|
||||
// Calls to AudioUnitInitialize() can fail if called back-to-back on different
|
||||
// ADM instances. A fall-back solution is to allow multiple sequential calls
|
||||
// with as small delay between each. This factor sets the max number of allowed
|
||||
// initialization attempts.
|
||||
static const int kMaxNumberOfAudioUnitInitializeAttempts = 5;
|
||||
// A VP I/O unit's bus 1 connects to input hardware (microphone).
|
||||
static const AudioUnitElement kInputBus = 1;
|
||||
// A VP I/O unit's bus 0 connects to output hardware (speaker).
|
||||
static const AudioUnitElement kOutputBus = 0;
|
||||
|
||||
// Returns the automatic gain control (AGC) state on the processed microphone
|
||||
// signal. Should be on by default for Voice Processing audio units.
|
||||
static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) {
|
||||
RTC_DCHECK(audio_unit);
|
||||
UInt32 size = sizeof(*enabled);
|
||||
OSStatus result = AudioUnitGetProperty(audio_unit,
|
||||
kAUVoiceIOProperty_VoiceProcessingEnableAGC,
|
||||
kAudioUnitScope_Global,
|
||||
kInputBus,
|
||||
enabled,
|
||||
&size);
|
||||
RTCLog(@"VPIO unit AGC: %u", static_cast<unsigned int>(*enabled));
|
||||
return result;
|
||||
}
|
||||
|
||||
VoiceProcessingAudioUnit::VoiceProcessingAudioUnit(bool bypass_voice_processing, bool disable_recording, bool enableSystemMute, int numChannels,
|
||||
VoiceProcessingAudioUnitObserver* observer)
|
||||
: bypass_voice_processing_(bypass_voice_processing),
|
||||
disable_recording_(disable_recording),
|
||||
numChannels_(numChannels),
|
||||
enableSystemMute_(enableSystemMute),
|
||||
observer_(observer),
|
||||
vpio_unit_(nullptr),
|
||||
state_(kInitRequired) {
|
||||
RTC_DCHECK(observer);
|
||||
}
|
||||
|
||||
VoiceProcessingAudioUnit::~VoiceProcessingAudioUnit() {
|
||||
DisposeAudioUnit();
|
||||
}
|
||||
|
||||
const UInt32 VoiceProcessingAudioUnit::kBytesPerSample = 2;
|
||||
|
||||
bool VoiceProcessingAudioUnit::Init() {
|
||||
RTC_DCHECK_EQ(state_, kInitRequired);
|
||||
|
||||
// Create an audio component description to identify the Voice Processing
|
||||
// I/O audio unit.
|
||||
AudioComponentDescription vpio_unit_description;
|
||||
vpio_unit_description.componentType = kAudioUnitType_Output;
|
||||
|
||||
if (disable_recording_) {
|
||||
vpio_unit_description.componentSubType = kAudioUnitSubType_RemoteIO;
|
||||
} else {
|
||||
vpio_unit_description.componentSubType = kAudioUnitSubType_VoiceProcessingIO;
|
||||
}
|
||||
vpio_unit_description.componentManufacturer = kAudioUnitManufacturer_Apple;
|
||||
vpio_unit_description.componentFlags = 0;
|
||||
vpio_unit_description.componentFlagsMask = 0;
|
||||
|
||||
// Obtain an audio unit instance given the description.
|
||||
AudioComponent found_vpio_unit_ref =
|
||||
AudioComponentFindNext(nullptr, &vpio_unit_description);
|
||||
|
||||
// Create a Voice Processing IO audio unit.
|
||||
OSStatus result = noErr;
|
||||
result = AudioComponentInstanceNew(found_vpio_unit_ref, &vpio_unit_);
|
||||
if (result != noErr) {
|
||||
vpio_unit_ = nullptr;
|
||||
RTCLogError(@"AudioComponentInstanceNew failed. Error=%ld.", (long)result);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!disable_recording_) {
|
||||
// Enable input on the input scope of the input element.
|
||||
UInt32 enable_input = 1;
|
||||
result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO,
|
||||
kAudioUnitScope_Input, kInputBus, &enable_input,
|
||||
sizeof(enable_input));
|
||||
if (result != noErr) {
|
||||
DisposeAudioUnit();
|
||||
RTCLogError(@"Failed to enable input on input scope of input element. "
|
||||
"Error=%ld.",
|
||||
(long)result);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Enable output on the output scope of the output element.
|
||||
UInt32 enable_output = 1;
|
||||
result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO,
|
||||
kAudioUnitScope_Output, kOutputBus,
|
||||
&enable_output, sizeof(enable_output));
|
||||
if (result != noErr) {
|
||||
DisposeAudioUnit();
|
||||
RTCLogError(@"Failed to enable output on output scope of output element. "
|
||||
"Error=%ld.",
|
||||
(long)result);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Specify the callback function that provides audio samples to the audio
|
||||
// unit.
|
||||
AURenderCallbackStruct render_callback;
|
||||
render_callback.inputProc = OnGetPlayoutData;
|
||||
render_callback.inputProcRefCon = this;
|
||||
result = AudioUnitSetProperty(
|
||||
vpio_unit_, kAudioUnitProperty_SetRenderCallback, kAudioUnitScope_Input,
|
||||
kOutputBus, &render_callback, sizeof(render_callback));
|
||||
if (result != noErr) {
|
||||
DisposeAudioUnit();
|
||||
RTCLogError(@"Failed to specify the render callback on the output bus. "
|
||||
"Error=%ld.",
|
||||
(long)result);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!disable_recording_) {
|
||||
// Disable AU buffer allocation for the recorder, we allocate our own.
|
||||
// TODO(henrika): not sure that it actually saves resource to make this call.
|
||||
UInt32 flag = 0;
|
||||
result = AudioUnitSetProperty(
|
||||
vpio_unit_, kAudioUnitProperty_ShouldAllocateBuffer,
|
||||
kAudioUnitScope_Output, kInputBus, &flag, sizeof(flag));
|
||||
if (result != noErr) {
|
||||
DisposeAudioUnit();
|
||||
RTCLogError(@"Failed to disable buffer allocation on the input bus. "
|
||||
"Error=%ld.",
|
||||
(long)result);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// Specify the callback to be called by the I/O thread to us when input audio
|
||||
// is available. The recorded samples can then be obtained by calling the
|
||||
// AudioUnitRender() method.
|
||||
AURenderCallbackStruct input_callback;
|
||||
input_callback.inputProc = OnDeliverRecordedData;
|
||||
input_callback.inputProcRefCon = this;
|
||||
result = AudioUnitSetProperty(vpio_unit_,
|
||||
kAudioOutputUnitProperty_SetInputCallback,
|
||||
kAudioUnitScope_Global, kInputBus,
|
||||
&input_callback, sizeof(input_callback));
|
||||
if (result != noErr) {
|
||||
DisposeAudioUnit();
|
||||
RTCLogError(@"Failed to specify the input callback on the input bus. "
|
||||
"Error=%ld.",
|
||||
(long)result);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (@available(iOS 15.0, macOS 14.0, *)) {
|
||||
VoiceProcessingAudioUnit *audio_unit = this;
|
||||
AUVoiceIOMutedSpeechActivityEventListener listener = ^(AUVoiceIOSpeechActivityEvent event) {
|
||||
if (event == kAUVoiceIOSpeechActivityHasStarted) {
|
||||
if (audio_unit->observer_) {
|
||||
audio_unit->observer_->OnMutedSpeechStatusChanged(true);
|
||||
}
|
||||
} else if (event == kAUVoiceIOSpeechActivityHasEnded) {
|
||||
if (audio_unit->observer_) {
|
||||
audio_unit->observer_->OnMutedSpeechStatusChanged(false);
|
||||
}
|
||||
}
|
||||
};
|
||||
result = AudioUnitSetProperty(vpio_unit_, kAUVoiceIOProperty_MutedSpeechActivityEventListener, kAudioUnitScope_Global, kInputBus, &listener, sizeof(AUVoiceIOMutedSpeechActivityEventListener));
|
||||
}
|
||||
}
|
||||
|
||||
if (enableSystemMute_) {
|
||||
UInt32 muteUplinkOutput = isMicrophoneMuted_ ? 1 : 0;
|
||||
OSStatus result = AudioUnitSetProperty(vpio_unit_, kAUVoiceIOProperty_MuteOutput, kAudioUnitScope_Global, kInputBus, &muteUplinkOutput, sizeof(muteUplinkOutput));
|
||||
if (result != noErr) {
|
||||
RTCLogError(@"Failed to set kAUVoiceIOProperty_MuteOutput. Error=%ld", (long)result);
|
||||
}
|
||||
}
|
||||
|
||||
state_ = kUninitialized;
|
||||
return true;
|
||||
}
|
||||
|
||||
VoiceProcessingAudioUnit::State VoiceProcessingAudioUnit::GetState() const {
|
||||
return state_;
|
||||
}
|
||||
|
||||
bool VoiceProcessingAudioUnit::Initialize(Float64 sample_rate) {
|
||||
RTC_DCHECK_GE(state_, kUninitialized);
|
||||
RTCLog(@"Initializing audio unit with sample rate: %f", sample_rate);
|
||||
|
||||
OSStatus result = noErr;
|
||||
AudioStreamBasicDescription outputFormat = GetFormat(sample_rate, numChannels_);
|
||||
AudioStreamBasicDescription inputFormat = GetFormat(sample_rate, 1);
|
||||
UInt32 size = sizeof(outputFormat);
|
||||
#if !defined(NDEBUG)
|
||||
LogStreamDescription(outputFormat);
|
||||
#endif
|
||||
|
||||
if (!disable_recording_) {
|
||||
// Set the format on the output scope of the input element/bus.
|
||||
result =
|
||||
AudioUnitSetProperty(vpio_unit_, kAudioUnitProperty_StreamFormat,
|
||||
kAudioUnitScope_Output, kInputBus, &inputFormat, size);
|
||||
if (result != noErr) {
|
||||
RTCLogError(@"Failed to set format on output scope of input bus. "
|
||||
"Error=%ld.",
|
||||
(long)result);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Set the format on the input scope of the output element/bus.
|
||||
result =
|
||||
AudioUnitSetProperty(vpio_unit_, kAudioUnitProperty_StreamFormat,
|
||||
kAudioUnitScope_Input, kOutputBus, &outputFormat, size);
|
||||
if (result != noErr) {
|
||||
RTCLogError(@"Failed to set format on input scope of output bus. "
|
||||
"Error=%ld.",
|
||||
(long)result);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Initialize the Voice Processing I/O unit instance.
|
||||
// Calls to AudioUnitInitialize() can fail if called back-to-back on
|
||||
// different ADM instances. The error message in this case is -66635 which is
|
||||
// undocumented. Tests have shown that calling AudioUnitInitialize a second
|
||||
// time, after a short sleep, avoids this issue.
|
||||
// See webrtc:5166 for details.
|
||||
int failed_initalize_attempts = 0;
|
||||
result = AudioUnitInitialize(vpio_unit_);
|
||||
while (result != noErr) {
|
||||
RTCLogError(@"Failed to initialize the Voice Processing I/O unit. "
|
||||
"Error=%ld.",
|
||||
(long)result);
|
||||
++failed_initalize_attempts;
|
||||
if (failed_initalize_attempts == kMaxNumberOfAudioUnitInitializeAttempts) {
|
||||
// Max number of initialization attempts exceeded, hence abort.
|
||||
RTCLogError(@"Too many initialization attempts.");
|
||||
return false;
|
||||
}
|
||||
RTCLog(@"Pause 100ms and try audio unit initialization again...");
|
||||
[NSThread sleepForTimeInterval:0.1f];
|
||||
result = AudioUnitInitialize(vpio_unit_);
|
||||
}
|
||||
if (result == noErr) {
|
||||
RTCLog(@"Voice Processing I/O unit is now initialized.");
|
||||
}
|
||||
|
||||
if (bypass_voice_processing_) {
|
||||
// Attempt to disable builtin voice processing.
|
||||
UInt32 toggle = 1;
|
||||
result = AudioUnitSetProperty(vpio_unit_,
|
||||
kAUVoiceIOProperty_BypassVoiceProcessing,
|
||||
kAudioUnitScope_Global,
|
||||
kInputBus,
|
||||
&toggle,
|
||||
sizeof(toggle));
|
||||
if (result == noErr) {
|
||||
RTCLog(@"Successfully bypassed voice processing.");
|
||||
} else {
|
||||
RTCLogError(@"Failed to bypass voice processing. Error=%ld.", (long)result);
|
||||
}
|
||||
state_ = kInitialized;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!disable_recording_) {
|
||||
// AGC should be enabled by default for Voice Processing I/O units but it is
|
||||
// checked below and enabled explicitly if needed. This scheme is used
|
||||
// to be absolutely sure that the AGC is enabled since we have seen cases
|
||||
// where only zeros are recorded and a disabled AGC could be one of the
|
||||
// reasons why it happens.
|
||||
int agc_was_enabled_by_default = 0;
|
||||
UInt32 agc_is_enabled = 0;
|
||||
result = GetAGCState(vpio_unit_, &agc_is_enabled);
|
||||
if (result != noErr) {
|
||||
RTCLogError(@"Failed to get AGC state (1st attempt). "
|
||||
"Error=%ld.",
|
||||
(long)result);
|
||||
// Example of error code: kAudioUnitErr_NoConnection (-10876).
|
||||
// All error codes related to audio units are negative and are therefore
|
||||
// converted into a postive value to match the UMA APIs.
|
||||
RTC_HISTOGRAM_COUNTS_SPARSE_100000(
|
||||
"WebRTC.Audio.GetAGCStateErrorCode1", (-1) * result);
|
||||
}
|
||||
|
||||
if (agc_is_enabled) {
|
||||
// Remember that the AGC was enabled by default. Will be used in UMA.
|
||||
agc_was_enabled_by_default = 1;
|
||||
} else {
|
||||
// AGC was initially disabled => try to enable it explicitly.
|
||||
UInt32 enable_agc = 1;
|
||||
result =
|
||||
AudioUnitSetProperty(vpio_unit_,
|
||||
kAUVoiceIOProperty_VoiceProcessingEnableAGC,
|
||||
kAudioUnitScope_Global, kInputBus, &enable_agc,
|
||||
sizeof(enable_agc));
|
||||
if (result != noErr) {
|
||||
RTCLogError(@"Failed to enable the built-in AGC. "
|
||||
"Error=%ld.",
|
||||
(long)result);
|
||||
RTC_HISTOGRAM_COUNTS_SPARSE_100000(
|
||||
"WebRTC.Audio.SetAGCStateErrorCode", (-1) * result);
|
||||
}
|
||||
result = GetAGCState(vpio_unit_, &agc_is_enabled);
|
||||
if (result != noErr) {
|
||||
RTCLogError(@"Failed to get AGC state (2nd attempt). "
|
||||
"Error=%ld.",
|
||||
(long)result);
|
||||
RTC_HISTOGRAM_COUNTS_SPARSE_100000(
|
||||
"WebRTC.Audio.GetAGCStateErrorCode2", (-1) * result);
|
||||
}
|
||||
}
|
||||
|
||||
// Track if the built-in AGC was enabled by default (as it should) or not.
|
||||
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.BuiltInAGCWasEnabledByDefault",
|
||||
agc_was_enabled_by_default);
|
||||
RTCLog(@"WebRTC.Audio.BuiltInAGCWasEnabledByDefault: %d",
|
||||
agc_was_enabled_by_default);
|
||||
// As a final step, add an UMA histogram for tracking the AGC state.
|
||||
// At this stage, the AGC should be enabled, and if it is not, more work is
|
||||
// needed to find out the root cause.
|
||||
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.BuiltInAGCIsEnabled", agc_is_enabled);
|
||||
RTCLog(@"WebRTC.Audio.BuiltInAGCIsEnabled: %u",
|
||||
static_cast<unsigned int>(agc_is_enabled));
|
||||
}
|
||||
|
||||
state_ = kInitialized;
|
||||
return true;
|
||||
}
|
||||
|
||||
OSStatus VoiceProcessingAudioUnit::Start() {
|
||||
RTC_DCHECK_GE(state_, kUninitialized);
|
||||
RTCLog(@"Starting audio unit.");
|
||||
|
||||
OSStatus result = AudioOutputUnitStart(vpio_unit_);
|
||||
if (result != noErr) {
|
||||
RTCLogError(@"Failed to start audio unit. Error=%ld", (long)result);
|
||||
return result;
|
||||
} else {
|
||||
RTCLog(@"Started audio unit");
|
||||
}
|
||||
state_ = kStarted;
|
||||
return noErr;
|
||||
}
|
||||
|
||||
bool VoiceProcessingAudioUnit::Stop() {
|
||||
RTC_DCHECK_GE(state_, kUninitialized);
|
||||
RTCLog(@"Stopping audio unit.");
|
||||
|
||||
OSStatus result = AudioOutputUnitStop(vpio_unit_);
|
||||
if (result != noErr) {
|
||||
RTCLogError(@"Failed to stop audio unit. Error=%ld", (long)result);
|
||||
return false;
|
||||
} else {
|
||||
RTCLog(@"Stopped audio unit");
|
||||
}
|
||||
|
||||
state_ = kInitialized;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool VoiceProcessingAudioUnit::Uninitialize() {
|
||||
RTC_DCHECK_GE(state_, kUninitialized);
|
||||
RTCLog(@"Unintializing audio unit.");
|
||||
|
||||
OSStatus result = AudioUnitUninitialize(vpio_unit_);
|
||||
if (result != noErr) {
|
||||
RTCLogError(@"Failed to uninitialize audio unit. Error=%ld", (long)result);
|
||||
return false;
|
||||
} else {
|
||||
RTCLog(@"Uninitialized audio unit.");
|
||||
}
|
||||
|
||||
state_ = kUninitialized;
|
||||
return true;
|
||||
}
|
||||
|
||||
void VoiceProcessingAudioUnit::setIsMicrophoneMuted(bool isMuted) {
|
||||
isMicrophoneMuted_ = isMuted;
|
||||
if (enableSystemMute_) {
|
||||
if (vpio_unit_ && state_ == kStarted) {
|
||||
UInt32 muteUplinkOutput = isMuted ? 1 : 0;
|
||||
OSStatus result = AudioUnitSetProperty(vpio_unit_, kAUVoiceIOProperty_MuteOutput, kAudioUnitScope_Global, kInputBus, &muteUplinkOutput, sizeof(muteUplinkOutput));
|
||||
if (result != noErr) {
|
||||
RTCLogError(@"Failed to set kAUVoiceIOProperty_MuteOutput. Error=%ld", (long)result);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
OSStatus VoiceProcessingAudioUnit::Render(AudioUnitRenderActionFlags* flags,
|
||||
const AudioTimeStamp* time_stamp,
|
||||
UInt32 output_bus_number,
|
||||
UInt32 num_frames,
|
||||
AudioBufferList* io_data) {
|
||||
RTC_DCHECK(vpio_unit_) << "Init() not called.";
|
||||
|
||||
OSStatus result = AudioUnitRender(vpio_unit_, flags, time_stamp,
|
||||
output_bus_number, num_frames, io_data);
|
||||
if (result != noErr) {
|
||||
RTCLogError(@"Failed to render audio unit. Error=%ld", (long)result);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
OSStatus VoiceProcessingAudioUnit::OnGetPlayoutData(
|
||||
void* in_ref_con,
|
||||
AudioUnitRenderActionFlags* flags,
|
||||
const AudioTimeStamp* time_stamp,
|
||||
UInt32 bus_number,
|
||||
UInt32 num_frames,
|
||||
AudioBufferList* io_data) {
|
||||
VoiceProcessingAudioUnit* audio_unit =
|
||||
static_cast<VoiceProcessingAudioUnit*>(in_ref_con);
|
||||
return audio_unit->NotifyGetPlayoutData(flags, time_stamp, bus_number,
|
||||
num_frames, io_data);
|
||||
}
|
||||
|
||||
OSStatus VoiceProcessingAudioUnit::OnDeliverRecordedData(
|
||||
void* in_ref_con,
|
||||
AudioUnitRenderActionFlags* flags,
|
||||
const AudioTimeStamp* time_stamp,
|
||||
UInt32 bus_number,
|
||||
UInt32 num_frames,
|
||||
AudioBufferList* io_data) {
|
||||
VoiceProcessingAudioUnit* audio_unit =
|
||||
static_cast<VoiceProcessingAudioUnit*>(in_ref_con);
|
||||
return audio_unit->NotifyDeliverRecordedData(flags, time_stamp, bus_number,
|
||||
num_frames, io_data);
|
||||
}
|
||||
|
||||
OSStatus VoiceProcessingAudioUnit::NotifyGetPlayoutData(
|
||||
AudioUnitRenderActionFlags* flags,
|
||||
const AudioTimeStamp* time_stamp,
|
||||
UInt32 bus_number,
|
||||
UInt32 num_frames,
|
||||
AudioBufferList* io_data) {
|
||||
return observer_->OnGetPlayoutData(flags, time_stamp, bus_number, num_frames,
|
||||
io_data);
|
||||
}
|
||||
|
||||
OSStatus VoiceProcessingAudioUnit::NotifyDeliverRecordedData(
|
||||
AudioUnitRenderActionFlags* flags,
|
||||
const AudioTimeStamp* time_stamp,
|
||||
UInt32 bus_number,
|
||||
UInt32 num_frames,
|
||||
AudioBufferList* io_data) {
|
||||
return observer_->OnDeliverRecordedData(flags, time_stamp, bus_number,
|
||||
num_frames, io_data);
|
||||
}
|
||||
|
||||
AudioStreamBasicDescription VoiceProcessingAudioUnit::GetFormat(
|
||||
Float64 sample_rate, int numChannels) const {
|
||||
// Set the application formats for input and output:
|
||||
// - use same format in both directions
|
||||
// - avoid resampling in the I/O unit by using the hardware sample rate
|
||||
// - linear PCM => noncompressed audio data format with one frame per packet
|
||||
// - no need to specify interleaving since only mono is supported
|
||||
AudioStreamBasicDescription format;
|
||||
format.mSampleRate = sample_rate;
|
||||
format.mFormatID = kAudioFormatLinearPCM;
|
||||
format.mFormatFlags =
|
||||
kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked | kAudioFormatFlagsNativeEndian;
|
||||
format.mBytesPerPacket = kBytesPerSample * numChannels;
|
||||
format.mFramesPerPacket = 1; // uncompressed.
|
||||
format.mBytesPerFrame = kBytesPerSample * numChannels;
|
||||
format.mChannelsPerFrame = numChannels;
|
||||
format.mBitsPerChannel = 8 * kBytesPerSample;
|
||||
return format;
|
||||
}
|
||||
|
||||
void VoiceProcessingAudioUnit::DisposeAudioUnit() {
|
||||
if (vpio_unit_) {
|
||||
switch (state_) {
|
||||
case kStarted:
|
||||
Stop();
|
||||
[[fallthrough]];
|
||||
case kInitialized:
|
||||
Uninitialize();
|
||||
break;
|
||||
case kUninitialized:
|
||||
case kInitRequired:
|
||||
break;
|
||||
}
|
||||
|
||||
RTCLog(@"Disposing audio unit.");
|
||||
OSStatus result = AudioComponentInstanceDispose(vpio_unit_);
|
||||
if (result != noErr) {
|
||||
RTCLogError(@"AudioComponentInstanceDispose failed. Error=%ld.",
|
||||
(long)result);
|
||||
}
|
||||
vpio_unit_ = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace tgcalls_ios_adm
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
#import "TGRTCMTLRenderer.h"
|
||||
|
||||
NS_AVAILABLE(10_11, 9_0)
|
||||
@interface TGRTCMTLI420Renderer : TGRTCMTLRenderer
|
||||
@end
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import "TGRTCMTLI420Renderer.h"
|
||||
|
||||
#import <Metal/Metal.h>
|
||||
#import <MetalKit/MetalKit.h>
|
||||
|
||||
#import "base/RTCI420Buffer.h"
|
||||
#import "base/RTCLogging.h"
|
||||
#import "base/RTCVideoFrame.h"
|
||||
#import "base/RTCVideoFrameBuffer.h"
|
||||
|
||||
#import "TGRTCMTLRenderer+Private.h"
|
||||
|
||||
|
||||
@implementation TGRTCMTLI420Renderer {
|
||||
// Textures.
|
||||
id<MTLTexture> _yTexture;
|
||||
id<MTLTexture> _uTexture;
|
||||
id<MTLTexture> _vTexture;
|
||||
|
||||
MTLTextureDescriptor *_descriptor;
|
||||
MTLTextureDescriptor *_chromaDescriptor;
|
||||
|
||||
int _width;
|
||||
int _height;
|
||||
int _chromaWidth;
|
||||
int _chromaHeight;
|
||||
}
|
||||
|
||||
|
||||
|
||||
- (void)getWidth:(nonnull int *)width
|
||||
height:(nonnull int *)height
|
||||
ofFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame {
|
||||
*width = frame.width;
|
||||
*height = frame.height;
|
||||
}
|
||||
|
||||
|
||||
- (BOOL)setupTexturesForFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame {
|
||||
if (![super setupTexturesForFrame:frame]) {
|
||||
return NO;
|
||||
}
|
||||
|
||||
id<MTLDevice> device = [self currentMetalDevice];
|
||||
if (!device) {
|
||||
return NO;
|
||||
}
|
||||
|
||||
id<RTC_OBJC_TYPE(RTCI420Buffer)> buffer = [frame.buffer toI420];
|
||||
|
||||
// Luma (y) texture.
|
||||
if (!_descriptor || _width != frame.width || _height != frame.height) {
|
||||
_width = frame.width;
|
||||
_height = frame.height;
|
||||
_descriptor = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatR8Unorm
|
||||
width:_width
|
||||
height:_height
|
||||
mipmapped:NO];
|
||||
_descriptor.usage = MTLTextureUsageShaderRead;
|
||||
_yTexture = [device newTextureWithDescriptor:_descriptor];
|
||||
}
|
||||
|
||||
// Chroma (u,v) textures
|
||||
[_yTexture replaceRegion:MTLRegionMake2D(0, 0, _width, _height)
|
||||
mipmapLevel:0
|
||||
withBytes:buffer.dataY
|
||||
bytesPerRow:buffer.strideY];
|
||||
|
||||
if (!_chromaDescriptor || _chromaWidth != frame.width / 2 || _chromaHeight != frame.height / 2) {
|
||||
_chromaWidth = frame.width / 2;
|
||||
_chromaHeight = frame.height / 2;
|
||||
_chromaDescriptor =
|
||||
[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatR8Unorm
|
||||
width:_chromaWidth
|
||||
height:_chromaHeight
|
||||
mipmapped:NO];
|
||||
_chromaDescriptor.usage = MTLTextureUsageShaderRead;
|
||||
_uTexture = [device newTextureWithDescriptor:_chromaDescriptor];
|
||||
_vTexture = [device newTextureWithDescriptor:_chromaDescriptor];
|
||||
}
|
||||
|
||||
[_uTexture replaceRegion:MTLRegionMake2D(0, 0, _chromaWidth, _chromaHeight)
|
||||
mipmapLevel:0
|
||||
withBytes:buffer.dataU
|
||||
bytesPerRow:buffer.strideU];
|
||||
[_vTexture replaceRegion:MTLRegionMake2D(0, 0, _chromaWidth, _chromaHeight)
|
||||
mipmapLevel:0
|
||||
withBytes:buffer.dataV
|
||||
bytesPerRow:buffer.strideV];
|
||||
|
||||
return (_uTexture != nil) && (_yTexture != nil) && (_vTexture != nil);
|
||||
}
|
||||
|
||||
- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder {
|
||||
[renderEncoder setFragmentTexture:_yTexture atIndex:0];
|
||||
[renderEncoder setFragmentTexture:_uTexture atIndex:1];
|
||||
[renderEncoder setFragmentTexture:_vTexture atIndex:2];
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import <Metal/Metal.h>
|
||||
|
||||
#import "TGRTCMTLRenderer.h"
|
||||
|
||||
#define MTL_STRINGIFY(s) @ #s
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
@interface TGRTCMTLRenderer (Private)
|
||||
- (nullable id<MTLDevice>)currentMetalDevice;
|
||||
- (NSString *)shaderSource;
|
||||
- (BOOL)setupTexturesForFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame;
|
||||
- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder;
|
||||
- (void)getWidth:(nonnull int *)width
|
||||
height:(nonnull int *)height
|
||||
cropWidth:(nonnull int *)cropWidth
|
||||
cropHeight:(nonnull int *)cropHeight
|
||||
cropX:(nonnull int *)cropX
|
||||
cropY:(nonnull int *)cropY
|
||||
ofFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame;
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <AppKit/AppKit.h>
|
||||
#import <MetalKit/MetalKit.h>
|
||||
|
||||
#import "base/RTCVideoFrame.h"
|
||||
|
||||
bool initMetal();
|
||||
|
||||
struct MTLFrameSize {
|
||||
float width = 0;
|
||||
float height = 0;
|
||||
};
|
||||
MTLFrameSize MTLAspectFilled(MTLFrameSize from, MTLFrameSize to);
|
||||
MTLFrameSize MTLAspectFitted(MTLFrameSize from, MTLFrameSize to);
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
typedef enum {
|
||||
TGRTCMTLRenderModeSinge,
|
||||
TGRTCMTLRenderModeDouble
|
||||
} TGRTCMTLRenderMode;
|
||||
|
||||
@protocol TGRTCMTLRenderer <NSObject>
|
||||
|
||||
- (void)drawFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame;
|
||||
- (BOOL)setSingleRendering:(__kindof CAMetalLayer *)view;
|
||||
- (BOOL)setDoubleRendering:(__kindof CAMetalLayer *)view foreground: (__kindof CAMetalLayer *)foreground;
|
||||
|
||||
@end
|
||||
|
||||
|
||||
NS_AVAILABLE(10_11, 9_0)
|
||||
@interface TGRTCMTLRenderer : NSObject <TGRTCMTLRenderer>
|
||||
|
||||
@property(atomic, nullable) NSValue *rotationOverride;
|
||||
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
|
|
@ -0,0 +1,641 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#import "TGRTCMTLRenderer+Private.h"
|
||||
|
||||
#import <Metal/Metal.h>
|
||||
#import <MetalKit/MetalKit.h>
|
||||
#import <MetalPerformanceShaders/MetalPerformanceShaders.h>
|
||||
#import "base/RTCLogging.h"
|
||||
#import "base/RTCVideoFrame.h"
|
||||
#import "base/RTCVideoFrameBuffer.h"
|
||||
#import "TGRTCMetalContextHolder.h"
|
||||
|
||||
#include "api/video/video_rotation.h"
|
||||
#include "rtc_base/checks.h"
|
||||
|
||||
|
||||
MTLFrameSize MTLAspectFitted(MTLFrameSize from, MTLFrameSize to) {
|
||||
double scale = std::min(
|
||||
from.width / std::max(1., double(to.width)),
|
||||
from.height / std::max(1., double(to.height)));
|
||||
return {
|
||||
float(std::ceil(to.width * scale)),
|
||||
float(std::ceil(to.height * scale))
|
||||
};
|
||||
}
|
||||
|
||||
MTLFrameSize MTLAspectFilled(MTLFrameSize from, MTLFrameSize to) {
|
||||
double scale = std::max(
|
||||
to.width / std::max(1., double(from.width)),
|
||||
to.height / std::max(1., double(from.height)));
|
||||
return {
|
||||
float(std::ceil(from.width * scale)),
|
||||
float(std::ceil(from.height * scale))
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
static NSString *const pipelineDescriptorLabel = @"RTCPipeline";
|
||||
static NSString *const commandBufferLabel = @"RTCCommandBuffer";
|
||||
static NSString *const renderEncoderLabel = @"RTCEncoder";
|
||||
static NSString *const renderEncoderDebugGroup = @"RTCDrawFrame";
|
||||
|
||||
|
||||
static TGRTCMetalContextHolder *metalContext = nil;
|
||||
|
||||
|
||||
|
||||
bool initMetal() {
|
||||
if (metalContext == nil) {
|
||||
metalContext = [[TGRTCMetalContextHolder alloc] init];
|
||||
} else if(metalContext.displayId != CGMainDisplayID()) {
|
||||
metalContext = [[TGRTCMetalContextHolder alloc] init];
|
||||
}
|
||||
return metalContext != nil;
|
||||
}
|
||||
|
||||
static inline void getCubeVertexData(size_t frameWidth,
|
||||
size_t frameHeight,
|
||||
RTCVideoRotation rotation,
|
||||
float *buffer) {
|
||||
// The computed values are the adjusted texture coordinates, in [0..1].
|
||||
// For the left and top, 0.0 means no cropping and e.g. 0.2 means we're skipping 20% of the
|
||||
// left/top edge.
|
||||
// For the right and bottom, 1.0 means no cropping and e.g. 0.8 means we're skipping 20% of the
|
||||
// right/bottom edge (i.e. render up to 80% of the width/height).
|
||||
float cropLeft = 0;
|
||||
float cropRight = 1;
|
||||
float cropTop = 0;
|
||||
float cropBottom = 1;
|
||||
|
||||
// These arrays map the view coordinates to texture coordinates, taking cropping and rotation
|
||||
// into account. The first two columns are view coordinates, the last two are texture coordinates.
|
||||
switch (rotation) {
|
||||
case RTCVideoRotation_0: {
|
||||
float values[16] = {-1.0, -1.0, cropLeft, cropBottom,
|
||||
1.0, -1.0, cropRight, cropBottom,
|
||||
-1.0, 1.0, cropLeft, cropTop,
|
||||
1.0, 1.0, cropRight, cropTop};
|
||||
memcpy(buffer, &values, sizeof(values));
|
||||
} break;
|
||||
case RTCVideoRotation_90: {
|
||||
float values[16] = {-1.0, -1.0, cropRight, cropBottom,
|
||||
1.0, -1.0, cropRight, cropTop,
|
||||
-1.0, 1.0, cropLeft, cropBottom,
|
||||
1.0, 1.0, cropLeft, cropTop};
|
||||
memcpy(buffer, &values, sizeof(values));
|
||||
} break;
|
||||
case RTCVideoRotation_180: {
|
||||
float values[16] = {-1.0, -1.0, cropRight, cropTop,
|
||||
1.0, -1.0, cropLeft, cropTop,
|
||||
-1.0, 1.0, cropRight, cropBottom,
|
||||
1.0, 1.0, cropLeft, cropBottom};
|
||||
memcpy(buffer, &values, sizeof(values));
|
||||
} break;
|
||||
case RTCVideoRotation_270: {
|
||||
float values[16] = {-1.0, -1.0, cropLeft, cropTop,
|
||||
1.0, -1.0, cropLeft, cropBottom,
|
||||
-1.0, 1.0, cropRight, cropTop,
|
||||
1.0, 1.0, cropRight, cropBottom};
|
||||
memcpy(buffer, &values, sizeof(values));
|
||||
} break;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@implementation TGRTCMTLRenderer {
|
||||
__kindof CAMetalLayer *_view;
|
||||
|
||||
__kindof CAMetalLayer *_foreground;
|
||||
|
||||
|
||||
TGRTCMetalContextHolder* _context;
|
||||
|
||||
id<MTLCommandQueue> _commandQueue;
|
||||
id<MTLBuffer> _vertexBuffer;
|
||||
id<MTLBuffer> _vertexBufferRotated;
|
||||
MTLFrameSize _frameSize;
|
||||
MTLFrameSize _scaledSize;
|
||||
|
||||
RTCVideoFrame *_frame;
|
||||
bool _frameIsUpdated;
|
||||
|
||||
RTCVideoRotation _rotation;
|
||||
bool _rotationInited;
|
||||
|
||||
id<MTLTexture> _rgbTexture;
|
||||
id<MTLTexture> _rgbScaledAndBlurredTexture;
|
||||
|
||||
id<MTLBuffer> _vertexBuffer0;
|
||||
id<MTLBuffer> _vertexBuffer1;
|
||||
id<MTLBuffer> _vertexBuffer2;
|
||||
|
||||
dispatch_semaphore_t _inflight1;
|
||||
dispatch_semaphore_t _inflight2;
|
||||
|
||||
}
|
||||
|
||||
|
||||
@synthesize rotationOverride = _rotationOverride;
|
||||
|
||||
- (instancetype)init {
|
||||
if (self = [super init]) {
|
||||
_inflight1 = dispatch_semaphore_create(0);
|
||||
_inflight2 = dispatch_semaphore_create(0);
|
||||
|
||||
_context = metalContext;
|
||||
_commandQueue = [_context.device newCommandQueueWithMaxCommandBufferCount:3];
|
||||
|
||||
|
||||
float vertexBufferArray[16] = {0};
|
||||
_vertexBuffer = [metalContext.device newBufferWithBytes:vertexBufferArray
|
||||
length:sizeof(vertexBufferArray)
|
||||
options:MTLResourceCPUCacheModeWriteCombined];
|
||||
float vertexBufferArrayRotated[16] = {0};
|
||||
_vertexBufferRotated = [metalContext.device newBufferWithBytes:vertexBufferArrayRotated
|
||||
length:sizeof(vertexBufferArrayRotated)
|
||||
options:MTLResourceCPUCacheModeWriteCombined];
|
||||
|
||||
|
||||
float verts[8] = {-1.0, 1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0};
|
||||
|
||||
_vertexBuffer0 = [metalContext.device newBufferWithBytes:verts length:sizeof(verts) options:0];
|
||||
|
||||
float values[8] = {0};
|
||||
|
||||
_vertexBuffer1 = [metalContext.device newBufferWithBytes:values
|
||||
length:sizeof(values)
|
||||
options:0];
|
||||
|
||||
_vertexBuffer2 = [metalContext.device newBufferWithBytes:values
|
||||
length:sizeof(values)
|
||||
options:0];
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
return self;
|
||||
}
|
||||
|
||||
- (BOOL)setSingleRendering:(__kindof CAMetalLayer *)view {
|
||||
return [self setupWithView:view foreground: nil];
|
||||
}
|
||||
- (BOOL)setDoubleRendering:(__kindof CAMetalLayer *)view foreground:(nonnull __kindof CAMetalLayer *)foreground {
|
||||
return [self setupWithView:view foreground: foreground];
|
||||
}
|
||||
|
||||
#pragma mark - Private
|
||||
|
||||
- (BOOL)setupWithView:(__kindof CAMetalLayer *)view foreground: (__kindof CAMetalLayer *)foreground {
|
||||
_view = view;
|
||||
_foreground = foreground;
|
||||
view.device = metalContext.device;
|
||||
foreground.device = metalContext.device;
|
||||
_context = metalContext;
|
||||
_rotationInited = false;
|
||||
return YES;
|
||||
}
|
||||
#pragma mark - Inheritance
|
||||
|
||||
- (id<MTLDevice>)currentMetalDevice {
|
||||
return metalContext.device;
|
||||
}
|
||||
|
||||
|
||||
- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder {
|
||||
// RTC_NOTREACHED() << "Virtual method not implemented in subclass.";
|
||||
}
|
||||
|
||||
- (void)getWidth:(int *)width
|
||||
height:(int *)height
|
||||
ofFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame {
|
||||
// RTC_NOTREACHED() << "Virtual method not implemented in subclass.";
|
||||
}
|
||||
|
||||
- (BOOL)setupTexturesForFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame {
|
||||
RTCVideoRotation rotation;
|
||||
NSValue *rotationOverride = self.rotationOverride;
|
||||
if (rotationOverride) {
|
||||
[rotationOverride getValue:&rotation];
|
||||
} else {
|
||||
rotation = frame.rotation;
|
||||
}
|
||||
|
||||
_frameIsUpdated = true;//_frame.timeStampNs != frame.timeStampNs;
|
||||
_frame = frame;
|
||||
|
||||
int frameWidth, frameHeight;
|
||||
[self getWidth:&frameWidth
|
||||
height:&frameHeight
|
||||
ofFrame:frame];
|
||||
|
||||
if (frameWidth != _frameSize.width || frameHeight != _frameSize.height || _rotation != rotation || !_rotationInited) {
|
||||
|
||||
bool rotationIsUpdated = _rotation != rotation || !_rotationInited;
|
||||
|
||||
_rotation = rotation;
|
||||
_frameSize.width = frameWidth;
|
||||
_frameSize.height = frameHeight;
|
||||
_frameIsUpdated = true;
|
||||
|
||||
|
||||
MTLFrameSize small;
|
||||
small.width = _frameSize.width / 4;
|
||||
small.height = _frameSize.height / 4;
|
||||
|
||||
_scaledSize = MTLAspectFitted(small, _frameSize);
|
||||
_rgbTexture = [self createTextureWithUsage: MTLTextureUsageShaderRead|MTLTextureUsageRenderTarget size:_frameSize];
|
||||
|
||||
_rgbScaledAndBlurredTexture = [self createTextureWithUsage:MTLTextureUsageShaderRead|MTLTextureUsageRenderTarget size:_scaledSize];
|
||||
|
||||
|
||||
if (rotationIsUpdated) {
|
||||
getCubeVertexData(frameWidth,
|
||||
frameHeight,
|
||||
RTCVideoRotation_0,
|
||||
(float *)_vertexBuffer.contents);
|
||||
|
||||
getCubeVertexData(frameWidth,
|
||||
frameHeight,
|
||||
rotation,
|
||||
(float *)_vertexBufferRotated.contents);
|
||||
|
||||
|
||||
switch (rotation) {
|
||||
case RTCVideoRotation_0: {
|
||||
float values[8] = {0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0};
|
||||
memcpy((float *)_vertexBuffer1.contents, &values, sizeof(values));
|
||||
memcpy((float *)_vertexBuffer2.contents, &values, sizeof(values));
|
||||
} break;
|
||||
case RTCVideoRotation_90: {
|
||||
float values[8] = {0.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0};
|
||||
memcpy((float *)_vertexBuffer1.contents, &values, sizeof(values));
|
||||
memcpy((float *)_vertexBuffer2.contents, &values, sizeof(values));
|
||||
} break;
|
||||
case RTCVideoRotation_180: {
|
||||
//[xLimit, yLimit, 0.0, yLimit, xLimit, 0.0, 0.0, 0.0]
|
||||
float values[8] = {1.0, 1.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0};
|
||||
memcpy(_vertexBuffer1.contents, &values, sizeof(values));
|
||||
memcpy(_vertexBuffer2.contents, &values, sizeof(values));
|
||||
} break;
|
||||
case RTCVideoRotation_270: {
|
||||
float values[8] = {1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0};
|
||||
memcpy(_vertexBuffer1.contents, &values, sizeof(values));
|
||||
memcpy(_vertexBuffer2.contents, &values, sizeof(values));
|
||||
} break;
|
||||
}
|
||||
_rotationInited = true;
|
||||
}
|
||||
}
|
||||
return YES;
|
||||
}
|
||||
|
||||
#pragma mark - GPU methods
|
||||
|
||||
|
||||
- (id<MTLTexture>)createTextureWithUsage:(MTLTextureUsage) usage size:(MTLFrameSize)size {
|
||||
MTLTextureDescriptor *rgbTextureDescriptor = [MTLTextureDescriptor
|
||||
texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm
|
||||
width:size.width
|
||||
height:size.height
|
||||
mipmapped:NO];
|
||||
rgbTextureDescriptor.usage = usage;
|
||||
return [metalContext.device newTextureWithDescriptor:rgbTextureDescriptor];
|
||||
}
|
||||
|
||||
- (id<MTLRenderCommandEncoder>)createRenderEncoderForTarget: (id<MTLTexture>)texture with: (id<MTLCommandBuffer>)commandBuffer {
|
||||
MTLRenderPassDescriptor *renderPassDescriptor = [[MTLRenderPassDescriptor alloc] init];
|
||||
renderPassDescriptor.colorAttachments[0].texture = texture;
|
||||
renderPassDescriptor.colorAttachments[0].loadAction = MTLLoadActionDontCare;
|
||||
renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColorMake(0, 0, 0, 0);
|
||||
|
||||
id<MTLRenderCommandEncoder> renderEncoder = [commandBuffer renderCommandEncoderWithDescriptor:renderPassDescriptor];
|
||||
renderEncoder.label = renderEncoderLabel;
|
||||
|
||||
return renderEncoder;
|
||||
}
|
||||
|
||||
|
||||
- (id<MTLTexture>)convertYUVtoRGV:(id<MTLBuffer>)buffer {
|
||||
id<MTLTexture> rgbTexture = _rgbTexture;
|
||||
if (_frameIsUpdated) {
|
||||
id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
|
||||
|
||||
id<MTLRenderCommandEncoder> renderEncoder = [self createRenderEncoderForTarget: rgbTexture with: commandBuffer];
|
||||
[renderEncoder pushDebugGroup:renderEncoderDebugGroup];
|
||||
[renderEncoder setRenderPipelineState:_context.pipelineYuvRgb];
|
||||
[renderEncoder setVertexBuffer:buffer offset:0 atIndex:0];
|
||||
[self uploadTexturesToRenderEncoder:renderEncoder];
|
||||
[renderEncoder setFragmentSamplerState:_context.sampler atIndex:0];
|
||||
|
||||
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip
|
||||
vertexStart:0
|
||||
vertexCount:4
|
||||
instanceCount:1];
|
||||
[renderEncoder popDebugGroup];
|
||||
[renderEncoder endEncoding];
|
||||
|
||||
[commandBuffer commit];
|
||||
}
|
||||
|
||||
return rgbTexture;
|
||||
}
|
||||
|
||||
- (id<MTLTexture>)scaleAndBlur:(id<MTLTexture>)inputTexture scale:(simd_float2)scale {
|
||||
|
||||
id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
|
||||
|
||||
id<MTLRenderCommandEncoder> renderEncoder = [self createRenderEncoderForTarget: _rgbScaledAndBlurredTexture with: commandBuffer];
|
||||
[renderEncoder pushDebugGroup:renderEncoderDebugGroup];
|
||||
[renderEncoder setRenderPipelineState:_context.pipelineScaleAndBlur];
|
||||
|
||||
[renderEncoder setFragmentTexture:inputTexture atIndex:0];
|
||||
|
||||
[renderEncoder setVertexBuffer:_vertexBuffer offset:0 atIndex:0];
|
||||
|
||||
[renderEncoder setFragmentBytes:&scale length:sizeof(scale) atIndex:0];
|
||||
[renderEncoder setFragmentSamplerState:_context.sampler atIndex:0];
|
||||
|
||||
bool vertical = true;
|
||||
[renderEncoder setFragmentBytes:&vertical length:sizeof(vertical) atIndex:1];
|
||||
|
||||
|
||||
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip
|
||||
vertexStart:0
|
||||
vertexCount:4
|
||||
instanceCount:1];
|
||||
[renderEncoder popDebugGroup];
|
||||
[renderEncoder endEncoding];
|
||||
|
||||
[commandBuffer commit];
|
||||
// [commandBuffer waitUntilCompleted];
|
||||
|
||||
return _rgbScaledAndBlurredTexture;
|
||||
}
|
||||
|
||||
- (void)mergeYUVTexturesInTarget:(id<MTLTexture>)targetTexture foregroundTexture: (id<MTLTexture>)foregroundTexture backgroundTexture:(id<MTLTexture>)backgroundTexture scale1:(simd_float2)scale1 scale2:(simd_float2)scale2 {
|
||||
id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
|
||||
|
||||
id<MTLRenderCommandEncoder> renderEncoder = [self createRenderEncoderForTarget: targetTexture with: commandBuffer];
|
||||
[renderEncoder pushDebugGroup:renderEncoderDebugGroup];
|
||||
[renderEncoder setRenderPipelineState:_context.pipelineTransformAndBlend];
|
||||
|
||||
|
||||
[renderEncoder setVertexBuffer:_vertexBuffer0 offset:0 atIndex:0];
|
||||
[renderEncoder setVertexBuffer:_vertexBuffer1 offset:0 atIndex:1];
|
||||
[renderEncoder setVertexBuffer:_vertexBuffer2 offset:0 atIndex:2];
|
||||
|
||||
[renderEncoder setFragmentTexture:foregroundTexture atIndex:0];
|
||||
[renderEncoder setFragmentTexture:backgroundTexture atIndex:1];
|
||||
|
||||
[renderEncoder setFragmentBytes:&scale1 length:sizeof(scale1) atIndex:0];
|
||||
[renderEncoder setFragmentBytes:&scale2 length:sizeof(scale2) atIndex:1];
|
||||
|
||||
[renderEncoder setFragmentSamplerState:_context.sampler atIndex:0];
|
||||
[renderEncoder setFragmentSamplerState:_context.sampler atIndex:1];
|
||||
|
||||
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip
|
||||
vertexStart:0
|
||||
vertexCount:4
|
||||
instanceCount:1];
|
||||
[renderEncoder popDebugGroup];
|
||||
[renderEncoder endEncoding];
|
||||
|
||||
[commandBuffer commit];
|
||||
// [commandBuffer waitUntilCompleted];
|
||||
}
|
||||
|
||||
- (void)doubleRender {
|
||||
id<CAMetalDrawable> background = _view.nextDrawable;
|
||||
id<CAMetalDrawable> foreground = _foreground.nextDrawable;
|
||||
|
||||
_rgbTexture = [self convertYUVtoRGV:_vertexBufferRotated];
|
||||
|
||||
CGSize drawableSize = _view.drawableSize;
|
||||
|
||||
MTLFrameSize from;
|
||||
MTLFrameSize to;
|
||||
|
||||
MTLFrameSize frameSize = _frameSize;
|
||||
MTLFrameSize scaledSize = _scaledSize;
|
||||
|
||||
|
||||
|
||||
from.width = _view.bounds.size.width;
|
||||
from.height = _view.bounds.size.height;
|
||||
|
||||
to.width = drawableSize.width;
|
||||
to.height = drawableSize.height;
|
||||
|
||||
// bool swap = _rotation == RTCVideoRotation_90 || _rotation == RTCVideoRotation_270;
|
||||
|
||||
// if (swap) {
|
||||
// frameSize.width = _frameSize.height;
|
||||
// frameSize.height = _frameSize.width;
|
||||
// scaledSize.width = _scaledSize.height;
|
||||
// scaledSize.height = _scaledSize.width;
|
||||
// }
|
||||
|
||||
_rgbScaledAndBlurredTexture = [self scaleAndBlur:_rgbTexture scale:simd_make_float2(frameSize.width / scaledSize.width, frameSize.height/ scaledSize.height)];
|
||||
|
||||
id<MTLCommandBuffer> commandBuffer_b = [_commandQueue commandBuffer];
|
||||
{
|
||||
id<MTLRenderCommandEncoder> renderEncoder = [self createRenderEncoderForTarget: background.texture with: commandBuffer_b];
|
||||
[renderEncoder pushDebugGroup:renderEncoderDebugGroup];
|
||||
[renderEncoder setRenderPipelineState:_context.pipelineScaleAndBlur];
|
||||
[renderEncoder setFragmentTexture:_rgbScaledAndBlurredTexture atIndex:0];
|
||||
[renderEncoder setVertexBuffer:_vertexBuffer offset:0 atIndex:0];
|
||||
simd_float2 scale = simd_make_float2(scaledSize.width / frameSize.width, scaledSize.height / frameSize.height);
|
||||
[renderEncoder setFragmentBytes:&scale length:sizeof(scale) atIndex:0];
|
||||
bool vertical = false;
|
||||
[renderEncoder setFragmentBytes:&vertical length:sizeof(vertical) atIndex:1];
|
||||
[renderEncoder setFragmentSamplerState:_context.sampler atIndex:0];
|
||||
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip
|
||||
vertexStart:0
|
||||
vertexCount:4
|
||||
instanceCount:1];
|
||||
[renderEncoder popDebugGroup];
|
||||
[renderEncoder endEncoding];
|
||||
}
|
||||
|
||||
|
||||
id<MTLCommandBuffer> commandBuffer_f = [_commandQueue commandBuffer];
|
||||
{
|
||||
id<MTLRenderCommandEncoder> renderEncoder = [self createRenderEncoderForTarget: foreground.texture with: commandBuffer_f];
|
||||
[renderEncoder pushDebugGroup:renderEncoderDebugGroup];
|
||||
[renderEncoder setRenderPipelineState:_context.pipelineThrough];
|
||||
[renderEncoder setFragmentTexture:_rgbTexture atIndex:0];
|
||||
[renderEncoder setVertexBuffer:_vertexBuffer offset:0 atIndex:0];
|
||||
[renderEncoder setFragmentSamplerState:_context.sampler atIndex:0];
|
||||
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip
|
||||
vertexStart:0
|
||||
vertexCount:4
|
||||
instanceCount:1];
|
||||
[renderEncoder popDebugGroup];
|
||||
[renderEncoder endEncoding];
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
dispatch_semaphore_t inflight = _inflight2;
|
||||
|
||||
[commandBuffer_f addCompletedHandler:^(id<MTLCommandBuffer> _Nonnull) {
|
||||
dispatch_semaphore_signal(inflight);
|
||||
}];
|
||||
[commandBuffer_b addCompletedHandler:^(id<MTLCommandBuffer> _Nonnull) {
|
||||
dispatch_semaphore_signal(inflight);
|
||||
}];
|
||||
|
||||
[commandBuffer_b addScheduledHandler:^(id<MTLCommandBuffer> _Nonnull) {
|
||||
[background present];
|
||||
}];
|
||||
[commandBuffer_f addScheduledHandler:^(id<MTLCommandBuffer> _Nonnull) {
|
||||
[foreground present];
|
||||
}];
|
||||
|
||||
|
||||
[commandBuffer_f commit];
|
||||
[commandBuffer_b commit];
|
||||
|
||||
dispatch_semaphore_wait(inflight, DISPATCH_TIME_FOREVER);
|
||||
dispatch_semaphore_wait(inflight, DISPATCH_TIME_FOREVER);
|
||||
|
||||
}
|
||||
- (void)singleRender {
|
||||
id<CAMetalDrawable> drawable = _view.nextDrawable;
|
||||
|
||||
CGSize drawableSize = _view.drawableSize;
|
||||
|
||||
MTLFrameSize from;
|
||||
MTLFrameSize to;
|
||||
|
||||
MTLFrameSize frameSize = _frameSize;
|
||||
MTLFrameSize scaledSize = _scaledSize;
|
||||
|
||||
|
||||
|
||||
from.width = _view.bounds.size.width;
|
||||
from.height = _view.bounds.size.height;
|
||||
|
||||
to.width = drawableSize.width;
|
||||
to.height = drawableSize.height;
|
||||
|
||||
bool swap = _rotation == RTCVideoRotation_90 || _rotation == RTCVideoRotation_270;
|
||||
|
||||
if (swap) {
|
||||
frameSize.width = _frameSize.height;
|
||||
frameSize.height = _frameSize.width;
|
||||
scaledSize.width = _scaledSize.height;
|
||||
scaledSize.height = _scaledSize.width;
|
||||
}
|
||||
|
||||
float ratio = (float)frameSize.height / (float)frameSize.width;
|
||||
|
||||
|
||||
|
||||
MTLFrameSize viewSize = MTLAspectFilled(to, from);
|
||||
|
||||
MTLFrameSize fitted = MTLAspectFitted(from, to);
|
||||
|
||||
|
||||
CGSize viewPortSize = CGSizeMake(viewSize.width, viewSize.height);
|
||||
|
||||
|
||||
|
||||
id<MTLTexture> targetTexture = drawable.texture;
|
||||
|
||||
|
||||
CGFloat heightAspectScale = viewPortSize.height / (fitted.width * ratio);
|
||||
CGFloat widthAspectScale = viewPortSize.width / (fitted.height * (1.0/ratio));
|
||||
|
||||
_rgbTexture = [self convertYUVtoRGV:_vertexBuffer];
|
||||
|
||||
simd_float2 smallScale = simd_make_float2(frameSize.width / scaledSize.width, frameSize.height / scaledSize.height);
|
||||
|
||||
_rgbScaledAndBlurredTexture = [self scaleAndBlur:_rgbTexture scale:smallScale];
|
||||
|
||||
simd_float2 scale1 = simd_make_float2(MAX(1.0, widthAspectScale), MAX(1.0, heightAspectScale));
|
||||
|
||||
float bgRatio_w = scaledSize.width / frameSize.width;
|
||||
float bgRatio_h = scaledSize.height / frameSize.height;
|
||||
|
||||
|
||||
|
||||
simd_float2 scale2 = simd_make_float2(MIN(bgRatio_w, widthAspectScale * bgRatio_w), MIN(bgRatio_h, heightAspectScale * bgRatio_h));
|
||||
|
||||
|
||||
if (swap) {
|
||||
scale1 = simd_make_float2(MAX(1.0, heightAspectScale), MAX(1.0, widthAspectScale));
|
||||
scale2 = simd_make_float2(MIN(1, heightAspectScale * 1), MIN(bgRatio_h, widthAspectScale * bgRatio_h));
|
||||
}
|
||||
|
||||
[self mergeYUVTexturesInTarget: targetTexture
|
||||
foregroundTexture: _rgbTexture
|
||||
backgroundTexture: _rgbScaledAndBlurredTexture
|
||||
scale1:scale1
|
||||
scale2:scale2];
|
||||
|
||||
id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
|
||||
|
||||
|
||||
dispatch_semaphore_t inflight = _inflight1;
|
||||
|
||||
[commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> _Nonnull) {
|
||||
dispatch_semaphore_signal(inflight);
|
||||
}];
|
||||
|
||||
|
||||
[commandBuffer addScheduledHandler:^(id<MTLCommandBuffer> _Nonnull) {
|
||||
[drawable present];
|
||||
}];
|
||||
|
||||
[commandBuffer commit];
|
||||
|
||||
dispatch_semaphore_wait(inflight, DISPATCH_TIME_FOREVER);
|
||||
|
||||
// [commandBuffer commit];
|
||||
|
||||
}
|
||||
|
||||
-(void)dealloc {
|
||||
dispatch_semaphore_signal(_inflight1);
|
||||
dispatch_semaphore_signal(_inflight2);
|
||||
dispatch_semaphore_signal(_inflight2);
|
||||
__block CAMetalLayer *view = _view;
|
||||
__block CAMetalLayer *foreground = _foreground;
|
||||
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
view = nil;
|
||||
foreground = nil;
|
||||
});
|
||||
}
|
||||
|
||||
#pragma mark - RTCMTLRenderer
|
||||
|
||||
- (void)drawFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame {
|
||||
@autoreleasepool {
|
||||
if (frame.width != 0 && frame.height != 0) {
|
||||
if ([self setupTexturesForFrame:frame]) {
|
||||
if (_foreground) {
|
||||
[self doubleRender];
|
||||
} else {
|
||||
[self singleRender];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
//
|
||||
// TGRTCMetalContextHolder.h
|
||||
// TgVoipWebrtc
|
||||
//
|
||||
// Created by Mikhail Filimonov on 28.06.2021.
|
||||
// Copyright © 2021 Mikhail Filimonov. All rights reserved.
|
||||
//
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <MetalKit/MetalKit.h>
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
@interface TGRTCMetalContextHolder : NSObject
|
||||
-(id __nullable)init;
|
||||
-(id<MTLDevice>)device;
|
||||
-(id<MTLRenderPipelineState>)pipelineYuvRgb;
|
||||
-(id<MTLRenderPipelineState>)pipelineTransformAndBlend;
|
||||
-(id<MTLRenderPipelineState>)pipelineScaleAndBlur;
|
||||
-(id<MTLRenderPipelineState>)pipelineThrough;
|
||||
-(id<MTLSamplerState>)sampler;
|
||||
-(CGDirectDisplayID)displayId;
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
|
|
@ -0,0 +1,136 @@
|
|||
//
|
||||
// TGRTCMetalContextHolder.m
|
||||
// TgVoipWebrtc
|
||||
//
|
||||
// Created by Mikhail Filimonov on 28.06.2021.
|
||||
// Copyright © 2021 Mikhail Filimonov. All rights reserved.
|
||||
//
|
||||
|
||||
#import "TGRTCMetalContextHolder.h"
|
||||
|
||||
static NSString *const vertexFunctionName = @"vertexPassthrough";
|
||||
static NSString *const fragmentFunctionName = @"fragmentColorConversion";
|
||||
static NSString *const fragmentDoTransformFilter = @"doTransformFilter";
|
||||
static NSString *const twoInputVertexName = @"twoInputVertex";
|
||||
static NSString *const transformAndBlendName = @"transformAndBlend";
|
||||
static NSString *const scaleAndBlurName = @"scaleAndBlur";
|
||||
static NSString *const fragmentPlainName = @"fragmentPlain";
|
||||
|
||||
|
||||
|
||||
@implementation TGRTCMetalContextHolder
|
||||
{
|
||||
id<MTLDevice> _device;
|
||||
|
||||
id<MTLRenderPipelineState> _pipelineYuvRgb;
|
||||
id<MTLRenderPipelineState> _pipelineTransformAndBlend;
|
||||
id<MTLRenderPipelineState> _pipelineScaleAndBlur;
|
||||
id<MTLRenderPipelineState> _pipelineThrough;
|
||||
|
||||
id<MTLSamplerState> _sampler;
|
||||
id<MTLLibrary> _defaultLibrary;
|
||||
|
||||
CGDirectDisplayID _displayId;
|
||||
}
|
||||
|
||||
-(id __nullable)init {
|
||||
if(self = [super init]) {
|
||||
_displayId = CGMainDisplayID();
|
||||
_device = CGDirectDisplayCopyCurrentMetalDevice(_displayId);
|
||||
_defaultLibrary = [_device newDefaultLibrary];
|
||||
}
|
||||
if (!_device) {
|
||||
return nil;
|
||||
}
|
||||
_sampler = [self defaultSamplerState:_device];
|
||||
[self loadPipelines];
|
||||
|
||||
return self;
|
||||
}
|
||||
|
||||
|
||||
- (id<MTLSamplerState>)defaultSamplerState:(id<MTLDevice>)device {
|
||||
MTLSamplerDescriptor *samplerDescriptor = [[MTLSamplerDescriptor alloc] init];
|
||||
samplerDescriptor.minFilter = MTLSamplerMinMagFilterLinear;
|
||||
samplerDescriptor.magFilter = MTLSamplerMinMagFilterLinear;
|
||||
samplerDescriptor.sAddressMode = MTLSamplerAddressModeClampToZero;
|
||||
samplerDescriptor.tAddressMode = MTLSamplerAddressModeClampToZero;
|
||||
|
||||
return [device newSamplerStateWithDescriptor:samplerDescriptor];
|
||||
}
|
||||
|
||||
- (void)loadPipelines {
|
||||
|
||||
{
|
||||
id<MTLFunction> vertexFunction = [_defaultLibrary newFunctionWithName:vertexFunctionName];
|
||||
id<MTLFunction> fragmentFunction = [_defaultLibrary newFunctionWithName:fragmentFunctionName];
|
||||
|
||||
MTLRenderPipelineDescriptor *pipelineDescriptor = [[MTLRenderPipelineDescriptor alloc] init];
|
||||
pipelineDescriptor.vertexFunction = vertexFunction;
|
||||
pipelineDescriptor.fragmentFunction = fragmentFunction;
|
||||
pipelineDescriptor.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm;
|
||||
pipelineDescriptor.depthAttachmentPixelFormat = MTLPixelFormatInvalid;
|
||||
NSError *error = nil;
|
||||
_pipelineYuvRgb = [_device newRenderPipelineStateWithDescriptor:pipelineDescriptor error:&error];
|
||||
}
|
||||
{
|
||||
id<MTLFunction> vertexFunction = [_defaultLibrary newFunctionWithName:vertexFunctionName];
|
||||
id<MTLFunction> fragmentFunction = [_defaultLibrary newFunctionWithName:fragmentPlainName];
|
||||
|
||||
MTLRenderPipelineDescriptor *pipelineDescriptor = [[MTLRenderPipelineDescriptor alloc] init];
|
||||
pipelineDescriptor.vertexFunction = vertexFunction;
|
||||
pipelineDescriptor.fragmentFunction = fragmentFunction;
|
||||
pipelineDescriptor.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm;
|
||||
pipelineDescriptor.depthAttachmentPixelFormat = MTLPixelFormatInvalid;
|
||||
NSError *error = nil;
|
||||
_pipelineThrough = [_device newRenderPipelineStateWithDescriptor:pipelineDescriptor error:&error];
|
||||
}
|
||||
|
||||
{
|
||||
id<MTLFunction> vertexFunction = [_defaultLibrary newFunctionWithName:twoInputVertexName];
|
||||
id<MTLFunction> fragmentFunction = [_defaultLibrary newFunctionWithName:transformAndBlendName];
|
||||
|
||||
MTLRenderPipelineDescriptor *pipelineDescriptor = [[MTLRenderPipelineDescriptor alloc] init];
|
||||
pipelineDescriptor.vertexFunction = vertexFunction;
|
||||
pipelineDescriptor.fragmentFunction = fragmentFunction;
|
||||
pipelineDescriptor.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm;
|
||||
pipelineDescriptor.depthAttachmentPixelFormat = MTLPixelFormatInvalid;
|
||||
NSError *error = nil;
|
||||
_pipelineTransformAndBlend = [_device newRenderPipelineStateWithDescriptor:pipelineDescriptor error:&error];
|
||||
}
|
||||
{
|
||||
id<MTLFunction> vertexFunction = [_defaultLibrary newFunctionWithName:vertexFunctionName];
|
||||
id<MTLFunction> fragmentFunction = [_defaultLibrary newFunctionWithName:scaleAndBlurName];
|
||||
|
||||
MTLRenderPipelineDescriptor *pipelineDescriptor = [[MTLRenderPipelineDescriptor alloc] init];
|
||||
pipelineDescriptor.vertexFunction = vertexFunction;
|
||||
pipelineDescriptor.fragmentFunction = fragmentFunction;
|
||||
pipelineDescriptor.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm;
|
||||
pipelineDescriptor.depthAttachmentPixelFormat = MTLPixelFormatInvalid;
|
||||
NSError *error = nil;
|
||||
_pipelineScaleAndBlur = [_device newRenderPipelineStateWithDescriptor:pipelineDescriptor error:&error];
|
||||
}
|
||||
}
|
||||
|
||||
-(id<MTLDevice>)device {
|
||||
return _device;
|
||||
}
|
||||
-(id<MTLRenderPipelineState>)pipelineYuvRgb {
|
||||
return _pipelineYuvRgb;
|
||||
}
|
||||
-(id<MTLRenderPipelineState>)pipelineTransformAndBlend {
|
||||
return _pipelineTransformAndBlend;
|
||||
}
|
||||
-(id<MTLRenderPipelineState>)pipelineScaleAndBlur {
|
||||
return _pipelineScaleAndBlur;
|
||||
}
|
||||
-(id<MTLRenderPipelineState>)pipelineThrough {
|
||||
return _pipelineThrough;
|
||||
}
|
||||
-(id<MTLSamplerState>)sampler {
|
||||
return _sampler;
|
||||
}
|
||||
-(CGDirectDisplayID)displayId {
|
||||
return _displayId;
|
||||
}
|
||||
@end
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_DECODER_FACTORY_H_
|
||||
#define SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_DECODER_FACTORY_H_
|
||||
|
||||
#import "base/RTCMacros.h"
|
||||
|
||||
#include "api/video_codecs/video_decoder_factory.h"
|
||||
#include "media/base/codec.h"
|
||||
|
||||
@protocol RTC_OBJC_TYPE
|
||||
(RTCVideoDecoderFactory);
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
class CustomObjCVideoDecoderFactory : public VideoDecoderFactory {
|
||||
public:
|
||||
explicit CustomObjCVideoDecoderFactory(id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>);
|
||||
~CustomObjCVideoDecoderFactory() override;
|
||||
|
||||
id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> wrapped_decoder_factory() const;
|
||||
|
||||
std::vector<SdpVideoFormat> GetSupportedFormats() const override;
|
||||
std::unique_ptr<VideoDecoder> CreateVideoDecoder(
|
||||
const SdpVideoFormat& format) override;
|
||||
|
||||
private:
|
||||
id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> decoder_factory_;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_DECODER_FACTORY_H_
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "objc_video_decoder_factory.h"
|
||||
|
||||
#import "base/RTCMacros.h"
|
||||
#import "base/RTCVideoDecoder.h"
|
||||
#import "base/RTCVideoDecoderFactory.h"
|
||||
#import "base/RTCVideoFrame.h"
|
||||
#import "base/RTCVideoFrameBuffer.h"
|
||||
#import "components/video_codec/RTCCodecSpecificInfoH264.h"
|
||||
#import "sdk/objc/api/peerconnection/RTCEncodedImage+Private.h"
|
||||
#import "sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.h"
|
||||
#import "sdk/objc/api/video_codec/RTCWrappedNativeVideoDecoder.h"
|
||||
#import "sdk/objc/helpers/NSString+StdString.h"
|
||||
|
||||
#include "api/video_codecs/sdp_video_format.h"
|
||||
#include "api/video_codecs/video_decoder.h"
|
||||
#include "modules/video_coding/include/video_codec_interface.h"
|
||||
#include "modules/video_coding/include/video_error_codes.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "rtc_base/time_utils.h"
|
||||
#include "sdk/objc/native/src/objc_frame_buffer.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
namespace {
|
||||
class ObjCVideoDecoder : public VideoDecoder {
|
||||
public:
|
||||
ObjCVideoDecoder(id<RTC_OBJC_TYPE(RTCVideoDecoder)> decoder)
|
||||
: decoder_(decoder), implementation_name_([decoder implementationName].stdString) {}
|
||||
|
||||
bool Configure(const Settings &settings) override {
|
||||
return
|
||||
[decoder_ startDecodeWithNumberOfCores:settings.number_of_cores()] == WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
int32_t Decode(const EncodedImage &input_image,
|
||||
bool missing_frames,
|
||||
int64_t render_time_ms = -1) override {
|
||||
RTC_OBJC_TYPE(RTCEncodedImage) *encodedImage =
|
||||
[[RTC_OBJC_TYPE(RTCEncodedImage) alloc] initWithNativeEncodedImage:input_image];
|
||||
|
||||
return (int32_t)[decoder_ decode:encodedImage
|
||||
missingFrames:missing_frames
|
||||
codecSpecificInfo:nil
|
||||
renderTimeMs:render_time_ms];
|
||||
}
|
||||
|
||||
int32_t RegisterDecodeCompleteCallback(DecodedImageCallback *callback) override {
|
||||
[decoder_ setCallback:^(RTC_OBJC_TYPE(RTCVideoFrame) * frame) {
|
||||
const auto buffer = rtc::make_ref_counted<ObjCFrameBuffer>(frame.buffer);
|
||||
|
||||
VideoFrame videoFrame =
|
||||
VideoFrame::Builder()
|
||||
.set_video_frame_buffer(buffer)
|
||||
.set_timestamp_rtp((uint32_t)(frame.timeStampNs / rtc::kNumNanosecsPerMicrosec))
|
||||
.set_timestamp_ms(0)
|
||||
.set_rotation((VideoRotation)frame.rotation)
|
||||
.build();
|
||||
videoFrame.set_timestamp(frame.timeStamp);
|
||||
|
||||
callback->Decoded(videoFrame);
|
||||
}];
|
||||
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
int32_t Release() override { return [decoder_ releaseDecoder]; }
|
||||
|
||||
const char *ImplementationName() const override { return implementation_name_.c_str(); }
|
||||
|
||||
private:
|
||||
id<RTC_OBJC_TYPE(RTCVideoDecoder)> decoder_;
|
||||
const std::string implementation_name_;
|
||||
};
|
||||
} // namespace
|
||||
|
||||
CustomObjCVideoDecoderFactory::CustomObjCVideoDecoderFactory(
|
||||
id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> decoder_factory)
|
||||
: decoder_factory_(decoder_factory) {}
|
||||
|
||||
CustomObjCVideoDecoderFactory::~CustomObjCVideoDecoderFactory() {}
|
||||
|
||||
id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> CustomObjCVideoDecoderFactory::wrapped_decoder_factory() const {
|
||||
return decoder_factory_;
|
||||
}
|
||||
|
||||
std::unique_ptr<VideoDecoder> CustomObjCVideoDecoderFactory::CreateVideoDecoder(
|
||||
const SdpVideoFormat &format) {
|
||||
NSString *codecName = [NSString stringWithUTF8String:format.name.c_str()];
|
||||
for (RTC_OBJC_TYPE(RTCVideoCodecInfo) * codecInfo in decoder_factory_.supportedCodecs) {
|
||||
if ([codecName isEqualToString:codecInfo.name]) {
|
||||
id<RTC_OBJC_TYPE(RTCVideoDecoder)> decoder = [decoder_factory_ createDecoder:codecInfo];
|
||||
|
||||
// Because of symbol conflict, isKindOfClass doesn't work as expected.
|
||||
// See https://bugs.webkit.org/show_bug.cgi?id=198782.
|
||||
if ([decoder isKindOfClass:[RTCWrappedNativeVideoDecoder class]]) {
|
||||
return [(RTCWrappedNativeVideoDecoder *)decoder releaseWrappedDecoder];
|
||||
} else {
|
||||
return std::unique_ptr<ObjCVideoDecoder>(new ObjCVideoDecoder(decoder));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
std::vector<SdpVideoFormat> CustomObjCVideoDecoderFactory::GetSupportedFormats() const {
|
||||
std::vector<SdpVideoFormat> supported_formats;
|
||||
for (RTC_OBJC_TYPE(RTCVideoCodecInfo) * supportedCodec in decoder_factory_.supportedCodecs) {
|
||||
SdpVideoFormat format = [supportedCodec nativeSdpVideoFormat];
|
||||
supported_formats.push_back(format);
|
||||
}
|
||||
|
||||
return supported_formats;
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_ENCODER_FACTORY_H_
|
||||
#define SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_ENCODER_FACTORY_H_
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
#import "base/RTCMacros.h"
|
||||
|
||||
#include "api/video_codecs/video_encoder_factory.h"
|
||||
|
||||
@protocol RTC_OBJC_TYPE
|
||||
(RTCVideoEncoderFactory);
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
class CustomObjCVideoEncoderFactory : public VideoEncoderFactory {
|
||||
public:
|
||||
explicit CustomObjCVideoEncoderFactory(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>);
|
||||
~CustomObjCVideoEncoderFactory() override;
|
||||
|
||||
id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> wrapped_encoder_factory() const;
|
||||
|
||||
std::vector<SdpVideoFormat> GetSupportedFormats() const override;
|
||||
std::vector<SdpVideoFormat> GetImplementations() const override;
|
||||
std::unique_ptr<VideoEncoder> CreateVideoEncoder(
|
||||
const SdpVideoFormat& format) override;
|
||||
std::unique_ptr<EncoderSelectorInterface> GetEncoderSelector() const override;
|
||||
|
||||
private:
|
||||
id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> encoder_factory_;
|
||||
};
|
||||
|
||||
|
||||
class SimulcastVideoEncoderFactory : public VideoEncoderFactory {
|
||||
public:
|
||||
explicit SimulcastVideoEncoderFactory(std::unique_ptr<CustomObjCVideoEncoderFactory> softwareFactory, std::unique_ptr<CustomObjCVideoEncoderFactory> hardwareFactory);
|
||||
~SimulcastVideoEncoderFactory() override;
|
||||
|
||||
std::vector<SdpVideoFormat> GetSupportedFormats() const override;
|
||||
std::vector<SdpVideoFormat> GetImplementations() const override;
|
||||
std::unique_ptr<VideoEncoder> CreateVideoEncoder(
|
||||
const SdpVideoFormat& format) override;
|
||||
std::unique_ptr<EncoderSelectorInterface> GetEncoderSelector() const override;
|
||||
|
||||
private:
|
||||
std::unique_ptr<CustomObjCVideoEncoderFactory> _softwareFactory;
|
||||
std::unique_ptr<CustomObjCVideoEncoderFactory> _hardwareFactory;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_ENCODER_FACTORY_H_
|
||||
|
|
@ -0,0 +1,240 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "objc_video_encoder_factory.h"
|
||||
|
||||
#include <string>
|
||||
|
||||
#import "base/RTCMacros.h"
|
||||
#import "base/RTCVideoEncoder.h"
|
||||
#import "base/RTCVideoEncoderFactory.h"
|
||||
#import "components/video_codec/RTCCodecSpecificInfoH264+Private.h"
|
||||
#ifndef WEBRTC_DISABLE_H265
|
||||
#import "RTCCodecSpecificInfoH265+Private.h"
|
||||
#endif
|
||||
#import "sdk/objc/api/peerconnection/RTCEncodedImage+Private.h"
|
||||
#import "sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.h"
|
||||
#import "sdk/objc/api/peerconnection/RTCVideoEncoderSettings+Private.h"
|
||||
#import "sdk/objc/api/video_codec/RTCVideoCodecConstants.h"
|
||||
#import "sdk/objc/api/video_codec/RTCWrappedNativeVideoEncoder.h"
|
||||
#import "sdk/objc/helpers/NSString+StdString.h"
|
||||
|
||||
#include "api/video/video_frame.h"
|
||||
#include "api/video_codecs/sdp_video_format.h"
|
||||
#include "api/video_codecs/video_encoder.h"
|
||||
#include "modules/video_coding/include/video_codec_interface.h"
|
||||
#include "modules/video_coding/include/video_error_codes.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "sdk/objc/native/src/objc_video_frame.h"
|
||||
#include "api/transport/field_trial_based_config.h"
|
||||
|
||||
#include "CustomSimulcastEncoderAdapter.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
namespace {
|
||||
|
||||
class ObjCVideoEncoder : public VideoEncoder {
|
||||
public:
|
||||
ObjCVideoEncoder(id<RTC_OBJC_TYPE(RTCVideoEncoder)> encoder)
|
||||
: encoder_(encoder), implementation_name_([encoder implementationName].stdString) {}
|
||||
|
||||
int32_t InitEncode(const VideoCodec *codec_settings, const Settings &encoder_settings) override {
|
||||
RTC_OBJC_TYPE(RTCVideoEncoderSettings) *settings =
|
||||
[[RTC_OBJC_TYPE(RTCVideoEncoderSettings) alloc] initWithNativeVideoCodec:codec_settings];
|
||||
return [encoder_ startEncodeWithSettings:settings
|
||||
numberOfCores:encoder_settings.number_of_cores];
|
||||
}
|
||||
|
||||
int32_t RegisterEncodeCompleteCallback(EncodedImageCallback *callback) override {
|
||||
[encoder_ setCallback:^BOOL(RTC_OBJC_TYPE(RTCEncodedImage) * _Nonnull frame,
|
||||
id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)> _Nonnull info) {
|
||||
EncodedImage encodedImage = [frame nativeEncodedImage];
|
||||
|
||||
// Handle types that can be converted into one of CodecSpecificInfo's hard coded cases.
|
||||
CodecSpecificInfo codecSpecificInfo;
|
||||
// Because of symbol conflict, isKindOfClass doesn't work as expected.
|
||||
// See https://bugs.webkit.org/show_bug.cgi?id=198782.
|
||||
if ([NSStringFromClass([info class]) isEqual:@"RTCCodecSpecificInfoH264"]) {
|
||||
// if ([info isKindOfClass:[RTCCodecSpecificInfoH264 class]]) {
|
||||
codecSpecificInfo = [(RTCCodecSpecificInfoH264 *)info nativeCodecSpecificInfo];
|
||||
#ifndef WEBRTC_DISABLE_H265
|
||||
} else if ([NSStringFromClass([info class]) isEqual:@"RTCCodecSpecificInfoH265"]) {
|
||||
// if ([info isKindOfClass:[RTCCodecSpecificInfoH265 class]]) {
|
||||
codecSpecificInfo = [(RTCCodecSpecificInfoH265 *)info nativeCodecSpecificInfo];
|
||||
#endif
|
||||
}
|
||||
|
||||
EncodedImageCallback::Result res = callback->OnEncodedImage(encodedImage, &codecSpecificInfo);
|
||||
return res.error == EncodedImageCallback::Result::OK;
|
||||
}];
|
||||
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
int32_t Release() override { return [encoder_ releaseEncoder]; }
|
||||
|
||||
int32_t Encode(const VideoFrame &frame,
|
||||
const std::vector<VideoFrameType> *frame_types) override {
|
||||
int32_t result = 0;
|
||||
@autoreleasepool {
|
||||
NSMutableArray<NSNumber *> *rtcFrameTypes = [NSMutableArray array];
|
||||
for (size_t i = 0; i < frame_types->size(); ++i) {
|
||||
[rtcFrameTypes addObject:@(RTCFrameType(frame_types->at(i)))];
|
||||
}
|
||||
|
||||
result = [encoder_ encode:ToObjCVideoFrame(frame)
|
||||
codecSpecificInfo:nil
|
||||
frameTypes:rtcFrameTypes];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
void SetRates(const RateControlParameters ¶meters) override {
|
||||
const uint32_t bitrate = parameters.bitrate.get_sum_kbps();
|
||||
const uint32_t framerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
|
||||
[encoder_ setBitrate:bitrate framerate:framerate];
|
||||
}
|
||||
|
||||
VideoEncoder::EncoderInfo GetEncoderInfo() const override {
|
||||
EncoderInfo info;
|
||||
info.supports_native_handle = true;
|
||||
info.implementation_name = implementation_name_;
|
||||
|
||||
RTC_OBJC_TYPE(RTCVideoEncoderQpThresholds) *qp_thresholds = [encoder_ scalingSettings];
|
||||
info.scaling_settings = qp_thresholds ? ScalingSettings(qp_thresholds.low, qp_thresholds.high) :
|
||||
ScalingSettings::kOff;
|
||||
|
||||
info.is_hardware_accelerated = true;
|
||||
return info;
|
||||
}
|
||||
|
||||
private:
|
||||
id<RTC_OBJC_TYPE(RTCVideoEncoder)> encoder_;
|
||||
const std::string implementation_name_;
|
||||
};
|
||||
|
||||
class ObjcVideoEncoderSelector : public VideoEncoderFactory::EncoderSelectorInterface {
|
||||
public:
|
||||
ObjcVideoEncoderSelector(id<RTC_OBJC_TYPE(RTCVideoEncoderSelector)> selector) {
|
||||
selector_ = selector;
|
||||
}
|
||||
void OnCurrentEncoder(const SdpVideoFormat &format) override {
|
||||
RTC_OBJC_TYPE(RTCVideoCodecInfo) *info =
|
||||
[[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithNativeSdpVideoFormat:format];
|
||||
[selector_ registerCurrentEncoderInfo:info];
|
||||
}
|
||||
absl::optional<SdpVideoFormat> OnEncoderBroken() override {
|
||||
RTC_OBJC_TYPE(RTCVideoCodecInfo) *info = [selector_ encoderForBrokenEncoder];
|
||||
if (info) {
|
||||
return [info nativeSdpVideoFormat];
|
||||
}
|
||||
return absl::nullopt;
|
||||
}
|
||||
absl::optional<SdpVideoFormat> OnAvailableBitrate(const DataRate &rate) override {
|
||||
RTC_OBJC_TYPE(RTCVideoCodecInfo) *info = [selector_ encoderForBitrate:rate.kbps<NSInteger>()];
|
||||
if (info) {
|
||||
return [info nativeSdpVideoFormat];
|
||||
}
|
||||
return absl::nullopt;
|
||||
}
|
||||
|
||||
private:
|
||||
id<RTC_OBJC_TYPE(RTCVideoEncoderSelector)> selector_;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
CustomObjCVideoEncoderFactory::CustomObjCVideoEncoderFactory(
|
||||
id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> encoder_factory)
|
||||
: encoder_factory_(encoder_factory) {}
|
||||
|
||||
CustomObjCVideoEncoderFactory::~CustomObjCVideoEncoderFactory() {}
|
||||
|
||||
id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> CustomObjCVideoEncoderFactory::wrapped_encoder_factory() const {
|
||||
return encoder_factory_;
|
||||
}
|
||||
|
||||
std::vector<SdpVideoFormat> CustomObjCVideoEncoderFactory::GetSupportedFormats() const {
|
||||
std::vector<SdpVideoFormat> supported_formats;
|
||||
for (RTC_OBJC_TYPE(RTCVideoCodecInfo) * supportedCodec in [encoder_factory_ supportedCodecs]) {
|
||||
SdpVideoFormat format = [supportedCodec nativeSdpVideoFormat];
|
||||
supported_formats.push_back(format);
|
||||
}
|
||||
|
||||
return supported_formats;
|
||||
}
|
||||
|
||||
std::vector<SdpVideoFormat> CustomObjCVideoEncoderFactory::GetImplementations() const {
|
||||
if ([encoder_factory_ respondsToSelector:@selector(implementations)]) {
|
||||
std::vector<SdpVideoFormat> supported_formats;
|
||||
for (RTC_OBJC_TYPE(RTCVideoCodecInfo) * supportedCodec in [encoder_factory_ implementations]) {
|
||||
SdpVideoFormat format = [supportedCodec nativeSdpVideoFormat];
|
||||
supported_formats.push_back(format);
|
||||
}
|
||||
return supported_formats;
|
||||
}
|
||||
return GetSupportedFormats();
|
||||
}
|
||||
|
||||
std::unique_ptr<VideoEncoder> CustomObjCVideoEncoderFactory::CreateVideoEncoder(
|
||||
const SdpVideoFormat &format) {
|
||||
RTCVideoCodecInfo *info = [[RTCVideoCodecInfo alloc] initWithNativeSdpVideoFormat:format];
|
||||
id<RTCVideoEncoder> encoder = [encoder_factory_ createEncoder:info];
|
||||
if ([encoder isKindOfClass:[RTCWrappedNativeVideoEncoder class]]) {
|
||||
return [(RTCWrappedNativeVideoEncoder *)encoder releaseWrappedEncoder];
|
||||
} else {
|
||||
return std::unique_ptr<ObjCVideoEncoder>(new ObjCVideoEncoder(encoder));
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<VideoEncoderFactory::EncoderSelectorInterface>
|
||||
CustomObjCVideoEncoderFactory::GetEncoderSelector() const {
|
||||
if ([encoder_factory_ respondsToSelector:@selector(encoderSelector)]) {
|
||||
id<RTC_OBJC_TYPE(RTCVideoEncoderSelector)> selector = [encoder_factory_ encoderSelector];
|
||||
if (selector) {
|
||||
return absl::make_unique<ObjcVideoEncoderSelector>(selector);
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
SimulcastVideoEncoderFactory::SimulcastVideoEncoderFactory(std::unique_ptr<CustomObjCVideoEncoderFactory> softwareFactory, std::unique_ptr<CustomObjCVideoEncoderFactory> hardwareFactory) :
|
||||
_softwareFactory(std::move(softwareFactory)),
|
||||
_hardwareFactory(std::move(hardwareFactory)) {
|
||||
}
|
||||
SimulcastVideoEncoderFactory::~SimulcastVideoEncoderFactory() {
|
||||
}
|
||||
|
||||
std::vector<SdpVideoFormat> SimulcastVideoEncoderFactory::GetSupportedFormats() const {
|
||||
return _hardwareFactory->GetSupportedFormats();
|
||||
}
|
||||
|
||||
std::vector<SdpVideoFormat> SimulcastVideoEncoderFactory::GetImplementations() const {
|
||||
return _hardwareFactory->GetImplementations();
|
||||
}
|
||||
|
||||
std::unique_ptr<VideoEncoder> SimulcastVideoEncoderFactory::CreateVideoEncoder(const SdpVideoFormat& format) {
|
||||
#ifndef __aarch64__
|
||||
#ifdef WEBRTC_MAC
|
||||
return std::make_unique<webrtc::CustomSimulcastEncoderAdapter>(_softwareFactory.get(), _softwareFactory.get(), format, webrtc::FieldTrialBasedConfig());
|
||||
#else
|
||||
return std::make_unique<webrtc::CustomSimulcastEncoderAdapter>(_hardwareFactory.get(), _hardwareFactory.get(), format, webrtc::FieldTrialBasedConfig());
|
||||
#endif //WEBRTC_MAC
|
||||
#else
|
||||
return std::make_unique<webrtc::CustomSimulcastEncoderAdapter>(_hardwareFactory.get(), _hardwareFactory.get(), format, webrtc::FieldTrialBasedConfig());
|
||||
#endif //__aarch64__
|
||||
}
|
||||
|
||||
std::unique_ptr<VideoEncoderFactory::EncoderSelectorInterface> SimulcastVideoEncoderFactory::GetEncoderSelector() const {
|
||||
return _hardwareFactory->GetEncoderSelector();
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
Loading…
Add table
Add a link
Reference in a new issue