Repo created

This commit is contained in:
Fr4nz D13trich 2025-11-22 14:04:28 +01:00
parent 81b91f4139
commit f8c34fa5ee
22732 changed files with 4815320 additions and 2 deletions

View file

@ -0,0 +1,363 @@
#ifndef TGCALLS_PLATFORM_INTERFACE_H
#define TGCALLS_PLATFORM_INTERFACE_H
#include "rtc_base/thread.h"
#include "api/video_codecs/video_encoder_factory.h"
#include "api/video_codecs/video_decoder_factory.h"
#include "api/media_stream_interface.h"
#include "rtc_base/network_monitor_factory.h"
#include "modules/audio_device/include/audio_device.h"
#include "rtc_base/ref_counted_object.h"
#include <string>
#include <map>
struct AVFrame;
struct AVCodecContext;
namespace tgcalls {
enum class VideoState;
class VideoCapturerInterface;
class PlatformContext;
struct PlatformCaptureInfo {
bool shouldBeAdaptedToReceiverAspectRate = false;
int rotation = 0;
};
class WrappedAudioDeviceModule : public webrtc::AudioDeviceModule {
public:
virtual void Stop() = 0;
virtual void setIsActive(bool isActive) = 0;
};
class DefaultWrappedAudioDeviceModule : public WrappedAudioDeviceModule {
public:
DefaultWrappedAudioDeviceModule(webrtc::scoped_refptr<webrtc::AudioDeviceModule> impl) :
_impl(impl) {
}
virtual ~DefaultWrappedAudioDeviceModule() {
}
virtual void Stop() override {
}
virtual void setIsActive(bool isActive) override {
}
virtual int32_t ActiveAudioLayer(AudioLayer *audioLayer) const override {
return _impl->ActiveAudioLayer(audioLayer);
}
virtual int32_t RegisterAudioCallback(webrtc::AudioTransport *audioCallback) override {
return _impl->RegisterAudioCallback(audioCallback);
}
virtual int32_t Init() override {
return _impl->Init();
}
virtual int32_t Terminate() override {
return _impl->Terminate();
}
virtual bool Initialized() const override {
return _impl->Initialized();
}
virtual int16_t PlayoutDevices() override {
return _impl->PlayoutDevices();
}
virtual int16_t RecordingDevices() override {
return _impl->RecordingDevices();
}
virtual int32_t PlayoutDeviceName(uint16_t index, char name[webrtc::kAdmMaxDeviceNameSize], char guid[webrtc::kAdmMaxGuidSize]) override {
return _impl->PlayoutDeviceName(index, name, guid);
}
virtual int32_t RecordingDeviceName(uint16_t index, char name[webrtc::kAdmMaxDeviceNameSize], char guid[webrtc::kAdmMaxGuidSize]) override {
return _impl->RecordingDeviceName(index, name, guid);
}
virtual int32_t SetPlayoutDevice(uint16_t index) override {
return _impl->SetPlayoutDevice(index);
}
#ifdef TGCALLS_UWP_DESKTOP
virtual int32_t SetPlayoutDevice(std::string deviceId) override {
return _impl->SetPlayoutDevice(deviceId);
}
#endif
virtual int32_t SetPlayoutDevice(WindowsDeviceType device) override {
return _impl->SetPlayoutDevice(device);
}
virtual int32_t SetRecordingDevice(uint16_t index) override {
return _impl->SetRecordingDevice(index);
}
#ifdef TGCALLS_UWP_DESKTOP
virtual int32_t SetRecordingDevice(std::string deviceId) override {
return _impl->SetRecordingDevice(deviceId);
}
#endif
virtual int32_t SetRecordingDevice(WindowsDeviceType device) override {
return _impl->SetRecordingDevice(device);
}
virtual int32_t PlayoutIsAvailable(bool *available) override {
return _impl->PlayoutIsAvailable(available);
}
virtual int32_t InitPlayout() override {
return _impl->InitPlayout();
}
virtual bool PlayoutIsInitialized() const override {
return _impl->PlayoutIsInitialized();
}
virtual int32_t RecordingIsAvailable(bool *available) override {
return _impl->RecordingIsAvailable(available);
}
virtual int32_t InitRecording() override {
return _impl->InitRecording();
}
virtual bool RecordingIsInitialized() const override {
return _impl->RecordingIsInitialized();
}
virtual int32_t StartPlayout() override {
return _impl->StartPlayout();
}
virtual int32_t StopPlayout() override {
return _impl->StopPlayout();
}
virtual bool Playing() const override {
return _impl->Playing();
}
virtual int32_t StartRecording() override {
return _impl->StartRecording();
}
virtual int32_t StopRecording() override {
return _impl->StopRecording();
}
virtual bool Recording() const override {
return _impl->Recording();
}
virtual int32_t InitSpeaker() override {
return _impl->InitSpeaker();
}
virtual bool SpeakerIsInitialized() const override {
return _impl->SpeakerIsInitialized();
}
virtual int32_t InitMicrophone() override {
return _impl->InitMicrophone();
}
virtual bool MicrophoneIsInitialized() const override {
return _impl->MicrophoneIsInitialized();
}
virtual int32_t SpeakerVolumeIsAvailable(bool *available) override {
return _impl->SpeakerVolumeIsAvailable(available);
}
virtual int32_t SetSpeakerVolume(uint32_t volume) override {
return _impl->SetSpeakerVolume(volume);
}
virtual int32_t SpeakerVolume(uint32_t* volume) const override {
return _impl->SpeakerVolume(volume);
}
virtual int32_t MaxSpeakerVolume(uint32_t *maxVolume) const override {
return _impl->MaxSpeakerVolume(maxVolume);
}
virtual int32_t MinSpeakerVolume(uint32_t *minVolume) const override {
return _impl->MinSpeakerVolume(minVolume);
}
virtual int32_t MicrophoneVolumeIsAvailable(bool *available) override {
return _impl->MicrophoneVolumeIsAvailable(available);
}
virtual int32_t SetMicrophoneVolume(uint32_t volume) override {
return _impl->SetMicrophoneVolume(volume);
}
virtual int32_t MicrophoneVolume(uint32_t *volume) const override {
return _impl->MicrophoneVolume(volume);
}
virtual int32_t MaxMicrophoneVolume(uint32_t *maxVolume) const override {
return _impl->MaxMicrophoneVolume(maxVolume);
}
virtual int32_t MinMicrophoneVolume(uint32_t *minVolume) const override {
return _impl->MinMicrophoneVolume(minVolume);
}
virtual int32_t SpeakerMuteIsAvailable(bool *available) override {
return _impl->SpeakerMuteIsAvailable(available);
}
virtual int32_t SetSpeakerMute(bool enable) override {
return _impl->SetSpeakerMute(enable);
}
virtual int32_t SpeakerMute(bool *enabled) const override {
return _impl->SpeakerMute(enabled);
}
virtual int32_t MicrophoneMuteIsAvailable(bool *available) override {
return _impl->MicrophoneMuteIsAvailable(available);
}
virtual int32_t SetMicrophoneMute(bool enable) override {
return _impl->SetMicrophoneMute(enable);
}
virtual int32_t MicrophoneMute(bool *enabled) const override {
return _impl->MicrophoneMute(enabled);
}
virtual int32_t StereoPlayoutIsAvailable(bool *available) const override {
return _impl->StereoPlayoutIsAvailable(available);
}
virtual int32_t SetStereoPlayout(bool enable) override {
return _impl->SetStereoPlayout(enable);
}
virtual int32_t StereoPlayout(bool *enabled) const override {
return _impl->StereoPlayout(enabled);
}
virtual int32_t StereoRecordingIsAvailable(bool *available) const override {
return _impl->StereoRecordingIsAvailable(available);
}
virtual int32_t SetStereoRecording(bool enable) override {
return _impl->SetStereoRecording(enable);
}
virtual int32_t StereoRecording(bool *enabled) const override {
return _impl->StereoRecording(enabled);
}
virtual int32_t PlayoutDelay(uint16_t* delayMS) const override {
return _impl->PlayoutDelay(delayMS);
}
virtual bool BuiltInAECIsAvailable() const override {
return _impl->BuiltInAECIsAvailable();
}
virtual bool BuiltInAGCIsAvailable() const override {
return _impl->BuiltInAGCIsAvailable();
}
virtual bool BuiltInNSIsAvailable() const override {
return _impl->BuiltInNSIsAvailable();
}
virtual int32_t EnableBuiltInAEC(bool enable) override {
return _impl->EnableBuiltInAEC(enable);
}
virtual int32_t EnableBuiltInAGC(bool enable) override {
return _impl->EnableBuiltInAGC(enable);
}
virtual int32_t EnableBuiltInNS(bool enable) override {
return _impl->EnableBuiltInNS(enable);
}
virtual int32_t GetPlayoutUnderrunCount() const override {
return _impl->GetPlayoutUnderrunCount();
}
#if defined(WEBRTC_IOS)
virtual int GetPlayoutAudioParameters(webrtc::AudioParameters *params) const override {
return _impl->GetPlayoutAudioParameters(params);
}
virtual int GetRecordAudioParameters(webrtc::AudioParameters *params) const override {
return _impl->GetRecordAudioParameters(params);
}
#endif // WEBRTC_IOS
webrtc::scoped_refptr<webrtc::AudioDeviceModule> WrappedInstance() const {
return _impl;
}
private:
webrtc::scoped_refptr<webrtc::AudioDeviceModule> _impl;
};
class PlatformVideoFrame {
public:
PlatformVideoFrame() {
}
virtual ~PlatformVideoFrame() = default;
};
class PlatformInterface {
public:
static PlatformInterface *SharedInstance();
virtual ~PlatformInterface() = default;
virtual void configurePlatformAudio(int numChannels = 1) {
}
virtual std::unique_ptr<rtc::NetworkMonitorFactory> createNetworkMonitorFactory() {
return nullptr;
}
virtual std::unique_ptr<webrtc::VideoEncoderFactory> makeVideoEncoderFactory(std::shared_ptr<PlatformContext> platformContext, bool preferHardwareEncoding = false, bool isScreencast = false) = 0;
virtual std::unique_ptr<webrtc::VideoDecoderFactory> makeVideoDecoderFactory(std::shared_ptr<PlatformContext> platformContext) = 0;
virtual bool supportsEncoding(const std::string &codecName, std::shared_ptr<PlatformContext> platformContext) = 0;
virtual webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> makeVideoSource(rtc::Thread *signalingThread, rtc::Thread *workerThread, bool screencapture) = 0;
virtual void adaptVideoSource(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> videoSource, int width, int height, int fps) = 0;
virtual std::unique_ptr<VideoCapturerInterface> makeVideoCapturer(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> source, std::string deviceId, std::function<void(VideoState)> stateUpdated, std::function<void(PlatformCaptureInfo)> captureInfoUpdated, std::shared_ptr<PlatformContext> platformContext, std::pair<int, int> &outResolution) = 0;
virtual webrtc::scoped_refptr<WrappedAudioDeviceModule> wrapAudioDeviceModule(webrtc::scoped_refptr<webrtc::AudioDeviceModule> module) {
return rtc::make_ref_counted<DefaultWrappedAudioDeviceModule>(module);
}
virtual void setupVideoDecoding(AVCodecContext *codecContext) {
}
virtual webrtc::scoped_refptr<webrtc::VideoFrameBuffer> createPlatformFrameFromData(AVFrame const *frame) {
return nullptr;
}
public:
bool preferX264 = false;
};
std::unique_ptr<PlatformInterface> CreatePlatformInterface();
inline PlatformInterface *PlatformInterface::SharedInstance() {
static const auto result = CreatePlatformInterface();
return result.get();
}
} // namespace tgcalls
#endif

View file

@ -0,0 +1,62 @@
#include "AndroidContext.h"
#include "sdk/android/native_api/jni/jvm.h"
#include "tgnet/FileLog.h"
namespace tgcalls {
AndroidContext::AndroidContext(JNIEnv *env, jobject peerInstance, jobject groupInstance, bool screencast) {
VideoCapturerDeviceClass = (jclass) env->NewGlobalRef(env->FindClass("org/telegram/messenger/voip/VideoCapturerDevice"));
jmethodID initMethodId = env->GetMethodID(VideoCapturerDeviceClass, "<init>", "(Z)V");
javaCapturer = env->NewGlobalRef(env->NewObject(VideoCapturerDeviceClass, initMethodId, screencast));
if (peerInstance) {
javaPeerInstance = env->NewGlobalRef(peerInstance);
}
if (groupInstance) {
javaGroupInstance = env->NewGlobalRef(groupInstance);
}
}
AndroidContext::~AndroidContext() {
JNIEnv *env = webrtc::AttachCurrentThreadIfNeeded();
jmethodID onDestroyMethodId = env->GetMethodID(VideoCapturerDeviceClass, "onDestroy", "()V");
env->CallVoidMethod(javaCapturer, onDestroyMethodId);
env->DeleteGlobalRef(javaCapturer);
javaCapturer = nullptr;
env->DeleteGlobalRef(VideoCapturerDeviceClass);
if (javaPeerInstance) {
env->DeleteGlobalRef(javaPeerInstance);
}
if (javaGroupInstance) {
env->DeleteGlobalRef(javaGroupInstance);
}
}
void AndroidContext::setJavaPeerInstance(JNIEnv *env, jobject instance) {
javaPeerInstance = env->NewGlobalRef(instance);
}
void AndroidContext::setJavaGroupInstance(JNIEnv *env, jobject instance) {
javaGroupInstance = env->NewGlobalRef(instance);
}
jobject AndroidContext::getJavaPeerInstance() {
return javaPeerInstance;
}
jobject AndroidContext::getJavaGroupInstance() {
return javaGroupInstance;
}
jobject AndroidContext::getJavaCapturer() {
return javaCapturer;
}
jclass AndroidContext::getJavaCapturerClass() {
return VideoCapturerDeviceClass;
}
} // namespace tgcalls

View file

@ -0,0 +1,39 @@
#ifndef TGCALLS_ANDROID_CONTEXT_H
#define TGCALLS_ANDROID_CONTEXT_H
#include "PlatformContext.h"
#include <jni.h>
#include <voip/tgcalls/group/GroupInstanceImpl.h>
namespace tgcalls {
class AndroidContext final : public PlatformContext {
public:
AndroidContext(JNIEnv *env, jobject peerInstance, jobject groupInstance, bool screencast);
~AndroidContext() override;
jobject getJavaCapturer();
jobject getJavaPeerInstance();
jobject getJavaGroupInstance();
jclass getJavaCapturerClass();
void setJavaPeerInstance(JNIEnv *env, jobject instance);
void setJavaGroupInstance(JNIEnv *env, jobject instance);
std::vector<std::shared_ptr<BroadcastPartTask>> audioStreamTasks;
std::vector<std::shared_ptr<BroadcastPartTask>> videoStreamTasks;
std::vector<std::shared_ptr<RequestMediaChannelDescriptionTask>> descriptionTasks;
private:
jclass VideoCapturerDeviceClass = nullptr;
jobject javaCapturer = nullptr;
jobject javaPeerInstance = nullptr;
jobject javaGroupInstance = nullptr;
};
} // namespace tgcalls
#endif

View file

@ -0,0 +1,160 @@
#include "AndroidInterface.h"
#include <rtc_base/ssl_adapter.h>
#include <modules/utility/include/jvm_android.h>
#include <sdk/android/src/jni/android_video_track_source.h>
#include <media/base/media_constants.h>
#include "VideoCapturerInterfaceImpl.h"
#include "sdk/android/native_api/base/init.h"
#include "sdk/android/native_api/codecs/wrapper.h"
#include "sdk/android/native_api/jni/class_loader.h"
#include "sdk/android/native_api/jni/jvm.h"
#include "sdk/android/native_api/jni/scoped_java_ref.h"
#include "sdk/android/native_api/video/video_source.h"
#include "api/video_codecs/builtin_video_encoder_factory.h"
#include "api/video_codecs/builtin_video_decoder_factory.h"
#include "pc/video_track_source_proxy.h"
#include "sdk/android/src/jni/android_network_monitor.h"
#include "api/video_track_source_proxy_factory.h"
#include "AndroidContext.h"
#include "media/engine/simulcast_encoder_adapter.h"
namespace tgcalls {
void AndroidInterface::configurePlatformAudio(int numChannels) {
}
class SimulcastVideoEncoderFactory : public webrtc::VideoEncoderFactory {
public:
std::unique_ptr<webrtc::VideoEncoderFactory> main_factory;
std::unique_ptr<webrtc::SimulcastEncoderAdapter> simulcast_adapter;
SimulcastVideoEncoderFactory(
std::unique_ptr<webrtc::VideoEncoderFactory> main_factory
): main_factory(std::move(main_factory)) {}
std::vector<webrtc::SdpVideoFormat> GetSupportedFormats() const override {
return main_factory->GetSupportedFormats();
}
std::vector<webrtc::SdpVideoFormat> GetImplementations() const override {
return main_factory->GetImplementations();
}
std::unique_ptr<EncoderSelectorInterface> GetEncoderSelector() const override {
return main_factory->GetEncoderSelector();
}
std::unique_ptr<webrtc::VideoEncoder> CreateVideoEncoder(const webrtc::SdpVideoFormat& format) override {
return std::make_unique<webrtc::SimulcastEncoderAdapter>(main_factory.get(), format);
}
CodecSupport QueryCodecSupport(
const webrtc::SdpVideoFormat& format,
absl::optional<std::string> scalability_mode) const override {
return main_factory->QueryCodecSupport(format, scalability_mode);
}
};
std::unique_ptr<webrtc::VideoEncoderFactory> AndroidInterface::makeVideoEncoderFactory(std::shared_ptr<PlatformContext> platformContext, bool preferHardwareEncoding, bool isScreencast) {
JNIEnv *env = webrtc::AttachCurrentThreadIfNeeded();
// AndroidContext *context = (AndroidContext *) platformContext.get();
// jmethodID methodId = env->GetMethodID(context->getJavaCapturerClass(), "getSharedEGLContext", "()Lorg/webrtc/EglBase$Context;");
// jobject eglContext = env->CallObjectMethod(context->getJavaCapturer(), methodId);
webrtc::ScopedJavaLocalRef<jclass> factory_class =
webrtc::GetClass(env, "org/webrtc/DefaultVideoEncoderFactory");
jmethodID factory_constructor = env->GetMethodID(
factory_class.obj(), "<init>", "(Lorg/webrtc/EglBase$Context;ZZ)V");
webrtc::ScopedJavaLocalRef<jobject> factory_object(
env, env->NewObject(factory_class.obj(), factory_constructor,
nullptr /* shared_context */,
false /* enable_intel_vp8_encoder */,
true /* enable_h264_high_profile */));
// return webrtc::JavaToNativeVideoEncoderFactory(env, factory_object.obj());
return std::make_unique<SimulcastVideoEncoderFactory>(webrtc::JavaToNativeVideoEncoderFactory(env, factory_object.obj()));
}
std::unique_ptr<webrtc::VideoDecoderFactory> AndroidInterface::makeVideoDecoderFactory(std::shared_ptr<PlatformContext> platformContext) {
JNIEnv *env = webrtc::AttachCurrentThreadIfNeeded();
// AndroidContext *context = (AndroidContext *) platformContext.get();
// jmethodID methodId = env->GetMethodID(context->getJavaCapturerClass(), "getSharedEGLContext", "()Lorg/webrtc/EglBase$Context;");
// jobject eglContext = env->CallObjectMethod(context->getJavaCapturer(), methodId);
webrtc::ScopedJavaLocalRef<jclass> factory_class =
webrtc::GetClass(env, "org/webrtc/DefaultVideoDecoderFactory");
jmethodID factory_constructor = env->GetMethodID(
factory_class.obj(), "<init>", "(Lorg/webrtc/EglBase$Context;)V");
webrtc::ScopedJavaLocalRef<jobject> factory_object(
env, env->NewObject(factory_class.obj(), factory_constructor,
nullptr /* shared_context */));
return webrtc::JavaToNativeVideoDecoderFactory(env, factory_object.obj());
}
void AndroidInterface::adaptVideoSource(rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> videoSource, int width, int height, int fps) {
}
rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> AndroidInterface::makeVideoSource(rtc::Thread *signalingThread, rtc::Thread *workerThread, bool screencapture) {
JNIEnv *env = webrtc::AttachCurrentThreadIfNeeded();
_source[screencapture ? 1 : 0] = webrtc::CreateJavaVideoSource(env, signalingThread, false, false);
return webrtc::CreateVideoTrackSourceProxy(signalingThread, workerThread, _source[screencapture ? 1 : 0].get());
}
bool AndroidInterface::supportsEncoding(const std::string &codecName, std::shared_ptr<PlatformContext> platformContext) {
if (hardwareVideoEncoderFactory == nullptr) {
JNIEnv *env = webrtc::AttachCurrentThreadIfNeeded();
// AndroidContext *context = (AndroidContext *) platformContext.get();
// jmethodID methodId = env->GetMethodID(context->getJavaCapturerClass(), "getSharedEGLContext", "()Lorg/webrtc/EglBase$Context;");
// jobject eglContext = env->CallObjectMethod(context->getJavaCapturer(), methodId);
webrtc::ScopedJavaLocalRef<jclass> factory_class =
webrtc::GetClass(env, "org/webrtc/HardwareVideoEncoderFactory");
jmethodID factory_constructor = env->GetMethodID(
factory_class.obj(), "<init>", "(Lorg/webrtc/EglBase$Context;ZZ)V");
webrtc::ScopedJavaLocalRef<jobject> factory_object(
env, env->NewObject(factory_class.obj(), factory_constructor,
nullptr,
false,
true));
hardwareVideoEncoderFactory = webrtc::JavaToNativeVideoEncoderFactory(env, factory_object.obj());
}
auto formats = hardwareVideoEncoderFactory->GetSupportedFormats();
for (auto format : formats) {
if (format.name == codecName) {
return true;
}
}
return codecName == cricket::kVp8CodecName;
}
std::unique_ptr<VideoCapturerInterface> AndroidInterface::makeVideoCapturer(
rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> source,
std::string deviceId,
std::function<void(VideoState)> stateUpdated,
std::function<void(PlatformCaptureInfo)> captureInfoUpdated,
std::shared_ptr<PlatformContext> platformContext,
std::pair<int, int> &outResolution
) {
return std::make_unique<VideoCapturerInterfaceImpl>(_source[deviceId == "screen" ? 1 : 0], deviceId, stateUpdated, platformContext);
}
std::unique_ptr<rtc::NetworkMonitorFactory> AndroidInterface::createNetworkMonitorFactory() {
return std::make_unique<webrtc::jni::AndroidNetworkMonitorFactory>();
}
std::unique_ptr<PlatformInterface> CreatePlatformInterface() {
return std::make_unique<AndroidInterface>();
}
} // namespace tgcalls

View file

@ -0,0 +1,30 @@
#ifndef TGCALLS_ANDROID_INTERFACE_H
#define TGCALLS_ANDROID_INTERFACE_H
#include "sdk/android/native_api/video/video_source.h"
#include "platform/PlatformInterface.h"
#include "VideoCapturerInterface.h"
namespace tgcalls {
class AndroidInterface : public PlatformInterface {
public:
void configurePlatformAudio(int numChannels = 1) override;
std::unique_ptr<webrtc::VideoEncoderFactory> makeVideoEncoderFactory(std::shared_ptr<PlatformContext> platformContext, bool preferHardwareEncoding = false, bool isScreencast = false) override;
std::unique_ptr<webrtc::VideoDecoderFactory> makeVideoDecoderFactory(std::shared_ptr<PlatformContext> platformContext) override;
bool supportsEncoding(const std::string &codecName, std::shared_ptr<PlatformContext> platformContext) override;
rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> makeVideoSource(rtc::Thread *signalingThread, rtc::Thread *workerThread, bool screencapture) override;
void adaptVideoSource(rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> videoSource, int width, int height, int fps) override;
std::unique_ptr<VideoCapturerInterface> makeVideoCapturer(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> source, std::string deviceId, std::function<void(VideoState)> stateUpdated, std::function<void(PlatformCaptureInfo)> captureInfoUpdated, std::shared_ptr<PlatformContext> platformContext, std::pair<int, int> &outResolution) override;
std::unique_ptr<rtc::NetworkMonitorFactory> createNetworkMonitorFactory() override;
private:
rtc::scoped_refptr<webrtc::JavaVideoTrackSourceInterface> _source[2];
std::unique_ptr<webrtc::VideoEncoderFactory> hardwareVideoEncoderFactory;
std::unique_ptr<webrtc::VideoEncoderFactory> softwareVideoEncoderFactory;
};
} // namespace tgcalls
#endif

View file

@ -0,0 +1,62 @@
#include "VideoCameraCapturer.h"
#include <stdint.h>
#include <memory>
#include <algorithm>
#include "AndroidInterface.h"
#include "AndroidContext.h"
#include "sdk/android/native_api/jni/jvm.h"
namespace tgcalls {
VideoCameraCapturer::VideoCameraCapturer(rtc::scoped_refptr<webrtc::JavaVideoTrackSourceInterface> source, std::string deviceId, std::function<void(VideoState)> stateUpdated, std::shared_ptr<PlatformContext> platformContext) : _source(source), _stateUpdated(stateUpdated), _platformContext(platformContext) {
AndroidContext *context = (AndroidContext *) platformContext.get();
JNIEnv *env = webrtc::AttachCurrentThreadIfNeeded();
jmethodID methodId = env->GetMethodID(context->getJavaCapturerClass(), "init", "(JLjava/lang/String;)V");
env->CallVoidMethod(context->getJavaCapturer(), methodId, (jlong) (intptr_t) this, env->NewStringUTF(deviceId.c_str()));
}
void VideoCameraCapturer::setState(VideoState state) {
_state = state;
if (_stateUpdated) {
_stateUpdated(_state);
}
JNIEnv *env = webrtc::AttachCurrentThreadIfNeeded();
auto context = (AndroidContext *) _platformContext.get();
jmethodID methodId = env->GetMethodID(context->getJavaCapturerClass(), "onStateChanged", "(JI)V");
env->CallVoidMethod(context->getJavaCapturer(), methodId, (jlong) (intptr_t) this, (jint) state);
}
void VideoCameraCapturer::setPreferredCaptureAspectRatio(float aspectRatio) {
_aspectRatio = aspectRatio;
JNIEnv *env = webrtc::AttachCurrentThreadIfNeeded();
auto context = (AndroidContext *) _platformContext.get();
jmethodID methodId = env->GetMethodID(context->getJavaCapturerClass(), "onAspectRatioRequested", "(F)V");
env->CallVoidMethod(context->getJavaCapturer(), methodId, (jfloat) aspectRatio);
}
void VideoCameraCapturer::setUncroppedSink(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink) {
if (_uncroppedSink != nullptr) {
_source->RemoveSink(_uncroppedSink.get());
}
if (sink != nullptr) {
_source->AddOrUpdateSink(sink.get(), rtc::VideoSinkWants());
}
_uncroppedSink = sink;
}
webrtc::ScopedJavaLocalRef<jobject> VideoCameraCapturer::GetJavaVideoCapturerObserver(JNIEnv *env) {
return _source->GetJavaVideoCapturerObserver(env);
}
} // namespace tgcalls
extern "C" {
JNIEXPORT jobject Java_org_telegram_messenger_voip_VideoCapturerDevice_nativeGetJavaVideoCapturerObserver(JNIEnv *env, jclass clazz, jlong ptr) {
tgcalls::VideoCameraCapturer *capturer = (tgcalls::VideoCameraCapturer *) (intptr_t) ptr;
return capturer->GetJavaVideoCapturerObserver(env).Release();
}
}

View file

@ -0,0 +1,45 @@
#ifndef TGCALLS_VIDEO_CAMERA_CAPTURER_H
#define TGCALLS_VIDEO_CAMERA_CAPTURER_H
#include "api/scoped_refptr.h"
#include "api/media_stream_interface.h"
#include "modules/video_capture/video_capture.h"
#include "sdk/android/native_api/jni/scoped_java_ref.h"
#include "sdk/android/native_api/video/video_source.h"
#include "VideoCaptureInterface.h"
#include <memory>
#include <vector>
#include <stddef.h>
#include <jni.h>
namespace tgcalls {
class VideoCameraCapturer;
class VideoCameraCapturer {
public:
VideoCameraCapturer(rtc::scoped_refptr<webrtc::JavaVideoTrackSourceInterface> source, std::string deviceId, std::function<void(VideoState)> stateUpdated, std::shared_ptr<PlatformContext> platformContext);
void setState(VideoState state);
void setPreferredCaptureAspectRatio(float aspectRatio);
void setUncroppedSink(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink);
webrtc::ScopedJavaLocalRef<jobject> GetJavaVideoCapturerObserver(JNIEnv* env);
private:
rtc::scoped_refptr<webrtc::JavaVideoTrackSourceInterface> _source;
std::function<void(VideoState)> _stateUpdated;
VideoState _state;
std::shared_ptr<PlatformContext> _platformContext;
float _aspectRatio;
std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> _uncroppedSink;
};
} // namespace tgcalls
#endif

View file

@ -0,0 +1,31 @@
#include "VideoCapturerInterfaceImpl.h"
#include "VideoCameraCapturer.h"
namespace tgcalls {
VideoCapturerInterfaceImpl::VideoCapturerInterfaceImpl(rtc::scoped_refptr<webrtc::JavaVideoTrackSourceInterface> source, std::string deviceId, std::function<void(VideoState)> stateUpdated, std::shared_ptr<PlatformContext> platformContext) {
_capturer = std::make_unique<VideoCameraCapturer>(source, deviceId, stateUpdated, platformContext);
}
void VideoCapturerInterfaceImpl::setState(VideoState state) {
_capturer->setState(state);
}
void VideoCapturerInterfaceImpl::setPreferredCaptureAspectRatio(float aspectRatio) {
_capturer->setPreferredCaptureAspectRatio(aspectRatio);
}
void VideoCapturerInterfaceImpl::setUncroppedOutput(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink) {
_capturer->setUncroppedSink(sink);
}
int VideoCapturerInterfaceImpl::VideoCapturerInterfaceImpl::getRotation() {
return 0;
}
void VideoCapturerInterfaceImpl::setOnFatalError(std::function<void()> error) {
}
} // namespace tgcalls

View file

@ -0,0 +1,32 @@
#ifndef TGCALLS_VIDEO_CAPTURER_INTERFACE_IMPL_H
#define TGCALLS_VIDEO_CAPTURER_INTERFACE_IMPL_H
#include "VideoCapturerInterface.h"
#include "VideoCameraCapturer.h"
#include "api/media_stream_interface.h"
namespace tgcalls {
class VideoCapturerInterfaceImpl final : public VideoCapturerInterface {
public:
VideoCapturerInterfaceImpl(
rtc::scoped_refptr<webrtc::JavaVideoTrackSourceInterface> source,
std::string deviceId,
std::function<void(VideoState)> stateUpdated,
std::shared_ptr<PlatformContext> platformContext
);
void setState(VideoState state) override;
void setPreferredCaptureAspectRatio(float aspectRatio) override;
void setUncroppedOutput(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink) override;
int getRotation() override;
void setOnFatalError(std::function<void()> error) override;
private:
std::unique_ptr<VideoCameraCapturer> _capturer;
};
} // namespace tgcalls
#endif

View file

@ -0,0 +1,39 @@
#ifndef TGCALLS_AUDIO_DEVICE_MODULE_IOS
#define TGCALLS_AUDIO_DEVICE_MODULE_IOS
#include "platform/PlatformInterface.h"
namespace tgcalls {
class AudioDeviceModuleIOS : public DefaultWrappedAudioDeviceModule {
public:
AudioDeviceModuleIOS(webrtc::scoped_refptr<webrtc::AudioDeviceModule> impl) :
DefaultWrappedAudioDeviceModule(impl) {
}
virtual ~AudioDeviceModuleIOS() {
}
virtual int32_t StopPlayout() override {
return 0;
}
virtual int32_t StopRecording() override {
return 0;
}
virtual int32_t Terminate() override {
return 0;
}
virtual void Stop() override {
WrappedInstance()->StopPlayout();
WrappedInstance()->StopRecording();
WrappedInstance()->Terminate();
}
};
}
#endif

View file

@ -0,0 +1,31 @@
#ifndef TGCALLS_AUDIO_DEVICE_MODULE_MACOS
#define TGCALLS_AUDIO_DEVICE_MODULE_MACOS
#include "platform/PlatformInterface.h"
namespace tgcalls {
class AudioDeviceModuleMacos : public DefaultWrappedAudioDeviceModule {
public:
AudioDeviceModuleMacos(webrtc::scoped_refptr<webrtc::AudioDeviceModule> impl) :
DefaultWrappedAudioDeviceModule(impl) {
}
virtual ~AudioDeviceModuleMacos() {
}
virtual int32_t SetStereoPlayout(bool enable) override {
return WrappedInstance()->SetStereoPlayout(enable);
}
virtual void Stop() override {
WrappedInstance()->StopPlayout();
WrappedInstance()->StopRecording();
WrappedInstance()->Terminate();
}
};
}
#endif

View file

@ -0,0 +1,21 @@
#ifndef TGCALLS_CUSTOM_EXTERNAL_CAPTURER_H
#define TGCALLS_CUSTOM_EXTERNAL_CAPTURER_H
#ifdef WEBRTC_IOS
#import <Foundation/Foundation.h>
#import <AVFoundation/AVFoundation.h>
#include <memory>
#include "api/scoped_refptr.h"
#include "api/media_stream_interface.h"
#import "base/RTCVideoFrame.h"
#include "Instance.h"
@interface CustomExternalCapturer : NSObject
- (instancetype)initWithSource:(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source;
+ (void)passPixelBuffer:(CVPixelBufferRef)pixelBuffer sampleBufferReference:(CMSampleBufferRef)sampleBufferReference rotation:(RTCVideoRotation)rotation toSource:(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source croppingBuffer:(std::vector<uint8_t> &)croppingBuffer;
@end
#endif // WEBRTC_IOS
#endif

View file

@ -0,0 +1,90 @@
#include "CustomExternalCapturer.h"
#import <AVFoundation/AVFoundation.h>
#include "rtc_base/logging.h"
#import "base/RTCLogging.h"
#import "base/RTCVideoFrameBuffer.h"
#import "TGRTCCVPixelBuffer.h"
#import "sdk/objc/native/src/objc_video_track_source.h"
#import "sdk/objc/native/src/objc_frame_buffer.h"
//#import "api/video_track_source_proxy.h"
#import "helpers/UIDevice+RTCDevice.h"
#import "helpers/AVCaptureSession+DevicePosition.h"
#import "helpers/RTCDispatcher+Private.h"
#import "base/RTCVideoFrame.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "third_party/libyuv/include/libyuv.h"
#include "pc/video_track_source_proxy.h"
#include "DarwinVideoSource.h"
static const int64_t kNanosecondsPerSecond = 1000000000;
static tgcalls::DarwinVideoTrackSource *getObjCVideoSource(const webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> nativeSource) {
webrtc::VideoTrackSourceProxy *proxy_source =
static_cast<webrtc::VideoTrackSourceProxy *>(nativeSource.get());
return static_cast<tgcalls::DarwinVideoTrackSource *>(proxy_source->internal());
}
@interface CustomExternalCapturer () {
webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> _source;
}
@end
@implementation CustomExternalCapturer
- (instancetype)initWithSource:(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source {
self = [super init];
if (self != nil) {
_source = source;
}
return self;
}
- (void)dealloc {
}
+ (void)passPixelBuffer:(CVPixelBufferRef)pixelBuffer sampleBufferReference:(CMSampleBufferRef)sampleBufferReference rotation:(RTCVideoRotation)rotation toSource:(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source croppingBuffer:(std::vector<uint8_t> &)croppingBuffer {
TGRTCCVPixelBuffer *rtcPixelBuffer = [[TGRTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer];
if (sampleBufferReference) {
[rtcPixelBuffer storeSampleBufferReference:sampleBufferReference];
}
rtcPixelBuffer.deviceRelativeVideoRotation = -1;
int width = rtcPixelBuffer.width;
int height = rtcPixelBuffer.height;
width -= width % 4;
height -= height % 4;
if (width != rtcPixelBuffer.width || height != rtcPixelBuffer.height) {
CVPixelBufferRef outputPixelBufferRef = NULL;
OSType pixelFormat = CVPixelBufferGetPixelFormatType(rtcPixelBuffer.pixelBuffer);
CVPixelBufferCreate(NULL, width, height, pixelFormat, NULL, &outputPixelBufferRef);
if (outputPixelBufferRef) {
int bufferSize = [rtcPixelBuffer bufferSizeForCroppingAndScalingToWidth:width height:height];
if (croppingBuffer.size() < bufferSize) {
croppingBuffer.resize(bufferSize);
}
if ([rtcPixelBuffer cropAndScaleTo:outputPixelBufferRef withTempBuffer:croppingBuffer.data()]) {
rtcPixelBuffer = [[TGRTCCVPixelBuffer alloc] initWithPixelBuffer:outputPixelBufferRef];
rtcPixelBuffer.deviceRelativeVideoRotation = -1;
}
CVPixelBufferRelease(outputPixelBufferRef);
}
}
int64_t timeStampNs = CACurrentMediaTime() * kNanosecondsPerSecond;
RTCVideoFrame *videoFrame = [[RTCVideoFrame alloc] initWithBuffer:(id<RTCVideoFrameBuffer>)[rtcPixelBuffer toI420] rotation:rotation timeStampNs:timeStampNs];
getObjCVideoSource(source)->OnCapturedFrame(videoFrame);
}
@end

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,200 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
#ifndef TGCALLS_CUSTOM_SIMULCAST_ENCODER_ADAPTER_H_
#define TGCALLS_CUSTOM_SIMULCAST_ENCODER_ADAPTER_H_
#include <atomic>
#include <list>
#include <memory>
#include <stack>
#include <string>
#include <utility>
#include <vector>
#include "absl/types/optional.h"
#include "api/fec_controller_override.h"
#include "api/sequence_checker.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/video_encoder_factory.h"
#include "common_video/framerate_controller.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "rtc_base/experiments/encoder_info_settings.h"
#include "rtc_base/system/no_unique_address.h"
#include "rtc_base/system/rtc_export.h"
#include "api/field_trials_view.h"
namespace webrtc {
// CustomSimulcastEncoderAdapter implements simulcast support by creating multiple
// webrtc::VideoEncoder instances with the given VideoEncoderFactory.
// The object is created and destroyed on the worker thread, but all public
// interfaces should be called from the encoder task queue.
class RTC_EXPORT CustomSimulcastEncoderAdapter : public VideoEncoder {
public:
// TODO(bugs.webrtc.org/11000): Remove when downstream usage is gone.
CustomSimulcastEncoderAdapter(VideoEncoderFactory* primarty_factory,
const SdpVideoFormat& format);
// `primary_factory` produces the first-choice encoders to use.
// `fallback_factory`, if non-null, is used to create fallback encoder that
// will be used if InitEncode() fails for the primary encoder.
CustomSimulcastEncoderAdapter(VideoEncoderFactory* primary_factory,
VideoEncoderFactory* hardware_factory,
const SdpVideoFormat& format,
const FieldTrialsView& field_trials);
~CustomSimulcastEncoderAdapter() override;
// Implements VideoEncoder.
void SetFecControllerOverride(
FecControllerOverride* fec_controller_override) override;
int Release() override;
int InitEncode(const VideoCodec* codec_settings,
const VideoEncoder::Settings& settings) override;
int Encode(const VideoFrame& input_image,
const std::vector<VideoFrameType>* frame_types) override;
int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override;
void SetRates(const RateControlParameters& parameters) override;
void OnPacketLossRateUpdate(float packet_loss_rate) override;
void OnRttUpdate(int64_t rtt_ms) override;
void OnLossNotification(const LossNotification& loss_notification) override;
EncoderInfo GetEncoderInfo() const override;
private:
class EncoderContext {
public:
EncoderContext(std::unique_ptr<VideoEncoder> encoder,
bool prefer_temporal_support,
VideoEncoder::EncoderInfo primary_info,
VideoEncoder::EncoderInfo fallback_info);
EncoderContext& operator=(EncoderContext&&) = delete;
VideoEncoder& encoder() { return *encoder_; }
bool prefer_temporal_support() { return prefer_temporal_support_; }
void Release();
const VideoEncoder::EncoderInfo& PrimaryInfo() { return primary_info_; }
const VideoEncoder::EncoderInfo& FallbackInfo() { return fallback_info_; }
private:
std::unique_ptr<VideoEncoder> encoder_;
bool prefer_temporal_support_;
const VideoEncoder::EncoderInfo primary_info_;
const VideoEncoder::EncoderInfo fallback_info_;
};
class StreamContext : public EncodedImageCallback {
public:
StreamContext(CustomSimulcastEncoderAdapter* parent,
std::unique_ptr<EncoderContext> encoder_context,
std::unique_ptr<FramerateController> framerate_controller,
int stream_idx,
uint16_t width,
uint16_t height,
bool send_stream);
StreamContext(StreamContext&& rhs);
StreamContext& operator=(StreamContext&&) = delete;
~StreamContext() override;
Result OnEncodedImage(
const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_specific_info) override;
void OnDroppedFrame(DropReason reason) override;
VideoEncoder& encoder() { return encoder_context_->encoder(); }
const VideoEncoder& encoder() const { return encoder_context_->encoder(); }
int stream_idx() const { return stream_idx_; }
uint16_t width() const { return width_; }
uint16_t height() const { return height_; }
bool is_keyframe_needed() const {
return !is_paused_ && is_keyframe_needed_;
}
void set_is_keyframe_needed() { is_keyframe_needed_ = true; }
bool is_paused() const { return is_paused_; }
void set_is_paused(bool is_paused) { is_paused_ = is_paused; }
absl::optional<double> target_fps() const {
return framerate_controller_ == nullptr
? absl::nullopt
: absl::optional<double>(
framerate_controller_->GetMaxFramerate());
}
std::unique_ptr<EncoderContext> ReleaseEncoderContext() &&;
void OnKeyframe(Timestamp timestamp);
bool ShouldDropFrame(Timestamp timestamp);
private:
CustomSimulcastEncoderAdapter* const parent_;
std::unique_ptr<EncoderContext> encoder_context_;
std::unique_ptr<FramerateController> framerate_controller_;
const int stream_idx_;
const uint16_t width_;
const uint16_t height_;
bool is_keyframe_needed_;
bool is_paused_;
};
bool Initialized() const;
void DestroyStoredEncoders();
// This method creates encoder. May reuse previously created encoders from
// `cached_encoder_contexts_`. It's const because it's used from
// const GetEncoderInfo().
std::unique_ptr<EncoderContext> FetchOrCreateEncoderContext(
bool is_lowest_quality_stream, bool is_highest_quality_stream) const;
webrtc::VideoCodec MakeStreamCodec(const webrtc::VideoCodec& codec,
int stream_idx,
uint32_t start_bitrate_kbps,
bool is_lowest_quality_stream,
bool is_highest_quality_stream);
EncodedImageCallback::Result OnEncodedImage(
size_t stream_idx,
const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_specific_info);
void OnDroppedFrame(size_t stream_idx);
void OverrideFromFieldTrial(VideoEncoder::EncoderInfo* info) const;
std::atomic<int> inited_;
VideoEncoderFactory* const primary_encoder_factory_;
VideoEncoderFactory* const hardware_encoder_factory_;
const SdpVideoFormat video_format_;
VideoCodec codec_;
int total_streams_count_;
bool bypass_mode_;
std::vector<StreamContext> stream_contexts_;
EncodedImageCallback* encoded_complete_callback_;
// Used for checking the single-threaded access of the encoder interface.
RTC_NO_UNIQUE_ADDRESS SequenceChecker encoder_queue_;
// Store previously created and released encoders , so they don't have to be
// recreated. Remaining encoders are destroyed by the destructor.
// Marked as `mutable` becuase we may need to temporarily create encoder in
// GetEncoderInfo(), which is const.
mutable std::list<std::unique_ptr<EncoderContext>> cached_encoder_contexts_;
const absl::optional<unsigned int> experimental_boosted_screenshare_qp_;
const bool boost_base_layer_quality_;
const bool prefer_temporal_support_on_base_layer_;
const SimulcastEncoderAdapterEncoderInfoSettings encoder_info_override_;
};
} // namespace webrtc
#endif // TGCALLS_CUSTOM_SIMULCAST_ENCODER_ADAPTER_H_

View file

@ -0,0 +1,13 @@
#ifndef TGCALLS_DARWIN_FFMPEG_H
#define TGCALLS_DARWIN_FFMPEG_H
#include "platform/PlatformInterface.h"
namespace tgcalls {
void setupDarwinVideoDecoding(AVCodecContext *codecContext);
webrtc::scoped_refptr<webrtc::VideoFrameBuffer> createDarwinPlatformFrameFromData(AVFrame const *frame);
} // namespace tgcalls
#endif

View file

@ -0,0 +1,41 @@
#include "DarwinFFMpeg.h"
extern "C" {
#include <libavutil/frame.h>
#include <libavutil/pixfmt.h>
#include <libavcodec/avcodec.h>
}
#import "ExtractCVPixelBuffer.h"
namespace tgcalls {
static enum AVPixelFormat getDarwinPreferredPixelFormat(__unused AVCodecContext *ctx, __unused const enum AVPixelFormat *pix_fmts) {
return AV_PIX_FMT_VIDEOTOOLBOX;
}
void setupDarwinVideoDecoding(AVCodecContext *codecContext) {
return;
#if TARGET_IPHONE_SIMULATOR
#else
if (!codecContext) {
return;
}
av_hwdevice_ctx_create(&codecContext->hw_device_ctx, AV_HWDEVICE_TYPE_VIDEOTOOLBOX, nullptr, nullptr, 0);
codecContext->get_format = getDarwinPreferredPixelFormat;
#endif
}
webrtc::scoped_refptr<webrtc::VideoFrameBuffer> createDarwinPlatformFrameFromData(AVFrame const *frame) {
if (!frame) {
return nullptr;
}
if (frame->format == AV_PIX_FMT_VIDEOTOOLBOX && frame->data[3]) {
return extractCVPixelBuffer((void *)frame->data[3]);
} else {
return nullptr;
}
}
}

View file

@ -0,0 +1,40 @@
#ifndef TGCALLS_DARWIN_INTERFACE_H
#define TGCALLS_DARWIN_INTERFACE_H
#include "platform/PlatformInterface.h"
#import <CoreVideo/CoreVideo.h>
namespace tgcalls {
class DarwinVideoFrame : public PlatformVideoFrame {
public:
DarwinVideoFrame(CVPixelBufferRef pixelBuffer);
virtual ~DarwinVideoFrame();
CVPixelBufferRef pixelBuffer() const {
return _pixelBuffer;
}
private:
CVPixelBufferRef _pixelBuffer = nullptr;
};
class DarwinInterface : public PlatformInterface {
public:
std::unique_ptr<rtc::NetworkMonitorFactory> createNetworkMonitorFactory() override;
void configurePlatformAudio(int numChannels) override;
std::unique_ptr<webrtc::VideoEncoderFactory> makeVideoEncoderFactory(bool preferHardwareEncoding, bool isScreencast) override;
std::unique_ptr<webrtc::VideoDecoderFactory> makeVideoDecoderFactory() override;
bool supportsEncoding(const std::string &codecName) override;
webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> makeVideoSource(rtc::Thread *signalingThread, rtc::Thread *workerThread) override;
virtual void adaptVideoSource(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> videoSource, int width, int height, int fps) override;
std::unique_ptr<VideoCapturerInterface> makeVideoCapturer(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> source, std::string deviceId, std::function<void(VideoState)> stateUpdated, std::function<void(PlatformCaptureInfo)> captureInfoUpdated, std::shared_ptr<PlatformContext> platformContext, std::pair<int, int> &outResolution) override;
virtual webrtc::scoped_refptr<WrappedAudioDeviceModule> wrapAudioDeviceModule(webrtc::scoped_refptr<webrtc::AudioDeviceModule> module) override;
virtual void setupVideoDecoding(AVCodecContext *codecContext) override;
virtual webrtc::scoped_refptr<webrtc::VideoFrameBuffer> createPlatformFrameFromData(AVFrame const *frame) override;
};
} // namespace tgcalls
#endif

View file

@ -0,0 +1,158 @@
#include "DarwinInterface.h"
#include "VideoCapturerInterfaceImpl.h"
#include "sdk/objc/native/src/objc_video_track_source.h"
#include "sdk/objc/native/api/network_monitor_factory.h"
#include "media/base/media_constants.h"
#include "TGRTCDefaultVideoEncoderFactory.h"
#include "TGRTCDefaultVideoDecoderFactory.h"
#include "sdk/objc/native/api/video_encoder_factory.h"
#include "sdk/objc/native/api/video_decoder_factory.h"
#include "pc/video_track_source_proxy.h"
#import "base/RTCLogging.h"
#include "AudioDeviceModuleIOS.h"
#include "AudioDeviceModuleMacos.h"
#include "DarwinVideoSource.h"
#include "objc_video_encoder_factory.h"
#include "objc_video_decoder_factory.h"
#include "sdk/objc/native/src/objc_frame_buffer.h"
#import "sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.h"
#import "DarwinFFMpeg.h"
#ifdef WEBRTC_IOS
#include "platform/darwin/iOS/RTCAudioSession.h"
#include "platform/darwin/iOS/RTCAudioSessionConfiguration.h"
#import <UIKit/UIKit.h>
#endif // WEBRTC_IOS
#import <AVFoundation/AVFoundation.h>
#include <sys/sysctl.h>
namespace tgcalls {
std::unique_ptr<webrtc::VideoDecoderFactory> CustomObjCToNativeVideoDecoderFactory(
id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> objc_video_decoder_factory) {
return std::make_unique<webrtc::CustomObjCVideoDecoderFactory>(objc_video_decoder_factory);
}
static DarwinVideoTrackSource *getObjCVideoSource(const webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> nativeSource) {
webrtc::VideoTrackSourceProxy *proxy_source =
static_cast<webrtc::VideoTrackSourceProxy *>(nativeSource.get());
return static_cast<DarwinVideoTrackSource *>(proxy_source->internal());
}
[[maybe_unused]] static NSString *getPlatformInfo() {
const char *typeSpecifier = "hw.machine";
size_t size;
sysctlbyname(typeSpecifier, NULL, &size, NULL, 0);
char *answer = (char *)malloc(size);
sysctlbyname(typeSpecifier, answer, &size, NULL, 0);
NSString *results = [NSString stringWithCString:answer encoding:NSUTF8StringEncoding];
free(answer);
return results;
}
std::unique_ptr<rtc::NetworkMonitorFactory> DarwinInterface::createNetworkMonitorFactory() {
return webrtc::CreateNetworkMonitorFactory();
}
void DarwinInterface::configurePlatformAudio(int numChanels) {
}
std::unique_ptr<webrtc::VideoEncoderFactory> DarwinInterface::makeVideoEncoderFactory(bool preferHardwareEncoding, bool isScreencast) {
auto nativeFactory = std::make_unique<webrtc::CustomObjCVideoEncoderFactory>([[TGRTCDefaultVideoEncoderFactory alloc] initWithPreferHardwareH264:preferHardwareEncoding preferX264:false]);
if (!preferHardwareEncoding) {
auto nativeHardwareFactory = std::make_unique<webrtc::CustomObjCVideoEncoderFactory>([[TGRTCDefaultVideoEncoderFactory alloc] initWithPreferHardwareH264:true preferX264:false]);
return std::make_unique<webrtc::SimulcastVideoEncoderFactory>(std::move(nativeFactory), std::move(nativeHardwareFactory));
}
return nativeFactory;
}
std::unique_ptr<webrtc::VideoDecoderFactory> DarwinInterface::makeVideoDecoderFactory() {
return CustomObjCToNativeVideoDecoderFactory([[TGRTCDefaultVideoDecoderFactory alloc] init]);
}
bool DarwinInterface::supportsEncoding(const std::string &codecName) {
if (false) {
}
#ifndef WEBRTC_DISABLE_H265
else if (codecName == cricket::kH265CodecName) {
#ifdef WEBRTC_IOS
if (@available(iOS 11.0, *)) {
return [[AVAssetExportSession allExportPresets] containsObject:AVAssetExportPresetHEVCHighestQuality];
}
#elif defined WEBRTC_MAC // WEBRTC_IOS
#ifdef __x86_64__
return NO;
#else
return YES;
#endif
#endif // WEBRTC_IOS || WEBRTC_MAC
}
#endif
else if (codecName == cricket::kH264CodecName) {
#ifdef __x86_64__
return YES;
#else
return NO;
#endif
} else if (codecName == cricket::kVp8CodecName) {
return true;
} else if (codecName == cricket::kVp9CodecName) {
return true;
}
return false;
}
webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> DarwinInterface::makeVideoSource(rtc::Thread *signalingThread, rtc::Thread *workerThread) {
webrtc::scoped_refptr<tgcalls::DarwinVideoTrackSource> objCVideoTrackSource(new rtc::RefCountedObject<tgcalls::DarwinVideoTrackSource>());
return webrtc::VideoTrackSourceProxy::Create(signalingThread, workerThread, objCVideoTrackSource);
}
void DarwinInterface::adaptVideoSource(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> videoSource, int width, int height, int fps) {
getObjCVideoSource(videoSource)->OnOutputFormatRequest(width, height, fps);
}
std::unique_ptr<VideoCapturerInterface> DarwinInterface::makeVideoCapturer(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> source, std::string deviceId, std::function<void(VideoState)> stateUpdated, std::function<void(PlatformCaptureInfo)> captureInfoUpdated, std::shared_ptr<PlatformContext> platformContext, std::pair<int, int> &outResolution) {
return std::make_unique<VideoCapturerInterfaceImpl>(source, deviceId, stateUpdated, captureInfoUpdated, outResolution);
}
webrtc::scoped_refptr<WrappedAudioDeviceModule> DarwinInterface::wrapAudioDeviceModule(webrtc::scoped_refptr<webrtc::AudioDeviceModule> module) {
#ifdef WEBRTC_IOS
return rtc::make_ref_counted<AudioDeviceModuleIOS>(module);
#else
return rtc::make_ref_counted<AudioDeviceModuleMacos>(module);
#endif
}
void DarwinInterface::setupVideoDecoding(AVCodecContext *codecContext) {
return setupDarwinVideoDecoding(codecContext);
}
webrtc::scoped_refptr<webrtc::VideoFrameBuffer> DarwinInterface::createPlatformFrameFromData(AVFrame const *frame) {
return createDarwinPlatformFrameFromData(frame);
}
std::unique_ptr<PlatformInterface> CreatePlatformInterface() {
return std::make_unique<DarwinInterface>();
}
DarwinVideoFrame::DarwinVideoFrame(CVPixelBufferRef pixelBuffer) {
_pixelBuffer = CVPixelBufferRetain(pixelBuffer);
}
DarwinVideoFrame::~DarwinVideoFrame() {
if (_pixelBuffer) {
CVPixelBufferRelease(_pixelBuffer);
}
}
} // namespace tgcalls

View file

@ -0,0 +1,57 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef TGCALLS_DARWIN_VIDEO_SOURCE_H
#define TGCALLS_DARWIN_VIDEO_SOURCE_H
//#ifdef WEBRTC_IOS
#import "base/RTCVideoCapturer.h"
#include "base/RTCMacros.h"
#include "media/base/adapted_video_track_source.h"
#include "rtc_base/timestamp_aligner.h"
RTC_FWD_DECL_OBJC_CLASS(RTC_OBJC_TYPE(RTCVideoFrame));
namespace tgcalls {
class DarwinVideoTrackSource : public rtc::AdaptedVideoTrackSource {
public:
DarwinVideoTrackSource();
// This class can not be used for implementing screen casting. Hopefully, this
// function will be removed before we add that to iOS/Mac.
bool is_screencast() const override;
// Indicates that the encoder should denoise video before encoding it.
// If it is not set, the default configuration is used which is different
// depending on video codec.
absl::optional<bool> needs_denoising() const override;
SourceState state() const override;
bool remote() const override;
void OnCapturedFrame(RTC_OBJC_TYPE(RTCVideoFrame) * frame);
bool OnCapturedFrame(const webrtc::VideoFrame& frame);
// Called by RTCVideoSource.
void OnOutputFormatRequest(int width, int height, int fps);
private:
rtc::VideoBroadcaster broadcaster_;
rtc::TimestampAligner timestamp_aligner_;
};
} // namespace tgcalls
//#endif //WEBRTC_IOS
#endif

View file

@ -0,0 +1,165 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "DarwinVideoSource.h"
#import "base/RTCVideoFrame.h"
#import "base/RTCVideoFrameBuffer.h"
#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
#include "api/video/i420_buffer.h"
#include "sdk/objc/native/src/objc_frame_buffer.h"
namespace tgcalls {
DarwinVideoTrackSource::DarwinVideoTrackSource()
: AdaptedVideoTrackSource(/* required resolution alignment */ 2) {}
bool DarwinVideoTrackSource::is_screencast() const {
return false;
}
absl::optional<bool> DarwinVideoTrackSource::needs_denoising() const {
return false;
}
webrtc::MediaSourceInterface::SourceState DarwinVideoTrackSource::state() const {
return SourceState::kLive;
}
bool DarwinVideoTrackSource::remote() const {
return false;
}
void DarwinVideoTrackSource::OnOutputFormatRequest(int width, int height, int fps) {
cricket::VideoFormat format(width, height, cricket::VideoFormat::FpsToInterval(fps), 0);
video_adapter()->OnOutputFormatRequest(format);
}
void DarwinVideoTrackSource::OnCapturedFrame(RTC_OBJC_TYPE(RTCVideoFrame) * frame) {
const int64_t timestamp_us = frame.timeStampNs / rtc::kNumNanosecsPerMicrosec;
const int64_t translated_timestamp_us =
timestamp_aligner_.TranslateTimestamp(timestamp_us, rtc::TimeMicros());
int adapted_width;
int adapted_height;
int crop_width;
int crop_height;
int crop_x;
int crop_y;
if (!AdaptFrame(frame.width,
frame.height,
timestamp_us,
&adapted_width,
&adapted_height,
&crop_width,
&crop_height,
&crop_x,
&crop_y)) {
return;
}
webrtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer;
if (adapted_width == frame.width && adapted_height == frame.height) {
// No adaption - optimized path.
@autoreleasepool {
buffer = new rtc::RefCountedObject<webrtc::ObjCFrameBuffer>(frame.buffer);
}
} else if ([frame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) {
// Adapted CVPixelBuffer frame.
@autoreleasepool {
RTC_OBJC_TYPE(RTCCVPixelBuffer) *rtcPixelBuffer =
(RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer;
buffer = new rtc::RefCountedObject<webrtc::ObjCFrameBuffer>([[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc]
initWithPixelBuffer:rtcPixelBuffer.pixelBuffer
adaptedWidth:adapted_width
adaptedHeight:adapted_height
cropWidth:crop_width
cropHeight:crop_height
cropX:crop_x + rtcPixelBuffer.cropX
cropY:crop_y + rtcPixelBuffer.cropY]);
}
} else {
@autoreleasepool {
// Adapted I420 frame.
// TODO(magjed): Optimize this I420 path.
webrtc::scoped_refptr<webrtc::I420Buffer> i420_buffer = webrtc::I420Buffer::Create(adapted_width, adapted_height);
buffer = new rtc::RefCountedObject<webrtc::ObjCFrameBuffer>(frame.buffer);
i420_buffer->CropAndScaleFrom(*buffer->ToI420(), crop_x, crop_y, crop_width, crop_height);
buffer = i420_buffer;
}
}
// Applying rotation is only supported for legacy reasons and performance is
// not critical here.
webrtc::VideoRotation rotation = static_cast<webrtc::VideoRotation>(frame.rotation);
if (apply_rotation() && rotation != webrtc::kVideoRotation_0) {
buffer = webrtc::I420Buffer::Rotate(*buffer->ToI420(), rotation);
rotation = webrtc::kVideoRotation_0;
}
OnFrame(webrtc::VideoFrame::Builder()
.set_video_frame_buffer(buffer)
.set_rotation(rotation)
.set_timestamp_us(translated_timestamp_us)
.build());
}
bool DarwinVideoTrackSource::OnCapturedFrame(const webrtc::VideoFrame& frame) {
const int64_t timestamp_us = frame.timestamp_us() / rtc::kNumNanosecsPerMicrosec;
const int64_t translated_timestamp_us =
timestamp_aligner_.TranslateTimestamp(timestamp_us, rtc::TimeMicros());
int adapted_width;
int adapted_height;
int crop_width;
int crop_height;
int crop_x;
int crop_y;
if (!AdaptFrame(frame.width(),
frame.height(),
timestamp_us,
&adapted_width,
&adapted_height,
&crop_width,
&crop_height,
&crop_x,
&crop_y)) {
return false;
}
webrtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer;
if (adapted_width == frame.width() && adapted_height == frame.height()) {
buffer = frame.video_frame_buffer();
} else {
webrtc::scoped_refptr<webrtc::I420Buffer> i420_buffer = webrtc::I420Buffer::Create(adapted_width, adapted_height);
buffer = frame.video_frame_buffer();
i420_buffer->CropAndScaleFrom(*buffer->ToI420(), crop_x, crop_y, crop_width, crop_height);
buffer = i420_buffer;
}
// Applying rotation is only supported for legacy reasons and performance is
// not critical here.
webrtc::VideoRotation rotation = frame.rotation();
if (apply_rotation() && rotation != webrtc::kVideoRotation_0) {
buffer = webrtc::I420Buffer::Rotate(*buffer->ToI420(), rotation);
rotation = webrtc::kVideoRotation_0;
}
OnFrame(webrtc::VideoFrame::Builder()
.set_video_frame_buffer(buffer)
.set_rotation(rotation)
.set_timestamp_us(translated_timestamp_us)
.build());
return true;
}
} // namespace webrtc

View file

@ -0,0 +1,55 @@
//
// DesktopCaptureSourceView.h
// TgVoipWebrtc
//
// Created by Mikhail Filimonov on 28.12.2020.
// Copyright © 2020 Mikhail Filimonov. All rights reserved.
//
#import <Foundation/Foundation.h>
#import <AppKit/AppKit.h>
NS_ASSUME_NONNULL_BEGIN
@protocol VideoSourceMac
-(NSString *)deviceIdKey;
-(NSString *)title;
-(NSString *)uniqueKey;
-(BOOL)isEqual:(id)another;
@end
@interface DesktopCaptureSourceDataMac : NSObject
@property CGSize aspectSize;
@property double fps;
@property bool captureMouse;
-(id)initWithSize:(CGSize)size fps:(double)fps captureMouse:(bool)captureMouse;
-(NSString *)cachedKey;
@end
@interface DesktopCaptureSourceMac : NSObject <VideoSourceMac>
-(long)uniqueId;
-(BOOL)isWindow;
@end
@interface DesktopCaptureSourceScopeMac : NSObject
@property(nonatomic, strong, readonly) DesktopCaptureSourceDataMac *data;
@property(nonatomic, strong, readonly) DesktopCaptureSourceMac *source;
-(id)initWithSource:(DesktopCaptureSourceMac *)source data:(DesktopCaptureSourceDataMac *)data;
-(NSString *)cachedKey;
@end
@interface DesktopCaptureSourceManagerMac : NSObject
-(instancetype)init_s;
-(instancetype)init_w;
-(NSArray<DesktopCaptureSourceMac *> *)list;
-(NSView *)createForScope:(DesktopCaptureSourceScopeMac *)scope;
-(void)start:(DesktopCaptureSourceScopeMac *)scope;
-(void)stop:(DesktopCaptureSourceScopeMac *)scope;
@end
NS_ASSUME_NONNULL_END

View file

@ -0,0 +1,207 @@
//
// DesktopCaptureSourceView.m
// TgVoipWebrtc
//
// Created by Mikhail Filimonov on 28.12.2020.
// Copyright © 2020 Mikhail Filimonov. All rights reserved.
//
#import <Cocoa/Cocoa.h>
#import "DesktopCaptureSourceViewMac.h"
#import "platform/darwin/VideoMetalViewMac.h"
#import "tgcalls/desktop_capturer/DesktopCaptureSource.h"
#import "tgcalls/desktop_capturer/DesktopCaptureSourceHelper.h"
#import "tgcalls/desktop_capturer/DesktopCaptureSourceManager.h"
#import "platform/darwin/VideoMetalViewMac.h"
@interface DesktopCaptureSourceViewMetal : VideoMetalView
@end
@implementation DesktopCaptureSourceViewMetal
-(id)initWithHelper:(tgcalls::DesktopCaptureSourceHelper)helper {
if (self = [super initWithFrame:CGRectZero]) {
std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink = [self getSink];
helper.setOutput(sink);
[self setVideoContentMode:kCAGravityResizeAspectFill];
}
return self;
}
@end
@implementation DesktopCaptureSourceDataMac
-(id)initWithSize:(CGSize)size fps:(double)fps captureMouse:(bool)captureMouse {
if (self = [super init]) {
self.aspectSize = size;
self.fps = fps;
self.captureMouse = captureMouse;
}
return self;
}
-(NSString *)cachedKey {
return [[NSString alloc] initWithFormat:@"%@:%f:%d", NSStringFromSize(self.aspectSize), self.fps, self.captureMouse];
}
@end
@interface DesktopCaptureSourceMac ()
{
absl::optional<tgcalls::DesktopCaptureSource> _source;
BOOL _isWindow;
}
-(id)initWithSource:(tgcalls::DesktopCaptureSource)source isWindow:(BOOL)isWindow;
-(tgcalls::DesktopCaptureSource)getSource;
@end
@implementation DesktopCaptureSourceMac
-(tgcalls::DesktopCaptureSource)getSource {
return _source.value();
}
-(NSString *)title {
if (_isWindow) {
const tgcalls::DesktopCaptureSource source = _source.value();
return [[NSString alloc] initWithCString:source.title().c_str() encoding:NSUTF8StringEncoding];
}
else
return [[NSString alloc] initWithFormat:@"Screen"];
}
-(long)uniqueId {
return _source.value().uniqueId();
}
-(BOOL)isWindow {
return _isWindow;
}
-(NSString *)uniqueKey {
return [[NSString alloc] initWithFormat:@"%ld:%@", self.uniqueId, _isWindow ? @"Window" : @"Screen"];
}
-(NSString *)deviceIdKey {
return [[NSString alloc] initWithFormat:@"desktop_capturer_%@_%ld", _isWindow ? @"window" : @"screen", self.uniqueId];
}
-(BOOL)isEqual:(id)object {
return [[((DesktopCaptureSourceMac *)object) uniqueKey] isEqualToString:[self uniqueKey]];
}
- (BOOL)isEqualTo:(id)object {
return [[((DesktopCaptureSourceMac *)object) uniqueKey] isEqualToString:[self uniqueKey]];
}
-(id)initWithSource:(tgcalls::DesktopCaptureSource)source isWindow:(BOOL)isWindow {
if (self = [super init]) {
_source = source;
_isWindow = isWindow;
}
return self;
}
@end
@interface DesktopCaptureSourceScopeMac ()
-(tgcalls::DesktopCaptureSourceData)getData;
-(tgcalls::DesktopCaptureSource)getSource;
@end
@implementation DesktopCaptureSourceScopeMac
-(id)initWithSource:(DesktopCaptureSourceMac *)source data:(DesktopCaptureSourceDataMac *)data {
if (self = [super init]) {
_data = data;
_source = source;
}
return self;
}
-(NSString *)cachedKey {
return [[NSString alloc] initWithFormat:@"%@:%@", _source.uniqueKey, _data.cachedKey];
}
-(tgcalls::DesktopCaptureSourceData)getData {
tgcalls::DesktopCaptureSourceData data{
/*.aspectSize = */{ (int)_data.aspectSize.width, (int)_data.aspectSize.height},
/*.fps = */_data.fps,
/*.captureMouse = */_data.captureMouse,
};
return data;
}
-(tgcalls::DesktopCaptureSource)getSource {
return [_source getSource];
}
@end
@implementation DesktopCaptureSourceManagerMac
{
std::map<std::string, tgcalls::DesktopCaptureSourceHelper> _cached;
std::unique_ptr<tgcalls::DesktopCaptureSourceManager> _manager;
BOOL _isWindow;
}
-(instancetype)init_s {
if (self = [super init]) {
_manager = std::make_unique<tgcalls::DesktopCaptureSourceManager>(tgcalls::DesktopCaptureType::Screen);
_isWindow = NO;
}
return self;
}
-(instancetype)init_w {
if (self = [super init]) {
_manager = std::make_unique<tgcalls::DesktopCaptureSourceManager>(tgcalls::DesktopCaptureType::Window);
_isWindow = YES;
}
return self;
}
-(NSArray<DesktopCaptureSourceMac *> *)list {
std::vector<tgcalls::DesktopCaptureSource> sources = _manager->sources();
NSMutableArray<DesktopCaptureSourceMac *> *macSources = [[NSMutableArray alloc] init];
for (auto i = sources.begin(); i != sources.end(); ++i) {
[macSources addObject:[[DesktopCaptureSourceMac alloc] initWithSource:*i isWindow:_isWindow]];
}
return macSources;
}
-(NSView *)createForScope:(DesktopCaptureSourceScopeMac*)scope {
auto i = _cached.find(std::string([scope.cachedKey UTF8String]));
if (i == _cached.end()) {
i = _cached.emplace(
std::string([scope.cachedKey UTF8String]),
tgcalls::DesktopCaptureSourceHelper([scope getSource], [scope getData])).first;
}
DesktopCaptureSourceViewMetal *view = [[DesktopCaptureSourceViewMetal alloc] initWithHelper:i->second];
if (scope.data.captureMouse) {
[view setVideoContentMode:kCAGravityResizeAspect];
}
return view;
}
-(void)start:(DesktopCaptureSourceScopeMac *)scope {
const auto i = _cached.find(std::string([scope.cachedKey UTF8String]));
if (i != _cached.end()) {
i->second.start();
}
}
-(void)stop:(DesktopCaptureSourceScopeMac *)scope {
const auto i = _cached.find(std::string([scope.cachedKey UTF8String]));
if (i != _cached.end()) {
i->second.stop();
}
}
-(void)dealloc {
for (auto &[key, helper] : _cached) {
helper.stop();
}
_manager.reset();
}
@end

View file

@ -0,0 +1,27 @@
#ifndef SCREEN_CAPTURER_H
#define SCREEN_CAPTURER_H
#ifndef WEBRTC_IOS
#import <Foundation/Foundation.h>
#import "api/video/video_sink_interface.h"
#import "api/media_stream_interface.h"
#import "rtc_base/time_utils.h"
#import "api/video/video_sink_interface.h"
#import "api/media_stream_interface.h"
#import "sdk/objc/native/src/objc_video_track_source.h"
#import "sdk/objc/native/src/objc_frame_buffer.h"
#import "pc/video_track_source_proxy.h"
#import "tgcalls/platform/darwin/VideoCameraCapturerMac.h"
#import "tgcalls/desktop_capturer/DesktopCaptureSource.h"
@interface DesktopSharingCapturer : NSObject<CapturerInterface>
- (instancetype)initWithSource:(rtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)trackSource captureSource:(tgcalls::DesktopCaptureSource)captureSource;
@end
#endif //WEBRTC_IOS
#endif

View file

@ -0,0 +1,117 @@
#import "DesktopSharingCapturer.h"
#include "modules/desktop_capture/mac/screen_capturer_mac.h"
#include "modules/desktop_capture/desktop_and_cursor_composer.h"
#include "modules/desktop_capture/desktop_capturer_differ_wrapper.h"
#include "third_party/libyuv/include/libyuv.h"
#include "api/video/i420_buffer.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "third_party/libyuv/include/libyuv.h"
#include "tgcalls/desktop_capturer/DesktopCaptureSource.h"
#include "tgcalls/desktop_capturer/DesktopCaptureSourceHelper.h"
#include "tgcalls/desktop_capturer/DesktopCaptureSourceManager.h"
#include "DarwinVideoSource.h"
#import "helpers/RTCDispatcher+Private.h"
#import <QuartzCore/QuartzCore.h>
static RTCVideoFrame *customToObjCVideoFrame(const webrtc::VideoFrame &frame, RTCVideoRotation &rotation) {
rotation = RTCVideoRotation(frame.rotation());
RTCVideoFrame *videoFrame =
[[RTCVideoFrame alloc] initWithBuffer:webrtc::ToObjCVideoFrameBuffer(frame.video_frame_buffer())
rotation:rotation
timeStampNs:frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec];
videoFrame.timeStamp = frame.timestamp();
return videoFrame;
}
static tgcalls::DarwinVideoTrackSource *getObjCVideoSource(const rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> nativeSource) {
webrtc::VideoTrackSourceProxy *proxy_source =
static_cast<webrtc::VideoTrackSourceProxy *>(nativeSource.get());
return static_cast<tgcalls::DarwinVideoTrackSource *>(proxy_source->internal());
}
class RendererAdapterImpl : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
public:
RendererAdapterImpl(void (^frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation)) {
_frameReceived = [frameReceived copy];
}
void OnFrame(const webrtc::VideoFrame& nativeVideoFrame) override {
RTCVideoRotation rotation = RTCVideoRotation_0;
RTCVideoFrame* videoFrame = customToObjCVideoFrame(nativeVideoFrame, rotation);
CGSize currentSize = (videoFrame.rotation % 180 == 0) ? CGSizeMake(videoFrame.width, videoFrame.height) : CGSizeMake(videoFrame.height, videoFrame.width);
if (_frameReceived) {
_frameReceived(currentSize, videoFrame, rotation);
}
}
private:
void (^_frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation);
};
@implementation DesktopSharingCapturer {
absl::optional<tgcalls::DesktopCaptureSourceHelper> renderer;
std::shared_ptr<RendererAdapterImpl> _sink;
BOOL _isPaused;
}
- (instancetype)initWithSource:(rtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)trackSource captureSource:(tgcalls::DesktopCaptureSource)captureSource {
self = [super init];
if (self != nil) {
_sink.reset(new RendererAdapterImpl(^(CGSize size, RTCVideoFrame *videoFrame, RTCVideoRotation rotation) {
getObjCVideoSource(trackSource)->OnCapturedFrame(videoFrame);
}));
tgcalls::DesktopCaptureSourceData data{
/*.aspectSize = */{ 1920, 1080 },
/*.fps = */25,
/*.captureMouse = */true,
};
renderer.emplace(captureSource, data);
renderer->setOutput(_sink);
}
return self;
}
-(void)setOnFatalError:(std::function<void ()>)error {
renderer->setOnFatalError(error);
}
-(void)setOnPause:(std::function<void (bool)>)pause {
renderer->setOnPause(pause);
}
-(void)start {
renderer->start();
}
-(void)stop {
renderer->stop();
}
- (void)setIsEnabled:(bool)isEnabled {
BOOL updated = _isPaused != !isEnabled;
_isPaused = !isEnabled;
if (updated) {
if (isEnabled) {
renderer->start();
} else {
renderer->stop();
}
}
}
- (void)setPreferredCaptureAspectRatio:(float)aspectRatio {
}
- (void)setUncroppedSink:(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame> >)sink {
renderer->setSecondaryOutput(sink);
}
@end

View file

@ -0,0 +1,12 @@
#ifndef TGCALLS_EXTRACT_CV_PIXEL_BUFFER_H
#define TGCALLS_EXTRACT_CV_PIXEL_BUFFER_H
#include "platform/PlatformInterface.h"
namespace tgcalls {
webrtc::scoped_refptr<webrtc::VideoFrameBuffer> extractCVPixelBuffer(void *data);
} // namespace tgcalls
#endif

View file

@ -0,0 +1,14 @@
#include "ExtractCVPixelBuffer.h"
#include "sdk/objc/native/src/objc_frame_buffer.h"
#import "sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.h"
namespace tgcalls {
webrtc::scoped_refptr<webrtc::VideoFrameBuffer> extractCVPixelBuffer(void *data) {
CVPixelBufferRef pixelBuffer = (CVPixelBufferRef)(void *)data;
return rtc::make_ref_counted<webrtc::ObjCFrameBuffer>([[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer]);
}
}

View file

@ -0,0 +1,25 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
/* This file is borrowed from sdk/objc/components/video_codec/RTCCodecSpecificInfoH264+Private.h */
#import "RTCCodecSpecificInfoH265.h"
#include "modules/video_coding/include/video_codec_interface.h"
NS_ASSUME_NONNULL_BEGIN
/* Interfaces for converting to/from internal C++ formats. */
@interface RTCCodecSpecificInfoH265 ()
- (webrtc::CodecSpecificInfo)nativeCodecSpecificInfo;
@end
NS_ASSUME_NONNULL_END

View file

@ -0,0 +1,28 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
/* This file is borrowed from sdk/objc/components/video_codec/RTCCodecSpecificInfoH264.h. */
#import <Foundation/Foundation.h>
#import "RTCCodecSpecificInfo.h"
#import "RTCMacros.h"
/** Class for H265 specific config. */
typedef NS_ENUM(NSUInteger, RTCH265PacketizationMode) {
RTCH265PacketizationModeNonInterleaved = 0, // Mode 1 - STAP-A, FU-A is allowed
RTCH265PacketizationModeSingleNalUnit // Mode 0 - only single NALU allowed
};
RTC_OBJC_EXPORT
@interface RTCCodecSpecificInfoH265 : NSObject <RTCCodecSpecificInfo>
@property(nonatomic, assign) RTCH265PacketizationMode packetizationMode;
@end

View file

@ -0,0 +1,27 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
/* This file is borrowed from sdk/objc/components/video_codec/RTCCodecSpecificInfoH264.mm */
#import "RTCCodecSpecificInfoH265+Private.h"
// H265 specific settings.
@implementation RTCCodecSpecificInfoH265
@synthesize packetizationMode = _packetizationMode;
- (webrtc::CodecSpecificInfo)nativeCodecSpecificInfo {
webrtc::CodecSpecificInfo codecSpecificInfo;
codecSpecificInfo.codecType = webrtc::kVideoCodecH265;
//codecSpecificInfo.codecSpecific.H265.packetization_mode = (webrtc::H265PacketizationMode)_packetizationMode;
return codecSpecificInfo;
}
@end

View file

@ -0,0 +1,16 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import <Foundation/Foundation.h>
#import "RTCMacros.h"
RTC_OBJC_EXPORT extern NSString *const kRTCVideoCodecH265Name;
RTC_OBJC_EXPORT extern NSString *const kRTCLevel31Main;

View file

@ -0,0 +1,18 @@
/*
* Copyright 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
#import "RTCH265ProfileLevelId.h"
#include "media/base/media_constants.h"
NSString *const kRTCVideoCodecH265Name = @(cricket::kH265CodecName);
// TODO(jianjunz): This is value is not correct.
NSString *const kRTCLevel31Main = @"4d001f";

View file

@ -0,0 +1,19 @@
#import <Foundation/Foundation.h>
@interface SQueue : NSObject
+ (SQueue * _Nonnull)mainQueue;
+ (SQueue * _Nonnull)concurrentDefaultQueue;
+ (SQueue * _Nonnull)concurrentBackgroundQueue;
+ (SQueue * _Nonnull)wrapConcurrentNativeQueue:(dispatch_queue_t _Nonnull)nativeQueue;
- (void)dispatch:(dispatch_block_t _Nonnull)block;
- (void)dispatchSync:(dispatch_block_t _Nonnull)block;
- (void)dispatch:(dispatch_block_t _Nonnull)block synchronous:(bool)synchronous;
- (dispatch_queue_t _Nonnull)_dispatch_queue;
- (bool)isCurrentQueue;
@end

View file

@ -0,0 +1,124 @@
#import "SQueue.h"
static const void *SQueueSpecificKey = &SQueueSpecificKey;
@interface SQueue ()
{
dispatch_queue_t _queue;
void *_queueSpecific;
bool _specialIsMainQueue;
}
@end
@implementation SQueue
+ (SQueue *)mainQueue
{
static SQueue *queue = nil;
static dispatch_once_t onceToken;
dispatch_once(&onceToken, ^
{
queue = [[SQueue alloc] initWithNativeQueue:dispatch_get_main_queue() queueSpecific:NULL];
queue->_specialIsMainQueue = true;
});
return queue;
}
+ (SQueue *)concurrentDefaultQueue
{
static SQueue *queue = nil;
static dispatch_once_t onceToken;
dispatch_once(&onceToken, ^
{
queue = [[SQueue alloc] initWithNativeQueue:dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0) queueSpecific:NULL];
});
return queue;
}
+ (SQueue *)concurrentBackgroundQueue
{
static SQueue *queue = nil;
static dispatch_once_t onceToken;
dispatch_once(&onceToken, ^
{
queue = [[SQueue alloc] initWithNativeQueue:dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0) queueSpecific:NULL];
});
return queue;
}
+ (SQueue *)wrapConcurrentNativeQueue:(dispatch_queue_t)nativeQueue
{
return [[SQueue alloc] initWithNativeQueue:nativeQueue queueSpecific:NULL];
}
- (instancetype)init
{
dispatch_queue_t queue = dispatch_queue_create(NULL, NULL);
dispatch_queue_set_specific(queue, SQueueSpecificKey, (__bridge void *)self, NULL);
return [self initWithNativeQueue:queue queueSpecific:(__bridge void *)self];
}
- (instancetype)initWithNativeQueue:(dispatch_queue_t)queue queueSpecific:(void *)queueSpecific
{
self = [super init];
if (self != nil)
{
_queue = queue;
_queueSpecific = queueSpecific;
}
return self;
}
- (dispatch_queue_t)_dispatch_queue
{
return _queue;
}
- (void)dispatch:(dispatch_block_t)block
{
if (_queueSpecific != NULL && dispatch_get_specific(SQueueSpecificKey) == _queueSpecific)
block();
else if (_specialIsMainQueue && [NSThread isMainThread])
block();
else
dispatch_async(_queue, block);
}
- (void)dispatchSync:(dispatch_block_t)block
{
if (_queueSpecific != NULL && dispatch_get_specific(SQueueSpecificKey) == _queueSpecific)
block();
else if (_specialIsMainQueue && [NSThread isMainThread])
block();
else
dispatch_sync(_queue, block);
}
- (void)dispatch:(dispatch_block_t)block synchronous:(bool)synchronous {
if (_queueSpecific != NULL && dispatch_get_specific(SQueueSpecificKey) == _queueSpecific)
block();
else if (_specialIsMainQueue && [NSThread isMainThread])
block();
else {
if (synchronous) {
dispatch_sync(_queue, block);
} else {
dispatch_async(_queue, block);
}
}
}
- (bool)isCurrentQueue
{
if (_queueSpecific != NULL && dispatch_get_specific(SQueueSpecificKey) == _queueSpecific)
return true;
else if (_specialIsMainQueue && [NSThread isMainThread])
return true;
return false;
}
@end

View file

@ -0,0 +1,19 @@
//
// SQueueLocalObject.h
// SSignalKit
//
// Created by Mikhail Filimonov on 13.01.2021.
// Copyright © 2021 Telegram. All rights reserved.
//
#import <Foundation/Foundation.h>
#import "SQueue.h"
NS_ASSUME_NONNULL_BEGIN
@interface SQueueLocalObject : NSObject
-(id)initWithQueue:(SQueue *)queue generate:(id (^)(void))next;
-(void)with:(void (^)(id object))f;
@end
NS_ASSUME_NONNULL_END

View file

@ -0,0 +1,40 @@
//
// SQueueLocalObject.m
// SSignalKit
//
// Created by Mikhail Filimonov on 13.01.2021.
// Copyright © 2021 Telegram. All rights reserved.
//
#import "SQueueLocalObject.h"
@implementation SQueueLocalObject {
SQueue *_queue;
id valueRef;
}
-(id)initWithQueue:(SQueue *)queue generate:(id _Nonnull (^)(void))next {
if (self = [super init]) {
self->_queue = queue;
[queue dispatch:^{
self->valueRef = next();
}];
}
return self;
}
-(void)with:(void (^)(id object))f {
[self->_queue dispatch:^{
f(self->valueRef);
}];
}
-(void)dealloc {
__block id value = self->valueRef;
self->valueRef = nil;
[_queue dispatch:^{
value = nil;
}];
}
@end

View file

@ -0,0 +1,29 @@
//
// Capturer.h
// CoreMediaMacCapture
//
// Created by Mikhail Filimonov on 21.06.2021.
//
#import <Foundation/Foundation.h>
#import <CoreMedia/CoreMedia.h>
#import <AVFoundation/AVFoundation.h>
NS_ASSUME_NONNULL_BEGIN
typedef void(^renderBlock)(CMSampleBufferRef);
@interface TGCMIOCapturer : NSObject
-(id)initWithDeviceId:(AVCaptureDevice *)device;
-(void)start:(renderBlock)renderBlock;
-(void)stop;
@end
NS_ASSUME_NONNULL_END

View file

@ -0,0 +1,45 @@
//
// Capturer.m
// CoreMediaMacCapture
//
// Created by Mikhail Filimonov on 21.06.2021.
//
#import "TGCMIOCapturer.h"
#import "TGCMIODevice.h"
@interface TGCMIOCapturer ()
@end
@implementation TGCMIOCapturer
{
AVCaptureDevice * _captureDevice;
TGCMIODevice * _device;
}
-(id)initWithDeviceId:(AVCaptureDevice *)device {
if (self = [super init]) {
_captureDevice = device;
}
return self;
}
-(void)start:(renderBlock)renderBlock {
_device = [TGCMIODevice FindDeviceByUniqueId:_captureDevice];
[_device run:^(CMSampleBufferRef sampleBuffer) {
renderBlock(sampleBuffer);
}];
}
-(void)stop {
[_device stop];
}
-(void)dealloc {
}
@end

View file

@ -0,0 +1,31 @@
//
// CoreMediaVideoHAL.h
// CoreMediaMacCapture
//
// Created by Mikhail Filimonov on 21.06.2021.
//
#import <Foundation/Foundation.h>
#import <CoreMedia/CoreMedia.h>
#import <CoreMediaIO/CMIOHardware.h>
#import <CoreMedia/CoreMedia.h>
#import <AVFoundation/AVFoundation.h>
NS_ASSUME_NONNULL_BEGIN
typedef void(^RenderBlock)(CMSampleBufferRef);
@interface TGCMIODevice : NSObject
+(TGCMIODevice * __nullable)FindDeviceByUniqueId:(AVCaptureDevice *)device;
-(void)run:(RenderBlock)render;
-(void)stop;
-(CMIODeviceID)cmioDevice;
@end
NS_ASSUME_NONNULL_END

View file

@ -0,0 +1,193 @@
//
// CoreMediaVideoHAL.m
// CoreMediaMacCapture
//
// Created by Mikhail Filimonov on 21.06.2021.
//
#import "TGCMIODevice.h"
#import <CoreMediaIO/CMIOHardware.h>
#import <CoreMediaIO/CMIOHardwareStream.h>
#import <CoreMedia/CoreMedia.h>
#import <AVFoundation/AVFoundation.h>
@interface TGCMIODevice ()
{
CMIODeviceID _deviceId;
CMIOStreamID _streamId;
CMSimpleQueueRef _queueRef;
RenderBlock _renderBlock;
}
-(CMSimpleQueueRef)queue;
-(RenderBlock)block;
@end
static void handleStreamQueueAltered(CMIOStreamID streamID, void* token, void* refCon) {
CMSampleBufferRef sb = 0;
TGCMIODevice *renderBlock = (__bridge TGCMIODevice *)refCon;
CMSimpleQueueRef queueRef = [renderBlock queue];
while(0 != (sb = (CMSampleBufferRef)CMSimpleQueueDequeue(queueRef))) {
renderBlock.block(sb);
CFRelease(sb);
}
}
OSStatus GetPropertyData(CMIOObjectID objID, int32_t sel, CMIOObjectPropertyScope scope,
UInt32 qualifierDataSize, const void* qualifierData, UInt32 dataSize,
UInt32& dataUsed, void* data) {
CMIOObjectPropertyAddress addr={ (CMIOObjectPropertySelector)sel, scope,
kCMIOObjectPropertyElementMaster };
return CMIOObjectGetPropertyData(objID, &addr, qualifierDataSize, qualifierData,
dataSize, &dataUsed, data);
}
OSStatus GetPropertyData(CMIOObjectID objID, int32_t selector, UInt32 qualifierDataSize,
const void* qualifierData, UInt32 dataSize, UInt32& dataUsed,
void* data) {
return GetPropertyData(objID, selector, 0, qualifierDataSize,
qualifierData, dataSize, dataUsed, data);
}
OSStatus GetPropertyDataSize(CMIOObjectID objID, int32_t sel,
CMIOObjectPropertyScope scope, uint32_t& size) {
CMIOObjectPropertyAddress addr={ (CMIOObjectPropertySelector)sel, scope,
kCMIOObjectPropertyElementMaster };
return CMIOObjectGetPropertyDataSize(objID, &addr, 0, 0, &size);
}
OSStatus GetPropertyDataSize(CMIOObjectID objID, int32_t selector, uint32_t& size) {
return GetPropertyDataSize(objID, selector, 0, size);
}
OSStatus GetNumberDevices(uint32_t& cnt) {
if(0 != GetPropertyDataSize(kCMIOObjectSystemObject, kCMIOHardwarePropertyDevices, cnt))
return -1;
cnt /= sizeof(CMIODeviceID);
return 0;
}
OSStatus GetDevices(uint32_t& cnt, CMIODeviceID* pDevs) {
OSStatus status;
uint32_t numberDevices = 0, used = 0;
if((status = GetNumberDevices(numberDevices)) < 0)
return status;
if(numberDevices > (cnt = numberDevices))
return -1;
uint32_t size = numberDevices * sizeof(CMIODeviceID);
return GetPropertyData(kCMIOObjectSystemObject, kCMIOHardwarePropertyDevices,
0, NULL, size, used, pDevs);
}
template< const int C_Size >
OSStatus GetDeviceStrProp(CMIOObjectID objID, CMIOObjectPropertySelector sel,
char (&pValue)[C_Size]) {
CFStringRef answer = NULL;
UInt32 dataUsed= 0;
OSStatus status = GetPropertyData(objID, sel, 0, NULL, sizeof(answer),
dataUsed, &answer);
if(0 == status)// SUCCESS
CFStringCopyUTF8String(answer, pValue);
return status;
}
template< const int C_Size >
Boolean CFStringCopyUTF8String(CFStringRef aString, char (&pText)[C_Size]) {
CFIndex length = CFStringGetLength(aString);
if(sizeof(pText) < (length + 1))
return false;
CFIndex maxSize = CFStringGetMaximumSizeForEncoding(length, kCFStringEncodingUTF8);
return CFStringGetCString(aString, pText, maxSize, kCFStringEncodingUTF8);
}
uint32_t GetNumberInputStreams(CMIODeviceID devID)
{
uint32 size = 0;
GetPropertyDataSize(devID, kCMIODevicePropertyStreams,
kCMIODevicePropertyScopeInput, size);
return size / sizeof(CMIOStreamID);
}
OSStatus GetInputStreams(CMIODeviceID devID, uint32_t&
ioNumberStreams, CMIOStreamID* streamList)
{
ioNumberStreams = MIN(GetNumberInputStreams(devID), ioNumberStreams);
uint32_t size = ioNumberStreams * sizeof(CMIOStreamID);
uint32_t dataUsed = 0;
OSStatus err = GetPropertyData(devID, kCMIODevicePropertyStreams,
kCMIODevicePropertyScopeInput, 0,
NULL, size, dataUsed, streamList);
if(0 != err)
return err;
ioNumberStreams = size / sizeof(CMIOStreamID);
CMIOStreamID* firstItem = &(streamList[0]);
CMIOStreamID* lastItem = firstItem + ioNumberStreams;
//std::sort(firstItem, lastItem);
return 0;
}
@implementation TGCMIODevice
-(id)initWithDeviceId:(CMIODeviceID)deviceId streamId:(CMIOStreamID)streamId {
if (self = [super init]) {
_deviceId = deviceId;
_streamId = streamId;
}
return self;
}
-(CMIODeviceID)cmioDevice {
return _deviceId;
}
+(TGCMIODevice *)FindDeviceByUniqueId:(AVCaptureDevice *)device {
NSNumber *_connectionID = ((NSNumber *)[device valueForKey:@"_connectionID"]);
CMIODeviceID deviceId = (CMIODeviceID)[_connectionID intValue];
uint32_t numStreams = GetNumberInputStreams(deviceId);
CMIOStreamID* pStreams = (CMIOStreamID*)alloca(numStreams * sizeof(CMIOStreamID));
GetInputStreams(deviceId, numStreams, pStreams);
if (numStreams <= 0)
return nil;
CMIOStreamID streamId = pStreams[0];
return [[TGCMIODevice alloc] initWithDeviceId:deviceId streamId:streamId];
}
-(CMSimpleQueueRef)queue {
return _queueRef;
}
-(RenderBlock)block {
return _renderBlock;
}
-(void)run:(RenderBlock)render {
_renderBlock = render;
CMIOStreamCopyBufferQueue(_streamId, handleStreamQueueAltered, (void*)CFBridgingRetain(self), &_queueRef);
CMIODeviceStartStream(_deviceId, _streamId);
}
-(void)stop {
CMIODeviceStopStream(_deviceId, _streamId);
CMIOStreamCopyBufferQueue(_streamId, nil, nil, &_queueRef);
if (_queueRef)
CFRelease(_queueRef);
}
-(void)dealloc {
[self stop];
}
@end

View file

@ -0,0 +1,15 @@
#ifndef TGRTCCVPIXELBUFFER_H
#define TGRTCCVPIXELBUFFER_H
#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
@interface TGRTCCVPixelBuffer : RTCCVPixelBuffer
@property (nonatomic) bool shouldBeMirrored;
@property (nonatomic) int deviceRelativeVideoRotation;
- (void)storeSampleBufferReference:(CMSampleBufferRef _Nonnull)sampleBuffer;
@end
#endif

View file

@ -0,0 +1,28 @@
#import "TGRTCCVPixelBuffer.h"
@interface TGRTCCVPixelBuffer () {
CMSampleBufferRef _sampleBuffer;
}
@end
@implementation TGRTCCVPixelBuffer
- (void)dealloc {
if (_sampleBuffer) {
CFRelease(_sampleBuffer);
}
}
- (void)storeSampleBufferReference:(CMSampleBufferRef _Nonnull)sampleBuffer {
if (_sampleBuffer) {
CFRelease(_sampleBuffer);
_sampleBuffer = nil;
}
if (sampleBuffer) {
_sampleBuffer = (CMSampleBufferRef)CFRetain(sampleBuffer);
}
}
@end

View file

@ -0,0 +1,25 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import <Foundation/Foundation.h>
#import "RTCMacros.h"
#import "RTCVideoDecoderFactory.h"
NS_ASSUME_NONNULL_BEGIN
/** This decoder factory include support for all codecs bundled with WebRTC. If using custom
* codecs, create custom implementations of RTCVideoEncoderFactory and RTCVideoDecoderFactory.
*/
RTC_OBJC_EXPORT
@interface TGRTCDefaultVideoDecoderFactory : NSObject <RTCVideoDecoderFactory>
@end
NS_ASSUME_NONNULL_END

View file

@ -0,0 +1,116 @@
/*
* Copyright 2017 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import "TGRTCDefaultVideoDecoderFactory.h"
#import "RTCH264ProfileLevelId.h"
#import "TGRTCVideoDecoderH264.h"
#import "api/video_codec/RTCVideoCodecConstants.h"
#import "api/video_codec/RTCVideoDecoderVP8.h"
#import "base/RTCVideoCodecInfo.h"
#if defined(RTC_ENABLE_VP9)
#import "api/video_codec/RTCVideoDecoderVP9.h"
#endif
#if !defined(WEBRTC_DISABLE_H265)
#import "RTCH265ProfileLevelId.h"
#import "TGRTCVideoDecoderH265.h"
#endif
#import "sdk/objc/api/video_codec/RTCWrappedNativeVideoDecoder.h"
#include "modules/video_coding/codecs/h264/include/h264.h"
@implementation TGRTCDefaultVideoDecoderFactory
- (NSArray<RTCVideoCodecInfo *> *)supportedCodecs {
NSDictionary<NSString *, NSString *> *constrainedHighParams = @{
@"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedHigh,
@"level-asymmetry-allowed" : @"1",
@"packetization-mode" : @"1",
};
RTCVideoCodecInfo *constrainedHighInfo =
[[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH264Name
parameters:constrainedHighParams];
NSDictionary<NSString *, NSString *> *constrainedBaselineParams = @{
@"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedBaseline,
@"level-asymmetry-allowed" : @"1",
@"packetization-mode" : @"1",
};
RTCVideoCodecInfo *constrainedBaselineInfo =
[[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH264Name
parameters:constrainedBaselineParams];
RTCVideoCodecInfo *vp8Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecVp8Name];
#if defined(RTC_ENABLE_VP9)
RTCVideoCodecInfo *vp9Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecVp9Name];
#endif
#if !defined(WEBRTC_DISABLE_H265)
RTCVideoCodecInfo *h265Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH265Name];
#endif
NSMutableArray<RTCVideoCodecInfo *> *result = [[NSMutableArray alloc] initWithArray:@[
constrainedHighInfo,
constrainedBaselineInfo,
vp8Info,
#if defined(RTC_ENABLE_VP9)
vp9Info,
#endif
]];
#if !defined(WEBRTC_DISABLE_H265)
#ifdef WEBRTC_IOS
if (@available(iOS 11.0, *)) {
[result addObject:h265Info];
}
#else // WEBRTC_IOS
if (@available(macOS 10.13, *)) {
[result addObject:h265Info];
}
#endif // WEBRTC_IOS
#endif
return result;
}
- (id<RTCVideoDecoder>)createDecoder:(RTCVideoCodecInfo *)info {
if ([info.name isEqualToString:kRTCVideoCodecH264Name]) {
return [[TGRTCVideoDecoderH264 alloc] init];
} else if ([info.name isEqualToString:kRTCVideoCodecVp8Name]) {
return [RTCVideoDecoderVP8 vp8Decoder];
}
#if defined(RTC_ENABLE_VP9)
if ([info.name isEqualToString:kRTCVideoCodecVp9Name]) {
return [RTCVideoDecoderVP9 vp9Decoder];
}
#endif
#if !defined(WEBRTC_DISABLE_H265)
#ifdef WEBRTC_IOS
if (@available(iOS 11.0, *)) {
if ([info.name isEqualToString:kRTCVideoCodecH265Name]) {
return [[TGRTCVideoDecoderH265 alloc] init];
}
}
#else // WEBRTC_IOS
if (@available(macOS 10.13, *)) {
if ([info.name isEqualToString:kRTCVideoCodecH265Name]) {
return [[TGRTCVideoDecoderH265 alloc] init];
}
}
#endif // WEBRTC_IOS
#endif // !DISABLE_H265
return nil;
}
@end

View file

@ -0,0 +1,32 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import <Foundation/Foundation.h>
#import "RTCMacros.h"
#import "RTCVideoEncoderFactory.h"
NS_ASSUME_NONNULL_BEGIN
/** This encoder factory include support for all codecs bundled with WebRTC. If using custom
* codecs, create custom implementations of RTCVideoEncoderFactory and RTCVideoDecoderFactory.
*/
RTC_OBJC_EXPORT
@interface TGRTCDefaultVideoEncoderFactory : NSObject <RTCVideoEncoderFactory>
@property(nonatomic, retain) RTCVideoCodecInfo *preferredCodec;
- (instancetype)initWithPreferHardwareH264:(bool)preferHardwareH264 preferX264:(bool)preferX264;
+ (NSArray<RTCVideoCodecInfo *> *)supportedCodecs;
@end
NS_ASSUME_NONNULL_END

View file

@ -0,0 +1,172 @@
/*
* Copyright 2017 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import "TGRTCDefaultVideoEncoderFactory.h"
#import "RTCH264ProfileLevelId.h"
#import "TGRTCVideoEncoderH264.h"
#import "api/video_codec/RTCVideoCodecConstants.h"
#import "api/video_codec/RTCVideoEncoderVP8.h"
#import "base/RTCVideoCodecInfo.h"
#if defined(RTC_ENABLE_VP9)
#import "api/video_codec/RTCVideoEncoderVP9.h"
#endif
#ifndef WEBRTC_DISABLE_H265
#import "RTCH265ProfileLevelId.h"
#import "TGRTCVideoEncoderH265.h"
#endif
#import "sdk/objc/api/video_codec/RTCWrappedNativeVideoEncoder.h"
#import "modules/video_coding/codecs/h264/include/h264.h"
#import "h264_encoder_impl.h"
@interface TGRTCDefaultVideoEncoderFactory () {
bool _preferHardwareH264;
bool _preferX264;
}
@end
@implementation TGRTCDefaultVideoEncoderFactory
@synthesize preferredCodec;
- (instancetype)initWithPreferHardwareH264:(bool)preferHardwareH264 preferX264:(bool)preferX264 {
self = [super init];
if (self != nil) {
_preferHardwareH264 = preferHardwareH264;
_preferX264 = preferX264;
}
return self;
}
+ (NSArray<RTCVideoCodecInfo *> *)supportedCodecs {
NSDictionary<NSString *, NSString *> *constrainedHighParams = @{
@"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedHigh,
@"level-asymmetry-allowed" : @"1",
@"packetization-mode" : @"1",
};
RTCVideoCodecInfo *constrainedHighInfo =
[[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH264Name
parameters:constrainedHighParams];
NSDictionary<NSString *, NSString *> *constrainedBaselineParams = @{
@"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedBaseline,
@"level-asymmetry-allowed" : @"1",
@"packetization-mode" : @"1",
};
RTCVideoCodecInfo *constrainedBaselineInfo =
[[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH264Name
parameters:constrainedBaselineParams];
RTCVideoCodecInfo *vp8Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecVp8Name];
#if defined(RTC_ENABLE_VP9)
RTCVideoCodecInfo *vp9Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecVp9Name];
#endif
#if !defined(WEBRTC_DISABLE_H265)
RTCVideoCodecInfo *h265Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH265Name];
#endif
NSMutableArray *result = [[NSMutableArray alloc] initWithArray:@[
constrainedHighInfo,
constrainedBaselineInfo,
vp8Info,
#if defined(RTC_ENABLE_VP9)
vp9Info,
#endif
]];
#if !defined(WEBRTC_DISABLE_H265)
#ifdef WEBRTC_IOS
if (@available(iOS 11.0, *)) {
if ([[AVAssetExportSession allExportPresets] containsObject:AVAssetExportPresetHEVCHighestQuality]) {
[result addObject:h265Info];
}
}
#else // WEBRTC_IOS
if (@available(macOS 10.13, *)) {
if ([[AVAssetExportSession allExportPresets] containsObject:AVAssetExportPresetHEVCHighestQuality]) {
[result addObject:h265Info];
}
}
#endif // WEBRTC_IOS
#endif
return result;
}
- (id<RTCVideoEncoder>)createEncoder:(RTCVideoCodecInfo *)info {
if ([info.name isEqualToString:kRTCVideoCodecH264Name]) {
if (_preferHardwareH264) {
return [[TGRTCVideoEncoderH264 alloc] initWithCodecInfo:info];
} else {
webrtc::SdpVideoFormat videoFormat(info.name.UTF8String);
for (NSString *key in info.parameters) {
videoFormat.parameters.insert(std::make_pair(key.UTF8String, info.parameters[key].UTF8String));
}
cricket::VideoCodec videoCodec = cricket::CreateVideoCodec(videoFormat);
#ifdef TGCALLS_ENABLE_X264
if (_preferX264) {
return [[RTC_OBJC_TYPE(RTCWrappedNativeVideoEncoder) alloc] initWithNativeEncoder:std::make_unique<webrtc::H264EncoderX264Impl>(videoCodec)];
} else {
#endif
return [[RTC_OBJC_TYPE(RTCWrappedNativeVideoEncoder) alloc] initWithNativeEncoder:std::unique_ptr<webrtc::VideoEncoder>(webrtc::H264Encoder::Create(videoCodec))];
#ifdef TGCALLS_ENABLE_X264
}
#endif
}
} else if ([info.name isEqualToString:kRTCVideoCodecVp8Name]) {
return [RTCVideoEncoderVP8 vp8Encoder];
}
#if defined(RTC_ENABLE_VP9)
if ([info.name isEqualToString:kRTCVideoCodecVp9Name]) {
return [RTCVideoEncoderVP9 vp9Encoder];
}
#endif
#if !defined(WEBRTC_DISABLE_H265)
#ifdef WEBRTC_IOS
if (@available(iOS 11, *)) {
if ([info.name isEqualToString:kRTCVideoCodecH265Name]) {
return [[RTCVideoEncoderH265 alloc] initWithCodecInfo:info];
}
}
#else // WEBRTC_IOS
if (@available(macOS 10.13, *)) {
if ([info.name isEqualToString:kRTCVideoCodecH265Name]) {
return [[RTCVideoEncoderH265 alloc] initWithCodecInfo:info];
}
}
#endif // WEBRTC_IOS
#endif // !DISABLE_H265
return nil;
}
- (NSArray<RTCVideoCodecInfo *> *)supportedCodecs {
NSMutableArray<RTCVideoCodecInfo *> *codecs = [[[self class] supportedCodecs] mutableCopy];
NSMutableArray<RTCVideoCodecInfo *> *orderedCodecs = [NSMutableArray array];
NSUInteger index = [codecs indexOfObject:self.preferredCodec];
if (index != NSNotFound) {
[orderedCodecs addObject:[codecs objectAtIndex:index]];
[codecs removeObjectAtIndex:index];
}
[orderedCodecs addObjectsFromArray:codecs];
return [orderedCodecs copy];
}
@end

View file

@ -0,0 +1,18 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import <Foundation/Foundation.h>
#import "RTCMacros.h"
#import "RTCVideoDecoder.h"
RTC_OBJC_EXPORT
@interface RTC_OBJC_TYPE (TGRTCVideoDecoderH264) : NSObject <RTC_OBJC_TYPE(RTCVideoDecoder)>
@end

View file

@ -0,0 +1,292 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
#import "TGRTCVideoDecoderH264.h"
#import <VideoToolbox/VideoToolbox.h>
#import "base/RTCVideoFrame.h"
#import "base/RTCVideoFrameBuffer.h"
#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
#import "helpers.h"
#import "helpers/scoped_cftyperef.h"
#if defined(WEBRTC_IOS)
#import "helpers/UIDevice+RTCDevice.h"
#endif
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/time_utils.h"
#include "sdk/objc/components/video_codec/nalu_rewriter.h"
// Struct that we pass to the decoder per frame to decode. We receive it again
// in the decoder callback.
struct RTCFrameDecodeParams {
RTCFrameDecodeParams(RTCVideoDecoderCallback cb, int64_t ts) : callback(cb), timestamp(ts) {}
RTCVideoDecoderCallback callback;
int64_t timestamp;
};
@interface RTC_OBJC_TYPE (TGRTCVideoDecoderH264)
() - (void)setError : (OSStatus)error;
@end
// This is the callback function that VideoToolbox calls when decode is
// complete.
static void decompressionOutputCallback(void *decoderRef,
void *params,
OSStatus status,
VTDecodeInfoFlags infoFlags,
CVImageBufferRef imageBuffer,
CMTime timestamp,
CMTime duration) {
std::unique_ptr<RTCFrameDecodeParams> decodeParams(
reinterpret_cast<RTCFrameDecodeParams *>(params));
if (status != noErr) {
RTC_OBJC_TYPE(TGRTCVideoDecoderH264) *decoder =
(__bridge RTC_OBJC_TYPE(TGRTCVideoDecoderH264) *)decoderRef;
[decoder setError:status];
RTC_LOG(LS_ERROR) << "Failed to decode frame. Status: " << status;
return;
}
// TODO(tkchin): Handle CVO properly.
@autoreleasepool {
RTC_OBJC_TYPE(RTCCVPixelBuffer) *frameBuffer =
[[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:imageBuffer];
RTC_OBJC_TYPE(RTCVideoFrame) *decodedFrame = [[RTC_OBJC_TYPE(RTCVideoFrame) alloc]
initWithBuffer:frameBuffer
rotation:RTCVideoRotation_0
timeStampNs:CMTimeGetSeconds(timestamp) * rtc::kNumNanosecsPerSec];
decodedFrame.timeStamp = decodeParams->timestamp;
decodeParams->callback(decodedFrame);
}
}
// Decoder.
@implementation RTC_OBJC_TYPE (TGRTCVideoDecoderH264) {
CMVideoFormatDescriptionRef _videoFormat;
CMMemoryPoolRef _memoryPool;
VTDecompressionSessionRef _decompressionSession;
RTCVideoDecoderCallback _callback;
OSStatus _error;
}
- (instancetype)init {
self = [super init];
if (self) {
_memoryPool = CMMemoryPoolCreate(nil);
}
return self;
}
- (void)dealloc {
CMMemoryPoolInvalidate(_memoryPool);
CFRelease(_memoryPool);
[self destroyDecompressionSession];
[self setVideoFormat:nullptr];
}
- (NSInteger)startDecodeWithNumberOfCores:(int)numberOfCores {
return WEBRTC_VIDEO_CODEC_OK;
}
- (NSInteger)decode:(RTC_OBJC_TYPE(RTCEncodedImage) *)inputImage
missingFrames:(BOOL)missingFrames
codecSpecificInfo:(nullable id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)>)info
renderTimeMs:(int64_t)renderTimeMs {
RTC_DCHECK(inputImage.buffer);
if (_error != noErr) {
RTC_LOG(LS_WARNING) << "Last frame decode failed.";
_error = noErr;
return WEBRTC_VIDEO_CODEC_ERROR;
}
rtc::ScopedCFTypeRef<CMVideoFormatDescriptionRef> inputFormat =
rtc::ScopedCF(webrtc::CreateVideoFormatDescription((uint8_t *)inputImage.buffer.bytes,
inputImage.buffer.length));
if (inputFormat) {
// Check if the video format has changed, and reinitialize decoder if
// needed.
if (!CMFormatDescriptionEqual(inputFormat.get(), _videoFormat)) {
[self setVideoFormat:inputFormat.get()];
int resetDecompressionSessionError = [self resetDecompressionSession];
if (resetDecompressionSessionError != WEBRTC_VIDEO_CODEC_OK) {
return resetDecompressionSessionError;
}
}
}
if (!_videoFormat) {
// We received a frame but we don't have format information so we can't
// decode it.
// This can happen after backgrounding. We need to wait for the next
// sps/pps before we can resume so we request a keyframe by returning an
// error.
RTC_LOG(LS_WARNING) << "Missing video format. Frame with sps/pps required.";
return WEBRTC_VIDEO_CODEC_ERROR;
}
CMSampleBufferRef sampleBuffer = nullptr;
if (!webrtc::H264AnnexBBufferToCMSampleBuffer((uint8_t *)inputImage.buffer.bytes,
inputImage.buffer.length,
_videoFormat,
&sampleBuffer,
_memoryPool)) {
return WEBRTC_VIDEO_CODEC_ERROR;
}
RTC_DCHECK(sampleBuffer);
VTDecodeFrameFlags decodeFlags = kVTDecodeFrame_EnableAsynchronousDecompression;
std::unique_ptr<RTCFrameDecodeParams> frameDecodeParams;
frameDecodeParams.reset(new RTCFrameDecodeParams(_callback, inputImage.timeStamp));
OSStatus status = VTDecompressionSessionDecodeFrame(
_decompressionSession, sampleBuffer, decodeFlags, frameDecodeParams.release(), nullptr);
#if defined(WEBRTC_IOS)
// Re-initialize the decoder if we have an invalid session while the app is
// active or decoder malfunctions and retry the decode request.
if ((status == kVTInvalidSessionErr || status == kVTVideoDecoderMalfunctionErr) &&
[self resetDecompressionSession] == WEBRTC_VIDEO_CODEC_OK) {
RTC_LOG(LS_INFO) << "Failed to decode frame with code: " << status
<< " retrying decode after decompression session reset";
frameDecodeParams.reset(new RTCFrameDecodeParams(_callback, inputImage.timeStamp));
status = VTDecompressionSessionDecodeFrame(
_decompressionSession, sampleBuffer, decodeFlags, frameDecodeParams.release(), nullptr);
}
#endif
CFRelease(sampleBuffer);
if (status != noErr) {
RTC_LOG(LS_ERROR) << "Failed to decode frame with code: " << status;
return WEBRTC_VIDEO_CODEC_ERROR;
}
return WEBRTC_VIDEO_CODEC_OK;
}
- (void)setCallback:(RTCVideoDecoderCallback)callback {
_callback = callback;
}
- (void)setError:(OSStatus)error {
_error = error;
}
- (NSInteger)releaseDecoder {
// Need to invalidate the session so that callbacks no longer occur and it
// is safe to null out the callback.
[self destroyDecompressionSession];
[self setVideoFormat:nullptr];
_callback = nullptr;
return WEBRTC_VIDEO_CODEC_OK;
}
#pragma mark - Private
- (int)resetDecompressionSession {
[self destroyDecompressionSession];
// Need to wait for the first SPS to initialize decoder.
if (!_videoFormat) {
return WEBRTC_VIDEO_CODEC_OK;
}
// Set keys for OpenGL and IOSurface compatibilty, which makes the encoder
// create pixel buffers with GPU backed memory. The intent here is to pass
// the pixel buffers directly so we avoid a texture upload later during
// rendering. This currently is moot because we are converting back to an
// I420 frame after decode, but eventually we will be able to plumb
// CVPixelBuffers directly to the renderer.
// TODO(tkchin): Maybe only set OpenGL/IOSurface keys if we know that that
// we can pass CVPixelBuffers as native handles in decoder output.
#if TARGET_OS_SIMULATOR
static size_t const attributesSize = 2;
#else
static size_t const attributesSize = 3;
#endif
CFTypeRef keys[attributesSize] = {
#if defined(WEBRTC_IOS)
kCVPixelBufferMetalCompatibilityKey,
#elif defined(WEBRTC_MAC)
kCVPixelBufferOpenGLCompatibilityKey,
#endif
#if !(TARGET_OS_SIMULATOR)
kCVPixelBufferIOSurfacePropertiesKey,
#endif
kCVPixelBufferPixelFormatTypeKey};
CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0);
int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
CFNumberRef pixelFormat = CFNumberCreate(nullptr, kCFNumberLongType, &nv12type);
#if TARGET_OS_SIMULATOR
CFTypeRef values[attributesSize] = {kCFBooleanTrue, pixelFormat};
#else
CFTypeRef values[attributesSize] = {kCFBooleanTrue, ioSurfaceValue, pixelFormat};
#endif
CFDictionaryRef attributes = CreateCFTypeDictionary(keys, values, attributesSize);
if (ioSurfaceValue) {
CFRelease(ioSurfaceValue);
ioSurfaceValue = nullptr;
}
if (pixelFormat) {
CFRelease(pixelFormat);
pixelFormat = nullptr;
}
VTDecompressionOutputCallbackRecord record = {
decompressionOutputCallback, (__bridge void *)self,
};
OSStatus status = VTDecompressionSessionCreate(
nullptr, _videoFormat, nullptr, attributes, &record, &_decompressionSession);
CFRelease(attributes);
if (status != noErr) {
RTC_LOG(LS_ERROR) << "Failed to create decompression session: " << status;
[self destroyDecompressionSession];
return WEBRTC_VIDEO_CODEC_ERROR;
}
[self configureDecompressionSession];
return WEBRTC_VIDEO_CODEC_OK;
}
- (void)configureDecompressionSession {
RTC_DCHECK(_decompressionSession);
#if defined(WEBRTC_IOS)
VTSessionSetProperty(_decompressionSession, kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue);
#endif
}
- (void)destroyDecompressionSession {
if (_decompressionSession) {
#if defined(WEBRTC_IOS)
VTDecompressionSessionWaitForAsynchronousFrames(_decompressionSession);
#endif
VTDecompressionSessionInvalidate(_decompressionSession);
CFRelease(_decompressionSession);
_decompressionSession = nullptr;
}
}
- (void)setVideoFormat:(CMVideoFormatDescriptionRef)videoFormat {
if (_videoFormat == videoFormat) {
return;
}
if (_videoFormat) {
CFRelease(_videoFormat);
}
_videoFormat = videoFormat;
if (_videoFormat) {
CFRetain(_videoFormat);
}
}
- (NSString *)implementationName {
return @"VideoToolbox";
}
@end

View file

@ -0,0 +1,23 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_DISABLE_H265
#import <Foundation/Foundation.h>
#import "RTCMacros.h"
#import "RTCVideoDecoder.h"
RTC_OBJC_EXPORT
API_AVAILABLE(ios(11.0))
@interface TGRTCVideoDecoderH265 : NSObject <RTCVideoDecoder>
@end
#endif

View file

@ -0,0 +1,359 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
#ifndef WEBRTC_DISABLE_H265
#import "TGRTCVideoDecoderH265.h"
#import <VideoToolbox/VideoToolbox.h>
#import "base/RTCVideoFrame.h"
#import "base/RTCVideoFrameBuffer.h"
#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
#import "helpers.h"
#import "helpers/scoped_cftyperef.h"
#if defined(WEBRTC_IOS)
#import "helpers/UIDevice+RTCDevice.h"
#endif
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/time_utils.h"
#include "rtc_base/thread.h"
#include "sdk/objc/components/video_codec/nalu_rewriter.h"
#include "h265_nalu_rewriter.h"
#include "StaticThreads.h"
@interface MarkedDecodedH2651RTCCVPixelBuffer : RTCCVPixelBuffer
@end
@implementation MarkedDecodedH2651RTCCVPixelBuffer
@end
typedef void (^TGRTCVideoDecoderRequestKeyframeCallback)();
// Struct that we pass to the decoder per frame to decode. We receive it again
// in the decoder callback.
struct RTCH265FrameDecodeParams {
RTCH265FrameDecodeParams(RTCVideoDecoderCallback cb, int64_t ts, TGRTCVideoDecoderRequestKeyframeCallback requestFrame)
: callback(cb), timestamp(ts), requestFrame(requestFrame) {}
RTCVideoDecoderCallback callback;
int64_t timestamp;
TGRTCVideoDecoderRequestKeyframeCallback requestFrame;
};
// This is the callback function that VideoToolbox calls when decode is
// complete.
static void tg_h265DecompressionOutputCallback(void* decoder,
void* params,
OSStatus status,
VTDecodeInfoFlags infoFlags,
CVImageBufferRef imageBuffer,
CMTime timestamp,
CMTime duration) {
std::unique_ptr<RTCH265FrameDecodeParams> decodeParams(
reinterpret_cast<RTCH265FrameDecodeParams*>(params));
if (status != noErr) {
RTC_LOG(LS_ERROR) << "Failed to decode frame. Status: " << status;
if (status == -12909) {
decodeParams->requestFrame();
}
return;
}
// TODO(tkchin): Handle CVO properly.
RTCCVPixelBuffer* frameBuffer =
[[MarkedDecodedH2651RTCCVPixelBuffer alloc] initWithPixelBuffer:imageBuffer];
RTCVideoFrame* decodedFrame = [[RTCVideoFrame alloc]
initWithBuffer:frameBuffer
rotation:RTCVideoRotation_0
timeStampNs:CMTimeGetSeconds(timestamp) * rtc::kNumNanosecsPerSec];
decodedFrame.timeStamp = (int32_t)decodeParams->timestamp;
decodeParams->callback(decodedFrame);
}
@interface TGRTCVideoDecoderH265RequestKeyframeHolder : NSObject
@property (nonatomic, strong) NSLock *lock;
@property (nonatomic) bool shouldRequestKeyframe;
@end
@implementation TGRTCVideoDecoderH265RequestKeyframeHolder
- (instancetype)init {
self = [super init];
if (self != nil) {
_lock = [[NSLock alloc] init];
}
return self;
}
@end
// Decoder.
@implementation TGRTCVideoDecoderH265 {
CMVideoFormatDescriptionRef _videoFormat;
VTDecompressionSessionRef _decompressionSession;
RTCVideoDecoderCallback _callback;
TGRTCVideoDecoderH265RequestKeyframeHolder *_requestKeyframeHolder;
TGRTCVideoDecoderRequestKeyframeCallback _requestFrame;
OSStatus _error;
}
- (instancetype)init {
if (self = [super init]) {
_requestKeyframeHolder = [[TGRTCVideoDecoderH265RequestKeyframeHolder alloc] init];
TGRTCVideoDecoderH265RequestKeyframeHolder *requestKeyframeHolder = _requestKeyframeHolder;
_requestFrame = ^{
[requestKeyframeHolder.lock lock];
requestKeyframeHolder.shouldRequestKeyframe = true;
[requestKeyframeHolder.lock unlock];
};
NSNotificationCenter *center = [NSNotificationCenter defaultCenter];
#ifdef WEBRTC_IOS
[center addObserver:self
selector:@selector(handleApplicationDidBecomeActive:)
name:UIApplicationWillEnterForegroundNotification
object:[UIApplication sharedApplication]];
#endif
}
return self;
}
- (void)dealloc {
[self destroyDecompressionSession];
[self setVideoFormat:nullptr];
[[NSNotificationCenter defaultCenter] removeObserver:self];
}
- (NSInteger)startDecodeWithNumberOfCores:(int)numberOfCores {
return WEBRTC_VIDEO_CODEC_OK;
}
- (void)handleApplicationDidBecomeActive:(NSNotification *)notification {
__weak TGRTCVideoDecoderH265 *weakSelf = self;
tgcalls::StaticThreads::getMediaThread()->PostTask([weakSelf]() {
__strong TGRTCVideoDecoderH265 *strongSelf = weakSelf;
if (strongSelf == nil) {
return;
}
strongSelf->_videoFormat = nil;
});
}
- (NSInteger)decode:(RTCEncodedImage*)inputImage
missingFrames:(BOOL)missingFrames
codecSpecificInfo:(__nullable id<RTCCodecSpecificInfo>)info
renderTimeMs:(int64_t)renderTimeMs {
RTC_DCHECK(inputImage.buffer);
if (_error != noErr) {
RTC_LOG(LS_WARNING) << "Last frame decode failed.";
_error = noErr;
return WEBRTC_VIDEO_CODEC_ERROR;
}
rtc::ScopedCFTypeRef<CMVideoFormatDescriptionRef> inputFormat =
rtc::ScopedCF(webrtc::CreateH265VideoFormatDescription(
(uint8_t*)inputImage.buffer.bytes, inputImage.buffer.length));
if (inputFormat) {
CMVideoDimensions dimensions =
CMVideoFormatDescriptionGetDimensions(inputFormat.get());
RTC_LOG(LS_INFO) << "Resolution: " << dimensions.width << " x "
<< dimensions.height;
// Check if the video format has changed, and reinitialize decoder if
// needed.
if (!CMFormatDescriptionEqual(inputFormat.get(), _videoFormat)) {
[self setVideoFormat:inputFormat.get()];
int resetDecompressionSessionError = [self resetDecompressionSession];
if (resetDecompressionSessionError != WEBRTC_VIDEO_CODEC_OK) {
return resetDecompressionSessionError;
}
}
}
if (!_videoFormat) {
// We received a frame but we don't have format information so we can't
// decode it.
// This can happen after backgrounding. We need to wait for the next
// sps/pps before we can resume so we request a keyframe by returning an
// error.
RTC_LOG(LS_WARNING) << "Missing video format. Frame with sps/pps required.";
return WEBRTC_VIDEO_CODEC_ERROR;
}
CMSampleBufferRef sampleBuffer = nullptr;
if (!webrtc::H265AnnexBBufferToCMSampleBuffer(
(uint8_t*)inputImage.buffer.bytes, inputImage.buffer.length,
_videoFormat, &sampleBuffer)) {
return WEBRTC_VIDEO_CODEC_ERROR;
}
RTC_DCHECK(sampleBuffer);
VTDecodeFrameFlags decodeFlags =
kVTDecodeFrame_EnableAsynchronousDecompression;
std::unique_ptr<RTCH265FrameDecodeParams> frameDecodeParams;
frameDecodeParams.reset(
new RTCH265FrameDecodeParams(_callback, inputImage.timeStamp, _requestFrame));
OSStatus status = VTDecompressionSessionDecodeFrame(
_decompressionSession, sampleBuffer, decodeFlags,
frameDecodeParams.release(), nullptr);
#if defined(WEBRTC_IOS)
// Re-initialize the decoder if we have an invalid session while the app is
// active and retry the decode request.
if (status == kVTInvalidSessionErr &&
[self resetDecompressionSession] == WEBRTC_VIDEO_CODEC_OK) {
frameDecodeParams.reset(
new RTCH265FrameDecodeParams(_callback, inputImage.timeStamp, _requestFrame));
status = VTDecompressionSessionDecodeFrame(
_decompressionSession, sampleBuffer, decodeFlags,
frameDecodeParams.release(), nullptr);
}
#endif
CFRelease(sampleBuffer);
if (status != noErr) {
RTC_LOG(LS_ERROR) << "Failed to decode frame with code: " << status;
return WEBRTC_VIDEO_CODEC_ERROR;
}
bool requestKeyframe = false;
[_requestKeyframeHolder.lock lock];
if (_requestKeyframeHolder.shouldRequestKeyframe) {
_requestKeyframeHolder.shouldRequestKeyframe = false;
requestKeyframe = true;
}
[_requestKeyframeHolder.lock unlock];
if (requestKeyframe) {
RTC_LOG(LS_ERROR) << "Decoder asynchronously asked to request keyframe";
return WEBRTC_VIDEO_CODEC_ERROR;
}
return WEBRTC_VIDEO_CODEC_OK;
}
- (void)setCallback:(RTCVideoDecoderCallback)callback {
_callback = callback;
}
- (NSInteger)releaseDecoder {
// Need to invalidate the session so that callbacks no longer occur and it
// is safe to null out the callback.
[self destroyDecompressionSession];
[self setVideoFormat:nullptr];
_callback = nullptr;
return WEBRTC_VIDEO_CODEC_OK;
}
#pragma mark - Private
- (int)resetDecompressionSession {
[self destroyDecompressionSession];
// Need to wait for the first SPS to initialize decoder.
if (!_videoFormat) {
return WEBRTC_VIDEO_CODEC_OK;
}
// Set keys for OpenGL and IOSurface compatibilty, which makes the encoder
// create pixel buffers with GPU backed memory. The intent here is to pass
// the pixel buffers directly so we avoid a texture upload later during
// rendering. This currently is moot because we are converting back to an
// I420 frame after decode, but eventually we will be able to plumb
// CVPixelBuffers directly to the renderer.
// TODO(tkchin): Maybe only set OpenGL/IOSurface keys if we know that that
// we can pass CVPixelBuffers as native handles in decoder output.
static size_t const attributesSize = 3;
CFTypeRef keys[attributesSize] = {
#if defined(WEBRTC_IOS)
kCVPixelBufferOpenGLESCompatibilityKey,
#elif defined(WEBRTC_MAC)
kCVPixelBufferOpenGLCompatibilityKey,
#endif
kCVPixelBufferIOSurfacePropertiesKey,
kCVPixelBufferPixelFormatTypeKey
};
CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0);
int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
CFNumberRef pixelFormat =
CFNumberCreate(nullptr, kCFNumberLongType, &nv12type);
CFTypeRef values[attributesSize] = {kCFBooleanTrue, ioSurfaceValue,
pixelFormat};
CFDictionaryRef attributes =
CreateCFTypeDictionary(keys, values, attributesSize);
if (ioSurfaceValue) {
CFRelease(ioSurfaceValue);
ioSurfaceValue = nullptr;
}
if (pixelFormat) {
CFRelease(pixelFormat);
pixelFormat = nullptr;
}
VTDecompressionOutputCallbackRecord record = {
tg_h265DecompressionOutputCallback,
nullptr,
};
OSStatus status =
VTDecompressionSessionCreate(nullptr, _videoFormat, nullptr, attributes,
&record, &_decompressionSession);
CFRelease(attributes);
if (status != noErr) {
[self destroyDecompressionSession];
return WEBRTC_VIDEO_CODEC_ERROR;
}
[self configureDecompressionSession];
return WEBRTC_VIDEO_CODEC_OK;
}
- (void)configureDecompressionSession {
RTC_DCHECK(_decompressionSession);
#if defined(WEBRTC_IOS)
// VTSessionSetProperty(_decompressionSession,
// kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue);
#endif
}
- (void)destroyDecompressionSession {
if (_decompressionSession) {
#if defined(WEBRTC_IOS)
VTDecompressionSessionWaitForAsynchronousFrames(_decompressionSession);
#endif
VTDecompressionSessionInvalidate(_decompressionSession);
CFRelease(_decompressionSession);
_decompressionSession = nullptr;
}
}
- (void)setVideoFormat:(CMVideoFormatDescriptionRef)videoFormat {
if (_videoFormat == videoFormat) {
return;
}
if (_videoFormat) {
CFRelease(_videoFormat);
}
_videoFormat = videoFormat;
if (_videoFormat) {
CFRetain(_videoFormat);
}
}
- (NSString*)implementationName {
return @"VideoToolbox";
}
@end
#endif

View file

@ -0,0 +1,22 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import <Foundation/Foundation.h>
#import "RTCMacros.h"
#import "RTCVideoCodecInfo.h"
#import "RTCVideoEncoder.h"
RTC_OBJC_EXPORT
@interface TGRTCVideoEncoderH264 : NSObject <RTCVideoEncoder>
- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo *)codecInfo;
@end

View file

@ -0,0 +1,851 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
#import "TGRTCVideoEncoderH264.h"
#import <VideoToolbox/VideoToolbox.h>
#include <vector>
#if defined(WEBRTC_IOS)
#import "helpers/UIDevice+RTCDevice.h"
#endif
#import "RTCCodecSpecificInfoH264.h"
#import "RTCH264ProfileLevelId.h"
#import "api/peerconnection/RTCVideoCodecInfo+Private.h"
#import "base/RTCCodecSpecificInfo.h"
#import "base/RTCI420Buffer.h"
#import "base/RTCVideoEncoder.h"
#import "base/RTCVideoFrame.h"
#import "base/RTCVideoFrameBuffer.h"
#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
#import "helpers.h"
#include "common_video/h264/h264_bitstream_parser.h"
#include "api/video_codecs/h264_profile_level_id.h"
#include "common_video/include/bitrate_adjuster.h"
#include "modules/include/module_common_types.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/buffer.h"
#include "rtc_base/logging.h"
#include "rtc_base/time_utils.h"
#include "sdk/objc/components/video_codec/nalu_rewriter.h"
#include "third_party/libyuv/include/libyuv/convert_from.h"
@interface TGRTCVideoEncoderH264 ()
- (void)frameWasEncoded:(OSStatus)status
flags:(VTEncodeInfoFlags)infoFlags
sampleBuffer:(CMSampleBufferRef)sampleBuffer
codecSpecificInfo:(id<RTCCodecSpecificInfo>)codecSpecificInfo
width:(int32_t)width
height:(int32_t)height
renderTimeMs:(int64_t)renderTimeMs
timestamp:(uint32_t)timestamp
rotation:(RTCVideoRotation)rotation;
@end
namespace { // anonymous namespace
// The ratio between kVTCompressionPropertyKey_DataRateLimits and
// kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher
// than the average bit rate to avoid undershooting the target.
const float kLimitToAverageBitRateFactor = 1.5f;
// These thresholds deviate from the default h264 QP thresholds, as they
// have been found to work better on devices that support VideoToolbox
const int kLowH264QpThreshold = 28;
const int kHighH264QpThreshold = 39;
const OSType kNV12PixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
// Struct that we pass to the encoder per frame to encode. We receive it again
// in the encoder callback.
struct RTCFrameEncodeParams {
RTCFrameEncodeParams(TGRTCVideoEncoderH264 *e,
RTCCodecSpecificInfoH264 *csi,
int32_t w,
int32_t h,
int64_t rtms,
uint32_t ts,
RTCVideoRotation r)
: encoder(e), width(w), height(h), render_time_ms(rtms), timestamp(ts), rotation(r) {
if (csi) {
codecSpecificInfo = csi;
} else {
codecSpecificInfo = [[RTCCodecSpecificInfoH264 alloc] init];
}
}
TGRTCVideoEncoderH264 *encoder;
RTCCodecSpecificInfoH264 *codecSpecificInfo;
int32_t width;
int32_t height;
int64_t render_time_ms;
uint32_t timestamp;
RTCVideoRotation rotation;
};
// We receive I420Frames as input, but we need to feed CVPixelBuffers into the
// encoder. This performs the copy and format conversion.
// TODO(tkchin): See if encoder will accept i420 frames and compare performance.
static bool CopyVideoFrameToNV12PixelBuffer(id<RTCI420Buffer> frameBuffer, CVPixelBufferRef pixelBuffer) {
RTC_DCHECK(pixelBuffer);
RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixelBuffer), kNV12PixelFormat);
RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixelBuffer, 0), frameBuffer.height);
RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixelBuffer, 0), frameBuffer.width);
CVReturn cvRet = CVPixelBufferLockBaseAddress(pixelBuffer, 0);
if (cvRet != kCVReturnSuccess) {
RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
return false;
}
uint8_t *dstY = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
int dstStrideY = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
uint8_t *dstUV = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
int dstStrideUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
// Convert I420 to NV12.
int ret = libyuv::I420ToNV12(frameBuffer.dataY,
frameBuffer.strideY,
frameBuffer.dataU,
frameBuffer.strideU,
frameBuffer.dataV,
frameBuffer.strideV,
dstY,
dstStrideY,
dstUV,
dstStrideUV,
frameBuffer.width,
frameBuffer.height);
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
if (ret) {
RTC_LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret;
return false;
}
return true;
}
static CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) {
if (!pixel_buffer_pool) {
RTC_LOG(LS_ERROR) << "Failed to get pixel buffer pool.";
return nullptr;
}
CVPixelBufferRef pixel_buffer;
CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool, &pixel_buffer);
if (ret != kCVReturnSuccess) {
RTC_LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret;
// We probably want to drop frames here, since failure probably means
// that the pool is empty.
return nullptr;
}
return pixel_buffer;
}
// This is the callback function that VideoToolbox calls when encode is
// complete. From inspection this happens on its own queue.
static void compressionOutputCallback(void *encoder,
void *params,
OSStatus status,
VTEncodeInfoFlags infoFlags,
CMSampleBufferRef sampleBuffer) {
if (!params) {
// If there are pending callbacks when the encoder is destroyed, this can happen.
return;
}
std::unique_ptr<RTCFrameEncodeParams> encodeParams(
reinterpret_cast<RTCFrameEncodeParams *>(params));
[encodeParams->encoder frameWasEncoded:status
flags:infoFlags
sampleBuffer:sampleBuffer
codecSpecificInfo:encodeParams->codecSpecificInfo
width:encodeParams->width
height:encodeParams->height
renderTimeMs:encodeParams->render_time_ms
timestamp:encodeParams->timestamp
rotation:encodeParams->rotation];
}
// Extract VideoToolbox profile out of the webrtc::SdpVideoFormat. If there is
// no specific VideoToolbox profile for the specified level, AutoLevel will be
// returned. The user must initialize the encoder with a resolution and
// framerate conforming to the selected H264 level regardless.
CFStringRef ExtractProfile(const webrtc::H264ProfileLevelId &profile_level_id) {
switch (profile_level_id.profile) {
case webrtc::H264Profile::kProfileConstrainedBaseline:
case webrtc::H264Profile::kProfileBaseline:
switch (profile_level_id.level) {
case webrtc::H264Level::kLevel3:
return kVTProfileLevel_H264_Baseline_3_0;
case webrtc::H264Level::kLevel3_1:
return kVTProfileLevel_H264_Baseline_3_1;
case webrtc::H264Level::kLevel3_2:
return kVTProfileLevel_H264_Baseline_3_2;
case webrtc::H264Level::kLevel4:
return kVTProfileLevel_H264_Baseline_4_0;
case webrtc::H264Level::kLevel4_1:
return kVTProfileLevel_H264_Baseline_4_1;
case webrtc::H264Level::kLevel4_2:
return kVTProfileLevel_H264_Baseline_4_2;
case webrtc::H264Level::kLevel5:
return kVTProfileLevel_H264_Baseline_5_0;
case webrtc::H264Level::kLevel5_1:
return kVTProfileLevel_H264_Baseline_5_1;
case webrtc::H264Level::kLevel5_2:
return kVTProfileLevel_H264_Baseline_5_2;
case webrtc::H264Level::kLevel1:
case webrtc::H264Level::kLevel1_b:
case webrtc::H264Level::kLevel1_1:
case webrtc::H264Level::kLevel1_2:
case webrtc::H264Level::kLevel1_3:
case webrtc::H264Level::kLevel2:
case webrtc::H264Level::kLevel2_1:
case webrtc::H264Level::kLevel2_2:
return kVTProfileLevel_H264_Baseline_AutoLevel;
}
case webrtc::H264Profile::kProfileMain:
switch (profile_level_id.level) {
case webrtc::H264Level::kLevel3:
return kVTProfileLevel_H264_Main_3_0;
case webrtc::H264Level::kLevel3_1:
return kVTProfileLevel_H264_Main_3_1;
case webrtc::H264Level::kLevel3_2:
return kVTProfileLevel_H264_Main_3_2;
case webrtc::H264Level::kLevel4:
return kVTProfileLevel_H264_Main_4_0;
case webrtc::H264Level::kLevel4_1:
return kVTProfileLevel_H264_Main_4_1;
case webrtc::H264Level::kLevel4_2:
return kVTProfileLevel_H264_Main_4_2;
case webrtc::H264Level::kLevel5:
return kVTProfileLevel_H264_Main_5_0;
case webrtc::H264Level::kLevel5_1:
return kVTProfileLevel_H264_Main_5_1;
case webrtc::H264Level::kLevel5_2:
return kVTProfileLevel_H264_Main_5_2;
case webrtc::H264Level::kLevel1:
case webrtc::H264Level::kLevel1_b:
case webrtc::H264Level::kLevel1_1:
case webrtc::H264Level::kLevel1_2:
case webrtc::H264Level::kLevel1_3:
case webrtc::H264Level::kLevel2:
case webrtc::H264Level::kLevel2_1:
case webrtc::H264Level::kLevel2_2:
return kVTProfileLevel_H264_Main_AutoLevel;
}
case webrtc::H264Profile::kProfileConstrainedHigh:
case webrtc::H264Profile::kProfileHigh:
#ifdef __aarch64__
if (profile_level_id.level < webrtc::H264Level::kLevel4) {
return kVTProfileLevel_H264_Main_4_0;
}
#endif
switch (profile_level_id.level) {
case webrtc::H264Level::kLevel3:
return kVTProfileLevel_H264_High_3_0;
case webrtc::H264Level::kLevel3_1:
return kVTProfileLevel_H264_High_3_1;
case webrtc::H264Level::kLevel3_2:
return kVTProfileLevel_H264_High_3_2;
case webrtc::H264Level::kLevel4:
return kVTProfileLevel_H264_High_4_0;
case webrtc::H264Level::kLevel4_1:
return kVTProfileLevel_H264_High_4_1;
case webrtc::H264Level::kLevel4_2:
return kVTProfileLevel_H264_High_4_2;
case webrtc::H264Level::kLevel5:
return kVTProfileLevel_H264_High_5_0;
case webrtc::H264Level::kLevel5_1:
return kVTProfileLevel_H264_High_5_1;
case webrtc::H264Level::kLevel5_2:
return kVTProfileLevel_H264_High_5_2;
case webrtc::H264Level::kLevel1:
case webrtc::H264Level::kLevel1_b:
case webrtc::H264Level::kLevel1_1:
case webrtc::H264Level::kLevel1_2:
case webrtc::H264Level::kLevel1_3:
case webrtc::H264Level::kLevel2:
case webrtc::H264Level::kLevel2_1:
case webrtc::H264Level::kLevel2_2:
return kVTProfileLevel_H264_High_AutoLevel;
}
}
return {};
}
// The function returns the max allowed sample rate (pixels per second) that
// can be processed by given encoder with |profile_level_id|.
// See https://www.itu.int/rec/dologin_pub.asp?lang=e&id=T-REC-H.264-201610-S!!PDF-E&type=items
// for details.
NSUInteger GetMaxSampleRate(const webrtc::H264ProfileLevelId &profile_level_id) {
switch (profile_level_id.level) {
case webrtc::H264Level::kLevel3:
return 10368000;
case webrtc::H264Level::kLevel3_1:
return 27648000;
case webrtc::H264Level::kLevel3_2:
return 55296000;
case webrtc::H264Level::kLevel4:
case webrtc::H264Level::kLevel4_1:
return 62914560;
case webrtc::H264Level::kLevel4_2:
return 133693440;
case webrtc::H264Level::kLevel5:
return 150994944;
case webrtc::H264Level::kLevel5_1:
return 251658240;
case webrtc::H264Level::kLevel5_2:
return 530841600;
case webrtc::H264Level::kLevel1:
case webrtc::H264Level::kLevel1_b:
case webrtc::H264Level::kLevel1_1:
case webrtc::H264Level::kLevel1_2:
case webrtc::H264Level::kLevel1_3:
case webrtc::H264Level::kLevel2:
case webrtc::H264Level::kLevel2_1:
case webrtc::H264Level::kLevel2_2:
// Zero means auto rate setting.
return 0;
}
}
} // namespace
@implementation TGRTCVideoEncoderH264 {
RTCVideoCodecInfo *_codecInfo;
std::unique_ptr<webrtc::BitrateAdjuster> _bitrateAdjuster;
uint32_t _targetBitrateBps;
uint32_t _encoderBitrateBps;
uint32_t _encoderFrameRate;
uint32_t _maxAllowedFrameRate;
RTCH264PacketizationMode _packetizationMode;
absl::optional<webrtc::H264ProfileLevelId> _profile_level_id;
RTCVideoEncoderCallback _callback;
int32_t _width;
int32_t _height;
VTCompressionSessionRef _compressionSession;
CVPixelBufferPoolRef _pixelBufferPool;
RTCVideoCodecMode _mode;
webrtc::H264BitstreamParser _h264BitstreamParser;
std::vector<uint8_t> _frameScaleBuffer;
}
// .5 is set as a mininum to prevent overcompensating for large temporary
// overshoots. We don't want to degrade video quality too badly.
// .95 is set to prevent oscillations. When a lower bitrate is set on the
// encoder than previously set, its output seems to have a brief period of
// drastically reduced bitrate, so we want to avoid that. In steady state
// conditions, 0.95 seems to give us better overall bitrate over long periods
// of time.
- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo *)codecInfo {
if (self = [super init]) {
_codecInfo = codecInfo;
_bitrateAdjuster.reset(new webrtc::BitrateAdjuster(.5, .95));
_packetizationMode = RTCH264PacketizationModeNonInterleaved;
_profile_level_id =
webrtc::ParseSdpForH264ProfileLevelId([codecInfo nativeSdpVideoFormat].parameters);
RTC_DCHECK(_profile_level_id);
RTC_LOG(LS_INFO) << "Using profile " << CFStringToString(ExtractProfile(*_profile_level_id));
RTC_CHECK([codecInfo.name isEqualToString:kRTCVideoCodecH264Name]);
}
return self;
}
- (void)dealloc {
[self destroyCompressionSession];
}
- (NSInteger)startEncodeWithSettings:(RTCVideoEncoderSettings *)settings
numberOfCores:(int)numberOfCores {
RTC_DCHECK(settings);
RTC_DCHECK([settings.name isEqualToString:kRTCVideoCodecH264Name]);
_width = settings.width;
_height = settings.height;
_mode = settings.mode;
uint32_t aligned_width = (((_width + 15) >> 4) << 4);
uint32_t aligned_height = (((_height + 15) >> 4) << 4);
_maxAllowedFrameRate = static_cast<uint32_t>(GetMaxSampleRate(*_profile_level_id) /
(aligned_width * aligned_height));
// We can only set average bitrate on the HW encoder.
_targetBitrateBps = settings.startBitrate * 1000; // startBitrate is in kbps.
_bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps);
_encoderFrameRate = MIN(settings.maxFramerate, _maxAllowedFrameRate);
if (settings.maxFramerate > _maxAllowedFrameRate && _maxAllowedFrameRate > 0) {
RTC_LOG(LS_WARNING) << "Initial encoder frame rate setting " << settings.maxFramerate
<< " is larger than the "
<< "maximal allowed frame rate " << _maxAllowedFrameRate << ".";
}
// TODO(tkchin): Try setting payload size via
// kVTCompressionPropertyKey_MaxH264SliceBytes.
return [self resetCompressionSessionWithPixelFormat:kNV12PixelFormat];
}
- (NSInteger)encode:(RTCVideoFrame *)frame
codecSpecificInfo:(nullable id<RTCCodecSpecificInfo>)codecSpecificInfo
frameTypes:(NSArray<NSNumber *> *)frameTypes {
//RTC_DCHECK_EQ(frame.width, _width);
//RTC_DCHECK_EQ(frame.height, _height);
if (!_callback || !_compressionSession) {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
BOOL isKeyframeRequired = NO;
// Get a pixel buffer from the pool and copy frame data over.
if ([self resetCompressionSessionIfNeededWithFrame:frame]) {
isKeyframeRequired = YES;
}
CVPixelBufferRef pixelBuffer = nullptr;
if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
// Native frame buffer
RTCCVPixelBuffer *rtcPixelBuffer = (RTCCVPixelBuffer *)frame.buffer;
if (![rtcPixelBuffer requiresCropping]) {
// This pixel buffer might have a higher resolution than what the
// compression session is configured to. The compression session can
// handle that and will output encoded frames in the configured
// resolution regardless of the input pixel buffer resolution.
pixelBuffer = rtcPixelBuffer.pixelBuffer;
CVBufferRetain(pixelBuffer);
} else {
// Cropping required, we need to crop and scale to a new pixel buffer.
pixelBuffer = CreatePixelBuffer(_pixelBufferPool);
if (!pixelBuffer) {
return WEBRTC_VIDEO_CODEC_ERROR;
}
int dstWidth = CVPixelBufferGetWidth(pixelBuffer);
int dstHeight = CVPixelBufferGetHeight(pixelBuffer);
if ([rtcPixelBuffer requiresScalingToWidth:dstWidth height:dstHeight]) {
int size =
[rtcPixelBuffer bufferSizeForCroppingAndScalingToWidth:dstWidth height:dstHeight];
_frameScaleBuffer.resize(size);
} else {
_frameScaleBuffer.clear();
}
_frameScaleBuffer.shrink_to_fit();
if (![rtcPixelBuffer cropAndScaleTo:pixelBuffer withTempBuffer:_frameScaleBuffer.data()]) {
CVBufferRelease(pixelBuffer);
return WEBRTC_VIDEO_CODEC_ERROR;
}
}
}
if (!pixelBuffer) {
// We did not have a native frame buffer
pixelBuffer = CreatePixelBuffer(_pixelBufferPool);
if (!pixelBuffer) {
return WEBRTC_VIDEO_CODEC_ERROR;
}
RTC_DCHECK(pixelBuffer);
if (!CopyVideoFrameToNV12PixelBuffer([frame.buffer toI420], pixelBuffer)) {
RTC_LOG(LS_ERROR) << "Failed to copy frame data.";
CVBufferRelease(pixelBuffer);
return WEBRTC_VIDEO_CODEC_ERROR;
}
}
// Check if we need a keyframe.
if (!isKeyframeRequired && frameTypes) {
for (NSNumber *frameType in frameTypes) {
if ((RTCFrameType)frameType.intValue == RTCFrameTypeVideoFrameKey) {
isKeyframeRequired = YES;
break;
}
}
}
CMTime presentationTimeStamp = CMTimeMake(frame.timeStampNs / rtc::kNumNanosecsPerMillisec, 1000);
CFDictionaryRef frameProperties = nullptr;
if (isKeyframeRequired) {
CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame};
CFTypeRef values[] = {kCFBooleanTrue};
frameProperties = CreateCFTypeDictionary(keys, values, 1);
}
std::unique_ptr<RTCFrameEncodeParams> encodeParams;
encodeParams.reset(new RTCFrameEncodeParams(self,
codecSpecificInfo,
_width,
_height,
frame.timeStampNs / rtc::kNumNanosecsPerMillisec,
frame.timeStamp,
frame.rotation));
encodeParams->codecSpecificInfo.packetizationMode = _packetizationMode;
// Update the bitrate if needed.
[self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps() frameRate:_encoderFrameRate];
OSStatus status = VTCompressionSessionEncodeFrame(_compressionSession,
pixelBuffer,
presentationTimeStamp,
kCMTimeInvalid,
frameProperties,
encodeParams.release(),
nullptr);
if (frameProperties) {
CFRelease(frameProperties);
}
if (pixelBuffer) {
CVBufferRelease(pixelBuffer);
}
if (status == kVTInvalidSessionErr) {
// This error occurs when entering foreground after backgrounding the app.
RTC_LOG(LS_ERROR) << "Invalid compression session, resetting.";
[self resetCompressionSessionWithPixelFormat:[self pixelFormatOfFrame:frame]];
return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
} else if (status == kVTVideoEncoderMalfunctionErr) {
// Sometimes the encoder malfunctions and needs to be restarted.
RTC_LOG(LS_ERROR)
<< "Encountered video encoder malfunction error. Resetting compression session.";
[self resetCompressionSessionWithPixelFormat:[self pixelFormatOfFrame:frame]];
return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
} else if (status != noErr) {
RTC_LOG(LS_ERROR) << "Failed to encode frame with code: " << status;
return WEBRTC_VIDEO_CODEC_ERROR;
}
return WEBRTC_VIDEO_CODEC_OK;
}
- (void)setCallback:(RTCVideoEncoderCallback)callback {
_callback = callback;
}
- (int)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate {
_targetBitrateBps = 1000 * bitrateKbit;
_bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps);
if (framerate > _maxAllowedFrameRate && _maxAllowedFrameRate > 0) {
RTC_LOG(LS_WARNING) << "Encoder frame rate setting " << framerate << " is larger than the "
<< "maximal allowed frame rate " << _maxAllowedFrameRate << ".";
}
framerate = MIN(framerate, _maxAllowedFrameRate);
[self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps() frameRate:framerate];
return WEBRTC_VIDEO_CODEC_OK;
}
- (NSInteger)resolutionAlignment {
return 1;
}
- (BOOL)applyAlignmentToAllSimulcastLayers {
return NO;
}
- (BOOL)supportsNativeHandle {
return YES;
}
#pragma mark - Private
- (NSInteger)releaseEncoder {
// Need to destroy so that the session is invalidated and won't use the
// callback anymore. Do not remove callback until the session is invalidated
// since async encoder callbacks can occur until invalidation.
[self destroyCompressionSession];
_callback = nullptr;
return WEBRTC_VIDEO_CODEC_OK;
}
- (OSType)pixelFormatOfFrame:(RTCVideoFrame *)frame {
// Use NV12 for non-native frames.
if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
RTCCVPixelBuffer *rtcPixelBuffer = (RTCCVPixelBuffer *)frame.buffer;
return CVPixelBufferGetPixelFormatType(rtcPixelBuffer.pixelBuffer);
}
return kNV12PixelFormat;
}
- (BOOL)resetCompressionSessionIfNeededWithFrame:(RTCVideoFrame *)frame {
BOOL resetCompressionSession = NO;
// If we're capturing native frames in another pixel format than the compression session is
// configured with, make sure the compression session is reset using the correct pixel format.
OSType framePixelFormat = [self pixelFormatOfFrame:frame];
if (_compressionSession) {
_pixelBufferPool = VTCompressionSessionGetPixelBufferPool(_compressionSession);
// The pool attribute `kCVPixelBufferPixelFormatTypeKey` can contain either an array of pixel
// formats or a single pixel format.
NSDictionary *poolAttributes =
(__bridge NSDictionary *)CVPixelBufferPoolGetPixelBufferAttributes(_pixelBufferPool);
id pixelFormats =
[poolAttributes objectForKey:(__bridge NSString *)kCVPixelBufferPixelFormatTypeKey];
NSArray<NSNumber *> *compressionSessionPixelFormats = nil;
if ([pixelFormats isKindOfClass:[NSArray class]]) {
compressionSessionPixelFormats = (NSArray *)pixelFormats;
} else if ([pixelFormats isKindOfClass:[NSNumber class]]) {
compressionSessionPixelFormats = @[ (NSNumber *)pixelFormats ];
}
if (![compressionSessionPixelFormats
containsObject:[NSNumber numberWithLong:framePixelFormat]]) {
resetCompressionSession = YES;
RTC_LOG(LS_INFO) << "Resetting compression session due to non-matching pixel format.";
}
} else {
resetCompressionSession = YES;
}
if (resetCompressionSession) {
[self resetCompressionSessionWithPixelFormat:framePixelFormat];
}
return resetCompressionSession;
}
- (int)resetCompressionSessionWithPixelFormat:(OSType)framePixelFormat {
[self destroyCompressionSession];
// Set source image buffer attributes. These attributes will be present on
// buffers retrieved from the encoder's pixel buffer pool.
const size_t attributesSize = 3;
CFTypeRef keys[attributesSize] = {
#if defined(WEBRTC_IOS)
kCVPixelBufferOpenGLESCompatibilityKey,
#elif defined(WEBRTC_MAC)
kCVPixelBufferOpenGLCompatibilityKey,
#endif
kCVPixelBufferIOSurfacePropertiesKey,
kCVPixelBufferPixelFormatTypeKey
};
CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0);
int64_t pixelFormatType = framePixelFormat;
CFNumberRef pixelFormat = CFNumberCreate(nullptr, kCFNumberLongType, &pixelFormatType);
CFTypeRef values[attributesSize] = {kCFBooleanTrue, ioSurfaceValue, pixelFormat};
CFDictionaryRef sourceAttributes = CreateCFTypeDictionary(keys, values, attributesSize);
if (ioSurfaceValue) {
CFRelease(ioSurfaceValue);
ioSurfaceValue = nullptr;
}
if (pixelFormat) {
CFRelease(pixelFormat);
pixelFormat = nullptr;
}
CFMutableDictionaryRef encoder_specs = nullptr;
#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
// Currently hw accl is supported above 360p on mac, below 360p
// the compression session will be created with hw accl disabled.
encoder_specs = CFDictionaryCreateMutable(
nullptr, 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
CFDictionarySetValue(encoder_specs,
kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
kCFBooleanTrue);
#endif
OSStatus status =
VTCompressionSessionCreate(nullptr, // use default allocator
_width,
_height,
kCMVideoCodecType_H264,
encoder_specs, // use hardware accelerated encoder if available
sourceAttributes,
nullptr, // use default compressed data allocator
compressionOutputCallback,
nullptr,
&_compressionSession);
if (sourceAttributes) {
CFRelease(sourceAttributes);
sourceAttributes = nullptr;
}
if (encoder_specs) {
CFRelease(encoder_specs);
encoder_specs = nullptr;
}
if (status != noErr) {
RTC_LOG(LS_ERROR) << "Failed to create compression session: " << status;
return WEBRTC_VIDEO_CODEC_ERROR;
}
#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
CFBooleanRef hwaccl_enabled = nullptr;
status = VTSessionCopyProperty(_compressionSession,
kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder,
nullptr,
&hwaccl_enabled);
if (status == noErr && (CFBooleanGetValue(hwaccl_enabled))) {
RTC_LOG(LS_INFO) << "Compression session created with hw accl enabled";
} else {
RTC_LOG(LS_INFO) << "Compression session created with hw accl disabled";
}
#endif
[self configureCompressionSession];
// The pixel buffer pool is dependent on the compression session so if the session is reset, the
// pool should be reset as well.
_pixelBufferPool = VTCompressionSessionGetPixelBufferPool(_compressionSession);
return WEBRTC_VIDEO_CODEC_OK;
}
- (void)configureCompressionSession {
RTC_DCHECK(_compressionSession);
SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_RealTime, false);
SetVTSessionProperty(_compressionSession,
kVTCompressionPropertyKey_ProfileLevel,
ExtractProfile(*_profile_level_id));
SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AllowFrameReordering, false);
[self setEncoderBitrateBps:_targetBitrateBps frameRate:_encoderFrameRate];
// TODO(tkchin): Look at entropy mode and colorspace matrices.
// TODO(tkchin): Investigate to see if there's any way to make this work.
// May need it to interop with Android. Currently this call just fails.
// On inspecting encoder output on iOS8, this value is set to 6.
// internal::SetVTSessionProperty(compression_session_,
// kVTCompressionPropertyKey_MaxFrameDelayCount,
// 1);
// Set a relatively large value for keyframe emission (7200 frames or 4 minutes).
SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200);
SetVTSessionProperty(
_compressionSession, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240);
}
- (void)destroyCompressionSession {
if (_compressionSession) {
VTCompressionSessionInvalidate(_compressionSession);
CFRelease(_compressionSession);
_compressionSession = nullptr;
_pixelBufferPool = nullptr;
}
}
- (NSString *)implementationName {
return @"VideoToolbox";
}
- (void)setBitrateBps:(uint32_t)bitrateBps frameRate:(uint32_t)frameRate {
if (_encoderBitrateBps != bitrateBps || _encoderFrameRate != frameRate) {
[self setEncoderBitrateBps:bitrateBps frameRate:frameRate];
}
}
- (void)setEncoderBitrateBps:(uint32_t)bitrateBps frameRate:(uint32_t)frameRate {
if (_compressionSession) {
RTC_LOG(LS_ERROR) << "TGRTCVideoEncoderH264: setting bitrate to " << bitrateBps / 1024 << " kbit/s";
SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AverageBitRate, bitrateBps);
// With zero |_maxAllowedFrameRate|, we fall back to automatic frame rate detection.
if (_maxAllowedFrameRate > 0) {
SetVTSessionProperty(
_compressionSession, kVTCompressionPropertyKey_ExpectedFrameRate, frameRate);
}
// TODO(tkchin): Add a helper method to set array value.
int64_t dataLimitBytesPerSecondValue =
static_cast<int64_t>(bitrateBps * kLimitToAverageBitRateFactor / 8);
CFNumberRef bytesPerSecond =
CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &dataLimitBytesPerSecondValue);
int64_t oneSecondValue = 1;
CFNumberRef oneSecond =
CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &oneSecondValue);
const void *nums[2] = {bytesPerSecond, oneSecond};
CFArrayRef dataRateLimits = CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks);
OSStatus status = VTSessionSetProperty(
_compressionSession, kVTCompressionPropertyKey_DataRateLimits, dataRateLimits);
if (bytesPerSecond) {
CFRelease(bytesPerSecond);
}
if (oneSecond) {
CFRelease(oneSecond);
}
if (dataRateLimits) {
CFRelease(dataRateLimits);
}
if (status != noErr) {
RTC_LOG(LS_ERROR) << "Failed to set data rate limit with code: " << status;
}
_encoderBitrateBps = bitrateBps;
_encoderFrameRate = frameRate;
}
}
- (void)frameWasEncoded:(OSStatus)status
flags:(VTEncodeInfoFlags)infoFlags
sampleBuffer:(CMSampleBufferRef)sampleBuffer
codecSpecificInfo:(id<RTCCodecSpecificInfo>)codecSpecificInfo
width:(int32_t)width
height:(int32_t)height
renderTimeMs:(int64_t)renderTimeMs
timestamp:(uint32_t)timestamp
rotation:(RTCVideoRotation)rotation {
if (status != noErr) {
RTC_LOG(LS_ERROR) << "H264 encode failed with code: " << status;
return;
}
if (infoFlags & kVTEncodeInfo_FrameDropped) {
RTC_LOG(LS_INFO) << "H264 encode dropped frame.";
return;
}
BOOL isKeyframe = NO;
CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, 0);
if (attachments != nullptr && CFArrayGetCount(attachments)) {
CFDictionaryRef attachment =
static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
isKeyframe = !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync);
}
if (isKeyframe) {
RTC_LOG(LS_INFO) << "Generated keyframe";
}
__block std::unique_ptr<rtc::Buffer> buffer = std::make_unique<rtc::Buffer>();
{
bool result = webrtc::H264CMSampleBufferToAnnexBBuffer(sampleBuffer, isKeyframe, buffer.get());
if (!result) {
return;
}
}
RTCEncodedImage *frame = [[RTCEncodedImage alloc] init];
// This assumes ownership of `buffer` and is responsible for freeing it when done.
frame.buffer = [[NSData alloc] initWithBytesNoCopy:buffer->data()
length:buffer->size()
deallocator:^(void *bytes, NSUInteger size) {
buffer.reset();
}];
frame.encodedWidth = width;
frame.encodedHeight = height;
//frame.completeFrame = YES;
frame.frameType = isKeyframe ? RTCFrameTypeVideoFrameKey : RTCFrameTypeVideoFrameDelta;
frame.captureTimeMs = renderTimeMs;
frame.timeStamp = timestamp;
frame.rotation = rotation;
frame.contentType = (_mode == RTCVideoCodecModeScreensharing) ? RTCVideoContentTypeScreenshare :
RTCVideoContentTypeUnspecified;
frame.flags = webrtc::VideoSendTiming::kInvalid;
int qp;
_h264BitstreamParser.ParseBitstream(*buffer);
frame.qp = @(_h264BitstreamParser.GetLastSliceQp().value_or(-1));
BOOL res = _callback(frame, codecSpecificInfo);
if (!res) {
RTC_LOG(LS_ERROR) << "Encode callback failed";
return;
}
_bitrateAdjuster->Update(frame.buffer.length);
}
- (nullable RTCVideoEncoderQpThresholds *)scalingSettings {
return [[RTCVideoEncoderQpThresholds alloc] initWithThresholdsLow:kLowH264QpThreshold
high:kHighH264QpThreshold];
}
@end

View file

@ -0,0 +1,40 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_DISABLE_H265
#import <Foundation/Foundation.h>
#import "RTCMacros.h"
#import "RTCVideoCodecInfo.h"
#import "RTCVideoEncoder.h"
RTC_OBJC_EXPORT
API_AVAILABLE(ios(11.0))
@interface RTCVideoEncoderH265 : NSObject <RTCVideoEncoder>
- (instancetype _Nonnull)initWithCodecInfo:(RTCVideoCodecInfo * _Nonnull)codecInfo;
- (nullable RTC_OBJC_TYPE(RTCVideoEncoderQpThresholds) *)scalingSettings;
/** Resolutions should be aligned to this value. */
@property(nonatomic, readonly) NSInteger resolutionAlignment;
/** If enabled, resolution alignment is applied to all simulcast layers simultaneously so that when
scaled, all resolutions comply with 'resolutionAlignment'. */
@property(nonatomic, readonly) BOOL applyAlignmentToAllSimulcastLayers;
/** If YES, the reciever is expected to resample/scale the source texture to the expected output
size. */
@property(nonatomic, readonly) BOOL supportsNativeHandle;
@end
#endif

View file

@ -0,0 +1,624 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
#import "TGRTCVideoEncoderH265.h"
#import <VideoToolbox/VideoToolbox.h>
#include <vector>
#import "RTCCodecSpecificInfoH265.h"
#import "api/peerconnection/RTCVideoCodecInfo+Private.h"
#import "base/RTCI420Buffer.h"
#import "base/RTCVideoFrame.h"
#import "base/RTCVideoFrameBuffer.h"
#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
#import "helpers.h"
#if defined(WEBRTC_IOS)
#import "helpers/UIDevice+RTCDevice.h"
#endif
#include "api/video_codecs/h264_profile_level_id.h"
#include "common_video/h265/h265_bitstream_parser.h"
#include "common_video/include/bitrate_adjuster.h"
#include "libyuv/convert_from.h"
#include "modules/include/module_common_types.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/buffer.h"
#include "rtc_base/logging.h"
#include "rtc_base/time_utils.h"
#include "sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h"
#include "system_wrappers/include/clock.h"
#include "h265_nalu_rewriter.h"
@interface RTC_OBJC_TYPE (RTCVideoEncoderH265)
()
- (void)frameWasEncoded : (OSStatus)status flags : (VTEncodeInfoFlags)infoFlags sampleBuffer
: (CMSampleBufferRef)sampleBuffer codecSpecificInfo
: (id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)>)codecSpecificInfo width : (int32_t)width height
: (int32_t)height renderTimeMs : (int64_t)renderTimeMs timestamp : (uint32_t)timestamp rotation
: (RTCVideoRotation)rotation;
@end
namespace { // anonymous namespace
// The ratio between kVTCompressionPropertyKey_DataRateLimits and
// kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher
// than the average bit rate to avoid undershooting the target.
const float kLimitToAverageBitRateFactor = 1.5f;
// These thresholds deviate from the default h265 QP thresholds, as they
// have been found to work better on devices that support VideoToolbox
const int kLowh265QpThreshold = 28;
const int kHighh265QpThreshold = 39;
// Struct that we pass to the encoder per frame to encode. We receive it again
// in the encoder callback.
struct API_AVAILABLE(ios(11.0)) RTCFrameEncodeParams {
RTCFrameEncodeParams(RTC_OBJC_TYPE(RTCVideoEncoderH265) * e,
RTC_OBJC_TYPE(RTCCodecSpecificInfoH265) * csi,
int32_t w,
int32_t h,
int64_t rtms,
uint32_t ts,
RTCVideoRotation r)
: encoder(e),
width(w),
height(h),
render_time_ms(rtms),
timestamp(ts),
rotation(r) {
if (csi) {
codecSpecificInfo = csi;
} else {
codecSpecificInfo = [[RTC_OBJC_TYPE(RTCCodecSpecificInfoH265) alloc] init];
}
}
RTC_OBJC_TYPE(RTCVideoEncoderH265) * encoder;
RTC_OBJC_TYPE(RTCCodecSpecificInfoH265) * codecSpecificInfo;
int32_t width;
int32_t height;
int64_t render_time_ms;
uint32_t timestamp;
RTCVideoRotation rotation;
};
// We receive I420Frames as input, but we need to feed CVPixelBuffers into the
// encoder. This performs the copy and format conversion.
// TODO(tkchin): See if encoder will accept i420 frames and compare performance.
bool CopyVideoFrameToPixelBuffer(id<RTCI420Buffer> frameBuffer,
CVPixelBufferRef pixelBuffer) {
RTC_DCHECK(pixelBuffer);
RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixelBuffer),
kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixelBuffer, 0),
frameBuffer.height);
RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixelBuffer, 0),
frameBuffer.width);
CVReturn cvRet = CVPixelBufferLockBaseAddress(pixelBuffer, 0);
if (cvRet != kCVReturnSuccess) {
RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
return false;
}
uint8_t* dstY = reinterpret_cast<uint8_t*>(
CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
int dstStrideY = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
uint8_t* dstUV = reinterpret_cast<uint8_t*>(
CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
int dstStrideUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
// Convert I420 to NV12.
int ret = libyuv::I420ToNV12(
frameBuffer.dataY, frameBuffer.strideY, frameBuffer.dataU,
frameBuffer.strideU, frameBuffer.dataV, frameBuffer.strideV, dstY,
dstStrideY, dstUV, dstStrideUV, frameBuffer.width, frameBuffer.height);
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
if (ret) {
RTC_LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret;
return false;
}
return true;
}
CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) {
if (!pixel_buffer_pool) {
RTC_LOG(LS_ERROR) << "Failed to get pixel buffer pool.";
return nullptr;
}
CVPixelBufferRef pixel_buffer;
CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool,
&pixel_buffer);
if (ret != kCVReturnSuccess) {
RTC_LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret;
// We probably want to drop frames here, since failure probably means
// that the pool is empty.
return nullptr;
}
return pixel_buffer;
}
// This is the callback function that VideoToolbox calls when encode is
// complete. From inspection this happens on its own queue.
void compressionOutputCallback(void* encoder,
void* params,
OSStatus status,
VTEncodeInfoFlags infoFlags,
CMSampleBufferRef sampleBuffer)
API_AVAILABLE(ios(11.0)) {
if (!params) {
// If there are pending callbacks when the encoder is destroyed, this can happen.
return;
}
std::unique_ptr<RTCFrameEncodeParams> encodeParams(
reinterpret_cast<RTCFrameEncodeParams*>(params));
RTC_CHECK(encodeParams->encoder);
[encodeParams->encoder frameWasEncoded:status
flags:infoFlags
sampleBuffer:sampleBuffer
codecSpecificInfo:encodeParams->codecSpecificInfo
width:encodeParams->width
height:encodeParams->height
renderTimeMs:encodeParams->render_time_ms
timestamp:encodeParams->timestamp
rotation:encodeParams->rotation];
}
} // namespace
@implementation RTCVideoEncoderH265 {
RTCVideoCodecInfo* _codecInfo;
std::unique_ptr<webrtc::BitrateAdjuster> _bitrateAdjuster;
uint32_t _targetBitrateBps;
uint32_t _encoderBitrateBps;
CFStringRef _profile;
RTCVideoEncoderCallback _callback;
int32_t _width;
int32_t _height;
VTCompressionSessionRef _compressionSession;
RTCVideoCodecMode _mode;
int framesLeft;
webrtc::H265BitstreamParser _h265BitstreamParser;
std::vector<uint8_t> _nv12ScaleBuffer;
}
// .5 is set as a mininum to prevent overcompensating for large temporary
// overshoots. We don't want to degrade video quality too badly.
// .95 is set to prevent oscillations. When a lower bitrate is set on the
// encoder than previously set, its output seems to have a brief period of
// drastically reduced bitrate, so we want to avoid that. In steady state
// conditions, 0.95 seems to give us better overall bitrate over long periods
// of time.
- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo*)codecInfo {
if (self = [super init]) {
_codecInfo = codecInfo;
_bitrateAdjuster.reset(new webrtc::BitrateAdjuster(.5, .95));
RTC_CHECK([codecInfo.name isEqualToString:@"H265"]);
}
return self;
}
- (void)dealloc {
[self destroyCompressionSession];
}
- (NSInteger)startEncodeWithSettings:(RTCVideoEncoderSettings*)settings
numberOfCores:(int)numberOfCores {
RTC_DCHECK(settings);
RTC_DCHECK([settings.name isEqualToString:@"H265"]);
_width = settings.width;
_height = settings.height;
_mode = settings.mode;
// We can only set average bitrate on the HW encoder.
_targetBitrateBps = settings.startBitrate;
_bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps);
// TODO(tkchin): Try setting payload size via
// kVTCompressionPropertyKey_Maxh265SliceBytes.
return [self resetCompressionSession];
}
- (NSInteger)encode:(RTCVideoFrame*)frame
codecSpecificInfo:(id<RTCCodecSpecificInfo>)codecSpecificInfo
frameTypes:(NSArray<NSNumber*>*)frameTypes {
RTC_DCHECK_EQ(frame.width, _width);
RTC_DCHECK_EQ(frame.height, _height);
if (!_callback || !_compressionSession) {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
BOOL isKeyframeRequired = NO;
// Get a pixel buffer from the pool and copy frame data over.
CVPixelBufferPoolRef pixelBufferPool =
VTCompressionSessionGetPixelBufferPool(_compressionSession);
#if defined(WEBRTC_IOS)
if (!pixelBufferPool) {
// Kind of a hack. On backgrounding, the compression session seems to get
// invalidated, which causes this pool call to fail when the application
// is foregrounded and frames are being sent for encoding again.
// Resetting the session when this happens fixes the issue.
// In addition we request a keyframe so video can recover quickly.
[self resetCompressionSession];
pixelBufferPool =
VTCompressionSessionGetPixelBufferPool(_compressionSession);
isKeyframeRequired = YES;
RTC_LOG(LS_INFO) << "Resetting compression session due to invalid pool.";
}
#endif
CVPixelBufferRef pixelBuffer = nullptr;
if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
// Native frame buffer
RTCCVPixelBuffer* rtcPixelBuffer = (RTCCVPixelBuffer*)frame.buffer;
if (![rtcPixelBuffer requiresCropping]) {
// This pixel buffer might have a higher resolution than what the
// compression session is configured to. The compression session can
// handle that and will output encoded frames in the configured
// resolution regardless of the input pixel buffer resolution.
pixelBuffer = rtcPixelBuffer.pixelBuffer;
CVBufferRetain(pixelBuffer);
} else {
// Cropping required, we need to crop and scale to a new pixel buffer.
pixelBuffer = CreatePixelBuffer(pixelBufferPool);
if (!pixelBuffer) {
return WEBRTC_VIDEO_CODEC_ERROR;
}
int dstWidth = CVPixelBufferGetWidth(pixelBuffer);
int dstHeight = CVPixelBufferGetHeight(pixelBuffer);
if ([rtcPixelBuffer requiresScalingToWidth:dstWidth height:dstHeight]) {
int size =
[rtcPixelBuffer bufferSizeForCroppingAndScalingToWidth:dstWidth
height:dstHeight];
_nv12ScaleBuffer.resize(size);
} else {
_nv12ScaleBuffer.clear();
}
_nv12ScaleBuffer.shrink_to_fit();
if (![rtcPixelBuffer cropAndScaleTo:pixelBuffer
withTempBuffer:_nv12ScaleBuffer.data()]) {
return WEBRTC_VIDEO_CODEC_ERROR;
}
}
}
if (!pixelBuffer) {
// We did not have a native frame buffer
pixelBuffer = CreatePixelBuffer(pixelBufferPool);
if (!pixelBuffer) {
return WEBRTC_VIDEO_CODEC_ERROR;
}
RTC_DCHECK(pixelBuffer);
if (!CopyVideoFrameToPixelBuffer([frame.buffer toI420], pixelBuffer)) {
RTC_LOG(LS_ERROR) << "Failed to copy frame data.";
CVBufferRelease(pixelBuffer);
return WEBRTC_VIDEO_CODEC_ERROR;
}
}
// Check if we need a keyframe.
if (!isKeyframeRequired && frameTypes) {
for (NSNumber* frameType in frameTypes) {
if ((RTCFrameType)frameType.intValue == RTCFrameTypeVideoFrameKey) {
isKeyframeRequired = YES;
break;
}
}
}
CMTime presentationTimeStamp =
CMTimeMake(frame.timeStampNs / rtc::kNumNanosecsPerMillisec, 1000);
CFDictionaryRef frameProperties = nullptr;
if (isKeyframeRequired) {
CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame};
CFTypeRef values[] = {kCFBooleanTrue};
frameProperties = CreateCFTypeDictionary(keys, values, 1);
}
std::unique_ptr<RTCFrameEncodeParams> encodeParams;
encodeParams.reset(new RTCFrameEncodeParams(
self, codecSpecificInfo,_width, _height,
frame.timeStampNs / rtc::kNumNanosecsPerMillisec, frame.timeStamp,
frame.rotation));
// Update the bitrate if needed.
[self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps()];
OSStatus status = VTCompressionSessionEncodeFrame(
_compressionSession, pixelBuffer, presentationTimeStamp, kCMTimeInvalid,
frameProperties, encodeParams.release(), nullptr);
if (frameProperties) {
CFRelease(frameProperties);
}
if (pixelBuffer) {
CVBufferRelease(pixelBuffer);
}
if (status != noErr) {
RTC_LOG(LS_ERROR) << "Failed to encode frame with code: " << status;
return WEBRTC_VIDEO_CODEC_ERROR;
}
return WEBRTC_VIDEO_CODEC_OK;
}
- (void)setCallback:(RTCVideoEncoderCallback)callback {
_callback = callback;
}
- (int)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate {
_targetBitrateBps = 1000 * bitrateKbit;
_bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps);
[self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps()];
return WEBRTC_VIDEO_CODEC_OK;
}
#pragma mark - Private
- (NSInteger)releaseEncoder {
// Need to destroy so that the session is invalidated and won't use the
// callback anymore. Do not remove callback until the session is invalidated
// since async encoder callbacks can occur until invalidation.
[self destroyCompressionSession];
_callback = nullptr;
return WEBRTC_VIDEO_CODEC_OK;
}
- (int)resetCompressionSession {
[self destroyCompressionSession];
// Set source image buffer attributes. These attributes will be present on
// buffers retrieved from the encoder's pixel buffer pool.
const size_t attributesSize = 3;
CFTypeRef keys[attributesSize] = {
#if defined(WEBRTC_IOS)
kCVPixelBufferOpenGLESCompatibilityKey,
#elif defined(WEBRTC_MAC)
kCVPixelBufferOpenGLCompatibilityKey,
#endif
kCVPixelBufferIOSurfacePropertiesKey,
kCVPixelBufferPixelFormatTypeKey
};
CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0);
int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
CFNumberRef pixelFormat =
CFNumberCreate(nullptr, kCFNumberLongType, &nv12type);
CFTypeRef values[attributesSize] = {kCFBooleanTrue, ioSurfaceValue,
pixelFormat};
CFDictionaryRef sourceAttributes =
CreateCFTypeDictionary(keys, values, attributesSize);
if (ioSurfaceValue) {
CFRelease(ioSurfaceValue);
ioSurfaceValue = nullptr;
}
if (pixelFormat) {
CFRelease(pixelFormat);
pixelFormat = nullptr;
}
CFMutableDictionaryRef encoder_specs = nullptr;
#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
// Currently hw accl is supported above 360p on mac, below 360p
// the compression session will be created with hw accl disabled.
encoder_specs =
CFDictionaryCreateMutable(nullptr, 1, &kCFTypeDictionaryKeyCallBacks,
&kCFTypeDictionaryValueCallBacks);
CFDictionarySetValue(
encoder_specs,
kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
kCFBooleanTrue);
#endif
OSStatus status = VTCompressionSessionCreate(
nullptr, // use default allocator
_width, _height, kCMVideoCodecType_HEVC,
encoder_specs, // use hardware accelerated encoder if available
sourceAttributes,
nullptr, // use default compressed data allocator
compressionOutputCallback, nullptr, &_compressionSession);
if (sourceAttributes) {
CFRelease(sourceAttributes);
sourceAttributes = nullptr;
}
if (encoder_specs) {
CFRelease(encoder_specs);
encoder_specs = nullptr;
}
if (status != noErr) {
RTC_LOG(LS_ERROR) << "Failed to create compression session: " << status;
return WEBRTC_VIDEO_CODEC_ERROR;
}
#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
CFBooleanRef hwaccl_enabled = nullptr;
status = VTSessionCopyProperty(
_compressionSession,
kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder, nullptr,
&hwaccl_enabled);
if (status == noErr && (CFBooleanGetValue(hwaccl_enabled))) {
RTC_LOG(LS_INFO) << "Compression session created with hw accl enabled";
} else {
RTC_LOG(LS_INFO) << "Compression session created with hw accl disabled";
}
#endif
[self configureCompressionSession];
return WEBRTC_VIDEO_CODEC_OK;
}
- (void)configureCompressionSession {
RTC_DCHECK(_compressionSession);
SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_RealTime,
false);
// SetVTSessionProperty(_compressionSession,
// kVTCompressionPropertyKey_ProfileLevel, _profile);
SetVTSessionProperty(_compressionSession,
kVTCompressionPropertyKey_AllowFrameReordering, false);
[self setEncoderBitrateBps:_targetBitrateBps];
// TODO(tkchin): Look at entropy mode and colorspace matrices.
// TODO(tkchin): Investigate to see if there's any way to make this work.
// May need it to interop with Android. Currently this call just fails.
// On inspecting encoder output on iOS8, this value is set to 6.
// internal::SetVTSessionProperty(compression_session_,
// kVTCompressionPropertyKey_MaxFrameDelayCount,
// 1);
// Set a relatively large value for keyframe emission (7200 frames or 4
// minutes).
SetVTSessionProperty(_compressionSession,
kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200);
SetVTSessionProperty(_compressionSession,
kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration,
240);
OSStatus status =
VTCompressionSessionPrepareToEncodeFrames(_compressionSession);
if (status != noErr) {
RTC_LOG(LS_ERROR) << "Compression session failed to prepare encode frames.";
}
}
- (void)destroyCompressionSession {
if (_compressionSession) {
VTCompressionSessionInvalidate(_compressionSession);
CFRelease(_compressionSession);
_compressionSession = nullptr;
}
}
- (NSString*)implementationName {
return @"VideoToolbox";
}
- (void)setBitrateBps:(uint32_t)bitrateBps {
if (_encoderBitrateBps != bitrateBps) {
[self setEncoderBitrateBps:bitrateBps];
}
}
- (void)setEncoderBitrateBps:(uint32_t)bitrateBps {
if (_compressionSession) {
SetVTSessionProperty(_compressionSession,
kVTCompressionPropertyKey_AverageBitRate, bitrateBps);
// TODO(tkchin): Add a helper method to set array value.
int64_t dataLimitBytesPerSecondValue =
static_cast<int64_t>(bitrateBps * kLimitToAverageBitRateFactor / 8);
CFNumberRef bytesPerSecond =
CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type,
&dataLimitBytesPerSecondValue);
int64_t oneSecondValue = 1;
CFNumberRef oneSecond = CFNumberCreate(
kCFAllocatorDefault, kCFNumberSInt64Type, &oneSecondValue);
const void* nums[2] = {bytesPerSecond, oneSecond};
CFArrayRef dataRateLimits =
CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks);
OSStatus status = VTSessionSetProperty(
_compressionSession, kVTCompressionPropertyKey_DataRateLimits,
dataRateLimits);
if (bytesPerSecond) {
CFRelease(bytesPerSecond);
}
if (oneSecond) {
CFRelease(oneSecond);
}
if (dataRateLimits) {
CFRelease(dataRateLimits);
}
if (status != noErr) {
RTC_LOG(LS_ERROR) << "Failed to set data rate limit";
}
_encoderBitrateBps = bitrateBps;
}
}
- (void)frameWasEncoded:(OSStatus)status
flags:(VTEncodeInfoFlags)infoFlags
sampleBuffer:(CMSampleBufferRef)sampleBuffer
codecSpecificInfo:(id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)>)codecSpecificInfo
width:(int32_t)width
height:(int32_t)height
renderTimeMs:(int64_t)renderTimeMs
timestamp:(uint32_t)timestamp
rotation:(RTCVideoRotation)rotation {
if (status != noErr) {
RTC_LOG(LS_ERROR) << "h265 encode failed.";
return;
}
if (infoFlags & kVTEncodeInfo_FrameDropped) {
RTC_LOG(LS_INFO) << "h265 encoder dropped a frame.";
return;
}
BOOL isKeyframe = NO;
CFArrayRef attachments =
CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, 0);
if (attachments != nullptr && CFArrayGetCount(attachments)) {
CFDictionaryRef attachment =
static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
isKeyframe =
!CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync);
}
if (isKeyframe) {
RTC_LOG(LS_INFO) << "Generated keyframe";
}
__block std::unique_ptr<rtc::Buffer> buffer = std::make_unique<rtc::Buffer>();
if (!webrtc::H265CMSampleBufferToAnnexBBuffer(sampleBuffer, isKeyframe, buffer.get())) {
return;
}
RTCEncodedImage* frame = [[RTCEncodedImage alloc] init];
frame.buffer = [NSData dataWithBytesNoCopy:buffer->data()
length:buffer->size()
freeWhenDone:NO];
frame.encodedWidth = width;
frame.encodedHeight = height;
frame.frameType =
isKeyframe ? RTCFrameTypeVideoFrameKey : RTCFrameTypeVideoFrameDelta;
frame.captureTimeMs = renderTimeMs;
frame.timeStamp = timestamp;
frame.rotation = rotation;
frame.contentType = (_mode == RTCVideoCodecModeScreensharing)
? RTCVideoContentTypeScreenshare
: RTCVideoContentTypeUnspecified;
frame.flags = webrtc::VideoSendTiming::kInvalid;
_h265BitstreamParser.ParseBitstream(rtc::ArrayView<const uint8_t>(buffer->data(), buffer->size()));
absl::optional<int> qp = _h265BitstreamParser.GetLastSliceQp();
frame.qp = @(qp.value_or(1));
BOOL res = _callback(frame, codecSpecificInfo);
if (!res) {
RTC_LOG(LS_ERROR) << "Encode callback failed.";
return;
}
_bitrateAdjuster->Update(frame.buffer.length);
}
- (RTCVideoEncoderQpThresholds*)scalingSettings {
return [[RTCVideoEncoderQpThresholds alloc]
initWithThresholdsLow:kLowh265QpThreshold
high:kHighh265QpThreshold];
}
- (NSInteger)resolutionAlignment {
return 1;
}
- (BOOL)applyAlignmentToAllSimulcastLayers {
return NO;
}
- (BOOL)supportsNativeHandle {
return YES;
}
@end

View file

@ -0,0 +1,18 @@
//
// VideoCMIOCapture.h
// TgVoipWebrtc
//
// Created by Mikhail Filimonov on 21.06.2021.
// Copyright © 2021 Mikhail Filimonov. All rights reserved.
//
#import <Foundation/Foundation.h>
#include "VideoCameraCapturerMac.h"
NS_ASSUME_NONNULL_BEGIN
@interface VideoCMIOCapture : NSObject<CapturerInterface>
- (instancetype)initWithSource:(rtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source;
- (void)setupCaptureWithDevice:(AVCaptureDevice *)device;
@end
NS_ASSUME_NONNULL_END

View file

@ -0,0 +1,237 @@
//
// VideoCMIOCapture.m
// TgVoipWebrtc
//
// Created by Mikhail Filimonov on 21.06.2021.
// Copyright © 2021 Mikhail Filimonov. All rights reserved.
//
#import "VideoCMIOCapture.h"
#import "TGCMIODevice.h"
#import "TGCMIOCapturer.h"
#import <VideoToolbox/VideoToolbox.h>
#import "TGRTCCVPixelBuffer.h"
#include "rtc_base/logging.h"
#import "base/RTCLogging.h"
#import "base/RTCVideoFrameBuffer.h"
#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
#import "sdk/objc/native/src/objc_video_track_source.h"
#import "sdk/objc/native/src/objc_frame_buffer.h"
#import <CoreMediaIO/CMIOHardware.h>
#import "helpers/AVCaptureSession+DevicePosition.h"
#import "helpers/RTCDispatcher+Private.h"
#import "base/RTCVideoFrame.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "pc/video_track_source_proxy.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "third_party/libyuv/include/libyuv.h"
#include "DarwinVideoSource.h"
struct MTLFrameSize {
int width = 0;
int height = 0;
};
MTLFrameSize AspectFitted(MTLFrameSize from, MTLFrameSize to) {
double scale = std::min(
from.width / std::max(1., double(to.width)),
from.height / std::max(1., double(to.height)));
return {
int(std::ceil(to.width * scale)),
int(std::ceil(to.height * scale))
};
}
static const int64_t kNanosecondsPerSecond = 1000000000;
@interface VideoCMIOCapture ()
-(void)applyPixelBuffer:(CVPixelBufferRef)pixelBuffer timeStampNs:(int64_t)timeStampNs;
@end
void decompressionSessionDecodeFrameCallback(void *decompressionOutputRefCon,
void *sourceFrameRefCon,
OSStatus status,
VTDecodeInfoFlags infoFlags,
CVImageBufferRef imageBuffer,
CMTime presentationTimeStamp,
CMTime presentationDuration)
{
VideoCMIOCapture *manager = (__bridge VideoCMIOCapture *)decompressionOutputRefCon;
if (status == noErr)
{
[manager applyPixelBuffer:imageBuffer timeStampNs: CMTimeGetSeconds(presentationTimeStamp) * kNanosecondsPerSecond];
}
}
static tgcalls::DarwinVideoTrackSource *getObjCVideoSource(const rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> nativeSource) {
webrtc::VideoTrackSourceProxy *proxy_source =
static_cast<webrtc::VideoTrackSourceProxy *>(nativeSource.get());
return static_cast<tgcalls::DarwinVideoTrackSource *>(proxy_source->internal());
}
@implementation VideoCMIOCapture
{
TGCMIOCapturer *_capturer;
rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> _source;
std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> _uncroppedSink;
std::function<void ()> _onFatalError;
BOOL _hadFatalError;
BOOL _isRunning;
BOOL _shouldBeMirrored;
VTDecompressionSessionRef _decompressionSession;
}
- (void)start {
__weak VideoCMIOCapture *weakSelf = self;
[_capturer start:^(CMSampleBufferRef sampleBuffer) {
[weakSelf apply:sampleBuffer];
}];
}
-(void)apply:(CMSampleBufferRef)sampleBuffer {
if (CMSampleBufferGetNumSamples(sampleBuffer) != 1 || !CMSampleBufferIsValid(sampleBuffer) ||
!CMSampleBufferDataIsReady(sampleBuffer)) {
return;
}
CMVideoFormatDescriptionRef formatDesc = CMSampleBufferGetFormatDescription(sampleBuffer);
CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
if (pixelBuffer == nil) {
if (_decompressionSession == nil) {
[self createDecompSession:formatDesc];
}
[self render:sampleBuffer];
return;
}
int64_t timeStampNs = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) *
kNanosecondsPerSecond;
[self applyPixelBuffer:pixelBuffer timeStampNs: timeStampNs];
}
-(void)applyPixelBuffer:(CVPixelBufferRef)pixelBuffer timeStampNs:(int64_t)timeStampNs {
int width = (int)CVPixelBufferGetWidth(pixelBuffer);
int height = (int)CVPixelBufferGetHeight(pixelBuffer);
MTLFrameSize fittedSize = AspectFitted({ 1280, 720 }, { width, height });
fittedSize.width -= (fittedSize.width % 4);
fittedSize.height -= (fittedSize.height % 4);
TGRTCCVPixelBuffer *rtcPixelBuffer = [[TGRTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer adaptedWidth:fittedSize.width adaptedHeight:fittedSize.height cropWidth:width cropHeight:height cropX:0 cropY:0];
rtcPixelBuffer.shouldBeMirrored = _shouldBeMirrored;
RTCVideoFrame *videoFrame = [[RTCVideoFrame alloc] initWithBuffer:rtcPixelBuffer
rotation:RTCVideoRotation_0
timeStampNs:timeStampNs];
if (_uncroppedSink) {
const int64_t timestamp_us = timeStampNs / rtc::kNumNanosecsPerMicrosec;
rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer;
buffer = new rtc::RefCountedObject<webrtc::ObjCFrameBuffer>(videoFrame.buffer);
webrtc::VideoRotation rotation = static_cast<webrtc::VideoRotation>(videoFrame.rotation);
_uncroppedSink->OnFrame(webrtc::VideoFrame::Builder()
.set_video_frame_buffer(buffer)
.set_rotation(rotation)
.set_timestamp_us(timestamp_us)
.build());
}
getObjCVideoSource(_source)->OnCapturedFrame(videoFrame);
}
- (void)stop {
[_capturer stop];
}
- (void)setIsEnabled:(bool)isEnabled {
}
- (void)setUncroppedSink:(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)sink {
self->_uncroppedSink = sink;
}
- (void)setPreferredCaptureAspectRatio:(float)aspectRatio {
}
- (void)setOnFatalError:(std::function<void()>)error {
if (!self->_hadFatalError) {
_onFatalError = std::move(error);
} else if (error) {
error();
}
}
- (void)setOnPause:(std::function<void(bool)>)pause {
}
- (instancetype)initWithSource:(rtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source {
self = [super init];
if (self != nil) {
_source = source;
}
return self;
}
- (void)setupCaptureWithDevice:(AVCaptureDevice *)device {
_shouldBeMirrored = NO;
_capturer = [[TGCMIOCapturer alloc] initWithDeviceId:device];
}
- (void) render:(CMSampleBufferRef)sampleBuffer
{
VTDecodeFrameFlags flags = kVTDecodeFrame_EnableAsynchronousDecompression | kVTDecodeFrame_1xRealTimePlayback;
VTDecodeInfoFlags flagOut;
NSDate* currentTime = [NSDate date];
VTDecompressionSessionDecodeFrame(_decompressionSession, sampleBuffer, flags,
(void*)CFBridgingRetain(currentTime), &flagOut);
}
-(void) createDecompSession:(CMVideoFormatDescriptionRef)formatDesc
{
if (_decompressionSession) {
CFRelease(_decompressionSession);
}
_decompressionSession = NULL;
VTDecompressionOutputCallbackRecord callBackRecord;
callBackRecord.decompressionOutputCallback = decompressionSessionDecodeFrameCallback;
callBackRecord.decompressionOutputRefCon = (__bridge void *)self;
VTDecompressionSessionCreate(NULL, formatDesc, NULL,
NULL,
&callBackRecord, &_decompressionSession);
}
-(void)dealloc {
if (_decompressionSession) {
CFRelease(_decompressionSession);
}
}
@end

View file

@ -0,0 +1,33 @@
#ifndef TGCALLS_VIDEO_CAMERA_CAPTURER_H
#define TGCALLS_VIDEO_CAMERA_CAPTURER_H
#ifdef WEBRTC_IOS
#import <Foundation/Foundation.h>
#import <AVFoundation/AVFoundation.h>
#include <memory>
#include "api/scoped_refptr.h"
#include "api/media_stream_interface.h"
#include "Instance.h"
@class VideoCaptureView;
@interface VideoCameraCapturer : NSObject
+ (NSArray<AVCaptureDevice *> *)captureDevices;
+ (NSArray<AVCaptureDeviceFormat *> *)supportedFormatsForDevice:(AVCaptureDevice *)device;
- (instancetype)initWithSource:(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source useFrontCamera:(bool)useFrontCamera keepLandscape:(bool)keepLandscape isActiveUpdated:(void (^)(bool))isActiveUpdated rotationUpdated:(void (^)(int))rotationUpdated;
- (void)startCaptureWithDevice:(AVCaptureDevice *)device format:(AVCaptureDeviceFormat *)format fps:(NSInteger)fps;
- (void)stopCapture;
- (void)setIsEnabled:(bool)isEnabled;
- (void)setPreferredCaptureAspectRatio:(float)aspectRatio;
- (void)setUncroppedSink:(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)sink;
- (int)getRotation;
- (void)addPreviewView:(VideoCaptureView *)previewView;
- (void)addDirectSink:(std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)directSink;
@end
#endif // WEBRTC_IOS
#endif

View file

@ -0,0 +1,935 @@
#include "VideoCameraCapturer.h"
#import <AVFoundation/AVFoundation.h>
#include "rtc_base/logging.h"
#import "base/RTCLogging.h"
#import "base/RTCVideoFrameBuffer.h"
#import "TGRTCCVPixelBuffer.h"
#import "sdk/objc/native/src/objc_video_track_source.h"
#import "sdk/objc/native/src/objc_frame_buffer.h"
#import "pc/video_track_source_proxy.h"
#import "helpers/UIDevice+RTCDevice.h"
#import "helpers/AVCaptureSession+DevicePosition.h"
#import "helpers/RTCDispatcher+Private.h"
#import "base/RTCVideoFrame.h"
#include "DarwinVideoSource.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "third_party/libyuv/include/libyuv.h"
#include "api/video/i420_buffer.h"
#include "api/video/nv12_buffer.h"
#include "VideoCaptureView.h"
namespace {
static const int64_t kNanosecondsPerSecond = 1000000000;
static tgcalls::DarwinVideoTrackSource *getObjCVideoSource(const webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> nativeSource) {
webrtc::VideoTrackSourceProxy *proxy_source =
static_cast<webrtc::VideoTrackSourceProxy *>(nativeSource.get());
return static_cast<tgcalls::DarwinVideoTrackSource *>(proxy_source->internal());
}
static UIDeviceOrientation deviceOrientation(UIInterfaceOrientation orientation) {
switch (orientation) {
case UIInterfaceOrientationPortrait:
return UIDeviceOrientationPortrait;
case UIInterfaceOrientationPortraitUpsideDown:
return UIDeviceOrientationPortraitUpsideDown;
case UIInterfaceOrientationLandscapeLeft:
return UIDeviceOrientationLandscapeRight;
case UIInterfaceOrientationLandscapeRight:
return UIDeviceOrientationLandscapeLeft;
default:
return UIDeviceOrientationPortrait;
}
}
}
@interface VideoCameraCapturerPreviewRecord : NSObject
@property (nonatomic, weak) VideoCaptureView *view;
@end
@implementation VideoCameraCapturerPreviewRecord
- (instancetype)initWithCaptureView:(VideoCaptureView *)view {
self = [super init];
if (self != nil) {
self.view = view;
}
return self;
}
@end
@interface VideoCameraCapturer () <AVCaptureVideoDataOutputSampleBufferDelegate> {
webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> _source;
// Live on main thread.
bool _isFrontCamera;
bool _keepLandscape;
dispatch_queue_t _frameQueue;
// Live on RTCDispatcherTypeCaptureSession.
AVCaptureDevice *_currentDevice;
BOOL _hasRetriedOnFatalError;
BOOL _isRunning;
// Live on RTCDispatcherTypeCaptureSession and main thread.
std::atomic<bool> _willBeRunning;
AVCaptureVideoDataOutput *_videoDataOutput;
AVCaptureSession *_captureSession;
FourCharCode _preferredOutputPixelFormat;
FourCharCode _outputPixelFormat;
RTCVideoRotation _rotation;
UIDeviceOrientation _orientation;
bool _didReceiveOrientationUpdate;
bool _rotationLock;
// Live on mainThread.
void (^_isActiveUpdated)(bool);
bool _isActiveValue;
bool _inForegroundValue;
void (^_rotationUpdated)(int);
// Live on frameQueue and main thread.
std::atomic<bool> _isPaused;
// Live on frameQueue.
float _aspectRatio;
std::vector<uint8_t> _croppingBuffer;
std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> _uncroppedSink;
// Live on frameQueue and RTCDispatcherTypeCaptureSession.
std::atomic<int> _warmupFrameCount;
webrtc::NV12ToI420Scaler _nv12ToI420Scaler;
NSMutableArray<VideoCameraCapturerPreviewRecord *> *_previews;
std::vector<std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>> _directSinks;
}
@end
@implementation VideoCameraCapturer
- (instancetype)initWithSource:(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source useFrontCamera:(bool)useFrontCamera keepLandscape:(bool)keepLandscape isActiveUpdated:(void (^)(bool))isActiveUpdated rotationUpdated:(void (^)(int))rotationUpdated {
self = [super init];
if (self != nil) {
_source = source;
_isFrontCamera = useFrontCamera;
_keepLandscape = keepLandscape;
_isActiveValue = true;
_inForegroundValue = true;
_isPaused = false;
_isActiveUpdated = [isActiveUpdated copy];
_rotationUpdated = [rotationUpdated copy];
_warmupFrameCount = 100;
_previews = [[NSMutableArray alloc] init];
if (![self setupCaptureSession:[[AVCaptureSession alloc] init]]) {
return nil;
}
NSNotificationCenter *center = [NSNotificationCenter defaultCenter];
_orientation = deviceOrientation([[UIApplication sharedApplication] statusBarOrientation]);
_rotation = RTCVideoRotation_90;
switch (_orientation) {
case UIDeviceOrientationPortrait:
_rotation = RTCVideoRotation_90;
break;
case UIDeviceOrientationPortraitUpsideDown:
_rotation = RTCVideoRotation_270;
break;
case UIDeviceOrientationLandscapeLeft:
_rotation = useFrontCamera ? RTCVideoRotation_180 : RTCVideoRotation_0;
break;
case UIDeviceOrientationLandscapeRight:
_rotation = useFrontCamera ? RTCVideoRotation_0 : RTCVideoRotation_180;
break;
case UIDeviceOrientationFaceUp:
case UIDeviceOrientationFaceDown:
case UIDeviceOrientationUnknown:
// Ignore.
break;
}
if (_rotationUpdated) {
int angle = 0;
switch (_rotation) {
case RTCVideoRotation_0: {
angle = 0;
break;
}
case RTCVideoRotation_90: {
angle = 90;
break;
}
case RTCVideoRotation_180: {
angle = 180;
break;
}
case RTCVideoRotation_270: {
angle = 270;
break;
}
default: {
break;
}
}
_rotationUpdated(angle);
}
[center addObserver:self
selector:@selector(deviceOrientationDidChange:)
name:UIDeviceOrientationDidChangeNotification
object:nil];
[center addObserver:self
selector:@selector(handleCaptureSessionInterruption:)
name:AVCaptureSessionWasInterruptedNotification
object:_captureSession];
[center addObserver:self
selector:@selector(handleCaptureSessionInterruptionEnded:)
name:AVCaptureSessionInterruptionEndedNotification
object:_captureSession];
[center addObserver:self
selector:@selector(handleApplicationDidBecomeActive:)
name:UIApplicationDidBecomeActiveNotification
object:[UIApplication sharedApplication]];
[center addObserver:self
selector:@selector(handleApplicationWillEnterForeground:)
name:UIApplicationWillEnterForegroundNotification
object:[UIApplication sharedApplication]];
[center addObserver:self
selector:@selector(handleCaptureSessionRuntimeError:)
name:AVCaptureSessionRuntimeErrorNotification
object:_captureSession];
[center addObserver:self
selector:@selector(handleCaptureSessionDidStartRunning:)
name:AVCaptureSessionDidStartRunningNotification
object:_captureSession];
[center addObserver:self
selector:@selector(handleCaptureSessionDidStopRunning:)
name:AVCaptureSessionDidStopRunningNotification
object:_captureSession];
}
return self;
}
- (void)dealloc {
NSAssert(!_willBeRunning, @"Session was still running in RTCCameraVideoCapturer dealloc. Forgot to call stopCapture?");
[[NSNotificationCenter defaultCenter] removeObserver:self];
}
+ (NSArray<AVCaptureDevice *> *)captureDevices {
if (@available(iOS 10.0, *)) {
AVCaptureDeviceDiscoverySession *session = [AVCaptureDeviceDiscoverySession discoverySessionWithDeviceTypes:@[AVCaptureDeviceTypeBuiltInWideAngleCamera] mediaType:AVMediaTypeVideo position:AVCaptureDevicePositionUnspecified];
return session.devices;
} else {
NSMutableArray<AVCaptureDevice *> *result = [[NSMutableArray alloc] init];
for (AVCaptureDevice *device in [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]) {
if (device.position == AVCaptureDevicePositionFront || device.position == AVCaptureDevicePositionBack) {
[result addObject:device];
}
}
return result;
}
}
+ (NSArray<AVCaptureDeviceFormat *> *)supportedFormatsForDevice:(AVCaptureDevice *)device {
// Support opening the device in any format. We make sure it's converted to a format we
// can handle, if needed, in the method `-setupVideoDataOutput`.
return device.formats;
}
- (FourCharCode)preferredOutputPixelFormat {
return _preferredOutputPixelFormat;
}
- (void)startCaptureWithDevice:(AVCaptureDevice *)device
format:(AVCaptureDeviceFormat *)format
fps:(NSInteger)fps {
[self startCaptureWithDevice:device format:format fps:fps completionHandler:nil];
}
- (void)stopCapture {
_isActiveUpdated = nil;
[self stopCaptureWithCompletionHandler:nil];
}
- (void)setIsEnabled:(bool)isEnabled {
_isPaused = !isEnabled;
[self updateIsActiveValue];
}
- (void)setUncroppedSink:(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)sink {
dispatch_async(self.frameQueue, ^{
_uncroppedSink = sink;
});
}
- (int)getRotation {
switch (_rotation) {
case RTCVideoRotation_0:
return 0;
case RTCVideoRotation_90:
return 90;
case RTCVideoRotation_180:
return 180;
case RTCVideoRotation_270:
return 270;
default:
return 0;
}
}
- (void)addPreviewView:(VideoCaptureView *)previewView {
[_previews addObject:[[VideoCameraCapturerPreviewRecord alloc] initWithCaptureView:previewView]];
[previewView previewLayer].session = _captureSession;
}
- (void)addDirectSink:(std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)directSink {
_directSinks.push_back(directSink);
}
- (void)setPreferredCaptureAspectRatio:(float)aspectRatio {
dispatch_async(self.frameQueue, ^{
_aspectRatio = aspectRatio;
});
}
- (void)startCaptureWithDevice:(AVCaptureDevice *)device
format:(AVCaptureDeviceFormat *)format
fps:(NSInteger)fps
completionHandler:(nullable void (^)(NSError *))completionHandler {
_willBeRunning = true;
[RTCDispatcher
dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
RTCLogInfo("startCaptureWithDevice %@ @ %ld fps", format, (long)fps);
dispatch_async(dispatch_get_main_queue(), ^{
[[UIDevice currentDevice] beginGeneratingDeviceOrientationNotifications];
});
_currentDevice = device;
NSError *error = nil;
if (![_currentDevice lockForConfiguration:&error]) {
RTCLogError(@"Failed to lock device %@. Error: %@",
_currentDevice,
error.userInfo);
if (completionHandler) {
completionHandler(error);
}
_willBeRunning = false;
return;
}
[self reconfigureCaptureSessionInput];
[self updateDeviceCaptureFormat:format fps:fps];
[self updateVideoDataOutputPixelFormat:format];
[_captureSession startRunning];
[_currentDevice unlockForConfiguration];
_isRunning = YES;
if (completionHandler) {
completionHandler(nil);
}
}];
}
- (void)stopCaptureWithCompletionHandler:(nullable void (^)(void))completionHandler {
_willBeRunning = false;
[RTCDispatcher
dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
RTCLogInfo("Stop");
_currentDevice = nil;
for (AVCaptureDeviceInput *oldInput in [_captureSession.inputs copy]) {
[_captureSession removeInput:oldInput];
}
[_captureSession stopRunning];
dispatch_async(dispatch_get_main_queue(), ^{
[[UIDevice currentDevice] endGeneratingDeviceOrientationNotifications];
});
_isRunning = NO;
if (completionHandler) {
completionHandler();
}
}];
}
#pragma mark iOS notifications
#if TARGET_OS_IPHONE
- (void)deviceOrientationDidChange:(NSNotification *)notification {
[RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession block:^{
_didReceiveOrientationUpdate = true;
[self updateOrientation];
}];
}
#endif
#pragma mark AVCaptureVideoDataOutputSampleBufferDelegate
- (webrtc::scoped_refptr<webrtc::VideoFrameBuffer>)prepareI420Buffer:(CVPixelBufferRef)pixelBuffer {
if (!pixelBuffer) {
return nullptr;
}
const OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer);
CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
auto resultBuffer = rtc::make_ref_counted<webrtc::I420Buffer>(CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer));
switch (pixelFormat) {
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: {
const uint8_t* srcY =
static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
const int srcYStride = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
const uint8_t* srcUV =
static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
const int srcUVStride = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
// TODO(magjed): Use a frame buffer pool.
_nv12ToI420Scaler.NV12ToI420Scale(srcY,
srcYStride,
srcUV,
srcUVStride,
resultBuffer->width(),
resultBuffer->height(),
resultBuffer->MutableDataY(),
resultBuffer->StrideY(),
resultBuffer->MutableDataU(),
resultBuffer->StrideU(),
resultBuffer->MutableDataV(),
resultBuffer->StrideV(),
resultBuffer->width(),
resultBuffer->height());
break;
}
case kCVPixelFormatType_32BGRA:
case kCVPixelFormatType_32ARGB: {
return nullptr;
}
default: { RTC_DCHECK_NOTREACHED() << "Unsupported pixel format."; }
}
CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
return resultBuffer;
}
- (webrtc::scoped_refptr<webrtc::VideoFrameBuffer>)prepareNV12Buffer:(CVPixelBufferRef)pixelBuffer {
if (!pixelBuffer) {
return nullptr;
}
const OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer);
CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
switch (pixelFormat) {
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: {
const uint8_t* srcY =
static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
const int srcYStride = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
const uint8_t* srcUV =
static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
const int srcUVStride = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
const int srcWidth = (int)CVPixelBufferGetWidth(pixelBuffer);
const int srcHeight = (int)CVPixelBufferGetHeight(pixelBuffer);
int resultWidth = (int)(srcWidth * 0.8f);
resultWidth &= ~1;
int resultHeight = (int)(srcHeight * 0.8f);
resultHeight &= ~1;
webrtc::scoped_refptr<webrtc::NV12Buffer> resultBuffer = rtc::make_ref_counted<webrtc::NV12Buffer>(resultWidth, resultHeight, srcYStride, srcUVStride);
libyuv::NV12Scale(srcY, srcYStride, srcUV, srcUVStride,
resultWidth, resultHeight, resultBuffer->MutableDataY(),
resultBuffer->StrideY(), resultBuffer->MutableDataUV(), resultBuffer->StrideUV(), resultBuffer->width(),
resultBuffer->height(), libyuv::kFilterBilinear);
CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
return resultBuffer;
}
case kCVPixelFormatType_32BGRA:
case kCVPixelFormatType_32ARGB: {
CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
return nullptr;
}
default: { RTC_DCHECK_NOTREACHED() << "Unsupported pixel format."; }
}
return nullptr;
}
- (void)captureOutput:(AVCaptureOutput *)captureOutput
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
fromConnection:(AVCaptureConnection *)connection {
NSParameterAssert(captureOutput == _videoDataOutput);
int minWarmupFrameCount = 12;
_warmupFrameCount++;
if (_warmupFrameCount < minWarmupFrameCount) {
return;
}
if (CMSampleBufferGetNumSamples(sampleBuffer) != 1 || !CMSampleBufferIsValid(sampleBuffer) ||
!CMSampleBufferDataIsReady(sampleBuffer)) {
return;
}
CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
if (pixelBuffer == nil) {
return;
}
// Default to portrait orientation on iPhone.
BOOL usingFrontCamera = NO;
// Check the image's EXIF for the camera the image came from as the image could have been
// delayed as we set alwaysDiscardsLateVideoFrames to NO.
AVCaptureDevicePosition cameraPosition =
[AVCaptureSession devicePositionForSampleBuffer:sampleBuffer];
if (cameraPosition != AVCaptureDevicePositionUnspecified) {
usingFrontCamera = AVCaptureDevicePositionFront == cameraPosition;
} else {
AVCaptureDeviceInput *deviceInput =
(AVCaptureDeviceInput *)((AVCaptureInputPort *)connection.inputPorts.firstObject).input;
usingFrontCamera = AVCaptureDevicePositionFront == deviceInput.device.position;
}
int deviceRelativeVideoRotation = usingFrontCamera ? RTCVideoRotation_90 : RTCVideoRotation_90;
if (!_rotationLock) {
RTCVideoRotation updatedRotation = _rotation;
switch (_orientation) {
case UIDeviceOrientationPortrait:
updatedRotation = RTCVideoRotation_90;
break;
case UIDeviceOrientationPortraitUpsideDown:
updatedRotation = RTCVideoRotation_270;
break;
case UIDeviceOrientationLandscapeLeft:
updatedRotation = usingFrontCamera ? RTCVideoRotation_180 : RTCVideoRotation_0;
break;
case UIDeviceOrientationLandscapeRight:
updatedRotation = usingFrontCamera ? RTCVideoRotation_0 : RTCVideoRotation_180;
break;
case UIDeviceOrientationFaceUp:
case UIDeviceOrientationFaceDown:
case UIDeviceOrientationUnknown:
// Ignore.
break;
}
if (_rotation != updatedRotation) {
_rotation = updatedRotation;
if (_rotationUpdated) {
int angle = 0;
switch (_rotation) {
case RTCVideoRotation_0: {
angle = 0;
break;
}
case RTCVideoRotation_90: {
angle = 90;
break;
}
case RTCVideoRotation_180: {
angle = 180;
break;
}
case RTCVideoRotation_270: {
angle = 270;
break;
}
default: {
break;
}
}
_rotationUpdated(angle);
}
}
}
TGRTCCVPixelBuffer *rtcPixelBuffer = [[TGRTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer];
rtcPixelBuffer.shouldBeMirrored = usingFrontCamera;
rtcPixelBuffer.deviceRelativeVideoRotation = deviceRelativeVideoRotation;
TGRTCCVPixelBuffer *uncroppedRtcPixelBuffer = rtcPixelBuffer;
CGSize initialSize = CGSizeMake(uncroppedRtcPixelBuffer.width, uncroppedRtcPixelBuffer.height);
if (_aspectRatio > FLT_EPSILON) {
float aspect = 1.0f / _aspectRatio;
int width = rtcPixelBuffer.width;
int height = rtcPixelBuffer.height;
int cropX = 0;
int cropY = 0;
if (_keepLandscape && width > height) {
float aspectWidth = 404.0f;
float aspectHeight = 720.0f;
cropX = (int)((width - aspectWidth) / 2.0f);
cropY = (int)((height - aspectHeight) / 2.0f);
width = aspectWidth;
height = aspectHeight;
} else {
float aspectWidth = width;
float aspectHeight = ((float)(width)) / aspect;
cropX = (int)((width - aspectWidth) / 2.0f);
cropY = (int)((height - aspectHeight) / 2.0f);
width = (int)aspectWidth;
width &= ~1;
height = (int)aspectHeight;
height &= ~1;
}
height = MIN(rtcPixelBuffer.height, height + 16);
if (width < rtcPixelBuffer.width || height < rtcPixelBuffer.height) {
rtcPixelBuffer = [[TGRTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer adaptedWidth:width adaptedHeight:height cropWidth:width cropHeight:height cropX:cropX cropY:cropY];
rtcPixelBuffer.shouldBeMirrored = usingFrontCamera;
rtcPixelBuffer.deviceRelativeVideoRotation = deviceRelativeVideoRotation;
CVPixelBufferRef outputPixelBufferRef = NULL;
OSType pixelFormat = CVPixelBufferGetPixelFormatType(rtcPixelBuffer.pixelBuffer);
CVPixelBufferCreate(NULL, width, height, pixelFormat, NULL, &outputPixelBufferRef);
if (outputPixelBufferRef) {
int bufferSize = [rtcPixelBuffer bufferSizeForCroppingAndScalingToWidth:width height:height];
if (_croppingBuffer.size() < bufferSize) {
_croppingBuffer.resize(bufferSize);
}
if ([rtcPixelBuffer cropAndScaleTo:outputPixelBufferRef withTempBuffer:_croppingBuffer.data()]) {
rtcPixelBuffer = [[TGRTCCVPixelBuffer alloc] initWithPixelBuffer:outputPixelBufferRef];
rtcPixelBuffer.shouldBeMirrored = usingFrontCamera;
rtcPixelBuffer.deviceRelativeVideoRotation = deviceRelativeVideoRotation;
}
CVPixelBufferRelease(outputPixelBufferRef);
}
}
}
int64_t timeStampNs = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) * kNanosecondsPerSecond;
//RTCVideoFrame *videoFrame = [[RTCVideoFrame alloc] initWithBuffer:rtcPixelBuffer rotation:_rotation timeStampNs:timeStampNs];
//RTCVideoFrame *videoFrame = [[RTCVideoFrame alloc] initWithBuffer:(id<RTCVideoFrameBuffer>)[rtcPixelBuffer toI420] rotation:_rotation timeStampNs:timeStampNs];
webrtc::VideoRotation rotation = static_cast<webrtc::VideoRotation>(_rotation);
int previewRotation = 0;
CGSize previewSize = initialSize;
if (rotation == 90 || rotation == 270) {
previewSize = CGSizeMake(previewSize.height, previewSize.width);
}
for (VideoCameraCapturerPreviewRecord *record in _previews) {
dispatch_async(dispatch_get_main_queue(), ^{
VideoCaptureView *captureView = record.view;
[captureView onFrameGenerated:previewSize isMirrored:true rotation:previewRotation];
});
}
auto i420Buffer = [self prepareI420Buffer:[rtcPixelBuffer pixelBuffer]];
if (!_isPaused && i420Buffer) {
auto videoFrame = webrtc::VideoFrame::Builder()
.set_video_frame_buffer(i420Buffer)
.set_rotation(rotation)
.set_timestamp_us(timeStampNs)
.build();
if (getObjCVideoSource(_source)->OnCapturedFrame(videoFrame)) {
if (!_directSinks.empty()) {
for (const auto &it : _directSinks) {
if (const auto value = it.lock()) {
value->OnFrame(videoFrame);
}
}
}
}
if (uncroppedRtcPixelBuffer) {
const auto uncroppedSink = _uncroppedSink.lock();
if (uncroppedSink) {
int64_t timeStampNs = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) *
kNanosecondsPerSecond;
RTCVideoFrame *frame = [[RTCVideoFrame alloc] initWithBuffer:uncroppedRtcPixelBuffer rotation:_rotation timeStampNs:timeStampNs];
const int64_t timestamp_us = frame.timeStampNs / rtc::kNumNanosecsPerMicrosec;
webrtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer;
buffer = new rtc::RefCountedObject<webrtc::ObjCFrameBuffer>(frame.buffer);
webrtc::VideoRotation rotation = static_cast<webrtc::VideoRotation>(frame.rotation);
uncroppedSink->OnFrame(webrtc::VideoFrame::Builder()
.set_video_frame_buffer(buffer)
.set_rotation(rotation)
.set_timestamp_us(timestamp_us)
.build());
}
}
}
}
- (void)captureOutput:(AVCaptureOutput *)captureOutput
didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer
fromConnection:(AVCaptureConnection *)connection {
NSString *droppedReason =
(__bridge NSString *)CMGetAttachment(sampleBuffer, kCMSampleBufferAttachmentKey_DroppedFrameReason, nil);
RTCLogError(@"Dropped sample buffer. Reason: %@", droppedReason);
}
#pragma mark - AVCaptureSession notifications
- (void)handleCaptureSessionInterruption:(NSNotification *)notification {
NSString *reasonString = nil;
NSNumber *reason = notification.userInfo[AVCaptureSessionInterruptionReasonKey];
if (reason) {
switch (reason.intValue) {
case AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableInBackground:
reasonString = @"VideoDeviceNotAvailableInBackground";
break;
case AVCaptureSessionInterruptionReasonAudioDeviceInUseByAnotherClient:
reasonString = @"AudioDeviceInUseByAnotherClient";
break;
case AVCaptureSessionInterruptionReasonVideoDeviceInUseByAnotherClient:
reasonString = @"VideoDeviceInUseByAnotherClient";
break;
case AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableWithMultipleForegroundApps:
reasonString = @"VideoDeviceNotAvailableWithMultipleForegroundApps";
break;
}
}
RTCLog(@"Capture session interrupted: %@", reasonString);
}
- (void)handleCaptureSessionInterruptionEnded:(NSNotification *)notification {
RTCLog(@"Capture session interruption ended.");
}
- (void)handleCaptureSessionRuntimeError:(NSNotification *)notification {
NSError *error = [notification.userInfo objectForKey:AVCaptureSessionErrorKey];
RTCLogError(@"Capture session runtime error: %@", error);
[RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
if (error.code == AVErrorMediaServicesWereReset) {
[self handleNonFatalError];
} else {
[self handleFatalError];
}
}];
}
- (void)handleCaptureSessionDidStartRunning:(NSNotification *)notification {
RTCLog(@"Capture session started.");
[RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
// If we successfully restarted after an unknown error,
// allow future retries on fatal errors.
_hasRetriedOnFatalError = NO;
}];
_inForegroundValue = true;
[self updateIsActiveValue];
}
- (void)handleCaptureSessionDidStopRunning:(NSNotification *)notification {
RTCLog(@"Capture session stopped.");
_inForegroundValue = false;
[self updateIsActiveValue];
}
- (void)updateIsActiveValue {
bool isActive = _inForegroundValue && !_isPaused;
if (isActive != _isActiveValue) {
_isActiveValue = isActive;
if (_isActiveUpdated) {
_isActiveUpdated(_isActiveValue);
}
}
}
- (void)handleFatalError {
[RTCDispatcher
dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
if (!_hasRetriedOnFatalError) {
RTCLogWarning(@"Attempting to recover from fatal capture error.");
[self handleNonFatalError];
_hasRetriedOnFatalError = YES;
} else {
RTCLogError(@"Previous fatal error recovery failed.");
}
}];
}
- (void)handleNonFatalError {
[RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
RTCLog(@"Restarting capture session after error.");
if (_isRunning) {
[_captureSession startRunning];
}
}];
}
#pragma mark - UIApplication notifications
- (void)handleApplicationDidBecomeActive:(NSNotification *)notification {
[RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
if (_isRunning && !_captureSession.isRunning) {
RTCLog(@"Restarting capture session on active.");
_warmupFrameCount = 0;
[_captureSession startRunning];
}
}];
}
- (void)handleApplicationWillEnterForeground:(NSNotification *)notification {
[RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
RTCLog(@"Resetting warmup due to backgrounding.");
_warmupFrameCount = 0;
}];
}
#pragma mark - Private
- (dispatch_queue_t)frameQueue {
if (!_frameQueue) {
_frameQueue =
dispatch_queue_create("org.webrtc.cameravideocapturer.video", DISPATCH_QUEUE_SERIAL);
dispatch_set_target_queue(_frameQueue,
dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0));
}
return _frameQueue;
}
- (BOOL)setupCaptureSession:(AVCaptureSession *)captureSession {
NSAssert(_captureSession == nil, @"Setup capture session called twice.");
_captureSession = captureSession;
_captureSession.sessionPreset = AVCaptureSessionPresetInputPriority;
_captureSession.usesApplicationAudioSession = true;
if (@available(iOS 16.0, *)) {
if (_captureSession.isMultitaskingCameraAccessSupported) {
_captureSession.multitaskingCameraAccessEnabled = true;
}
}
[self setupVideoDataOutput];
// Add the output.
if (![_captureSession canAddOutput:_videoDataOutput]) {
RTCLogError(@"Video data output unsupported.");
return NO;
}
[_captureSession addOutput:_videoDataOutput];
return YES;
}
- (void)setupVideoDataOutput {
NSAssert(_videoDataOutput == nil, @"Setup video data output called twice.");
AVCaptureVideoDataOutput *videoDataOutput = [[AVCaptureVideoDataOutput alloc] init];
// `videoDataOutput.availableVideoCVPixelFormatTypes` returns the pixel formats supported by the
// device with the most efficient output format first. Find the first format that we support.
NSSet<NSNumber *> *supportedPixelFormats = [TGRTCCVPixelBuffer supportedPixelFormats];
NSMutableOrderedSet *availablePixelFormats =
[NSMutableOrderedSet orderedSetWithArray:videoDataOutput.availableVideoCVPixelFormatTypes];
[availablePixelFormats intersectSet:supportedPixelFormats];
NSNumber *pixelFormat = availablePixelFormats.firstObject;
NSAssert(pixelFormat, @"Output device has no supported formats.");
_preferredOutputPixelFormat = [pixelFormat unsignedIntValue];
_outputPixelFormat = _preferredOutputPixelFormat;
videoDataOutput.videoSettings = @{(NSString *)kCVPixelBufferPixelFormatTypeKey : pixelFormat};
videoDataOutput.alwaysDiscardsLateVideoFrames = NO;
[videoDataOutput setSampleBufferDelegate:self queue:self.frameQueue];
_videoDataOutput = videoDataOutput;
}
- (void)updateVideoDataOutputPixelFormat:(AVCaptureDeviceFormat *)format {
FourCharCode mediaSubType = CMFormatDescriptionGetMediaSubType(format.formatDescription);
if (![[TGRTCCVPixelBuffer supportedPixelFormats] containsObject:@(mediaSubType)]) {
mediaSubType = _preferredOutputPixelFormat;
}
if (mediaSubType != _outputPixelFormat) {
_outputPixelFormat = mediaSubType;
_videoDataOutput.videoSettings =
@{ (NSString *)kCVPixelBufferPixelFormatTypeKey : @(mediaSubType) };
}
}
#pragma mark - Private, called inside capture queue
- (void)updateDeviceCaptureFormat:(AVCaptureDeviceFormat *)format fps:(NSInteger)fps {
NSAssert([RTCDispatcher isOnQueueForType:RTCDispatcherTypeCaptureSession],
@"updateDeviceCaptureFormat must be called on the capture queue.");
@try {
_currentDevice.activeFormat = format;
_currentDevice.activeVideoMinFrameDuration = CMTimeMake(1, (int32_t)fps);
} @catch (NSException *exception) {
RTCLogError(@"Failed to set active format!\n User info:%@", exception.userInfo);
return;
}
}
- (void)reconfigureCaptureSessionInput {
NSAssert([RTCDispatcher isOnQueueForType:RTCDispatcherTypeCaptureSession],
@"reconfigureCaptureSessionInput must be called on the capture queue.");
NSError *error = nil;
AVCaptureDeviceInput *input =
[AVCaptureDeviceInput deviceInputWithDevice:_currentDevice error:&error];
if (!input) {
RTCLogError(@"Failed to create front camera input: %@", error.localizedDescription);
return;
}
[_captureSession beginConfiguration];
for (AVCaptureDeviceInput *oldInput in [_captureSession.inputs copy]) {
[_captureSession removeInput:oldInput];
}
if ([_captureSession canAddInput:input]) {
[_captureSession addInput:input];
} else {
RTCLogError(@"Cannot add camera as an input to the session.");
}
[_captureSession commitConfiguration];
}
- (void)updateOrientation {
NSAssert([RTCDispatcher isOnQueueForType:RTCDispatcherTypeCaptureSession],
@"updateOrientation must be called on the capture queue.");
if (_didReceiveOrientationUpdate) {
_orientation = [UIDevice currentDevice].orientation;
}
}
@end
//

View file

@ -0,0 +1,35 @@
#ifndef TGCALLS_VIDEO_CAMERA_CAPTURER_MAC_H
#define TGCALLS_VIDEO_CAMERA_CAPTURER_MAC_H
#ifndef WEBRTC_IOS
#import <Foundation/Foundation.h>
#import <AVFoundation/AVFoundation.h>
#include <memory>
#include "api/scoped_refptr.h"
#include "api/media_stream_interface.h"
@protocol CapturerInterface
- (void)start;
- (void)stop;
- (void)setIsEnabled:(bool)isEnabled;
- (void)setUncroppedSink:(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)sink;
- (void)setPreferredCaptureAspectRatio:(float)aspectRatio;
- (void)setOnFatalError:(std::function<void()>)error;
- (void)setOnPause:(std::function<void(bool)>)pause;
@end
@interface VideoCameraCapturer : NSObject<CapturerInterface>
+ (NSArray<AVCaptureDevice *> *)captureDevices;
+ (NSArray<AVCaptureDeviceFormat *> *)supportedFormatsForDevice:(AVCaptureDevice *)device;
- (instancetype)initWithSource:(rtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source isActiveUpdated:(void (^)(bool))isActiveUpdated;
- (void)setupCaptureWithDevice:(AVCaptureDevice *)device format:(AVCaptureDeviceFormat *)format fps:(NSInteger)fps;
- (void)setUncroppedSink:(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)sink;
- (BOOL)deviceIsCaptureCompitable:(AVCaptureDevice *)device;
@end
#endif //WEBRTC_MAC
#endif

View file

@ -0,0 +1,725 @@
#include "VideoCameraCapturerMac.h"
#import <AVFoundation/AVFoundation.h>
#import "TGRTCCVPixelBuffer.h"
#include "rtc_base/logging.h"
#import "base/RTCLogging.h"
#import "base/RTCVideoFrameBuffer.h"
#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
#import "sdk/objc/native/src/objc_video_track_source.h"
#import "sdk/objc/native/src/objc_frame_buffer.h"
#import <CoreMediaIO/CMIOHardware.h>
#import "TGCMIODevice.h"
#import "helpers/AVCaptureSession+DevicePosition.h"
#import "helpers/RTCDispatcher+Private.h"
#import "base/RTCVideoFrame.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "pc/video_track_source_proxy.h"
#include "third_party/libyuv/include/libyuv.h"
#include "DarwinVideoSource.h"
struct CameraFrameSize {
int width = 0;
int height = 0;
};
CameraFrameSize AspectFitted(CameraFrameSize from, CameraFrameSize to) {
double scale = std::min(
from.width / std::max(1., double(to.width)),
from.height / std::max(1., double(to.height)));
return {
int(std::ceil(to.width * scale)),
int(std::ceil(to.height * scale))
};
}
static const int64_t kNanosecondsPerSecond = 1000000000;
static tgcalls::DarwinVideoTrackSource *getObjCVideoSource(const rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> nativeSource) {
webrtc::VideoTrackSourceProxy *proxy_source =
static_cast<webrtc::VideoTrackSourceProxy *>(nativeSource.get());
return static_cast<tgcalls::DarwinVideoTrackSource *>(proxy_source->internal());
}
@interface RTCCVPixelBuffer (CustomCropping)
@end
@implementation RTCCVPixelBuffer (CustomCropping)
- (BOOL)custom_cropAndScaleTo:(CVPixelBufferRef)outputPixelBuffer
withTempBuffer:(nullable uint8_t*)tmpBuffer {
const OSType srcPixelFormat = CVPixelBufferGetPixelFormatType(self.pixelBuffer);
const OSType dstPixelFormat = CVPixelBufferGetPixelFormatType(outputPixelBuffer);
switch (srcPixelFormat) {
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: {
size_t dstWidth = CVPixelBufferGetWidth(outputPixelBuffer);
size_t dstHeight = CVPixelBufferGetHeight(outputPixelBuffer);
if (dstWidth > 0 && dstHeight > 0) {
RTC_DCHECK(dstPixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ||
dstPixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange);
if ([self requiresScalingToWidth:(int)dstWidth height:(int)dstHeight]) {
RTC_DCHECK(tmpBuffer);
}
[self custom_cropAndScaleNV12To:outputPixelBuffer withTempBuffer:tmpBuffer];
}
break;
}
case kCVPixelFormatType_32BGRA:
case kCVPixelFormatType_32ARGB: {
RTC_DCHECK(srcPixelFormat == dstPixelFormat);
[self custom_cropAndScaleARGBTo:outputPixelBuffer];
break;
}
default: { RTC_DCHECK_NOTREACHED() << "Unsupported pixel format."; }
}
return YES;
}
- (void)custom_cropAndScaleNV12To:(CVPixelBufferRef)outputPixelBuffer withTempBuffer:(uint8_t*)tmpBuffer {
// Prepare output pointers.
CVReturn cvRet = CVPixelBufferLockBaseAddress(outputPixelBuffer, 0);
if (cvRet != kCVReturnSuccess) {
RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
}
const int dstWidth = (int)CVPixelBufferGetWidth(outputPixelBuffer);
const int dstHeight = (int)CVPixelBufferGetHeight(outputPixelBuffer);
uint8_t* dstY =
reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 0));
const int dstYStride = (int)CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 0);
uint8_t* dstUV =
reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 1));
const int dstUVStride = (int)CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 1);
// Prepare source pointers.
CVPixelBufferLockBaseAddress(self.pixelBuffer, kCVPixelBufferLock_ReadOnly);
const uint8_t* srcY = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(self.pixelBuffer, 0));
const int srcYStride = (int)CVPixelBufferGetBytesPerRowOfPlane(self.pixelBuffer, 0);
const uint8_t* srcUV = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(self.pixelBuffer, 1));
const int srcUVStride = (int)CVPixelBufferGetBytesPerRowOfPlane(self.pixelBuffer, 1);
// Crop just by modifying pointers.
srcY += srcYStride * self.cropY + self.cropX;
srcUV += srcUVStride * (self.cropY / 2) + self.cropX;
webrtc::NV12Scale(tmpBuffer,
srcY,
srcYStride,
srcUV,
srcUVStride,
self.cropWidth,
self.cropHeight,
dstY,
dstYStride,
dstUV,
dstUVStride,
dstWidth,
dstHeight);
CVPixelBufferUnlockBaseAddress(self.pixelBuffer, kCVPixelBufferLock_ReadOnly);
CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0);
}
- (void)custom_cropAndScaleARGBTo:(CVPixelBufferRef)outputPixelBuffer {
// Prepare output pointers.
CVReturn cvRet = CVPixelBufferLockBaseAddress(outputPixelBuffer, 0);
if (cvRet != kCVReturnSuccess) {
RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
}
const int dstWidth = (int)CVPixelBufferGetWidth(outputPixelBuffer);
const int dstHeight = (int)CVPixelBufferGetHeight(outputPixelBuffer);
uint8_t* dst = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddress(outputPixelBuffer));
const int dstStride = (int)CVPixelBufferGetBytesPerRow(outputPixelBuffer);
// Prepare source pointers.
CVPixelBufferLockBaseAddress(self.pixelBuffer, kCVPixelBufferLock_ReadOnly);
const uint8_t* src = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(self.pixelBuffer));
const int srcStride = (int)CVPixelBufferGetBytesPerRow(self.pixelBuffer);
// Crop just by modifying pointers. Need to ensure that src pointer points to a byte corresponding
// to the start of a new pixel (byte with B for BGRA) so that libyuv scales correctly.
const int bytesPerPixel = 4;
src += srcStride * self.cropY + (self.cropX * bytesPerPixel);
// kCVPixelFormatType_32BGRA corresponds to libyuv::FOURCC_ARGB
libyuv::ARGBScale(src,
srcStride,
self.cropWidth,
self.cropHeight,
dst,
dstStride,
dstWidth,
dstHeight,
libyuv::kFilterBox);
CVPixelBufferUnlockBaseAddress(self.pixelBuffer, kCVPixelBufferLock_ReadOnly);
CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0);
}
@end
@interface VideoCameraCapturer () <AVCaptureVideoDataOutputSampleBufferDelegate> {
rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> _source;
dispatch_queue_t _frameQueue;
AVCaptureDevice *_currentDevice;
AVCaptureInput *_currentInput;
// Live on RTCDispatcherTypeCaptureSession.
BOOL _hasRetriedOnFatalError;
BOOL _hadFatalError;
BOOL _isRunning;
BOOL _shouldBeMirrored;
// Live on RTCDispatcherTypeCaptureSession and main thread.
std::atomic<bool> _willBeRunning;
AVCaptureVideoDataOutput *_videoDataOutput;
AVCaptureSession *_captureSession;
AVCaptureConnection *_videoConnection;
AVCaptureDevice *_videoDevice;
AVCaptureDeviceInput *_videoInputDevice;
FourCharCode _preferredOutputPixelFormat;
FourCharCode _outputPixelFormat;
RTCVideoRotation _rotation;
// Live on mainThread.
void (^_isActiveUpdated)(bool);
bool _isActiveValue;
bool _inForegroundValue;
// Live on frameQueue and main thread.
std::atomic<bool> _isPaused;
std::atomic<int> _skippedFrame;
// Live on frameQueue;
float _aspectRatio;
std::vector<uint8_t> _croppingBuffer;
std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> _uncroppedSink;
std::function<void ()> _onFatalError;
int _warmupFrameCount;
}
@end
@implementation VideoCameraCapturer
- (instancetype)initWithSource:(rtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source isActiveUpdated:(void (^)(bool))isActiveUpdated {
self = [super init];
if (self != nil) {
_source = source;
_isActiveUpdated = [isActiveUpdated copy];
_isActiveValue = true;
_inForegroundValue = true;
_isPaused = false;
_skippedFrame = 0;
_rotation = RTCVideoRotation_0;
_warmupFrameCount = 100;
if (![self setupCaptureSession:[[AVCaptureSession alloc] init]]) {
return nil;
}
}
return self;
}
- (void)dealloc {
NSAssert(!_willBeRunning, @"Session was still running in RTCCameraVideoCapturer dealloc. Forgot to call stopCapture?");
[[NSNotificationCenter defaultCenter] removeObserver:self];
}
+ (NSArray<AVCaptureDevice *> *)captureDevices {
AVCaptureDevice * defaultDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
NSMutableArray<AVCaptureDevice *> * devices = [[AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo] mutableCopy];
[devices addObjectsFromArray:[AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed]];
if ([devices count] > 0) {
[devices insertObject:defaultDevice atIndex:0];
}
return devices;
}
- (BOOL)deviceIsCaptureCompitable:(AVCaptureDevice *)device {
if (![device isConnected] || [device isSuspended]) {
return NO;
}
AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:device error:nil];
return [_captureSession canAddInput:input];
}
+ (NSArray<AVCaptureDeviceFormat *> *)supportedFormatsForDevice:(AVCaptureDevice *)device {
// Support opening the device in any format. We make sure it's converted to a format we
// can handle, if needed, in the method `-setupVideoDataOutput`.
return device.formats;
}
- (FourCharCode)preferredOutputPixelFormat {
return _preferredOutputPixelFormat;
}
- (void)setupCaptureWithDevice:(AVCaptureDevice *)device
format:(AVCaptureDeviceFormat *)format
fps:(NSInteger)fps {
[self setupCaptureWithDevice:device format:format fps:fps completionHandler:nil];
}
-(void)setOnFatalError:(std::function<void ()>)error {
if (!self->_hadFatalError) {
_onFatalError = std::move(error);
} else if (error) {
error();
}
}
-(void)setOnPause:(std::function<void (bool)>)pause {
}
- (void)stop {
_isActiveUpdated = nil;
[self stopCaptureWithCompletionHandler:nil];
}
- (void)setIsEnabled:(bool)isEnabled {
BOOL updated = _isPaused != !isEnabled;
_isPaused = !isEnabled;
_skippedFrame = 0;
if (updated) {
if (_isPaused) {
// [RTCDispatcher
// dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
// block:^{
[self->_captureSession stopRunning];
self->_isRunning = NO;
// }];
} else {
// [RTCDispatcher
// dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
// block:^{
[self->_captureSession startRunning];
self->_isRunning = YES;
// }];
}
}
[self updateIsActiveValue];
}
- (void)setUncroppedSink:(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)sink {
dispatch_async(self.frameQueue, ^{
self->_uncroppedSink = sink;
});
}
- (void)setPreferredCaptureAspectRatio:(float)aspectRatio {
dispatch_async(self.frameQueue, ^{
self->_aspectRatio = aspectRatio;
});
}
- (void)updateIsActiveValue {
bool isActive = _inForegroundValue && !_isPaused;
if (isActive != _isActiveValue) {
_isActiveValue = isActive;
if (_isActiveUpdated) {
_isActiveUpdated(_isActiveValue);
}
}
}
- (void)setupCaptureWithDevice:(AVCaptureDevice *)device
format:(AVCaptureDeviceFormat *)format
fps:(NSInteger)fps
completionHandler:(nullable void (^)(NSError *))completionHandler {
CMIOObjectPropertyAddress latency_pa = {
kCMIODevicePropertyLatency,
kCMIOObjectPropertyScopeWildcard,
kCMIOObjectPropertyElementWildcard
};
UInt32 dataSize = 0;
NSNumber *_connectionID = ((NSNumber *)[device valueForKey:@"_connectionID"]);
CMIODeviceID deviceId = (CMIODeviceID)[_connectionID intValue];
if (device) {
if (CMIOObjectGetPropertyDataSize(deviceId, &latency_pa, 0, nil, &dataSize) == noErr) {
_shouldBeMirrored = NO;
} else {
_shouldBeMirrored = YES;
}
} else {
_shouldBeMirrored = [device hasMediaType:AVMediaTypeVideo];
}
// [RTCDispatcher
// dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
// block:^{
RTCLogInfo("startCaptureWithDevice %@ @ %ld fps", format, (long)fps);
NSError *error = nil;
self->_currentDevice = device;
self->_currentInput = [[AVCaptureDeviceInput alloc] initWithDevice:device error:&error];
if (![self->_currentDevice lockForConfiguration:&error]) {
RTCLogError(@"Failed to lock device %@. Error: %@",
self->_currentDevice,
error.userInfo);
if (completionHandler) {
completionHandler(error);
}
self->_willBeRunning = false;
return;
}
[self updateDeviceCaptureFormat:format fps:fps];
[self updateVideoDataOutputPixelFormat:format];
[self->_currentDevice unlockForConfiguration];
[self reconfigureCaptureSessionInput];
if (completionHandler) {
completionHandler(nil);
}
// }];
}
-(void)start {
_willBeRunning = true;
// [RTCDispatcher
// dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
// block:^{
[self->_captureSession startRunning];
self->_isRunning = YES;
// }];
}
- (void)stopCaptureWithCompletionHandler:(nullable void (^)(void))completionHandler {
_willBeRunning = false;
// [RTCDispatcher
// dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
// block:^{
RTCLogInfo("Stop");
self->_currentDevice = nil;
for (AVCaptureDeviceInput *oldInput in [self->_captureSession.inputs copy]) {
[self->_captureSession removeInput:oldInput];
}
[self->_captureSession stopRunning];
self->_isRunning = NO;
if (completionHandler) {
completionHandler();
}
// }];
}
#pragma mark AVCaptureVideoDataOutputSampleBufferDelegate
- (void)captureOutput:(AVCaptureOutput *)captureOutput
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
fromConnection:(AVCaptureConnection *)connection {
NSParameterAssert(captureOutput == _videoDataOutput);
int minWarmupFrameCount = 12;
_warmupFrameCount++;
if (_warmupFrameCount < minWarmupFrameCount) {
return;
}
if (CMSampleBufferGetNumSamples(sampleBuffer) != 1 || !CMSampleBufferIsValid(sampleBuffer) ||
!CMSampleBufferDataIsReady(sampleBuffer)) {
return;
}
CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
if (pixelBuffer == nil) {
return;
}
int width = (int)CVPixelBufferGetWidth(pixelBuffer);
int height = (int)CVPixelBufferGetHeight(pixelBuffer);
CameraFrameSize fittedSize = { width, height };
fittedSize.width -= (fittedSize.width % 32);
fittedSize.height -= (fittedSize.height % 4);
TGRTCCVPixelBuffer *rtcPixelBuffer = [[TGRTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer adaptedWidth:fittedSize.width adaptedHeight:fittedSize.height cropWidth:width cropHeight:height cropX:0 cropY:0];
rtcPixelBuffer.shouldBeMirrored = _shouldBeMirrored;
if (!_isPaused && _uncroppedSink) {
int64_t timeStampNs = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) *
kNanosecondsPerSecond;
RTCVideoFrame *frame = [[RTCVideoFrame alloc] initWithBuffer:rtcPixelBuffer
rotation:_rotation
timeStampNs:timeStampNs];
const int64_t timestamp_us = frame.timeStampNs / rtc::kNumNanosecsPerMicrosec;
rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer;
buffer = new rtc::RefCountedObject<webrtc::ObjCFrameBuffer>(frame.buffer);
webrtc::VideoRotation rotation = static_cast<webrtc::VideoRotation>(frame.rotation);
_uncroppedSink->OnFrame(webrtc::VideoFrame::Builder()
.set_video_frame_buffer(buffer)
.set_rotation(rotation)
.set_timestamp_us(timestamp_us)
.build());
}
int64_t timeStampNs = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) *
kNanosecondsPerSecond;
RTCVideoFrame *videoFrame = [[RTCVideoFrame alloc] initWithBuffer:rtcPixelBuffer
rotation:_rotation
timeStampNs:timeStampNs];
if (!_isPaused) {
getObjCVideoSource(_source)->OnCapturedFrame(videoFrame);
}
_skippedFrame = MIN(_skippedFrame + 1, 16);
}
- (void)captureOutput:(AVCaptureOutput *)captureOutput
didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer
fromConnection:(AVCaptureConnection *)connection {
NSString *droppedReason =
(__bridge NSString *)CMGetAttachment(sampleBuffer, kCMSampleBufferAttachmentKey_DroppedFrameReason, nil);
RTCLogError(@"Dropped sample buffer. Reason: %@", droppedReason);
}
#pragma mark - AVCaptureSession notifications
- (void)handleCaptureSessionInterruption:(NSNotification *)notification {
}
- (void)handleCaptureSessionInterruptionEnded:(NSNotification *)notification {
RTCLog(@"Capture session interruption ended.");
}
- (void)handleCaptureSessionRuntimeError:(NSNotification *)notification {
NSError *error = [notification.userInfo objectForKey:AVCaptureSessionErrorKey];
RTCLogError(@"Capture session runtime error: %@", error);
// [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
// block:^{
[self handleFatalError];
// }];
}
- (void)handleCaptureSessionDidStartRunning:(NSNotification *)notification {
RTCLog(@"Capture session started.");
// [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
// block:^{
// If we successfully restarted after an unknown error,
// allow future retries on fatal errors.
self->_hasRetriedOnFatalError = NO;
self->_hadFatalError = NO;
// }];
_inForegroundValue = true;
[self updateIsActiveValue];
}
- (void)handleCaptureSessionDidStopRunning:(NSNotification *)notification {
RTCLog(@"Capture session stopped.");
_inForegroundValue = false;
[self updateIsActiveValue];
}
- (void)handleFatalError {
// [RTCDispatcher
// dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
// block:^{
if (!self->_hasRetriedOnFatalError) {
RTCLogWarning(@"Attempting to recover from fatal capture error.");
[self handleNonFatalError];
self->_warmupFrameCount = 0;
self->_hasRetriedOnFatalError = YES;
} else {
RTCLogError(@"Previous fatal error recovery failed.");
if (_onFatalError) {
_onFatalError();
} else {
self->_hadFatalError = YES;
}
}
// }];
}
- (void)handleNonFatalError {
// [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
// block:^{
RTCLog(@"Restarting capture session after error.");
if (self->_isRunning) {
self->_warmupFrameCount = 0;
[self->_captureSession startRunning];
}
// }];
}
#pragma mark - UIApplication notifications
- (void)handleApplicationDidBecomeActive:(NSNotification *)notification {
// [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
// block:^{
if (self->_isRunning && !self->_captureSession.isRunning) {
RTCLog(@"Restarting capture session on active.");
self->_warmupFrameCount = 0;
[self->_captureSession startRunning];
}
// }];
}
#pragma mark - Private
- (dispatch_queue_t)frameQueue {
if (!_frameQueue) {
_frameQueue =
dispatch_queue_create("org.webrtc.cameravideocapturer.video", DISPATCH_QUEUE_SERIAL);
dispatch_set_target_queue(_frameQueue,
dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0));
}
return _frameQueue;
}
- (BOOL)setupCaptureSession:(AVCaptureSession *)captureSession {
NSAssert(_captureSession == nil, @"Setup capture session called twice.");
_captureSession = captureSession;
[self setupVideoDataOutput];
// Add the output.
if (![_captureSession canAddOutput:_videoDataOutput]) {
RTCLogError(@"Video data output unsupported.");
return NO;
}
[_captureSession addOutput:_videoDataOutput];
return YES;
}
- (void)setupVideoDataOutput {
NSAssert(_videoDataOutput == nil, @"Setup video data output called twice.");
AVCaptureVideoDataOutput *videoDataOutput = [[AVCaptureVideoDataOutput alloc] init];
// `videoDataOutput.availableVideoCVPixelFormatTypes` returns the pixel formats supported by the
// device with the most efficient output format first. Find the first format that we support.
NSSet<NSNumber *> *supportedPixelFormats = [RTCCVPixelBuffer supportedPixelFormats];
NSMutableOrderedSet *availablePixelFormats =
[NSMutableOrderedSet orderedSetWithArray:videoDataOutput.availableVideoCVPixelFormatTypes];
[availablePixelFormats intersectSet:supportedPixelFormats];
NSNumber *pixelFormat = availablePixelFormats.firstObject;
NSAssert(pixelFormat, @"Output device has no supported formats.");
_preferredOutputPixelFormat = [pixelFormat unsignedIntValue];
_outputPixelFormat = _preferredOutputPixelFormat;
videoDataOutput.videoSettings = @{(NSString *)kCVPixelBufferPixelFormatTypeKey : pixelFormat};
videoDataOutput.alwaysDiscardsLateVideoFrames = YES;
[videoDataOutput setSampleBufferDelegate:self queue:self.frameQueue];
_videoDataOutput = videoDataOutput;
}
- (void)updateVideoDataOutputPixelFormat:(AVCaptureDeviceFormat *)format {
FourCharCode mediaSubType = CMFormatDescriptionGetMediaSubType(format.formatDescription);
if (![[RTCCVPixelBuffer supportedPixelFormats] containsObject:@(mediaSubType)]) {
mediaSubType = _preferredOutputPixelFormat;
}
CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(format.formatDescription);
if (mediaSubType != _outputPixelFormat) {
_outputPixelFormat = mediaSubType;
_videoDataOutput.videoSettings =
@{ (NSString *)kCVPixelBufferPixelFormatTypeKey : @(mediaSubType) };
} else {
// _videoDataOutput.videoSettings =
// @{ (NSString *)kCVPixelBufferWidthKey: @(dimensions.width), (NSString *)kCVPixelBufferHeightKey: @(dimensions.height) };
}
AVCaptureConnection *connection = [_videoDataOutput connectionWithMediaType:AVMediaTypeVideo];
}
#pragma mark - Private, called inside capture queue
- (void)updateDeviceCaptureFormat:(AVCaptureDeviceFormat *)format fps:(NSInteger)fps {
// NSAssert([RTCDispatcher isOnQueueForType:RTCDispatcherTypeCaptureSession],
// @"updateDeviceCaptureFormat must be called on the capture queue.");
@try {
_currentDevice.activeFormat = format;
if (format.videoSupportedFrameRateRanges.count > 0) {
int target = 24;
int closest = -1;
CMTime result;
for (int i = 0; i < format.videoSupportedFrameRateRanges.count; i++) {
const auto rateRange = format.videoSupportedFrameRateRanges[i];
int gap = abs(rateRange.minFrameRate - target);
if (gap <= closest || closest == -1) {
closest = gap;
result = rateRange.maxFrameDuration;
}
}
if (closest >= 0) {
_currentDevice.activeVideoMaxFrameDuration = result;
}
}
} @catch (NSException *exception) {
RTCLogError(@"Failed to set active format!\n User info:%@", exception.userInfo);
return;
}
}
- (void)reconfigureCaptureSessionInput {
// NSAssert([RTCDispatcher isOnQueueForType:RTCDispatcherTypeCaptureSession],
// @"reconfigureCaptureSessionInput must be called on the capture queue.");
NSError *error = nil;
AVCaptureInput *input = _currentInput;
if (!input) {
RTCLogError(@"Failed to create front camera input: %@", error.localizedDescription);
return;
}
[_captureSession beginConfiguration];
for (AVCaptureInput *oldInput in [_captureSession.inputs copy]) {
[_captureSession removeInput:oldInput];
}
if ([_captureSession canAddInput:input]) {
[_captureSession addInput:input];
} else {
RTCLogError(@"Cannot add camera as an input to the session.");
}
[_captureSession commitConfiguration];
}
@end

View file

@ -0,0 +1,33 @@
#ifndef TGCALLS_VIDEO_CAPTURE_VIEW_H
#define TGCALLS_VIDEO_CAPTURE_VIEW_H
#ifdef WEBRTC_IOS
#import <Foundation/Foundation.h>
#import <UIKit/UIKit.h>
#include <memory>
@class AVCaptureVideoPreviewLayer;
@interface VideoCaptureView : UIView
@property(nonatomic) UIViewContentMode videoContentMode;
@property(nonatomic, getter=isEnabled) BOOL enabled;
@property(nonatomic, nullable) NSValue* rotationOverride;
@property (nonatomic, readwrite) int internalOrientation;
@property (nonatomic, readwrite) CGFloat internalAspect;
- (void)setOnFirstFrameReceived:(void (^ _Nullable)())onFirstFrameReceived;
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated;
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated;
- (void)onFrameGenerated:(CGSize)size isMirrored:(bool)isMirrored rotation:(int)rotation;
- (AVCaptureVideoPreviewLayer * _Nonnull)previewLayer;
@end
#endif //WEBRTC_IOS
#endif

View file

@ -0,0 +1,264 @@
#import "VideoCaptureView.h"
#import <CoreVideo/CoreVideo.h>
#import "base/RTCLogging.h"
#import "base/RTCVideoFrame.h"
#import "base/RTCVideoFrameBuffer.h"
#import "TGRTCCVPixelBuffer.h"
#include "sdk/objc/native/api/video_frame.h"
#include "sdk/objc/native/src/objc_frame_buffer.h"
#include "sdk/objc/base/RTCI420Buffer.h"
#import "api/video/video_sink_interface.h"
#import "api/media_stream_interface.h"
#import "rtc_base/time_utils.h"
@interface VideoCaptureContentView : UIView
@end
@implementation VideoCaptureContentView
+ (Class)layerClass {
return [AVCaptureVideoPreviewLayer class];
}
- (AVCaptureVideoPreviewLayer * _Nonnull)videoLayer {
return (AVCaptureVideoPreviewLayer *)self.layer;
}
@end
@interface VideoCaptureFrame : NSObject
@property (nonatomic, readonly) CGSize size;
@property (nonatomic, readonly) bool isMirrored;
@property (nonatomic, readonly) int rotation;
@end
@implementation VideoCaptureFrame
- (instancetype)initWithSize:(CGSize)size isMirrored:(bool)isMirrored rotation:(int)rotation {
self = [super init];
if (self != nil) {
_size = size;
_isMirrored = isMirrored;
_rotation = rotation;
}
return self;
}
@end
@interface VideoCaptureView () {
VideoCaptureContentView *_captureView;
VideoCaptureFrame *_videoFrame;
VideoCaptureFrame *_stashedVideoFrame;
int _isWaitingForLayoutFrameCount;
bool _didStartWaitingForLayout;
CGSize _videoFrameSize;
int64_t _lastFrameTimeNs;
CGSize _currentSize;
void (^_onFirstFrameReceived)();
bool _firstFrameReceivedReported;
void (^_onOrientationUpdated)(int, CGFloat);
void (^_onIsMirroredUpdated)(bool);
bool _didSetShouldBeMirrored;
bool _shouldBeMirrored;
}
@end
@implementation VideoCaptureView
- (instancetype)initWithFrame:(CGRect)frameRect {
self = [super initWithFrame:frameRect];
if (self) {
[self configure];
_enabled = true;
_currentSize = CGSizeZero;
_rotationOverride = @(RTCVideoRotation_0);
}
return self;
}
- (void)dealloc {
}
- (AVCaptureVideoPreviewLayer * _Nonnull)previewLayer {
return _captureView.videoLayer;
}
- (void)setEnabled:(BOOL)enabled {
if (_enabled != enabled) {
_enabled = enabled;
}
}
- (void)setVideoContentMode:(UIViewContentMode)mode {
_videoContentMode = mode;
}
- (void)configure {
_captureView = [[VideoCaptureContentView alloc] init];
[_captureView videoLayer].videoGravity = AVLayerVideoGravityResizeAspectFill;
[self addSubview:_captureView];
_videoFrameSize = CGSizeZero;
}
- (void)layoutSubviews {
[super layoutSubviews];
CGRect bounds = self.bounds;
_captureView.frame = bounds;
if (_didStartWaitingForLayout) {
_didStartWaitingForLayout = false;
_isWaitingForLayoutFrameCount = 0;
if (_stashedVideoFrame != nil) {
_videoFrame = _stashedVideoFrame;
_stashedVideoFrame = nil;
}
}
}
#pragma mark -
- (void)setRotationOverride:(NSValue *)rotationOverride {
_rotationOverride = rotationOverride;
[self setNeedsLayout];
}
- (RTCVideoRotation)frameRotation {
if (_rotationOverride) {
RTCVideoRotation rotation;
if (@available(iOS 11, *)) {
[_rotationOverride getValue:&rotation size:sizeof(rotation)];
} else {
[_rotationOverride getValue:&rotation];
}
return rotation;
}
if (_videoFrame) {
switch (_videoFrame.rotation) {
case webrtc::kVideoRotation_0:
return RTCVideoRotation_0;
case webrtc::kVideoRotation_90:
return RTCVideoRotation_90;
case webrtc::kVideoRotation_180:
return RTCVideoRotation_180;
case webrtc::kVideoRotation_270:
return RTCVideoRotation_270;
default:
return RTCVideoRotation_0;
}
} else {
return RTCVideoRotation_0;
}
}
- (void)onFrameGenerated:(CGSize)size isMirrored:(bool)isMirrored rotation:(int)rotation {
assert([NSThread isMainThread]);
if (!CGSizeEqualToSize(size, _currentSize)) {
_currentSize = size;
}
int mappedValue = 0;
switch (RTCVideoRotation(rotation)) {
case RTCVideoRotation_90: {
mappedValue = 1;
break;
}
case RTCVideoRotation_180: {
mappedValue = 2;
break;
}
case RTCVideoRotation_270: {
mappedValue = 3;
break;
}
default: {
mappedValue = 0;
break;
}
}
[self setInternalOrientationAndSize:mappedValue size:size];
if (!_firstFrameReceivedReported && _onFirstFrameReceived) {
_firstFrameReceivedReported = true;
_onFirstFrameReceived();
}
if (!self.isEnabled) {
return;
}
VideoCaptureFrame *frame = [[VideoCaptureFrame alloc] initWithSize:size isMirrored:isMirrored rotation:rotation];
if (_isWaitingForLayoutFrameCount > 0) {
_stashedVideoFrame = frame;
_isWaitingForLayoutFrameCount--;
return;
}
if (!_didStartWaitingForLayout) {
if (_videoFrame && _videoFrame.size.width > 0 && _videoFrame.size.height > 0 && frame.size.width > 0 && frame.size.height > 0) {
float previousAspect = ((float)_videoFrame.size.width) / ((float)_videoFrame.size.height);
float updatedAspect = ((float)frame.size.width) / ((float)frame.size.height);
if ((previousAspect < 1.0f) != (updatedAspect < 1.0f)) {
_stashedVideoFrame = frame;
_didStartWaitingForLayout = true;
_isWaitingForLayoutFrameCount = 5;
return;
}
}
}
_videoFrame = frame;
}
- (void)setOnFirstFrameReceived:(void (^ _Nullable)())onFirstFrameReceived {
_onFirstFrameReceived = [onFirstFrameReceived copy];
_firstFrameReceivedReported = false;
}
- (void)setInternalOrientationAndSize:(int)internalOrientation size:(CGSize)size {
CGFloat aspect = 1.0f;
if (size.width > 1.0f && size.height > 1.0f) {
aspect = size.width / size.height;
}
if (_internalOrientation != internalOrientation || ABS(_internalAspect - aspect) > 0.001) {
RTCLogInfo(@"VideoCaptureView@%lx orientation: %d, aspect: %f", (intptr_t)self, internalOrientation, (float)aspect);
_internalOrientation = internalOrientation;
_internalAspect = aspect;
if (_onOrientationUpdated) {
_onOrientationUpdated(internalOrientation, aspect);
}
}
}
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated {
_onOrientationUpdated = [onOrientationUpdated copy];
}
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated {
_onIsMirroredUpdated = [onIsMirroredUpdated copy];
}
@end

View file

@ -0,0 +1,41 @@
#ifndef TGCALLS_VIDEO_CAPTURER_INTERFACE_IMPL_H
#define TGCALLS_VIDEO_CAPTURER_INTERFACE_IMPL_H
#include "VideoCapturerInterface.h"
#include "sdk/objc/native/src/objc_video_track_source.h"
//#include "api/video_track_source_proxy.h"
@interface VideoCapturerInterfaceImplHolder : NSObject
@property (nonatomic) void *reference;
@end
namespace tgcalls {
struct PlatformCaptureInfo;
class VideoCapturerInterfaceImpl : public VideoCapturerInterface {
public:
VideoCapturerInterfaceImpl(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> source, std::string deviceId, std::function<void(VideoState)> stateUpdated, std::function<void(PlatformCaptureInfo)> captureInfoUpdated, std::pair<int, int> &outResolution);
~VideoCapturerInterfaceImpl() override;
void setState(VideoState state) override;
void setPreferredCaptureAspectRatio(float aspectRatio) override;
void withNativeImplementation(std::function<void(void *)> completion) override;
void setUncroppedOutput(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink) override;
void setOnFatalError(std::function<void()> error) override;
void setOnPause(std::function<void(bool)> pause) override;
int getRotation() override;
id getInternalReference();
private:
webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> _source;
VideoCapturerInterfaceImplHolder *_implReference;
};
} // namespace tgcalls
#endif

View file

@ -0,0 +1,480 @@
#include "VideoCapturerInterfaceImpl.h"
#include "absl/strings/match.h"
#include "api/audio_codecs/audio_decoder_factory_template.h"
#include "api/audio_codecs/audio_encoder_factory_template.h"
#include "api/audio_codecs/opus/audio_decoder_opus.h"
#include "api/audio_codecs/opus/audio_encoder_opus.h"
#include "api/rtp_parameters.h"
#include "api/task_queue/default_task_queue_factory.h"
#include "media/base/codec.h"
#include "media/base/media_constants.h"
#include "media/engine/webrtc_media_engine.h"
#include "modules/audio_device/include/audio_device_default.h"
#include "rtc_base/task_utils/repeating_task.h"
#include "system_wrappers/include/field_trial.h"
#include "api/video/builtin_video_bitrate_allocator_factory.h"
#include "api/video/video_bitrate_allocation.h"
#include "sdk/objc/native/api/video_encoder_factory.h"
#include "sdk/objc/native/api/video_decoder_factory.h"
#include "sdk/objc/api/RTCVideoRendererAdapter.h"
#include "sdk/objc/native/api/video_frame.h"
#include "api/media_types.h"
#ifndef WEBRTC_IOS
#import "VideoCameraCapturerMac.h"
#import "tgcalls/platform/darwin/DesktopSharingCapturer.h"
#import "tgcalls/desktop_capturer/DesktopCaptureSourceHelper.h"
#import "CustomExternalCapturer.h"
#import "VideoCMIOCapture.h"
#else
#import "VideoCameraCapturer.h"
#import "CustomExternalCapturer.h"
#endif
#import <AVFoundation/AVFoundation.h>
#import "VideoCaptureInterface.h"
#import "platform/PlatformInterface.h"
@interface VideoCapturerInterfaceImplSourceDescription : NSObject
@property (nonatomic, readonly) bool isFrontCamera;
@property (nonatomic, readonly) bool keepLandscape;
@property (nonatomic, readonly) NSString *deviceId;
@property (nonatomic, strong, readonly, nullable) AVCaptureDevice *device;
@property (nonatomic, strong, readonly, nullable) AVCaptureDeviceFormat *format;
@end
@implementation VideoCapturerInterfaceImplSourceDescription
- (instancetype)initWithIsFrontCamera:(bool)isFrontCamera keepLandscape:(bool)keepLandscape deviceId:(NSString *)deviceId device:(AVCaptureDevice * _Nullable)device format:(AVCaptureDeviceFormat * _Nullable)format {
self = [super init];
if (self != nil) {
_isFrontCamera = isFrontCamera;
_keepLandscape = keepLandscape;
_deviceId = deviceId;
_device = device;
_format = format;
}
return self;
}
@end
@interface VideoCapturerInterfaceImplReference : NSObject {
#ifdef WEBRTC_IOS
CustomExternalCapturer *_customExternalCapturer;
VideoCameraCapturer *_videoCameraCapturer;
#else
id<CapturerInterface> _videoCapturer;
#endif
}
@end
@implementation VideoCapturerInterfaceImplReference
- (id)videoCameraCapturer {
#ifdef WEBRTC_IOS
return _videoCameraCapturer;
#else
return _videoCapturer;
#endif
}
+ (AVCaptureDevice *)selectCapturerDeviceWithDeviceId:(NSString *)deviceId {
AVCaptureDevice *selectedCamera = nil;
#ifdef WEBRTC_IOS
bool useFrontCamera = ![deviceId hasPrefix:@"back"];
AVCaptureDevice *frontCamera = nil;
AVCaptureDevice *backCamera = nil;
for (AVCaptureDevice *device in [VideoCameraCapturer captureDevices]) {
if (device.position == AVCaptureDevicePositionFront) {
frontCamera = device;
} else if (device.position == AVCaptureDevicePositionBack) {
backCamera = device;
}
}
if (useFrontCamera && frontCamera != nil) {
selectedCamera = frontCamera;
} else {
selectedCamera = backCamera;
}
#else
NSArray *deviceComponents = [deviceId componentsSeparatedByString:@":"];
if (deviceComponents.count == 2) {
deviceId = deviceComponents[0];
}
//&& [devices[i] hasMediaType:AVMediaTypeVideo]
NSArray<AVCaptureDevice *> *devices = [VideoCameraCapturer captureDevices];
for (int i = 0; i < devices.count; i++) {
if (devices[i].isConnected && !devices[i].isSuspended ) {
if ([deviceId isEqualToString:@""] || [deviceId isEqualToString:devices[i].uniqueID]) {
selectedCamera = devices[i];
break;
}
}
}
if (selectedCamera == nil) {
for (int i = 0; i < devices.count; i++) {
if (devices[i].isConnected && !devices[i].isSuspended) {
selectedCamera = devices[i];
break;
}
}
}
#endif
return selectedCamera;
}
+ (AVCaptureDeviceFormat *)selectCaptureDeviceFormatForDevice:(AVCaptureDevice *)selectedCamera {
NSMutableArray<AVCaptureDeviceFormat *> *sortedFormats = [NSMutableArray arrayWithArray:[[VideoCameraCapturer supportedFormatsForDevice:selectedCamera] sortedArrayUsingComparator:^NSComparisonResult(AVCaptureDeviceFormat* lhs, AVCaptureDeviceFormat *rhs) {
int32_t width1 = CMVideoFormatDescriptionGetDimensions(lhs.formatDescription).width;
int32_t width2 = CMVideoFormatDescriptionGetDimensions(rhs.formatDescription).width;
return width1 < width2 ? NSOrderedAscending : NSOrderedDescending;
}]];
for (int i = (int)[sortedFormats count] - 1; i >= 0; i--) {
if ([[sortedFormats[i] description] containsString:@"x420"]) {
[sortedFormats removeObjectAtIndex:i];
}
}
AVCaptureDeviceFormat *bestFormat = sortedFormats.firstObject;
bool didSelectPreferredFormat = false;
#ifdef WEBRTC_IOS
for (AVCaptureDeviceFormat *format in sortedFormats) {
CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(format.formatDescription);
if (dimensions.width == 1280 && dimensions.height == 720) {
if (format.videoFieldOfView > 60.0f && format.videoSupportedFrameRateRanges.lastObject.maxFrameRate == 30) {
didSelectPreferredFormat = true;
bestFormat = format;
break;
}
}
}
#endif
if (!didSelectPreferredFormat) {
for (AVCaptureDeviceFormat *format in sortedFormats) {
CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(format.formatDescription);
if (dimensions.width >= 720 && dimensions.height >= 720) {
didSelectPreferredFormat = true;
bestFormat = format;
break;
}
}
}
if (!didSelectPreferredFormat) {
for (AVCaptureDeviceFormat *format in sortedFormats) {
CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(format.formatDescription);
if (dimensions.width >= 640 && dimensions.height >= 640) {
didSelectPreferredFormat = true;
bestFormat = format;
break;
}
}
}
if (bestFormat == nil) {
assert(false);
return nil;
}
return bestFormat;
}
+ (VideoCapturerInterfaceImplSourceDescription *)selectCapturerDescriptionWithDeviceId:(NSString *)deviceId {
if ([deviceId isEqualToString:@":ios_custom"]) {
return [[VideoCapturerInterfaceImplSourceDescription alloc] initWithIsFrontCamera:false keepLandscape:false deviceId:deviceId device:nil format:nil];
}
if ([deviceId hasPrefix:@"desktop_capturer_"]) {
return [[VideoCapturerInterfaceImplSourceDescription alloc] initWithIsFrontCamera:false keepLandscape:true deviceId: deviceId device: nil format: nil];
}
AVCaptureDevice *selectedCamera = [VideoCapturerInterfaceImplReference selectCapturerDeviceWithDeviceId:deviceId];
if (selectedCamera == nil) {
return [[VideoCapturerInterfaceImplSourceDescription alloc] initWithIsFrontCamera:![deviceId hasPrefix:@"back"] keepLandscape:[deviceId containsString:@"landscape"] deviceId: deviceId device: nil format: nil];
}
AVCaptureDeviceFormat *bestFormat = [VideoCapturerInterfaceImplReference selectCaptureDeviceFormatForDevice:selectedCamera];
return [[VideoCapturerInterfaceImplSourceDescription alloc] initWithIsFrontCamera:![deviceId hasPrefix:@"back"] keepLandscape:[deviceId containsString:@"landscape"] deviceId:deviceId device:selectedCamera format:bestFormat];
}
- (instancetype)initWithSource:(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)source sourceDescription:(VideoCapturerInterfaceImplSourceDescription *)sourceDescription isActiveUpdated:(void (^)(bool))isActiveUpdated rotationUpdated:(void (^)(int))rotationUpdated {
self = [super init];
if (self != nil) {
assert([NSThread isMainThread]);
#ifdef WEBRTC_IOS
if ([sourceDescription.deviceId isEqualToString:@":ios_custom"]) {
_customExternalCapturer = [[CustomExternalCapturer alloc] initWithSource:source];
} else {
_videoCameraCapturer = [[VideoCameraCapturer alloc] initWithSource:source useFrontCamera:sourceDescription.isFrontCamera keepLandscape:sourceDescription.keepLandscape isActiveUpdated:isActiveUpdated rotationUpdated:rotationUpdated];
[_videoCameraCapturer startCaptureWithDevice:sourceDescription.device format:sourceDescription.format fps:30];
}
#else
if (const auto desktopCaptureSource = tgcalls::DesktopCaptureSourceForKey([sourceDescription.deviceId UTF8String])) {
DesktopSharingCapturer *sharing = [[DesktopSharingCapturer alloc] initWithSource:source captureSource:desktopCaptureSource];
_videoCapturer = sharing;
} else if (!tgcalls::ShouldBeDesktopCapture([sourceDescription.deviceId UTF8String])) {
id<CapturerInterface> camera;
if ([sourceDescription.device hasMediaType:AVMediaTypeMuxed]) {
VideoCMIOCapture *value = [[VideoCMIOCapture alloc] initWithSource:source];
[value setupCaptureWithDevice:sourceDescription.device];
camera = value;
} else {
VideoCameraCapturer *value = [[VideoCameraCapturer alloc] initWithSource:source isActiveUpdated:isActiveUpdated];
[value setupCaptureWithDevice:sourceDescription.device format:sourceDescription.format fps:30];
camera = value;
}
_videoCapturer = camera;
} else {
_videoCapturer = nil;
}
if (_videoCapturer) {
[_videoCapturer start];
}
#endif
}
return self;
}
- (void)dealloc {
assert([NSThread isMainThread]);
#ifdef WEBRTC_IOS
[_videoCameraCapturer stopCapture];
#elif TARGET_OS_OSX
if (_videoCapturer) {
[_videoCapturer stop];
}
#endif
}
-(void)setOnFatalError:(std::function<void()>)error {
#ifdef WEBRTC_IOS
#else
if (_videoCapturer) {
[_videoCapturer setOnFatalError:error];
} else if (error) {
error();
}
#endif
}
-(void)setOnPause:(std::function<void(bool)>)pause {
#ifdef WEBRTC_IOS
#else
if (_videoCapturer) {
[_videoCapturer setOnPause:pause];
}
#endif
}
- (void)setIsEnabled:(bool)isEnabled {
#ifdef WEBRTC_IOS
if (_videoCameraCapturer) {
[_videoCameraCapturer setIsEnabled:isEnabled];
}
#else
if (_videoCapturer) {
[_videoCapturer setIsEnabled:isEnabled];
}
#endif
}
- (void)setUncroppedSink:(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)sink {
#ifdef WEBRTC_IOS
if (_videoCameraCapturer) {
[_videoCameraCapturer setUncroppedSink:sink];
}
#else
if (_videoCapturer) {
[_videoCapturer setUncroppedSink:sink];
}
#endif
}
- (void)setPreferredCaptureAspectRatio:(float)aspectRatio {
#ifdef WEBRTC_IOS
if (_videoCameraCapturer) {
[_videoCameraCapturer setPreferredCaptureAspectRatio:aspectRatio];
}
#else
if (_videoCapturer) {
[_videoCapturer setPreferredCaptureAspectRatio:aspectRatio];
}
#endif
}
- (int)getRotation {
#ifdef WEBRTC_IOS
if (_videoCameraCapturer) {
return [_videoCameraCapturer getRotation];
} else {
return 0;
}
#elif TARGET_OS_OSX
return 0;
#else
#error "Unsupported platform"
#endif
}
- (id)getInternalReference {
#ifdef WEBRTC_IOS
if (_videoCameraCapturer) {
return _videoCameraCapturer;
} else if (_customExternalCapturer) {
return _customExternalCapturer;
} else {
return nil;
}
#endif
return nil;
}
@end
@implementation VideoCapturerInterfaceImplHolder
@end
namespace tgcalls {
VideoCapturerInterfaceImpl::VideoCapturerInterfaceImpl(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> source, std::string deviceId, std::function<void(VideoState)> stateUpdated, std::function<void(PlatformCaptureInfo)> captureInfoUpdated, std::pair<int, int> &outResolution) :
_source(source) {
VideoCapturerInterfaceImplSourceDescription *sourceDescription = [VideoCapturerInterfaceImplReference selectCapturerDescriptionWithDeviceId:[NSString stringWithUTF8String:deviceId.c_str()]];
CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(sourceDescription.format.formatDescription);
#ifdef WEBRTC_IOS
outResolution.first = dimensions.height;
outResolution.second = dimensions.width;
#else
outResolution.first = dimensions.width;
outResolution.second = dimensions.height;
#endif
_implReference = [[VideoCapturerInterfaceImplHolder alloc] init];
VideoCapturerInterfaceImplHolder *implReference = _implReference;
dispatch_async(dispatch_get_main_queue(), ^{
VideoCapturerInterfaceImplReference *value = [[VideoCapturerInterfaceImplReference alloc] initWithSource:source sourceDescription:sourceDescription isActiveUpdated:^(bool isActive) {
stateUpdated(isActive ? VideoState::Active : VideoState::Paused);
} rotationUpdated:^(int angle) {
PlatformCaptureInfo info;
bool isLandscape = angle == 180 || angle == 0;
info.shouldBeAdaptedToReceiverAspectRate = !isLandscape;
info.rotation = angle;
captureInfoUpdated(info);
}];
if (value != nil) {
implReference.reference = (void *)CFBridgingRetain(value);
}
});
}
VideoCapturerInterfaceImpl::~VideoCapturerInterfaceImpl() {
VideoCapturerInterfaceImplHolder *implReference = _implReference;
dispatch_async(dispatch_get_main_queue(), ^{
if (implReference.reference != nil) {
CFBridgingRelease(implReference.reference);
}
});
}
void VideoCapturerInterfaceImpl::setState(VideoState state) {
VideoCapturerInterfaceImplHolder *implReference = _implReference;
dispatch_async(dispatch_get_main_queue(), ^{
if (implReference.reference != nil) {
VideoCapturerInterfaceImplReference *reference = (__bridge VideoCapturerInterfaceImplReference *)implReference.reference;
[reference setIsEnabled:(state == VideoState::Active)];
}
});
}
void VideoCapturerInterfaceImpl::setPreferredCaptureAspectRatio(float aspectRatio) {
VideoCapturerInterfaceImplHolder *implReference = _implReference;
dispatch_async(dispatch_get_main_queue(), ^{
if (implReference.reference != nil) {
VideoCapturerInterfaceImplReference *reference = (__bridge VideoCapturerInterfaceImplReference *)implReference.reference;
[reference setPreferredCaptureAspectRatio:aspectRatio];
}
});
}
void VideoCapturerInterfaceImpl::withNativeImplementation(std::function<void(void *)> completion) {
VideoCapturerInterfaceImplHolder *implReference = _implReference;
dispatch_async(dispatch_get_main_queue(), ^{
if (implReference.reference != nil) {
VideoCapturerInterfaceImplReference *reference = (__bridge VideoCapturerInterfaceImplReference *)implReference.reference;
completion((__bridge void *)[reference videoCameraCapturer]);
}
});
}
void VideoCapturerInterfaceImpl::setUncroppedOutput(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink) {
VideoCapturerInterfaceImplHolder *implReference = _implReference;
dispatch_async(dispatch_get_main_queue(), ^{
if (implReference.reference != nil) {
VideoCapturerInterfaceImplReference *reference = (__bridge VideoCapturerInterfaceImplReference *)implReference.reference;
[reference setUncroppedSink:sink];
}
});
}
void VideoCapturerInterfaceImpl::setOnFatalError(std::function<void()> error) {
VideoCapturerInterfaceImplHolder *implReference = _implReference;
dispatch_async(dispatch_get_main_queue(), ^{
if (implReference.reference != nil) {
VideoCapturerInterfaceImplReference *reference = (__bridge VideoCapturerInterfaceImplReference *)implReference.reference;
[reference setOnFatalError:error];
}
});
}
void VideoCapturerInterfaceImpl::setOnPause(std::function<void(bool)> pause) {
VideoCapturerInterfaceImplHolder *implReference = _implReference;
dispatch_async(dispatch_get_main_queue(), ^{
if (implReference.reference != nil) {
VideoCapturerInterfaceImplReference *reference = (__bridge VideoCapturerInterfaceImplReference *)implReference.reference;
[reference setOnPause: pause];
}
});
}
int VideoCapturerInterfaceImpl::getRotation() {
__block int value = 0;
VideoCapturerInterfaceImplHolder *implReference = _implReference;
dispatch_sync(dispatch_get_main_queue(), ^{
if (implReference.reference != nil) {
VideoCapturerInterfaceImplReference *reference = (__bridge VideoCapturerInterfaceImplReference *)implReference.reference;
value = [reference getRotation];
}
});
return value;
}
id VideoCapturerInterfaceImpl::getInternalReference() {
__block id value = nil;
VideoCapturerInterfaceImplHolder *implReference = _implReference;
dispatch_sync(dispatch_get_main_queue(), ^{
if (implReference.reference != nil) {
VideoCapturerInterfaceImplReference *reference = (__bridge VideoCapturerInterfaceImplReference *)implReference.reference;
value = [reference getInternalReference];
}
});
return value;
}
} // namespace tgcalls

View file

@ -0,0 +1,37 @@
#ifndef TGCALLS_VIDEO_METAL_VIEW_H
#define TGCALLS_VIDEO_METAL_VIEW_H
#ifdef WEBRTC_IOS
#import <Foundation/Foundation.h>
#import <UIKit/UIKit.h>
#import "api/media_stream_interface.h"
#include <memory>
@class RTCVideoFrame;
@interface VideoMetalView : UIView
+ (bool)isSupported;
@property(nonatomic) UIViewContentMode videoContentMode;
@property(nonatomic, getter=isEnabled) BOOL enabled;
@property(nonatomic, nullable) NSValue* rotationOverride;
@property (nonatomic, readwrite) int internalOrientation;
@property (nonatomic, readwrite) CGFloat internalAspect;
- (void)setSize:(CGSize)size;
- (void)renderFrame:(nullable RTCVideoFrame *)frame;
- (std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)getSink;
- (void)setOnFirstFrameReceived:(void (^ _Nullable)())onFirstFrameReceived;
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated;
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated;
- (void)setClone:(VideoMetalView * _Nullable)clone;
@end
#endif //WEBRTC_IOS
#endif

View file

@ -0,0 +1,459 @@
#import "VideoMetalView.h"
#import <Metal/Metal.h>
#import <MetalKit/MetalKit.h>
#import "base/RTCLogging.h"
#import "base/RTCVideoFrame.h"
#import "base/RTCVideoFrameBuffer.h"
#import "TGRTCCVPixelBuffer.h"
#include "sdk/objc/native/api/video_frame.h"
#include "sdk/objc/native/src/objc_frame_buffer.h"
#import "api/video/video_sink_interface.h"
#import "api/media_stream_interface.h"
#import "rtc_base/time_utils.h"
#import "RTCMTLI420Renderer.h"
#import "RTCMTLNV12Renderer.h"
#import "RTCMTLRGBRenderer.h"
#define MTKViewClass NSClassFromString(@"MTKView")
#define RTCMTLNV12RendererClass NSClassFromString(@"RTCMTLNV12Renderer")
#define RTCMTLI420RendererClass NSClassFromString(@"RTCMTLI420Renderer")
#define RTCMTLRGBRendererClass NSClassFromString(@"RTCMTLRGBRenderer")
namespace {
static RTCVideoFrame *customToObjCVideoFrame(const webrtc::VideoFrame &frame, RTCVideoRotation &rotation) {
rotation = RTCVideoRotation(frame.rotation());
RTCVideoFrame *videoFrame =
[[RTCVideoFrame alloc] initWithBuffer:webrtc::ToObjCVideoFrameBuffer(frame.video_frame_buffer())
rotation:rotation
timeStampNs:frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec];
videoFrame.timeStamp = frame.timestamp();
return videoFrame;
}
class VideoRendererAdapterImpl : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
public:
VideoRendererAdapterImpl(void (^frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation)) {
_frameReceived = [frameReceived copy];
}
void OnFrame(const webrtc::VideoFrame& nativeVideoFrame) override {
@autoreleasepool {
RTCVideoRotation rotation = RTCVideoRotation_90;
RTCVideoFrame* videoFrame = customToObjCVideoFrame(nativeVideoFrame, rotation);
//CGSize currentSize = (videoFrame.rotation % 180 == 0) ? CGSizeMake(videoFrame.width, videoFrame.height) : CGSizeMake(videoFrame.height, videoFrame.width);
CGSize currentSize = CGSizeMake(videoFrame.width, videoFrame.height);
if (_frameReceived) {
_frameReceived(currentSize, videoFrame, rotation);
}
}
}
private:
void (^_frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation);
};
}
@interface VideoMetalView () <MTKViewDelegate> {
RTCMTLI420Renderer *_rendererI420;
RTCMTLNV12Renderer *_rendererNV12;
RTCMTLRGBRenderer *_rendererRGB;
MTKView *_metalView;
RTCVideoFrame *_videoFrame;
RTCVideoFrame *_stashedVideoFrame;
int _isWaitingForLayoutFrameCount;
bool _didStartWaitingForLayout;
CGSize _videoFrameSize;
int64_t _lastFrameTimeNs;
CGSize _currentSize;
std::shared_ptr<VideoRendererAdapterImpl> _sink;
void (^_onFirstFrameReceived)();
bool _firstFrameReceivedReported;
void (^_onOrientationUpdated)(int, CGFloat);
void (^_onIsMirroredUpdated)(bool);
bool _didSetShouldBeMirrored;
bool _shouldBeMirrored;
bool _shouldBeMirroredVertically;
__weak VideoMetalView *_cloneView;
}
@end
@implementation VideoMetalView
+ (bool)isSupported {
static bool value;
static dispatch_once_t onceToken;
dispatch_once(&onceToken, ^{
id<MTLDevice> device = MTLCreateSystemDefaultDevice();
value = device != nil;
});
return value;
}
- (instancetype)initWithFrame:(CGRect)frameRect {
self = [super initWithFrame:frameRect];
if (self) {
[self configure];
_currentSize = CGSizeZero;
_rotationOverride = @(RTCVideoRotation_0);
__weak VideoMetalView *weakSelf = self;
_sink.reset(new VideoRendererAdapterImpl(^(CGSize size, RTCVideoFrame *videoFrame, RTCVideoRotation rotation) {
dispatch_async(dispatch_get_main_queue(), ^{
__strong VideoMetalView *strongSelf = weakSelf;
if (strongSelf == nil) {
return;
}
if (!CGSizeEqualToSize(size, strongSelf->_currentSize)) {
strongSelf->_currentSize = size;
[strongSelf setSize:size];
}
int mappedValue = 0;
switch (rotation) {
case RTCVideoRotation_90:
mappedValue = 1;
break;
case RTCVideoRotation_180:
mappedValue = 2;
break;
case RTCVideoRotation_270:
mappedValue = 3;
break;
default:
mappedValue = 0;
break;
}
[strongSelf setInternalOrientationAndSize:mappedValue size:size];
[strongSelf renderFrame:videoFrame];
VideoMetalView *cloneView = strongSelf->_cloneView;
if (cloneView) {
if (!CGSizeEqualToSize(size, cloneView->_currentSize)) {
cloneView->_currentSize = size;
[cloneView setSize:size];
}
[cloneView setInternalOrientationAndSize:mappedValue size:size];
[cloneView renderFrame:videoFrame];
}
});
}));
}
return self;
}
- (void)dealloc {
_sink.reset();
}
- (BOOL)isEnabled {
return !_metalView.paused;
}
- (void)setEnabled:(BOOL)enabled {
_metalView.paused = !enabled;
}
- (UIViewContentMode)videoContentMode {
return _metalView.contentMode;
}
- (void)setVideoContentMode:(UIViewContentMode)mode {
_metalView.contentMode = mode;
}
#pragma mark - Private
+ (BOOL)isMetalAvailable {
return MTLCreateSystemDefaultDevice() != nil;
}
+ (MTKView *)createMetalView:(CGRect)frame {
return [[MTKViewClass alloc] initWithFrame:frame];
}
+ (RTCMTLNV12Renderer *)createNV12Renderer {
return [[RTCMTLNV12RendererClass alloc] init];
}
+ (RTCMTLI420Renderer *)createI420Renderer {
return [[RTCMTLI420RendererClass alloc] init];
}
+ (RTCMTLRGBRenderer *)createRGBRenderer {
return [[RTCMTLRGBRenderer alloc] init];
}
- (void)configure {
NSAssert([VideoMetalView isMetalAvailable], @"Metal not availiable on this device");
_metalView = [VideoMetalView createMetalView:self.bounds];
_metalView.delegate = self;
_metalView.contentMode = UIViewContentModeScaleToFill;
_metalView.preferredFramesPerSecond = 30;
[self addSubview:_metalView];
_videoFrameSize = CGSizeZero;
}
- (void)setMultipleTouchEnabled:(BOOL)multipleTouchEnabled {
[super setMultipleTouchEnabled:multipleTouchEnabled];
_metalView.multipleTouchEnabled = multipleTouchEnabled;
}
- (void)layoutSubviews {
[super layoutSubviews];
CGRect bounds = self.bounds;
_metalView.frame = bounds;
if (!CGSizeEqualToSize(_videoFrameSize, CGSizeZero)) {
_metalView.drawableSize = [self drawableSize];
} else {
_metalView.drawableSize = bounds.size;
}
if (_didStartWaitingForLayout) {
_didStartWaitingForLayout = false;
_isWaitingForLayoutFrameCount = 0;
if (_stashedVideoFrame != nil) {
_videoFrame = _stashedVideoFrame;
_stashedVideoFrame = nil;
}
}
}
#pragma mark - MTKViewDelegate methods
- (void)drawInMTKView:(nonnull MTKView *)view {
NSAssert(view == _metalView, @"Receiving draw callbacks from foreign instance.");
RTCVideoFrame *videoFrame = _videoFrame;
// Skip rendering if we've already rendered this frame.
if (!videoFrame || videoFrame.timeStampNs == _lastFrameTimeNs) {
return;
}
if (CGRectIsEmpty(view.bounds)) {
return;
}
RTCMTLRenderer *renderer;
if ([videoFrame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
RTCCVPixelBuffer *buffer = (RTCCVPixelBuffer*)videoFrame.buffer;
if ([buffer isKindOfClass:[TGRTCCVPixelBuffer class]]) {
bool shouldBeMirrored = ((TGRTCCVPixelBuffer *)buffer).shouldBeMirrored;
bool shouldBeMirroredVertically = _internalOrientation == 1 || _internalOrientation == 3;
if (shouldBeMirrored != _shouldBeMirrored || shouldBeMirroredVertically != _shouldBeMirroredVertically) {
_shouldBeMirrored = shouldBeMirrored;
_shouldBeMirroredVertically = shouldBeMirroredVertically;
if (_shouldBeMirrored) {
if (_shouldBeMirroredVertically) {
_metalView.transform = CGAffineTransformMakeScale(1.0f, -1.0f);
} else {
_metalView.transform = CGAffineTransformMakeScale(-1.0f, 1.0f);
}
} else {
_metalView.transform = CGAffineTransformIdentity;
}
if (_didSetShouldBeMirrored) {
if (_onIsMirroredUpdated) {
_onIsMirroredUpdated(_shouldBeMirrored);
}
} else {
_didSetShouldBeMirrored = true;
}
}
}
const OSType pixelFormat = CVPixelBufferGetPixelFormatType(buffer.pixelBuffer);
if (pixelFormat == kCVPixelFormatType_32BGRA || pixelFormat == kCVPixelFormatType_32ARGB) {
if (!_rendererRGB) {
_rendererRGB = [VideoMetalView createRGBRenderer];
if (![_rendererRGB addRenderingDestination:_metalView]) {
_rendererRGB = nil;
RTCLogError(@"Failed to create RGB renderer");
return;
}
}
renderer = _rendererRGB;
} else {
if (!_rendererNV12) {
_rendererNV12 = [VideoMetalView createNV12Renderer];
if (![_rendererNV12 addRenderingDestination:_metalView]) {
_rendererNV12 = nil;
RTCLogError(@"Failed to create NV12 renderer");
return;
}
}
renderer = _rendererNV12;
}
} else {
if (!_rendererI420) {
_rendererI420 = [VideoMetalView createI420Renderer];
if (![_rendererI420 addRenderingDestination:_metalView]) {
_rendererI420 = nil;
RTCLogError(@"Failed to create I420 renderer");
return;
}
}
renderer = _rendererI420;
}
renderer.rotationOverride = _rotationOverride;
[renderer drawFrame:videoFrame];
_lastFrameTimeNs = videoFrame.timeStampNs;
}
- (void)mtkView:(MTKView *)view drawableSizeWillChange:(CGSize)size {
}
#pragma mark -
- (void)setRotationOverride:(NSValue *)rotationOverride {
_rotationOverride = rotationOverride;
_metalView.drawableSize = [self drawableSize];
[self setNeedsLayout];
}
- (RTCVideoRotation)frameRotation {
if (_rotationOverride) {
RTCVideoRotation rotation;
if (@available(iOS 11, *)) {
[_rotationOverride getValue:&rotation size:sizeof(rotation)];
} else {
[_rotationOverride getValue:&rotation];
}
return rotation;
}
return _videoFrame.rotation;
}
- (CGSize)drawableSize {
// Flip width/height if the rotations are not the same.
CGSize videoFrameSize = _videoFrameSize;
return videoFrameSize;
/*RTCVideoRotation frameRotation = [self frameRotation];
BOOL useLandscape =
(frameRotation == RTCVideoRotation_0) || (frameRotation == RTCVideoRotation_180);
BOOL sizeIsLandscape = (_videoFrame.rotation == RTCVideoRotation_0) ||
(_videoFrame.rotation == RTCVideoRotation_180);
if (useLandscape == sizeIsLandscape) {
return videoFrameSize;
} else {
return CGSizeMake(videoFrameSize.height, videoFrameSize.width);
}*/
}
#pragma mark - RTCVideoRenderer
- (void)setSize:(CGSize)size {
assert([NSThread isMainThread]);
_videoFrameSize = size;
_metalView.drawableSize = [self drawableSize];
//_metalView.drawableSize = drawableSize;
//[self setNeedsLayout];
//[strongSelf.delegate videoView:self didChangeVideoSize:size];
}
- (void)renderFrame:(nullable RTCVideoFrame *)frame {
assert([NSThread isMainThread]);
if (!_firstFrameReceivedReported && _onFirstFrameReceived) {
_firstFrameReceivedReported = true;
_onFirstFrameReceived();
}
if (!self.isEnabled) {
return;
}
if (frame == nil) {
RTCLogInfo(@"Incoming frame is nil. Exiting render callback.");
return;
}
if (_isWaitingForLayoutFrameCount > 0) {
_stashedVideoFrame = frame;
_isWaitingForLayoutFrameCount--;
return;
}
if (!_didStartWaitingForLayout) {
if (_videoFrame != nil && _videoFrame.width > 0 && _videoFrame.height > 0 && frame.width > 0 && frame.height > 0) {
float previousAspect = ((float)_videoFrame.width) / ((float)_videoFrame.height);
float updatedAspect = ((float)frame.width) / ((float)frame.height);
if ((previousAspect < 1.0f) != (updatedAspect < 1.0f)) {
_stashedVideoFrame = frame;
_didStartWaitingForLayout = true;
_isWaitingForLayoutFrameCount = 5;
return;
}
}
}
_videoFrame = frame;
}
- (std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)getSink {
assert([NSThread isMainThread]);
return _sink;
}
- (void)setOnFirstFrameReceived:(void (^ _Nullable)())onFirstFrameReceived {
_onFirstFrameReceived = [onFirstFrameReceived copy];
_firstFrameReceivedReported = false;
}
- (void)setInternalOrientationAndSize:(int)internalOrientation size:(CGSize)size {
CGFloat aspect = 1.0f;
if (size.width > 1.0f && size.height > 1.0f) {
aspect = size.width / size.height;
}
if (_internalOrientation != internalOrientation || ABS(_internalAspect - aspect) > 0.001) {
RTCLogInfo(@"VideoMetalView@%lx orientation: %d, aspect: %f", (intptr_t)self, internalOrientation, (float)aspect);
_internalOrientation = internalOrientation;
_internalAspect = aspect;
if (_onOrientationUpdated) {
_onOrientationUpdated(internalOrientation, aspect);
}
}
}
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated {
_onOrientationUpdated = [onOrientationUpdated copy];
}
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated {
_onIsMirroredUpdated = [onIsMirroredUpdated copy];
}
- (void)setClone:(VideoMetalView * _Nullable)clone {
_cloneView = clone;
}
@end

View file

@ -0,0 +1,41 @@
#ifndef TGCALLS_VIDEO_METAL_VIEW_MAC_H
#define TGCALLS_VIDEO_METAL_VIEW_MAC_H
#ifndef WEBRTC_IOS
#import <Foundation/Foundation.h>
#import <AppKit/AppKit.h>
#import "api/media_stream_interface.h"
#include <memory>
@class RTCVideoFrame;
@interface VideoMetalView : NSView
+ (bool)isSupported;
@property(nonatomic) CALayerContentsGravity _Nullable videoContentMode;
@property(nonatomic, getter=isEnabled) BOOL enabled;
@property(nonatomic, nullable) NSValue* rotationOverride;
@property (nonatomic, readwrite) int internalOrientation;
@property (nonatomic, readwrite) CGFloat internalAspect;
- (void)setSize:(CGSize)size;
- (void)renderFrame:(nullable RTCVideoFrame *)frame;
- (std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)getSink;
- (void)setOnFirstFrameReceived:(void (^ _Nullable)(float))onFirstFrameReceived;
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated;
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated;
- (void)setForceMirrored:(BOOL)forceMirrored;
-(void)setIsPaused:(bool)paused;
-(void)renderToSize:(NSSize)size animated: (bool)animated;
@end
#endif // WEBRTC_MAC
#endif

View file

@ -0,0 +1,503 @@
#import "VideoMetalViewMac.h"
#import <Metal/Metal.h>
#import <MetalKit/MetalKit.h>
#import "TGRTCCVPixelBuffer.h"
#import "base/RTCLogging.h"
#import "base/RTCVideoFrame.h"
#import "base/RTCVideoFrameBuffer.h"
#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
#include "sdk/objc/native/api/video_frame.h"
#include "sdk/objc/native/src/objc_frame_buffer.h"
#import "api/video/video_sink_interface.h"
#import "api/media_stream_interface.h"
#import "rtc_base/time_utils.h"
#import "SQueueLocalObject.h"
#import "api/video/video_sink_interface.h"
#import "api/media_stream_interface.h"
#import "tgcalls/platform/darwin/macOS/TGRTCMTLI420Renderer.h"
#define MTKViewClass NSClassFromString(@"MTKView")
#define TGRTCMTLI420RendererClass NSClassFromString(@"TGRTCMTLI420Renderer")
SQueue *renderQueue = [[SQueue alloc] init];
namespace {
static RTCVideoFrame *customToObjCVideoFrame(const webrtc::VideoFrame &frame, RTCVideoRotation &rotation) {
rotation = RTCVideoRotation(frame.rotation());
RTCVideoFrame *videoFrame =
[[RTCVideoFrame alloc] initWithBuffer:webrtc::ToObjCVideoFrameBuffer(frame.video_frame_buffer())
rotation:rotation
timeStampNs:frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec];
videoFrame.timeStamp = frame.timestamp();
return videoFrame;
}
class VideoRendererAdapterImpl : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
public:
VideoRendererAdapterImpl(void (^frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation)) {
_frameReceived = [frameReceived copy];
}
void OnFrame(const webrtc::VideoFrame& nativeVideoFrame) override {
RTCVideoRotation rotation = RTCVideoRotation_0;
RTCVideoFrame* videoFrame = customToObjCVideoFrame(nativeVideoFrame, rotation);
CGSize currentSize = (videoFrame.rotation % 180 == 0) ? CGSizeMake(videoFrame.width, videoFrame.height) : CGSizeMake(videoFrame.height, videoFrame.width);
if (_frameReceived) {
_frameReceived(currentSize, videoFrame, rotation);
}
}
private:
void (^_frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation);
};
}
@interface TGCAMetalLayer : CAMetalLayer
@end
@implementation TGCAMetalLayer
-(void)dealloc {
}
@end
@interface VideoMetalView () {
SQueueLocalObject *_rendererI420;
CAMetalLayer *_metalView;
NSView *_foregroundView;
CGSize _videoFrameSize;
RTCVideoRotation _rotation;
int64_t _lastFrameTimeNs;
CGSize _currentSize;
std::shared_ptr<VideoRendererAdapterImpl> _sink;
void (^_onFirstFrameReceived)(float);
bool _firstFrameReceivedReported;
void (^_onOrientationUpdated)(int, CGFloat);
void (^_onIsMirroredUpdated)(bool);
bool _didSetShouldBeMirrored;
bool _shouldBeMirrored;
bool _forceMirrored;
bool _isPaused;
NSMutableArray<RTCVideoFrame *> *_frames;
RTCVideoFrame *_videoFrame;
BOOL _drawing;
BOOL _deleteForegroundOnNextDrawing;
}
@end
@implementation VideoMetalView
+ (bool)isSupported {
return [VideoMetalView isMetalAvailable];
}
- (instancetype)initWithFrame:(CGRect)frameRect {
self = [super initWithFrame:frameRect];
if (self) {
[self configure];
_lastFrameTimeNs = INT32_MAX;
_currentSize = CGSizeZero;
_frames = [[NSMutableArray alloc] init];
_drawing = false;
_isPaused = false;
_deleteForegroundOnNextDrawing = false;
__weak VideoMetalView *weakSelf = self;
_sink.reset(new VideoRendererAdapterImpl(^(CGSize size, RTCVideoFrame *videoFrame, RTCVideoRotation rotation) {
dispatch_async(dispatch_get_main_queue(), ^{
__strong VideoMetalView *strongSelf = weakSelf;
if (strongSelf == nil) {
return;
}
strongSelf->_rotation = videoFrame.rotation;
if (!CGSizeEqualToSize(size, strongSelf->_currentSize)) {
strongSelf->_currentSize = size;
[strongSelf setSize:size];
}
int mappedValue = 0;
switch (rotation) {
case RTCVideoRotation_90:
mappedValue = 0;
break;
case RTCVideoRotation_180:
mappedValue = 1;
break;
case RTCVideoRotation_270:
mappedValue = 2;
break;
default:
mappedValue = 0;
break;
}
[strongSelf setInternalOrientation:mappedValue];
[strongSelf renderFrame:videoFrame];
if ([videoFrame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
RTCCVPixelBuffer *buffer = (RTCCVPixelBuffer*)videoFrame.buffer;
if ([buffer isKindOfClass:[TGRTCCVPixelBuffer class]]) {
bool shouldBeMirrored = ((TGRTCCVPixelBuffer *)buffer).shouldBeMirrored;
if (shouldBeMirrored != strongSelf->_shouldBeMirrored) {
strongSelf->_shouldBeMirrored = shouldBeMirrored;
if (strongSelf->_onIsMirroredUpdated) {
strongSelf->_onIsMirroredUpdated(strongSelf->_shouldBeMirrored);
}
}
}
}
if (!strongSelf->_firstFrameReceivedReported && strongSelf->_onFirstFrameReceived) {
strongSelf->_firstFrameReceivedReported = true;
strongSelf->_onFirstFrameReceived((float)videoFrame.width / (float)videoFrame.height);
}
});
}));
}
return self;
}
- (BOOL)isEnabled {
return !_isPaused;
}
- (void)setEnabled:(BOOL)enabled {
_isPaused = enabled;
}
-(void)setIsPaused:(bool)paused {
_isPaused = paused;
[self updateDrawingSize:self.frame.size];
}
-(void)renderToSize:(NSSize)size animated: (bool)animated {
[CATransaction begin];
[CATransaction setDisableActions:YES];
if (animated) {
if (!_foregroundView) {
_foregroundView = [[NSView alloc] initWithFrame:self.bounds];
_foregroundView.wantsLayer = YES;
_foregroundView.autoresizingMask = 0;
_foregroundView.layer = [VideoMetalView createMetalView:self.bounds];
_foregroundView.layer.contentsGravity = kCAGravityResizeAspect;
[self addSubview:_foregroundView];
}
CAMetalLayer *layer = _metalView;
CAMetalLayer *foreground = (CAMetalLayer *)_foregroundView.layer;
[_rendererI420 with:^(TGRTCMTLI420Renderer * renderer) {
[renderer setDoubleRendering:layer foreground:foreground];
}];
_deleteForegroundOnNextDrawing = false;
} else {
_deleteForegroundOnNextDrawing = true;
CAMetalLayer *layer = _metalView;
[_rendererI420 with:^(TGRTCMTLI420Renderer * renderer) {
[renderer setSingleRendering:layer];
}];
}
[self updateDrawingSize:size];
[CATransaction commit];
}
- (CALayerContentsGravity)videoContentMode {
return _metalView.contentsGravity;
}
- (void)setVideoContentMode:(CALayerContentsGravity)mode {
// _metalView.contentsGravity = mode;
}
#pragma mark - Private
+ (BOOL)isMetalAvailable {
return initMetal();
}
+ (CAMetalLayer *)createMetalView:(CGRect)frame {
CAMetalLayer *layer = [[TGCAMetalLayer alloc] init];
[CATransaction begin];
[CATransaction setDisableActions:true];
layer.framebufferOnly = true;
layer.opaque = false;
// layer.cornerRadius = 4;
if (@available(macOS 10.13, *)) {
layer.displaySyncEnabled = NO;
}
// layer.presentsWithTransaction = YES;
layer.backgroundColor = [NSColor clearColor].CGColor;
layer.contentsGravity = kCAGravityResizeAspectFill;
layer.frame = frame;
[CATransaction commit];
return layer;
}
+ (TGRTCMTLI420Renderer *)createI420Renderer {
return [[TGRTCMTLI420RendererClass alloc] init];
}
- (void)configure {
NSAssert([VideoMetalView isMetalAvailable], @"Metal not availiable on this device");
self.wantsLayer = YES;
self.layerContentsRedrawPolicy = NSViewLayerContentsRedrawOnSetNeedsDisplay;
_metalView = [VideoMetalView createMetalView:self.bounds];
self.layer = _metalView;
_videoFrameSize = CGSizeZero;
CAMetalLayer *layer = _metalView;
_rendererI420 = [[SQueueLocalObject alloc] initWithQueue:renderQueue generate: ^{
TGRTCMTLI420Renderer *renderer = [VideoMetalView createI420Renderer];
[renderer setSingleRendering:layer];
return renderer;
}];
}
-(void)setFrameSize:(NSSize)newSize {
[super setFrameSize:newSize];
[self updateDrawingSize: newSize];
}
- (void)layout {
[super layout];
}
-(void)updateDrawingSize:(NSSize)size {
if (_isPaused) {
return;
}
_metalView.frame = CGRectMake(0, 0, size.width, size.height);
_foregroundView.frame = self.bounds;
if (!CGSizeEqualToSize(_videoFrameSize, CGSizeZero)) {
_metalView.drawableSize = [self drawableSize:size];
((CAMetalLayer *)_foregroundView.layer).drawableSize = _videoFrameSize;
} else {
_metalView.drawableSize = size;
((CAMetalLayer *)_foregroundView.layer).drawableSize = size;
}
if(!_isPaused) {
RTCVideoFrame *frame = [_frames lastObject];
if (frame == nil) {
frame = _videoFrame;
}
if (frame) {
[self renderFrame:frame];
}
}
}
-(void)dealloc {
int bp = 0;
bp += 1;
}
#pragma mark -
- (void)setRotationOverride:(NSValue *)rotationOverride {
_rotationOverride = rotationOverride;
[self setNeedsLayout:YES];
}
- (RTCVideoRotation)rtcFrameRotation {
if (_rotationOverride) {
RTCVideoRotation rotation;
if (@available(macOS 10.13, *)) {
[_rotationOverride getValue:&rotation size:sizeof(rotation)];
} else {
[_rotationOverride getValue:&rotation];
}
return rotation;
}
return _rotation;
}
- (CGSize)drawableSize:(NSSize)forSize {
MTLFrameSize from;
MTLFrameSize to;
from.width = _videoFrameSize.width;
from.height = _videoFrameSize.height;
if (CGSizeEqualToSize(CGSizeZero, forSize)) {
to.width = _videoFrameSize.width;
to.height = _videoFrameSize.height;
} else {
to.width = forSize.width;
to.height = forSize.height;
}
MTLFrameSize size = MTLAspectFilled(to, from);
return CGSizeMake(size.width, size.height);
}
#pragma mark - RTCVideoRenderer
- (void)setSize:(CGSize)size {
assert([NSThread isMainThread]);
_videoFrameSize = size;
[self updateDrawingSize:self.frame.size];
_internalAspect = _videoFrameSize.width / _videoFrameSize.height;
}
- (void)renderFrame:(nullable RTCVideoFrame *)frame {
assert([NSThread isMainThread]);
if (!self.isEnabled) {
return;
}
if (frame == nil) {
RTCLogInfo(@"Incoming frame is nil. Exiting render callback.");
return;
}
RTCVideoFrame *videoFrame = frame;
// Skip rendering if we've already rendered this frame.
if (!videoFrame) {
return;
}
if (CGRectIsEmpty(self.bounds)) {
return;
}
if (CGRectIsEmpty(self.visibleRect)) {
return;
}
if (self.window == nil || self.superview == nil) {
return;
}
if ((self.window.occlusionState & NSWindowOcclusionStateVisible) == 0) {
return;
}
if (_frames.count >= 5) {
[_frames removeAllObjects];
[_frames addObject:videoFrame];
[self enqueue];
return;
}
[_frames addObject:videoFrame];
[self enqueue];
}
-(void)enqueue {
if(_frames.count > 0 && !_drawing) {
RTCVideoFrame *videoFrame = [_frames firstObject];
NSValue * rotationOverride = _rotationOverride;
int64_t timeStampNs = videoFrame.timeStampNs;
__weak VideoMetalView *weakSelf = self;
dispatch_block_t completion = ^{
__strong VideoMetalView *strongSelf = weakSelf;
if (strongSelf && strongSelf->_frames.count > 0) {
[strongSelf->_frames removeObjectAtIndex:0];
strongSelf->_drawing = false;
strongSelf->_lastFrameTimeNs = timeStampNs;
if (strongSelf->_deleteForegroundOnNextDrawing) {
[strongSelf->_foregroundView removeFromSuperview];
strongSelf->_foregroundView = nil;
strongSelf->_deleteForegroundOnNextDrawing = false;
}
[strongSelf enqueue];
}
};
_videoFrame = videoFrame;
self->_drawing = true;
[_rendererI420 with:^(TGRTCMTLI420Renderer * object) {
object.rotationOverride = rotationOverride;
[object drawFrame:videoFrame];
dispatch_async(dispatch_get_main_queue(), completion);
}];
}
}
- (std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)getSink {
assert([NSThread isMainThread]);
return _sink;
}
- (void)setOnFirstFrameReceived:(void (^ _Nullable)(float))onFirstFrameReceived {
_onFirstFrameReceived = [onFirstFrameReceived copy];
_firstFrameReceivedReported = false;
}
- (void)setInternalOrientationAndSize:(int)internalOrientation size:(CGSize)size {
CGFloat aspect = 1.0f;
if (size.width > 1.0f && size.height > 1.0f) {
aspect = size.width / size.height;
}
if (_internalOrientation != internalOrientation || ABS(_internalAspect - aspect) > 0.001) {
RTCLogInfo(@"VideoMetalView@%lx orientation: %d, aspect: %f", (intptr_t)self, internalOrientation, (float)aspect);
_internalOrientation = internalOrientation;
_internalAspect = aspect;
if (_onOrientationUpdated) {
_onOrientationUpdated(internalOrientation, aspect);
}
}
}
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated {
_onOrientationUpdated = [onOrientationUpdated copy];
}
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated {
_onIsMirroredUpdated = [onIsMirroredUpdated copy];
}
- (void)setForceMirrored:(BOOL)forceMirrored {
_forceMirrored = forceMirrored;
[self setNeedsLayout:YES];
}
@end

View file

@ -0,0 +1,35 @@
#ifndef TGCALLS_VIDEO_SAMPLE_BUFFER_VIEW_H
#define TGCALLS_VIDEO_SAMPLE_BUFFER_VIEW_H
#ifdef WEBRTC_IOS
#import <Foundation/Foundation.h>
#import <UIKit/UIKit.h>
#import "api/media_stream_interface.h"
#include <memory>
@interface VideoSampleBufferView : UIView
@property(nonatomic) UIViewContentMode videoContentMode;
@property(nonatomic, getter=isEnabled) BOOL enabled;
@property(nonatomic, nullable) NSValue* rotationOverride;
@property (nonatomic, readwrite) int internalOrientation;
@property (nonatomic, readwrite) CGFloat internalAspect;
- (void)setSize:(CGSize)size;
- (std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)getSink;
- (void)setOnFirstFrameReceived:(void (^ _Nullable)())onFirstFrameReceived;
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated;
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated;
- (void)addFrame:(const webrtc::VideoFrame&)frame;
- (void)setCloneTarget:(VideoSampleBufferView * _Nullable)cloneTarget;
@end
#endif //WEBRTC_IOS
#endif

View file

@ -0,0 +1,798 @@
#import "VideoSampleBufferView.h"
#import <CoreVideo/CoreVideo.h>
#import "base/RTCLogging.h"
#import "base/RTCVideoFrame.h"
#import "base/RTCVideoFrameBuffer.h"
#import "TGRTCCVPixelBuffer.h"
#include "sdk/objc/native/api/video_frame.h"
#include "sdk/objc/native/src/objc_frame_buffer.h"
#include "sdk/objc/base/RTCI420Buffer.h"
#import "api/video/video_sink_interface.h"
#import "api/media_stream_interface.h"
#import "rtc_base/time_utils.h"
#import "RTCMTLI420Renderer.h"
#import "RTCMTLNV12Renderer.h"
#import "RTCMTLRGBRenderer.h"
#include "libyuv.h"
#define MTKViewClass NSClassFromString(@"MTKView")
#define RTCMTLNV12RendererClass NSClassFromString(@"RTCMTLNV12Renderer")
#define RTCMTLI420RendererClass NSClassFromString(@"RTCMTLI420Renderer")
#define RTCMTLRGBRendererClass NSClassFromString(@"RTCMTLRGBRenderer")
namespace {
class VideoRendererAdapterImpl : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
public:
VideoRendererAdapterImpl(void (^frameReceived)(std::shared_ptr<webrtc::VideoFrame>)) {
_frameReceived = [frameReceived copy];
}
void OnFrame(const webrtc::VideoFrame& nativeVideoFrame) override {
@autoreleasepool {
if (_frameReceived) {
std::shared_ptr<webrtc::VideoFrame> videoFrame = std::make_shared<webrtc::VideoFrame>(nativeVideoFrame);
_frameReceived(videoFrame);
}
}
}
private:
void (^_frameReceived)(std::shared_ptr<webrtc::VideoFrame>);
};
}
@interface VideoSampleBufferContentView : UIView
@property (nonatomic) bool isPaused;
@end
@implementation VideoSampleBufferContentView
+ (Class)layerClass {
return [AVSampleBufferDisplayLayer class];
}
- (AVSampleBufferDisplayLayer * _Nonnull)videoLayer {
return (AVSampleBufferDisplayLayer *)self.layer;
}
@end
@interface VideoSampleBufferViewRenderingContext : NSObject {
__weak VideoSampleBufferContentView *_sampleBufferView;
__weak VideoSampleBufferContentView *_cloneTarget;
CVPixelBufferPoolRef _pixelBufferPool;
int _pixelBufferPoolWidth;
int _pixelBufferPoolHeight;
bool _isBusy;
}
@end
@implementation VideoSampleBufferViewRenderingContext
+ (dispatch_queue_t)sharedQueue {
static dispatch_queue_t queue;
static dispatch_once_t onceToken;
dispatch_once(&onceToken, ^{
queue = dispatch_queue_create("VideoSampleBufferViewRenderingContext", 0);
});
return queue;
}
- (instancetype)initWithView:(VideoSampleBufferContentView *)view {
self = [super init];
if (self != nil) {
_sampleBufferView = view;
}
return self;
}
- (void)dealloc {
_isBusy = true;
if (_pixelBufferPool) {
CFRelease(_pixelBufferPool);
}
void *opaqueReference = (__bridge_retained void *)_sampleBufferView;
dispatch_async(dispatch_get_main_queue(), ^{
__strong VideoSampleBufferContentView *object = (__bridge_transfer VideoSampleBufferContentView *)opaqueReference;
[object description];
});
}
static bool CopyVideoFrameToNV12PixelBuffer(const webrtc::I420BufferInterface *frameBuffer, CVPixelBufferRef pixelBuffer) {
if (!frameBuffer) {
return false;
}
RTC_DCHECK(pixelBuffer);
RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixelBuffer), kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixelBuffer, 0), frameBuffer->height());
RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixelBuffer, 0), frameBuffer->width());
CVReturn cvRet = CVPixelBufferLockBaseAddress(pixelBuffer, 0);
if (cvRet != kCVReturnSuccess) {
return false;
}
uint8_t *dstY = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
int dstStrideY = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
uint8_t *dstUV = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
int dstStrideUV = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
// Convert I420 to NV12.
int ret = libyuv::I420ToNV12(frameBuffer->DataY(),
frameBuffer->StrideY(),
frameBuffer->DataU(),
frameBuffer->StrideU(),
frameBuffer->DataV(),
frameBuffer->StrideV(),
dstY,
dstStrideY,
dstUV,
dstStrideUV,
frameBuffer->width(),
frameBuffer->height());
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
if (ret) {
return false;
}
return true;
}
static bool CopyNV12VideoFrameToNV12PixelBuffer(const webrtc::NV12BufferInterface *frameBuffer, CVPixelBufferRef pixelBuffer) {
if (!frameBuffer) {
return false;
}
RTC_DCHECK(pixelBuffer);
RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixelBuffer), kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixelBuffer, 0), frameBuffer->height());
RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixelBuffer, 0), frameBuffer->width());
CVReturn cvRet = CVPixelBufferLockBaseAddress(pixelBuffer, 0);
if (cvRet != kCVReturnSuccess) {
return false;
}
uint8_t *dstY = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
int dstStrideY = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
uint8_t *dstUV = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
int dstStrideUV = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
// Convert I420 to NV12.
int ret = libyuv::NV12Copy(frameBuffer->DataY(), frameBuffer->StrideY(), frameBuffer->DataUV(), frameBuffer->StrideUV(), dstY, dstStrideY, dstUV, dstStrideUV, frameBuffer->width(), frameBuffer->height());
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
if (ret) {
return false;
}
return true;
}
- (CVPixelBufferPoolRef)createPixelBufferPoolWithWidth:(int32_t)width height:(int32_t)height pixelFormat:(FourCharCode)pixelFormat maxBufferCount:(int32_t) maxBufferCount {
CVPixelBufferPoolRef outputPool = NULL;
NSDictionary *sourcePixelBufferOptions = @{
(id)kCVPixelBufferPixelFormatTypeKey : @(pixelFormat),
(id)kCVPixelBufferWidthKey : @(width),
(id)kCVPixelBufferHeightKey : @(height),
(id)kCVPixelBufferIOSurfacePropertiesKey : @{}
};
NSDictionary *pixelBufferPoolOptions = @{ (id)kCVPixelBufferPoolMinimumBufferCountKey : @(maxBufferCount) };
CVPixelBufferPoolCreate(kCFAllocatorDefault, (__bridge CFDictionaryRef)pixelBufferPoolOptions, (__bridge CFDictionaryRef)sourcePixelBufferOptions, &outputPool);
return outputPool;
}
- (CMSampleBufferRef)createSampleBufferFromI420Buffer:(const webrtc::I420BufferInterface *)buffer {
if (!buffer) {
return nil;
}
NSMutableDictionary *ioSurfaceProperties = [[NSMutableDictionary alloc] init];
//ioSurfaceProperties[@"IOSurfaceIsGlobal"] = @(true);
NSMutableDictionary *options = [[NSMutableDictionary alloc] init];
//options[(__bridge NSString *)kCVPixelBufferBytesPerRowAlignmentKey] = @(buffer.strideY);
options[(__bridge NSString *)kCVPixelBufferIOSurfacePropertiesKey] = ioSurfaceProperties;
CVPixelBufferRef pixelBufferRef = nil;
if (!(_pixelBufferPool != nil && _pixelBufferPoolWidth == buffer->width() && _pixelBufferPoolHeight == buffer->height())) {
if (_pixelBufferPool) {
CFRelease(_pixelBufferPool);
_pixelBufferPool = nil;
}
_pixelBufferPool = [self createPixelBufferPoolWithWidth:buffer->width() height:buffer->height() pixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange maxBufferCount:10];
_pixelBufferPoolWidth = buffer->width();
_pixelBufferPoolHeight = buffer->height();
}
if (_pixelBufferPool != nil && _pixelBufferPoolWidth == buffer->width() && _pixelBufferPoolHeight == buffer->height()) {
CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(kCFAllocatorDefault, _pixelBufferPool, nil, &pixelBufferRef);
} else {
CVPixelBufferCreate(
kCFAllocatorDefault,
buffer->width(),
buffer->height(),
kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
(__bridge CFDictionaryRef)options,
&pixelBufferRef
);
}
if (pixelBufferRef == nil) {
return nil;
}
CopyVideoFrameToNV12PixelBuffer(buffer, pixelBufferRef);
CMVideoFormatDescriptionRef formatRef = nil;
OSStatus status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBufferRef, &formatRef);
if (status != 0) {
return nil;
}
if (formatRef == nil) {
return nil;
}
CMSampleTimingInfo timingInfo;
timingInfo.duration = CMTimeMake(1, 30);
timingInfo.presentationTimeStamp = CMTimeMake(0, 30);
timingInfo.decodeTimeStamp = CMTimeMake(0, 30);
CMSampleBufferRef sampleBuffer = nil;
OSStatus bufferStatus = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, pixelBufferRef, formatRef, &timingInfo, &sampleBuffer);
CFRelease(formatRef);
CFRelease(pixelBufferRef);
if (bufferStatus != noErr) {
return nil;
}
if (sampleBuffer == nil) {
return nil;
}
NSArray *attachments = (__bridge NSArray *)CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true);
NSMutableDictionary *dict = (NSMutableDictionary *)attachments[0];
dict[(__bridge NSString *)kCMSampleAttachmentKey_DisplayImmediately] = @(true);
return sampleBuffer;
}
- (CMSampleBufferRef)createSampleBufferFromNV12Buffer:(const webrtc::NV12BufferInterface *)buffer {
if (!buffer) {
return nil;
}
NSMutableDictionary *ioSurfaceProperties = [[NSMutableDictionary alloc] init];
//ioSurfaceProperties[@"IOSurfaceIsGlobal"] = @(true);
NSMutableDictionary *options = [[NSMutableDictionary alloc] init];
//options[(__bridge NSString *)kCVPixelBufferBytesPerRowAlignmentKey] = @(buffer.strideY);
options[(__bridge NSString *)kCVPixelBufferIOSurfacePropertiesKey] = ioSurfaceProperties;
CVPixelBufferRef pixelBufferRef = nil;
if (!(_pixelBufferPool != nil && _pixelBufferPoolWidth == buffer->width() && _pixelBufferPoolHeight == buffer->height())) {
if (_pixelBufferPool) {
CFRelease(_pixelBufferPool);
_pixelBufferPool = nil;
}
_pixelBufferPool = [self createPixelBufferPoolWithWidth:buffer->width() height:buffer->height() pixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange maxBufferCount:10];
_pixelBufferPoolWidth = buffer->width();
_pixelBufferPoolHeight = buffer->height();
}
if (_pixelBufferPool != nil && _pixelBufferPoolWidth == buffer->width() && _pixelBufferPoolHeight == buffer->height()) {
CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(kCFAllocatorDefault, _pixelBufferPool, nil, &pixelBufferRef);
} else {
CVPixelBufferCreate(
kCFAllocatorDefault,
buffer->width(),
buffer->height(),
kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
(__bridge CFDictionaryRef)options,
&pixelBufferRef
);
}
if (pixelBufferRef == nil) {
return nil;
}
CopyNV12VideoFrameToNV12PixelBuffer(buffer, pixelBufferRef);
CMVideoFormatDescriptionRef formatRef = nil;
OSStatus status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBufferRef, &formatRef);
if (status != 0) {
return nil;
}
if (formatRef == nil) {
return nil;
}
CMSampleTimingInfo timingInfo;
timingInfo.duration = CMTimeMake(1, 30);
timingInfo.presentationTimeStamp = CMTimeMake(0, 30);
timingInfo.decodeTimeStamp = CMTimeMake(0, 30);
CMSampleBufferRef sampleBuffer = nil;
OSStatus bufferStatus = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, pixelBufferRef, formatRef, &timingInfo, &sampleBuffer);
CFRelease(formatRef);
CFRelease(pixelBufferRef);
if (bufferStatus != noErr) {
return nil;
}
if (sampleBuffer == nil) {
return nil;
}
NSArray *attachments = (__bridge NSArray *)CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true);
NSMutableDictionary *dict = (NSMutableDictionary *)attachments[0];
dict[(__bridge NSString *)kCMSampleAttachmentKey_DisplayImmediately] = @(true);
return sampleBuffer;
}
- (CMSampleBufferRef)createSampleBufferFromPixelBuffer:(CVPixelBufferRef)pixelBufferRef {
CMVideoFormatDescriptionRef formatRef = nil;
OSStatus status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBufferRef, &formatRef);
if (status != 0) {
return nil;
}
if (formatRef == nil) {
return nil;
}
CMSampleTimingInfo timingInfo;
timingInfo.duration = CMTimeMake(1, 30);
timingInfo.presentationTimeStamp = CMTimeMake(0, 30);
timingInfo.decodeTimeStamp = CMTimeMake(0, 30);
CMSampleBufferRef sampleBuffer = nil;
OSStatus bufferStatus = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, pixelBufferRef, formatRef, &timingInfo, &sampleBuffer);
CFRelease(formatRef);
if (bufferStatus != noErr) {
return nil;
}
if (sampleBuffer == nil) {
return nil;
}
NSArray *attachments = (__bridge NSArray *)CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true);
NSMutableDictionary *dict = (NSMutableDictionary *)attachments[0];
dict[(__bridge NSString *)kCMSampleAttachmentKey_DisplayImmediately] = @(true);
return sampleBuffer;
}
- (void)renderFrameIfReady:(std::shared_ptr<webrtc::VideoFrame>)frame {
if (_isBusy) {
return;
}
VideoSampleBufferContentView *sampleBufferView = _sampleBufferView;
if (!sampleBufferView) {
return;
}
AVSampleBufferDisplayLayer *layer = [sampleBufferView videoLayer];
VideoSampleBufferContentView *cloneTarget = _cloneTarget;
__weak AVSampleBufferDisplayLayer *cloneLayer = nil;
if (cloneTarget) {
cloneLayer = [cloneTarget videoLayer];
}
_isBusy = true;
dispatch_async([VideoSampleBufferViewRenderingContext sharedQueue], ^{
__strong AVSampleBufferDisplayLayer *strongCloneLayer = cloneLayer;
switch (frame->video_frame_buffer()->type()) {
case webrtc::VideoFrameBuffer::Type::kI420:
case webrtc::VideoFrameBuffer::Type::kI420A: {
CMSampleBufferRef sampleBuffer = [self createSampleBufferFromI420Buffer:frame->video_frame_buffer()->GetI420()];
if (sampleBuffer) {
[layer enqueueSampleBuffer:sampleBuffer];
[cloneLayer enqueueSampleBuffer:sampleBuffer];
if ([layer status] == AVQueuedSampleBufferRenderingStatusFailed) {
[layer flush];
}
if ([cloneLayer status] == AVQueuedSampleBufferRenderingStatusFailed) {
[cloneLayer flush];
}
CFRelease(sampleBuffer);
}
break;
}
case webrtc::VideoFrameBuffer::Type::kNV12: {
CMSampleBufferRef sampleBuffer = [self createSampleBufferFromNV12Buffer:(webrtc::NV12BufferInterface *)frame->video_frame_buffer().get()];
if (sampleBuffer) {
[layer enqueueSampleBuffer:sampleBuffer];
[cloneLayer enqueueSampleBuffer:sampleBuffer];
CFRelease(sampleBuffer);
}
break;
}
case webrtc::VideoFrameBuffer::Type::kNative: {
id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> nativeBuffer = static_cast<webrtc::ObjCFrameBuffer *>(frame->video_frame_buffer().get())->wrapped_frame_buffer();
if ([nativeBuffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) {
RTCCVPixelBuffer *pixelBuffer = (RTCCVPixelBuffer *)nativeBuffer;
CMSampleBufferRef sampleBuffer = [self createSampleBufferFromPixelBuffer:pixelBuffer.pixelBuffer];
if (sampleBuffer) {
[layer enqueueSampleBuffer:sampleBuffer];
[cloneLayer enqueueSampleBuffer:sampleBuffer];
CFRelease(sampleBuffer);
}
}
break;
}
default: {
break;
}
}
_isBusy = false;
void *opaqueReference = (__bridge_retained void *)layer;
void *cloneLayerReference = (__bridge_retained void *)strongCloneLayer;
strongCloneLayer = nil;
dispatch_async(dispatch_get_main_queue(), ^{
__strong AVSampleBufferDisplayLayer *object = (__bridge_transfer AVSampleBufferDisplayLayer *)opaqueReference;
object = nil;
__strong AVSampleBufferDisplayLayer *cloneObject = (__bridge_transfer AVSampleBufferDisplayLayer *)cloneLayerReference;
cloneObject = nil;
});
});
}
- (void)setCloneTarget:(VideoSampleBufferContentView * _Nullable)cloneTarget {
_cloneTarget = cloneTarget;
}
@end
@protocol ClonePortalView
- (void)setSourceView:(UIView * _Nullable)sourceView;
- (void)setHidesSourceView:(bool)arg1;
- (void)setMatchesAlpha:(bool)arg1;
- (void)setMatchesPosition:(bool)arg1;
- (void)setMatchesTransform:(bool)arg1;
@end
@interface VideoSampleBufferView () {
VideoSampleBufferContentView *_sampleBufferView;
VideoSampleBufferViewRenderingContext *_renderingContext;
std::shared_ptr<webrtc::VideoFrame> _videoFrame;
std::shared_ptr<webrtc::VideoFrame> _stashedVideoFrame;
int _isWaitingForLayoutFrameCount;
bool _didStartWaitingForLayout;
CGSize _videoFrameSize;
int64_t _lastFrameTimeNs;
CGSize _currentSize;
std::shared_ptr<VideoRendererAdapterImpl> _sink;
void (^_onFirstFrameReceived)();
bool _firstFrameReceivedReported;
void (^_onOrientationUpdated)(int, CGFloat);
void (^_onIsMirroredUpdated)(bool);
bool _didSetShouldBeMirrored;
bool _shouldBeMirrored;
__weak VideoSampleBufferView *_cloneTarget;
UIView<ClonePortalView> *_portalView;
}
@end
@implementation VideoSampleBufferView
- (instancetype)initWithFrame:(CGRect)frameRect {
self = [super initWithFrame:frameRect];
if (self) {
[self configure];
_enabled = true;
_currentSize = CGSizeZero;
_rotationOverride = @(RTCVideoRotation_0);
__weak VideoSampleBufferView *weakSelf = self;
_sink.reset(new VideoRendererAdapterImpl(^(std::shared_ptr<webrtc::VideoFrame> videoFrame) {
if (!videoFrame) {
return;
}
dispatch_async(dispatch_get_main_queue(), ^{
__strong VideoSampleBufferView *strongSelf = weakSelf;
if (strongSelf == nil) {
return;
}
[strongSelf renderFrame:videoFrame];
});
}));
}
return self;
}
- (void)dealloc {
}
- (void)setEnabled:(BOOL)enabled {
if (_enabled != enabled) {
_enabled = enabled;
_sampleBufferView.isPaused = !enabled;
}
}
- (void)setVideoContentMode:(UIViewContentMode)mode {
_videoContentMode = mode;
}
#pragma mark - Private
- (void)configure {
_sampleBufferView = [[VideoSampleBufferContentView alloc] init];
[self addSubview:_sampleBufferView];
_renderingContext = [[VideoSampleBufferViewRenderingContext alloc] initWithView:_sampleBufferView];
_videoFrameSize = CGSizeZero;
}
- (void)layoutSubviews {
[super layoutSubviews];
CGRect bounds = self.bounds;
if (!CGRectEqualToRect(_sampleBufferView.frame, bounds)) {
_sampleBufferView.frame = bounds;
_portalView.frame = bounds;
}
if (_didStartWaitingForLayout) {
_didStartWaitingForLayout = false;
_isWaitingForLayoutFrameCount = 0;
if (_stashedVideoFrame != nil) {
_videoFrame = _stashedVideoFrame;
_stashedVideoFrame = nil;
}
}
}
#pragma mark -
- (void)setRotationOverride:(NSValue *)rotationOverride {
_rotationOverride = rotationOverride;
[self setNeedsLayout];
}
- (RTCVideoRotation)frameRotation {
if (_rotationOverride) {
RTCVideoRotation rotation;
if (@available(iOS 11, *)) {
[_rotationOverride getValue:&rotation size:sizeof(rotation)];
} else {
[_rotationOverride getValue:&rotation];
}
return rotation;
}
if (_videoFrame) {
switch (_videoFrame->rotation()) {
case webrtc::kVideoRotation_0:
return RTCVideoRotation_0;
case webrtc::kVideoRotation_90:
return RTCVideoRotation_90;
case webrtc::kVideoRotation_180:
return RTCVideoRotation_180;
case webrtc::kVideoRotation_270:
return RTCVideoRotation_270;
default:
return RTCVideoRotation_0;
}
} else {
return RTCVideoRotation_0;
}
}
- (void)setSize:(CGSize)size {
assert([NSThread isMainThread]);
}
- (void)renderFrame:(std::shared_ptr<webrtc::VideoFrame>)frame {
[self renderFrameInternal:frame skipRendering:false];
VideoSampleBufferView *cloneTarget = _cloneTarget;
if (cloneTarget) {
[cloneTarget renderFrameInternal:frame skipRendering:true];
}
}
- (void)renderFrameInternal:(std::shared_ptr<webrtc::VideoFrame>)frame skipRendering:(bool)skipRendering {
assert([NSThread isMainThread]);
CGSize size = CGSizeMake(frame->width(), frame->height());
if (!CGSizeEqualToSize(size, _currentSize)) {
_currentSize = size;
[self setSize:size];
}
int mappedValue = 0;
switch (RTCVideoRotation(frame->rotation())) {
case RTCVideoRotation_90: {
mappedValue = 1;
break;
}
case RTCVideoRotation_180: {
mappedValue = 2;
break;
}
case RTCVideoRotation_270: {
mappedValue = 3;
break;
}
default: {
mappedValue = 0;
break;
}
}
[self setInternalOrientationAndSize:mappedValue size:size];
if (!_firstFrameReceivedReported && _onFirstFrameReceived) {
_firstFrameReceivedReported = true;
_onFirstFrameReceived();
}
if (!self.isEnabled) {
return;
}
if (!frame) {
return;
}
if (_isWaitingForLayoutFrameCount > 0) {
_stashedVideoFrame = frame;
_isWaitingForLayoutFrameCount--;
return;
}
if (!_didStartWaitingForLayout) {
if (_videoFrame && _videoFrame->width() > 0 && _videoFrame->height() > 0 && frame->width() > 0 && frame->height() > 0) {
float previousAspect = ((float)_videoFrame->width()) / ((float)_videoFrame->height());
float updatedAspect = ((float)frame->width()) / ((float)frame->height());
if ((previousAspect < 1.0f) != (updatedAspect < 1.0f)) {
_stashedVideoFrame = frame;
_didStartWaitingForLayout = true;
_isWaitingForLayoutFrameCount = 5;
return;
}
}
}
_videoFrame = frame;
if (!skipRendering) {
[_renderingContext renderFrameIfReady:frame];
}
}
- (std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)getSink {
assert([NSThread isMainThread]);
return _sink;
}
- (void)addFrame:(const webrtc::VideoFrame&)frame {
std::shared_ptr<webrtc::VideoFrame> videoFrame = std::make_shared<webrtc::VideoFrame>(frame);
[self renderFrame:videoFrame];
}
static NSString * _Nonnull shiftString(NSString *string, int key) {
NSMutableString *result = [[NSMutableString alloc] init];
for (int i = 0; i < (int)[string length]; i++) {
unichar c = [string characterAtIndex:i];
c += key;
[result appendString:[NSString stringWithCharacters:&c length:1]];
}
return result;
}
/*- (void)addAsCloneTarget:(VideoSampleBufferView *)sourceView {
if (_portalView) {
[_portalView setSourceView:nil];
[_portalView removeFromSuperview];
}
if (!sourceView) {
return;
}
static Class portalViewClass = nil;
static dispatch_once_t onceToken;
dispatch_once(&onceToken, ^{
portalViewClass = NSClassFromString(@"_UIPortalView");
});
if (portalViewClass) {
_portalView = (UIView<ClonePortalView> *)[[portalViewClass alloc] init];
_portalView.frame = sourceView->_sampleBufferView.frame;
_portalView.backgroundColor = [UIColor redColor];
[_portalView setSourceView:sourceView->_sampleBufferView];
[_portalView setHidesSourceView:true];
[_portalView setMatchesAlpha:false];
[_portalView setMatchesPosition:false];
[_portalView setMatchesTransform:false];
[self addSubview:_portalView];
}
}*/
- (void)setCloneTarget:(VideoSampleBufferView * _Nullable)cloneTarget {
_cloneTarget = cloneTarget;
if (cloneTarget) {
[_renderingContext setCloneTarget:cloneTarget->_sampleBufferView];
//[cloneTarget addAsCloneTarget:self];
}
}
- (void)setOnFirstFrameReceived:(void (^ _Nullable)())onFirstFrameReceived {
_onFirstFrameReceived = [onFirstFrameReceived copy];
_firstFrameReceivedReported = false;
}
- (void)setInternalOrientationAndSize:(int)internalOrientation size:(CGSize)size {
CGFloat aspect = 1.0f;
if (size.width > 1.0f && size.height > 1.0f) {
aspect = size.width / size.height;
}
if (_internalOrientation != internalOrientation || ABS(_internalAspect - aspect) > 0.001) {
RTCLogInfo(@"VideoSampleBufferView@%lx orientation: %d, aspect: %f", (intptr_t)self, internalOrientation, (float)aspect);
_internalOrientation = internalOrientation;
_internalAspect = aspect;
if (_onOrientationUpdated) {
_onOrientationUpdated(internalOrientation, aspect);
}
}
}
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated {
_onOrientationUpdated = [onOrientationUpdated copy];
}
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated {
_onIsMirroredUpdated = [onIsMirroredUpdated copy];
}
@end

View file

@ -0,0 +1,36 @@
#ifndef TGCALLS_VIDEO_SAMPLE_BUFFER_VIEW_H
#define TGCALLS_VIDEO_SAMPLE_BUFFER_VIEW_H
#ifdef WEBRTC_MAC
#import <Foundation/Foundation.h>
#import <Cocoa/Cocoa.h>
#import "api/media_stream_interface.h"
#include <memory>
@interface VideoSampleBufferView : NSView
@property(nonatomic) CALayerContentsGravity _Nullable videoContentMode;
@property(nonatomic, getter=isEnabled) BOOL enabled;
@property(nonatomic, nullable) NSValue* rotationOverride;
@property (nonatomic, readwrite) int internalOrientation;
@property (nonatomic, readwrite) CGFloat internalAspect;
- (void)setSize:(CGSize)size;
- (std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)getSink;
- (void)setOnFirstFrameReceived:(void (^ _Nullable)())onFirstFrameReceived;
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated;
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated;
+(BOOL)isAvailable;
@end
#endif //WEBRTC_MAC
#endif

View file

@ -0,0 +1,427 @@
#import "VideoSampleBufferViewMac.h"
#import <CoreVideo/CoreVideo.h>
#import "base/RTCLogging.h"
#import "base/RTCVideoFrame.h"
#import "base/RTCVideoFrameBuffer.h"
#import "TGRTCCVPixelBuffer.h"
#include "sdk/objc/native/api/video_frame.h"
#include "sdk/objc/native/src/objc_frame_buffer.h"
#include "sdk/objc/base/RTCI420Buffer.h"
#import "api/video/video_sink_interface.h"
#import "api/media_stream_interface.h"
#import "rtc_base/time_utils.h"
#import "RTCMTLI420Renderer.h"
#import "RTCMTLNV12Renderer.h"
#import "RTCMTLRGBRenderer.h"
#include "libyuv.h"
#define MTKViewClass NSClassFromString(@"MTKView")
#define RTCMTLNV12RendererClass NSClassFromString(@"RTCMTLNV12Renderer")
#define RTCMTLI420RendererClass NSClassFromString(@"RTCMTLI420Renderer")
#define RTCMTLRGBRendererClass NSClassFromString(@"RTCMTLRGBRenderer")
namespace {
static RTCVideoFrame *customToObjCVideoFrame(const webrtc::VideoFrame &frame, RTCVideoRotation &rotation) {
rotation = RTCVideoRotation(frame.rotation());
RTCVideoFrame *videoFrame =
[[RTCVideoFrame alloc] initWithBuffer:webrtc::ToObjCVideoFrameBuffer(frame.video_frame_buffer())
rotation:rotation
timeStampNs:frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec];
videoFrame.timeStamp = frame.timestamp();
return videoFrame;
}
class VideoRendererAdapterImpl : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
public:
VideoRendererAdapterImpl(void (^frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation)) {
_frameReceived = [frameReceived copy];
}
void OnFrame(const webrtc::VideoFrame& nativeVideoFrame) override {
@autoreleasepool {
RTCVideoRotation rotation = RTCVideoRotation_90;
RTCVideoFrame* videoFrame = customToObjCVideoFrame(nativeVideoFrame, rotation);
//CGSize currentSize = (videoFrame.rotation % 180 == 0) ? CGSizeMake(videoFrame.width, videoFrame.height) : CGSizeMake(videoFrame.height, videoFrame.width);
CGSize currentSize = CGSizeMake(videoFrame.width, videoFrame.height);
if (_frameReceived) {
_frameReceived(currentSize, videoFrame, rotation);
}
}
}
private:
void (^_frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation);
};
}
@interface VideoSampleBufferView () {
AVSampleBufferDisplayLayer *_sampleBufferLayer;
RTCVideoFrame *_videoFrame;
RTCVideoFrame *_stashedVideoFrame;
int _isWaitingForLayoutFrameCount;
bool _didStartWaitingForLayout;
CGSize _videoFrameSize;
int64_t _lastFrameTimeNs;
CGSize _currentSize;
std::shared_ptr<VideoRendererAdapterImpl> _sink;
void (^_onFirstFrameReceived)();
bool _firstFrameReceivedReported;
void (^_onOrientationUpdated)(int, CGFloat);
void (^_onIsMirroredUpdated)(bool);
bool _didSetShouldBeMirrored;
bool _shouldBeMirrored;
CVPixelBufferPoolRef _pixelBufferPool;
int _pixelBufferPoolWidth;
int _pixelBufferPoolHeight;
}
@end
@implementation VideoSampleBufferView
- (instancetype)initWithFrame:(CGRect)frameRect {
self = [super initWithFrame:frameRect];
if (self) {
[self configure];
_enabled = true;
_currentSize = CGSizeZero;
_rotationOverride = @(RTCVideoRotation_0);
__weak VideoSampleBufferView *weakSelf = self;
_sink.reset(new VideoRendererAdapterImpl(^(CGSize size, RTCVideoFrame *videoFrame, RTCVideoRotation rotation) {
dispatch_async(dispatch_get_main_queue(), ^{
__strong VideoSampleBufferView *strongSelf = weakSelf;
if (strongSelf == nil) {
return;
}
if (!CGSizeEqualToSize(size, strongSelf->_currentSize)) {
strongSelf->_currentSize = size;
[strongSelf setSize:size];
}
int mappedValue = 0;
switch (rotation) {
case RTCVideoRotation_90:
mappedValue = 1;
break;
case RTCVideoRotation_180:
mappedValue = 2;
break;
case RTCVideoRotation_270:
mappedValue = 3;
break;
default:
mappedValue = 0;
break;
}
[strongSelf setInternalOrientationAndSize:mappedValue size:size];
[strongSelf renderFrame:videoFrame];
});
}));
}
return self;
}
- (void)dealloc {
if (_pixelBufferPool) {
CFRelease(_pixelBufferPool);
}
}
- (void)setEnabled:(BOOL)enabled {
_enabled = enabled;
}
- (void)setVideoContentMode:(CALayerContentsGravity)mode {
_videoContentMode = mode;
}
#pragma mark - Private
- (void)configure {
self.wantsLayer = YES;
_sampleBufferLayer = [[AVSampleBufferDisplayLayer alloc] init];
self.layer = _sampleBufferLayer;
// [self.layer addSublayer:_sampleBufferLayer];
_videoFrameSize = CGSizeZero;
}
- (void)layout {
[super layout];
CGRect bounds = self.bounds;
[CATransaction begin];
[CATransaction setAnimationDuration:0];
[CATransaction setDisableActions:true];
_sampleBufferLayer.frame = bounds;
[CATransaction commit];
if (_didStartWaitingForLayout) {
_didStartWaitingForLayout = false;
_isWaitingForLayoutFrameCount = 0;
if (_stashedVideoFrame != nil) {
_videoFrame = _stashedVideoFrame;
_stashedVideoFrame = nil;
}
}
}
#pragma mark -
- (void)setRotationOverride:(NSValue *)rotationOverride {
_rotationOverride = rotationOverride;
[self setNeedsLayout:YES];
}
- (RTCVideoRotation)frameRotation {
if (_rotationOverride) {
RTCVideoRotation rotation;
if (@available(iOS 11, *)) {
[_rotationOverride getValue:&rotation size:sizeof(rotation)];
} else {
[_rotationOverride getValue:&rotation];
}
return rotation;
}
return _videoFrame.rotation;
}
#pragma mark - RTCVideoRenderer
bool CopyVideoFrameToNV12PixelBuffer(id<RTC_OBJC_TYPE(RTCI420Buffer)> frameBuffer,
CVPixelBufferRef pixelBuffer) {
RTC_DCHECK(pixelBuffer);
RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixelBuffer), kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixelBuffer, 0), frameBuffer.height);
RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixelBuffer, 0), frameBuffer.width);
CVReturn cvRet = CVPixelBufferLockBaseAddress(pixelBuffer, 0);
if (cvRet != kCVReturnSuccess) {
return false;
}
uint8_t *dstY = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
size_t dstStrideY = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
uint8_t *dstUV = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
size_t dstStrideUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
int ret = libyuv::I420ToNV12(frameBuffer.dataY,
frameBuffer.strideY,
frameBuffer.dataU,
frameBuffer.strideU,
frameBuffer.dataV,
frameBuffer.strideV,
dstY,
(int)dstStrideY,
dstUV,
(int)dstStrideUV,
frameBuffer.width,
frameBuffer.height);
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
if (ret) {
return false;
}
return true;
}
+ (CVPixelBufferPoolRef)createPixelBufferPoolWithWidth:(int32_t)width height:(int32_t)height pixelFormat:(FourCharCode)pixelFormat maxBufferCount:(int32_t) maxBufferCount {
CVPixelBufferPoolRef outputPool = NULL;
NSDictionary *sourcePixelBufferOptions = @{
(id)kCVPixelBufferPixelFormatTypeKey : @(pixelFormat),
(id)kCVPixelBufferWidthKey : @(width),
(id)kCVPixelBufferHeightKey : @(height),
(id)kCVPixelBufferIOSurfacePropertiesKey : @{}
};
NSDictionary *pixelBufferPoolOptions = @{ (id)kCVPixelBufferPoolMinimumBufferCountKey : @(maxBufferCount) };
CVPixelBufferPoolCreate(kCFAllocatorDefault, (__bridge CFDictionaryRef)pixelBufferPoolOptions, (__bridge CFDictionaryRef)sourcePixelBufferOptions, &outputPool);
return outputPool;
}
- (CMSampleBufferRef)createSampleBufferFromBuffer:(id<RTC_OBJC_TYPE(RTCI420Buffer)>)buffer {
NSMutableDictionary *ioSurfaceProperties = [[NSMutableDictionary alloc] init];
//ioSurfaceProperties[@"IOSurfaceIsGlobal"] = @(true);
NSMutableDictionary *options = [[NSMutableDictionary alloc] init];
//options[(__bridge NSString *)kCVPixelBufferBytesPerRowAlignmentKey] = @(buffer.strideY);
options[(__bridge NSString *)kCVPixelBufferIOSurfacePropertiesKey] = ioSurfaceProperties;
CVPixelBufferRef pixelBufferRef = nil;
if (!(_pixelBufferPool != nil && _pixelBufferPoolWidth == buffer.width && _pixelBufferPoolHeight == buffer.height)) {
if (_pixelBufferPool) {
CFRelease(_pixelBufferPool);
_pixelBufferPool = nil;
}
_pixelBufferPool = [VideoSampleBufferView createPixelBufferPoolWithWidth:buffer.width height:buffer.height pixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange maxBufferCount:10];
_pixelBufferPoolWidth = buffer.width;
_pixelBufferPoolHeight = buffer.height;
}
if (_pixelBufferPool != nil && _pixelBufferPoolWidth == buffer.width && _pixelBufferPoolHeight == buffer.height) {
CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(kCFAllocatorDefault, _pixelBufferPool, nil, &pixelBufferRef);
} else {
CVPixelBufferCreate(
kCFAllocatorDefault,
buffer.width,
buffer.height,
kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
(__bridge CFDictionaryRef)options,
&pixelBufferRef
);
}
if (pixelBufferRef == nil) {
return nil;
}
CopyVideoFrameToNV12PixelBuffer([buffer toI420], pixelBufferRef);
CMVideoFormatDescriptionRef formatRef = nil;
OSStatus status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBufferRef, &formatRef);
if (status != 0) {
return nil;
}
if (formatRef == nil) {
return nil;
}
CMSampleTimingInfo timingInfo;
timingInfo.duration = CMTimeMake(1, 30);
timingInfo.presentationTimeStamp = CMTimeMake(0, 30);
timingInfo.decodeTimeStamp = CMTimeMake(0, 30);
CMSampleBufferRef sampleBuffer = nil;
OSStatus bufferStatus = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, pixelBufferRef, formatRef, &timingInfo, &sampleBuffer);
CFRelease(pixelBufferRef);
if (bufferStatus != noErr) {
return nil;
}
if (sampleBuffer == nil) {
return nil;
}
NSArray *attachments = (__bridge NSArray *)CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true);
NSMutableDictionary *dict = (NSMutableDictionary *)attachments[0];
dict[(__bridge NSString *)kCMSampleAttachmentKey_DisplayImmediately] = @(true);
return sampleBuffer;
}
- (void)setSize:(CGSize)size {
assert([NSThread isMainThread]);
}
- (void)renderFrame:(nullable RTCVideoFrame *)frame {
assert([NSThread isMainThread]);
if (!_firstFrameReceivedReported && _onFirstFrameReceived) {
_firstFrameReceivedReported = true;
_onFirstFrameReceived();
}
if (!self.isEnabled) {
return;
}
if (frame == nil) {
RTCLogInfo(@"Incoming frame is nil. Exiting render callback.");
return;
}
if (_isWaitingForLayoutFrameCount > 0) {
_stashedVideoFrame = frame;
_isWaitingForLayoutFrameCount--;
return;
}
if (!_didStartWaitingForLayout) {
if (_videoFrame != nil && _videoFrame.width > 0 && _videoFrame.height > 0 && frame.width > 0 && frame.height > 0) {
float previousAspect = ((float)_videoFrame.width) / ((float)_videoFrame.height);
float updatedAspect = ((float)frame.width) / ((float)frame.height);
if ((previousAspect < 1.0f) != (updatedAspect < 1.0f)) {
_stashedVideoFrame = frame;
_didStartWaitingForLayout = true;
_isWaitingForLayoutFrameCount = 5;
return;
}
}
}
_videoFrame = frame;
id<RTC_OBJC_TYPE(RTCI420Buffer)> buffer = [frame.buffer toI420];
CMSampleBufferRef sampleBuffer = [self createSampleBufferFromBuffer:buffer];
if (sampleBuffer) {
[_sampleBufferLayer enqueueSampleBuffer:sampleBuffer];
CFRelease(sampleBuffer);
}
}
- (std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)getSink {
assert([NSThread isMainThread]);
return _sink;
}
- (void)setOnFirstFrameReceived:(void (^ _Nullable)())onFirstFrameReceived {
_onFirstFrameReceived = [onFirstFrameReceived copy];
_firstFrameReceivedReported = false;
}
- (void)setInternalOrientationAndSize:(int)internalOrientation size:(CGSize)size {
CGFloat aspect = 1.0f;
if (size.width > 1.0f && size.height > 1.0f) {
aspect = size.width / size.height;
}
if (_internalOrientation != internalOrientation || ABS(_internalAspect - aspect) > 0.001) {
RTCLogInfo(@"VideoSampleBufferView@%lx orientation: %d, aspect: %f", (intptr_t)self, internalOrientation, (float)aspect);
_internalOrientation = internalOrientation;
_internalAspect = aspect;
if (_onOrientationUpdated) {
_onOrientationUpdated(internalOrientation, aspect);
}
}
}
- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated {
_onOrientationUpdated = [onOrientationUpdated copy];
}
- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated {
_onIsMirroredUpdated = [onIsMirroredUpdated copy];
}
+(BOOL)isAvailable {
return YES;
}
@end

View file

@ -0,0 +1,349 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
#ifdef TGCALLS_ENABLE_X264
#include "h264_encoder_impl.h"
#include <limits>
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
namespace webrtc {
namespace {
int NumberOfThreads(int width, int height, int number_of_cores) {
if (width * height >= 1920 * 1080 && number_of_cores > 8) {
return 8; // 8 threads for 1080p on high perf machines.
} else if (width * height > 1280 * 960 && number_of_cores >= 6) {
return 3; // 3 threads for 1080p.
} else if (width * height > 640 * 480 && number_of_cores >= 3) {
return 2; // 2 threads for qHD/HD.
} else {
return 1; // 1 thread for VGA or less.
}
}
} // namespace
H264EncoderX264Impl::H264EncoderX264Impl(cricket::VideoCodec const &videoCodec) :
packetization_mode_(H264PacketizationMode::SingleNalUnit),
encoded_image_callback_(nullptr),
inited_(false),
encoder_(nullptr) {
}
H264EncoderX264Impl::~H264EncoderX264Impl() {
Release();
}
int32_t H264EncoderX264Impl::InitEncode(const VideoCodec* inst,
int32_t number_of_cores,
size_t max_payload_size) {
if (inst == NULL) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
if (inst->maxFramerate < 1) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
// allow zero to represent an unspecified maxBitRate
if (inst->maxBitrate > 0 && inst->startBitrate > inst->maxBitrate) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
if (inst->width < 1 || inst->height < 1) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
if (number_of_cores < 1) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
int ret_val = Release();
if (ret_val < 0) {
return ret_val;
}
codec_settings_ = *inst;
/* Get default params for preset/tuning */
x264_param_t param;
memset(&param, 0, sizeof(param));
x264_param_default(&param);
ret_val = x264_param_default_preset(&param, "fast", "zerolatency");
if (ret_val != 0) {
RTC_LOG(LS_ERROR) << "H264EncoderX264Impl::InitEncode() fails to initialize encoder ret_val " << ret_val;
x264_encoder_close(encoder_);
encoder_ = NULL;
return WEBRTC_VIDEO_CODEC_ERROR;
}
param.i_log_level = X264_LOG_WARNING;
param.i_threads = 1;
param.i_width = inst->width;
param.i_height = inst->height;
param.i_frame_total = 0;
param.b_sliced_threads = 0;
//param.i_keyint_min = 5000;
param.i_keyint_max = 60;
/*param.i_bframe = 0;
param.b_open_gop = 0;
param.i_bframe_pyramid = 0;
param.i_bframe_adaptive = X264_B_ADAPT_TRELLIS;
param.i_nal_hrd = 0;
param.i_fps_den = 1;
param.i_fps_num = inst->maxFramerate;
param.i_timebase_den = 1;
param.i_timebase_num = inst->maxFramerate;
param.b_intra_refresh = 0;
param.i_frame_reference = 1;
param.b_annexb = 1;
param.i_csp = X264_CSP_I420;
param.b_aud = 1;
param.b_vfr_input = 0;
param.b_repeat_headers = 1;*/
param.rc.i_rc_method = X264_RC_ABR;
param.rc.i_bitrate = codec_settings_.maxBitrate;
param.rc.i_vbv_max_bitrate = param.rc.i_bitrate;
param.rc.i_vbv_buffer_size = param.rc.i_bitrate * 2;
param.rc.f_vbv_buffer_init = 1.0;
//param.rc.i_qp_max = codec_settings_.qpMax;
/*param.rc.i_rc_method = X264_RC_CRF;
param.rc.f_rf_constant_max = 20;
param.rc.f_rf_constant_max = 25;*/
//param.rc.b_filler = 0;
param.i_slice_max_size = (int)max_payload_size;
param.i_sps_id = sps_id_;
ret_val = x264_param_apply_profile(&param, "high");
if (ret_val != 0) {
RTC_LOG(LS_ERROR) << "H264EncoderX264Impl::InitEncode() fails to initialize encoder ret_val " << ret_val;
x264_encoder_close(encoder_);
encoder_ = NULL;
return WEBRTC_VIDEO_CODEC_ERROR;
}
param.analyse.i_weighted_pred = X264_WEIGHTP_NONE;
ret_val = x264_picture_alloc(&pic_, param.i_csp, param.i_width, param.i_height);
if (ret_val != 0) {
RTC_LOG(LS_ERROR) << "H264EncoderX264Impl::InitEncode() fails to initialize encoder ret_val " << ret_val;
x264_encoder_close(encoder_);
encoder_ = NULL;
return WEBRTC_VIDEO_CODEC_ERROR;
}
encoder_ = x264_encoder_open(&param);
if (!encoder_){
RTC_LOG(LS_ERROR) << "H264EncoderX264Impl::InitEncode() fails to initialize encoder ret_val " << ret_val;
x264_encoder_close(encoder_);
x264_picture_clean(&pic_);
encoder_ = NULL;
return WEBRTC_VIDEO_CODEC_ERROR;
}
const size_t new_capacity = CalcBufferSize(VideoType::kI420, codec_settings_.width, codec_settings_.height);
encoded_image_.SetEncodedData(EncodedImageBuffer::Create(new_capacity));
encoded_image_._encodedWidth = codec_settings_.width;
encoded_image_._encodedHeight = codec_settings_.height;
encoded_image_.ClearEncodedData();
inited_ = true;
RTC_LOG(LS_INFO) << "H264EncoderX264Impl::InitEncode(width: " << inst->width << ", height: " << inst->height << ", framerate: " << inst->maxFramerate << ", start_bitrate: " << inst->startBitrate << ", max_bitrate: " << inst->maxBitrate << ")";
sps_id_++;
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t H264EncoderX264Impl::Release() {
if (encoder_) {
encoder_ = nullptr;
}
encoded_image_.ClearEncodedData();
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t H264EncoderX264Impl::RegisterEncodeCompleteCallback(EncodedImageCallback* callback) {
encoded_image_callback_ = callback;
return WEBRTC_VIDEO_CODEC_OK;
}
void H264EncoderX264Impl::SetRates(const RateControlParameters& parameters) {
codec_settings_.maxBitrate = parameters.bitrate.GetSpatialLayerSum(0);
codec_settings_.maxFramerate = (uint32_t)parameters.framerate_fps;
}
int32_t H264EncoderX264Impl::Encode(const VideoFrame& frame, const std::vector<VideoFrameType>* frame_types) {
if (!inited_) {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
if (frame.size() == 0) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
if (encoded_image_callback_ == NULL) {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
VideoFrameType frame_type = VideoFrameType::kVideoFrameDelta;
if (frame_types && frame_types->size() > 0) {
frame_type = (*frame_types)[0];
}
pic_.img.i_csp = X264_CSP_I420;
pic_.img.i_plane = 3;
switch (frame_type) {
case VideoFrameType::kEmptyFrame: {
pic_.i_type = X264_TYPE_AUTO;
break;
}
case VideoFrameType::kVideoFrameDelta: {
pic_.i_type = X264_TYPE_AUTO;
break;
}
case VideoFrameType::kVideoFrameKey: {
pic_.i_type = X264_TYPE_AUTO;
x264_encoder_intra_refresh(encoder_);
break;
}
default: {
pic_.i_type = X264_TYPE_AUTO;
break;
}
}
webrtc::scoped_refptr<I420BufferInterface> frame_buffer = frame.video_frame_buffer()->ToI420();
pic_.img.plane[0] = (uint8_t *)frame_buffer->DataY();
pic_.img.i_stride[0] = frame_buffer->StrideY();
pic_.img.plane[1] = (uint8_t *)frame_buffer->DataU();
pic_.img.i_stride[1] = frame_buffer->StrideU();
pic_.img.plane[2] = (uint8_t *)frame_buffer->DataV();
pic_.img.i_stride[2] = frame_buffer->StrideV();
pic_.i_pts++;
int n_nal = 0;
int i_frame_size = x264_encoder_encode(encoder_, &nal_t_, &n_nal, &pic_, &pic_out_);
if (i_frame_size < 0) {
RTC_LOG(LS_ERROR) << "H264EncoderX264Impl::Encode() fails to encode " << i_frame_size;
x264_encoder_close(encoder_);
x264_picture_clean(&pic_);
encoder_ = NULL;
return WEBRTC_VIDEO_CODEC_ERROR;
}
if (i_frame_size) {
if (n_nal == 0) {
return WEBRTC_VIDEO_CODEC_OK;
}
// Encoder can skip frames to save bandwidth in which case
// `encoded_images_[i]._length` == 0.
/*if (encoded_images_[i].size() > 0) {
// Parse QP.
h264_bitstream_parser_.ParseBitstream(encoded_images_[i]);
encoded_images_[i].qp_ =
h264_bitstream_parser_.GetLastSliceQp().value_or(-1);
// Deliver encoded image.
CodecSpecificInfo codec_specific;
codec_specific.codecType = kVideoCodecH264;
codec_specific.codecSpecific.H264.packetization_mode =
packetization_mode_;
codec_specific.codecSpecific.H264.temporal_idx = kNoTemporalIdx;
codec_specific.codecSpecific.H264.idr_frame =
info.eFrameType == videoFrameTypeIDR;
codec_specific.codecSpecific.H264.base_layer_sync = false;
if (configurations_[i].num_temporal_layers > 1) {
const uint8_t tid = info.sLayerInfo[0].uiTemporalId;
codec_specific.codecSpecific.H264.temporal_idx = tid;
codec_specific.codecSpecific.H264.base_layer_sync =
tid > 0 && tid < tl0sync_limit_[i];
if (codec_specific.codecSpecific.H264.base_layer_sync) {
tl0sync_limit_[i] = tid;
}
if (tid == 0) {
tl0sync_limit_[i] = configurations_[i].num_temporal_layers;
}
}
encoded_image_callback_->OnEncodedImage(encoded_images_[i],
&codec_specific);
}
}*/
size_t required_capacity = 0;
size_t fragments_count = 0;
for (int layer = 0; layer < 1; ++layer) {
for (int nal = 0; nal < n_nal; ++nal, ++fragments_count) {
int currentNaluSize = nal_t_[nal].i_payload;
RTC_CHECK_GE(currentNaluSize, 0);
// Ensure `required_capacity` will not overflow.
RTC_CHECK_LE(currentNaluSize,
std::numeric_limits<size_t>::max() - required_capacity);
required_capacity += currentNaluSize;
}
}
// TODO(nisse): Use a cache or buffer pool to avoid allocation?
auto buffer = EncodedImageBuffer::Create(required_capacity);
encoded_image_.SetEncodedData(buffer);
int currentBufferLength = 0;
uint32_t totalNaluIndex = 0;
for (int nal_index = 0; nal_index < n_nal; nal_index++) {
uint32_t currentNaluSize = 0;
currentNaluSize = nal_t_[nal_index].i_payload;
memcpy(buffer->data() + currentBufferLength, nal_t_[nal_index].p_payload, currentNaluSize);
currentBufferLength += currentNaluSize;
totalNaluIndex++;
}
}
i_frame++;
if (encoded_image_.size() > 0) {
encoded_image_._encodedWidth = codec_settings_.width;
encoded_image_._encodedHeight = codec_settings_.height;
encoded_image_.SetTimestamp(frame.timestamp());
encoded_image_._frameType = frame_type;
encoded_image_.SetSpatialIndex(0);
CodecSpecificInfo codec_specific;
codec_specific.codecType = kVideoCodecH264;
codec_specific.codecSpecific.H264.packetization_mode = packetization_mode_;
codec_specific.codecSpecific.H264.temporal_idx = kNoTemporalIdx;
if (pic_out_.i_type == X264_TYPE_IDR) {
codec_specific.codecSpecific.H264.idr_frame = true;
} else {
codec_specific.codecSpecific.H264.idr_frame = false;
}
codec_specific.codecSpecific.H264.base_layer_sync = false;
encoded_image_callback_->OnEncodedImage(encoded_image_, &codec_specific);
}
return WEBRTC_VIDEO_CODEC_OK;
}
bool H264EncoderX264Impl::IsInitialized() const {
return encoder_ != nullptr;
}
} // namespace webrtc
#endif

View file

@ -0,0 +1,80 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
#ifdef TGCALLS_ENABLE_x264
#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_H264_H264_ENCODER_IMPL_H_
#define WEBRTC_MODULES_VIDEO_CODING_CODECS_H264_H264_ENCODER_IMPL_H_
#include "modules/video_coding/codecs/h264/include/h264.h"
#include <vector>
#include "api/scoped_refptr.h"
#include <libx264/x264.h>
class ISVCEncoder;
namespace webrtc {
class H264EncoderX264Impl : public H264Encoder {
public:
H264EncoderX264Impl(cricket::VideoCodec const &videoCodec);
~H264EncoderX264Impl() override;
// |max_payload_size| is ignored.
// The following members of |codec_settings| are used. The rest are ignored.;
// - codecType (must be kVideoCodecH264)
// - targetBitrate
// - maxFramerate
// - width
// - height
int32_t InitEncode(const VideoCodec* codec_settings,
int32_t number_of_cores,
size_t /*max_payload_size*/) override;
int32_t Release() override;
int32_t RegisterEncodeCompleteCallback(
EncodedImageCallback* callback) override;
void SetRates(const RateControlParameters& parameters) override;
// The result of encoding - an EncodedImage and RTPFragmentationHeader - are
// passed to the encode complete callback.
int32_t Encode(const VideoFrame& frame,
const std::vector<VideoFrameType>* frame_types) override;
private:
bool IsInitialized() const;
// ISVCEncoder* openh264_encoder_;
VideoCodec codec_settings_;
H264PacketizationMode packetization_mode_;
EncodedImage encoded_image_;
EncodedImageCallback* encoded_image_callback_;
bool inited_;
x264_picture_t pic_;
x264_picture_t pic_out_;
x264_t *encoder_;
int i_frame = 0;//frame index
x264_nal_t *nal_t_;
int sps_id_ = 1;
// x264_param_t x264_parameter;
};
} // namespace webrtc
#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_H264_H264_ENCODER_IMPL_H_
#endif

View file

@ -0,0 +1,406 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
#include "h265_nalu_rewriter.h"
#include <CoreFoundation/CoreFoundation.h>
#include <memory>
#include <vector>
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "common_video/h265/h265_common.h"
namespace webrtc {
namespace {
const char kAnnexBHeaderBytes[4] = {0, 0, 0, 1};
const size_t kAvccHeaderByteSize = sizeof(uint32_t);
// Helper class for reading NALUs from an RTP Annex B buffer.
class H265AnnexBBufferReader final {
public:
H265AnnexBBufferReader(const uint8_t* annexb_buffer, size_t length);
~H265AnnexBBufferReader();
H265AnnexBBufferReader(const H265AnnexBBufferReader& other) = delete;
void operator=(const H265AnnexBBufferReader& other) = delete;
// Returns a pointer to the beginning of the next NALU slice without the
// header bytes and its length. Returns false if no more slices remain.
bool ReadNalu(const uint8_t** out_nalu, size_t* out_length);
// Returns the number of unread NALU bytes, including the size of the header.
// If the buffer has no remaining NALUs this will return zero.
size_t BytesRemaining() const;
// Reset the reader to start reading from the first NALU
void SeekToStart();
// Seek to the next position that holds a NALU of the desired type,
// or the end if no such NALU is found.
// Return true if a NALU of the desired type is found, false if we
// reached the end instead
bool SeekToNextNaluOfType(H265::NaluType type);
private:
// Returns the the next offset that contains NALU data.
size_t FindNextNaluHeader(const uint8_t* start,
size_t length,
size_t offset) const;
const uint8_t* const start_;
std::vector<NaluIndex> offsets_;
std::vector<NaluIndex>::iterator offset_;
const size_t length_;
};
// Helper class for writing NALUs using avcc format into a buffer.
class H265AvccBufferWriter final {
public:
H265AvccBufferWriter(uint8_t* const avcc_buffer, size_t length);
~H265AvccBufferWriter() {}
H265AvccBufferWriter(const H265AvccBufferWriter& other) = delete;
void operator=(const H265AvccBufferWriter& other) = delete;
// Writes the data slice into the buffer. Returns false if there isn't
// enough space left.
bool WriteNalu(const uint8_t* data, size_t data_size);
// Returns the unused bytes in the buffer.
size_t BytesRemaining() const;
private:
uint8_t* const start_;
size_t offset_;
const size_t length_;
};
H265AnnexBBufferReader::H265AnnexBBufferReader(const uint8_t* annexb_buffer,
size_t length)
: start_(annexb_buffer), length_(length) {
RTC_DCHECK(annexb_buffer);
offsets_ = H264::FindNaluIndices(annexb_buffer, length);
offset_ = offsets_.begin();
}
H265AnnexBBufferReader::~H265AnnexBBufferReader() = default;
bool H265AnnexBBufferReader::ReadNalu(const uint8_t** out_nalu,
size_t* out_length) {
RTC_DCHECK(out_nalu);
RTC_DCHECK(out_length);
*out_nalu = nullptr;
*out_length = 0;
if (offset_ == offsets_.end()) {
return false;
}
*out_nalu = start_ + offset_->payload_start_offset;
*out_length = offset_->payload_size;
++offset_;
return true;
}
size_t H265AnnexBBufferReader::BytesRemaining() const {
if (offset_ == offsets_.end()) {
return 0;
}
return length_ - offset_->start_offset;
}
void H265AnnexBBufferReader::SeekToStart() {
offset_ = offsets_.begin();
}
bool H265AnnexBBufferReader::SeekToNextNaluOfType(H265::NaluType type) {
for (; offset_ != offsets_.end(); ++offset_) {
if (offset_->payload_size < 1)
continue;
if (H265::ParseNaluType(*(start_ + offset_->payload_start_offset)) == type)
return true;
}
return false;
}
H265AvccBufferWriter::H265AvccBufferWriter(uint8_t* const avcc_buffer, size_t length)
: start_(avcc_buffer), offset_(0), length_(length) {
RTC_DCHECK(avcc_buffer);
}
bool H265AvccBufferWriter::WriteNalu(const uint8_t* data, size_t data_size) {
// Check if we can write this length of data.
if (data_size + kAvccHeaderByteSize > BytesRemaining()) {
return false;
}
// Write length header, which needs to be big endian.
uint32_t big_endian_length = CFSwapInt32HostToBig(data_size);
memcpy(start_ + offset_, &big_endian_length, sizeof(big_endian_length));
offset_ += sizeof(big_endian_length);
// Write data.
memcpy(start_ + offset_, data, data_size);
offset_ += data_size;
return true;
}
size_t H265AvccBufferWriter::BytesRemaining() const {
return length_ - offset_;
}
}
bool H265CMSampleBufferToAnnexBBuffer(
CMSampleBufferRef hvcc_sample_buffer,
bool is_keyframe,
rtc::Buffer* annexb_buffer) {
RTC_DCHECK(hvcc_sample_buffer);
// Get format description from the sample buffer.
CMVideoFormatDescriptionRef description =
CMSampleBufferGetFormatDescription(hvcc_sample_buffer);
if (description == nullptr) {
RTC_LOG(LS_ERROR) << "Failed to get sample buffer's description.";
return false;
}
// Get parameter set information.
int nalu_header_size = 0;
size_t param_set_count = 0;
OSStatus status = CMVideoFormatDescriptionGetHEVCParameterSetAtIndex(
description, 0, nullptr, nullptr, &param_set_count, &nalu_header_size);
if (status != noErr) {
RTC_LOG(LS_ERROR) << "Failed to get parameter set.";
return false;
}
RTC_CHECK_EQ(nalu_header_size, kAvccHeaderByteSize);
RTC_DCHECK_EQ(param_set_count, 3);
// Truncate any previous data in the buffer without changing its capacity.
annexb_buffer->SetSize(0);
size_t nalu_offset = 0;
std::vector<size_t> frag_offsets;
std::vector<size_t> frag_lengths;
// Place all parameter sets at the front of buffer.
if (is_keyframe) {
size_t param_set_size = 0;
const uint8_t* param_set = nullptr;
for (size_t i = 0; i < param_set_count; ++i) {
status = CMVideoFormatDescriptionGetHEVCParameterSetAtIndex(
description, i, &param_set, &param_set_size, nullptr, nullptr);
if (status != noErr) {
RTC_LOG(LS_ERROR) << "Failed to get parameter set.";
return false;
}
// Update buffer.
annexb_buffer->AppendData(kAnnexBHeaderBytes, sizeof(kAnnexBHeaderBytes));
annexb_buffer->AppendData(reinterpret_cast<const char*>(param_set),
param_set_size);
// Update fragmentation.
frag_offsets.push_back(nalu_offset + sizeof(kAnnexBHeaderBytes));
frag_lengths.push_back(param_set_size);
nalu_offset += sizeof(kAnnexBHeaderBytes) + param_set_size;
}
}
// Get block buffer from the sample buffer.
CMBlockBufferRef block_buffer =
CMSampleBufferGetDataBuffer(hvcc_sample_buffer);
if (block_buffer == nullptr) {
RTC_LOG(LS_ERROR) << "Failed to get sample buffer's block buffer.";
return false;
}
CMBlockBufferRef contiguous_buffer = nullptr;
// Make sure block buffer is contiguous.
if (!CMBlockBufferIsRangeContiguous(block_buffer, 0, 0)) {
status = CMBlockBufferCreateContiguous(
nullptr, block_buffer, nullptr, nullptr, 0, 0, 0, &contiguous_buffer);
if (status != noErr) {
RTC_LOG(LS_ERROR) << "Failed to flatten non-contiguous block buffer: "
<< status;
return false;
}
} else {
contiguous_buffer = block_buffer;
// Retain to make cleanup easier.
CFRetain(contiguous_buffer);
block_buffer = nullptr;
}
// Now copy the actual data.
char* data_ptr = nullptr;
size_t block_buffer_size = CMBlockBufferGetDataLength(contiguous_buffer);
status = CMBlockBufferGetDataPointer(contiguous_buffer, 0, nullptr, nullptr,
&data_ptr);
if (status != noErr) {
RTC_LOG(LS_ERROR) << "Failed to get block buffer data.";
CFRelease(contiguous_buffer);
return false;
}
size_t bytes_remaining = block_buffer_size;
while (bytes_remaining > 0) {
// The size type here must match |nalu_header_size|, we expect 4 bytes.
// Read the length of the next packet of data. Must convert from big endian
// to host endian.
RTC_DCHECK_GE(bytes_remaining, (size_t)nalu_header_size);
uint32_t* uint32_data_ptr = reinterpret_cast<uint32_t*>(data_ptr);
uint32_t packet_size = CFSwapInt32BigToHost(*uint32_data_ptr);
// Update buffer.
annexb_buffer->AppendData(kAnnexBHeaderBytes, sizeof(kAnnexBHeaderBytes));
annexb_buffer->AppendData(data_ptr + nalu_header_size, packet_size);
// Update fragmentation.
frag_offsets.push_back(nalu_offset + sizeof(kAnnexBHeaderBytes));
frag_lengths.push_back(packet_size);
nalu_offset += sizeof(kAnnexBHeaderBytes) + packet_size;
size_t bytes_written = packet_size + sizeof(kAnnexBHeaderBytes);
bytes_remaining -= bytes_written;
data_ptr += bytes_written;
}
RTC_DCHECK_EQ(bytes_remaining, (size_t)0);
CFRelease(contiguous_buffer);
return true;
}
bool H265AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer,
size_t annexb_buffer_size,
CMVideoFormatDescriptionRef video_format,
CMSampleBufferRef* out_sample_buffer) {
RTC_DCHECK(annexb_buffer);
RTC_DCHECK(out_sample_buffer);
RTC_DCHECK(video_format);
*out_sample_buffer = nullptr;
H265AnnexBBufferReader reader(annexb_buffer, annexb_buffer_size);
if (reader.SeekToNextNaluOfType(H265::kVps)) {
// Buffer contains an SPS NALU - skip it and the following PPS
const uint8_t* data;
size_t data_len;
if (!reader.ReadNalu(&data, &data_len)) {
RTC_LOG(LS_ERROR) << "Failed to read VPS";
return false;
}
if (!reader.ReadNalu(&data, &data_len)) {
RTC_LOG(LS_ERROR) << "Failed to read SPS";
return false;
}
if (!reader.ReadNalu(&data, &data_len)) {
RTC_LOG(LS_ERROR) << "Failed to read PPS";
return false;
}
} else {
// No SPS NALU - start reading from the first NALU in the buffer
reader.SeekToStart();
}
// Allocate memory as a block buffer.
// TODO(tkchin): figure out how to use a pool.
CMBlockBufferRef block_buffer = nullptr;
OSStatus status = CMBlockBufferCreateWithMemoryBlock(
nullptr, nullptr, reader.BytesRemaining(), nullptr, nullptr, 0,
reader.BytesRemaining(), kCMBlockBufferAssureMemoryNowFlag,
&block_buffer);
if (status != kCMBlockBufferNoErr) {
RTC_LOG(LS_ERROR) << "Failed to create block buffer.";
return false;
}
// Make sure block buffer is contiguous.
CMBlockBufferRef contiguous_buffer = nullptr;
if (!CMBlockBufferIsRangeContiguous(block_buffer, 0, 0)) {
status = CMBlockBufferCreateContiguous(
nullptr, block_buffer, nullptr, nullptr, 0, 0, 0, &contiguous_buffer);
if (status != noErr) {
RTC_LOG(LS_ERROR) << "Failed to flatten non-contiguous block buffer: "
<< status;
CFRelease(block_buffer);
return false;
}
} else {
contiguous_buffer = block_buffer;
block_buffer = nullptr;
}
// Get a raw pointer into allocated memory.
size_t block_buffer_size = 0;
char* data_ptr = nullptr;
status = CMBlockBufferGetDataPointer(contiguous_buffer, 0, nullptr,
&block_buffer_size, &data_ptr);
if (status != kCMBlockBufferNoErr) {
RTC_LOG(LS_ERROR) << "Failed to get block buffer data pointer.";
CFRelease(contiguous_buffer);
return false;
}
RTC_DCHECK(block_buffer_size == reader.BytesRemaining());
// Write Avcc NALUs into block buffer memory.
H265AvccBufferWriter writer(reinterpret_cast<uint8_t*>(data_ptr),
block_buffer_size);
while (reader.BytesRemaining() > 0) {
const uint8_t* nalu_data_ptr = nullptr;
size_t nalu_data_size = 0;
if (reader.ReadNalu(&nalu_data_ptr, &nalu_data_size)) {
writer.WriteNalu(nalu_data_ptr, nalu_data_size);
}
}
// Create sample buffer.
status = CMSampleBufferCreate(nullptr, contiguous_buffer, true, nullptr,
nullptr, video_format, 1, 0, nullptr, 0,
nullptr, out_sample_buffer);
if (status != noErr) {
RTC_LOG(LS_ERROR) << "Failed to create sample buffer.";
CFRelease(contiguous_buffer);
return false;
}
CFRelease(contiguous_buffer);
return true;
}
CMVideoFormatDescriptionRef CreateH265VideoFormatDescription(
const uint8_t* annexb_buffer,
size_t annexb_buffer_size) {
const uint8_t* param_set_ptrs[3] = {};
size_t param_set_sizes[3] = {};
H265AnnexBBufferReader reader(annexb_buffer, annexb_buffer_size);
// Skip everyting before the VPS, then read the VPS, SPS and PPS
if (!reader.SeekToNextNaluOfType(H265::kVps)) {
return nullptr;
}
if (!reader.ReadNalu(&param_set_ptrs[0], &param_set_sizes[0])) {
RTC_LOG(LS_ERROR) << "Failed to read VPS";
return nullptr;
}
if (!reader.ReadNalu(&param_set_ptrs[1], &param_set_sizes[1])) {
RTC_LOG(LS_ERROR) << "Failed to read SPS";
return nullptr;
}
if (!reader.ReadNalu(&param_set_ptrs[2], &param_set_sizes[2])) {
RTC_LOG(LS_ERROR) << "Failed to read PPS";
return nullptr;
}
// Parse the SPS and PPS into a CMVideoFormatDescription.
CMVideoFormatDescriptionRef description = nullptr;
OSStatus status = CMVideoFormatDescriptionCreateFromHEVCParameterSets(
kCFAllocatorDefault, 3, param_set_ptrs, param_set_sizes, 4, nullptr,
&description);
if (status != noErr) {
RTC_LOG(LS_ERROR) << "Failed to create video format description.";
return nullptr;
}
return description;
}
} // namespace webrtc

View file

@ -0,0 +1,53 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
#ifndef TGCALLS_PLATFORM_DARWIN_H265_NALU_REWRITER_H_
#define TGCALLS_PLATFORM_DARWIN_H265_NALU_REWRITER_H_
#include "modules/video_coding/codecs/h264/include/h264.h"
#include <CoreMedia/CoreMedia.h>
#include <vector>
#include "common_video/h264/h264_common.h"
#include "common_video/h265/h265_common.h"
#include "rtc_base/buffer.h"
using webrtc::H264::NaluIndex;
namespace webrtc {
bool H265CMSampleBufferToAnnexBBuffer(
CMSampleBufferRef hvcc_sample_buffer,
bool is_keyframe,
rtc::Buffer* annexb_buffer)
__OSX_AVAILABLE_STARTING(__MAC_10_12, __IPHONE_11_0);
// Converts a buffer received from RTP into a sample buffer suitable for the
// VideoToolbox decoder. The RTP buffer is in annex b format whereas the sample
// buffer is in hvcc format.
// If |is_keyframe| is true then |video_format| is ignored since the format will
// be read from the buffer. Otherwise |video_format| must be provided.
// Caller is responsible for releasing the created sample buffer.
bool H265AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer,
size_t annexb_buffer_size,
CMVideoFormatDescriptionRef video_format,
CMSampleBufferRef* out_sample_buffer)
__OSX_AVAILABLE_STARTING(__MAC_10_12, __IPHONE_11_0);
CMVideoFormatDescriptionRef CreateH265VideoFormatDescription(
const uint8_t* annexb_buffer,
size_t annexb_buffer_size)
__OSX_AVAILABLE_STARTING(__MAC_10_12, __IPHONE_11_0);
} // namespace webrtc
#endif // TGCALLS_PLATFORM_DARWIN_H265_NALU_REWRITER_H_

View file

@ -0,0 +1,48 @@
#ifndef TGCALLS_CALL_AUDIO_TONE_H_
#define TGCALLS_CALL_AUDIO_TONE_H_
#include <vector>
namespace tgcalls {
class CallAudioTone {
public:
CallAudioTone(std::vector<int16_t> &&samples, int sampleRate, int loopCount) :
_samples(std::move(samples)), _sampleRate(sampleRate), _loopCount(loopCount) {
}
public:
std::vector<int16_t> const samples() const {
return _samples;
}
int sampleRate() const {
return _sampleRate;
}
int loopCount() const {
return _loopCount;
}
void setLoopCount(int loopCount) {
_loopCount = loopCount;
}
size_t offset() const {
return _offset;
}
void setOffset(size_t offset) {
_offset = offset;
}
private:
std::vector<int16_t> _samples;
int _sampleRate = 48000;
int _loopCount = 1;
size_t _offset = 0;
};
}
#endif // TGCALLS_CALL_AUDIO_TONE_H_

View file

@ -0,0 +1,180 @@
/*
* Copyright 2016 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import "RTCAudioSession+Private.h"
#import "RTCAudioSessionConfiguration.h"
#import "base/RTCLogging.h"
@implementation RTC_OBJC_TYPE (RTCAudioSession)
(Configuration)
- (BOOL)setConfiguration : (RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration error
: (NSError **)outError disableRecording:(BOOL)disableRecording {
return [self setConfiguration:configuration
active:NO
shouldSetActive:NO
error:outError
disableRecording:disableRecording];
}
- (BOOL)setConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration
active:(BOOL)active
error:(NSError **)outError disableRecording:(BOOL)disableRecording {
return [self setConfiguration:configuration
active:active
shouldSetActive:YES
error:outError
disableRecording:disableRecording];
}
#pragma mark - Private
- (BOOL)setConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration
active:(BOOL)active
shouldSetActive:(BOOL)shouldSetActive
error:(NSError **)outError disableRecording:(BOOL)disableRecording {
NSParameterAssert(configuration);
if (outError) {
*outError = nil;
}
// Provide an error even if there isn't one so we can log it. We will not
// return immediately on error in this function and instead try to set
// everything we can.
NSError *error = nil;
if (!disableRecording) {
if (self.category != configuration.category ||
self.categoryOptions != configuration.categoryOptions) {
NSError *categoryError = nil;
if (![self setCategory:configuration.category
withOptions:configuration.categoryOptions
error:&categoryError]) {
RTCLogError(@"Failed to set category: %@",
categoryError.localizedDescription);
error = categoryError;
} else {
RTCLog(@"Set category to: %@", configuration.category);
}
}
if (self.mode != configuration.mode) {
NSError *modeError = nil;
if (![self setMode:configuration.mode error:&modeError]) {
RTCLogError(@"Failed to set mode: %@",
modeError.localizedDescription);
error = modeError;
} else {
RTCLog(@"Set mode to: %@", configuration.mode);
}
}
// Sometimes category options don't stick after setting mode.
if (self.categoryOptions != configuration.categoryOptions) {
NSError *categoryError = nil;
if (![self setCategory:configuration.category
withOptions:configuration.categoryOptions
error:&categoryError]) {
RTCLogError(@"Failed to set category options: %@",
categoryError.localizedDescription);
error = categoryError;
} else {
RTCLog(@"Set category options to: %ld",
(long)configuration.categoryOptions);
}
}
}
if (self.preferredSampleRate != configuration.sampleRate) {
NSError *sampleRateError = nil;
if (![self setPreferredSampleRate:configuration.sampleRate
error:&sampleRateError]) {
RTCLogError(@"Failed to set preferred sample rate: %@",
sampleRateError.localizedDescription);
if (!self.ignoresPreferredAttributeConfigurationErrors) {
error = sampleRateError;
}
} else {
RTCLog(@"Set preferred sample rate to: %.2f",
configuration.sampleRate);
}
}
if (self.preferredIOBufferDuration != configuration.ioBufferDuration) {
NSError *bufferDurationError = nil;
if (![self setPreferredIOBufferDuration:configuration.ioBufferDuration
error:&bufferDurationError]) {
RTCLogError(@"Failed to set preferred IO buffer duration: %@",
bufferDurationError.localizedDescription);
if (!self.ignoresPreferredAttributeConfigurationErrors) {
error = bufferDurationError;
}
} else {
RTCLog(@"Set preferred IO buffer duration to: %f",
configuration.ioBufferDuration);
}
}
if (shouldSetActive) {
NSError *activeError = nil;
if (![self setActive:active error:&activeError]) {
RTCLogError(@"Failed to setActive to %d: %@",
active, activeError.localizedDescription);
error = activeError;
}
}
if (self.isActive &&
// TODO(tkchin): Figure out which category/mode numChannels is valid for.
[self.mode isEqualToString:AVAudioSessionModeVoiceChat]) {
// Try to set the preferred number of hardware audio channels. These calls
// must be done after setting the audio sessions category and mode and
// activating the session.
NSInteger inputNumberOfChannels = configuration.inputNumberOfChannels;
if (self.inputNumberOfChannels != inputNumberOfChannels) {
NSError *inputChannelsError = nil;
if (![self setPreferredInputNumberOfChannels:inputNumberOfChannels
error:&inputChannelsError]) {
RTCLogError(@"Failed to set preferred input number of channels: %@",
inputChannelsError.localizedDescription);
if (!self.ignoresPreferredAttributeConfigurationErrors) {
error = inputChannelsError;
}
} else {
RTCLog(@"Set input number of channels to: %ld",
(long)inputNumberOfChannels);
}
}
NSInteger outputNumberOfChannels = configuration.outputNumberOfChannels;
if (self.outputNumberOfChannels != outputNumberOfChannels) {
NSError *outputChannelsError = nil;
if (![self setPreferredOutputNumberOfChannels:outputNumberOfChannels
error:&outputChannelsError]) {
RTCLogError(@"Failed to set preferred output number of channels: %@",
outputChannelsError.localizedDescription);
if (!self.ignoresPreferredAttributeConfigurationErrors) {
error = outputChannelsError;
}
} else {
RTCLog(@"Set output number of channels to: %ld",
(long)outputNumberOfChannels);
}
}
}
if (outError) {
*outError = error;
}
return error == nil;
}
@end

View file

@ -0,0 +1,95 @@
/*
* Copyright 2016 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import "RTCAudioSession.h"
NS_ASSUME_NONNULL_BEGIN
@class RTC_OBJC_TYPE(RTCAudioSessionConfiguration);
@interface RTC_OBJC_TYPE (RTCAudioSession)
()
/** Number of times setActive:YES has succeeded without a balanced call to
* setActive:NO.
*/
@property(nonatomic, readonly) int activationCount;
/** The number of times `beginWebRTCSession` was called without a balanced call
* to `endWebRTCSession`.
*/
@property(nonatomic, readonly) int webRTCSessionCount;
/** Convenience BOOL that checks useManualAudio and isAudioEnebled. */
@property(readonly) BOOL canPlayOrRecord;
/** Tracks whether we have been sent an interruption event that hasn't been matched by either an
* interrupted end event or a foreground event.
*/
@property(nonatomic, assign) BOOL isInterrupted;
/** Adds the delegate to the list of delegates, and places it at the front of
* the list. This delegate will be notified before other delegates of
* audio events.
*/
- (void)pushDelegate:(id<RTC_OBJC_TYPE(RTCAudioSessionDelegate)>)delegate;
/** Signals RTCAudioSession that a WebRTC session is about to begin and
* audio configuration is needed. Will configure the audio session for WebRTC
* if not already configured and if configuration is not delayed.
* Successful calls must be balanced by a call to endWebRTCSession.
*/
- (BOOL)beginWebRTCSession:(NSError **)outError;
/** Signals RTCAudioSession that a WebRTC session is about to end and audio
* unconfiguration is needed. Will unconfigure the audio session for WebRTC
* if this is the last unmatched call and if configuration is not delayed.
*/
- (BOOL)endWebRTCSession:(NSError **)outError;
/** Configure the audio session for WebRTC. This call will fail if the session
* is already configured. On other failures, we will attempt to restore the
* previously used audio session configuration.
* `lockForConfiguration` must be called first.
* Successful calls to configureWebRTCSession must be matched by calls to
* `unconfigureWebRTCSession`.
*/
- (BOOL)configureWebRTCSession:(NSError **)outError disableRecording:(BOOL)disableRecording;
/** Unconfigures the session for WebRTC. This will attempt to restore the
* audio session to the settings used before `configureWebRTCSession` was
* called.
* `lockForConfiguration` must be called first.
*/
- (BOOL)unconfigureWebRTCSession:(NSError **)outError;
/** Returns a configuration error with the given description. */
- (NSError *)configurationErrorWithDescription:(NSString *)description;
/** Notifies the receiver that a playout glitch was detected. */
- (void)notifyDidDetectPlayoutGlitch:(int64_t)totalNumberOfGlitches;
/** Notifies the receiver that there was an error when starting an audio unit. */
- (void)notifyAudioUnitStartFailedWithError:(OSStatus)error;
// Properties and methods for tests.
- (void)notifyDidBeginInterruption;
- (void)notifyDidEndInterruptionWithShouldResumeSession:(BOOL)shouldResumeSession;
- (void)notifyDidChangeRouteWithReason:(AVAudioSessionRouteChangeReason)reason
previousRoute:(AVAudioSessionRouteDescription *)previousRoute;
- (void)notifyMediaServicesWereLost;
- (void)notifyMediaServicesWereReset;
- (void)notifyDidChangeCanPlayOrRecord:(BOOL)canPlayOrRecord;
- (void)notifyDidStartPlayOrRecord;
- (void)notifyDidStopPlayOrRecord;
@end
NS_ASSUME_NONNULL_END

View file

@ -0,0 +1,266 @@
/*
* Copyright 2016 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import <AVFoundation/AVFoundation.h>
#import <Foundation/Foundation.h>
#import "RTCMacros.h"
NS_ASSUME_NONNULL_BEGIN
extern NSString *const kRTCAudioSessionErrorDomain;
/** Method that requires lock was called without lock. */
extern NSInteger const kRTCAudioSessionErrorLockRequired;
/** Unknown configuration error occurred. */
extern NSInteger const kRTCAudioSessionErrorConfiguration;
@class RTC_OBJC_TYPE(RTCAudioSession);
@class RTC_OBJC_TYPE(RTCAudioSessionConfiguration);
// Surfaces AVAudioSession events. WebRTC will listen directly for notifications
// from AVAudioSession and handle them before calling these delegate methods,
// at which point applications can perform additional processing if required.
RTC_OBJC_EXPORT
@protocol RTC_OBJC_TYPE
(RTCAudioSessionDelegate)<NSObject>
@optional
/** Called on a system notification thread when AVAudioSession starts an
* interruption event.
*/
- (void)audioSessionDidBeginInterruption:(RTC_OBJC_TYPE(RTCAudioSession) *)session;
/** Called on a system notification thread when AVAudioSession ends an
* interruption event.
*/
- (void)audioSessionDidEndInterruption:(RTC_OBJC_TYPE(RTCAudioSession) *)session
shouldResumeSession:(BOOL)shouldResumeSession;
/** Called on a system notification thread when AVAudioSession changes the
* route.
*/
- (void)audioSessionDidChangeRoute:(RTC_OBJC_TYPE(RTCAudioSession) *)session
reason:(AVAudioSessionRouteChangeReason)reason
previousRoute:(AVAudioSessionRouteDescription *)previousRoute;
/** Called on a system notification thread when AVAudioSession media server
* terminates.
*/
- (void)audioSessionMediaServerTerminated:(RTC_OBJC_TYPE(RTCAudioSession) *)session;
/** Called on a system notification thread when AVAudioSession media server
* restarts.
*/
- (void)audioSessionMediaServerReset:(RTC_OBJC_TYPE(RTCAudioSession) *)session;
// TODO(tkchin): Maybe handle SilenceSecondaryAudioHintNotification.
- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)session
didChangeCanPlayOrRecord:(BOOL)canPlayOrRecord;
/** Called on a WebRTC thread when the audio device is notified to begin
* playback or recording.
*/
- (void)audioSessionDidStartPlayOrRecord:(RTC_OBJC_TYPE(RTCAudioSession) *)session;
/** Called on a WebRTC thread when the audio device is notified to stop
* playback or recording.
*/
- (void)audioSessionDidStopPlayOrRecord:(RTC_OBJC_TYPE(RTCAudioSession) *)session;
/** Called when the AVAudioSession output volume value changes. */
- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession
didChangeOutputVolume:(float)outputVolume;
/** Called when the audio device detects a playout glitch. The argument is the
* number of glitches detected so far in the current audio playout session.
*/
- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession
didDetectPlayoutGlitch:(int64_t)totalNumberOfGlitches;
/** Called when the audio session is about to change the active state.
*/
- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession willSetActive:(BOOL)active;
/** Called after the audio session sucessfully changed the active state.
*/
- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession didSetActive:(BOOL)active;
/** Called after the audio session failed to change the active state.
*/
- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession
failedToSetActive:(BOOL)active
error:(NSError *)error;
- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession
audioUnitStartFailedWithError:(NSError *)error;
@end
/** This is a protocol used to inform RTCAudioSession when the audio session
* activation state has changed outside of RTCAudioSession. The current known use
* case of this is when CallKit activates the audio session for the application
*/
RTC_OBJC_EXPORT
@protocol RTC_OBJC_TYPE
(RTCAudioSessionActivationDelegate)<NSObject>
/** Called when the audio session is activated outside of the app by iOS. */
- (void)audioSessionDidActivate : (AVAudioSession *)session;
/** Called when the audio session is deactivated outside of the app by iOS. */
- (void)audioSessionDidDeactivate:(AVAudioSession *)session;
@end
/** Proxy class for AVAudioSession that adds a locking mechanism similar to
* AVCaptureDevice. This is used to that interleaving configurations between
* WebRTC and the application layer are avoided.
*
* RTCAudioSession also coordinates activation so that the audio session is
* activated only once. See `setActive:error:`.
*/
RTC_OBJC_EXPORT
@interface RTC_OBJC_TYPE (RTCAudioSession) : NSObject <RTC_OBJC_TYPE(RTCAudioSessionActivationDelegate)>
/** Convenience property to access the AVAudioSession singleton. Callers should
* not call setters on AVAudioSession directly, but other method invocations
* are fine.
*/
@property(nonatomic, readonly) AVAudioSession *session;
/** Our best guess at whether the session is active based on results of calls to
* AVAudioSession.
*/
@property(nonatomic, readonly) BOOL isActive;
/** If YES, WebRTC will not initialize the audio unit automatically when an
* audio track is ready for playout or recording. Instead, applications should
* call setIsAudioEnabled. If NO, WebRTC will initialize the audio unit
* as soon as an audio track is ready for playout or recording.
*/
@property(nonatomic, assign) BOOL useManualAudio;
/** This property is only effective if useManualAudio is YES.
* Represents permission for WebRTC to initialize the VoIP audio unit.
* When set to NO, if the VoIP audio unit used by WebRTC is active, it will be
* stopped and uninitialized. This will stop incoming and outgoing audio.
* When set to YES, WebRTC will initialize and start the audio unit when it is
* needed (e.g. due to establishing an audio connection).
* This property was introduced to work around an issue where if an AVPlayer is
* playing audio while the VoIP audio unit is initialized, its audio would be
* either cut off completely or played at a reduced volume. By preventing
* the audio unit from being initialized until after the audio has completed,
* we are able to prevent the abrupt cutoff.
*/
@property(nonatomic, assign) BOOL isAudioEnabled;
// Proxy properties.
@property(readonly) NSString *category;
@property(readonly) AVAudioSessionCategoryOptions categoryOptions;
@property(readonly) NSString *mode;
@property(readonly) BOOL secondaryAudioShouldBeSilencedHint;
@property(readonly) AVAudioSessionRouteDescription *currentRoute;
@property(readonly) NSInteger maximumInputNumberOfChannels;
@property(readonly) NSInteger maximumOutputNumberOfChannels;
@property(readonly) float inputGain;
@property(readonly) BOOL inputGainSettable;
@property(readonly) BOOL inputAvailable;
@property(readonly, nullable) NSArray<AVAudioSessionDataSourceDescription *> *inputDataSources;
@property(readonly, nullable) AVAudioSessionDataSourceDescription *inputDataSource;
@property(readonly, nullable) NSArray<AVAudioSessionDataSourceDescription *> *outputDataSources;
@property(readonly, nullable) AVAudioSessionDataSourceDescription *outputDataSource;
@property(readonly) double sampleRate;
@property(readonly) double preferredSampleRate;
@property(readonly) NSInteger inputNumberOfChannels;
@property(readonly) NSInteger outputNumberOfChannels;
@property(readonly) float outputVolume;
@property(readonly) NSTimeInterval inputLatency;
@property(readonly) NSTimeInterval outputLatency;
@property(readonly) NSTimeInterval IOBufferDuration;
@property(readonly) NSTimeInterval preferredIOBufferDuration;
/**
When YES, calls to -setConfiguration:error: and -setConfiguration:active:error: ignore errors in
configuring the audio session's "preferred" attributes (e.g. preferredInputNumberOfChannels).
Typically, configurations to preferred attributes are optimizations, and ignoring this type of
configuration error allows code flow to continue along the happy path when these optimization are
not available. The default value of this property is NO.
*/
@property(nonatomic) BOOL ignoresPreferredAttributeConfigurationErrors;
/** Default constructor. */
+ (instancetype)sharedInstance;
- (instancetype)init NS_UNAVAILABLE;
/** Adds a delegate, which is held weakly. */
- (void)addDelegate:(id<RTC_OBJC_TYPE(RTCAudioSessionDelegate)>)delegate;
/** Removes an added delegate. */
- (void)removeDelegate:(id<RTC_OBJC_TYPE(RTCAudioSessionDelegate)>)delegate;
/** Request exclusive access to the audio session for configuration. This call
* will block if the lock is held by another object.
*/
- (void)lockForConfiguration;
/** Relinquishes exclusive access to the audio session. */
- (void)unlockForConfiguration;
/** If `active`, activates the audio session if it isn't already active.
* Successful calls must be balanced with a setActive:NO when activation is no
* longer required. If not `active`, deactivates the audio session if one is
* active and this is the last balanced call. When deactivating, the
* AVAudioSessionSetActiveOptionNotifyOthersOnDeactivation option is passed to
* AVAudioSession.
*/
- (BOOL)setActive:(BOOL)active error:(NSError **)outError;
// The following methods are proxies for the associated methods on
// AVAudioSession. `lockForConfiguration` must be called before using them
// otherwise they will fail with kRTCAudioSessionErrorLockRequired.
- (BOOL)setCategory:(NSString *)category
withOptions:(AVAudioSessionCategoryOptions)options
error:(NSError **)outError;
- (BOOL)setMode:(NSString *)mode error:(NSError **)outError;
- (BOOL)setInputGain:(float)gain error:(NSError **)outError;
- (BOOL)setPreferredSampleRate:(double)sampleRate error:(NSError **)outError;
- (BOOL)setPreferredIOBufferDuration:(NSTimeInterval)duration error:(NSError **)outError;
- (BOOL)setPreferredInputNumberOfChannels:(NSInteger)count error:(NSError **)outError;
- (BOOL)setPreferredOutputNumberOfChannels:(NSInteger)count error:(NSError **)outError;
- (BOOL)overrideOutputAudioPort:(AVAudioSessionPortOverride)portOverride error:(NSError **)outError;
- (BOOL)setPreferredInput:(AVAudioSessionPortDescription *)inPort error:(NSError **)outError;
- (BOOL)setInputDataSource:(AVAudioSessionDataSourceDescription *)dataSource
error:(NSError **)outError;
- (BOOL)setOutputDataSource:(AVAudioSessionDataSourceDescription *)dataSource
error:(NSError **)outError;
@end
@interface RTC_OBJC_TYPE (RTCAudioSession)
(Configuration)
/** Applies the configuration to the current session. Attempts to set all
* properties even if previous ones fail. Only the last error will be
* returned.
* `lockForConfiguration` must be called first.
*/
- (BOOL)setConfiguration : (RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration error
: (NSError **)outError;
/** Convenience method that calls both setConfiguration and setActive.
* `lockForConfiguration` must be called first.
*/
- (BOOL)setConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration
active:(BOOL)active
error:(NSError **)outError
disableRecording:(BOOL)disableRecording;
@end
NS_ASSUME_NONNULL_END

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,47 @@
/*
* Copyright 2016 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import <AVFoundation/AVFoundation.h>
#import <Foundation/Foundation.h>
#import "RTCMacros.h"
NS_ASSUME_NONNULL_BEGIN
RTC_EXTERN const double kRTCAudioSessionHighPerformanceSampleRate;
RTC_EXTERN const double kRTCAudioSessionLowComplexitySampleRate;
RTC_EXTERN const double kRTCAudioSessionHighPerformanceIOBufferDuration;
RTC_EXTERN const double kRTCAudioSessionLowComplexityIOBufferDuration;
// Struct to hold configuration values.
RTC_OBJC_EXPORT
@interface RTC_OBJC_TYPE (RTCAudioSessionConfiguration) : NSObject
@property(nonatomic, strong) NSString *category;
@property(nonatomic, assign) AVAudioSessionCategoryOptions categoryOptions;
@property(nonatomic, strong) NSString *mode;
@property(nonatomic, assign) double sampleRate;
@property(nonatomic, assign) NSTimeInterval ioBufferDuration;
@property(nonatomic, assign) NSInteger inputNumberOfChannels;
@property(nonatomic, assign) NSInteger outputNumberOfChannels;
/** Initializes configuration to defaults. */
- (instancetype)init NS_DESIGNATED_INITIALIZER;
/** Returns the current configuration of the audio session. */
+ (instancetype)currentConfiguration;
/** Returns the configuration that WebRTC needs. */
+ (instancetype)webRTCConfiguration;
/** Provide a way to override the default configuration. */
+ (void)setWebRTCConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration;
@end
NS_ASSUME_NONNULL_END

View file

@ -0,0 +1,133 @@
/*
* Copyright 2016 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import "RTCAudioSessionConfiguration.h"
#import "RTCAudioSession.h"
#import "helpers/RTCDispatcher.h"
#import "helpers/UIDevice+RTCDevice.h"
// Try to use mono to save resources. Also avoids channel format conversion
// in the I/O audio unit. Initial tests have shown that it is possible to use
// mono natively for built-in microphones and for BT headsets but not for
// wired headsets. Wired headsets only support stereo as native channel format
// but it is a low cost operation to do a format conversion to mono in the
// audio unit. Hence, we will not hit a RTC_CHECK in
// VerifyAudioParametersForActiveAudioSession() for a mismatch between the
// preferred number of channels and the actual number of channels.
const int kRTCAudioSessionPreferredNumberOfChannels = 1;
// Preferred hardware sample rate (unit is in Hertz). The client sample rate
// will be set to this value as well to avoid resampling the the audio unit's
// format converter. Note that, some devices, e.g. BT headsets, only supports
// 8000Hz as native sample rate.
const double kRTCAudioSessionHighPerformanceSampleRate = 48000.0;
// A lower sample rate will be used for devices with only one core
// (e.g. iPhone 4). The goal is to reduce the CPU load of the application.
const double kRTCAudioSessionLowComplexitySampleRate = 16000.0;
// Use a hardware I/O buffer size (unit is in seconds) that matches the 10ms
// size used by WebRTC. The exact actual size will differ between devices.
// Example: using 48kHz on iPhone 6 results in a native buffer size of
// ~10.6667ms or 512 audio frames per buffer. The FineAudioBuffer instance will
// take care of any buffering required to convert between native buffers and
// buffers used by WebRTC. It is beneficial for the performance if the native
// size is as an even multiple of 10ms as possible since it results in "clean"
// callback sequence without bursts of callbacks back to back.
const double kRTCAudioSessionHighPerformanceIOBufferDuration = 0.02;
// Use a larger buffer size on devices with only one core (e.g. iPhone 4).
// It will result in a lower CPU consumption at the cost of a larger latency.
// The size of 60ms is based on instrumentation that shows a significant
// reduction in CPU load compared with 10ms on low-end devices.
// TODO(henrika): monitor this size and determine if it should be modified.
const double kRTCAudioSessionLowComplexityIOBufferDuration = 0.06;
static RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *gWebRTCConfiguration = nil;
@implementation RTC_OBJC_TYPE (RTCAudioSessionConfiguration)
@synthesize category = _category;
@synthesize categoryOptions = _categoryOptions;
@synthesize mode = _mode;
@synthesize sampleRate = _sampleRate;
@synthesize ioBufferDuration = _ioBufferDuration;
@synthesize inputNumberOfChannels = _inputNumberOfChannels;
@synthesize outputNumberOfChannels = _outputNumberOfChannels;
- (instancetype)init {
if (self = [super init]) {
// Use a category which supports simultaneous recording and playback.
// By default, using this category implies that our apps audio is
// nonmixable, hence activating the session will interrupt any other
// audio sessions which are also nonmixable.
_category = AVAudioSessionCategoryPlayAndRecord;
_categoryOptions = AVAudioSessionCategoryOptionAllowBluetooth;
// Specify mode for two-way voice communication (e.g. VoIP).
_mode = AVAudioSessionModeVoiceChat;
// Set the session's sample rate or the hardware sample rate.
// It is essential that we use the same sample rate as stream format
// to ensure that the I/O unit does not have to do sample rate conversion.
// Set the preferred audio I/O buffer duration, in seconds.
NSUInteger processorCount = [NSProcessInfo processInfo].processorCount;
// Use best sample rate and buffer duration if the CPU has more than one
// core.
if (processorCount > 1) {
_sampleRate = kRTCAudioSessionHighPerformanceSampleRate;
_ioBufferDuration = kRTCAudioSessionHighPerformanceIOBufferDuration;
} else {
_sampleRate = kRTCAudioSessionLowComplexitySampleRate;
_ioBufferDuration = kRTCAudioSessionLowComplexityIOBufferDuration;
}
// We try to use mono in both directions to save resources and format
// conversions in the audio unit. Some devices does only support stereo;
// e.g. wired headset on iPhone 6.
// TODO(henrika): add support for stereo if needed.
_inputNumberOfChannels = kRTCAudioSessionPreferredNumberOfChannels;
_outputNumberOfChannels = kRTCAudioSessionPreferredNumberOfChannels;
}
return self;
}
+ (void)initialize {
gWebRTCConfiguration = [[self alloc] init];
}
+ (instancetype)currentConfiguration {
RTC_OBJC_TYPE(RTCAudioSession) *session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance];
RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *config =
[[RTC_OBJC_TYPE(RTCAudioSessionConfiguration) alloc] init];
config.category = session.category;
config.categoryOptions = session.categoryOptions;
config.mode = session.mode;
config.sampleRate = session.sampleRate;
config.ioBufferDuration = session.IOBufferDuration;
config.inputNumberOfChannels = session.inputNumberOfChannels;
config.outputNumberOfChannels = session.outputNumberOfChannels;
return config;
}
+ (instancetype)webRTCConfiguration {
@synchronized(self) {
return (RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)gWebRTCConfiguration;
}
}
+ (void)setWebRTCConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration {
@synchronized(self) {
gWebRTCConfiguration = configuration;
}
}
@end

View file

@ -0,0 +1,33 @@
/*
* Copyright 2018 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import "RTCAudioSession.h"
NS_ASSUME_NONNULL_BEGIN
namespace webrtc {
class AudioSessionObserver;
}
/** Adapter that forwards RTCAudioSessionDelegate calls to the appropriate
* methods on the AudioSessionObserver.
*/
@interface RTCNativeAudioSessionDelegateAdapter : NSObject <RTC_OBJC_TYPE (RTCAudioSessionDelegate)>
- (instancetype)init NS_UNAVAILABLE;
/** `observer` is a raw pointer and should be kept alive
* for this object's lifetime.
*/
- (instancetype)initWithObserver:(webrtc::AudioSessionObserver *)observer NS_DESIGNATED_INITIALIZER;
@end
NS_ASSUME_NONNULL_END

View file

@ -0,0 +1,89 @@
/*
* Copyright 2018 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import "RTCNativeAudioSessionDelegateAdapter.h"
#include "sdk/objc/native/src/audio/audio_session_observer.h"
#import "base/RTCLogging.h"
@implementation RTCNativeAudioSessionDelegateAdapter {
webrtc::AudioSessionObserver *_observer;
}
- (instancetype)initWithObserver:(webrtc::AudioSessionObserver *)observer {
RTC_DCHECK(observer);
if (self = [super init]) {
_observer = observer;
}
return self;
}
#pragma mark - RTC_OBJC_TYPE(RTCAudioSessionDelegate)
- (void)audioSessionDidBeginInterruption:(RTC_OBJC_TYPE(RTCAudioSession) *)session {
_observer->OnInterruptionBegin();
}
- (void)audioSessionDidEndInterruption:(RTC_OBJC_TYPE(RTCAudioSession) *)session
shouldResumeSession:(BOOL)shouldResumeSession {
_observer->OnInterruptionEnd();
}
- (void)audioSessionDidChangeRoute:(RTC_OBJC_TYPE(RTCAudioSession) *)session
reason:(AVAudioSessionRouteChangeReason)reason
previousRoute:(AVAudioSessionRouteDescription *)previousRoute {
switch (reason) {
case AVAudioSessionRouteChangeReasonUnknown:
case AVAudioSessionRouteChangeReasonNewDeviceAvailable:
case AVAudioSessionRouteChangeReasonOldDeviceUnavailable:
case AVAudioSessionRouteChangeReasonCategoryChange:
// It turns out that we see a category change (at least in iOS 9.2)
// when making a switch from a BT device to e.g. Speaker using the
// iOS Control Center and that we therefore must check if the sample
// rate has changed. And if so is the case, restart the audio unit.
case AVAudioSessionRouteChangeReasonOverride:
case AVAudioSessionRouteChangeReasonWakeFromSleep:
case AVAudioSessionRouteChangeReasonNoSuitableRouteForCategory:
_observer->OnValidRouteChange();
break;
case AVAudioSessionRouteChangeReasonRouteConfigurationChange:
// The set of input and output ports has not changed, but their
// configuration has, e.g., a ports selected data source has
// changed. Ignore this type of route change since we are focusing
// on detecting headset changes.
RTCLog(@"Ignoring RouteConfigurationChange");
break;
}
}
- (void)audioSessionMediaServerTerminated:(RTC_OBJC_TYPE(RTCAudioSession) *)session {
}
- (void)audioSessionMediaServerReset:(RTC_OBJC_TYPE(RTCAudioSession) *)session {
}
- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)session
didChangeCanPlayOrRecord:(BOOL)canPlayOrRecord {
_observer->OnCanPlayOrRecordChange(canPlayOrRecord);
}
- (void)audioSessionDidStartPlayOrRecord:(RTC_OBJC_TYPE(RTCAudioSession) *)session {
}
- (void)audioSessionDidStopPlayOrRecord:(RTC_OBJC_TYPE(RTCAudioSession) *)session {
}
- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession
didChangeOutputVolume:(float)outputVolume {
_observer->OnChangedOutputVolume();
}
@end

View file

@ -0,0 +1,329 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef TGCALLS_AUDIO_AUDIO_DEVICE_IOS_H_
#define TGCALLS_AUDIO_AUDIO_DEVICE_IOS_H_
#include <atomic>
#include <memory>
#include "api/scoped_refptr.h"
#include "api/sequence_checker.h"
#include "sdk/objc/native/src/audio/audio_session_observer.h"
#include "api/task_queue/pending_task_safety_flag.h"
#include "modules/audio_device/audio_device_generic.h"
#include "rtc_base/buffer.h"
#include "rtc_base/thread.h"
#include "rtc_base/thread_annotations.h"
#include "sdk/objc/base/RTCMacros.h"
#include "tgcalls_voice_processing_audio_unit.h"
#include "platform/darwin/iOS/CallAudioTone.h"
RTC_FWD_DECL_OBJC_CLASS(RTCNativeAudioSessionDelegateAdapter);
namespace webrtc {
class FineAudioBuffer;
namespace tgcalls_ios_adm {
// Implements full duplex 16-bit mono PCM audio support for iOS using a
// Voice-Processing (VP) I/O audio unit in Core Audio. The VP I/O audio unit
// supports audio echo cancellation. It also adds automatic gain control,
// adjustment of voice-processing quality and muting.
//
// An instance must be created and destroyed on one and the same thread.
// All supported public methods must also be called on the same thread.
// A thread checker will RTC_DCHECK if any supported method is called on an
// invalid thread.
//
// Recorded audio will be delivered on a real-time internal I/O thread in the
// audio unit. The audio unit will also ask for audio data to play out on this
// same thread.
class AudioDeviceIOS : public AudioDeviceGeneric,
public AudioSessionObserver,
public VoiceProcessingAudioUnitObserver {
public:
explicit AudioDeviceIOS(bool bypass_voice_processing, bool disable_recording, bool enableSystemMute, int numChannels);
~AudioDeviceIOS() override;
void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) override;
InitStatus Init() override;
int32_t Terminate() override;
bool Initialized() const override;
int32_t InitPlayout() override;
bool PlayoutIsInitialized() const override;
int32_t InitRecording() override;
bool RecordingIsInitialized() const override;
int32_t StartPlayout() override;
int32_t StopPlayout() override;
bool Playing() const override;
int32_t StartRecording() override;
int32_t StopRecording() override;
bool Recording() const override;
void setIsBufferPlaying(bool isBufferPlaying);
void setIsBufferRecording(bool isBufferRecording);
// These methods returns hard-coded delay values and not dynamic delay
// estimates. The reason is that iOS supports a built-in AEC and the WebRTC
// AEC will always be disabled in the Libjingle layer to avoid running two
// AEC implementations at the same time. And, it saves resources to avoid
// updating these delay values continuously.
// TODO(henrika): it would be possible to mark these two methods as not
// implemented since they are only called for A/V-sync purposes today and
// A/V-sync is not supported on iOS. However, we avoid adding error messages
// the log by using these dummy implementations instead.
int32_t PlayoutDelay(uint16_t& delayMS) const override;
// No implementation for playout underrun on iOS. We override it to avoid a
// periodic log that it isn't available from the base class.
int32_t GetPlayoutUnderrunCount() const override { return -1; }
// Native audio parameters stored during construction.
// These methods are unique for the iOS implementation.
int GetPlayoutAudioParameters(AudioParameters* params) const override;
int GetRecordAudioParameters(AudioParameters* params) const override;
// These methods are currently not fully implemented on iOS:
// See audio_device_not_implemented.cc for trivial implementations.
int32_t ActiveAudioLayer(
AudioDeviceModule::AudioLayer& audioLayer) const override;
int32_t PlayoutIsAvailable(bool& available) override;
int32_t RecordingIsAvailable(bool& available) override;
int16_t PlayoutDevices() override;
int16_t RecordingDevices() override;
int32_t PlayoutDeviceName(uint16_t index,
char name[kAdmMaxDeviceNameSize],
char guid[kAdmMaxGuidSize]) override;
int32_t RecordingDeviceName(uint16_t index,
char name[kAdmMaxDeviceNameSize],
char guid[kAdmMaxGuidSize]) override;
int32_t SetPlayoutDevice(uint16_t index) override;
int32_t SetPlayoutDevice(
AudioDeviceModule::WindowsDeviceType device) override;
int32_t SetRecordingDevice(uint16_t index) override;
int32_t SetRecordingDevice(
AudioDeviceModule::WindowsDeviceType device) override;
int32_t InitSpeaker() override;
bool SpeakerIsInitialized() const override;
int32_t InitMicrophone() override;
bool MicrophoneIsInitialized() const override;
int32_t SpeakerVolumeIsAvailable(bool& available) override;
int32_t SetSpeakerVolume(uint32_t volume) override;
int32_t SpeakerVolume(uint32_t& volume) const override;
int32_t MaxSpeakerVolume(uint32_t& maxVolume) const override;
int32_t MinSpeakerVolume(uint32_t& minVolume) const override;
int32_t MicrophoneVolumeIsAvailable(bool& available) override;
int32_t SetMicrophoneVolume(uint32_t volume) override;
int32_t MicrophoneVolume(uint32_t& volume) const override;
int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const override;
int32_t MinMicrophoneVolume(uint32_t& minVolume) const override;
int32_t MicrophoneMuteIsAvailable(bool& available) override;
int32_t SetMicrophoneMute(bool enable) override;
int32_t MicrophoneMute(bool& enabled) const override;
int32_t SpeakerMuteIsAvailable(bool& available) override;
int32_t SetSpeakerMute(bool enable) override;
int32_t SpeakerMute(bool& enabled) const override;
int32_t StereoPlayoutIsAvailable(bool& available) override;
int32_t SetStereoPlayout(bool enable) override;
int32_t StereoPlayout(bool& enabled) const override;
int32_t StereoRecordingIsAvailable(bool& available) override;
int32_t SetStereoRecording(bool enable) override;
int32_t StereoRecording(bool& enabled) const override;
// AudioSessionObserver methods. May be called from any thread.
void OnInterruptionBegin() override;
void OnInterruptionEnd() override;
void OnValidRouteChange() override;
void OnCanPlayOrRecordChange(bool can_play_or_record) override;
void OnChangedOutputVolume() override;
void setTone(std::shared_ptr<tgcalls::CallAudioTone> tone);
// VoiceProcessingAudioUnitObserver methods.
OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags,
const AudioTimeStamp* time_stamp,
UInt32 bus_number,
UInt32 num_frames,
AudioBufferList* io_data) override;
OSStatus OnGetPlayoutData(AudioUnitRenderActionFlags* flags,
const AudioTimeStamp* time_stamp,
UInt32 bus_number,
UInt32 num_frames,
AudioBufferList* io_data) override;
void OnMutedSpeechStatusChanged(bool isDetectingSpeech) override;
bool IsInterrupted();
public:
void (^mutedSpeechDetectionChanged)(bool);
private:
// Called by the relevant AudioSessionObserver methods on `thread_`.
void HandleInterruptionBegin();
void HandleInterruptionEnd();
void HandleValidRouteChange();
void HandleCanPlayOrRecordChange(bool can_play_or_record);
void HandleSampleRateChange();
void HandlePlayoutGlitchDetected();
void HandleOutputVolumeChange();
// Uses current `playout_parameters_` and `record_parameters_` to inform the
// audio device buffer (ADB) about our internal audio parameters.
void UpdateAudioDeviceBuffer();
// Since the preferred audio parameters are only hints to the OS, the actual
// values may be different once the AVAudioSession has been activated.
// This method asks for the current hardware parameters and takes actions
// if they should differ from what we have asked for initially. It also
// defines `playout_parameters_` and `record_parameters_`.
void SetupAudioBuffersForActiveAudioSession();
// Creates the audio unit.
bool CreateAudioUnit();
// Updates the audio unit state based on current state.
void UpdateAudioUnit(bool can_play_or_record);
// Configures the audio session for WebRTC.
bool ConfigureAudioSession();
// Like above, but requires caller to already hold session lock.
bool ConfigureAudioSessionLocked();
// Unconfigures the audio session.
void UnconfigureAudioSession();
// Activates our audio session, creates and initializes the voice-processing
// audio unit and verifies that we got the preferred native audio parameters.
bool InitPlayOrRecord();
// Closes and deletes the voice-processing I/O unit.
void ShutdownPlayOrRecord();
// Resets thread-checkers before a call is restarted.
void PrepareForNewStart();
// Determines whether voice processing should be enabled or disabled.
const bool bypass_voice_processing_;
const bool disable_recording_;
const bool enableSystemMute_ = false;
const int numChannels_;
// Native I/O audio thread checker.
SequenceChecker io_thread_checker_;
// Thread that this object is created on.
rtc::Thread* thread_;
// Raw pointer handle provided to us in AttachAudioBuffer(). Owned by the
// AudioDeviceModuleImpl class and called by AudioDeviceModule::Create().
// The AudioDeviceBuffer is a member of the AudioDeviceModuleImpl instance
// and therefore outlives this object.
AudioDeviceBuffer* audio_device_buffer_;
// Contains audio parameters (sample rate, #channels, buffer size etc.) for
// the playout and recording sides. These structure is set in two steps:
// first, native sample rate and #channels are defined in Init(). Next, the
// audio session is activated and we verify that the preferred parameters
// were granted by the OS. At this stage it is also possible to add a third
// component to the parameters; the native I/O buffer duration.
// A RTC_CHECK will be hit if we for some reason fail to open an audio session
// using the specified parameters.
AudioParameters playout_parameters_;
AudioParameters record_parameters_;
// The AudioUnit used to play and record audio.
std::unique_ptr<VoiceProcessingAudioUnit> audio_unit_;
// FineAudioBuffer takes an AudioDeviceBuffer which delivers audio data
// in chunks of 10ms. It then allows for this data to be pulled in
// a finer or coarser granularity. I.e. interacting with this class instead
// of directly with the AudioDeviceBuffer one can ask for any number of
// audio data samples. Is also supports a similar scheme for the recording
// side.
// Example: native buffer size can be 128 audio frames at 16kHz sample rate.
// WebRTC will provide 480 audio frames per 10ms but iOS asks for 128
// in each callback (one every 8ms). This class can then ask for 128 and the
// FineAudioBuffer will ask WebRTC for new data only when needed and also
// cache non-utilized audio between callbacks. On the recording side, iOS
// can provide audio data frames of size 128 and these are accumulated until
// enough data to supply one 10ms call exists. This 10ms chunk is then sent
// to WebRTC and the remaining part is stored.
std::unique_ptr<FineAudioBuffer> fine_audio_buffer_;
// Temporary storage for recorded data. AudioUnitRender() renders into this
// array as soon as a frame of the desired buffer size has been recorded.
// On real iOS devices, the size will be fixed and set once. For iOS
// simulators, the size can vary from callback to callback and the size
// will be changed dynamically to account for this behavior.
rtc::BufferT<int16_t> record_audio_buffer_;
// Set to 1 when recording is active and 0 otherwise.
std::atomic<int> recording_;
// Set to 1 when playout is active and 0 otherwise.
std::atomic<int> playing_;
// Set to true after successful call to Init(), false otherwise.
bool initialized_ RTC_GUARDED_BY(thread_);
// Set to true after successful call to InitRecording() or InitPlayout(),
// false otherwise.
bool audio_is_initialized_;
// Set to true if audio session is interrupted, false otherwise.
bool is_interrupted_;
// Audio interruption observer instance.
RTCNativeAudioSessionDelegateAdapter* audio_session_observer_
RTC_GUARDED_BY(thread_);
// Set to true if we've activated the audio session.
bool has_configured_session_ RTC_GUARDED_BY(thread_);
// Counts number of detected audio glitches on the playout side.
int64_t num_detected_playout_glitches_ RTC_GUARDED_BY(thread_);
int64_t last_playout_time_ RTC_GUARDED_BY(io_thread_checker_);
// Counts number of playout callbacks per call.
// The value is updated on the native I/O thread and later read on the
// creating `thread_` but at this stage no audio is active.
// Hence, it is a "thread safe" design and no lock is needed.
int64_t num_playout_callbacks_;
// Contains the time for when the last output volume change was detected.
int64_t last_output_volume_change_time_ RTC_GUARDED_BY(thread_);
// Avoids running pending task after `this` is Terminated.
webrtc::scoped_refptr<PendingTaskSafetyFlag> safety_ =
PendingTaskSafetyFlag::Create();
std::atomic<bool> _hasTone;
std::shared_ptr<tgcalls::CallAudioTone> _tone;
bool isBufferPlaying_ = false;
bool isBufferRecording_ = false;
bool isMicrophoneMuted_ = false;
};
} // namespace tgcalls_ios_adm
} // namespace webrtc
#endif // SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_IOS_H_

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,161 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef TGCALLS_AUDIO_AUDIO_DEVICE_MODULE_IOS_H_
#define TGCALLS_AUDIO_AUDIO_DEVICE_MODULE_IOS_H_
#include <memory>
#include "tgcalls_audio_device_ios.h"
#include "api/task_queue/task_queue_factory.h"
#include "modules/audio_device/audio_device_buffer.h"
#include "modules/audio_device/include/audio_device.h"
#include "rtc_base/checks.h"
#include "platform/darwin/iOS/CallAudioTone.h"
namespace webrtc {
class AudioDeviceGeneric;
namespace tgcalls_ios_adm {
class AudioDeviceModuleIOS : public AudioDeviceModule {
public:
int32_t AttachAudioBuffer();
explicit AudioDeviceModuleIOS(bool bypass_voice_processing, bool disable_recording, bool enableSystemMute, int numChannels);
~AudioDeviceModuleIOS() override;
// Retrieve the currently utilized audio layer
int32_t ActiveAudioLayer(AudioLayer* audioLayer) const override;
// Full-duplex transportation of PCM audio
int32_t RegisterAudioCallback(AudioTransport* audioCallback) override;
// Main initializaton and termination
int32_t Init() override;
int32_t Terminate() override;
bool Initialized() const override;
// Device enumeration
int16_t PlayoutDevices() override;
int16_t RecordingDevices() override;
int32_t PlayoutDeviceName(uint16_t index,
char name[kAdmMaxDeviceNameSize],
char guid[kAdmMaxGuidSize]) override;
int32_t RecordingDeviceName(uint16_t index,
char name[kAdmMaxDeviceNameSize],
char guid[kAdmMaxGuidSize]) override;
// Device selection
int32_t SetPlayoutDevice(uint16_t index) override;
int32_t SetPlayoutDevice(WindowsDeviceType device) override;
int32_t SetRecordingDevice(uint16_t index) override;
int32_t SetRecordingDevice(WindowsDeviceType device) override;
// Audio transport initialization
int32_t PlayoutIsAvailable(bool* available) override;
int32_t InitPlayout() override;
bool PlayoutIsInitialized() const override;
int32_t RecordingIsAvailable(bool* available) override;
int32_t InitRecording() override;
bool RecordingIsInitialized() const override;
// Audio transport control
int32_t InternalStartPlayout();
int32_t StartPlayout() override;
int32_t StopPlayout() override;
bool Playing() const override;
int32_t InternalStartRecording();
int32_t StartRecording() override;
int32_t StopRecording() override;
bool Recording() const override;
// Audio mixer initialization
int32_t InitSpeaker() override;
bool SpeakerIsInitialized() const override;
int32_t InitMicrophone() override;
bool MicrophoneIsInitialized() const override;
// Speaker volume controls
int32_t SpeakerVolumeIsAvailable(bool* available) override;
int32_t SetSpeakerVolume(uint32_t volume) override;
int32_t SpeakerVolume(uint32_t* volume) const override;
int32_t MaxSpeakerVolume(uint32_t* maxVolume) const override;
int32_t MinSpeakerVolume(uint32_t* minVolume) const override;
// Microphone volume controls
int32_t MicrophoneVolumeIsAvailable(bool* available) override;
int32_t SetMicrophoneVolume(uint32_t volume) override;
int32_t MicrophoneVolume(uint32_t* volume) const override;
int32_t MaxMicrophoneVolume(uint32_t* maxVolume) const override;
int32_t MinMicrophoneVolume(uint32_t* minVolume) const override;
// Speaker mute control
int32_t SpeakerMuteIsAvailable(bool* available) override;
int32_t SetSpeakerMute(bool enable) override;
int32_t SpeakerMute(bool* enabled) const override;
// Microphone mute control
int32_t MicrophoneMuteIsAvailable(bool* available) override;
int32_t SetMicrophoneMute(bool enable) override;
int32_t MicrophoneMute(bool* enabled) const override;
// Stereo support
int32_t StereoPlayoutIsAvailable(bool* available) const override;
int32_t SetStereoPlayout(bool enable) override;
int32_t StereoPlayout(bool* enabled) const override;
int32_t StereoRecordingIsAvailable(bool* available) const override;
int32_t SetStereoRecording(bool enable) override;
int32_t StereoRecording(bool* enabled) const override;
// Delay information and control
int32_t PlayoutDelay(uint16_t* delayMS) const override;
bool BuiltInAECIsAvailable() const override;
int32_t EnableBuiltInAEC(bool enable) override;
bool BuiltInAGCIsAvailable() const override;
int32_t EnableBuiltInAGC(bool enable) override;
bool BuiltInNSIsAvailable() const override;
int32_t EnableBuiltInNS(bool enable) override;
int32_t GetPlayoutUnderrunCount() const override;
void setTone(std::shared_ptr<tgcalls::CallAudioTone> tone);
#if defined(WEBRTC_IOS)
int GetPlayoutAudioParameters(AudioParameters* params) const override;
int GetRecordAudioParameters(AudioParameters* params) const override;
#endif // WEBRTC_IOS
public:
void (^mutedSpeechDetectionChanged)(bool);
private:
const bool bypass_voice_processing_;
const bool disable_recording_;
const bool enableSystemMute_;
const int numChannels_;
bool initialized_ = false;
bool internalIsPlaying_ = false;
bool audioBufferPlayoutStarted_ = false;
bool audioBufferRecordingStarted_ = false;
const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
std::unique_ptr<AudioDeviceIOS> audio_device_;
std::unique_ptr<AudioDeviceBuffer> audio_device_buffer_;
std::shared_ptr<tgcalls::CallAudioTone> pendingAudioTone_;
};
} // namespace tgcalls_ios_adm
} // namespace webrtc
#endif // SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_MODULE_IOS_H_

View file

@ -0,0 +1,756 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "tgcalls_audio_device_module_ios.h"
#include "api/task_queue/default_task_queue_factory.h"
#include "modules/audio_device/audio_device_config.h"
#include "modules/audio_device/audio_device_generic.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/ref_count.h"
#include "system_wrappers/include/metrics.h"
#include "api/make_ref_counted.h"
#if defined(WEBRTC_IOS)
#include "tgcalls_audio_device_ios.h"
#endif
#define CHECKinitialized_() \
{ \
if (!initialized_) { \
return -1; \
}; \
}
#define CHECKinitialized__BOOL() \
{ \
if (!initialized_) { \
return false; \
}; \
}
namespace webrtc {
webrtc::scoped_refptr<AudioDeviceModule> CreateAudioDeviceModule(bool bypass_voice_processing) {
return rtc::make_ref_counted<webrtc::tgcalls_ios_adm::AudioDeviceModuleIOS>(bypass_voice_processing, false, false, 1);
}
webrtc::scoped_refptr<AudioDeviceModule> AudioDeviceModule::Create(
AudioLayer audio_layer,
TaskQueueFactory* task_queue_factory) {
RTC_DLOG(LS_INFO) << __FUNCTION__;
return CreateAudioDeviceModule(false);
}
}
namespace webrtc {
namespace tgcalls_ios_adm {
AudioDeviceModuleIOS::AudioDeviceModuleIOS(bool bypass_voice_processing, bool disable_recording, bool enableSystemMute, int numChannels)
: bypass_voice_processing_(bypass_voice_processing),
disable_recording_(disable_recording),
enableSystemMute_(enableSystemMute),
numChannels_(numChannels),
task_queue_factory_(CreateDefaultTaskQueueFactory()) {
RTC_LOG(LS_INFO) << "current platform is IOS";
RTC_LOG(LS_INFO) << "iPhone Audio APIs will be utilized.";
}
int32_t AudioDeviceModuleIOS::AttachAudioBuffer() {
RTC_DLOG(LS_INFO) << __FUNCTION__;
audio_device_->AttachAudioBuffer(audio_device_buffer_.get());
return 0;
}
AudioDeviceModuleIOS::~AudioDeviceModuleIOS() {
RTC_DLOG(LS_INFO) << __FUNCTION__;
}
int32_t AudioDeviceModuleIOS::ActiveAudioLayer(AudioLayer* audioLayer) const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
AudioLayer activeAudio;
if (audio_device_->ActiveAudioLayer(activeAudio) == -1) {
return -1;
}
*audioLayer = activeAudio;
return 0;
}
int32_t AudioDeviceModuleIOS::Init() {
RTC_DLOG(LS_INFO) << __FUNCTION__;
if (initialized_)
return 0;
audio_device_buffer_.reset(new webrtc::AudioDeviceBuffer(task_queue_factory_.get()));
audio_device_.reset(new tgcalls_ios_adm::AudioDeviceIOS(bypass_voice_processing_, disable_recording_, enableSystemMute_, numChannels_));
RTC_CHECK(audio_device_);
if (audio_device_) {
audio_device_->mutedSpeechDetectionChanged = [mutedSpeechDetectionChanged copy];
}
this->AttachAudioBuffer();
AudioDeviceGeneric::InitStatus status = audio_device_->Init();
RTC_HISTOGRAM_ENUMERATION(
"WebRTC.Audio.InitializationResult", static_cast<int>(status),
static_cast<int>(AudioDeviceGeneric::InitStatus::NUM_STATUSES));
if (status != AudioDeviceGeneric::InitStatus::OK) {
RTC_LOG(LS_ERROR) << "Audio device initialization failed.";
return -1;
}
initialized_ = true;
if (pendingAudioTone_) {
audio_device_->setTone(pendingAudioTone_);
pendingAudioTone_ = nullptr;
}
return 0;
}
int32_t AudioDeviceModuleIOS::Terminate() {
RTC_DLOG(LS_INFO) << __FUNCTION__;
if (!initialized_)
return 0;
if (audio_device_->Terminate() == -1) {
return -1;
}
initialized_ = false;
return 0;
}
bool AudioDeviceModuleIOS::Initialized() const {
RTC_DLOG(LS_INFO) << __FUNCTION__ << ": " << initialized_;
return initialized_;
}
int32_t AudioDeviceModuleIOS::InitSpeaker() {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
return audio_device_->InitSpeaker();
}
int32_t AudioDeviceModuleIOS::InitMicrophone() {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
return audio_device_->InitMicrophone();
}
int32_t AudioDeviceModuleIOS::SpeakerVolumeIsAvailable(bool* available) {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
bool isAvailable = false;
if (audio_device_->SpeakerVolumeIsAvailable(isAvailable) == -1) {
return -1;
}
*available = isAvailable;
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
return 0;
}
int32_t AudioDeviceModuleIOS::SetSpeakerVolume(uint32_t volume) {
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << volume << ")";
CHECKinitialized_();
return audio_device_->SetSpeakerVolume(volume);
}
int32_t AudioDeviceModuleIOS::SpeakerVolume(uint32_t* volume) const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
uint32_t level = 0;
if (audio_device_->SpeakerVolume(level) == -1) {
return -1;
}
*volume = level;
RTC_DLOG(LS_INFO) << "output: " << *volume;
return 0;
}
bool AudioDeviceModuleIOS::SpeakerIsInitialized() const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized__BOOL();
bool isInitialized = audio_device_->SpeakerIsInitialized();
RTC_DLOG(LS_INFO) << "output: " << isInitialized;
return isInitialized;
}
bool AudioDeviceModuleIOS::MicrophoneIsInitialized() const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized__BOOL();
bool isInitialized = audio_device_->MicrophoneIsInitialized();
RTC_DLOG(LS_INFO) << "output: " << isInitialized;
return isInitialized;
}
int32_t AudioDeviceModuleIOS::MaxSpeakerVolume(uint32_t* maxVolume) const {
CHECKinitialized_();
uint32_t maxVol = 0;
if (audio_device_->MaxSpeakerVolume(maxVol) == -1) {
return -1;
}
*maxVolume = maxVol;
return 0;
}
int32_t AudioDeviceModuleIOS::MinSpeakerVolume(uint32_t* minVolume) const {
CHECKinitialized_();
uint32_t minVol = 0;
if (audio_device_->MinSpeakerVolume(minVol) == -1) {
return -1;
}
*minVolume = minVol;
return 0;
}
int32_t AudioDeviceModuleIOS::SpeakerMuteIsAvailable(bool* available) {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
bool isAvailable = false;
if (audio_device_->SpeakerMuteIsAvailable(isAvailable) == -1) {
return -1;
}
*available = isAvailable;
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
return 0;
}
int32_t AudioDeviceModuleIOS::SetSpeakerMute(bool enable) {
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")";
CHECKinitialized_();
return audio_device_->SetSpeakerMute(enable);
}
int32_t AudioDeviceModuleIOS::SpeakerMute(bool* enabled) const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
bool muted = false;
if (audio_device_->SpeakerMute(muted) == -1) {
return -1;
}
*enabled = muted;
RTC_DLOG(LS_INFO) << "output: " << muted;
return 0;
}
int32_t AudioDeviceModuleIOS::MicrophoneMuteIsAvailable(bool* available) {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
bool isAvailable = false;
if (audio_device_->MicrophoneMuteIsAvailable(isAvailable) == -1) {
return -1;
}
*available = isAvailable;
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
return 0;
}
int32_t AudioDeviceModuleIOS::SetMicrophoneMute(bool enable) {
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")";
CHECKinitialized_();
return (audio_device_->SetMicrophoneMute(enable));
}
int32_t AudioDeviceModuleIOS::MicrophoneMute(bool* enabled) const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
bool muted = false;
if (audio_device_->MicrophoneMute(muted) == -1) {
return -1;
}
*enabled = muted;
RTC_DLOG(LS_INFO) << "output: " << muted;
return 0;
}
int32_t AudioDeviceModuleIOS::MicrophoneVolumeIsAvailable(bool* available) {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
bool isAvailable = false;
if (audio_device_->MicrophoneVolumeIsAvailable(isAvailable) == -1) {
return -1;
}
*available = isAvailable;
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
return 0;
}
int32_t AudioDeviceModuleIOS::SetMicrophoneVolume(uint32_t volume) {
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << volume << ")";
CHECKinitialized_();
return (audio_device_->SetMicrophoneVolume(volume));
}
int32_t AudioDeviceModuleIOS::MicrophoneVolume(uint32_t* volume) const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
uint32_t level = 0;
if (audio_device_->MicrophoneVolume(level) == -1) {
return -1;
}
*volume = level;
RTC_DLOG(LS_INFO) << "output: " << *volume;
return 0;
}
int32_t AudioDeviceModuleIOS::StereoRecordingIsAvailable(
bool* available) const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
bool isAvailable = false;
if (audio_device_->StereoRecordingIsAvailable(isAvailable) == -1) {
return -1;
}
*available = isAvailable;
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
return 0;
}
int32_t AudioDeviceModuleIOS::SetStereoRecording(bool enable) {
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")";
CHECKinitialized_();
if (enable) {
RTC_LOG(LS_WARNING) << "recording in stereo is not supported";
}
return -1;
}
int32_t AudioDeviceModuleIOS::StereoRecording(bool* enabled) const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
bool stereo = false;
if (audio_device_->StereoRecording(stereo) == -1) {
return -1;
}
*enabled = stereo;
RTC_DLOG(LS_INFO) << "output: " << stereo;
return 0;
}
int32_t AudioDeviceModuleIOS::StereoPlayoutIsAvailable(bool* available) const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
bool isAvailable = false;
if (audio_device_->StereoPlayoutIsAvailable(isAvailable) == -1) {
return -1;
}
*available = isAvailable;
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
return 0;
}
int32_t AudioDeviceModuleIOS::SetStereoPlayout(bool enable) {
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")";
CHECKinitialized_();
if (audio_device_->PlayoutIsInitialized()) {
RTC_LOG(LS_ERROR) << "unable to set stereo mode while playing side is initialized";
return -1;
}
if (audio_device_->SetStereoPlayout(enable)) {
RTC_LOG(LS_WARNING) << "stereo playout is not supported";
return -1;
}
int8_t nChannels(1);
if (enable) {
nChannels = 2;
}
audio_device_buffer_.get()->SetPlayoutChannels(nChannels);
return 0;
}
int32_t AudioDeviceModuleIOS::StereoPlayout(bool* enabled) const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
bool stereo = false;
if (audio_device_->StereoPlayout(stereo) == -1) {
return -1;
}
*enabled = stereo;
RTC_DLOG(LS_INFO) << "output: " << stereo;
return 0;
}
int32_t AudioDeviceModuleIOS::PlayoutIsAvailable(bool* available) {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
bool isAvailable = false;
if (audio_device_->PlayoutIsAvailable(isAvailable) == -1) {
return -1;
}
*available = isAvailable;
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
return 0;
}
int32_t AudioDeviceModuleIOS::RecordingIsAvailable(bool* available) {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
bool isAvailable = false;
if (audio_device_->RecordingIsAvailable(isAvailable) == -1) {
return -1;
}
*available = isAvailable;
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
return 0;
}
int32_t AudioDeviceModuleIOS::MaxMicrophoneVolume(uint32_t* maxVolume) const {
CHECKinitialized_();
uint32_t maxVol(0);
if (audio_device_->MaxMicrophoneVolume(maxVol) == -1) {
return -1;
}
*maxVolume = maxVol;
return 0;
}
int32_t AudioDeviceModuleIOS::MinMicrophoneVolume(uint32_t* minVolume) const {
CHECKinitialized_();
uint32_t minVol(0);
if (audio_device_->MinMicrophoneVolume(minVol) == -1) {
return -1;
}
*minVolume = minVol;
return 0;
}
int16_t AudioDeviceModuleIOS::PlayoutDevices() {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
uint16_t nPlayoutDevices = audio_device_->PlayoutDevices();
RTC_DLOG(LS_INFO) << "output: " << nPlayoutDevices;
return (int16_t)(nPlayoutDevices);
}
int32_t AudioDeviceModuleIOS::SetPlayoutDevice(uint16_t index) {
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << index << ")";
CHECKinitialized_();
return audio_device_->SetPlayoutDevice(index);
}
int32_t AudioDeviceModuleIOS::SetPlayoutDevice(WindowsDeviceType device) {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
return audio_device_->SetPlayoutDevice(device);
}
int32_t AudioDeviceModuleIOS::PlayoutDeviceName(
uint16_t index,
char name[kAdmMaxDeviceNameSize],
char guid[kAdmMaxGuidSize]) {
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << index << ", ...)";
CHECKinitialized_();
if (name == NULL) {
return -1;
}
if (audio_device_->PlayoutDeviceName(index, name, guid) == -1) {
return -1;
}
if (name != NULL) {
RTC_DLOG(LS_INFO) << "output: name = " << name;
}
if (guid != NULL) {
RTC_DLOG(LS_INFO) << "output: guid = " << guid;
}
return 0;
}
int32_t AudioDeviceModuleIOS::RecordingDeviceName(
uint16_t index,
char name[kAdmMaxDeviceNameSize],
char guid[kAdmMaxGuidSize]) {
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << index << ", ...)";
CHECKinitialized_();
if (name == NULL) {
return -1;
}
if (audio_device_->RecordingDeviceName(index, name, guid) == -1) {
return -1;
}
if (name != NULL) {
RTC_DLOG(LS_INFO) << "output: name = " << name;
}
if (guid != NULL) {
RTC_DLOG(LS_INFO) << "output: guid = " << guid;
}
return 0;
}
int16_t AudioDeviceModuleIOS::RecordingDevices() {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
uint16_t nRecordingDevices = audio_device_->RecordingDevices();
RTC_DLOG(LS_INFO) << "output: " << nRecordingDevices;
return (int16_t)nRecordingDevices;
}
int32_t AudioDeviceModuleIOS::SetRecordingDevice(uint16_t index) {
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << index << ")";
CHECKinitialized_();
return audio_device_->SetRecordingDevice(index);
}
int32_t AudioDeviceModuleIOS::SetRecordingDevice(WindowsDeviceType device) {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
return audio_device_->SetRecordingDevice(device);
}
int32_t AudioDeviceModuleIOS::InitPlayout() {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
if (PlayoutIsInitialized()) {
return 0;
}
int32_t result = audio_device_->InitPlayout();
RTC_DLOG(LS_INFO) << "output: " << result;
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.InitPlayoutSuccess",
static_cast<int>(result == 0));
return result;
}
int32_t AudioDeviceModuleIOS::InitRecording() {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
if (RecordingIsInitialized()) {
return 0;
}
int32_t result = audio_device_->InitRecording();
RTC_DLOG(LS_INFO) << "output: " << result;
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.InitRecordingSuccess",
static_cast<int>(result == 0));
return result;
}
bool AudioDeviceModuleIOS::PlayoutIsInitialized() const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized__BOOL();
return audio_device_->PlayoutIsInitialized();
}
bool AudioDeviceModuleIOS::RecordingIsInitialized() const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized__BOOL();
return audio_device_->RecordingIsInitialized();
}
int32_t AudioDeviceModuleIOS::InternalStartPlayout() {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
if (audio_device_->Playing()) {
return 0;
}
int32_t result = audio_device_->StartPlayout();
RTC_DLOG(LS_INFO) << "output: " << result;
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StartPlayoutSuccess",
static_cast<int>(result == 0));
return result;
}
int32_t AudioDeviceModuleIOS::StartPlayout() {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
if (audio_device_->Playing()) {
if (!audioBufferPlayoutStarted_) {
audioBufferPlayoutStarted_ = true;
audio_device_buffer_.get()->StartPlayout();
audio_device_->setIsBufferPlaying(true);
}
internalIsPlaying_ = true;
return 0;
}
audio_device_buffer_.get()->StartPlayout();
audio_device_->setIsBufferPlaying(true);
audioBufferPlayoutStarted_ = true;
int32_t result = audio_device_->StartPlayout();
RTC_DLOG(LS_INFO) << "output: " << result;
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StartPlayoutSuccess",
static_cast<int>(result == 0));
internalIsPlaying_ = true;
return result;
}
int32_t AudioDeviceModuleIOS::StopPlayout() {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
int32_t result = audio_device_->StopPlayout();
if (audioBufferPlayoutStarted_) {
audio_device_buffer_.get()->StopPlayout();
audioBufferPlayoutStarted_ = false;
audio_device_->setIsBufferPlaying(false);
}
RTC_DLOG(LS_INFO) << "output: " << result;
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StopPlayoutSuccess",
static_cast<int>(result == 0));
return result;
}
bool AudioDeviceModuleIOS::Playing() const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized__BOOL();
return internalIsPlaying_;
}
int32_t AudioDeviceModuleIOS::InternalStartRecording() {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
if (Recording()) {
return 0;
}
int32_t result = audio_device_->StartRecording();
RTC_DLOG(LS_INFO) << "output: " << result;
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StartRecordingSuccess",
static_cast<int>(result == 0));
return result;
}
int32_t AudioDeviceModuleIOS::StartRecording() {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
if (Recording()) {
if (!audioBufferRecordingStarted_) {
audioBufferRecordingStarted_ = true;
audio_device_buffer_.get()->StartRecording();
audio_device_->setIsBufferRecording(true);
}
return 0;
}
audio_device_buffer_.get()->StartRecording();
audioBufferRecordingStarted_ = true;
audio_device_->setIsBufferRecording(true);
int32_t result = audio_device_->StartRecording();
RTC_DLOG(LS_INFO) << "output: " << result;
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StartRecordingSuccess",
static_cast<int>(result == 0));
return result;
}
int32_t AudioDeviceModuleIOS::StopRecording() {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized_();
int32_t result = audio_device_->StopRecording();
if (audioBufferRecordingStarted_) {
audioBufferRecordingStarted_ = false;
audio_device_buffer_.get()->StopRecording();
audio_device_->setIsBufferRecording(false);
}
RTC_DLOG(LS_INFO) << "output: " << result;
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StopRecordingSuccess",
static_cast<int>(result == 0));
return result;
}
bool AudioDeviceModuleIOS::Recording() const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized__BOOL();
return audio_device_->Recording();
}
int32_t AudioDeviceModuleIOS::RegisterAudioCallback(
AudioTransport* audioCallback) {
RTC_DLOG(LS_INFO) << __FUNCTION__;
return audio_device_buffer_.get()->RegisterAudioCallback(audioCallback);
}
int32_t AudioDeviceModuleIOS::PlayoutDelay(uint16_t* delayMS) const {
CHECKinitialized_();
uint16_t delay = 0;
if (audio_device_->PlayoutDelay(delay) == -1) {
RTC_LOG(LS_ERROR) << "failed to retrieve the playout delay";
return -1;
}
*delayMS = delay;
return 0;
}
bool AudioDeviceModuleIOS::BuiltInAECIsAvailable() const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized__BOOL();
bool isAvailable = audio_device_->BuiltInAECIsAvailable();
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
return isAvailable;
}
int32_t AudioDeviceModuleIOS::EnableBuiltInAEC(bool enable) {
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")";
CHECKinitialized_();
int32_t ok = audio_device_->EnableBuiltInAEC(enable);
RTC_DLOG(LS_INFO) << "output: " << ok;
return ok;
}
bool AudioDeviceModuleIOS::BuiltInAGCIsAvailable() const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized__BOOL();
bool isAvailable = audio_device_->BuiltInAGCIsAvailable();
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
return isAvailable;
}
int32_t AudioDeviceModuleIOS::EnableBuiltInAGC(bool enable) {
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")";
CHECKinitialized_();
int32_t ok = audio_device_->EnableBuiltInAGC(enable);
RTC_DLOG(LS_INFO) << "output: " << ok;
return ok;
}
bool AudioDeviceModuleIOS::BuiltInNSIsAvailable() const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
CHECKinitialized__BOOL();
bool isAvailable = audio_device_->BuiltInNSIsAvailable();
RTC_DLOG(LS_INFO) << "output: " << isAvailable;
return isAvailable;
}
int32_t AudioDeviceModuleIOS::EnableBuiltInNS(bool enable) {
RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")";
CHECKinitialized_();
int32_t ok = audio_device_->EnableBuiltInNS(enable);
RTC_DLOG(LS_INFO) << "output: " << ok;
return ok;
}
int32_t AudioDeviceModuleIOS::GetPlayoutUnderrunCount() const {
// Don't log here, as this method can be called very often.
CHECKinitialized_();
int32_t ok = audio_device_->GetPlayoutUnderrunCount();
return ok;
}
void AudioDeviceModuleIOS::setTone(std::shared_ptr<tgcalls::CallAudioTone> tone) {
if (audio_device_) {
audio_device_->setTone(tone);
} else {
pendingAudioTone_ = tone;
}
}
#if defined(WEBRTC_IOS)
int AudioDeviceModuleIOS::GetPlayoutAudioParameters(
AudioParameters* params) const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
int r = audio_device_->GetPlayoutAudioParameters(params);
RTC_DLOG(LS_INFO) << "output: " << r;
return r;
}
int AudioDeviceModuleIOS::GetRecordAudioParameters(
AudioParameters* params) const {
RTC_DLOG(LS_INFO) << __FUNCTION__;
int r = audio_device_->GetRecordAudioParameters(params);
RTC_DLOG(LS_INFO) << "output: " << r;
return r;
}
#endif // WEBRTC_IOS
}
}

View file

@ -0,0 +1,149 @@
/*
* Copyright 2016 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef TGCALLS_AUDIO_VOICE_PROCESSING_AUDIO_UNIT_H_
#define TGCALLS_AUDIO_VOICE_PROCESSING_AUDIO_UNIT_H_
#include <AudioUnit/AudioUnit.h>
namespace webrtc {
namespace tgcalls_ios_adm {
class VoiceProcessingAudioUnitObserver {
public:
// Callback function called on a real-time priority I/O thread from the audio
// unit. This method is used to signal that recorded audio is available.
virtual OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags,
const AudioTimeStamp* time_stamp,
UInt32 bus_number,
UInt32 num_frames,
AudioBufferList* io_data) = 0;
// Callback function called on a real-time priority I/O thread from the audio
// unit. This method is used to provide audio samples to the audio unit.
virtual OSStatus OnGetPlayoutData(AudioUnitRenderActionFlags* io_action_flags,
const AudioTimeStamp* time_stamp,
UInt32 bus_number,
UInt32 num_frames,
AudioBufferList* io_data) = 0;
virtual void OnMutedSpeechStatusChanged(bool isDetectingSpeech) {}
protected:
~VoiceProcessingAudioUnitObserver() {}
};
// Convenience class to abstract away the management of a Voice Processing
// I/O Audio Unit. The Voice Processing I/O unit has the same characteristics
// as the Remote I/O unit (supports full duplex low-latency audio input and
// output) and adds AEC for for two-way duplex communication. It also adds AGC,
// adjustment of voice-processing quality, and muting. Hence, ideal for
// VoIP applications.
class VoiceProcessingAudioUnit {
public:
VoiceProcessingAudioUnit(bool bypass_voice_processing, bool disable_recording, bool enableSystemMute, int numChannels,
VoiceProcessingAudioUnitObserver* observer);
~VoiceProcessingAudioUnit();
// TODO(tkchin): enum for state and state checking.
enum State : int32_t {
// Init() should be called.
kInitRequired,
// Audio unit created but not initialized.
kUninitialized,
// Initialized but not started. Equivalent to stopped.
kInitialized,
// Initialized and started.
kStarted,
};
// Number of bytes per audio sample for 16-bit signed integer representation.
static const UInt32 kBytesPerSample;
// Initializes this class by creating the underlying audio unit instance.
// Creates a Voice-Processing I/O unit and configures it for full-duplex
// audio. The selected stream format is selected to avoid internal resampling
// and to match the 10ms callback rate for WebRTC as well as possible.
// Does not intialize the audio unit.
bool Init();
VoiceProcessingAudioUnit::State GetState() const;
// Initializes the underlying audio unit with the given sample rate.
bool Initialize(Float64 sample_rate);
// Starts the underlying audio unit.
OSStatus Start();
// Stops the underlying audio unit.
bool Stop();
// Uninitializes the underlying audio unit.
bool Uninitialize();
void setIsMicrophoneMuted(bool isMuted);
// Calls render on the underlying audio unit.
OSStatus Render(AudioUnitRenderActionFlags* flags,
const AudioTimeStamp* time_stamp,
UInt32 output_bus_number,
UInt32 num_frames,
AudioBufferList* io_data);
private:
// The C API used to set callbacks requires static functions. When these are
// called, they will invoke the relevant instance method by casting
// in_ref_con to VoiceProcessingAudioUnit*.
static OSStatus OnGetPlayoutData(void* in_ref_con,
AudioUnitRenderActionFlags* flags,
const AudioTimeStamp* time_stamp,
UInt32 bus_number,
UInt32 num_frames,
AudioBufferList* io_data);
static OSStatus OnDeliverRecordedData(void* in_ref_con,
AudioUnitRenderActionFlags* flags,
const AudioTimeStamp* time_stamp,
UInt32 bus_number,
UInt32 num_frames,
AudioBufferList* io_data);
// Notifies observer that samples are needed for playback.
OSStatus NotifyGetPlayoutData(AudioUnitRenderActionFlags* flags,
const AudioTimeStamp* time_stamp,
UInt32 bus_number,
UInt32 num_frames,
AudioBufferList* io_data);
// Notifies observer that recorded samples are available for render.
OSStatus NotifyDeliverRecordedData(AudioUnitRenderActionFlags* flags,
const AudioTimeStamp* time_stamp,
UInt32 bus_number,
UInt32 num_frames,
AudioBufferList* io_data);
// Returns the predetermined format with a specific sample rate. See
// implementation file for details on format.
AudioStreamBasicDescription GetFormat(Float64 sample_rate, int numChannels) const;
// Deletes the underlying audio unit.
void DisposeAudioUnit();
const bool bypass_voice_processing_;
const bool disable_recording_;
const int numChannels_;
bool enableSystemMute_ = false;
bool isMicrophoneMuted_ = true;
VoiceProcessingAudioUnitObserver* observer_;
AudioUnit vpio_unit_;
VoiceProcessingAudioUnit::State state_;
};
} // namespace tgcalls_ios_adm
} // namespace webrtc
#endif // SDK_OBJC_NATIVE_SRC_AUDIO_VOICE_PROCESSING_AUDIO_UNIT_H_

View file

@ -0,0 +1,544 @@
/*
* Copyright 2016 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import "tgcalls_voice_processing_audio_unit.h"
#include "rtc_base/checks.h"
#include "system_wrappers/include/metrics.h"
#import "base/RTCLogging.h"
#import "platform/darwin/iOS/RTCAudioSessionConfiguration.h"
#if !defined(NDEBUG)
static void LogStreamDescription(AudioStreamBasicDescription description) {
char formatIdString[5];
UInt32 formatId = CFSwapInt32HostToBig(description.mFormatID);
bcopy(&formatId, formatIdString, 4);
formatIdString[4] = '\0';
RTCLog(@"AudioStreamBasicDescription: {\n"
" mSampleRate: %.2f\n"
" formatIDString: %s\n"
" mFormatFlags: 0x%X\n"
" mBytesPerPacket: %u\n"
" mFramesPerPacket: %u\n"
" mBytesPerFrame: %u\n"
" mChannelsPerFrame: %u\n"
" mBitsPerChannel: %u\n"
" mReserved: %u\n}",
description.mSampleRate, formatIdString,
static_cast<unsigned int>(description.mFormatFlags),
static_cast<unsigned int>(description.mBytesPerPacket),
static_cast<unsigned int>(description.mFramesPerPacket),
static_cast<unsigned int>(description.mBytesPerFrame),
static_cast<unsigned int>(description.mChannelsPerFrame),
static_cast<unsigned int>(description.mBitsPerChannel),
static_cast<unsigned int>(description.mReserved));
}
#endif
namespace webrtc {
namespace tgcalls_ios_adm {
// Calls to AudioUnitInitialize() can fail if called back-to-back on different
// ADM instances. A fall-back solution is to allow multiple sequential calls
// with as small delay between each. This factor sets the max number of allowed
// initialization attempts.
static const int kMaxNumberOfAudioUnitInitializeAttempts = 5;
// A VP I/O unit's bus 1 connects to input hardware (microphone).
static const AudioUnitElement kInputBus = 1;
// A VP I/O unit's bus 0 connects to output hardware (speaker).
static const AudioUnitElement kOutputBus = 0;
// Returns the automatic gain control (AGC) state on the processed microphone
// signal. Should be on by default for Voice Processing audio units.
static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) {
RTC_DCHECK(audio_unit);
UInt32 size = sizeof(*enabled);
OSStatus result = AudioUnitGetProperty(audio_unit,
kAUVoiceIOProperty_VoiceProcessingEnableAGC,
kAudioUnitScope_Global,
kInputBus,
enabled,
&size);
RTCLog(@"VPIO unit AGC: %u", static_cast<unsigned int>(*enabled));
return result;
}
VoiceProcessingAudioUnit::VoiceProcessingAudioUnit(bool bypass_voice_processing, bool disable_recording, bool enableSystemMute, int numChannels,
VoiceProcessingAudioUnitObserver* observer)
: bypass_voice_processing_(bypass_voice_processing),
disable_recording_(disable_recording),
numChannels_(numChannels),
enableSystemMute_(enableSystemMute),
observer_(observer),
vpio_unit_(nullptr),
state_(kInitRequired) {
RTC_DCHECK(observer);
}
VoiceProcessingAudioUnit::~VoiceProcessingAudioUnit() {
DisposeAudioUnit();
}
const UInt32 VoiceProcessingAudioUnit::kBytesPerSample = 2;
bool VoiceProcessingAudioUnit::Init() {
RTC_DCHECK_EQ(state_, kInitRequired);
// Create an audio component description to identify the Voice Processing
// I/O audio unit.
AudioComponentDescription vpio_unit_description;
vpio_unit_description.componentType = kAudioUnitType_Output;
if (disable_recording_) {
vpio_unit_description.componentSubType = kAudioUnitSubType_RemoteIO;
} else {
vpio_unit_description.componentSubType = kAudioUnitSubType_VoiceProcessingIO;
}
vpio_unit_description.componentManufacturer = kAudioUnitManufacturer_Apple;
vpio_unit_description.componentFlags = 0;
vpio_unit_description.componentFlagsMask = 0;
// Obtain an audio unit instance given the description.
AudioComponent found_vpio_unit_ref =
AudioComponentFindNext(nullptr, &vpio_unit_description);
// Create a Voice Processing IO audio unit.
OSStatus result = noErr;
result = AudioComponentInstanceNew(found_vpio_unit_ref, &vpio_unit_);
if (result != noErr) {
vpio_unit_ = nullptr;
RTCLogError(@"AudioComponentInstanceNew failed. Error=%ld.", (long)result);
return false;
}
if (!disable_recording_) {
// Enable input on the input scope of the input element.
UInt32 enable_input = 1;
result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO,
kAudioUnitScope_Input, kInputBus, &enable_input,
sizeof(enable_input));
if (result != noErr) {
DisposeAudioUnit();
RTCLogError(@"Failed to enable input on input scope of input element. "
"Error=%ld.",
(long)result);
return false;
}
}
// Enable output on the output scope of the output element.
UInt32 enable_output = 1;
result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO,
kAudioUnitScope_Output, kOutputBus,
&enable_output, sizeof(enable_output));
if (result != noErr) {
DisposeAudioUnit();
RTCLogError(@"Failed to enable output on output scope of output element. "
"Error=%ld.",
(long)result);
return false;
}
// Specify the callback function that provides audio samples to the audio
// unit.
AURenderCallbackStruct render_callback;
render_callback.inputProc = OnGetPlayoutData;
render_callback.inputProcRefCon = this;
result = AudioUnitSetProperty(
vpio_unit_, kAudioUnitProperty_SetRenderCallback, kAudioUnitScope_Input,
kOutputBus, &render_callback, sizeof(render_callback));
if (result != noErr) {
DisposeAudioUnit();
RTCLogError(@"Failed to specify the render callback on the output bus. "
"Error=%ld.",
(long)result);
return false;
}
if (!disable_recording_) {
// Disable AU buffer allocation for the recorder, we allocate our own.
// TODO(henrika): not sure that it actually saves resource to make this call.
UInt32 flag = 0;
result = AudioUnitSetProperty(
vpio_unit_, kAudioUnitProperty_ShouldAllocateBuffer,
kAudioUnitScope_Output, kInputBus, &flag, sizeof(flag));
if (result != noErr) {
DisposeAudioUnit();
RTCLogError(@"Failed to disable buffer allocation on the input bus. "
"Error=%ld.",
(long)result);
return false;
}
// Specify the callback to be called by the I/O thread to us when input audio
// is available. The recorded samples can then be obtained by calling the
// AudioUnitRender() method.
AURenderCallbackStruct input_callback;
input_callback.inputProc = OnDeliverRecordedData;
input_callback.inputProcRefCon = this;
result = AudioUnitSetProperty(vpio_unit_,
kAudioOutputUnitProperty_SetInputCallback,
kAudioUnitScope_Global, kInputBus,
&input_callback, sizeof(input_callback));
if (result != noErr) {
DisposeAudioUnit();
RTCLogError(@"Failed to specify the input callback on the input bus. "
"Error=%ld.",
(long)result);
return false;
}
if (@available(iOS 15.0, macOS 14.0, *)) {
VoiceProcessingAudioUnit *audio_unit = this;
AUVoiceIOMutedSpeechActivityEventListener listener = ^(AUVoiceIOSpeechActivityEvent event) {
if (event == kAUVoiceIOSpeechActivityHasStarted) {
if (audio_unit->observer_) {
audio_unit->observer_->OnMutedSpeechStatusChanged(true);
}
} else if (event == kAUVoiceIOSpeechActivityHasEnded) {
if (audio_unit->observer_) {
audio_unit->observer_->OnMutedSpeechStatusChanged(false);
}
}
};
result = AudioUnitSetProperty(vpio_unit_, kAUVoiceIOProperty_MutedSpeechActivityEventListener, kAudioUnitScope_Global, kInputBus, &listener, sizeof(AUVoiceIOMutedSpeechActivityEventListener));
}
}
if (enableSystemMute_) {
UInt32 muteUplinkOutput = isMicrophoneMuted_ ? 1 : 0;
OSStatus result = AudioUnitSetProperty(vpio_unit_, kAUVoiceIOProperty_MuteOutput, kAudioUnitScope_Global, kInputBus, &muteUplinkOutput, sizeof(muteUplinkOutput));
if (result != noErr) {
RTCLogError(@"Failed to set kAUVoiceIOProperty_MuteOutput. Error=%ld", (long)result);
}
}
state_ = kUninitialized;
return true;
}
VoiceProcessingAudioUnit::State VoiceProcessingAudioUnit::GetState() const {
return state_;
}
bool VoiceProcessingAudioUnit::Initialize(Float64 sample_rate) {
RTC_DCHECK_GE(state_, kUninitialized);
RTCLog(@"Initializing audio unit with sample rate: %f", sample_rate);
OSStatus result = noErr;
AudioStreamBasicDescription outputFormat = GetFormat(sample_rate, numChannels_);
AudioStreamBasicDescription inputFormat = GetFormat(sample_rate, 1);
UInt32 size = sizeof(outputFormat);
#if !defined(NDEBUG)
LogStreamDescription(outputFormat);
#endif
if (!disable_recording_) {
// Set the format on the output scope of the input element/bus.
result =
AudioUnitSetProperty(vpio_unit_, kAudioUnitProperty_StreamFormat,
kAudioUnitScope_Output, kInputBus, &inputFormat, size);
if (result != noErr) {
RTCLogError(@"Failed to set format on output scope of input bus. "
"Error=%ld.",
(long)result);
return false;
}
}
// Set the format on the input scope of the output element/bus.
result =
AudioUnitSetProperty(vpio_unit_, kAudioUnitProperty_StreamFormat,
kAudioUnitScope_Input, kOutputBus, &outputFormat, size);
if (result != noErr) {
RTCLogError(@"Failed to set format on input scope of output bus. "
"Error=%ld.",
(long)result);
return false;
}
// Initialize the Voice Processing I/O unit instance.
// Calls to AudioUnitInitialize() can fail if called back-to-back on
// different ADM instances. The error message in this case is -66635 which is
// undocumented. Tests have shown that calling AudioUnitInitialize a second
// time, after a short sleep, avoids this issue.
// See webrtc:5166 for details.
int failed_initalize_attempts = 0;
result = AudioUnitInitialize(vpio_unit_);
while (result != noErr) {
RTCLogError(@"Failed to initialize the Voice Processing I/O unit. "
"Error=%ld.",
(long)result);
++failed_initalize_attempts;
if (failed_initalize_attempts == kMaxNumberOfAudioUnitInitializeAttempts) {
// Max number of initialization attempts exceeded, hence abort.
RTCLogError(@"Too many initialization attempts.");
return false;
}
RTCLog(@"Pause 100ms and try audio unit initialization again...");
[NSThread sleepForTimeInterval:0.1f];
result = AudioUnitInitialize(vpio_unit_);
}
if (result == noErr) {
RTCLog(@"Voice Processing I/O unit is now initialized.");
}
if (bypass_voice_processing_) {
// Attempt to disable builtin voice processing.
UInt32 toggle = 1;
result = AudioUnitSetProperty(vpio_unit_,
kAUVoiceIOProperty_BypassVoiceProcessing,
kAudioUnitScope_Global,
kInputBus,
&toggle,
sizeof(toggle));
if (result == noErr) {
RTCLog(@"Successfully bypassed voice processing.");
} else {
RTCLogError(@"Failed to bypass voice processing. Error=%ld.", (long)result);
}
state_ = kInitialized;
return true;
}
if (!disable_recording_) {
// AGC should be enabled by default for Voice Processing I/O units but it is
// checked below and enabled explicitly if needed. This scheme is used
// to be absolutely sure that the AGC is enabled since we have seen cases
// where only zeros are recorded and a disabled AGC could be one of the
// reasons why it happens.
int agc_was_enabled_by_default = 0;
UInt32 agc_is_enabled = 0;
result = GetAGCState(vpio_unit_, &agc_is_enabled);
if (result != noErr) {
RTCLogError(@"Failed to get AGC state (1st attempt). "
"Error=%ld.",
(long)result);
// Example of error code: kAudioUnitErr_NoConnection (-10876).
// All error codes related to audio units are negative and are therefore
// converted into a postive value to match the UMA APIs.
RTC_HISTOGRAM_COUNTS_SPARSE_100000(
"WebRTC.Audio.GetAGCStateErrorCode1", (-1) * result);
}
if (agc_is_enabled) {
// Remember that the AGC was enabled by default. Will be used in UMA.
agc_was_enabled_by_default = 1;
} else {
// AGC was initially disabled => try to enable it explicitly.
UInt32 enable_agc = 1;
result =
AudioUnitSetProperty(vpio_unit_,
kAUVoiceIOProperty_VoiceProcessingEnableAGC,
kAudioUnitScope_Global, kInputBus, &enable_agc,
sizeof(enable_agc));
if (result != noErr) {
RTCLogError(@"Failed to enable the built-in AGC. "
"Error=%ld.",
(long)result);
RTC_HISTOGRAM_COUNTS_SPARSE_100000(
"WebRTC.Audio.SetAGCStateErrorCode", (-1) * result);
}
result = GetAGCState(vpio_unit_, &agc_is_enabled);
if (result != noErr) {
RTCLogError(@"Failed to get AGC state (2nd attempt). "
"Error=%ld.",
(long)result);
RTC_HISTOGRAM_COUNTS_SPARSE_100000(
"WebRTC.Audio.GetAGCStateErrorCode2", (-1) * result);
}
}
// Track if the built-in AGC was enabled by default (as it should) or not.
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.BuiltInAGCWasEnabledByDefault",
agc_was_enabled_by_default);
RTCLog(@"WebRTC.Audio.BuiltInAGCWasEnabledByDefault: %d",
agc_was_enabled_by_default);
// As a final step, add an UMA histogram for tracking the AGC state.
// At this stage, the AGC should be enabled, and if it is not, more work is
// needed to find out the root cause.
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.BuiltInAGCIsEnabled", agc_is_enabled);
RTCLog(@"WebRTC.Audio.BuiltInAGCIsEnabled: %u",
static_cast<unsigned int>(agc_is_enabled));
}
state_ = kInitialized;
return true;
}
OSStatus VoiceProcessingAudioUnit::Start() {
RTC_DCHECK_GE(state_, kUninitialized);
RTCLog(@"Starting audio unit.");
OSStatus result = AudioOutputUnitStart(vpio_unit_);
if (result != noErr) {
RTCLogError(@"Failed to start audio unit. Error=%ld", (long)result);
return result;
} else {
RTCLog(@"Started audio unit");
}
state_ = kStarted;
return noErr;
}
bool VoiceProcessingAudioUnit::Stop() {
RTC_DCHECK_GE(state_, kUninitialized);
RTCLog(@"Stopping audio unit.");
OSStatus result = AudioOutputUnitStop(vpio_unit_);
if (result != noErr) {
RTCLogError(@"Failed to stop audio unit. Error=%ld", (long)result);
return false;
} else {
RTCLog(@"Stopped audio unit");
}
state_ = kInitialized;
return true;
}
bool VoiceProcessingAudioUnit::Uninitialize() {
RTC_DCHECK_GE(state_, kUninitialized);
RTCLog(@"Unintializing audio unit.");
OSStatus result = AudioUnitUninitialize(vpio_unit_);
if (result != noErr) {
RTCLogError(@"Failed to uninitialize audio unit. Error=%ld", (long)result);
return false;
} else {
RTCLog(@"Uninitialized audio unit.");
}
state_ = kUninitialized;
return true;
}
void VoiceProcessingAudioUnit::setIsMicrophoneMuted(bool isMuted) {
isMicrophoneMuted_ = isMuted;
if (enableSystemMute_) {
if (vpio_unit_ && state_ == kStarted) {
UInt32 muteUplinkOutput = isMuted ? 1 : 0;
OSStatus result = AudioUnitSetProperty(vpio_unit_, kAUVoiceIOProperty_MuteOutput, kAudioUnitScope_Global, kInputBus, &muteUplinkOutput, sizeof(muteUplinkOutput));
if (result != noErr) {
RTCLogError(@"Failed to set kAUVoiceIOProperty_MuteOutput. Error=%ld", (long)result);
}
}
}
}
OSStatus VoiceProcessingAudioUnit::Render(AudioUnitRenderActionFlags* flags,
const AudioTimeStamp* time_stamp,
UInt32 output_bus_number,
UInt32 num_frames,
AudioBufferList* io_data) {
RTC_DCHECK(vpio_unit_) << "Init() not called.";
OSStatus result = AudioUnitRender(vpio_unit_, flags, time_stamp,
output_bus_number, num_frames, io_data);
if (result != noErr) {
RTCLogError(@"Failed to render audio unit. Error=%ld", (long)result);
}
return result;
}
OSStatus VoiceProcessingAudioUnit::OnGetPlayoutData(
void* in_ref_con,
AudioUnitRenderActionFlags* flags,
const AudioTimeStamp* time_stamp,
UInt32 bus_number,
UInt32 num_frames,
AudioBufferList* io_data) {
VoiceProcessingAudioUnit* audio_unit =
static_cast<VoiceProcessingAudioUnit*>(in_ref_con);
return audio_unit->NotifyGetPlayoutData(flags, time_stamp, bus_number,
num_frames, io_data);
}
OSStatus VoiceProcessingAudioUnit::OnDeliverRecordedData(
void* in_ref_con,
AudioUnitRenderActionFlags* flags,
const AudioTimeStamp* time_stamp,
UInt32 bus_number,
UInt32 num_frames,
AudioBufferList* io_data) {
VoiceProcessingAudioUnit* audio_unit =
static_cast<VoiceProcessingAudioUnit*>(in_ref_con);
return audio_unit->NotifyDeliverRecordedData(flags, time_stamp, bus_number,
num_frames, io_data);
}
OSStatus VoiceProcessingAudioUnit::NotifyGetPlayoutData(
AudioUnitRenderActionFlags* flags,
const AudioTimeStamp* time_stamp,
UInt32 bus_number,
UInt32 num_frames,
AudioBufferList* io_data) {
return observer_->OnGetPlayoutData(flags, time_stamp, bus_number, num_frames,
io_data);
}
OSStatus VoiceProcessingAudioUnit::NotifyDeliverRecordedData(
AudioUnitRenderActionFlags* flags,
const AudioTimeStamp* time_stamp,
UInt32 bus_number,
UInt32 num_frames,
AudioBufferList* io_data) {
return observer_->OnDeliverRecordedData(flags, time_stamp, bus_number,
num_frames, io_data);
}
AudioStreamBasicDescription VoiceProcessingAudioUnit::GetFormat(
Float64 sample_rate, int numChannels) const {
// Set the application formats for input and output:
// - use same format in both directions
// - avoid resampling in the I/O unit by using the hardware sample rate
// - linear PCM => noncompressed audio data format with one frame per packet
// - no need to specify interleaving since only mono is supported
AudioStreamBasicDescription format;
format.mSampleRate = sample_rate;
format.mFormatID = kAudioFormatLinearPCM;
format.mFormatFlags =
kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked | kAudioFormatFlagsNativeEndian;
format.mBytesPerPacket = kBytesPerSample * numChannels;
format.mFramesPerPacket = 1; // uncompressed.
format.mBytesPerFrame = kBytesPerSample * numChannels;
format.mChannelsPerFrame = numChannels;
format.mBitsPerChannel = 8 * kBytesPerSample;
return format;
}
void VoiceProcessingAudioUnit::DisposeAudioUnit() {
if (vpio_unit_) {
switch (state_) {
case kStarted:
Stop();
[[fallthrough]];
case kInitialized:
Uninitialize();
break;
case kUninitialized:
case kInitRequired:
break;
}
RTCLog(@"Disposing audio unit.");
OSStatus result = AudioComponentInstanceDispose(vpio_unit_);
if (result != noErr) {
RTCLogError(@"AudioComponentInstanceDispose failed. Error=%ld.",
(long)result);
}
vpio_unit_ = nullptr;
}
}
} // namespace tgcalls_ios_adm
} // namespace webrtc

View file

@ -0,0 +1,17 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import <Foundation/Foundation.h>
#import "TGRTCMTLRenderer.h"
NS_AVAILABLE(10_11, 9_0)
@interface TGRTCMTLI420Renderer : TGRTCMTLRenderer
@end

View file

@ -0,0 +1,111 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import "TGRTCMTLI420Renderer.h"
#import <Metal/Metal.h>
#import <MetalKit/MetalKit.h>
#import "base/RTCI420Buffer.h"
#import "base/RTCLogging.h"
#import "base/RTCVideoFrame.h"
#import "base/RTCVideoFrameBuffer.h"
#import "TGRTCMTLRenderer+Private.h"
@implementation TGRTCMTLI420Renderer {
// Textures.
id<MTLTexture> _yTexture;
id<MTLTexture> _uTexture;
id<MTLTexture> _vTexture;
MTLTextureDescriptor *_descriptor;
MTLTextureDescriptor *_chromaDescriptor;
int _width;
int _height;
int _chromaWidth;
int _chromaHeight;
}
- (void)getWidth:(nonnull int *)width
height:(nonnull int *)height
ofFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame {
*width = frame.width;
*height = frame.height;
}
- (BOOL)setupTexturesForFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame {
if (![super setupTexturesForFrame:frame]) {
return NO;
}
id<MTLDevice> device = [self currentMetalDevice];
if (!device) {
return NO;
}
id<RTC_OBJC_TYPE(RTCI420Buffer)> buffer = [frame.buffer toI420];
// Luma (y) texture.
if (!_descriptor || _width != frame.width || _height != frame.height) {
_width = frame.width;
_height = frame.height;
_descriptor = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatR8Unorm
width:_width
height:_height
mipmapped:NO];
_descriptor.usage = MTLTextureUsageShaderRead;
_yTexture = [device newTextureWithDescriptor:_descriptor];
}
// Chroma (u,v) textures
[_yTexture replaceRegion:MTLRegionMake2D(0, 0, _width, _height)
mipmapLevel:0
withBytes:buffer.dataY
bytesPerRow:buffer.strideY];
if (!_chromaDescriptor || _chromaWidth != frame.width / 2 || _chromaHeight != frame.height / 2) {
_chromaWidth = frame.width / 2;
_chromaHeight = frame.height / 2;
_chromaDescriptor =
[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatR8Unorm
width:_chromaWidth
height:_chromaHeight
mipmapped:NO];
_chromaDescriptor.usage = MTLTextureUsageShaderRead;
_uTexture = [device newTextureWithDescriptor:_chromaDescriptor];
_vTexture = [device newTextureWithDescriptor:_chromaDescriptor];
}
[_uTexture replaceRegion:MTLRegionMake2D(0, 0, _chromaWidth, _chromaHeight)
mipmapLevel:0
withBytes:buffer.dataU
bytesPerRow:buffer.strideU];
[_vTexture replaceRegion:MTLRegionMake2D(0, 0, _chromaWidth, _chromaHeight)
mipmapLevel:0
withBytes:buffer.dataV
bytesPerRow:buffer.strideV];
return (_uTexture != nil) && (_yTexture != nil) && (_vTexture != nil);
}
- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder {
[renderEncoder setFragmentTexture:_yTexture atIndex:0];
[renderEncoder setFragmentTexture:_uTexture atIndex:1];
[renderEncoder setFragmentTexture:_vTexture atIndex:2];
}
@end

View file

@ -0,0 +1,33 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import <Metal/Metal.h>
#import "TGRTCMTLRenderer.h"
#define MTL_STRINGIFY(s) @ #s
NS_ASSUME_NONNULL_BEGIN
@interface TGRTCMTLRenderer (Private)
- (nullable id<MTLDevice>)currentMetalDevice;
- (NSString *)shaderSource;
- (BOOL)setupTexturesForFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame;
- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder;
- (void)getWidth:(nonnull int *)width
height:(nonnull int *)height
cropWidth:(nonnull int *)cropWidth
cropHeight:(nonnull int *)cropHeight
cropX:(nonnull int *)cropX
cropY:(nonnull int *)cropY
ofFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame;
@end
NS_ASSUME_NONNULL_END

View file

@ -0,0 +1,49 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import <Foundation/Foundation.h>
#import <AppKit/AppKit.h>
#import <MetalKit/MetalKit.h>
#import "base/RTCVideoFrame.h"
bool initMetal();
struct MTLFrameSize {
float width = 0;
float height = 0;
};
MTLFrameSize MTLAspectFilled(MTLFrameSize from, MTLFrameSize to);
MTLFrameSize MTLAspectFitted(MTLFrameSize from, MTLFrameSize to);
NS_ASSUME_NONNULL_BEGIN
typedef enum {
TGRTCMTLRenderModeSinge,
TGRTCMTLRenderModeDouble
} TGRTCMTLRenderMode;
@protocol TGRTCMTLRenderer <NSObject>
- (void)drawFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame;
- (BOOL)setSingleRendering:(__kindof CAMetalLayer *)view;
- (BOOL)setDoubleRendering:(__kindof CAMetalLayer *)view foreground: (__kindof CAMetalLayer *)foreground;
@end
NS_AVAILABLE(10_11, 9_0)
@interface TGRTCMTLRenderer : NSObject <TGRTCMTLRenderer>
@property(atomic, nullable) NSValue *rotationOverride;
@end
NS_ASSUME_NONNULL_END

View file

@ -0,0 +1,641 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import "TGRTCMTLRenderer+Private.h"
#import <Metal/Metal.h>
#import <MetalKit/MetalKit.h>
#import <MetalPerformanceShaders/MetalPerformanceShaders.h>
#import "base/RTCLogging.h"
#import "base/RTCVideoFrame.h"
#import "base/RTCVideoFrameBuffer.h"
#import "TGRTCMetalContextHolder.h"
#include "api/video/video_rotation.h"
#include "rtc_base/checks.h"
MTLFrameSize MTLAspectFitted(MTLFrameSize from, MTLFrameSize to) {
double scale = std::min(
from.width / std::max(1., double(to.width)),
from.height / std::max(1., double(to.height)));
return {
float(std::ceil(to.width * scale)),
float(std::ceil(to.height * scale))
};
}
MTLFrameSize MTLAspectFilled(MTLFrameSize from, MTLFrameSize to) {
double scale = std::max(
to.width / std::max(1., double(from.width)),
to.height / std::max(1., double(from.height)));
return {
float(std::ceil(from.width * scale)),
float(std::ceil(from.height * scale))
};
}
static NSString *const pipelineDescriptorLabel = @"RTCPipeline";
static NSString *const commandBufferLabel = @"RTCCommandBuffer";
static NSString *const renderEncoderLabel = @"RTCEncoder";
static NSString *const renderEncoderDebugGroup = @"RTCDrawFrame";
static TGRTCMetalContextHolder *metalContext = nil;
bool initMetal() {
if (metalContext == nil) {
metalContext = [[TGRTCMetalContextHolder alloc] init];
} else if(metalContext.displayId != CGMainDisplayID()) {
metalContext = [[TGRTCMetalContextHolder alloc] init];
}
return metalContext != nil;
}
static inline void getCubeVertexData(size_t frameWidth,
size_t frameHeight,
RTCVideoRotation rotation,
float *buffer) {
// The computed values are the adjusted texture coordinates, in [0..1].
// For the left and top, 0.0 means no cropping and e.g. 0.2 means we're skipping 20% of the
// left/top edge.
// For the right and bottom, 1.0 means no cropping and e.g. 0.8 means we're skipping 20% of the
// right/bottom edge (i.e. render up to 80% of the width/height).
float cropLeft = 0;
float cropRight = 1;
float cropTop = 0;
float cropBottom = 1;
// These arrays map the view coordinates to texture coordinates, taking cropping and rotation
// into account. The first two columns are view coordinates, the last two are texture coordinates.
switch (rotation) {
case RTCVideoRotation_0: {
float values[16] = {-1.0, -1.0, cropLeft, cropBottom,
1.0, -1.0, cropRight, cropBottom,
-1.0, 1.0, cropLeft, cropTop,
1.0, 1.0, cropRight, cropTop};
memcpy(buffer, &values, sizeof(values));
} break;
case RTCVideoRotation_90: {
float values[16] = {-1.0, -1.0, cropRight, cropBottom,
1.0, -1.0, cropRight, cropTop,
-1.0, 1.0, cropLeft, cropBottom,
1.0, 1.0, cropLeft, cropTop};
memcpy(buffer, &values, sizeof(values));
} break;
case RTCVideoRotation_180: {
float values[16] = {-1.0, -1.0, cropRight, cropTop,
1.0, -1.0, cropLeft, cropTop,
-1.0, 1.0, cropRight, cropBottom,
1.0, 1.0, cropLeft, cropBottom};
memcpy(buffer, &values, sizeof(values));
} break;
case RTCVideoRotation_270: {
float values[16] = {-1.0, -1.0, cropLeft, cropTop,
1.0, -1.0, cropLeft, cropBottom,
-1.0, 1.0, cropRight, cropTop,
1.0, 1.0, cropRight, cropBottom};
memcpy(buffer, &values, sizeof(values));
} break;
}
}
@implementation TGRTCMTLRenderer {
__kindof CAMetalLayer *_view;
__kindof CAMetalLayer *_foreground;
TGRTCMetalContextHolder* _context;
id<MTLCommandQueue> _commandQueue;
id<MTLBuffer> _vertexBuffer;
id<MTLBuffer> _vertexBufferRotated;
MTLFrameSize _frameSize;
MTLFrameSize _scaledSize;
RTCVideoFrame *_frame;
bool _frameIsUpdated;
RTCVideoRotation _rotation;
bool _rotationInited;
id<MTLTexture> _rgbTexture;
id<MTLTexture> _rgbScaledAndBlurredTexture;
id<MTLBuffer> _vertexBuffer0;
id<MTLBuffer> _vertexBuffer1;
id<MTLBuffer> _vertexBuffer2;
dispatch_semaphore_t _inflight1;
dispatch_semaphore_t _inflight2;
}
@synthesize rotationOverride = _rotationOverride;
- (instancetype)init {
if (self = [super init]) {
_inflight1 = dispatch_semaphore_create(0);
_inflight2 = dispatch_semaphore_create(0);
_context = metalContext;
_commandQueue = [_context.device newCommandQueueWithMaxCommandBufferCount:3];
float vertexBufferArray[16] = {0};
_vertexBuffer = [metalContext.device newBufferWithBytes:vertexBufferArray
length:sizeof(vertexBufferArray)
options:MTLResourceCPUCacheModeWriteCombined];
float vertexBufferArrayRotated[16] = {0};
_vertexBufferRotated = [metalContext.device newBufferWithBytes:vertexBufferArrayRotated
length:sizeof(vertexBufferArrayRotated)
options:MTLResourceCPUCacheModeWriteCombined];
float verts[8] = {-1.0, 1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0};
_vertexBuffer0 = [metalContext.device newBufferWithBytes:verts length:sizeof(verts) options:0];
float values[8] = {0};
_vertexBuffer1 = [metalContext.device newBufferWithBytes:values
length:sizeof(values)
options:0];
_vertexBuffer2 = [metalContext.device newBufferWithBytes:values
length:sizeof(values)
options:0];
}
return self;
}
- (BOOL)setSingleRendering:(__kindof CAMetalLayer *)view {
return [self setupWithView:view foreground: nil];
}
- (BOOL)setDoubleRendering:(__kindof CAMetalLayer *)view foreground:(nonnull __kindof CAMetalLayer *)foreground {
return [self setupWithView:view foreground: foreground];
}
#pragma mark - Private
- (BOOL)setupWithView:(__kindof CAMetalLayer *)view foreground: (__kindof CAMetalLayer *)foreground {
_view = view;
_foreground = foreground;
view.device = metalContext.device;
foreground.device = metalContext.device;
_context = metalContext;
_rotationInited = false;
return YES;
}
#pragma mark - Inheritance
- (id<MTLDevice>)currentMetalDevice {
return metalContext.device;
}
- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder {
// RTC_NOTREACHED() << "Virtual method not implemented in subclass.";
}
- (void)getWidth:(int *)width
height:(int *)height
ofFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame {
// RTC_NOTREACHED() << "Virtual method not implemented in subclass.";
}
- (BOOL)setupTexturesForFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame {
RTCVideoRotation rotation;
NSValue *rotationOverride = self.rotationOverride;
if (rotationOverride) {
[rotationOverride getValue:&rotation];
} else {
rotation = frame.rotation;
}
_frameIsUpdated = true;//_frame.timeStampNs != frame.timeStampNs;
_frame = frame;
int frameWidth, frameHeight;
[self getWidth:&frameWidth
height:&frameHeight
ofFrame:frame];
if (frameWidth != _frameSize.width || frameHeight != _frameSize.height || _rotation != rotation || !_rotationInited) {
bool rotationIsUpdated = _rotation != rotation || !_rotationInited;
_rotation = rotation;
_frameSize.width = frameWidth;
_frameSize.height = frameHeight;
_frameIsUpdated = true;
MTLFrameSize small;
small.width = _frameSize.width / 4;
small.height = _frameSize.height / 4;
_scaledSize = MTLAspectFitted(small, _frameSize);
_rgbTexture = [self createTextureWithUsage: MTLTextureUsageShaderRead|MTLTextureUsageRenderTarget size:_frameSize];
_rgbScaledAndBlurredTexture = [self createTextureWithUsage:MTLTextureUsageShaderRead|MTLTextureUsageRenderTarget size:_scaledSize];
if (rotationIsUpdated) {
getCubeVertexData(frameWidth,
frameHeight,
RTCVideoRotation_0,
(float *)_vertexBuffer.contents);
getCubeVertexData(frameWidth,
frameHeight,
rotation,
(float *)_vertexBufferRotated.contents);
switch (rotation) {
case RTCVideoRotation_0: {
float values[8] = {0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0};
memcpy((float *)_vertexBuffer1.contents, &values, sizeof(values));
memcpy((float *)_vertexBuffer2.contents, &values, sizeof(values));
} break;
case RTCVideoRotation_90: {
float values[8] = {0.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0};
memcpy((float *)_vertexBuffer1.contents, &values, sizeof(values));
memcpy((float *)_vertexBuffer2.contents, &values, sizeof(values));
} break;
case RTCVideoRotation_180: {
//[xLimit, yLimit, 0.0, yLimit, xLimit, 0.0, 0.0, 0.0]
float values[8] = {1.0, 1.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0};
memcpy(_vertexBuffer1.contents, &values, sizeof(values));
memcpy(_vertexBuffer2.contents, &values, sizeof(values));
} break;
case RTCVideoRotation_270: {
float values[8] = {1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0};
memcpy(_vertexBuffer1.contents, &values, sizeof(values));
memcpy(_vertexBuffer2.contents, &values, sizeof(values));
} break;
}
_rotationInited = true;
}
}
return YES;
}
#pragma mark - GPU methods
- (id<MTLTexture>)createTextureWithUsage:(MTLTextureUsage) usage size:(MTLFrameSize)size {
MTLTextureDescriptor *rgbTextureDescriptor = [MTLTextureDescriptor
texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm
width:size.width
height:size.height
mipmapped:NO];
rgbTextureDescriptor.usage = usage;
return [metalContext.device newTextureWithDescriptor:rgbTextureDescriptor];
}
- (id<MTLRenderCommandEncoder>)createRenderEncoderForTarget: (id<MTLTexture>)texture with: (id<MTLCommandBuffer>)commandBuffer {
MTLRenderPassDescriptor *renderPassDescriptor = [[MTLRenderPassDescriptor alloc] init];
renderPassDescriptor.colorAttachments[0].texture = texture;
renderPassDescriptor.colorAttachments[0].loadAction = MTLLoadActionDontCare;
renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColorMake(0, 0, 0, 0);
id<MTLRenderCommandEncoder> renderEncoder = [commandBuffer renderCommandEncoderWithDescriptor:renderPassDescriptor];
renderEncoder.label = renderEncoderLabel;
return renderEncoder;
}
- (id<MTLTexture>)convertYUVtoRGV:(id<MTLBuffer>)buffer {
id<MTLTexture> rgbTexture = _rgbTexture;
if (_frameIsUpdated) {
id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
id<MTLRenderCommandEncoder> renderEncoder = [self createRenderEncoderForTarget: rgbTexture with: commandBuffer];
[renderEncoder pushDebugGroup:renderEncoderDebugGroup];
[renderEncoder setRenderPipelineState:_context.pipelineYuvRgb];
[renderEncoder setVertexBuffer:buffer offset:0 atIndex:0];
[self uploadTexturesToRenderEncoder:renderEncoder];
[renderEncoder setFragmentSamplerState:_context.sampler atIndex:0];
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip
vertexStart:0
vertexCount:4
instanceCount:1];
[renderEncoder popDebugGroup];
[renderEncoder endEncoding];
[commandBuffer commit];
}
return rgbTexture;
}
- (id<MTLTexture>)scaleAndBlur:(id<MTLTexture>)inputTexture scale:(simd_float2)scale {
id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
id<MTLRenderCommandEncoder> renderEncoder = [self createRenderEncoderForTarget: _rgbScaledAndBlurredTexture with: commandBuffer];
[renderEncoder pushDebugGroup:renderEncoderDebugGroup];
[renderEncoder setRenderPipelineState:_context.pipelineScaleAndBlur];
[renderEncoder setFragmentTexture:inputTexture atIndex:0];
[renderEncoder setVertexBuffer:_vertexBuffer offset:0 atIndex:0];
[renderEncoder setFragmentBytes:&scale length:sizeof(scale) atIndex:0];
[renderEncoder setFragmentSamplerState:_context.sampler atIndex:0];
bool vertical = true;
[renderEncoder setFragmentBytes:&vertical length:sizeof(vertical) atIndex:1];
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip
vertexStart:0
vertexCount:4
instanceCount:1];
[renderEncoder popDebugGroup];
[renderEncoder endEncoding];
[commandBuffer commit];
// [commandBuffer waitUntilCompleted];
return _rgbScaledAndBlurredTexture;
}
- (void)mergeYUVTexturesInTarget:(id<MTLTexture>)targetTexture foregroundTexture: (id<MTLTexture>)foregroundTexture backgroundTexture:(id<MTLTexture>)backgroundTexture scale1:(simd_float2)scale1 scale2:(simd_float2)scale2 {
id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
id<MTLRenderCommandEncoder> renderEncoder = [self createRenderEncoderForTarget: targetTexture with: commandBuffer];
[renderEncoder pushDebugGroup:renderEncoderDebugGroup];
[renderEncoder setRenderPipelineState:_context.pipelineTransformAndBlend];
[renderEncoder setVertexBuffer:_vertexBuffer0 offset:0 atIndex:0];
[renderEncoder setVertexBuffer:_vertexBuffer1 offset:0 atIndex:1];
[renderEncoder setVertexBuffer:_vertexBuffer2 offset:0 atIndex:2];
[renderEncoder setFragmentTexture:foregroundTexture atIndex:0];
[renderEncoder setFragmentTexture:backgroundTexture atIndex:1];
[renderEncoder setFragmentBytes:&scale1 length:sizeof(scale1) atIndex:0];
[renderEncoder setFragmentBytes:&scale2 length:sizeof(scale2) atIndex:1];
[renderEncoder setFragmentSamplerState:_context.sampler atIndex:0];
[renderEncoder setFragmentSamplerState:_context.sampler atIndex:1];
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip
vertexStart:0
vertexCount:4
instanceCount:1];
[renderEncoder popDebugGroup];
[renderEncoder endEncoding];
[commandBuffer commit];
// [commandBuffer waitUntilCompleted];
}
- (void)doubleRender {
id<CAMetalDrawable> background = _view.nextDrawable;
id<CAMetalDrawable> foreground = _foreground.nextDrawable;
_rgbTexture = [self convertYUVtoRGV:_vertexBufferRotated];
CGSize drawableSize = _view.drawableSize;
MTLFrameSize from;
MTLFrameSize to;
MTLFrameSize frameSize = _frameSize;
MTLFrameSize scaledSize = _scaledSize;
from.width = _view.bounds.size.width;
from.height = _view.bounds.size.height;
to.width = drawableSize.width;
to.height = drawableSize.height;
// bool swap = _rotation == RTCVideoRotation_90 || _rotation == RTCVideoRotation_270;
// if (swap) {
// frameSize.width = _frameSize.height;
// frameSize.height = _frameSize.width;
// scaledSize.width = _scaledSize.height;
// scaledSize.height = _scaledSize.width;
// }
_rgbScaledAndBlurredTexture = [self scaleAndBlur:_rgbTexture scale:simd_make_float2(frameSize.width / scaledSize.width, frameSize.height/ scaledSize.height)];
id<MTLCommandBuffer> commandBuffer_b = [_commandQueue commandBuffer];
{
id<MTLRenderCommandEncoder> renderEncoder = [self createRenderEncoderForTarget: background.texture with: commandBuffer_b];
[renderEncoder pushDebugGroup:renderEncoderDebugGroup];
[renderEncoder setRenderPipelineState:_context.pipelineScaleAndBlur];
[renderEncoder setFragmentTexture:_rgbScaledAndBlurredTexture atIndex:0];
[renderEncoder setVertexBuffer:_vertexBuffer offset:0 atIndex:0];
simd_float2 scale = simd_make_float2(scaledSize.width / frameSize.width, scaledSize.height / frameSize.height);
[renderEncoder setFragmentBytes:&scale length:sizeof(scale) atIndex:0];
bool vertical = false;
[renderEncoder setFragmentBytes:&vertical length:sizeof(vertical) atIndex:1];
[renderEncoder setFragmentSamplerState:_context.sampler atIndex:0];
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip
vertexStart:0
vertexCount:4
instanceCount:1];
[renderEncoder popDebugGroup];
[renderEncoder endEncoding];
}
id<MTLCommandBuffer> commandBuffer_f = [_commandQueue commandBuffer];
{
id<MTLRenderCommandEncoder> renderEncoder = [self createRenderEncoderForTarget: foreground.texture with: commandBuffer_f];
[renderEncoder pushDebugGroup:renderEncoderDebugGroup];
[renderEncoder setRenderPipelineState:_context.pipelineThrough];
[renderEncoder setFragmentTexture:_rgbTexture atIndex:0];
[renderEncoder setVertexBuffer:_vertexBuffer offset:0 atIndex:0];
[renderEncoder setFragmentSamplerState:_context.sampler atIndex:0];
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip
vertexStart:0
vertexCount:4
instanceCount:1];
[renderEncoder popDebugGroup];
[renderEncoder endEncoding];
}
dispatch_semaphore_t inflight = _inflight2;
[commandBuffer_f addCompletedHandler:^(id<MTLCommandBuffer> _Nonnull) {
dispatch_semaphore_signal(inflight);
}];
[commandBuffer_b addCompletedHandler:^(id<MTLCommandBuffer> _Nonnull) {
dispatch_semaphore_signal(inflight);
}];
[commandBuffer_b addScheduledHandler:^(id<MTLCommandBuffer> _Nonnull) {
[background present];
}];
[commandBuffer_f addScheduledHandler:^(id<MTLCommandBuffer> _Nonnull) {
[foreground present];
}];
[commandBuffer_f commit];
[commandBuffer_b commit];
dispatch_semaphore_wait(inflight, DISPATCH_TIME_FOREVER);
dispatch_semaphore_wait(inflight, DISPATCH_TIME_FOREVER);
}
- (void)singleRender {
id<CAMetalDrawable> drawable = _view.nextDrawable;
CGSize drawableSize = _view.drawableSize;
MTLFrameSize from;
MTLFrameSize to;
MTLFrameSize frameSize = _frameSize;
MTLFrameSize scaledSize = _scaledSize;
from.width = _view.bounds.size.width;
from.height = _view.bounds.size.height;
to.width = drawableSize.width;
to.height = drawableSize.height;
bool swap = _rotation == RTCVideoRotation_90 || _rotation == RTCVideoRotation_270;
if (swap) {
frameSize.width = _frameSize.height;
frameSize.height = _frameSize.width;
scaledSize.width = _scaledSize.height;
scaledSize.height = _scaledSize.width;
}
float ratio = (float)frameSize.height / (float)frameSize.width;
MTLFrameSize viewSize = MTLAspectFilled(to, from);
MTLFrameSize fitted = MTLAspectFitted(from, to);
CGSize viewPortSize = CGSizeMake(viewSize.width, viewSize.height);
id<MTLTexture> targetTexture = drawable.texture;
CGFloat heightAspectScale = viewPortSize.height / (fitted.width * ratio);
CGFloat widthAspectScale = viewPortSize.width / (fitted.height * (1.0/ratio));
_rgbTexture = [self convertYUVtoRGV:_vertexBuffer];
simd_float2 smallScale = simd_make_float2(frameSize.width / scaledSize.width, frameSize.height / scaledSize.height);
_rgbScaledAndBlurredTexture = [self scaleAndBlur:_rgbTexture scale:smallScale];
simd_float2 scale1 = simd_make_float2(MAX(1.0, widthAspectScale), MAX(1.0, heightAspectScale));
float bgRatio_w = scaledSize.width / frameSize.width;
float bgRatio_h = scaledSize.height / frameSize.height;
simd_float2 scale2 = simd_make_float2(MIN(bgRatio_w, widthAspectScale * bgRatio_w), MIN(bgRatio_h, heightAspectScale * bgRatio_h));
if (swap) {
scale1 = simd_make_float2(MAX(1.0, heightAspectScale), MAX(1.0, widthAspectScale));
scale2 = simd_make_float2(MIN(1, heightAspectScale * 1), MIN(bgRatio_h, widthAspectScale * bgRatio_h));
}
[self mergeYUVTexturesInTarget: targetTexture
foregroundTexture: _rgbTexture
backgroundTexture: _rgbScaledAndBlurredTexture
scale1:scale1
scale2:scale2];
id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
dispatch_semaphore_t inflight = _inflight1;
[commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> _Nonnull) {
dispatch_semaphore_signal(inflight);
}];
[commandBuffer addScheduledHandler:^(id<MTLCommandBuffer> _Nonnull) {
[drawable present];
}];
[commandBuffer commit];
dispatch_semaphore_wait(inflight, DISPATCH_TIME_FOREVER);
// [commandBuffer commit];
}
-(void)dealloc {
dispatch_semaphore_signal(_inflight1);
dispatch_semaphore_signal(_inflight2);
dispatch_semaphore_signal(_inflight2);
__block CAMetalLayer *view = _view;
__block CAMetalLayer *foreground = _foreground;
dispatch_async(dispatch_get_main_queue(), ^{
view = nil;
foreground = nil;
});
}
#pragma mark - RTCMTLRenderer
- (void)drawFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame {
@autoreleasepool {
if (frame.width != 0 && frame.height != 0) {
if ([self setupTexturesForFrame:frame]) {
if (_foreground) {
[self doubleRender];
} else {
[self singleRender];
}
}
}
}
}
@end

View file

@ -0,0 +1,25 @@
//
// TGRTCMetalContextHolder.h
// TgVoipWebrtc
//
// Created by Mikhail Filimonov on 28.06.2021.
// Copyright © 2021 Mikhail Filimonov. All rights reserved.
//
#import <Foundation/Foundation.h>
#import <MetalKit/MetalKit.h>
NS_ASSUME_NONNULL_BEGIN
@interface TGRTCMetalContextHolder : NSObject
-(id __nullable)init;
-(id<MTLDevice>)device;
-(id<MTLRenderPipelineState>)pipelineYuvRgb;
-(id<MTLRenderPipelineState>)pipelineTransformAndBlend;
-(id<MTLRenderPipelineState>)pipelineScaleAndBlur;
-(id<MTLRenderPipelineState>)pipelineThrough;
-(id<MTLSamplerState>)sampler;
-(CGDirectDisplayID)displayId;
@end
NS_ASSUME_NONNULL_END

View file

@ -0,0 +1,136 @@
//
// TGRTCMetalContextHolder.m
// TgVoipWebrtc
//
// Created by Mikhail Filimonov on 28.06.2021.
// Copyright © 2021 Mikhail Filimonov. All rights reserved.
//
#import "TGRTCMetalContextHolder.h"
static NSString *const vertexFunctionName = @"vertexPassthrough";
static NSString *const fragmentFunctionName = @"fragmentColorConversion";
static NSString *const fragmentDoTransformFilter = @"doTransformFilter";
static NSString *const twoInputVertexName = @"twoInputVertex";
static NSString *const transformAndBlendName = @"transformAndBlend";
static NSString *const scaleAndBlurName = @"scaleAndBlur";
static NSString *const fragmentPlainName = @"fragmentPlain";
@implementation TGRTCMetalContextHolder
{
id<MTLDevice> _device;
id<MTLRenderPipelineState> _pipelineYuvRgb;
id<MTLRenderPipelineState> _pipelineTransformAndBlend;
id<MTLRenderPipelineState> _pipelineScaleAndBlur;
id<MTLRenderPipelineState> _pipelineThrough;
id<MTLSamplerState> _sampler;
id<MTLLibrary> _defaultLibrary;
CGDirectDisplayID _displayId;
}
-(id __nullable)init {
if(self = [super init]) {
_displayId = CGMainDisplayID();
_device = CGDirectDisplayCopyCurrentMetalDevice(_displayId);
_defaultLibrary = [_device newDefaultLibrary];
}
if (!_device) {
return nil;
}
_sampler = [self defaultSamplerState:_device];
[self loadPipelines];
return self;
}
- (id<MTLSamplerState>)defaultSamplerState:(id<MTLDevice>)device {
MTLSamplerDescriptor *samplerDescriptor = [[MTLSamplerDescriptor alloc] init];
samplerDescriptor.minFilter = MTLSamplerMinMagFilterLinear;
samplerDescriptor.magFilter = MTLSamplerMinMagFilterLinear;
samplerDescriptor.sAddressMode = MTLSamplerAddressModeClampToZero;
samplerDescriptor.tAddressMode = MTLSamplerAddressModeClampToZero;
return [device newSamplerStateWithDescriptor:samplerDescriptor];
}
- (void)loadPipelines {
{
id<MTLFunction> vertexFunction = [_defaultLibrary newFunctionWithName:vertexFunctionName];
id<MTLFunction> fragmentFunction = [_defaultLibrary newFunctionWithName:fragmentFunctionName];
MTLRenderPipelineDescriptor *pipelineDescriptor = [[MTLRenderPipelineDescriptor alloc] init];
pipelineDescriptor.vertexFunction = vertexFunction;
pipelineDescriptor.fragmentFunction = fragmentFunction;
pipelineDescriptor.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm;
pipelineDescriptor.depthAttachmentPixelFormat = MTLPixelFormatInvalid;
NSError *error = nil;
_pipelineYuvRgb = [_device newRenderPipelineStateWithDescriptor:pipelineDescriptor error:&error];
}
{
id<MTLFunction> vertexFunction = [_defaultLibrary newFunctionWithName:vertexFunctionName];
id<MTLFunction> fragmentFunction = [_defaultLibrary newFunctionWithName:fragmentPlainName];
MTLRenderPipelineDescriptor *pipelineDescriptor = [[MTLRenderPipelineDescriptor alloc] init];
pipelineDescriptor.vertexFunction = vertexFunction;
pipelineDescriptor.fragmentFunction = fragmentFunction;
pipelineDescriptor.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm;
pipelineDescriptor.depthAttachmentPixelFormat = MTLPixelFormatInvalid;
NSError *error = nil;
_pipelineThrough = [_device newRenderPipelineStateWithDescriptor:pipelineDescriptor error:&error];
}
{
id<MTLFunction> vertexFunction = [_defaultLibrary newFunctionWithName:twoInputVertexName];
id<MTLFunction> fragmentFunction = [_defaultLibrary newFunctionWithName:transformAndBlendName];
MTLRenderPipelineDescriptor *pipelineDescriptor = [[MTLRenderPipelineDescriptor alloc] init];
pipelineDescriptor.vertexFunction = vertexFunction;
pipelineDescriptor.fragmentFunction = fragmentFunction;
pipelineDescriptor.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm;
pipelineDescriptor.depthAttachmentPixelFormat = MTLPixelFormatInvalid;
NSError *error = nil;
_pipelineTransformAndBlend = [_device newRenderPipelineStateWithDescriptor:pipelineDescriptor error:&error];
}
{
id<MTLFunction> vertexFunction = [_defaultLibrary newFunctionWithName:vertexFunctionName];
id<MTLFunction> fragmentFunction = [_defaultLibrary newFunctionWithName:scaleAndBlurName];
MTLRenderPipelineDescriptor *pipelineDescriptor = [[MTLRenderPipelineDescriptor alloc] init];
pipelineDescriptor.vertexFunction = vertexFunction;
pipelineDescriptor.fragmentFunction = fragmentFunction;
pipelineDescriptor.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm;
pipelineDescriptor.depthAttachmentPixelFormat = MTLPixelFormatInvalid;
NSError *error = nil;
_pipelineScaleAndBlur = [_device newRenderPipelineStateWithDescriptor:pipelineDescriptor error:&error];
}
}
-(id<MTLDevice>)device {
return _device;
}
-(id<MTLRenderPipelineState>)pipelineYuvRgb {
return _pipelineYuvRgb;
}
-(id<MTLRenderPipelineState>)pipelineTransformAndBlend {
return _pipelineTransformAndBlend;
}
-(id<MTLRenderPipelineState>)pipelineScaleAndBlur {
return _pipelineScaleAndBlur;
}
-(id<MTLRenderPipelineState>)pipelineThrough {
return _pipelineThrough;
}
-(id<MTLSamplerState>)sampler {
return _sampler;
}
-(CGDirectDisplayID)displayId {
return _displayId;
}
@end

View file

@ -0,0 +1,41 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_DECODER_FACTORY_H_
#define SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_DECODER_FACTORY_H_
#import "base/RTCMacros.h"
#include "api/video_codecs/video_decoder_factory.h"
#include "media/base/codec.h"
@protocol RTC_OBJC_TYPE
(RTCVideoDecoderFactory);
namespace webrtc {
class CustomObjCVideoDecoderFactory : public VideoDecoderFactory {
public:
explicit CustomObjCVideoDecoderFactory(id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>);
~CustomObjCVideoDecoderFactory() override;
id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> wrapped_decoder_factory() const;
std::vector<SdpVideoFormat> GetSupportedFormats() const override;
std::unique_ptr<VideoDecoder> CreateVideoDecoder(
const SdpVideoFormat& format) override;
private:
id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> decoder_factory_;
};
} // namespace webrtc
#endif // SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_DECODER_FACTORY_H_

View file

@ -0,0 +1,126 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "objc_video_decoder_factory.h"
#import "base/RTCMacros.h"
#import "base/RTCVideoDecoder.h"
#import "base/RTCVideoDecoderFactory.h"
#import "base/RTCVideoFrame.h"
#import "base/RTCVideoFrameBuffer.h"
#import "components/video_codec/RTCCodecSpecificInfoH264.h"
#import "sdk/objc/api/peerconnection/RTCEncodedImage+Private.h"
#import "sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.h"
#import "sdk/objc/api/video_codec/RTCWrappedNativeVideoDecoder.h"
#import "sdk/objc/helpers/NSString+StdString.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_decoder.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/logging.h"
#include "rtc_base/time_utils.h"
#include "sdk/objc/native/src/objc_frame_buffer.h"
namespace webrtc {
namespace {
class ObjCVideoDecoder : public VideoDecoder {
public:
ObjCVideoDecoder(id<RTC_OBJC_TYPE(RTCVideoDecoder)> decoder)
: decoder_(decoder), implementation_name_([decoder implementationName].stdString) {}
bool Configure(const Settings &settings) override {
return
[decoder_ startDecodeWithNumberOfCores:settings.number_of_cores()] == WEBRTC_VIDEO_CODEC_OK;
}
int32_t Decode(const EncodedImage &input_image,
bool missing_frames,
int64_t render_time_ms = -1) override {
RTC_OBJC_TYPE(RTCEncodedImage) *encodedImage =
[[RTC_OBJC_TYPE(RTCEncodedImage) alloc] initWithNativeEncodedImage:input_image];
return (int32_t)[decoder_ decode:encodedImage
missingFrames:missing_frames
codecSpecificInfo:nil
renderTimeMs:render_time_ms];
}
int32_t RegisterDecodeCompleteCallback(DecodedImageCallback *callback) override {
[decoder_ setCallback:^(RTC_OBJC_TYPE(RTCVideoFrame) * frame) {
const auto buffer = rtc::make_ref_counted<ObjCFrameBuffer>(frame.buffer);
VideoFrame videoFrame =
VideoFrame::Builder()
.set_video_frame_buffer(buffer)
.set_timestamp_rtp((uint32_t)(frame.timeStampNs / rtc::kNumNanosecsPerMicrosec))
.set_timestamp_ms(0)
.set_rotation((VideoRotation)frame.rotation)
.build();
videoFrame.set_timestamp(frame.timeStamp);
callback->Decoded(videoFrame);
}];
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t Release() override { return [decoder_ releaseDecoder]; }
const char *ImplementationName() const override { return implementation_name_.c_str(); }
private:
id<RTC_OBJC_TYPE(RTCVideoDecoder)> decoder_;
const std::string implementation_name_;
};
} // namespace
CustomObjCVideoDecoderFactory::CustomObjCVideoDecoderFactory(
id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> decoder_factory)
: decoder_factory_(decoder_factory) {}
CustomObjCVideoDecoderFactory::~CustomObjCVideoDecoderFactory() {}
id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> CustomObjCVideoDecoderFactory::wrapped_decoder_factory() const {
return decoder_factory_;
}
std::unique_ptr<VideoDecoder> CustomObjCVideoDecoderFactory::CreateVideoDecoder(
const SdpVideoFormat &format) {
NSString *codecName = [NSString stringWithUTF8String:format.name.c_str()];
for (RTC_OBJC_TYPE(RTCVideoCodecInfo) * codecInfo in decoder_factory_.supportedCodecs) {
if ([codecName isEqualToString:codecInfo.name]) {
id<RTC_OBJC_TYPE(RTCVideoDecoder)> decoder = [decoder_factory_ createDecoder:codecInfo];
// Because of symbol conflict, isKindOfClass doesn't work as expected.
// See https://bugs.webkit.org/show_bug.cgi?id=198782.
if ([decoder isKindOfClass:[RTCWrappedNativeVideoDecoder class]]) {
return [(RTCWrappedNativeVideoDecoder *)decoder releaseWrappedDecoder];
} else {
return std::unique_ptr<ObjCVideoDecoder>(new ObjCVideoDecoder(decoder));
}
}
}
return nullptr;
}
std::vector<SdpVideoFormat> CustomObjCVideoDecoderFactory::GetSupportedFormats() const {
std::vector<SdpVideoFormat> supported_formats;
for (RTC_OBJC_TYPE(RTCVideoCodecInfo) * supportedCodec in decoder_factory_.supportedCodecs) {
SdpVideoFormat format = [supportedCodec nativeSdpVideoFormat];
supported_formats.push_back(format);
}
return supported_formats;
}
} // namespace webrtc

Some files were not shown because too many files have changed in this diff Show more