c2c/signal-android
c2c/signal-android
2
0
Fork 0
Code Issues Pull requests Projects Releases 7 Packages Wiki Activity Actions

Source added

Browse source
This commit is contained in:
Fr4nz D13trich 2025-11-20 09:26:33 +01:00
parent b2864b500e
commit ba28ca859e
8352 changed files with 1487182 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

1
video/app/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
/build

76
video/app/build.gradle.kts Normal file
View file

@ -0,0 +1,76 @@
/*
* Copyright 2023 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
plugins {
id("signal-sample-app")
alias(libs.plugins.compose.compiler)
}
val signalBuildToolsVersion: String by rootProject.extra
val signalCompileSdkVersion: String by rootProject.extra
val signalTargetSdkVersion: Int by rootProject.extra
val signalMinSdkVersion: Int by rootProject.extra
val signalJavaVersion: JavaVersion by rootProject.extra
val signalKotlinJvmTarget: String by rootProject.extra
android {
namespace = "org.thoughtcrime.video.app"
compileSdkVersion = signalCompileSdkVersion
defaultConfig {
applicationId = "org.thoughtcrime.video.app"
minSdk = 23
targetSdk = signalTargetSdkVersion
versionCode = 1
versionName = "1.0"
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
vectorDrawables {
useSupportLibrary = true
}
}
buildTypes {
release {
isMinifyEnabled = false
proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro")
}
}
compileOptions {
sourceCompatibility = signalJavaVersion
targetCompatibility = signalJavaVersion
}
kotlinOptions {
jvmTarget = signalKotlinJvmTarget
}
buildFeatures {
compose = true
}
composeOptions {
kotlinCompilerExtensionVersion = "1.5.4"
}
packaging {
resources {
excludes += "/META-INF/{AL2.0,LGPL2.1}"
}
}
}
dependencies {
implementation(libs.androidx.fragment.ktx)
implementation(libs.androidx.activity.compose)
implementation(platform(libs.androidx.compose.bom))
implementation(libs.androidx.compose.material3)
implementation(libs.bundles.media3)
implementation(project(":video"))
implementation(project(":core-util"))
implementation("androidx.work:work-runtime-ktx:2.9.1")
implementation("androidx.lifecycle:lifecycle-viewmodel-compose:2.8.4")
implementation(libs.androidx.compose.ui.tooling.core)
implementation(libs.androidx.compose.ui.test.manifest)
androidTestImplementation(testLibs.junit.junit)
androidTestImplementation(testLibs.androidx.test.runner)
androidTestImplementation(testLibs.androidx.test.ext.junit.ktx)
}

21
video/app/proguard-rules.pro vendored Normal file
View file

@ -0,0 +1,21 @@
# Add project specific ProGuard rules here.
# You can control the set of applied configuration files using the
# proguardFiles setting in build.gradle.
#
# For more details, see
# http://developer.android.com/guide/developing/tools/proguard.html
# If your project uses WebView with JS, uncomment the following
# and specify the fully qualified class name to the JavaScript interface
# class:
#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
# public *;
#}
# Uncomment this to preserve the line number information for
# debugging stack traces.
#-keepattributes SourceFile,LineNumberTable
# If you keep the line number information, uncomment this to
# hide the original source file name.
#-renamesourcefileattribute SourceFile

27
video/app/src/androidTest/java/org/thoughtcrime/video/app/ExampleInstrumentedTest.kt Normal file
View file

@ -0,0 +1,27 @@
/*
* Copyright 2023 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.video.app
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import org.junit.Assert.assertEquals
import org.junit.Test
import org.junit.runner.RunWith
/**
* Instrumented test, which will execute on an Android device.
*
* See [testing documentation](http://d.android.com/tools/testing).
*/
@RunWith(AndroidJUnit4::class)
class ExampleInstrumentedTest {
@Test
fun useAppContext() {
// Context of the app under test.
val appContext = InstrumentationRegistry.getInstrumentation().targetContext
assertEquals("org.thoughtcrime.video.app", appContext.packageName)
}
}

47
video/app/src/main/AndroidManifest.xml Normal file
View file

@ -0,0 +1,47 @@
<?xml version="1.0" encoding="utf-8"?><!--
~ Copyright 2023 Signal Messenger, LLC
~ SPDX-License-Identifier: AGPL-3.0-only
-->
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:tools="http://schemas.android.com/tools">
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC" />
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
<application
android:allowBackup="true"
android:icon="@mipmap/ic_launcher"
android:label="@string/app_name"
android:roundIcon="@mipmap/ic_launcher_round"
android:supportsRtl="true"
android:theme="@style/Theme.Signal">
<activity
android:name=".MainActivity"
android:exported="true"
android:theme="@style/Theme.Signal">
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
</activity>
<activity
android:name=".transcode.TranscodeTestActivity"
android:exported="false"
android:parentActivityName=".MainActivity"
android:theme="@style/Theme.Signal" />
<activity
android:name=".playback.PlaybackTestActivity"
android:exported="false"
android:parentActivityName=".MainActivity"
android:theme="@style/Theme.Signal" />
<service
android:name="androidx.work.impl.foreground.SystemForegroundService"
android:foregroundServiceType="dataSync"
tools:node="merge" />
</application>
</manifest>

157
video/app/src/main/java/org/thoughtcrime/video/app/MainActivity.kt Normal file
View file

@ -0,0 +1,157 @@
/*
* Copyright 2023 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.video.app
import android.content.Context
import android.content.Intent
import android.content.SharedPreferences
import android.media.MediaScannerConnection
import android.os.Bundle
import android.os.Environment
import androidx.activity.compose.setContent
import androidx.appcompat.app.AppCompatActivity
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.material3.Checkbox
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.tooling.preview.Preview
import org.signal.core.util.logging.AndroidLogger
import org.signal.core.util.logging.Log
import org.thoughtcrime.video.app.playback.PlaybackTestActivity
import org.thoughtcrime.video.app.transcode.TranscodeTestActivity
import org.thoughtcrime.video.app.ui.composables.LabeledButton
import org.thoughtcrime.video.app.ui.theme.SignalTheme
/**
* Main activity for this sample app.
*/
class MainActivity : AppCompatActivity() {
companion object {
private val TAG = Log.tag(MainActivity::class.java)
private var appLaunch = true
}
private val sharedPref: SharedPreferences by lazy {
getSharedPreferences(
getString(R.string.preference_file_key),
Context.MODE_PRIVATE
)
}
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
Log.initialize(AndroidLogger)
val startPlaybackScreen = { saveChoice: Boolean -> proceed(Screen.TEST_PLAYBACK, saveChoice) }
val startTranscodeScreen = { saveChoice: Boolean -> proceed(Screen.TEST_TRANSCODE, saveChoice) }
setContent {
Body(startPlaybackScreen, startTranscodeScreen)
}
refreshMediaProviderForExternalStorage(this, arrayOf("video/*"))
if (appLaunch) {
appLaunch = false
getLaunchChoice()?.let {
proceed(it, false)
}
}
}
@Composable
private fun Body(startPlaybackScreen: (Boolean) -> Unit, startTranscodeScreen: (Boolean) -> Unit) {
var rememberChoice by remember { mutableStateOf(getLaunchChoice() != null) }
SignalTheme {
Surface(modifier = Modifier.fillMaxSize(), color = MaterialTheme.colorScheme.background) {
Column(
verticalArrangement = Arrangement.Center,
horizontalAlignment = Alignment.CenterHorizontally
) {
LabeledButton("Test Playback") {
startPlaybackScreen(rememberChoice)
}
LabeledButton("Test Transcode") {
startTranscodeScreen(rememberChoice)
}
Row(
verticalAlignment = Alignment.CenterVertically
) {
Checkbox(
checked = rememberChoice,
onCheckedChange = { isChecked ->
rememberChoice = isChecked
if (!isChecked) {
clearLaunchChoice()
}
}
)
Text(text = "Remember & Skip This Screen", style = MaterialTheme.typography.labelLarge)
}
}
}
}
}
private fun getLaunchChoice(): Screen? {
val screenName = sharedPref.getString(getString(R.string.preference_activity_shortcut_key), null) ?: return null
return Screen.valueOf(screenName)
}
private fun clearLaunchChoice() {
with(sharedPref.edit()) {
remove(getString(R.string.preference_activity_shortcut_key))
apply()
}
}
private fun saveLaunchChoice(choice: Screen) {
with(sharedPref.edit()) {
putString(getString(R.string.preference_activity_shortcut_key), choice.name)
apply()
}
}
private fun refreshMediaProviderForExternalStorage(context: Context, mimeTypes: Array<String>) {
val rootPath = Environment.getExternalStorageDirectory().absolutePath
MediaScannerConnection.scanFile(
context,
arrayOf<String>(rootPath),
mimeTypes
) { _, _ ->
Log.i(TAG, "Re-scan of external storage for media completed.")
}
}
private fun proceed(screen: Screen, saveChoice: Boolean) {
if (saveChoice) {
saveLaunchChoice(screen)
}
when (screen) {
Screen.TEST_PLAYBACK -> startActivity(Intent(this, PlaybackTestActivity::class.java))
Screen.TEST_TRANSCODE -> startActivity(Intent(this, TranscodeTestActivity::class.java))
}
}
private enum class Screen {
TEST_PLAYBACK,
TEST_TRANSCODE
}
@Preview
@Composable
private fun PreviewBody() {
Body({}, {})
}
}

107
video/app/src/main/java/org/thoughtcrime/video/app/playback/PlaybackTestActivity.kt Normal file
View file

@ -0,0 +1,107 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.video.app.playback
import android.os.Bundle
import android.util.Log
import androidx.activity.compose.setContent
import androidx.activity.result.PickVisualMediaRequest
import androidx.activity.result.contract.ActivityResultContracts
import androidx.activity.viewModels
import androidx.annotation.OptIn
import androidx.appcompat.app.AppCompatActivity
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.runtime.Composable
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.tooling.preview.Preview
import androidx.compose.ui.viewinterop.AndroidView
import androidx.media3.common.util.UnstableApi
import androidx.media3.exoplayer.ExoPlayer
import androidx.media3.exoplayer.source.MediaSource
import androidx.media3.ui.PlayerView
import org.thoughtcrime.video.app.ui.composables.LabeledButton
import org.thoughtcrime.video.app.ui.theme.SignalTheme
class PlaybackTestActivity : AppCompatActivity() {
private val viewModel: PlaybackTestViewModel by viewModels()
private lateinit var exoPlayer: ExoPlayer
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
viewModel.initialize(this)
exoPlayer = ExoPlayer.Builder(this).build()
setContent {
SignalTheme {
Surface(modifier = Modifier.fillMaxSize(), color = MaterialTheme.colorScheme.background) {
Column(
verticalArrangement = Arrangement.Center,
horizontalAlignment = Alignment.CenterHorizontally
) {
val videoUri = viewModel.selectedVideo
if (videoUri == null) {
LabeledButton("Select Video") { pickMedia.launch(PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.VideoOnly)) }
} else {
LabeledButton("Play Video") { viewModel.updateMediaSource(this@PlaybackTestActivity) }
LabeledButton("Play Video with slow download") { viewModel.updateMediaSourceTrickle(this@PlaybackTestActivity) }
ExoVideoView(source = viewModel.mediaSource, exoPlayer = exoPlayer)
}
}
}
}
}
}
override fun onPause() {
super.onPause()
exoPlayer.pause()
}
override fun onDestroy() {
super.onDestroy()
viewModel.releaseCache()
exoPlayer.stop()
exoPlayer.release()
}
/**
* This launches the system media picker and stores the resulting URI.
*/
private val pickMedia = registerForActivityResult(ActivityResultContracts.PickVisualMedia()) { uri ->
if (uri != null) {
Log.d("PlaybackPicker", "Selected URI: $uri")
viewModel.selectedVideo = uri
viewModel.updateMediaSource(this)
} else {
Log.d("PlaybackPicker", "No media selected")
}
}
}
@OptIn(UnstableApi::class)
@Composable
fun ExoVideoView(source: MediaSource, exoPlayer: ExoPlayer, modifier: Modifier = Modifier) {
exoPlayer.playWhenReady = false
exoPlayer.setMediaSource(source)
exoPlayer.prepare()
AndroidView(factory = { context ->
PlayerView(context).apply {
player = exoPlayer
}
}, modifier = modifier)
}
@Preview(showBackground = true)
@Composable
fun GreetingPreview() {
SignalTheme {
LabeledButton("Preview Render") {}
}
}

93
video/app/src/main/java/org/thoughtcrime/video/app/playback/PlaybackTestViewModel.kt Normal file
View file

@ -0,0 +1,93 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.video.app.playback
import android.content.Context
import android.net.Uri
import androidx.annotation.OptIn
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.setValue
import androidx.lifecycle.ViewModel
import androidx.media3.common.MediaItem
import androidx.media3.common.util.UnstableApi
import androidx.media3.datasource.DefaultDataSource
import androidx.media3.datasource.cache.Cache
import androidx.media3.datasource.cache.CacheDataSource
import androidx.media3.datasource.cache.NoOpCacheEvictor
import androidx.media3.datasource.cache.SimpleCache
import androidx.media3.exoplayer.source.MediaSource
import androidx.media3.exoplayer.source.ProgressiveMediaSource
import androidx.media3.exoplayer.source.SilenceMediaSource
import java.io.File
/**
* Main screen view model for the video sample app.
*/
@OptIn(UnstableApi::class)
class PlaybackTestViewModel : ViewModel() {
// Initialize an silent media source before the user selects a video. This is the closest I could find to an "empty" media source while still being nullsafe.
private val value by lazy {
val factory = SilenceMediaSource.Factory()
factory.setDurationUs(1000)
factory.createMediaSource()
}
private lateinit var cache: Cache
var selectedVideo: Uri? by mutableStateOf(null)
var mediaSource: MediaSource by mutableStateOf(value)
private set
/**
* Initialize the backing cache. This is a file in the app's cache directory that has a random suffix to ensure you get cache misses on a new app launch.
*
* @param context required to get the file path of the cache directory.
*/
fun initialize(context: Context) {
val cacheDir = File(context.cacheDir.absolutePath)
cache = SimpleCache(File(cacheDir, getRandomString(12)), NoOpCacheEvictor())
}
fun updateMediaSource(context: Context) {
selectedVideo?.let {
mediaSource = ProgressiveMediaSource.Factory(DefaultDataSource.Factory(context)).createMediaSource(MediaItem.fromUri(it))
}
}
/**
* Replaces the media source with one that has a latency to each read from the media source, simulating network latency.
* It stores the result in a cache (that does not have a penalty) to better mimic real-world performance:
* once a chunk is downloaded from the network, it will not have to be re-fetched.
*
* @param context
*/
fun updateMediaSourceTrickle(context: Context) {
selectedVideo?.let {
val cacheFactory = CacheDataSource.Factory()
.setCache(cache)
.setUpstreamDataSourceFactory(SlowDataSource.Factory(context, 10))
mediaSource = ProgressiveMediaSource.Factory(cacheFactory).createMediaSource(MediaItem.fromUri(it))
}
}
fun releaseCache() {
cache.release()
}
/**
* Get random string. Will always return at least one character.
*
* @param length length of the returned string.
* @return a string composed of random alphanumeric characters of the specified length (minimum of 1).
*/
private fun getRandomString(length: Int): String {
val allowedChars = ('A'..'Z') + ('a'..'z') + ('0'..'9')
return (1..length.coerceAtLeast(1))
.map { allowedChars.random() }
.joinToString("")
}
}

55
video/app/src/main/java/org/thoughtcrime/video/app/playback/SlowDataSource.kt Normal file
View file

@ -0,0 +1,55 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.video.app.playback
import android.content.Context
import android.net.Uri
import androidx.annotation.OptIn
import androidx.media3.common.util.UnstableApi
import androidx.media3.datasource.DataSource
import androidx.media3.datasource.DataSpec
import androidx.media3.datasource.DefaultDataSource
import androidx.media3.datasource.TransferListener
/**
* This wraps a [DefaultDataSource] and adds [latency] to each read. This is intended to approximate a slow/shoddy network connection that drip-feeds in data.
*
* @property latency the amount, in milliseconds, that each read should be delayed. A good proxy for network ping.
* @constructor
*
* @param context used to initialize the underlying [DefaultDataSource.Factory]
*/
@OptIn(UnstableApi::class)
class SlowDataSource(context: Context, private val latency: Long) : DataSource {
private val internalDataSource: DataSource = DefaultDataSource.Factory(context).createDataSource()
override fun read(buffer: ByteArray, offset: Int, length: Int): Int {
Thread.sleep(latency)
return internalDataSource.read(buffer, offset, length)
}
override fun addTransferListener(transferListener: TransferListener) {
internalDataSource.addTransferListener(transferListener)
}
override fun open(dataSpec: DataSpec): Long {
return internalDataSource.open(dataSpec)
}
override fun getUri(): Uri? {
return internalDataSource.uri
}
override fun close() {
return internalDataSource.close()
}
class Factory(private val context: Context, private val latency: Long) : DataSource.Factory {
override fun createDataSource(): DataSource {
return SlowDataSource(context, latency)
}
}
}

25
video/app/src/main/java/org/thoughtcrime/video/app/transcode/Constants.kt Normal file
View file

@ -0,0 +1,25 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.video.app.transcode
/**
* A dumping ground for constants that should be referenced across the sample app.
*/
internal const val MIN_VIDEO_MEGABITRATE = 0.5f
internal const val MAX_VIDEO_MEGABITRATE = 4f
internal val OPTIONS_AUDIO_KILOBITRATES = listOf(64, 96, 128, 160, 192)
enum class VideoResolution(val longEdge: Int, val shortEdge: Int) {
SD(854, 480),
HD(1280, 720),
FHD(1920, 1080),
WQHD(2560, 1440),
UHD(3840, 2160);
fun getContentDescription(): String {
return "Resolution with a long edge of $longEdge and a short edge of $shortEdge."
}
}

141
video/app/src/main/java/org/thoughtcrime/video/app/transcode/TranscodeTestActivity.kt Normal file
View file

@ -0,0 +1,141 @@
/*
* Copyright 2024 Signal Messenger, LLC
* 2SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.video.app.transcode
import android.Manifest
import android.app.NotificationChannel
import android.app.NotificationManager
import android.content.DialogInterface
import android.content.Intent
import android.content.pm.PackageManager
import android.net.Uri
import android.os.Build
import android.os.Bundle
import android.util.Log
import android.view.ViewGroup
import androidx.activity.compose.setContent
import androidx.activity.result.PickVisualMediaRequest
import androidx.activity.result.contract.ActivityResultContracts
import androidx.activity.viewModels
import androidx.appcompat.app.AppCompatActivity
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.verticalScroll
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.ComposeView
import androidx.compose.ui.unit.dp
import androidx.core.app.ActivityCompat
import androidx.core.content.ContextCompat
import androidx.lifecycle.compose.collectAsStateWithLifecycle
import com.google.android.material.dialog.MaterialAlertDialogBuilder
import org.thoughtcrime.video.app.R
import org.thoughtcrime.video.app.transcode.composables.ConfigureEncodingParameters
import org.thoughtcrime.video.app.transcode.composables.SelectInput
import org.thoughtcrime.video.app.transcode.composables.SelectOutput
import org.thoughtcrime.video.app.transcode.composables.TranscodingJobProgress
import org.thoughtcrime.video.app.transcode.composables.WorkState
import org.thoughtcrime.video.app.ui.theme.SignalTheme
/**
* Visual entry point for testing transcoding in the video sample app.
*/
class TranscodeTestActivity : AppCompatActivity() {
private val TAG = "TranscodeTestActivity"
private val viewModel: TranscodeTestViewModel by viewModels()
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
viewModel.initialize(this)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
val name = applicationContext.getString(R.string.channel_name)
val descriptionText = applicationContext.getString(R.string.channel_description)
val importance = NotificationManager.IMPORTANCE_DEFAULT
val mChannel = NotificationChannel(getString(R.string.notification_channel_id), name, importance)
mChannel.description = descriptionText
val notificationManager = applicationContext.getSystemService(NOTIFICATION_SERVICE) as NotificationManager
notificationManager.createNotificationChannel(mChannel)
}
setContent {
SignalTheme {
Surface(modifier = Modifier.fillMaxSize(), color = MaterialTheme.colorScheme.background) {
val transcodingJobs = viewModel.getTranscodingJobsAsState().collectAsStateWithLifecycle(emptyList())
if (transcodingJobs.value.isNotEmpty()) {
TranscodingJobProgress(transcodingJobs = transcodingJobs.value.map { WorkState.fromInfo(it) }, resetButtonOnClick = { viewModel.reset() })
} else if (viewModel.selectedVideos.isEmpty()) {
SelectInput { pickMedia.launch(PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.VideoOnly)) }
} else if (viewModel.outputDirectory == null) {
SelectOutput { outputDirRequest.launch(null) }
} else {
ConfigureEncodingParameters(
modifier = Modifier
.padding(horizontal = 16.dp)
.verticalScroll(rememberScrollState()),
viewModel = viewModel
)
}
}
}
}
getComposeView()?.keepScreenOn = true
if (Build.VERSION.SDK_INT >= 33) {
val notificationPermissionStatus = ContextCompat.checkSelfPermission(this, Manifest.permission.POST_NOTIFICATIONS)
Log.v(TAG, "Notification permission status: $notificationPermissionStatus")
if (notificationPermissionStatus != PackageManager.PERMISSION_GRANTED) {
if (ActivityCompat.shouldShowRequestPermissionRationale(this, Manifest.permission.POST_NOTIFICATIONS)) {
showPermissionRationaleDialog { _, _ -> requestPermissionLauncher.launch(Manifest.permission.POST_NOTIFICATIONS) }
} else {
requestPermissionLauncher.launch(Manifest.permission.POST_NOTIFICATIONS)
}
}
}
}
private fun showPermissionRationaleDialog(okListener: DialogInterface.OnClickListener) {
MaterialAlertDialogBuilder(this)
.setTitle("The system will request the notification permission.")
.setMessage("This permission is required to show the transcoding progress in the notification tray.")
.setPositiveButton("Ok", okListener)
.show()
}
/**
* This launches the system media picker and stores the resulting URI.
*/
private val pickMedia = registerForActivityResult(ActivityResultContracts.PickMultipleVisualMedia()) { uris: List<Uri> ->
if (uris.isNotEmpty()) {
Log.d(TAG, "Selected URI: $uris")
viewModel.selectedVideos = uris
viewModel.resetOutputDirectory()
} else {
Log.d(TAG, "No media selected")
}
}
private val outputDirRequest = registerForActivityResult(ActivityResultContracts.OpenDocumentTree()) { uri ->
uri?.let {
contentResolver.takePersistableUriPermission(uri, Intent.FLAG_GRANT_READ_URI_PERMISSION)
viewModel.setOutputDirectoryAndCleanFailedTranscodes(this, it)
}
}
private val requestPermissionLauncher =
registerForActivityResult(
ActivityResultContracts.RequestPermission()
) { isGranted: Boolean ->
Log.d(TAG, "Notification permission allowed: $isGranted")
}
private fun getComposeView(): ComposeView? {
return window.decorView
.findViewById<ViewGroup>(android.R.id.content)
.getChildAt(0) as? ComposeView
}
}

114
video/app/src/main/java/org/thoughtcrime/video/app/transcode/TranscodeTestRepository.kt Normal file
View file

@ -0,0 +1,114 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.video.app.transcode
import android.content.Context
import android.net.Uri
import androidx.work.Data
import androidx.work.OneTimeWorkRequestBuilder
import androidx.work.WorkInfo
import androidx.work.WorkManager
import androidx.work.WorkQuery
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.emptyFlow
import org.thoughtcrime.securesms.video.TranscodingPreset
import java.util.UUID
import kotlin.math.absoluteValue
import kotlin.random.Random
/**
* Repository to perform various transcoding functions.
*/
class TranscodeTestRepository(context: Context) {
private val workManager = WorkManager.getInstance(context)
private val usedNotificationIds = emptySet<Int>()
private fun transcode(selectedVideos: List<Uri>, outputDirectory: Uri, forceSequentialProcessing: Boolean, transcodingPreset: TranscodingPreset? = null, customTranscodingOptions: CustomTranscodingOptions? = null): Map<UUID, Uri> {
if (customTranscodingOptions == null && transcodingPreset == null) {
throw IllegalArgumentException("Must define either custom options or transcoding preset!")
} else if (customTranscodingOptions != null && transcodingPreset != null) {
throw IllegalArgumentException("Cannot define both custom options and transcoding preset!")
}
if (selectedVideos.isEmpty()) {
return emptyMap()
}
val urisAndRequests = selectedVideos.map {
var notificationId = Random.nextInt().absoluteValue
while (usedNotificationIds.contains(notificationId)) {
notificationId = Random.nextInt().absoluteValue
}
val inputData = Data.Builder()
.putString(TranscodeWorker.KEY_INPUT_URI, it.toString())
.putString(TranscodeWorker.KEY_OUTPUT_URI, outputDirectory.toString())
.putInt(TranscodeWorker.KEY_NOTIFICATION_ID, notificationId)
if (transcodingPreset != null) {
inputData.putString(TranscodeWorker.KEY_TRANSCODING_PRESET_NAME, transcodingPreset.name)
} else if (customTranscodingOptions != null) {
inputData.putString(TranscodeWorker.KEY_VIDEO_CODEC, customTranscodingOptions.videoCodec)
inputData.putInt(TranscodeWorker.KEY_LONG_EDGE, customTranscodingOptions.videoResolution.longEdge)
inputData.putInt(TranscodeWorker.KEY_SHORT_EDGE, customTranscodingOptions.videoResolution.shortEdge)
inputData.putInt(TranscodeWorker.KEY_VIDEO_BIT_RATE, customTranscodingOptions.videoBitrate)
inputData.putInt(TranscodeWorker.KEY_AUDIO_BIT_RATE, customTranscodingOptions.audioBitrate)
inputData.putBoolean(TranscodeWorker.KEY_ENABLE_FASTSTART, customTranscodingOptions.enableFastStart)
inputData.putBoolean(TranscodeWorker.KEY_ENABLE_AUDIO_REMUX, customTranscodingOptions.enableAudioRemux)
}
val transcodeRequest = OneTimeWorkRequestBuilder<TranscodeWorker>()
.setInputData(inputData.build())
.addTag(TRANSCODING_WORK_TAG)
.build()
it to transcodeRequest
}
val idsToUris = urisAndRequests.associateBy({ it.second.id }, { it.first })
val requests = urisAndRequests.map { it.second }
if (forceSequentialProcessing) {
var continuation = workManager.beginWith(requests.first())
for (request in requests.drop(1)) {
continuation = continuation.then(request)
}
continuation.enqueue()
} else {
workManager.enqueue(requests)
}
return idsToUris
}
fun transcodeWithCustomOptions(selectedVideos: List<Uri>, outputDirectory: Uri, forceSequentialProcessing: Boolean, customTranscodingOptions: CustomTranscodingOptions?): Map<UUID, Uri> {
return transcode(selectedVideos, outputDirectory, forceSequentialProcessing, customTranscodingOptions = customTranscodingOptions)
}
fun transcodeWithPresetOptions(selectedVideos: List<Uri>, outputDirectory: Uri, forceSequentialProcessing: Boolean, transcodingPreset: TranscodingPreset): Map<UUID, Uri> {
return transcode(selectedVideos, outputDirectory, forceSequentialProcessing, transcodingPreset)
}
fun getTranscodingJobsAsFlow(jobIds: List<UUID>): Flow<MutableList<WorkInfo>> {
if (jobIds.isEmpty()) {
return emptyFlow()
}
return workManager.getWorkInfosFlow(WorkQuery.fromIds(jobIds))
}
fun cancelAllTranscodes() {
workManager.cancelAllWorkByTag(TRANSCODING_WORK_TAG)
workManager.pruneWork()
}
fun cleanPrivateStorage(context: Context) {
context.filesDir.listFiles()?.forEach {
it.delete()
}
}
data class CustomTranscodingOptions(val videoCodec: String, val videoResolution: VideoResolution, val videoBitrate: Int, val audioBitrate: Int, val enableFastStart: Boolean, val enableAudioRemux: Boolean)
companion object {
private const val TAG = "TranscodingTestRepository"
const val TRANSCODING_WORK_TAG = "transcoding"
}
}

133
video/app/src/main/java/org/thoughtcrime/video/app/transcode/TranscodeTestViewModel.kt Normal file
View file

@ -0,0 +1,133 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.video.app.transcode
import android.content.Context
import android.net.Uri
import android.widget.Toast
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableFloatStateOf
import androidx.compose.runtime.mutableIntStateOf
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.setValue
import androidx.lifecycle.ViewModel
import androidx.work.WorkInfo
import kotlinx.coroutines.flow.Flow
import org.thoughtcrime.securesms.video.TranscodingPreset
import org.thoughtcrime.securesms.video.TranscodingQuality
import org.thoughtcrime.securesms.video.videoconverter.MediaConverter
import java.util.UUID
import kotlin.math.roundToInt
/**
* ViewModel for the transcoding screen of the video sample app. See [TranscodeTestActivity].
*/
class TranscodeTestViewModel : ViewModel() {
private lateinit var repository: TranscodeTestRepository
private var backPressedRunnable = {}
private var transcodingJobs: Map<UUID, Uri> = emptyMap()
var transcodingPreset by mutableStateOf(TranscodingPreset.LEVEL_2)
private set
var outputDirectory: Uri? by mutableStateOf(null)
private set
var selectedVideos: List<Uri> by mutableStateOf(emptyList())
var videoMegaBitrate by mutableFloatStateOf(calculateVideoMegaBitrateFromPreset(transcodingPreset))
var videoResolution by mutableStateOf(convertPresetToVideoResolution(transcodingPreset))
var audioKiloBitrate by mutableIntStateOf(calculateAudioKiloBitrateFromPreset(transcodingPreset))
var useHevc by mutableStateOf(false)
var useAutoTranscodingSettings by mutableStateOf(true)
var enableFastStart by mutableStateOf(true)
var enableAudioRemux by mutableStateOf(true)
var forceSequentialQueueProcessing by mutableStateOf(false)
fun initialize(context: Context) {
repository = TranscodeTestRepository(context)
backPressedRunnable = { Toast.makeText(context, "Cancelling all transcoding jobs!", Toast.LENGTH_LONG).show() }
}
fun transcode() {
val output = outputDirectory ?: throw IllegalStateException("No output directory selected!")
transcodingJobs = if (useAutoTranscodingSettings) {
repository.transcodeWithPresetOptions(
selectedVideos,
output,
forceSequentialQueueProcessing,
transcodingPreset
)
} else {
repository.transcodeWithCustomOptions(
selectedVideos,
output,
forceSequentialQueueProcessing,
TranscodeTestRepository.CustomTranscodingOptions(
if (useHevc) MediaConverter.VIDEO_CODEC_H265 else MediaConverter.VIDEO_CODEC_H264,
videoResolution,
(videoMegaBitrate * MEGABIT).roundToInt(),
audioKiloBitrate * KILOBIT,
enableAudioRemux,
enableFastStart
)
)
}
}
fun updateTranscodingPreset(preset: TranscodingPreset) {
transcodingPreset = preset
videoResolution = convertPresetToVideoResolution(preset)
videoMegaBitrate = calculateVideoMegaBitrateFromPreset(preset)
audioKiloBitrate = calculateAudioKiloBitrateFromPreset(preset)
}
fun getTranscodingJobsAsState(): Flow<MutableList<WorkInfo>> {
return repository.getTranscodingJobsAsFlow(transcodingJobs.keys.toList())
}
fun setOutputDirectoryAndCleanFailedTranscodes(context: Context, folderUri: Uri) {
outputDirectory = folderUri
repository.cleanPrivateStorage(context)
}
fun reset() {
cancelAllTranscodes()
resetOutputDirectory()
selectedVideos = emptyList()
}
private fun cancelAllTranscodes() {
repository.cancelAllTranscodes()
transcodingJobs = emptyMap()
}
fun resetOutputDirectory() {
outputDirectory = null
}
companion object {
private const val MEGABIT = 1000000
private const val KILOBIT = 1000
@JvmStatic
private fun calculateVideoMegaBitrateFromPreset(preset: TranscodingPreset): Float {
val quality = TranscodingQuality.createFromPreset(preset, -1)
return quality.targetVideoBitRate.toFloat() / MEGABIT
}
@JvmStatic
private fun calculateAudioKiloBitrateFromPreset(preset: TranscodingPreset): Int {
val quality = TranscodingQuality.createFromPreset(preset, -1)
return quality.targetAudioBitRate / KILOBIT
}
@JvmStatic
private fun convertPresetToVideoResolution(preset: TranscodingPreset) = when (preset) {
TranscodingPreset.LEVEL_3 -> VideoResolution.HD
else -> VideoResolution.SD
}
}
}

266
video/app/src/main/java/org/thoughtcrime/video/app/transcode/TranscodeWorker.kt Normal file
View file

@ -0,0 +1,266 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.video.app.transcode
import android.app.PendingIntent
import android.content.Context
import android.content.Intent
import android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC
import android.net.Uri
import android.os.Build
import android.util.Log
import androidx.core.app.NotificationCompat
import androidx.core.app.TaskStackBuilder
import androidx.documentfile.provider.DocumentFile
import androidx.media3.common.util.UnstableApi
import androidx.work.CoroutineWorker
import androidx.work.Data
import androidx.work.ForegroundInfo
import androidx.work.WorkManager
import androidx.work.WorkerParameters
import org.signal.core.util.readLength
import org.thoughtcrime.securesms.video.StreamingTranscoder
import org.thoughtcrime.securesms.video.TranscodingPreset
import org.thoughtcrime.securesms.video.postprocessing.Mp4FaststartPostProcessor
import org.thoughtcrime.securesms.video.videoconverter.MediaConverter.VideoCodec
import org.thoughtcrime.securesms.video.videoconverter.mediadatasource.InputStreamMediaDataSource
import org.thoughtcrime.securesms.video.videoconverter.utils.VideoConstants
import org.thoughtcrime.video.app.R
import java.io.File
import java.io.FileInputStream
import java.io.IOException
import java.io.InputStream
import java.time.Instant
/**
* A WorkManager worker to transcode videos in the background. This utilizes [StreamingTranscoder].
*/
class TranscodeWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx, params) {
private var lastProgress = 0
@UnstableApi
override suspend fun doWork(): Result {
val logPrefix = "[Job ${id.toString().takeLast(4)}]"
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) {
Log.w(TAG, "$logPrefix Transcoder is only supported on API 26+!")
return Result.failure()
}
val inputParams = InputParams(inputData)
val inputFilename = DocumentFile.fromSingleUri(applicationContext, inputParams.inputUri)?.name?.removeFileExtension()
if (inputFilename == null) {
Log.w(TAG, "$logPrefix Could not read input file name!")
return Result.failure()
}
val filenameBase = "transcoded-${Instant.now()}-$inputFilename"
val tempFilename = "$filenameBase$TEMP_FILE_EXTENSION"
val finalFilename = "$filenameBase$OUTPUT_FILE_EXTENSION"
setForeground(createForegroundInfo(-1, inputParams.notificationId))
applicationContext.openFileOutput(tempFilename, Context.MODE_PRIVATE).use { outputStream ->
if (outputStream == null) {
Log.w(TAG, "$logPrefix Could not open temp file for I/O!")
return Result.failure()
}
applicationContext.contentResolver.openInputStream(inputParams.inputUri).use { inputStream ->
applicationContext.openFileOutput(inputFilename, Context.MODE_PRIVATE).use { outputStream ->
Log.i(TAG, "Started copying input to internal storage.")
inputStream?.copyTo(outputStream)
Log.i(TAG, "Finished copying input to internal storage.")
}
}
}
val datasource = WorkerMediaDataSource(File(applicationContext.filesDir, inputFilename))
val transcoder = if (inputParams.resolution > 0 && inputParams.videoBitrate > 0) {
if (inputParams.videoCodec == null) {
Log.w(TAG, "$logPrefix Video codec was null!")
return Result.failure()
}
Log.d(TAG, "$logPrefix Initializing StreamingTranscoder with custom parameters: CODEC:${inputParams.videoCodec} B:V=${inputParams.videoBitrate}, B:A=${inputParams.audioBitrate}, res=${inputParams.resolution}, audioRemux=${inputParams.audioRemux}")
StreamingTranscoder.createManuallyForTesting(datasource, null, inputParams.videoCodec, inputParams.videoBitrate, inputParams.audioBitrate, inputParams.resolution, inputParams.audioRemux)
} else if (inputParams.transcodingPreset != null) {
StreamingTranscoder(datasource, null, inputParams.transcodingPreset, DEFAULT_FILE_SIZE_LIMIT, inputParams.audioRemux)
} else {
throw IllegalArgumentException("Improper input data! No TranscodingPreset defined, or invalid manual parameters!")
}
applicationContext.openFileOutput(tempFilename, Context.MODE_PRIVATE).use { outputStream ->
transcoder.transcode({ percent: Int ->
if (lastProgress != percent) {
lastProgress = percent
Log.v(TAG, "$logPrefix Updating progress percent to $percent%")
setProgressAsync(Data.Builder().putInt(KEY_PROGRESS, percent).build())
setForegroundAsync(createForegroundInfo(percent, inputParams.notificationId))
}
}, outputStream, { isStopped })
}
Log.v(TAG, "$logPrefix Initial transcode completed successfully!")
val finalFile = createFile(inputParams.outputDirUri, finalFilename) ?: run {
Log.w(TAG, "$logPrefix Could not create final file for faststart processing!")
return Result.failure()
}
if (!inputParams.postProcessForFastStart) {
applicationContext.openFileInput(tempFilename).use { tempFileStream ->
if (tempFileStream == null) {
Log.w(TAG, "$logPrefix Could not open temp file for I/O!")
return Result.failure()
}
applicationContext.contentResolver.openOutputStream(finalFile.uri, "w").use { finalFileStream ->
if (finalFileStream == null) {
Log.w(TAG, "$logPrefix Could not open output file for I/O!")
return Result.failure()
}
tempFileStream.copyTo(finalFileStream)
}
}
Log.v(TAG, "$logPrefix Rename successful.")
} else {
val tempFileLength: Long
applicationContext.openFileInput(tempFilename).use { tempFileStream ->
if (tempFileStream == null) {
Log.w(TAG, "$logPrefix Could not open temp file for I/O!")
return Result.failure()
}
tempFileLength = tempFileStream.readLength()
}
applicationContext.contentResolver.openOutputStream(finalFile.uri, "w").use { finalFileStream ->
if (finalFileStream == null) {
Log.w(TAG, "$logPrefix Could not open output file for I/O!")
return Result.failure()
}
val inputStreamFactory = { applicationContext.openFileInput(tempFilename) ?: throw IOException("Could not open temp file for reading!") }
val bytesCopied = Mp4FaststartPostProcessor(inputStreamFactory).processAndWriteTo(finalFileStream)
if (bytesCopied != tempFileLength) {
Log.w(TAG, "$logPrefix Postprocessing failed! Original transcoded filesize ($tempFileLength) did not match postprocessed filesize ($bytesCopied)")
return Result.failure()
}
Log.v(TAG, "$logPrefix Faststart postprocess successful.")
}
val tempFile = File(applicationContext.filesDir, tempFilename)
if (!tempFile.delete()) {
Log.w(TAG, "$logPrefix Failed to delete temp file after processing!")
return Result.failure()
}
}
Log.v(TAG, "$logPrefix Overall transcode job successful.")
return Result.success()
}
private fun createForegroundInfo(progress: Int, notificationId: Int): ForegroundInfo {
val id = applicationContext.getString(R.string.notification_channel_id)
val title = applicationContext.getString(R.string.notification_title)
val cancel = applicationContext.getString(R.string.cancel_transcode)
val intent = WorkManager.getInstance(applicationContext)
.createCancelPendingIntent(getId())
val transcodeActivityIntent = Intent(applicationContext, TranscodeTestActivity::class.java)
val pendingIntent: PendingIntent? = TaskStackBuilder.create(applicationContext).run {
addNextIntentWithParentStack(transcodeActivityIntent)
getPendingIntent(
0,
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
)
}
val notification = NotificationCompat.Builder(applicationContext, id)
.setContentTitle(title)
.setTicker(title)
.setProgress(100, progress, progress <= 0)
.setSmallIcon(R.drawable.ic_work_notification)
.setOngoing(true)
.setContentIntent(pendingIntent)
.setPriority(NotificationCompat.PRIORITY_LOW)
.addAction(android.R.drawable.ic_delete, cancel, intent)
.build()
return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
ForegroundInfo(notificationId, notification, FOREGROUND_SERVICE_TYPE_DATA_SYNC)
} else {
ForegroundInfo(notificationId, notification)
}
}
private fun createFile(treeUri: Uri, filename: String): DocumentFile? {
return DocumentFile.fromTreeUri(applicationContext, treeUri)?.createFile(VideoConstants.VIDEO_MIME_TYPE, filename)
}
private fun String.removeFileExtension(): String {
val lastDot = this.lastIndexOf('.')
return if (lastDot != -1) {
this.substring(0, lastDot)
} else {
this
}
}
private class WorkerMediaDataSource(private val file: File) : InputStreamMediaDataSource() {
private val size = file.length()
private var inputStream: InputStream? = null
override fun close() {
inputStream?.close()
}
override fun getSize(): Long {
return size
}
override fun createInputStream(position: Long): InputStream {
inputStream?.close()
val openedInputStream = FileInputStream(file)
openedInputStream.skip(position)
inputStream = openedInputStream
return openedInputStream
}
}
private data class InputParams(private val inputData: Data) {
val notificationId: Int = inputData.getInt(KEY_NOTIFICATION_ID, -1)
val inputUri: Uri = Uri.parse(inputData.getString(KEY_INPUT_URI))
val outputDirUri: Uri = Uri.parse(inputData.getString(KEY_OUTPUT_URI))
val postProcessForFastStart: Boolean = inputData.getBoolean(KEY_ENABLE_FASTSTART, true)
val transcodingPreset: TranscodingPreset? = inputData.getString(KEY_TRANSCODING_PRESET_NAME)?.let { TranscodingPreset.valueOf(it) }
@VideoCodec val videoCodec: String? = inputData.getString(KEY_VIDEO_CODEC)
val resolution: Int = inputData.getInt(KEY_SHORT_EDGE, -1)
val videoBitrate: Int = inputData.getInt(KEY_VIDEO_BIT_RATE, -1)
val audioBitrate: Int = inputData.getInt(KEY_AUDIO_BIT_RATE, -1)
val audioRemux: Boolean = inputData.getBoolean(KEY_ENABLE_AUDIO_REMUX, true)
}
companion object {
private const val TAG = "TranscodeWorker"
private const val OUTPUT_FILE_EXTENSION = ".mp4"
const val TEMP_FILE_EXTENSION = ".tmp"
private const val DEFAULT_FILE_SIZE_LIMIT: Long = 100 * 1024 * 1024
const val KEY_INPUT_URI = "input_uri"
const val KEY_OUTPUT_URI = "output_uri"
const val KEY_TRANSCODING_PRESET_NAME = "transcoding_quality_preset"
const val KEY_PROGRESS = "progress"
const val KEY_VIDEO_CODEC = "video_codec"
const val KEY_LONG_EDGE = "resolution_long_edge"
const val KEY_SHORT_EDGE = "resolution_short_edge"
const val KEY_VIDEO_BIT_RATE = "video_bit_rate"
const val KEY_AUDIO_BIT_RATE = "audio_bit_rate"
const val KEY_ENABLE_AUDIO_REMUX = "audio_remux"
const val KEY_ENABLE_FASTSTART = "video_enable_faststart"
const val KEY_NOTIFICATION_ID = "notification_id"
}
}

325
video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/ConfigurationSelection.kt Normal file
View file

@ -0,0 +1,325 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.video.app.transcode.composables
import android.net.Uri
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.selection.selectable
import androidx.compose.foundation.selection.selectableGroup
import androidx.compose.material3.Checkbox
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.RadioButton
import androidx.compose.material3.Slider
import androidx.compose.material3.SliderDefaults
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.semantics.Role
import androidx.compose.ui.semantics.contentDescription
import androidx.compose.ui.semantics.semantics
import androidx.compose.ui.text.font.FontFamily
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.tooling.preview.Preview
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import androidx.lifecycle.viewmodel.compose.viewModel
import org.thoughtcrime.securesms.video.TranscodingPreset
import org.thoughtcrime.securesms.video.videoconverter.utils.DeviceCapabilities
import org.thoughtcrime.video.app.transcode.MAX_VIDEO_MEGABITRATE
import org.thoughtcrime.video.app.transcode.MIN_VIDEO_MEGABITRATE
import org.thoughtcrime.video.app.transcode.OPTIONS_AUDIO_KILOBITRATES
import org.thoughtcrime.video.app.transcode.TranscodeTestViewModel
import org.thoughtcrime.video.app.transcode.VideoResolution
import org.thoughtcrime.video.app.ui.composables.LabeledButton
import kotlin.math.roundToInt
/**
* A view that shows the queue of video URIs to encode, and allows you to change the encoding options.
*/
@Composable
fun ConfigureEncodingParameters(
hevcCapable: Boolean = DeviceCapabilities.canEncodeHevc(),
modifier: Modifier = Modifier,
viewModel: TranscodeTestViewModel = viewModel()
) {
Column(
verticalArrangement = Arrangement.Center,
horizontalAlignment = Alignment.CenterHorizontally,
modifier = modifier
) {
Text(
text = "Selected videos:",
modifier = Modifier
.padding(horizontal = 8.dp)
.align(Alignment.Start)
)
viewModel.selectedVideos.forEach {
Text(
text = it.toString(),
fontSize = 8.sp,
fontFamily = FontFamily.Monospace,
modifier = Modifier
.padding(horizontal = 8.dp)
.align(Alignment.Start)
)
}
Row(
verticalAlignment = Alignment.CenterVertically,
modifier = Modifier
.fillMaxWidth()
) {
Checkbox(
checked = viewModel.forceSequentialQueueProcessing,
onCheckedChange = { viewModel.forceSequentialQueueProcessing = it }
)
Text(text = "Force Sequential Queue Processing", style = MaterialTheme.typography.bodySmall)
}
Row(
verticalAlignment = Alignment.CenterVertically,
modifier = Modifier
.fillMaxWidth()
) {
Checkbox(
checked = viewModel.useAutoTranscodingSettings,
onCheckedChange = { viewModel.useAutoTranscodingSettings = it }
)
Text(
text = "Match Signal App Transcoding Settings",
style = MaterialTheme.typography.bodySmall
)
}
if (viewModel.useAutoTranscodingSettings) {
PresetPicker(
viewModel.transcodingPreset,
viewModel::updateTranscodingPreset,
modifier = Modifier.padding(vertical = 16.dp)
)
} else {
CustomSettings(
selectedResolution = viewModel.videoResolution,
onResolutionSelected = { viewModel.videoResolution = it },
useHevc = viewModel.useHevc,
onUseHevcSettingChanged = { viewModel.useHevc = it },
fastStartChecked = viewModel.enableFastStart,
onFastStartSettingCheckChanged = { viewModel.enableFastStart = it },
audioRemuxChecked = viewModel.enableAudioRemux,
onAudioRemuxCheckChanged = { viewModel.enableAudioRemux = it },
videoSliderPosition = viewModel.videoMegaBitrate,
updateVideoSliderPosition = { viewModel.videoMegaBitrate = it },
audioSliderPosition = viewModel.audioKiloBitrate,
updateAudioSliderPosition = { viewModel.audioKiloBitrate = it.roundToInt() },
hevcCapable = hevcCapable,
modifier = Modifier.padding(vertical = 16.dp)
)
}
LabeledButton(
buttonLabel = "Transcode",
onClick = {
viewModel.transcode()
viewModel.selectedVideos = emptyList()
viewModel.resetOutputDirectory()
},
modifier = Modifier.padding(vertical = 8.dp)
)
}
}
@Composable
private fun PresetPicker(
selectedTranscodingPreset: TranscodingPreset,
onPresetSelected: (TranscodingPreset) -> Unit,
modifier: Modifier = Modifier
) {
Row(
horizontalArrangement = Arrangement.SpaceEvenly,
modifier = modifier
.fillMaxWidth()
.selectableGroup()
) {
TranscodingPreset.entries.forEach {
Column(
horizontalAlignment = Alignment.CenterHorizontally,
modifier = Modifier
.selectable(
selected = selectedTranscodingPreset == it,
onClick = {
onPresetSelected(it)
},
role = Role.RadioButton
)
) {
RadioButton(
selected = selectedTranscodingPreset == it,
onClick = null,
modifier = Modifier.semantics { contentDescription = it.name }
)
Text(
text = it.name,
textAlign = TextAlign.Center
)
}
}
}
}
@Composable
private fun CustomSettings(
selectedResolution: VideoResolution,
onResolutionSelected: (VideoResolution) -> Unit,
useHevc: Boolean,
onUseHevcSettingChanged: (Boolean) -> Unit,
fastStartChecked: Boolean,
onFastStartSettingCheckChanged: (Boolean) -> Unit,
audioRemuxChecked: Boolean,
onAudioRemuxCheckChanged: (Boolean) -> Unit,
videoSliderPosition: Float,
updateVideoSliderPosition: (Float) -> Unit,
audioSliderPosition: Int,
updateAudioSliderPosition: (Float) -> Unit,
hevcCapable: Boolean,
modifier: Modifier = Modifier
) {
Row(
horizontalArrangement = Arrangement.SpaceEvenly,
modifier = modifier
.fillMaxWidth()
.selectableGroup()
) {
VideoResolution.entries.forEach {
Column(
horizontalAlignment = Alignment.CenterHorizontally,
modifier = Modifier
.selectable(
selected = selectedResolution == it,
onClick = { onResolutionSelected(it) },
role = Role.RadioButton
)
.padding(start = 16.dp)
) {
RadioButton(
selected = selectedResolution == it,
onClick = null,
modifier = Modifier.semantics { contentDescription = it.getContentDescription() }
)
Text(
text = "${it.shortEdge}p",
textAlign = TextAlign.Center
)
}
}
}
VideoBitrateSlider(videoSliderPosition, updateVideoSliderPosition)
AudioBitrateSlider(audioSliderPosition, updateAudioSliderPosition)
if (hevcCapable) {
Row(
verticalAlignment = Alignment.CenterVertically,
modifier = Modifier
.padding(vertical = 8.dp, horizontal = 8.dp)
.fillMaxWidth()
) {
Checkbox(
checked = useHevc,
onCheckedChange = { onUseHevcSettingChanged(it) }
)
Text(text = "Use HEVC encoder", style = MaterialTheme.typography.bodySmall)
}
}
Row(
verticalAlignment = Alignment.CenterVertically,
modifier = Modifier
.padding(vertical = 8.dp, horizontal = 8.dp)
.fillMaxWidth()
) {
Checkbox(
checked = audioRemuxChecked,
onCheckedChange = { onAudioRemuxCheckChanged(it) }
)
Text(text = "Allow audio remuxing", style = MaterialTheme.typography.bodySmall)
}
Row(
verticalAlignment = Alignment.CenterVertically,
modifier = Modifier
.padding(vertical = 8.dp, horizontal = 8.dp)
.fillMaxWidth()
) {
Checkbox(
checked = fastStartChecked,
onCheckedChange = { onFastStartSettingCheckChanged(it) }
)
Text(text = "Perform Mp4San Postprocessing", style = MaterialTheme.typography.bodySmall)
}
}
@Composable
private fun VideoBitrateSlider(
videoSliderPosition: Float,
updateSliderPosition: (Float) -> Unit,
modifier: Modifier = Modifier
) {
Slider(
value = videoSliderPosition,
onValueChange = updateSliderPosition,
colors = SliderDefaults.colors(
thumbColor = MaterialTheme.colorScheme.secondary,
activeTrackColor = MaterialTheme.colorScheme.secondary,
inactiveTrackColor = MaterialTheme.colorScheme.secondaryContainer
),
valueRange = MIN_VIDEO_MEGABITRATE..MAX_VIDEO_MEGABITRATE,
modifier = modifier.padding(top = 16.dp, start = 16.dp, end = 16.dp)
)
Text(text = String.format("Video: %.2f Mbit/s", videoSliderPosition))
}
@Composable
private fun AudioBitrateSlider(
audioSliderPosition: Int,
updateSliderPosition: (Float) -> Unit,
modifier: Modifier = Modifier
) {
val minValue = OPTIONS_AUDIO_KILOBITRATES.first().toFloat()
val maxValue = OPTIONS_AUDIO_KILOBITRATES.last().toFloat()
val steps = OPTIONS_AUDIO_KILOBITRATES.size - 2
Slider(
value = audioSliderPosition.toFloat(),
onValueChange = updateSliderPosition,
colors = SliderDefaults.colors(
thumbColor = MaterialTheme.colorScheme.secondary,
activeTrackColor = MaterialTheme.colorScheme.secondary,
inactiveTrackColor = MaterialTheme.colorScheme.secondaryContainer
),
valueRange = minValue..maxValue,
steps = steps,
modifier = modifier.padding(top = 16.dp, start = 16.dp, end = 16.dp)
)
Text(text = String.format("Audio: %d Kbit/s", audioSliderPosition))
}
@Preview(showBackground = true)
@Composable
private fun ConfigurationScreenPreviewChecked() {
val vm: TranscodeTestViewModel = viewModel()
vm.selectedVideos = listOf(Uri.parse("content://1"), Uri.parse("content://2"))
vm.forceSequentialQueueProcessing = true
ConfigureEncodingParameters()
}
@Preview(showBackground = true)
@Composable
private fun ConfigurationScreenPreviewUnchecked() {
val vm: TranscodeTestViewModel = viewModel()
vm.selectedVideos = listOf(Uri.parse("content://1"), Uri.parse("content://2"))
vm.useAutoTranscodingSettings = false
ConfigureEncodingParameters(hevcCapable = true)
}

33
video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/InputSelection.kt Normal file
View file

@ -0,0 +1,33 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.video.app.transcode.composables
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.runtime.Composable
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.tooling.preview.Preview
import org.thoughtcrime.video.app.ui.composables.LabeledButton
/**
* A view that prompts you to select input videos for transcoding.
*/
@Composable
fun SelectInput(modifier: Modifier = Modifier, onClick: () -> Unit) {
Column(
verticalArrangement = Arrangement.Center,
horizontalAlignment = Alignment.CenterHorizontally
) {
LabeledButton("Select Videos", onClick = onClick, modifier = modifier)
}
}
@Preview
@Composable
private fun InputSelectionPreview() {
SelectInput { }
}

33
video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/OutputSelection.kt Normal file
View file

@ -0,0 +1,33 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.video.app.transcode.composables
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.runtime.Composable
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.tooling.preview.Preview
import org.thoughtcrime.video.app.ui.composables.LabeledButton
/**
* A view that prompts you to select an output directory that transcoded videos will be saved to.
*/
@Composable
fun SelectOutput(modifier: Modifier = Modifier, onClick: () -> Unit) {
Column(
verticalArrangement = Arrangement.Center,
horizontalAlignment = Alignment.CenterHorizontally
) {
LabeledButton("Select Output Directory", onClick = onClick, modifier = modifier)
}
}
@Preview
@Composable
private fun OutputSelectionPreview() {
SelectOutput { }
}

78
video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/Progress.kt Normal file
View file

@ -0,0 +1,78 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.video.app.transcode.composables
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.padding
import androidx.compose.material3.LinearProgressIndicator
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.tooling.preview.Preview
import androidx.compose.ui.unit.dp
import androidx.work.WorkInfo
import org.thoughtcrime.video.app.transcode.TranscodeWorker
import org.thoughtcrime.video.app.ui.composables.LabeledButton
/**
* A view that shows the current encodes in progress.
*/
@Composable
fun TranscodingJobProgress(transcodingJobs: List<WorkState>, resetButtonOnClick: () -> Unit, modifier: Modifier = Modifier) {
Column(
verticalArrangement = Arrangement.Center,
horizontalAlignment = Alignment.CenterHorizontally
) {
transcodingJobs.forEach { workInfo ->
val currentProgress = workInfo.progress
Row(
verticalAlignment = Alignment.CenterVertically,
modifier = modifier.padding(horizontal = 16.dp)
) {
val progressIndicatorModifier = Modifier.weight(3f)
Text(
text = "Job ${workInfo.id.takeLast(4)}",
modifier = Modifier
.padding(end = 16.dp)
.weight(1f)
)
if (workInfo.state.isFinished) {
Text(text = workInfo.state.toString(), textAlign = TextAlign.Center, modifier = progressIndicatorModifier)
} else if (currentProgress >= 0) {
LinearProgressIndicator(progress = currentProgress / 100f, modifier = progressIndicatorModifier)
} else {
LinearProgressIndicator(modifier = progressIndicatorModifier)
}
}
}
LabeledButton("Reset/Cancel", onClick = resetButtonOnClick)
}
}
data class WorkState(val id: String, val state: WorkInfo.State, val progress: Int) {
companion object {
fun fromInfo(info: WorkInfo): WorkState {
return WorkState(info.id.toString(), info.state, info.progress.getInt(TranscodeWorker.KEY_PROGRESS, -1))
}
}
}
@Preview
@Composable
private fun ProgressScreenPreview() {
TranscodingJobProgress(
listOf(
WorkState("abcde", WorkInfo.State.RUNNING, 47),
WorkState("fghij", WorkInfo.State.ENQUEUED, -1),
WorkState("klmnop", WorkInfo.State.FAILED, -1)
),
resetButtonOnClick = {}
)
}

18
video/app/src/main/java/org/thoughtcrime/video/app/ui/composables/LabeledButton.kt Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.video.app.ui.composables
import androidx.compose.material3.Button
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.ui.Modifier
@Composable
fun LabeledButton(buttonLabel: String, modifier: Modifier = Modifier, onClick: () -> Unit) {
Button(onClick = onClick, modifier = modifier) {
Text(buttonLabel)
}
}

11
video/app/src/main/java/org/thoughtcrime/video/app/ui/theme/Color.kt Normal file
View file

@ -0,0 +1,11 @@
package org.thoughtcrime.video.app.ui.theme
import androidx.compose.ui.graphics.Color
val Purple80 = Color(0xFFD0BCFF)
val PurpleGrey80 = Color(0xFFCCC2DC)
val Pink80 = Color(0xFFEFB8C8)
val Purple40 = Color(0xFF6650a4)
val PurpleGrey40 = Color(0xFF625b71)
val Pink40 = Color(0xFF7D5260)

75
video/app/src/main/java/org/thoughtcrime/video/app/ui/theme/Theme.kt Normal file
View file

@ -0,0 +1,75 @@
/*
* Copyright 2023 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.video.app.ui.theme
import android.app.Activity
import android.os.Build
import androidx.compose.foundation.isSystemInDarkTheme
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.darkColorScheme
import androidx.compose.material3.dynamicDarkColorScheme
import androidx.compose.material3.dynamicLightColorScheme
import androidx.compose.material3.lightColorScheme
import androidx.compose.runtime.Composable
import androidx.compose.runtime.SideEffect
import androidx.compose.ui.graphics.toArgb
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.platform.LocalView
import androidx.core.view.WindowCompat
private val DarkColorScheme = darkColorScheme(
primary = Purple80,
secondary = PurpleGrey80,
tertiary = Pink80
)
private val LightColorScheme = lightColorScheme(
primary = Purple40,
secondary = PurpleGrey40,
tertiary = Pink40
/* Other default colors to override
background = Color(0xFFFFFBFE),
surface = Color(0xFFFFFBFE),
onPrimary = Color.White,
onSecondary = Color.White,
onTertiary = Color.White,
onBackground = Color(0xFF1C1B1F),
onSurface = Color(0xFF1C1B1F),
*/
)
@Composable
fun SignalTheme(
darkTheme: Boolean = isSystemInDarkTheme(),
// Dynamic color is available on Android 12+
dynamicColor: Boolean = true,
content: @Composable () -> Unit
) {
val colorScheme = when {
dynamicColor && Build.VERSION.SDK_INT >= Build.VERSION_CODES.S -> {
val context = LocalContext.current
if (darkTheme) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context)
}
darkTheme -> DarkColorScheme
else -> LightColorScheme
}
val view = LocalView.current
if (!view.isInEditMode) {
SideEffect {
val window = (view.context as Activity).window
window.statusBarColor = colorScheme.primary.toArgb()
WindowCompat.getInsetsController(window, view).isAppearanceLightStatusBars = darkTheme
}
}
MaterialTheme(
colorScheme = colorScheme,
typography = Typography,
content = content
)
}

39
video/app/src/main/java/org/thoughtcrime/video/app/ui/theme/Type.kt Normal file
View file

@ -0,0 +1,39 @@
/*
* Copyright 2023 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.video.app.ui.theme
import androidx.compose.material3.Typography
import androidx.compose.ui.text.TextStyle
import androidx.compose.ui.text.font.FontFamily
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.unit.sp
// Set of Material typography styles to start with
val Typography = Typography(
bodyLarge = TextStyle(
fontFamily = FontFamily.Default,
fontWeight = FontWeight.Normal,
fontSize = 16.sp,
lineHeight = 24.sp,
letterSpacing = 0.5.sp
)
/* Other default text styles to override
titleLarge = TextStyle(
fontFamily = FontFamily.Default,
fontWeight = FontWeight.Normal,
fontSize = 22.sp,
lineHeight = 28.sp,
letterSpacing = 0.sp
),
labelSmall = TextStyle(
fontFamily = FontFamily.Default,
fontWeight = FontWeight.Medium,
fontSize = 11.sp,
lineHeight = 16.sp,
letterSpacing = 0.5.sp
)
*/
)

15
video/app/src/main/res/drawable-anydpi-v24/ic_work_notification.xml Normal file
View file

@ -0,0 +1,15 @@
<vector xmlns:android="http://schemas.android.com/apk/res/android"
android:width="24dp"
android:height="24dp"
android:viewportWidth="24"
android:viewportHeight="24"
android:tint="#FFFFFF">
<group android:scaleX="0.92"
android:scaleY="0.92"
android:translateX="0.96"
android:translateY="0.96">
<path
android:pathData="M17.6,11.48 L19.44,8.3a0.63,0.63 0,0 0,-1.09 -0.63l-1.88,3.24a11.43,11.43 0,0 0,-8.94 0L5.65,7.67a0.63,0.63 0,0 0,-1.09 0.63L6.4,11.48A10.81,10.81 0,0 0,1 20L23,20A10.81,10.81 0,0 0,17.6 11.48ZM7,17.25A1.25,1.25 0,1 1,8.25 16,1.25 1.25,0 0,1 7,17.25ZM17,17.25A1.25,1.25 0,1 1,18.25 16,1.25 1.25,0 0,1 17,17.25Z"
android:fillColor="#FF000000"/>
</group>
</vector>

BIN
video/app/src/main/res/drawable-hdpi/ic_work_notification.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 400 B

BIN
video/app/src/main/res/drawable-mdpi/ic_work_notification.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 263 B

35
video/app/src/main/res/drawable-v24/ic_launcher_foreground.xml Normal file
View file

@ -0,0 +1,35 @@
<!--
~ Copyright 2023 Signal Messenger, LLC
~ SPDX-License-Identifier: AGPL-3.0-only
-->
<vector xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:aapt="http://schemas.android.com/aapt"
android:width="108dp"
android:height="108dp"
android:viewportWidth="108"
android:viewportHeight="108">
<path android:pathData="M31,63.928c0,0 6.4,-11 12.1,-13.1c7.2,-2.6 26,-1.4 26,-1.4l38.1,38.1L107,108.928l-32,-1L31,63.928z">
<aapt:attr name="android:fillColor">
<gradient
android:endX="85.84757"
android:endY="92.4963"
android:startX="42.9492"
android:startY="49.59793"
android:type="linear">
<item
android:color="#44000000"
android:offset="0.0" />
<item
android:color="#00000000"
android:offset="1.0" />
</gradient>
</aapt:attr>
</path>
<path
android:fillColor="#FFFFFF"
android:fillType="nonZero"
android:pathData="M65.3,45.828l3.8,-6.6c0.2,-0.4 0.1,-0.9 -0.3,-1.1c-0.4,-0.2 -0.9,-0.1 -1.1,0.3l-3.9,6.7c-6.3,-2.8 -13.4,-2.8 -19.7,0l-3.9,-6.7c-0.2,-0.4 -0.7,-0.5 -1.1,-0.3C38.8,38.328 38.7,38.828 38.9,39.228l3.8,6.6C36.2,49.428 31.7,56.028 31,63.928h46C76.3,56.028 71.8,49.428 65.3,45.828zM43.4,57.328c-0.8,0 -1.5,-0.5 -1.8,-1.2c-0.3,-0.7 -0.1,-1.5 0.4,-2.1c0.5,-0.5 1.4,-0.7 2.1,-0.4c0.7,0.3 1.2,1 1.2,1.8C45.3,56.528 44.5,57.328 43.4,57.328L43.4,57.328zM64.6,57.328c-0.8,0 -1.5,-0.5 -1.8,-1.2s-0.1,-1.5 0.4,-2.1c0.5,-0.5 1.4,-0.7 2.1,-0.4c0.7,0.3 1.2,1 1.2,1.8C66.5,56.528 65.6,57.328 64.6,57.328L64.6,57.328z"
android:strokeWidth="1"
android:strokeColor="#00000000" />
</vector>

BIN
video/app/src/main/res/drawable-xhdpi/ic_work_notification.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 508 B

BIN
video/app/src/main/res/drawable-xxhdpi/ic_work_notification.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 769 B

175
video/app/src/main/res/drawable/ic_launcher_background.xml Normal file
View file

@ -0,0 +1,175 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
~ Copyright 2023 Signal Messenger, LLC
~ SPDX-License-Identifier: AGPL-3.0-only
-->
<vector xmlns:android="http://schemas.android.com/apk/res/android"
android:width="108dp"
android:height="108dp"
android:viewportWidth="108"
android:viewportHeight="108">
<path
android:fillColor="#3DDC84"
android:pathData="M0,0h108v108h-108z" />
<path
android:fillColor="#00000000"
android:pathData="M9,0L9,108"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M19,0L19,108"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M29,0L29,108"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M39,0L39,108"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M49,0L49,108"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M59,0L59,108"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M69,0L69,108"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M79,0L79,108"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M89,0L89,108"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M99,0L99,108"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M0,9L108,9"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M0,19L108,19"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M0,29L108,29"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M0,39L108,39"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M0,49L108,49"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M0,59L108,59"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M0,69L108,69"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M0,79L108,79"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M0,89L108,89"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M0,99L108,99"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M19,29L89,29"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M19,39L89,39"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M19,49L89,49"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M19,59L89,59"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M19,69L89,69"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M19,79L89,79"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M29,19L29,89"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M39,19L39,89"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M49,19L49,89"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M59,19L59,89"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M69,19L69,89"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
<path
android:fillColor="#00000000"
android:pathData="M79,19L79,89"
android:strokeWidth="0.8"
android:strokeColor="#33FFFFFF" />
</vector>

11
video/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml Normal file
View file

@ -0,0 +1,11 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
~ Copyright 2023 Signal Messenger, LLC
~ SPDX-License-Identifier: AGPL-3.0-only
-->
<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
<background android:drawable="@drawable/ic_launcher_background" />
<foreground android:drawable="@drawable/ic_launcher_foreground" />
<monochrome android:drawable="@drawable/ic_launcher_foreground" />
</adaptive-icon>

11
video/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml Normal file
View file

@ -0,0 +1,11 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
~ Copyright 2023 Signal Messenger, LLC
~ SPDX-License-Identifier: AGPL-3.0-only
-->
<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
<background android:drawable="@drawable/ic_launcher_background" />
<foreground android:drawable="@drawable/ic_launcher_foreground" />
<monochrome android:drawable="@drawable/ic_launcher_foreground" />
</adaptive-icon>

BIN
video/app/src/main/res/mipmap-hdpi/ic_launcher.webp Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1.4 KiB

BIN
video/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 2.8 KiB

BIN
video/app/src/main/res/mipmap-mdpi/ic_launcher.webp Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 982 B

BIN
video/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1.7 KiB

BIN
video/app/src/main/res/mipmap-xhdpi/ic_launcher.webp Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1.9 KiB

BIN
video/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 3.8 KiB

BIN
video/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 2.8 KiB

BIN
video/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 5.8 KiB

BIN
video/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 3.8 KiB

BIN
video/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 7.6 KiB

15
video/app/src/main/res/values/colors.xml Normal file
View file

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
~ Copyright 2023 Signal Messenger, LLC
~ SPDX-License-Identifier: AGPL-3.0-only
-->
<resources>
<color name="purple_200">#FFBB86FC</color>
<color name="purple_500">#FF6200EE</color>
<color name="purple_700">#FF3700B3</color>
<color name="teal_200">#FF03DAC5</color>
<color name="teal_700">#FF018786</color>
<color name="black">#FF000000</color>
<color name="white">#FFFFFFFF</color>
</resources>

15
video/app/src/main/res/values/strings.xml Normal file
View file

@ -0,0 +1,15 @@
<!--
~ Copyright 2023 Signal Messenger, LLC
~ SPDX-License-Identifier: AGPL-3.0-only
-->
<resources>
<string name="app_name">Video Framework Tester</string>
<string name="notification_channel_id">transcode-progress</string>
<string name="notification_title">Encoding video…</string>
<string name="cancel_transcode">Cancel</string>
<string name="channel_name">Transcoding progress updates.</string>
<string name="channel_description">Persistent notifications that allow the transcode job to complete when the app is in the background.</string>
<string name="preference_file_key">settings</string>
<string name="preference_activity_shortcut_key">activity_shortcut</string>
</resources>

10
video/app/src/main/res/values/themes.xml Normal file
View file

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
~ Copyright 2023 Signal Messenger, LLC
~ SPDX-License-Identifier: AGPL-3.0-only
-->
<resources>
<style name="Theme.Signal" parent="Theme.AppCompat.DayNight" />
</resources>

21
video/app/src/test/java/org/thoughtcrime/video/app/ExampleUnitTest.kt Normal file
View file

@ -0,0 +1,21 @@
/*
* Copyright 2023 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.video.app
import org.junit.Assert.assertEquals
import org.junit.Test
/**
* Example local unit test, which will execute on the development machine (host).
*
* See [testing documentation](http://d.android.com/tools/testing).
*/
class ExampleUnitTest {
@Test
fun addition_isCorrect() {
assertEquals(4, 2 + 2)
}
}

17
video/lib/build.gradle.kts Normal file
View file

@ -0,0 +1,17 @@
plugins {
id("signal-library")
}
android {
namespace = "org.signal.video"
}
dependencies {
implementation(project(":core-util"))
implementation(libs.libsignal.android)
implementation(libs.google.guava.android)
implementation(libs.bundles.mp4parser) {
exclude(group = "junit", module = "junit")
}
}

2
video/lib/src/main/AndroidManifest.xml Normal file
View file

@ -0,0 +1,2 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest />

285
video/lib/src/main/java/org/thoughtcrime/securesms/video/StreamingTranscoder.java Normal file
View file

@ -0,0 +1,285 @@
package org.thoughtcrime.securesms.video;
import android.media.MediaDataSource;
import android.media.MediaMetadataRetriever;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.annotation.VisibleForTesting;
import com.google.common.io.CountingOutputStream;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.video.exceptions.VideoSizeException;
import org.thoughtcrime.securesms.video.exceptions.VideoSourceException;
import org.thoughtcrime.securesms.video.interfaces.TranscoderCancelationSignal;
import org.thoughtcrime.securesms.video.videoconverter.MediaConverter;
import org.thoughtcrime.securesms.video.videoconverter.exceptions.EncodingException;
import org.thoughtcrime.securesms.video.videoconverter.mediadatasource.MediaDataSourceMediaInput;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.text.NumberFormat;
import java.util.Locale;
import java.util.concurrent.TimeUnit;
@RequiresApi(26)
public final class StreamingTranscoder {
private static final String TAG = Log.tag(StreamingTranscoder.class);
private final MediaDataSource dataSource;
private final long upperSizeLimit;
private final long inSize;
private final long duration;
private final int inputBitRate;
private final TranscodingQuality targetQuality;
private final boolean transcodeRequired;
private final long fileSizeEstimate;
private final @Nullable TranscoderOptions options;
private final boolean allowAudioRemux;
/**
* @param upperSizeLimit A upper size to transcode to. The actual output size can be up to 10% smaller.
*/
public StreamingTranscoder(@NonNull MediaDataSource dataSource,
@Nullable TranscoderOptions options,
@NonNull TranscodingPreset preset,
long upperSizeLimit,
boolean allowAudioRemux)
throws IOException, VideoSourceException
{
this.dataSource = dataSource;
this.options = options;
this.allowAudioRemux = allowAudioRemux;
final MediaMetadataRetriever mediaMetadataRetriever = new MediaMetadataRetriever();
try {
mediaMetadataRetriever.setDataSource(dataSource);
} catch (RuntimeException e) {
Log.w(TAG, "Unable to read datasource", e);
throw new VideoSourceException("Unable to read datasource", e);
}
if (options != null && options.endTimeUs != 0) {
this.duration = TimeUnit.MICROSECONDS.toMillis(options.endTimeUs - options.startTimeUs);
} else {
this.duration = getDuration(mediaMetadataRetriever);
}
this.inSize = dataSource.getSize();
this.inputBitRate = TranscodingQuality.bitRate(inSize, duration);
this.targetQuality = TranscodingQuality.createFromPreset(preset, duration);
this.upperSizeLimit = upperSizeLimit;
this.transcodeRequired = inputBitRate >= targetQuality.getTargetTotalBitRate() * 1.2 || inSize > upperSizeLimit || containsLocation(mediaMetadataRetriever) || options != null;
if (!transcodeRequired) {
Log.i(TAG, "Video is within 20% of target bitrate, below the size limit, contained no location metadata or custom options.");
}
this.fileSizeEstimate = targetQuality.getByteCountEstimate();
}
private StreamingTranscoder(@NonNull MediaDataSource dataSource,
@Nullable TranscoderOptions options,
String codec,
int videoBitrate,
int audioBitrate,
int shortEdge,
boolean allowAudioRemux)
throws IOException, VideoSourceException
{
this.dataSource = dataSource;
this.options = options;
this.allowAudioRemux = allowAudioRemux;
final MediaMetadataRetriever mediaMetadataRetriever = new MediaMetadataRetriever();
try {
mediaMetadataRetriever.setDataSource(dataSource);
} catch (RuntimeException e) {
Log.w(TAG, "Unable to read datasource", e);
throw new VideoSourceException("Unable to read datasource", e);
}
this.inSize = dataSource.getSize();
this.duration = getDuration(mediaMetadataRetriever);
this.inputBitRate = TranscodingQuality.bitRate(inSize, duration);
this.targetQuality = TranscodingQuality.createManuallyForTesting(codec, shortEdge, videoBitrate, audioBitrate, duration);
this.upperSizeLimit = 0L;
this.transcodeRequired = true;
this.fileSizeEstimate = targetQuality.getByteCountEstimate();
}
@VisibleForTesting
public static StreamingTranscoder createManuallyForTesting(@NonNull MediaDataSource dataSource,
@Nullable TranscoderOptions options,
@NonNull @MediaConverter.VideoCodec String codec,
int videoBitrate,
int audioBitrate,
int shortEdge,
boolean allowAudioRemux)
throws VideoSourceException, IOException
{
return new StreamingTranscoder(dataSource, options, codec, videoBitrate, audioBitrate, shortEdge, allowAudioRemux);
}
/**
* @return The total content size of the MP4 mdat box.
*/
public long transcode(@NonNull Progress progress,
@NonNull OutputStream stream,
@Nullable TranscoderCancelationSignal cancelationSignal)
throws IOException, EncodingException
{
float durationSec = duration / 1000f;
NumberFormat numberFormat = NumberFormat.getInstance(Locale.US);
Log.i(TAG, String.format(Locale.US,
"Transcoding:\n" +
"Target bitrate : %s + %s = %s\n" +
"Target format : %dp\n" +
"Video duration : %.1fs\n" +
"Size limit : %s kB\n" +
"Estimate : %s kB\n" +
"Input size : %s kB\n" +
"Input bitrate : %s bps",
numberFormat.format(targetQuality.getTargetVideoBitRate()),
numberFormat.format(targetQuality.getTargetAudioBitRate()),
numberFormat.format(targetQuality.getTargetTotalBitRate()),
targetQuality.getOutputResolution(),
durationSec,
numberFormat.format(upperSizeLimit / 1024),
numberFormat.format(fileSizeEstimate / 1024),
numberFormat.format(inSize / 1024),
numberFormat.format(inputBitRate)));
final boolean sizeLimitEnabled = 0 < upperSizeLimit;
if (sizeLimitEnabled && upperSizeLimit < fileSizeEstimate) {
throw new VideoSizeException("Size constraints could not be met!");
}
final long startTime = System.currentTimeMillis();
final MediaConverter converter = new MediaConverter();
converter.setInput(new MediaDataSourceMediaInput(dataSource));
final CountingOutputStream outStream;
if (sizeLimitEnabled) {
outStream = new CountingOutputStream(new LimitedSizeOutputStream(stream, upperSizeLimit));
} else {
outStream = new CountingOutputStream(stream);
}
converter.setOutput(outStream);
converter.setVideoCodec(targetQuality.getCodec());
converter.setVideoResolution(targetQuality.getOutputResolution());
converter.setVideoBitrate(targetQuality.getTargetVideoBitRate());
converter.setAudioBitrate(targetQuality.getTargetAudioBitRate());
converter.setAllowAudioRemux(allowAudioRemux);
if (options != null) {
if (options.endTimeUs > 0) {
long timeFrom = options.startTimeUs / 1000;
long timeTo = options.endTimeUs / 1000;
converter.setTimeRange(timeFrom, timeTo);
Log.i(TAG, String.format(Locale.US, "Trimming:\nTotal duration: %d\nKeeping: %d..%d\nFinal duration:(%d)", duration, timeFrom, timeTo, timeTo - timeFrom));
}
}
converter.setListener(percent -> {
progress.onProgress(percent);
return cancelationSignal != null && cancelationSignal.isCanceled();
});
long mdatSize = converter.convert();
long outSize = outStream.getCount();
float encodeDurationSec = (System.currentTimeMillis() - startTime) / 1000f;
Log.i(TAG, String.format(Locale.US,
"Transcoding complete:\n" +
"Transcode time : %.1fs (%.1fx)\n" +
"Output size : %s kB\n" +
" of Original : %.1f%%\n" +
" of Estimate : %.1f%%\n" +
"Output bitrate : %s bps",
encodeDurationSec,
durationSec / encodeDurationSec,
numberFormat.format(outSize / 1024),
(outSize * 100d) / inSize,
(outSize * 100d) / fileSizeEstimate,
numberFormat.format(TranscodingQuality.bitRate(outSize, duration))));
if (sizeLimitEnabled && outSize > upperSizeLimit) {
throw new VideoSizeException("Size constraints could not be met!");
}
stream.flush();
return mdatSize;
}
public boolean isTranscodeRequired() {
return transcodeRequired;
}
private static long getDuration(MediaMetadataRetriever mediaMetadataRetriever) throws VideoSourceException {
String durationString = mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION);
if (durationString == null) {
throw new VideoSourceException("Cannot determine duration of video, null meta data");
}
try {
long duration = Long.parseLong(durationString);
if (duration <= 0) {
throw new VideoSourceException("Cannot determine duration of video, meta data: " + durationString);
}
return duration;
} catch (NumberFormatException e) {
throw new VideoSourceException("Cannot determine duration of video, meta data: " + durationString, e);
}
}
private static boolean containsLocation(MediaMetadataRetriever mediaMetadataRetriever) {
String locationString = mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_LOCATION);
return locationString != null;
}
public interface Progress {
void onProgress(int percent);
}
private static class LimitedSizeOutputStream extends FilterOutputStream {
private final long sizeLimit;
private long written;
LimitedSizeOutputStream(@NonNull OutputStream inner, long sizeLimit) {
super(inner);
this.sizeLimit = sizeLimit;
}
@Override public void write(int b) throws IOException {
incWritten(1);
out.write(b);
}
@Override public void write(byte[] b, int off, int len) throws IOException {
incWritten(len);
out.write(b, off, len);
}
private void incWritten(int len) throws IOException {
long newWritten = written + len;
if (newWritten > sizeLimit) {
Log.w(TAG, String.format(Locale.US, "File size limit hit. Wrote %d, tried to write %d more. Limit is %d", written, len, sizeLimit));
throw new VideoSizeException("File size limit hit");
}
written = newWritten;
}
}
}

3
video/lib/src/main/java/org/thoughtcrime/securesms/video/TranscoderOptions.kt Normal file
View file

@ -0,0 +1,3 @@
package org.thoughtcrime.securesms.video
data class TranscoderOptions(@JvmField val startTimeUs: Long, @JvmField val endTimeUs: Long)

54
video/lib/src/main/java/org/thoughtcrime/securesms/video/TranscodingQuality.kt Normal file
View file

@ -0,0 +1,54 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.video
import org.thoughtcrime.securesms.video.videoconverter.MediaConverter
import org.thoughtcrime.securesms.video.videoconverter.MediaConverter.VideoCodec
import org.thoughtcrime.securesms.video.videoconverter.utils.VideoConstants
/**
* A data class to hold various video transcoding parameters, such as bitrate.
*/
class TranscodingQuality private constructor(@VideoCodec val codec: String, val outputResolution: Int, val targetVideoBitRate: Int, val targetAudioBitRate: Int, private val durationMs: Long) {
companion object {
@JvmStatic
fun createFromPreset(preset: TranscodingPreset, durationMs: Long): TranscodingQuality {
return TranscodingQuality(preset.videoCodec, preset.videoShortEdge, preset.videoBitRate, preset.audioBitRate, durationMs)
}
@JvmStatic
fun createManuallyForTesting(codec: String, outputShortEdge: Int, videoBitrate: Int, audioBitrate: Int, durationMs: Long): TranscodingQuality {
return TranscodingQuality(codec, outputShortEdge, videoBitrate, audioBitrate, durationMs)
}
@JvmStatic
fun bitRate(bytes: Long, durationMs: Long): Int {
return (bytes * 8 / (durationMs / 1000f)).toInt()
}
}
val targetTotalBitRate = targetVideoBitRate + targetAudioBitRate
val byteCountEstimate = ((targetTotalBitRate / 8f) * (durationMs / 1000f)).toInt()
override fun toString(): String {
return "Quality{codec=$codec, targetVideoBitRate=$targetVideoBitRate, targetAudioBitRate=$targetAudioBitRate, duration=$durationMs, filesize=$byteCountEstimate}"
}
}
enum class TranscodingPreset(@VideoCodec val videoCodec: String, val videoShortEdge: Int, val videoBitRate: Int, val audioBitRate: Int) {
LEVEL_1(MediaConverter.VIDEO_CODEC_H264, VideoConstants.VIDEO_SHORT_EDGE_SD, VideoConstants.VIDEO_BITRATE_L1, VideoConstants.AUDIO_BITRATE),
LEVEL_2(MediaConverter.VIDEO_CODEC_H264, VideoConstants.VIDEO_SHORT_EDGE_HD, VideoConstants.VIDEO_BITRATE_L2, VideoConstants.AUDIO_BITRATE),
LEVEL_3(MediaConverter.VIDEO_CODEC_H264, VideoConstants.VIDEO_SHORT_EDGE_HD, VideoConstants.VIDEO_BITRATE_L3, VideoConstants.AUDIO_BITRATE),
/** Experimetnal H265 level */
LEVEL_3_H265(MediaConverter.VIDEO_CODEC_H265, VideoConstants.VIDEO_SHORT_EDGE_HD, VideoConstants.VIDEO_BITRATE_L3, VideoConstants.AUDIO_BITRATE);
fun calculateMaxVideoUploadDurationInSeconds(upperFileSizeLimit: Long): Int {
val upperFileSizeLimitWithMargin = (upperFileSizeLimit / 1.1).toLong()
val totalBitRate = videoBitRate + audioBitRate
return Math.toIntExact((upperFileSizeLimitWithMargin * 8) / totalBitRate)
}
}

11
video/lib/src/main/java/org/thoughtcrime/securesms/video/exceptions/VideoPostProcessingException.kt Normal file
View file

@ -0,0 +1,11 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.video.exceptions
class VideoPostProcessingException : RuntimeException {
internal constructor(message: String?) : super(message)
internal constructor(message: String?, inner: Exception?) : super(message, inner)
}

12
video/lib/src/main/java/org/thoughtcrime/securesms/video/exceptions/VideoSizeException.kt Normal file
View file

@ -0,0 +1,12 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.video.exceptions
import java.io.IOException
/**
* Exception to denote when video processing has been unable to meet its output file size requirements.
*/
class VideoSizeException(message: String?) : IOException(message)

13
video/lib/src/main/java/org/thoughtcrime/securesms/video/exceptions/VideoSourceException.kt Normal file
View file

@ -0,0 +1,13 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.video.exceptions
/**
* Exception to denote when video processing has had an issue with its source input.
*/
class VideoSourceException : Exception {
internal constructor(message: String?) : super(message)
internal constructor(message: String?, inner: Exception?) : super(message, inner)
}

20
video/lib/src/main/java/org/thoughtcrime/securesms/video/interfaces/MediaInput.kt Normal file
View file

@ -0,0 +1,20 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.video.interfaces
import android.media.MediaExtractor
import java.io.Closeable
import java.io.IOException
/**
* Abstraction over the different sources of media input for transcoding.
*/
interface MediaInput : Closeable {
@Throws(IOException::class)
fun createExtractor(): MediaExtractor
fun hasSameInput(other: MediaInput): Boolean
}

29
video/lib/src/main/java/org/thoughtcrime/securesms/video/interfaces/Muxer.java Normal file
View file

@ -0,0 +1,29 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.video.interfaces;
import android.media.MediaCodec;
import android.media.MediaFormat;
import androidx.annotation.NonNull;
import java.io.IOException;
import java.nio.ByteBuffer;
public interface Muxer {
void start() throws IOException;
long stop() throws IOException;
int addTrack(@NonNull MediaFormat format) throws IOException;
void writeSampleData(int trackIndex, @NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException;
void release();
boolean supportsAudioRemux();
}

5
video/lib/src/main/java/org/thoughtcrime/securesms/video/interfaces/TranscoderCancelationSignal.kt Normal file
View file

@ -0,0 +1,5 @@
package org.thoughtcrime.securesms.video.interfaces
fun interface TranscoderCancelationSignal {
fun isCanceled(): Boolean
}

84
video/lib/src/main/java/org/thoughtcrime/securesms/video/postprocessing/Mp4FaststartPostProcessor.kt Normal file
View file

@ -0,0 +1,84 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.video.postprocessing
import org.signal.core.util.readLength
import org.signal.core.util.stream.LimitedInputStream
import org.signal.libsignal.media.Mp4Sanitizer
import org.signal.libsignal.media.SanitizedMetadata
import org.thoughtcrime.securesms.video.exceptions.VideoPostProcessingException
import java.io.ByteArrayInputStream
import java.io.InputStream
import java.io.OutputStream
import java.io.SequenceInputStream
/**
* A post processor that takes a stream of bytes, and using [Mp4Sanitizer], moves the metadata to the front of the file.
*
* @property inputStreamFactory factory for the [InputStream]. Expected to be called multiple times.
*/
class Mp4FaststartPostProcessor(private val inputStreamFactory: InputStreamFactory) {
/**
* It is the responsibility of the caller to close the resulting [InputStream].
*/
fun process(inputLength: Long = calculateStreamLength(inputStreamFactory.create())): SequenceInputStream {
val metadata = inputStreamFactory.create().use { inputStream ->
sanitizeMetadata(inputStream, inputLength)
}
if (metadata.sanitizedMetadata == null) {
throw VideoPostProcessingException("Sanitized metadata was null!")
}
val inputStream = inputStreamFactory.create()
inputStream.skip(metadata.dataOffset)
return SequenceInputStream(ByteArrayInputStream(metadata.sanitizedMetadata), LimitedInputStream(inputStream, metadata.dataLength))
}
fun processAndWriteTo(outputStream: OutputStream, inputLength: Long = calculateStreamLength(inputStreamFactory.create())): Long {
process(inputLength).use { inStream ->
return inStream.copyTo(outputStream)
}
}
/**
* It is the responsibility of the caller to close the resulting [InputStream].
*/
fun processWithMdatLength(inputLength: Long, mdatLength: Int): SequenceInputStream {
val metadata = inputStreamFactory.create().use { inputStream ->
inputStream.use {
Mp4Sanitizer.sanitizeFileWithCompoundedMdatBoxes(it, inputLength, mdatLength)
}
}
if (metadata.sanitizedMetadata == null) {
throw VideoPostProcessingException("Sanitized metadata was null!")
}
val inputStream = inputStreamFactory.create()
inputStream.skip(metadata.dataOffset)
return SequenceInputStream(ByteArrayInputStream(metadata.sanitizedMetadata), LimitedInputStream(inputStream, metadata.dataLength))
}
fun interface InputStreamFactory {
fun create(): InputStream
}
companion object {
const val TAG = "Mp4Faststart"
@JvmStatic
fun calculateStreamLength(inputStream: InputStream): Long {
inputStream.use {
return it.readLength()
}
}
@JvmStatic
private fun sanitizeMetadata(inputStream: InputStream, inputLength: Long): SanitizedMetadata {
inputStream.use {
return Mp4Sanitizer.sanitize(it, inputLength)
}
}
}
}

60
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AndroidMuxer.java Normal file
View file

@ -0,0 +1,60 @@
package org.thoughtcrime.securesms.video.videoconverter;
import android.media.MediaCodec;
import android.media.MediaFormat;
import android.media.MediaMuxer;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import org.thoughtcrime.securesms.video.interfaces.Muxer;
import java.io.File;
import java.io.FileDescriptor;
import java.io.IOException;
import java.nio.ByteBuffer;
final class AndroidMuxer implements Muxer {
private final MediaMuxer muxer;
AndroidMuxer(final @NonNull File file) throws IOException {
muxer = new MediaMuxer(file.getAbsolutePath(), MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
}
@RequiresApi(26)
AndroidMuxer(final @NonNull FileDescriptor fileDescriptor) throws IOException {
muxer = new MediaMuxer(fileDescriptor, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
}
@Override
public void start() {
muxer.start();
}
@Override
public long stop() {
muxer.stop();
return 0;
}
@Override
public int addTrack(final @NonNull MediaFormat format) {
return muxer.addTrack(format);
}
@Override
public void writeSampleData(final int trackIndex, final @NonNull ByteBuffer byteBuf, final @NonNull MediaCodec.BufferInfo bufferInfo) {
muxer.writeSampleData(trackIndex, byteBuf, bufferInfo);
}
@Override
public void release() {
muxer.release();
}
@Override
public boolean supportsAudioRemux() {
return false;
}
}

506
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AudioTrackConverter.java Normal file
View file

@ -0,0 +1,506 @@
package org.thoughtcrime.securesms.video.videoconverter;
import android.annotation.SuppressLint;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.video.interfaces.MediaInput;
import org.thoughtcrime.securesms.video.interfaces.Muxer;
import org.thoughtcrime.securesms.video.videoconverter.utils.Preconditions;
import org.thoughtcrime.securesms.video.videoconverter.utils.VideoConstants;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Locale;
final class AudioTrackConverter {
private static final String TAG = "media-converter";
private static final boolean VERBOSE = false; // lots of logging
private static final String OUTPUT_AUDIO_MIME_TYPE = VideoConstants.AUDIO_MIME_TYPE; // Advanced Audio Coding
private static final int OUTPUT_AUDIO_AAC_PROFILE = MediaCodecInfo.CodecProfileLevel.AACObjectLC; //MediaCodecInfo.CodecProfileLevel.AACObjectHE;
private static final int SAMPLE_BUFFER_SIZE = 16 * 1024;
private static final int TIMEOUT_USEC = 10000;
private final long mTimeFrom;
private final long mTimeTo;
private final int mAudioBitrate;
final long mInputDuration;
private final MediaExtractor mAudioExtractor;
private final MediaCodec mAudioDecoder;
private final MediaCodec mAudioEncoder;
private final ByteBuffer instanceSampleBuffer = ByteBuffer.allocateDirect(SAMPLE_BUFFER_SIZE);
private final MediaCodec.BufferInfo instanceBufferInfo = new MediaCodec.BufferInfo();
private final ByteBuffer[] mAudioDecoderInputBuffers;
private ByteBuffer[] mAudioDecoderOutputBuffers;
private final ByteBuffer[] mAudioEncoderInputBuffers;
private ByteBuffer[] mAudioEncoderOutputBuffers;
private final MediaCodec.BufferInfo mAudioDecoderOutputBufferInfo;
private final MediaCodec.BufferInfo mAudioEncoderOutputBufferInfo;
MediaFormat mEncoderOutputAudioFormat;
boolean mAudioExtractorDone;
private boolean mAudioDecoderDone;
boolean mAudioEncoderDone;
private boolean skipTrancode;
private int mOutputAudioTrack = -1;
private int mPendingAudioDecoderOutputBufferIndex = -1;
long mMuxingAudioPresentationTime;
private int mAudioExtractedFrameCount;
private int mAudioDecodedFrameCount;
private int mAudioEncodedFrameCount;
private Muxer mMuxer;
static @Nullable
AudioTrackConverter create(
final @NonNull MediaInput input,
final long timeFrom,
final long timeTo,
final int audioBitrate,
final boolean allowSkipTranscode) throws IOException {
final MediaExtractor audioExtractor = input.createExtractor();
final int audioInputTrack = getAndSelectAudioTrackIndex(audioExtractor);
if (audioInputTrack == -1) {
audioExtractor.release();
return null;
}
return new AudioTrackConverter(audioExtractor, audioInputTrack, timeFrom, timeTo, audioBitrate, allowSkipTranscode);
}
private AudioTrackConverter(
final @NonNull MediaExtractor audioExtractor,
final int audioInputTrack,
long timeFrom,
long timeTo,
int audioBitrate,
final boolean allowSkipTranscode) throws IOException {
mTimeFrom = timeFrom;
mTimeTo = timeTo;
mAudioExtractor = audioExtractor;
mAudioBitrate = audioBitrate;
final MediaCodecInfo audioCodecInfo = MediaConverter.selectCodec(OUTPUT_AUDIO_MIME_TYPE);
if (audioCodecInfo == null) {
// Don't fail CTS if they don't have an AAC codec (not here, anyway).
Log.e(TAG, "Unable to find an appropriate codec for " + OUTPUT_AUDIO_MIME_TYPE);
throw new FileNotFoundException();
}
if (VERBOSE) Log.d(TAG, "audio found codec: " + audioCodecInfo.getName());
final MediaFormat inputAudioFormat = mAudioExtractor.getTrackFormat(audioInputTrack);
mInputDuration = inputAudioFormat.containsKey(MediaFormat.KEY_DURATION) ? inputAudioFormat.getLong(MediaFormat.KEY_DURATION) : 0;
skipTrancode = allowSkipTranscode && formatCanSkipTranscode(inputAudioFormat, audioBitrate);
if (skipTrancode) {
mEncoderOutputAudioFormat = inputAudioFormat;
}
if (VERBOSE) Log.d(TAG, "audio skipping transcoding: " + skipTrancode);
final MediaFormat outputAudioFormat =
MediaFormat.createAudioFormat(
OUTPUT_AUDIO_MIME_TYPE,
inputAudioFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE),
inputAudioFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT));
outputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, audioBitrate);
outputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE);
outputAudioFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, SAMPLE_BUFFER_SIZE);
// Create a MediaCodec for the desired codec, then configure it as an encoder with
// our desired properties. Request a Surface to use for input.
mAudioEncoder = createAudioEncoder(audioCodecInfo, outputAudioFormat);
// Create a MediaCodec for the decoder, based on the extractor's format.
mAudioDecoder = createAudioDecoder(inputAudioFormat);
mAudioDecoderInputBuffers = mAudioDecoder.getInputBuffers();
mAudioDecoderOutputBuffers = mAudioDecoder.getOutputBuffers();
mAudioEncoderInputBuffers = mAudioEncoder.getInputBuffers();
mAudioEncoderOutputBuffers = mAudioEncoder.getOutputBuffers();
mAudioDecoderOutputBufferInfo = new MediaCodec.BufferInfo();
mAudioEncoderOutputBufferInfo = new MediaCodec.BufferInfo();
if (mTimeFrom > 0) {
mAudioExtractor.seekTo(mTimeFrom * 1000, MediaExtractor.SEEK_TO_PREVIOUS_SYNC);
Log.i(TAG, "Seek audio:" + mTimeFrom + " " + mAudioExtractor.getSampleTime());
}
}
void setMuxer(final @NonNull Muxer muxer) throws IOException {
mMuxer = muxer;
if (mEncoderOutputAudioFormat != null) {
Log.d(TAG, "muxer: adding audio track.");
if (!mEncoderOutputAudioFormat.containsKey(MediaFormat.KEY_BIT_RATE)) {
Log.d(TAG, "muxer: fixed MediaFormat to add bitrate.");
mEncoderOutputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, mAudioBitrate);
}
if (!mEncoderOutputAudioFormat.containsKey(MediaFormat.KEY_AAC_PROFILE)) {
Log.d(TAG, "muxer: fixed MediaFormat to add AAC profile.");
mEncoderOutputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE);
}
mOutputAudioTrack = muxer.addTrack(mEncoderOutputAudioFormat);
}
}
void step() throws IOException {
if (skipTrancode && mEncoderOutputAudioFormat != null) {
try {
extractAndRemux();
return;
} catch (IllegalArgumentException e) {
Log.w(TAG, "Remuxer threw an exception! Disabling remux.", e);
skipTrancode = false;
}
}
// Extract audio from file and feed to decoder.
// Do not extract audio if we have determined the output format but we are not yet
// ready to mux the frames.
while (!mAudioExtractorDone && (mEncoderOutputAudioFormat == null || mMuxer != null)) {
int decoderInputBufferIndex = mAudioDecoder.dequeueInputBuffer(TIMEOUT_USEC);
if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no audio decoder input buffer");
break;
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: returned input buffer: " + decoderInputBufferIndex);
}
final ByteBuffer decoderInputBuffer = mAudioDecoderInputBuffers[decoderInputBufferIndex];
final int size = mAudioExtractor.readSampleData(decoderInputBuffer, 0);
final long presentationTime = mAudioExtractor.getSampleTime();
if (VERBOSE) {
Log.d(TAG, "audio extractor: returned buffer of size " + size);
Log.d(TAG, "audio extractor: returned buffer for time " + presentationTime);
}
mAudioExtractorDone = isAudioExtractorDone(size, presentationTime);
if (mAudioExtractorDone) {
if (VERBOSE) Log.d(TAG, "audio extractor: EOS");
mAudioDecoder.queueInputBuffer(
decoderInputBufferIndex,
0,
0,
0,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
} else {
mAudioDecoder.queueInputBuffer(
decoderInputBufferIndex,
0,
size,
presentationTime,
mAudioExtractor.getSampleFlags());
}
mAudioExtractor.advance();
mAudioExtractedFrameCount++;
// We extracted a frame, let's try something else next.
break;
}
// Poll output frames from the audio decoder.
// Do not poll if we already have a pending buffer to feed to the encoder.
while (!mAudioDecoderDone && mPendingAudioDecoderOutputBufferIndex == -1
&& (mEncoderOutputAudioFormat == null || mMuxer != null)) {
final int decoderOutputBufferIndex =
mAudioDecoder.dequeueOutputBuffer(
mAudioDecoderOutputBufferInfo, TIMEOUT_USEC);
if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no audio decoder output buffer");
break;
}
if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
if (VERBOSE) Log.d(TAG, "audio decoder: output buffers changed");
mAudioDecoderOutputBuffers = mAudioDecoder.getOutputBuffers();
break;
}
if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
if (VERBOSE) {
MediaFormat decoderOutputAudioFormat = mAudioDecoder.getOutputFormat();
Log.d(TAG, "audio decoder: output format changed: " + decoderOutputAudioFormat);
}
break;
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: returned output buffer: " + decoderOutputBufferIndex);
Log.d(TAG, "audio decoder: returned buffer of size " + mAudioDecoderOutputBufferInfo.size);
}
if ((mAudioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
if (VERBOSE) Log.d(TAG, "audio decoder: codec config buffer");
mAudioDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false);
break;
}
if (mAudioDecoderOutputBufferInfo.presentationTimeUs < mTimeFrom * 1000 &&
(mAudioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == 0) {
if (VERBOSE)
Log.d(TAG, "audio decoder: frame prior to " + mAudioDecoderOutputBufferInfo.presentationTimeUs);
mAudioDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false);
break;
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: returned buffer for time " + mAudioDecoderOutputBufferInfo.presentationTimeUs);
Log.d(TAG, "audio decoder: output buffer is now pending: " + mPendingAudioDecoderOutputBufferIndex);
}
mPendingAudioDecoderOutputBufferIndex = decoderOutputBufferIndex;
mAudioDecodedFrameCount++;
// We extracted a pending frame, let's try something else next.
break;
}
// Feed the pending decoded audio buffer to the audio encoder.
while (mPendingAudioDecoderOutputBufferIndex != -1) {
if (VERBOSE) {
Log.d(TAG, "audio decoder: attempting to process pending buffer: " + mPendingAudioDecoderOutputBufferIndex);
}
final int encoderInputBufferIndex = mAudioEncoder.dequeueInputBuffer(TIMEOUT_USEC);
if (encoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no audio encoder input buffer");
break;
}
if (VERBOSE) {
Log.d(TAG, "audio encoder: returned input buffer: " + encoderInputBufferIndex);
}
final ByteBuffer encoderInputBuffer = mAudioEncoderInputBuffers[encoderInputBufferIndex];
final int size = mAudioDecoderOutputBufferInfo.size;
final long presentationTime = mAudioDecoderOutputBufferInfo.presentationTimeUs;
if (VERBOSE) {
Log.d(TAG, "audio decoder: processing pending buffer: " + mPendingAudioDecoderOutputBufferIndex);
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: pending buffer of size " + size);
Log.d(TAG, "audio decoder: pending buffer for time " + presentationTime);
}
if (size >= 0) {
final ByteBuffer decoderOutputBuffer = mAudioDecoderOutputBuffers[mPendingAudioDecoderOutputBufferIndex].duplicate();
decoderOutputBuffer.position(mAudioDecoderOutputBufferInfo.offset);
decoderOutputBuffer.limit(mAudioDecoderOutputBufferInfo.offset + size);
encoderInputBuffer.position(0);
encoderInputBuffer.put(decoderOutputBuffer);
mAudioEncoder.queueInputBuffer(
encoderInputBufferIndex,
0,
size,
presentationTime,
mAudioDecoderOutputBufferInfo.flags);
}
mAudioDecoder.releaseOutputBuffer(mPendingAudioDecoderOutputBufferIndex, false);
mPendingAudioDecoderOutputBufferIndex = -1;
if ((mAudioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "audio decoder: EOS");
mAudioDecoderDone = true;
}
// We enqueued a pending frame, let's try something else next.
break;
}
// Poll frames from the audio encoder and send them to the muxer.
while (!mAudioEncoderDone && (mEncoderOutputAudioFormat == null || mMuxer != null)) {
final int encoderOutputBufferIndex = mAudioEncoder.dequeueOutputBuffer(mAudioEncoderOutputBufferInfo, TIMEOUT_USEC);
if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no audio encoder output buffer");
break;
}
if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
if (VERBOSE) Log.d(TAG, "audio encoder: output buffers changed");
mAudioEncoderOutputBuffers = mAudioEncoder.getOutputBuffers();
break;
}
if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
if (VERBOSE) Log.d(TAG, "audio encoder: output format changed");
Preconditions.checkState("audio encoder changed its output format again?", mOutputAudioTrack < 0);
mEncoderOutputAudioFormat = mAudioEncoder.getOutputFormat();
break;
}
Preconditions.checkState("should have added track before processing output", mMuxer != null);
if (VERBOSE) {
Log.d(TAG, "audio encoder: returned output buffer: " + encoderOutputBufferIndex);
Log.d(TAG, "audio encoder: returned buffer of size " + mAudioEncoderOutputBufferInfo.size);
}
final ByteBuffer encoderOutputBuffer = mAudioEncoderOutputBuffers[encoderOutputBufferIndex];
if ((mAudioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
if (VERBOSE) Log.d(TAG, "audio encoder: codec config buffer");
// Simply ignore codec config buffers.
mAudioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
break;
}
if (VERBOSE) {
Log.d(TAG, "audio encoder: returned buffer for time " + mAudioEncoderOutputBufferInfo.presentationTimeUs);
}
if (mAudioEncoderOutputBufferInfo.size != 0) {
mMuxer.writeSampleData(mOutputAudioTrack, encoderOutputBuffer, mAudioEncoderOutputBufferInfo);
mMuxingAudioPresentationTime = Math.max(mMuxingAudioPresentationTime, mAudioEncoderOutputBufferInfo.presentationTimeUs);
}
if ((mAudioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "audio encoder: EOS");
mAudioEncoderDone = true;
}
mAudioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
mAudioEncodedFrameCount++;
// We enqueued an encoded frame, let's try something else next.
break;
}
}
void release() throws Exception {
Exception exception = null;
try {
if (mAudioExtractor != null) {
mAudioExtractor.release();
}
} catch (Exception e) {
Log.e(TAG, "error while releasing mAudioExtractor", e);
exception = e;
}
try {
if (mAudioDecoder != null) {
mAudioDecoder.stop();
mAudioDecoder.release();
}
} catch (Exception e) {
Log.e(TAG, "error while releasing mAudioDecoder", e);
if (exception == null) {
exception = e;
}
}
try {
if (mAudioEncoder != null) {
mAudioEncoder.stop();
mAudioEncoder.release();
}
} catch (Exception e) {
Log.e(TAG, "error while releasing mAudioEncoder", e);
if (exception == null) {
exception = e;
}
}
if (exception != null) {
throw exception;
}
}
AudioTrackConverterState dumpState() {
return new AudioTrackConverterState(
mAudioExtractedFrameCount, mAudioExtractorDone,
mAudioDecodedFrameCount, mAudioDecoderDone,
mAudioEncodedFrameCount, mAudioEncoderDone,
mPendingAudioDecoderOutputBufferIndex,
mMuxer != null, mOutputAudioTrack);
}
void verifyEndState() {
Preconditions.checkState("no frame should be pending", -1 == mPendingAudioDecoderOutputBufferIndex);
}
@SuppressLint("WrongConstant") // flags extracted from sample by MediaExtractor should be safe for MediaCodec.BufferInfo
private void extractAndRemux() throws IOException {
if (mMuxer == null) {
Log.d(TAG, "audio remuxer: tried to execute before muxer was ready");
return;
}
int size = mAudioExtractor.readSampleData(instanceSampleBuffer, 0);
long presentationTime = mAudioExtractor.getSampleTime();
int sampleFlags = mAudioExtractor.getSampleFlags();
if (VERBOSE) {
Log.d(TAG, "audio extractor: returned buffer of size " + size);
Log.d(TAG, "audio extractor: returned buffer for time " + presentationTime);
Log.d(TAG, "audio extractor: returned buffer with flags " + Integer.toBinaryString(sampleFlags));
}
mAudioExtractorDone = isAudioExtractorDone(size, presentationTime);
if (mAudioExtractorDone) {
if (VERBOSE) Log.d(TAG, "audio encoder: EOS");
instanceBufferInfo.set(0, 0, presentationTime, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
mAudioEncoderDone = true;
} else {
instanceBufferInfo.set(0, size, presentationTime, sampleFlags);
}
mMuxer.writeSampleData(mOutputAudioTrack, instanceSampleBuffer, instanceBufferInfo);
if (VERBOSE) {
Log.d(TAG, "audio extractor: wrote sample at " + presentationTime);
}
mAudioExtractor.advance();
mAudioExtractedFrameCount++;
mAudioEncodedFrameCount++;
mMuxingAudioPresentationTime = Math.max(mMuxingAudioPresentationTime, presentationTime);
}
private boolean isAudioExtractorDone(int size, long presentationTime) {
return presentationTime == -1 || size < 0 || (mTimeTo > 0 && presentationTime > mTimeTo * 1000);
}
private static @NonNull
MediaCodec createAudioDecoder(final @NonNull MediaFormat inputFormat) throws IOException {
final MediaCodec decoder = MediaCodec.createDecoderByType(MediaConverter.getMimeTypeFor(inputFormat));
decoder.configure(inputFormat, null, null, 0);
decoder.start();
return decoder;
}
private static @NonNull
MediaCodec createAudioEncoder(final @NonNull MediaCodecInfo codecInfo, final @NonNull MediaFormat format) throws IOException {
final MediaCodec encoder = MediaCodec.createByCodecName(codecInfo.getName());
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
encoder.start();
return encoder;
}
private static int getAndSelectAudioTrackIndex(MediaExtractor extractor) {
for (int index = 0; index < extractor.getTrackCount(); ++index) {
if (VERBOSE) {
Log.d(TAG, "format for track " + index + " is " + MediaConverter.getMimeTypeFor(extractor.getTrackFormat(index)));
}
if (isAudioFormat(extractor.getTrackFormat(index))) {
extractor.selectTrack(index);
return index;
}
}
return -1;
}
private static boolean isAudioFormat(final @NonNull MediaFormat format) {
return MediaConverter.getMimeTypeFor(format).startsWith("audio/");
}
/**
* HE-AAC input bitstreams exhibit bad decoder behavior: the decoder's output buffer's presentation timestamp is way larger than the input sample's.
* This mismatch propagates throughout the transcoding pipeline and results in slowed, distorted audio in the output file.
* To sidestep this: AAC and its variants are a supported output codec, and HE-AAC bitrates are almost always lower than our target bitrate,
* so we can pass through the input bitstream unaltered, relying on consumers of the output file to render HE-AAC correctly.
*/
private static boolean formatCanSkipTranscode(MediaFormat audioFormat, int desiredBitrate) {
try {
int inputBitrate = audioFormat.getInteger(MediaFormat.KEY_BIT_RATE);
String inputMimeType = audioFormat.getString(MediaFormat.KEY_MIME);
return OUTPUT_AUDIO_MIME_TYPE.equals(inputMimeType) && inputBitrate <= desiredBitrate;
} catch (NullPointerException exception) {
if (VERBOSE) {
Log.d(TAG, "could not find bitrate in mediaFormat, can't skip transcoding.");
}
return false;
}
}
}

187
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/InputSurface.java Normal file
View file

@ -0,0 +1,187 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* This file has been modified by Signal.
*/
package org.thoughtcrime.securesms.video.videoconverter;
import android.opengl.EGL14;
import android.opengl.EGLConfig;
import android.opengl.EGLContext;
import android.opengl.EGLDisplay;
import android.opengl.EGLExt;
import android.opengl.EGLSurface;
import android.view.Surface;
import org.signal.core.util.logging.Log;
/**
* Holds state associated with a Surface used for MediaCodec encoder input.
* <p>
* The constructor takes a Surface obtained from MediaCodec.createInputSurface(), and uses that
* to create an EGL window surface. Calls to eglSwapBuffers() cause a frame of data to be sent
* to the video encoder.
*/
final class InputSurface {
private static final String TAG = "InputSurface";
private static final boolean VERBOSE = false;
private static final int EGL_RECORDABLE_ANDROID = 0x3142;
private static final int EGL_OPENGL_ES2_BIT = 4;
private EGLDisplay mEGLDisplay;
private EGLContext mEGLContext;
private EGLSurface mEGLSurface;
private Surface mSurface;
/**
* Creates an InputSurface from a Surface.
*/
InputSurface(Surface surface) throws TranscodingException {
if (surface == null) {
throw new NullPointerException();
}
mSurface = surface;
eglSetup();
}
/**
* Prepares EGL. We want a GLES 2.0 context and a surface that supports recording.
*/
private void eglSetup() throws TranscodingException {
mEGLDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY);
if (mEGLDisplay == EGL14.EGL_NO_DISPLAY) {
throw new TranscodingException("unable to get EGL14 display");
}
int[] version = new int[2];
if (!EGL14.eglInitialize(mEGLDisplay, version, 0, version, 1)) {
mEGLDisplay = null;
throw new TranscodingException("unable to initialize EGL14");
}
// Configure EGL for pbuffer and OpenGL ES 2.0. We want enough RGB bits
// to be able to tell if the frame is reasonable.
int[] attribList = {
EGL14.EGL_RED_SIZE, 8,
EGL14.EGL_GREEN_SIZE, 8,
EGL14.EGL_BLUE_SIZE, 8,
EGL14.EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
EGL_RECORDABLE_ANDROID, 1,
EGL14.EGL_NONE
};
EGLConfig[] configs = new EGLConfig[1];
int[] numConfigs = new int[1];
if (!EGL14.eglChooseConfig(mEGLDisplay, attribList, 0, configs, 0, configs.length,
numConfigs, 0)) {
throw new TranscodingException("unable to find RGB888+recordable ES2 EGL config");
}
// Configure context for OpenGL ES 2.0.
int[] attrib_list = {
EGL14.EGL_CONTEXT_CLIENT_VERSION, 2,
EGL14.EGL_NONE
};
mEGLContext = EGL14.eglCreateContext(mEGLDisplay, configs[0], EGL14.EGL_NO_CONTEXT,
attrib_list, 0);
checkEglError("eglCreateContext");
if (mEGLContext == null) {
throw new TranscodingException("null context");
}
// Create a window surface, and attach it to the Surface we received.
int[] surfaceAttribs = {
EGL14.EGL_NONE
};
mEGLSurface = EGL14.eglCreateWindowSurface(mEGLDisplay, configs[0], mSurface,
surfaceAttribs, 0);
checkEglError("eglCreateWindowSurface");
if (mEGLSurface == null) {
throw new TranscodingException("surface was null");
}
}
/**
* Discard all resources held by this class, notably the EGL context. Also releases the
* Surface that was passed to our constructor.
*/
public void release() {
if (EGL14.eglGetCurrentContext().equals(mEGLContext)) {
// Clear the current context and surface to ensure they are discarded immediately.
EGL14.eglMakeCurrent(mEGLDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE,
EGL14.EGL_NO_CONTEXT);
}
EGL14.eglDestroySurface(mEGLDisplay, mEGLSurface);
EGL14.eglDestroyContext(mEGLDisplay, mEGLContext);
//EGL14.eglTerminate(mEGLDisplay);
mSurface.release();
// null everything out so future attempts to use this object will cause an NPE
mEGLDisplay = null;
mEGLContext = null;
mEGLSurface = null;
mSurface = null;
}
/**
* Makes our EGL context and surface current.
*/
void makeCurrent() throws TranscodingException {
if (!EGL14.eglMakeCurrent(mEGLDisplay, mEGLSurface, mEGLSurface, mEGLContext)) {
throw new TranscodingException("eglMakeCurrent failed");
}
}
/**
* Calls eglSwapBuffers. Use this to "publish" the current frame.
*/
boolean swapBuffers() {
return EGL14.eglSwapBuffers(mEGLDisplay, mEGLSurface);
}
/**
* Returns the Surface that the MediaCodec receives buffers from.
*/
public Surface getSurface() {
return mSurface;
}
/**
* Sends the presentation time stamp to EGL. Time is expressed in nanoseconds.
*/
void setPresentationTime(long nsecs) {
EGLExt.eglPresentationTimeANDROID(mEGLDisplay, mEGLSurface, nsecs);
}
/**
* Checks for EGL errors.
*/
private static void checkEglError(String msg) throws TranscodingException {
boolean failed = false;
int error;
while ((error = EGL14.eglGetError()) != EGL14.EGL_SUCCESS) {
Log.e(TAG, msg + ": EGL error: 0x" + Integer.toHexString(error));
failed = true;
}
if (failed) {
throw new TranscodingException("EGL error encountered (see log)");
}
}
}

386
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java Normal file
View file

@ -0,0 +1,386 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* This file has been modified by Signal.
*/
package org.thoughtcrime.securesms.video.videoconverter;
import android.media.MediaCodecInfo;
import android.media.MediaCodecList;
import android.media.MediaFormat;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.annotation.StringDef;
import androidx.annotation.WorkerThread;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.video.interfaces.MediaInput;
import org.thoughtcrime.securesms.video.interfaces.Muxer;
import org.thoughtcrime.securesms.video.videoconverter.exceptions.EncodingException;
import org.thoughtcrime.securesms.video.videoconverter.muxer.StreamingMuxer;
import java.io.File;
import java.io.FileDescriptor;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@SuppressWarnings("WeakerAccess")
public final class MediaConverter {
private static final String TAG = "media-converter";
private static final boolean VERBOSE = false; // lots of logging
private static final int STUCK_FRAME_THRESHOLD = 100;
// Describes when the annotation will be discarded
@Retention(RetentionPolicy.SOURCE)
@StringDef({VIDEO_CODEC_H264, VIDEO_CODEC_H265})
public @interface VideoCodec {}
public static final String VIDEO_CODEC_H264 = "video/avc";
public static final String VIDEO_CODEC_H265 = "video/hevc";
private MediaInput mInput;
private Output mOutput;
private long mTimeFrom;
private long mTimeTo;
private int mVideoResolution;
private int mVideoBitrate = 2000000; // 2Mbps
private @VideoCodec String mVideoCodec = VIDEO_CODEC_H264;
private int mAudioBitrate = 128000; // 128Kbps
private boolean mAllowAudioRemux = false;
private Listener mListener;
private boolean mCancelled;
public interface Listener {
boolean onProgress(int percent);
}
public MediaConverter() {
}
public void setInput(final @NonNull MediaInput videoInput) {
mInput = videoInput;
}
@SuppressWarnings("unused")
public void setOutput(final @NonNull File file) {
mOutput = new FileOutput(file);
}
@SuppressWarnings("unused")
@RequiresApi(26)
public void setOutput(final @NonNull FileDescriptor fileDescriptor) {
mOutput = new FileDescriptorOutput(fileDescriptor);
}
public void setOutput(final @NonNull OutputStream stream) {
mOutput = new StreamOutput(stream);
}
@SuppressWarnings("unused")
public void setTimeRange(long timeFrom, long timeTo) {
mTimeFrom = timeFrom;
mTimeTo = timeTo;
if (timeTo > 0 && timeFrom >= timeTo) {
throw new IllegalArgumentException("timeFrom:" + timeFrom + " timeTo:" + timeTo);
}
}
@SuppressWarnings("unused")
public void setVideoResolution(int videoResolution) {
mVideoResolution = videoResolution;
}
@SuppressWarnings("unused")
public void setVideoCodec(final @VideoCodec String videoCodec) throws FileNotFoundException {
if (selectCodec(videoCodec) == null) {
throw new FileNotFoundException();
}
mVideoCodec = videoCodec;
}
@SuppressWarnings("unused")
public void setVideoBitrate(final int videoBitrate) {
mVideoBitrate = videoBitrate;
}
@SuppressWarnings("unused")
public void setAudioBitrate(final int audioBitrate) {
mAudioBitrate = audioBitrate;
}
@SuppressWarnings("unused")
public void setListener(final Listener listener) {
mListener = listener;
}
public void setAllowAudioRemux(boolean allow) {
mAllowAudioRemux = allow;
}
/**
* @return The total content size of the MP4 mdat box.
*/
@WorkerThread
public long convert() throws EncodingException, IOException {
// Exception that may be thrown during release.
Exception exception = null;
Muxer muxer = null;
VideoTrackConverter videoTrackConverter = null;
AudioTrackConverter audioTrackConverter = null;
long mdatContentLength = 0;
boolean muxerStopped = false;
try {
muxer = mOutput.createMuxer();
videoTrackConverter = VideoTrackConverter.create(mInput, mTimeFrom, mTimeTo, mVideoResolution, mVideoBitrate, mVideoCodec);
audioTrackConverter = AudioTrackConverter.create(mInput, mTimeFrom, mTimeTo, mAudioBitrate, mAllowAudioRemux && muxer.supportsAudioRemux());
if (videoTrackConverter == null && audioTrackConverter == null) {
throw new EncodingException("No video and audio tracks");
}
doExtractDecodeEditEncodeMux(
videoTrackConverter,
audioTrackConverter,
muxer);
mdatContentLength = muxer.stop();
muxerStopped = true;
} catch (EncodingException | IOException e) {
Log.e(TAG, "error converting", e);
exception = e;
throw e;
} catch (Exception e) {
Log.e(TAG, "error converting", e);
exception = e;
} finally {
if (VERBOSE) Log.d(TAG, "releasing extractor, decoder, encoder, and muxer");
// Try to release everything we acquired, even if one of the releases fails, in which
// case we save the first exception we got and re-throw at the end (unless something
// other exception has already been thrown). This guarantees the first exception thrown
// is reported as the cause of the error, everything is (attempted) to be released, and
// all other exceptions appear in the logs.
try {
if (videoTrackConverter != null) {
videoTrackConverter.release();
}
} catch (Exception e) {
if (exception == null) {
exception = e;
}
}
try {
if (audioTrackConverter != null) {
audioTrackConverter.release();
}
} catch (Exception e) {
if (exception == null) {
exception = e;
}
}
try {
if (muxer != null) {
if (!muxerStopped) {
muxer.stop();
}
muxer.release();
}
} catch (Exception e) {
Log.e(TAG, "error while releasing muxer", e);
if (exception == null) {
exception = e;
}
}
}
if (exception != null) {
throw new EncodingException("Transcode failed", exception);
}
return mdatContentLength;
}
/**
* Does the actual work for extracting, decoding, encoding and muxing.
*/
private void doExtractDecodeEditEncodeMux(
final @Nullable VideoTrackConverter videoTrackConverter,
final @Nullable AudioTrackConverter audioTrackConverter,
final @NonNull Muxer muxer) throws IOException, TranscodingException {
MediaConverterState oldState = null;
int stuckFrames = 0;
boolean muxing = false;
int percentProcessed = 0;
long inputDuration = Math.max(
videoTrackConverter == null ? 0 : videoTrackConverter.mInputDuration,
audioTrackConverter == null ? 0 : audioTrackConverter.mInputDuration);
while (!mCancelled &&
((videoTrackConverter != null && !videoTrackConverter.mVideoEncoderDone) ||
(audioTrackConverter != null &&!audioTrackConverter.mAudioEncoderDone))) {
final MediaConverterState currentState = new MediaConverterState(videoTrackConverter != null ? videoTrackConverter.dumpState() : null, audioTrackConverter != null ? audioTrackConverter.dumpState() : null, muxing);
if (VERBOSE) {
Log.d(TAG, "loop: " + currentState);
}
if (currentState.equals(oldState)) {
if (++stuckFrames >= STUCK_FRAME_THRESHOLD) {
mCancelled = true;
}
} else {
oldState = currentState;
stuckFrames = 0;
}
if (videoTrackConverter != null && (audioTrackConverter == null || audioTrackConverter.mAudioExtractorDone || videoTrackConverter.mMuxingVideoPresentationTime <= audioTrackConverter.mMuxingAudioPresentationTime)) {
videoTrackConverter.step();
}
if (audioTrackConverter != null && (videoTrackConverter == null || videoTrackConverter.mVideoExtractorDone || videoTrackConverter.mMuxingVideoPresentationTime >= audioTrackConverter.mMuxingAudioPresentationTime)) {
audioTrackConverter.step();
}
if (inputDuration != 0 && mListener != null) {
final long timeFromUs = mTimeFrom <= 0 ? 0 : mTimeFrom * 1000;
final long timeToUs = mTimeTo <= 0 ? inputDuration : mTimeTo * 1000;
final int curPercentProcessed = (int) (100 *
(Math.max(
videoTrackConverter == null ? 0 : videoTrackConverter.mMuxingVideoPresentationTime,
audioTrackConverter == null ? 0 : audioTrackConverter.mMuxingAudioPresentationTime)
- timeFromUs) / (timeToUs - timeFromUs));
if (curPercentProcessed != percentProcessed) {
percentProcessed = curPercentProcessed;
mCancelled = mCancelled || mListener.onProgress(percentProcessed);
}
}
if (!muxing
&& (videoTrackConverter == null || videoTrackConverter.mEncoderOutputVideoFormat != null)
&& (audioTrackConverter == null || audioTrackConverter.mEncoderOutputAudioFormat != null)) {
if (videoTrackConverter != null) {
videoTrackConverter.setMuxer(muxer);
}
if (audioTrackConverter != null) {
audioTrackConverter.setMuxer(muxer);
}
Log.d(TAG, "muxer: starting");
muxer.start();
muxing = true;
}
}
// Basic sanity checks.
if (videoTrackConverter != null) {
videoTrackConverter.verifyEndState();
}
if (audioTrackConverter != null) {
audioTrackConverter.verifyEndState();
}
// TODO: Check the generated output file.
}
static String getMimeTypeFor(MediaFormat format) {
return format.getString(MediaFormat.KEY_MIME);
}
/**
* Returns the first codec capable of encoding the specified MIME type, or null if no match was
* found.
*/
static MediaCodecInfo selectCodec(final String mimeType) {
final int numCodecs = MediaCodecList.getCodecCount();
for (int i = 0; i < numCodecs; i++) {
final MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i);
if (!codecInfo.isEncoder()) {
continue;
}
final String[] types = codecInfo.getSupportedTypes();
for (String type : types) {
if (type.equalsIgnoreCase(mimeType)) {
return codecInfo;
}
}
}
return null;
}
interface Output {
@NonNull
Muxer createMuxer() throws IOException;
}
private static class FileOutput implements Output {
final File file;
FileOutput(final @NonNull File file) {
this.file = file;
}
@Override
public @NonNull
Muxer createMuxer() throws IOException {
return new AndroidMuxer(file);
}
}
@RequiresApi(26)
private static class FileDescriptorOutput implements Output {
final FileDescriptor fileDescriptor;
FileDescriptorOutput(final @NonNull FileDescriptor fileDescriptor) {
this.fileDescriptor = fileDescriptor;
}
@Override
public @NonNull
Muxer createMuxer() throws IOException {
return new AndroidMuxer(fileDescriptor);
}
}
private static class StreamOutput implements Output {
final OutputStream outputStream;
StreamOutput(final @NonNull OutputStream outputStream) {
this.outputStream = outputStream;
}
@Override
public @NonNull Muxer createMuxer() {
return new StreamingMuxer(outputStream);
}
}
}

12
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverterState.kt Normal file
View file

@ -0,0 +1,12 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.video.videoconverter
data class MediaConverterState(val videoTrack: VideoTrackConverterState?, val audioTrack: AudioTrackConverterState?, val muxing: Boolean)
data class VideoTrackConverterState(val extractedCount: Long, val extractedDone: Boolean, val decodedCount: Long, val decodedDone: Boolean, val encodedCount: Long, val encodedDone: Boolean, val muxing: Boolean, val trackIndex: Int)
data class AudioTrackConverterState(val extractedCount: Long, val extractedDone: Boolean, val decodedCount: Long, val decodedDone: Boolean, val encodedCount: Long, val encodedDone: Boolean, val pendingBufferIndex: Int, val muxing: Boolean, val trackIndex: Int)

303
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/OutputSurface.java Normal file
View file

@ -0,0 +1,303 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* This file has been modified by Signal.
*/
package org.thoughtcrime.securesms.video.videoconverter;
import android.graphics.SurfaceTexture;
import android.opengl.EGL14;
import android.view.Surface;
import org.signal.core.util.logging.Log;
import javax.microedition.khronos.egl.EGL10;
import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.egl.EGLContext;
import javax.microedition.khronos.egl.EGLDisplay;
import javax.microedition.khronos.egl.EGLSurface;
/**
* Holds state associated with a Surface used for MediaCodec decoder output.
* <p>
* The (width,height) constructor for this class will prepare GL, create a SurfaceTexture,
* and then create a Surface for that SurfaceTexture. The Surface can be passed to
* MediaCodec.configure() to receive decoder output. When a frame arrives, we latch the
* texture with updateTexImage, then render the texture with GL to a pbuffer.
* <p>
* The no-arg constructor skips the GL preparation step and doesn't allocate a pbuffer.
* Instead, it just creates the Surface and SurfaceTexture, and when a frame arrives
* we just draw it on whatever surface is current.
* <p>
* By default, the Surface will be using a BufferQueue in asynchronous mode, so we
* can potentially drop frames.
*/
final class OutputSurface implements SurfaceTexture.OnFrameAvailableListener {
private static final String TAG = "OutputSurface";
private static final boolean VERBOSE = false;
private static final int EGL_OPENGL_ES2_BIT = 4;
private EGL10 mEGL;
private EGLDisplay mEGLDisplay;
private EGLContext mEGLContext;
private EGLSurface mEGLSurface;
private SurfaceTexture mSurfaceTexture;
private Surface mSurface;
private final Object mFrameSyncObject = new Object(); // guards mFrameAvailable
private boolean mFrameAvailable;
private TextureRender mTextureRender;
/**
* Creates an OutputSurface backed by a pbuffer with the specifed dimensions. The new
* EGL context and surface will be made current. Creates a Surface that can be passed
* to MediaCodec.configure().
*/
OutputSurface(int width, int height, boolean flipX) throws TranscodingException {
if (width <= 0 || height <= 0) {
throw new IllegalArgumentException();
}
eglSetup(width, height);
makeCurrent();
setup(flipX);
}
/**
* Creates an OutputSurface using the current EGL context. Creates a Surface that can be
* passed to MediaCodec.configure().
*/
OutputSurface() throws TranscodingException {
setup(false);
}
/**
* Creates instances of TextureRender and SurfaceTexture, and a Surface associated
* with the SurfaceTexture.
*/
private void setup(boolean flipX) throws TranscodingException {
mTextureRender = new TextureRender(flipX);
mTextureRender.surfaceCreated();
// Even if we don't access the SurfaceTexture after the constructor returns, we
// still need to keep a reference to it. The Surface doesn't retain a reference
// at the Java level, so if we don't either then the object can get GCed, which
// causes the native finalizer to run.
if (VERBOSE) Log.d(TAG, "textureID=" + mTextureRender.getTextureId());
mSurfaceTexture = new SurfaceTexture(mTextureRender.getTextureId());
// This doesn't work if OutputSurface is created on the thread that CTS started for
// these test cases.
//
// The CTS-created thread has a Looper, and the SurfaceTexture constructor will
// create a Handler that uses it. The "frame available" message is delivered
// there, but since we're not a Looper-based thread we'll never see it. For
// this to do anything useful, OutputSurface must be created on a thread without
// a Looper, so that SurfaceTexture uses the main application Looper instead.
//
// Java language note: passing "this" out of a constructor is generally unwise,
// but we should be able to get away with it here.
mSurfaceTexture.setOnFrameAvailableListener(this);
mSurface = new Surface(mSurfaceTexture);
}
/**
* Prepares EGL. We want a GLES 2.0 context and a surface that supports pbuffer.
*/
private void eglSetup(int width, int height) throws TranscodingException {
mEGL = (EGL10)EGLContext.getEGL();
mEGLDisplay = mEGL.eglGetDisplay(EGL10.EGL_DEFAULT_DISPLAY);
if (!mEGL.eglInitialize(mEGLDisplay, null)) {
throw new TranscodingException("unable to initialize EGL10");
}
// Configure EGL for pbuffer and OpenGL ES 2.0. We want enough RGB bits
// to be able to tell if the frame is reasonable.
int[] attribList = {
EGL10.EGL_RED_SIZE, 8,
EGL10.EGL_GREEN_SIZE, 8,
EGL10.EGL_BLUE_SIZE, 8,
EGL10.EGL_SURFACE_TYPE, EGL10.EGL_PBUFFER_BIT,
EGL10.EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
EGL10.EGL_NONE
};
EGLConfig[] configs = new EGLConfig[1];
int[] numConfigs = new int[1];
if (!mEGL.eglChooseConfig(mEGLDisplay, attribList, configs, 1, numConfigs)) {
throw new TranscodingException("unable to find RGB888+pbuffer EGL config");
}
// Configure context for OpenGL ES 2.0.
int[] attrib_list = {
EGL14.EGL_CONTEXT_CLIENT_VERSION, 2,
EGL10.EGL_NONE
};
mEGLContext = mEGL.eglCreateContext(mEGLDisplay, configs[0], EGL10.EGL_NO_CONTEXT,
attrib_list);
checkEglError("eglCreateContext");
if (mEGLContext == null) {
throw new TranscodingException("null context");
}
// Create a pbuffer surface. By using this for output, we can use glReadPixels
// to test values in the output.
int[] surfaceAttribs = {
EGL10.EGL_WIDTH, width,
EGL10.EGL_HEIGHT, height,
EGL10.EGL_NONE
};
mEGLSurface = mEGL.eglCreatePbufferSurface(mEGLDisplay, configs[0], surfaceAttribs);
checkEglError("eglCreatePbufferSurface");
if (mEGLSurface == null) {
throw new TranscodingException("surface was null");
}
}
/**
* Discard all resources held by this class, notably the EGL context.
*/
public void release() {
if (mEGL != null) {
if (mEGL.eglGetCurrentContext().equals(mEGLContext)) {
// Clear the current context and surface to ensure they are discarded immediately.
mEGL.eglMakeCurrent(mEGLDisplay, EGL10.EGL_NO_SURFACE, EGL10.EGL_NO_SURFACE,
EGL10.EGL_NO_CONTEXT);
}
mEGL.eglDestroySurface(mEGLDisplay, mEGLSurface);
mEGL.eglDestroyContext(mEGLDisplay, mEGLContext);
//mEGL.eglTerminate(mEGLDisplay);
}
mSurface.release();
// this causes a bunch of warnings that appear harmless but might confuse someone:
// W BufferQueue: [unnamed-3997-2] cancelBuffer: BufferQueue has been abandoned!
//mSurfaceTexture.release();
// null everything out so future attempts to use this object will cause an NPE
mEGLDisplay = null;
mEGLContext = null;
mEGLSurface = null;
mEGL = null;
mTextureRender = null;
mSurface = null;
mSurfaceTexture = null;
}
/**
* Makes our EGL context and surface current.
*/
private void makeCurrent() throws TranscodingException {
if (mEGL == null) {
throw new TranscodingException("not configured for makeCurrent");
}
checkEglError("before makeCurrent");
if (!mEGL.eglMakeCurrent(mEGLDisplay, mEGLSurface, mEGLSurface, mEGLContext)) {
throw new TranscodingException("eglMakeCurrent failed");
}
}
/**
* Returns the Surface that we draw onto.
*/
public Surface getSurface() {
return mSurface;
}
/**
* Replaces the fragment shader.
*/
void changeFragmentShader(String fragmentShader) throws TranscodingException {
mTextureRender.changeFragmentShader(fragmentShader);
}
/**
* Latches the next buffer into the texture. Must be called from the thread that created
* the OutputSurface object, after the onFrameAvailable callback has signaled that new
* data is available.
*/
void awaitNewImage() throws TranscodingException {
final int TIMEOUT_MS = 750;
synchronized (mFrameSyncObject) {
final long expireTime = System.currentTimeMillis() + TIMEOUT_MS;
while (!mFrameAvailable) {
try {
// Wait for onFrameAvailable() to signal us. Use a timeout to avoid
// stalling the test if it doesn't arrive.
mFrameSyncObject.wait(TIMEOUT_MS);
if (!mFrameAvailable && System.currentTimeMillis() > expireTime) {
throw new TranscodingException("Surface frame wait timed out");
}
} catch (InterruptedException ie) {
// shouldn't happen
throw new TranscodingException(ie);
}
}
mFrameAvailable = false;
}
// Latch the data.
TextureRender.checkGlError("before updateTexImage");
mSurfaceTexture.updateTexImage();
}
/**
* Draws the data from SurfaceTexture onto the current EGL surface.
*/
void drawImage() throws TranscodingException {
mTextureRender.drawFrame(mSurfaceTexture);
}
@Override
public void onFrameAvailable(SurfaceTexture st) {
if (VERBOSE) Log.d(TAG, "new frame available");
synchronized (mFrameSyncObject) {
if (mFrameAvailable) {
try {
throw new TranscodingException("mFrameAvailable already set, frame could be dropped");
} catch (TranscodingException e) {
e.printStackTrace();
}
}
mFrameAvailable = true;
mFrameSyncObject.notifyAll();
}
}
/**
* Checks for EGL errors.
*/
private void checkEglError(String msg) throws TranscodingException {
boolean failed = false;
int error;
while ((error = mEGL.eglGetError()) != EGL10.EGL_SUCCESS) {
Log.e(TAG, msg + ": EGL error: 0x" + Integer.toHexString(error));
failed = true;
}
if (failed) {
throw new TranscodingException("EGL error encountered (see log)");
}
}
}

258
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/TextureRender.java Normal file
View file

@ -0,0 +1,258 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* This file has been modified by Signal.
*/
package org.thoughtcrime.securesms.video.videoconverter;
import android.graphics.SurfaceTexture;
import android.opengl.GLES11Ext;
import android.opengl.GLES20;
import android.opengl.Matrix;
import org.signal.core.util.logging.Log;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
/**
* Code for rendering a texture onto a surface using OpenGL ES 2.0.
*/
final class TextureRender {
private static final String TAG = "TextureRender";
private static final int FLOAT_SIZE_BYTES = 4;
private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
private final float[] mTriangleVerticesData = {
// X, Y, Z, U, V
-1.0f, -1.0f, 0, 0.f, 0.f,
1.0f, -1.0f, 0, 1.f, 0.f,
-1.0f, 1.0f, 0, 0.f, 1.f,
1.0f, 1.0f, 0, 1.f, 1.f,
};
private final float[] mTriangleVerticesDataFlippedX = {
// X, Y, Z, U, V
-1.0f, -1.0f, 0, 1.f, 0.f,
1.0f, -1.0f, 0, 0.f, 0.f,
-1.0f, 1.0f, 0, 1.f, 1.f,
1.0f, 1.0f, 0, 0.f, 1.f,
};
private final FloatBuffer mTriangleVertices;
private static final String VERTEX_SHADER =
"uniform mat4 uMVPMatrix;\n" +
"uniform mat4 uSTMatrix;\n" +
"attribute vec4 aPosition;\n" +
"attribute vec4 aTextureCoord;\n" +
"varying vec2 vTextureCoord;\n" +
"void main() {\n" +
" gl_Position = uMVPMatrix * aPosition;\n" +
" vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
"}\n";
private static final String FRAGMENT_SHADER =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" + // highp here doesn't seem to matter
"varying vec2 vTextureCoord;\n" +
"uniform samplerExternalOES sTexture;\n" +
"void main() {\n" +
" gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
"}\n";
private final float[] mMVPMatrix = new float[16];
private final float[] mSTMatrix = new float[16];
private int mProgram;
private int mTextureID = -12345;
private int muMVPMatrixHandle;
private int muSTMatrixHandle;
private int maPositionHandle;
private int maTextureHandle;
TextureRender(boolean flipX) {
float[] verticesData = flipX ? mTriangleVerticesDataFlippedX : mTriangleVerticesData;
mTriangleVertices = ByteBuffer.allocateDirect(
verticesData.length * FLOAT_SIZE_BYTES)
.order(ByteOrder.nativeOrder()).asFloatBuffer();
mTriangleVertices.put(verticesData).position(0);
Matrix.setIdentityM(mSTMatrix, 0);
}
int getTextureId() {
return mTextureID;
}
void drawFrame(SurfaceTexture st) throws TranscodingException {
checkGlError("onDrawFrame start");
st.getTransformMatrix(mSTMatrix);
GLES20.glClearColor(0.0f, 1.0f, 0.0f, 1.0f);
GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);
GLES20.glUseProgram(mProgram);
checkGlError("glUseProgram");
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
checkGlError("glVertexAttribPointer maPosition");
GLES20.glEnableVertexAttribArray(maPositionHandle);
checkGlError("glEnableVertexAttribArray maPositionHandle");
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
GLES20.glVertexAttribPointer(maTextureHandle, 2, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
checkGlError("glVertexAttribPointer maTextureHandle");
GLES20.glEnableVertexAttribArray(maTextureHandle);
checkGlError("glEnableVertexAttribArray maTextureHandle");
Matrix.setIdentityM(mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
checkGlError("glDrawArrays");
GLES20.glFinish();
}
/**
* Initializes GL state. Call this after the EGL surface has been created and made current.
*/
void surfaceCreated() throws TranscodingException {
mProgram = createProgram(VERTEX_SHADER, FRAGMENT_SHADER);
if (mProgram == 0) {
throw new TranscodingException("failed creating program");
}
maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
checkGlError("glGetAttribLocation aPosition");
if (maPositionHandle == -1) {
throw new TranscodingException("Could not get attrib location for aPosition");
}
maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
checkGlError("glGetAttribLocation aTextureCoord");
if (maTextureHandle == -1) {
throw new TranscodingException("Could not get attrib location for aTextureCoord");
}
muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
checkGlError("glGetUniformLocation uMVPMatrix");
if (muMVPMatrixHandle == -1) {
throw new TranscodingException("Could not get attrib location for uMVPMatrix");
}
muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
checkGlError("glGetUniformLocation uSTMatrix");
if (muSTMatrixHandle == -1) {
throw new TranscodingException("Could not get attrib location for uSTMatrix");
}
int[] textures = new int[1];
GLES20.glGenTextures(1, textures, 0);
mTextureID = textures[0];
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);
checkGlError("glBindTexture mTextureID");
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
GLES20.GL_LINEAR);
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
GLES20.GL_LINEAR);
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S,
GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T,
GLES20.GL_CLAMP_TO_EDGE);
checkGlError("glTexParameter");
}
/**
* Replaces the fragment shader.
*/
public void changeFragmentShader(String fragmentShader) throws TranscodingException {
GLES20.glDeleteProgram(mProgram);
mProgram = createProgram(VERTEX_SHADER, fragmentShader);
if (mProgram == 0) {
throw new TranscodingException("failed creating program");
}
}
private static int loadShader(int shaderType, String source) throws TranscodingException {
int shader = GLES20.glCreateShader(shaderType);
checkGlError("glCreateShader type=" + shaderType);
GLES20.glShaderSource(shader, source);
GLES20.glCompileShader(shader);
int[] compiled = new int[1];
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
if (compiled[0] == 0) {
Log.e(TAG, "Could not compile shader " + shaderType + ":");
Log.e(TAG, " " + GLES20.glGetShaderInfoLog(shader));
GLES20.glDeleteShader(shader);
shader = 0;
}
return shader;
}
private int createProgram(String vertexSource, String fragmentSource) throws TranscodingException {
int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
if (vertexShader == 0) {
return 0;
}
int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
if (pixelShader == 0) {
return 0;
}
int program = GLES20.glCreateProgram();
checkGlError("glCreateProgram");
if (program == 0) {
Log.e(TAG, "Could not create program");
}
GLES20.glAttachShader(program, vertexShader);
checkGlError("glAttachShader");
GLES20.glAttachShader(program, pixelShader);
checkGlError("glAttachShader");
GLES20.glLinkProgram(program);
int[] linkStatus = new int[1];
GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
if (linkStatus[0] != GLES20.GL_TRUE) {
Log.e(TAG, "Could not link program: ");
Log.e(TAG, GLES20.glGetProgramInfoLog(program));
GLES20.glDeleteProgram(program);
program = 0;
}
return program;
}
static void checkGlError(String msg) throws TranscodingException {
boolean failed = false;
int error;
while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
Log.e(TAG, msg + ": GLES20 error: 0x" + Integer.toHexString(error));
failed = true;
}
if (failed) {
throw new TranscodingException("GLES20 error encountered (see log)");
}
}
}

16
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/TranscodingException.java Normal file
View file

@ -0,0 +1,16 @@
package org.thoughtcrime.securesms.video.videoconverter;
final class TranscodingException extends Exception {
TranscodingException(String message) {
super(message);
}
TranscodingException(Throwable inner) {
super(inner);
}
TranscodingException(String message, Throwable inner) {
super(message, inner);
}
}

207
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsExtractor.java Normal file
View file

@ -0,0 +1,207 @@
package org.thoughtcrime.securesms.video.videoconverter;
import android.graphics.Bitmap;
import android.media.MediaCodec;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.opengl.GLES20;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.video.interfaces.MediaInput;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
@RequiresApi(api = 23)
final class VideoThumbnailsExtractor {
private static final String TAG = Log.tag(VideoThumbnailsExtractor.class);
interface Callback {
void durationKnown(long duration);
boolean publishProgress(int index, Bitmap thumbnail);
void failed();
}
static void extractThumbnails(final @NonNull MediaInput input,
final int thumbnailCount,
final int thumbnailResolution,
final @NonNull Callback callback)
{
MediaExtractor extractor = null;
MediaCodec decoder = null;
OutputSurface outputSurface = null;
try {
extractor = input.createExtractor();
MediaFormat mediaFormat = null;
for (int index = 0; index < extractor.getTrackCount(); ++index) {
final String mimeType = extractor.getTrackFormat(index).getString(MediaFormat.KEY_MIME);
if (mimeType != null && mimeType.startsWith("video/")) {
extractor.selectTrack(index);
mediaFormat = extractor.getTrackFormat(index);
break;
}
}
if (mediaFormat != null) {
final String mime = mediaFormat.getString(MediaFormat.KEY_MIME);
if (mime == null) {
throw new IllegalArgumentException("Mime type for MediaFormat was null: \t" + mediaFormat);
}
final int rotation = mediaFormat.containsKey(MediaFormat.KEY_ROTATION) ? mediaFormat.getInteger(MediaFormat.KEY_ROTATION) : 0;
final int width = mediaFormat.getInteger(MediaFormat.KEY_WIDTH);
final int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
final int outputWidth;
final int outputHeight;
if (width < height) {
outputWidth = thumbnailResolution;
outputHeight = height * outputWidth / width;
} else {
outputHeight = thumbnailResolution;
outputWidth = width * outputHeight / height;
}
final int outputWidthRotated;
final int outputHeightRotated;
if ((rotation % 180 == 90)) {
//noinspection SuspiciousNameCombination
outputWidthRotated = outputHeight;
//noinspection SuspiciousNameCombination
outputHeightRotated = outputWidth;
} else {
outputWidthRotated = outputWidth;
outputHeightRotated = outputHeight;
}
Log.i(TAG, "video :" + width + "x" + height + " " + rotation);
Log.i(TAG, "output: " + outputWidthRotated + "x" + outputHeightRotated);
outputSurface = new OutputSurface(outputWidthRotated, outputHeightRotated, true);
decoder = MediaCodec.createDecoderByType(mime);
decoder.configure(mediaFormat, outputSurface.getSurface(), null, 0);
decoder.start();
long duration = 0;
if (mediaFormat.containsKey(MediaFormat.KEY_DURATION)) {
duration = mediaFormat.getLong(MediaFormat.KEY_DURATION);
} else {
Log.w(TAG, "Video is missing duration!");
}
callback.durationKnown(duration);
doExtract(extractor, decoder, outputSurface, outputWidthRotated, outputHeightRotated, duration, thumbnailCount, callback);
}
} catch (Throwable t) {
Log.w(TAG, t);
callback.failed();
} finally {
if (outputSurface != null) {
outputSurface.release();
}
if (decoder != null) {
try {
decoder.stop();
} catch (MediaCodec.CodecException codecException) {
Log.w(TAG, "Decoder stop failed: " + codecException.getDiagnosticInfo(), codecException);
} catch (IllegalStateException ise) {
Log.w(TAG, "Decoder stop failed", ise);
}
decoder.release();
}
if (extractor != null) {
extractor.release();
}
}
}
private static void doExtract(final @NonNull MediaExtractor extractor,
final @NonNull MediaCodec decoder,
final @NonNull OutputSurface outputSurface,
final int outputWidth, int outputHeight, long duration, int thumbnailCount,
final @NonNull Callback callback)
throws TranscodingException
{
final int TIMEOUT_USEC = 10000;
final ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers();
final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
int samplesExtracted = 0;
int thumbnailsCreated = 0;
Log.i(TAG, "doExtract started");
final ByteBuffer pixelBuf = ByteBuffer.allocateDirect(outputWidth * outputHeight * 4);
pixelBuf.order(ByteOrder.LITTLE_ENDIAN);
boolean outputDone = false;
boolean inputDone = false;
while (!outputDone) {
if (!inputDone) {
int inputBufIndex = decoder.dequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0) {
final ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex];
final int sampleSize = extractor.readSampleData(inputBuf, 0);
if (sampleSize < 0 || samplesExtracted >= thumbnailCount) {
decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
Log.i(TAG, "input done");
} else {
final long presentationTimeUs = extractor.getSampleTime();
decoder.queueInputBuffer(inputBufIndex, 0, sampleSize, presentationTimeUs, 0 /*flags*/);
samplesExtracted++;
extractor.seekTo(duration * samplesExtracted / thumbnailCount, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
Log.i(TAG, "seek to " + duration * samplesExtracted / thumbnailCount + ", actual " + extractor.getSampleTime());
}
}
}
final int outputBufIndex;
try {
outputBufIndex = decoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
} catch (IllegalStateException e) {
Log.w(TAG, "Decoder not in the Executing state, or codec is configured in asynchronous mode.", e);
throw new TranscodingException("Decoder not in the Executing state, or codec is configured in asynchronous mode.", e);
}
if (outputBufIndex >= 0) {
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
outputDone = true;
}
final boolean shouldRender = (info.size != 0) /*&& (info.presentationTimeUs >= duration * decodeCount / thumbnailCount)*/;
decoder.releaseOutputBuffer(outputBufIndex, shouldRender);
if (shouldRender) {
outputSurface.awaitNewImage();
outputSurface.drawImage();
if (thumbnailsCreated < thumbnailCount) {
pixelBuf.rewind();
GLES20.glReadPixels(0, 0, outputWidth, outputHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, pixelBuf);
final Bitmap bitmap = Bitmap.createBitmap(outputWidth, outputHeight, Bitmap.Config.ARGB_8888);
pixelBuf.rewind();
bitmap.copyPixelsFromBuffer(pixelBuf);
if (!callback.publishProgress(thumbnailsCreated, bitmap)) {
break;
}
Log.i(TAG, "publishProgress for frame " + thumbnailsCreated + " at " + info.presentationTimeUs + " (target " + duration * thumbnailsCreated / thumbnailCount + ")");
}
thumbnailsCreated++;
}
}
}
Log.i(TAG, "doExtract finished");
}
}

552
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java Normal file
View file

@ -0,0 +1,552 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.video.videoconverter;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.os.Build;
import android.view.Surface;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.video.interfaces.MediaInput;
import org.thoughtcrime.securesms.video.interfaces.Muxer;
import org.thoughtcrime.securesms.video.videoconverter.utils.Extensions;
import org.thoughtcrime.securesms.video.videoconverter.utils.MediaCodecCompat;
import org.thoughtcrime.securesms.video.videoconverter.utils.Preconditions;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicReference;
import kotlin.Pair;
final class VideoTrackConverter {
private static final String TAG = "media-converter";
private static final boolean VERBOSE = false; // lots of logging
private static final int OUTPUT_VIDEO_IFRAME_INTERVAL = 1; // 1 second between I-frames
private static final int OUTPUT_VIDEO_FRAME_RATE = 30; // needed only for MediaFormat.KEY_I_FRAME_INTERVAL to work; the actual frame rate matches the source
private static final int TIMEOUT_USEC = 10000;
private static final String MEDIA_FORMAT_KEY_DISPLAY_WIDTH = "display-width";
private static final String MEDIA_FORMAT_KEY_DISPLAY_HEIGHT = "display-height";
private static final float FRAME_RATE_TOLERANCE = 0.05f; // tolerance for transcoding VFR -> CFR
private final long mTimeFrom;
private final long mTimeTo;
final long mInputDuration;
private final MediaExtractor mVideoExtractor;
private final MediaCodec mVideoDecoder;
private final MediaCodec mVideoEncoder;
private final InputSurface mInputSurface;
private final OutputSurface mOutputSurface;
private final ByteBuffer[] mVideoDecoderInputBuffers;
private ByteBuffer[] mVideoEncoderOutputBuffers;
private final MediaCodec.BufferInfo mVideoDecoderOutputBufferInfo;
private final MediaCodec.BufferInfo mVideoEncoderOutputBufferInfo;
MediaFormat mEncoderOutputVideoFormat;
boolean mVideoExtractorDone;
private boolean mVideoDecoderDone;
boolean mVideoEncoderDone;
private int mOutputVideoTrack = -1;
long mMuxingVideoPresentationTime;
private int mVideoExtractedFrameCount;
private int mVideoDecodedFrameCount;
private int mVideoEncodedFrameCount;
private Muxer mMuxer;
static @Nullable VideoTrackConverter create(
final @NonNull MediaInput input,
final long timeFrom,
final long timeTo,
final int videoResolution,
final int videoBitrate,
final @NonNull String videoCodec) throws IOException, TranscodingException {
final MediaExtractor videoExtractor = input.createExtractor();
final int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor);
if (videoInputTrack == -1) {
videoExtractor.release();
return null;
}
return new VideoTrackConverter(videoExtractor, videoInputTrack, timeFrom, timeTo, videoResolution, videoBitrate, videoCodec);
}
private VideoTrackConverter(
final @NonNull MediaExtractor videoExtractor,
final int videoInputTrack,
final long timeFrom,
final long timeTo,
final int videoResolution,
final int videoBitrate,
final @NonNull String videoCodec) throws IOException, TranscodingException {
mTimeFrom = timeFrom;
mTimeTo = timeTo;
mVideoExtractor = videoExtractor;
final MediaCodecInfo videoCodecInfo = MediaConverter.selectCodec(videoCodec);
if (videoCodecInfo == null) {
// Don't fail CTS if they don't have an AVC codec (not here, anyway).
Log.e(TAG, "Unable to find an appropriate codec for " + videoCodec);
throw new FileNotFoundException();
}
if (VERBOSE) Log.d(TAG, "video found codec: " + videoCodecInfo.getName());
final MediaFormat inputVideoFormat = mVideoExtractor.getTrackFormat(videoInputTrack);
mInputDuration = inputVideoFormat.containsKey(MediaFormat.KEY_DURATION) ? inputVideoFormat.getLong(MediaFormat.KEY_DURATION) : 0;
final int rotation = inputVideoFormat.containsKey(MediaFormat.KEY_ROTATION) ? inputVideoFormat.getInteger(MediaFormat.KEY_ROTATION) : 0;
final int width = inputVideoFormat.containsKey(MEDIA_FORMAT_KEY_DISPLAY_WIDTH)
? inputVideoFormat.getInteger(MEDIA_FORMAT_KEY_DISPLAY_WIDTH)
: inputVideoFormat.getInteger(MediaFormat.KEY_WIDTH);
final int height = inputVideoFormat.containsKey(MEDIA_FORMAT_KEY_DISPLAY_HEIGHT)
? inputVideoFormat.getInteger(MEDIA_FORMAT_KEY_DISPLAY_HEIGHT)
: inputVideoFormat.getInteger(MediaFormat.KEY_HEIGHT);
int outputWidth = width;
int outputHeight = height;
if (outputWidth < outputHeight) {
outputWidth = videoResolution;
outputHeight = height * outputWidth / width;
} else {
outputHeight = videoResolution;
outputWidth = width * outputHeight / height;
}
// many encoders do not work when height and width are not multiple of 16 (also, some iPhones do not play some heights)
outputHeight = (outputHeight + 7) & ~0xF;
outputWidth = (outputWidth + 7) & ~0xF;
final int outputWidthRotated;
final int outputHeightRotated;
if ((rotation % 180 == 90)) {
//noinspection SuspiciousNameCombination
outputWidthRotated = outputHeight;
//noinspection SuspiciousNameCombination
outputHeightRotated = outputWidth;
} else {
outputWidthRotated = outputWidth;
outputHeightRotated = outputHeight;
}
final MediaFormat outputVideoFormat = MediaFormat.createVideoFormat(videoCodec, outputWidthRotated, outputHeightRotated);
// Set some properties. Failing to specify some of these can cause the MediaCodec
// configure() call to throw an unhelpful exception.
outputVideoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
outputVideoFormat.setInteger(MediaFormat.KEY_BIT_RATE, videoBitrate);
outputVideoFormat.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR);
outputVideoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, OUTPUT_VIDEO_FRAME_RATE);
outputVideoFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL);
if (Build.VERSION.SDK_INT >= 31 && isHdr(inputVideoFormat)) {
outputVideoFormat.setInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST, MediaFormat.COLOR_TRANSFER_SDR_VIDEO);
}
if (VERBOSE) Log.d(TAG, "video format: " + outputVideoFormat);
// Create a MediaCodec for the desired codec, then configure it as an encoder with
// our desired properties. Request a Surface to use for input.
final AtomicReference<Surface> inputSurfaceReference = new AtomicReference<>();
mVideoEncoder = createVideoEncoder(videoCodecInfo, outputVideoFormat, inputSurfaceReference);
mInputSurface = new InputSurface(inputSurfaceReference.get());
mInputSurface.makeCurrent();
// Create a MediaCodec for the decoder, based on the extractor's format.
mOutputSurface = new OutputSurface();
mOutputSurface.changeFragmentShader(createFragmentShader(
inputVideoFormat.getInteger(MediaFormat.KEY_WIDTH), inputVideoFormat.getInteger(MediaFormat.KEY_HEIGHT),
outputWidth, outputHeight));
mVideoDecoder = createVideoDecoder(inputVideoFormat, mOutputSurface.getSurface());
mVideoDecoderInputBuffers = mVideoDecoder.getInputBuffers();
mVideoEncoderOutputBuffers = mVideoEncoder.getOutputBuffers();
mVideoDecoderOutputBufferInfo = new MediaCodec.BufferInfo();
mVideoEncoderOutputBufferInfo = new MediaCodec.BufferInfo();
if (mTimeFrom > 0) {
mVideoExtractor.seekTo(mTimeFrom * 1000, MediaExtractor.SEEK_TO_PREVIOUS_SYNC);
Log.i(TAG, "Seek video:" + mTimeFrom + " " + mVideoExtractor.getSampleTime());
}
}
private boolean isHdr(MediaFormat inputVideoFormat) {
if (Build.VERSION.SDK_INT < 24) {
return false;
}
try {
final int colorInfo = inputVideoFormat.getInteger(MediaFormat.KEY_COLOR_TRANSFER);
return colorInfo == MediaFormat.COLOR_TRANSFER_ST2084 || colorInfo == MediaFormat.COLOR_TRANSFER_HLG;
} catch (NullPointerException npe) {
// color transfer key does not exist, no color data supplied
return false;
}
}
void setMuxer(final @NonNull Muxer muxer) throws IOException {
mMuxer = muxer;
if (mEncoderOutputVideoFormat != null) {
Log.d(TAG, "muxer: adding video track.");
mOutputVideoTrack = muxer.addTrack(mEncoderOutputVideoFormat);
}
}
void step() throws IOException, TranscodingException {
// Extract video from file and feed to decoder.
// Do not extract video if we have determined the output format but we are not yet
// ready to mux the frames.
while (!mVideoExtractorDone
&& (mEncoderOutputVideoFormat == null || mMuxer != null)) {
int decoderInputBufferIndex = mVideoDecoder.dequeueInputBuffer(TIMEOUT_USEC);
if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no video decoder input buffer");
break;
}
if (VERBOSE) {
Log.d(TAG, "video decoder: returned input buffer: " + decoderInputBufferIndex);
}
final ByteBuffer decoderInputBuffer = mVideoDecoderInputBuffers[decoderInputBufferIndex];
final int size = mVideoExtractor.readSampleData(decoderInputBuffer, 0);
final long presentationTime = mVideoExtractor.getSampleTime();
if (VERBOSE) {
Log.d(TAG, "video extractor: returned buffer of size " + size);
Log.d(TAG, "video extractor: returned buffer for time " + presentationTime);
}
mVideoExtractorDone = size < 0 || (mTimeTo > 0 && presentationTime > mTimeTo * 1000);
if (mVideoExtractorDone) {
if (VERBOSE) Log.d(TAG, "video extractor: EOS");
mVideoDecoder.queueInputBuffer(
decoderInputBufferIndex,
0,
0,
0,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
} else {
mVideoDecoder.queueInputBuffer(
decoderInputBufferIndex,
0,
size,
presentationTime,
mVideoExtractor.getSampleFlags());
}
mVideoExtractor.advance();
mVideoExtractedFrameCount++;
// We extracted a frame, let's try something else next.
break;
}
// Poll output frames from the video decoder and feed the encoder.
while (!mVideoDecoderDone && (mEncoderOutputVideoFormat == null || mMuxer != null)) {
final int decoderOutputBufferIndex =
mVideoDecoder.dequeueOutputBuffer(
mVideoDecoderOutputBufferInfo, TIMEOUT_USEC);
if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no video decoder output buffer");
break;
}
if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
if (VERBOSE) Log.d(TAG, "video decoder: output buffers changed");
break;
}
if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
if (VERBOSE) {
Log.d(TAG, "video decoder: output format changed: " + mVideoDecoder.getOutputFormat());
}
break;
}
if (VERBOSE) {
Log.d(TAG, "video decoder: returned output buffer: "
+ decoderOutputBufferIndex);
Log.d(TAG, "video decoder: returned buffer of size "
+ mVideoDecoderOutputBufferInfo.size);
}
if ((mVideoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
if (VERBOSE) Log.d(TAG, "video decoder: codec config buffer");
mVideoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false);
break;
}
if (mVideoDecoderOutputBufferInfo.presentationTimeUs < mTimeFrom * 1000 &&
(mVideoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == 0) {
if (VERBOSE) Log.d(TAG, "video decoder: frame prior to " + mVideoDecoderOutputBufferInfo.presentationTimeUs);
mVideoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false);
break;
}
if (VERBOSE) {
Log.d(TAG, "video decoder: returned buffer for time " + mVideoDecoderOutputBufferInfo.presentationTimeUs);
}
boolean render = mVideoDecoderOutputBufferInfo.size != 0;
mVideoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, render);
if (render) {
if (VERBOSE) Log.d(TAG, "output surface: await new image");
mOutputSurface.awaitNewImage();
// Edit the frame and send it to the encoder.
if (VERBOSE) Log.d(TAG, "output surface: draw image");
mOutputSurface.drawImage();
mInputSurface.setPresentationTime(mVideoDecoderOutputBufferInfo.presentationTimeUs * 1000);
if (VERBOSE) Log.d(TAG, "input surface: swap buffers");
mInputSurface.swapBuffers();
if (VERBOSE) Log.d(TAG, "video encoder: notified of new frame");
}
if ((mVideoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "video decoder: EOS");
mVideoDecoderDone = true;
mVideoEncoder.signalEndOfInputStream();
}
mVideoDecodedFrameCount++;
// We extracted a pending frame, let's try something else next.
break;
}
// Poll frames from the video encoder and send them to the muxer.
while (!mVideoEncoderDone && (mEncoderOutputVideoFormat == null || mMuxer != null)) {
final int encoderOutputBufferIndex = mVideoEncoder.dequeueOutputBuffer(mVideoEncoderOutputBufferInfo, TIMEOUT_USEC);
if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no video encoder output buffer");
if (mVideoDecoderDone) {
// on some devices and encoder stops after signalEndOfInputStream
Log.w(TAG, "mVideoDecoderDone, but didn't get BUFFER_FLAG_END_OF_STREAM");
mVideoEncodedFrameCount = mVideoDecodedFrameCount;
mVideoEncoderDone = true;
}
break;
}
if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
if (VERBOSE) Log.d(TAG, "video encoder: output buffers changed");
mVideoEncoderOutputBuffers = mVideoEncoder.getOutputBuffers();
break;
}
if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
if (VERBOSE) Log.d(TAG, "video encoder: output format changed");
Preconditions.checkState("video encoder changed its output format again?", mOutputVideoTrack < 0);
mEncoderOutputVideoFormat = mVideoEncoder.getOutputFormat();
break;
}
Preconditions.checkState("should have added track before processing output", mMuxer != null);
if (VERBOSE) {
Log.d(TAG, "video encoder: returned output buffer: " + encoderOutputBufferIndex);
Log.d(TAG, "video encoder: returned buffer of size " + mVideoEncoderOutputBufferInfo.size);
}
final ByteBuffer encoderOutputBuffer = mVideoEncoderOutputBuffers[encoderOutputBufferIndex];
if ((mVideoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
if (VERBOSE) Log.d(TAG, "video encoder: codec config buffer");
// Simply ignore codec config buffers.
mVideoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
break;
}
if (VERBOSE) {
Log.d(TAG, "video encoder: returned buffer for time " + mVideoEncoderOutputBufferInfo.presentationTimeUs);
}
if (mVideoEncoderOutputBufferInfo.size != 0) {
mMuxer.writeSampleData(mOutputVideoTrack, encoderOutputBuffer, mVideoEncoderOutputBufferInfo);
mMuxingVideoPresentationTime = Math.max(mMuxingVideoPresentationTime, mVideoEncoderOutputBufferInfo.presentationTimeUs);
}
if ((mVideoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "video encoder: EOS");
mVideoEncoderDone = true;
}
mVideoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
mVideoEncodedFrameCount++;
// We enqueued an encoded frame, let's try something else next.
break;
}
}
void release() throws Exception {
Exception exception = null;
try {
if (mVideoExtractor != null) {
mVideoExtractor.release();
}
} catch (Exception e) {
Log.e(TAG, "error while releasing mVideoExtractor", e);
exception = e;
}
try {
if (mVideoDecoder != null) {
mVideoDecoder.stop();
mVideoDecoder.release();
}
} catch (Exception e) {
Log.e(TAG, "error while releasing mVideoDecoder", e);
if (exception == null) {
exception = e;
}
}
try {
if (mOutputSurface != null) {
mOutputSurface.release();
}
} catch (Exception e) {
Log.e(TAG, "error while releasing mOutputSurface", e);
if (exception == null) {
exception = e;
}
}
try {
if (mInputSurface != null) {
mInputSurface.release();
}
} catch (Exception e) {
Log.e(TAG, "error while releasing mInputSurface", e);
if (exception == null) {
exception = e;
}
}
try {
if (mVideoEncoder != null) {
mVideoEncoder.stop();
mVideoEncoder.release();
}
} catch (Exception e) {
Log.e(TAG, "error while releasing mVideoEncoder", e);
if (exception == null) {
exception = e;
}
}
if (exception != null) {
throw exception;
}
}
VideoTrackConverterState dumpState() {
return new VideoTrackConverterState(
mVideoExtractedFrameCount, mVideoExtractorDone,
mVideoDecodedFrameCount, mVideoDecoderDone,
mVideoEncodedFrameCount, mVideoEncoderDone,
mMuxer != null, mOutputVideoTrack);
}
void verifyEndState() {
Preconditions.checkState("encoded (" + mVideoEncodedFrameCount + ") and decoded (" + mVideoDecodedFrameCount + ") video frame counts should match", Extensions.isWithin(mVideoDecodedFrameCount, mVideoEncodedFrameCount, FRAME_RATE_TOLERANCE));
Preconditions.checkState("decoded frame count should be less than extracted frame count", mVideoDecodedFrameCount <= mVideoExtractedFrameCount);
}
private static String createFragmentShader(
final int srcWidth,
final int srcHeight,
final int dstWidth,
final int dstHeight) {
final float kernelSizeX = (float) srcWidth / (float) dstWidth;
final float kernelSizeY = (float) srcHeight / (float) dstHeight;
Log.i(TAG, "kernel " + kernelSizeX + "x" + kernelSizeY);
final String shader;
if (kernelSizeX <= 2 && kernelSizeY <= 2) {
shader =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" + // highp here doesn't seem to matter
"varying vec2 vTextureCoord;\n" +
"uniform samplerExternalOES sTexture;\n" +
"void main() {\n" +
" gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
"}\n";
} else {
final int kernelRadiusX = (int) Math.ceil(kernelSizeX - .1f) / 2;
final int kernelRadiusY = (int) Math.ceil(kernelSizeY - .1f) / 2;
final float stepX = kernelSizeX / (1 + 2 * kernelRadiusX) * (1f / srcWidth);
final float stepY = kernelSizeY / (1 + 2 * kernelRadiusY) * (1f / srcHeight);
final float sum = (1 + 2 * kernelRadiusX) * (1 + 2 * kernelRadiusY);
final StringBuilder colorLoop = new StringBuilder();
for (int i = -kernelRadiusX; i <=kernelRadiusX; i++) {
for (int j = -kernelRadiusY; j <=kernelRadiusY; j++) {
if (i != 0 || j != 0) {
colorLoop.append(" + texture2D(sTexture, vTextureCoord.xy + vec2(")
.append(i * stepX).append(", ").append(j * stepY).append("))\n");
}
}
}
shader =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" + // highp here doesn't seem to matter
"varying vec2 vTextureCoord;\n" +
"uniform samplerExternalOES sTexture;\n" +
"void main() {\n" +
" gl_FragColor = (texture2D(sTexture, vTextureCoord)\n" +
colorLoop +
" ) / " + sum + ";\n" +
"}\n";
}
Log.i(TAG, shader);
return shader;
}
private @NonNull
MediaCodec createVideoDecoder(
final @NonNull MediaFormat inputFormat,
final @NonNull Surface surface) {
final Pair<MediaCodec, MediaFormat> decoderPair = MediaCodecCompat.findDecoder(inputFormat);
final MediaCodec decoder = decoderPair.getFirst();
decoder.configure(decoderPair.getSecond(), surface, null, 0);
decoder.start();
return decoder;
}
private @NonNull
MediaCodec createVideoEncoder(
final @NonNull MediaCodecInfo codecInfo,
final @NonNull MediaFormat format,
final @NonNull AtomicReference<Surface> surfaceReference) throws IOException {
boolean tonemapRequested = isTonemapEnabled(format);
final MediaCodec encoder = MediaCodec.createByCodecName(codecInfo.getName());
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
if (tonemapRequested && !isTonemapEnabled(format)) {
Log.d(TAG, "HDR tone-mapping requested but not supported by the decoder.");
}
// Must be called before start()
surfaceReference.set(encoder.createInputSurface());
encoder.start();
return encoder;
}
private static boolean isTonemapEnabled(@NonNull MediaFormat format) {
if (Build.VERSION.SDK_INT < 31) {
return false;
}
try {
int request = format.getInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST);
return request == MediaFormat.COLOR_TRANSFER_SDR_VIDEO;
} catch (NullPointerException npe) {
// transfer request key does not exist, tone mapping not requested
return false;
}
}
private static int getAndSelectVideoTrackIndex(@NonNull MediaExtractor extractor) {
for (int index = 0; index < extractor.getTrackCount(); ++index) {
if (VERBOSE) {
Log.d(TAG, "format for track " + index + " is " + MediaConverter.getMimeTypeFor(extractor.getTrackFormat(index)));
}
if (isVideoFormat(extractor.getTrackFormat(index))) {
extractor.selectTrack(index);
return index;
}
}
return -1;
}
private static boolean isVideoFormat(final @NonNull MediaFormat format) {
return MediaConverter.getMimeTypeFor(format).startsWith("video/");
}
}

10
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/exceptions/EncodingException.kt Normal file
View file

@ -0,0 +1,10 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.video.videoconverter.exceptions
class EncodingException : Exception {
constructor(message: String?) : super(message)
constructor(message: String?, inner: Exception?) : super(message, inner)
}

45
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/mediadatasource/InputStreamMediaDataSource.kt Normal file
View file

@ -0,0 +1,45 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.video.videoconverter.mediadatasource
import android.media.MediaDataSource
import java.io.IOException
import java.io.InputStream
/**
* Extend this class in order to be able to use the system media framework with any arbitrary [InputStream] of bytes.
*/
abstract class InputStreamMediaDataSource : MediaDataSource() {
@Throws(IOException::class)
override fun readAt(position: Long, bytes: ByteArray?, offset: Int, length: Int): Int {
if (position >= size) {
return -1
}
createInputStream(position).use { inputStream ->
var totalRead = 0
while (totalRead < length) {
val read: Int = inputStream.read(bytes, offset + totalRead, length - totalRead)
if (read == -1) {
return if (totalRead == 0) {
-1
} else {
totalRead
}
}
totalRead += read
}
return totalRead
}
}
abstract override fun close()
abstract override fun getSize(): Long
@Throws(IOException::class)
abstract fun createInputStream(position: Long): InputStream
}

32
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/mediadatasource/MediaDataSourceMediaInput.kt Normal file
View file

@ -0,0 +1,32 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.video.videoconverter.mediadatasource
import android.media.MediaDataSource
import android.media.MediaExtractor
import org.thoughtcrime.securesms.video.interfaces.MediaInput
import java.io.IOException
/**
* [MediaInput] implementation that adds support for the system framework's media data source.
*/
class MediaDataSourceMediaInput(private val mediaDataSource: MediaDataSource) : MediaInput {
@Throws(IOException::class)
override fun createExtractor(): MediaExtractor {
return MediaExtractor().apply {
setDataSource(mediaDataSource)
}
}
override fun hasSameInput(other: MediaInput): Boolean {
return other is MediaDataSourceMediaInput && other.mediaDataSource == this.mediaDataSource
}
@Throws(IOException::class)
override fun close() {
mediaDataSource.close()
}
}

123
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/AacTrack.java Normal file
View file

@ -0,0 +1,123 @@
package org.thoughtcrime.securesms.video.videoconverter.muxer;
import android.util.SparseIntArray;
import org.mp4parser.boxes.iso14496.part1.objectdescriptors.AudioSpecificConfig;
import org.mp4parser.boxes.iso14496.part1.objectdescriptors.DecoderConfigDescriptor;
import org.mp4parser.boxes.iso14496.part1.objectdescriptors.DecoderSpecificInfo;
import org.mp4parser.boxes.iso14496.part1.objectdescriptors.ESDescriptor;
import org.mp4parser.boxes.iso14496.part1.objectdescriptors.SLConfigDescriptor;
import org.mp4parser.boxes.iso14496.part12.SampleDescriptionBox;
import org.mp4parser.boxes.iso14496.part14.ESDescriptorBox;
import org.mp4parser.boxes.sampleentry.AudioSampleEntry;
import org.mp4parser.streaming.extensions.DefaultSampleFlagsTrackExtension;
import org.mp4parser.streaming.input.AbstractStreamingTrack;
import org.mp4parser.streaming.input.StreamingSampleImpl;
import java.io.IOException;
import java.nio.ByteBuffer;
import androidx.annotation.Nullable;
abstract class AacTrack extends AbstractStreamingTrack {
private static final SparseIntArray SAMPLING_FREQUENCY_INDEX_MAP = new SparseIntArray();
static {
SAMPLING_FREQUENCY_INDEX_MAP.put(96000, 0);
SAMPLING_FREQUENCY_INDEX_MAP.put(88200, 1);
SAMPLING_FREQUENCY_INDEX_MAP.put(64000, 2);
SAMPLING_FREQUENCY_INDEX_MAP.put(48000, 3);
SAMPLING_FREQUENCY_INDEX_MAP.put(44100, 4);
SAMPLING_FREQUENCY_INDEX_MAP.put(32000, 5);
SAMPLING_FREQUENCY_INDEX_MAP.put(24000, 6);
SAMPLING_FREQUENCY_INDEX_MAP.put(22050, 7);
SAMPLING_FREQUENCY_INDEX_MAP.put(16000, 8);
SAMPLING_FREQUENCY_INDEX_MAP.put(12000, 9);
SAMPLING_FREQUENCY_INDEX_MAP.put(11025, 10);
SAMPLING_FREQUENCY_INDEX_MAP.put(8000, 11);
}
private final SampleDescriptionBox stsd;
private int sampleRate;
AacTrack(long avgBitrate, long maxBitrate, int sampleRate, int channelCount, int aacProfile, @Nullable DecoderSpecificInfo decoderSpecificInfo) {
this.sampleRate = sampleRate;
final DefaultSampleFlagsTrackExtension defaultSampleFlagsTrackExtension = new DefaultSampleFlagsTrackExtension();
defaultSampleFlagsTrackExtension.setIsLeading(2);
defaultSampleFlagsTrackExtension.setSampleDependsOn(2);
defaultSampleFlagsTrackExtension.setSampleIsDependedOn(2);
defaultSampleFlagsTrackExtension.setSampleHasRedundancy(2);
defaultSampleFlagsTrackExtension.setSampleIsNonSyncSample(false);
this.addTrackExtension(defaultSampleFlagsTrackExtension);
stsd = new SampleDescriptionBox();
final AudioSampleEntry audioSampleEntry = new AudioSampleEntry("mp4a");
if (channelCount == 7) {
audioSampleEntry.setChannelCount(8);
} else {
audioSampleEntry.setChannelCount(channelCount);
}
audioSampleEntry.setSampleRate(sampleRate);
audioSampleEntry.setDataReferenceIndex(1);
audioSampleEntry.setSampleSize(16);
final ESDescriptorBox esds = new ESDescriptorBox();
ESDescriptor descriptor = new ESDescriptor();
descriptor.setEsId(0);
final SLConfigDescriptor slConfigDescriptor = new SLConfigDescriptor();
slConfigDescriptor.setPredefined(2);
descriptor.setSlConfigDescriptor(slConfigDescriptor);
final DecoderConfigDescriptor decoderConfigDescriptor = new DecoderConfigDescriptor();
decoderConfigDescriptor.setObjectTypeIndication(0x40 /*Audio ISO/IEC 14496-3*/);
decoderConfigDescriptor.setStreamType(5 /*audio stream*/);
decoderConfigDescriptor.setBufferSizeDB(1536);
decoderConfigDescriptor.setMaxBitRate(maxBitrate);
decoderConfigDescriptor.setAvgBitRate(avgBitrate);
final AudioSpecificConfig audioSpecificConfig = new AudioSpecificConfig();
audioSpecificConfig.setOriginalAudioObjectType(aacProfile);
audioSpecificConfig.setSamplingFrequencyIndex(SAMPLING_FREQUENCY_INDEX_MAP.get(sampleRate));
audioSpecificConfig.setChannelConfiguration(channelCount);
decoderConfigDescriptor.setAudioSpecificInfo(audioSpecificConfig);
if (decoderSpecificInfo != null) {
decoderConfigDescriptor.setDecoderSpecificInfo(decoderSpecificInfo);
}
descriptor.setDecoderConfigDescriptor(decoderConfigDescriptor);
esds.setEsDescriptor(descriptor);
audioSampleEntry.addBox(esds);
stsd.addBox(audioSampleEntry);
}
public long getTimescale() {
return sampleRate;
}
public String getHandler() {
return "soun";
}
public String getLanguage() {
return "\u0060\u0060\u0060"; // 0 in Iso639
}
public synchronized SampleDescriptionBox getSampleDescriptionBox() {
return stsd;
}
public void close() {
}
void processSample(ByteBuffer frame) throws IOException {
sampleSink.acceptSample(new StreamingSampleImpl(frame, 1024), this);
}
}

478
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/AvcTrack.java Normal file
View file

@ -0,0 +1,478 @@
package org.thoughtcrime.securesms.video.videoconverter.muxer;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.mp4parser.boxes.iso14496.part12.SampleDescriptionBox;
import org.mp4parser.boxes.iso14496.part15.AvcConfigurationBox;
import org.mp4parser.boxes.sampleentry.VisualSampleEntry;
import org.mp4parser.streaming.SampleExtension;
import org.mp4parser.streaming.StreamingSample;
import org.mp4parser.streaming.extensions.CompositionTimeSampleExtension;
import org.mp4parser.streaming.extensions.CompositionTimeTrackExtension;
import org.mp4parser.streaming.extensions.DimensionTrackExtension;
import org.mp4parser.streaming.extensions.SampleFlagsSampleExtension;
import org.mp4parser.streaming.input.AbstractStreamingTrack;
import org.mp4parser.streaming.input.StreamingSampleImpl;
import org.mp4parser.streaming.input.h264.H264NalUnitHeader;
import org.mp4parser.streaming.input.h264.H264NalUnitTypes;
import org.mp4parser.streaming.input.h264.spspps.PictureParameterSet;
import org.mp4parser.streaming.input.h264.spspps.SeqParameterSet;
import org.mp4parser.streaming.input.h264.spspps.SliceHeader;
import org.signal.core.util.logging.Log;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
abstract class AvcTrack extends AbstractStreamingTrack {
private static final String TAG = "AvcTrack";
private int maxDecFrameBuffering = 16;
private final List<StreamingSample> decFrameBuffer = new ArrayList<>();
private final List<StreamingSample> decFrameBuffer2 = new ArrayList<>();
private final LinkedHashMap<Integer, ByteBuffer> spsIdToSpsBytes = new LinkedHashMap<>();
private final LinkedHashMap<Integer, SeqParameterSet> spsIdToSps = new LinkedHashMap<>();
private final LinkedHashMap<Integer, ByteBuffer> ppsIdToPpsBytes = new LinkedHashMap<>();
private final LinkedHashMap<Integer, PictureParameterSet> ppsIdToPps = new LinkedHashMap<>();
private int timescale = 90000;
private int frametick = 3000;
private final SampleDescriptionBox stsd;
private final List<ByteBuffer> bufferedNals = new ArrayList<>();
private FirstVclNalDetector fvnd;
private H264NalUnitHeader sliceNalUnitHeader;
private long currentPresentationTimeUs;
AvcTrack(final @NonNull ByteBuffer spsBuffer, final @NonNull ByteBuffer ppsBuffer) {
handlePPS(ppsBuffer);
final SeqParameterSet sps = handleSPS(spsBuffer);
int width = (sps.pic_width_in_mbs_minus1 + 1) * 16;
int mult = 2;
if (sps.frame_mbs_only_flag) {
mult = 1;
}
int height = 16 * (sps.pic_height_in_map_units_minus1 + 1) * mult;
if (sps.frame_cropping_flag) {
int chromaArrayType = 0;
if (!sps.residual_color_transform_flag) {
chromaArrayType = sps.chroma_format_idc.getId();
}
int cropUnitX = 1;
int cropUnitY = mult;
if (chromaArrayType != 0) {
cropUnitX = sps.chroma_format_idc.getSubWidth();
cropUnitY = sps.chroma_format_idc.getSubHeight() * mult;
}
width -= cropUnitX * (sps.frame_crop_left_offset + sps.frame_crop_right_offset);
height -= cropUnitY * (sps.frame_crop_top_offset + sps.frame_crop_bottom_offset);
}
final VisualSampleEntry visualSampleEntry = new VisualSampleEntry("avc1");
visualSampleEntry.setDataReferenceIndex(1);
visualSampleEntry.setDepth(24);
visualSampleEntry.setFrameCount(1);
visualSampleEntry.setHorizresolution(72);
visualSampleEntry.setVertresolution(72);
final DimensionTrackExtension dte = this.getTrackExtension(DimensionTrackExtension.class);
if (dte == null) {
this.addTrackExtension(new DimensionTrackExtension(width, height));
}
visualSampleEntry.setWidth(width);
visualSampleEntry.setHeight(height);
visualSampleEntry.setCompressorname("AVC Coding");
final AvcConfigurationBox avcConfigurationBox = new AvcConfigurationBox();
avcConfigurationBox.setSequenceParameterSets(Collections.singletonList(spsBuffer));
avcConfigurationBox.setPictureParameterSets(Collections.singletonList(ppsBuffer));
avcConfigurationBox.setAvcLevelIndication(sps.level_idc);
avcConfigurationBox.setAvcProfileIndication(sps.profile_idc);
avcConfigurationBox.setBitDepthLumaMinus8(sps.bit_depth_luma_minus8);
avcConfigurationBox.setBitDepthChromaMinus8(sps.bit_depth_chroma_minus8);
avcConfigurationBox.setChromaFormat(sps.chroma_format_idc.getId());
avcConfigurationBox.setConfigurationVersion(1);
avcConfigurationBox.setLengthSizeMinusOne(3);
avcConfigurationBox.setProfileCompatibility(
(sps.constraint_set_0_flag ? 128 : 0) +
(sps.constraint_set_1_flag ? 64 : 0) +
(sps.constraint_set_2_flag ? 32 : 0) +
(sps.constraint_set_3_flag ? 16 : 0) +
(sps.constraint_set_4_flag ? 8 : 0) +
(int) (sps.reserved_zero_2bits & 0x3)
);
visualSampleEntry.addBox(avcConfigurationBox);
stsd = new SampleDescriptionBox();
stsd.addBox(visualSampleEntry);
int _timescale;
int _frametick;
if (sps.vuiParams != null) {
_timescale = sps.vuiParams.time_scale >> 1; // Not sure why, but I found this in several places, and it works...
_frametick = sps.vuiParams.num_units_in_tick;
if (_timescale == 0 || _frametick == 0) {
Log.w(TAG, "vuiParams contain invalid values: time_scale: " + _timescale + " and frame_tick: " + _frametick + ". Setting frame rate to 30fps");
_timescale = 0;
_frametick = 0;
}
if (_frametick > 0) {
if (_timescale / _frametick > 100) {
Log.w(TAG, "Framerate is " + (_timescale / _frametick) + ". That is suspicious.");
}
} else {
Log.w(TAG, "Frametick is " + _frametick + ". That is suspicious.");
}
if (sps.vuiParams.bitstreamRestriction != null) {
maxDecFrameBuffering = sps.vuiParams.bitstreamRestriction.max_dec_frame_buffering;
}
} else {
Log.w(TAG, "Can't determine frame rate as SPS does not contain vuiParama");
_timescale = 0;
_frametick = 0;
}
if (_timescale != 0 && _frametick != 0) {
timescale = _timescale;
frametick = _frametick;
}
if (sps.pic_order_cnt_type == 0) {
addTrackExtension(new CompositionTimeTrackExtension());
} else if (sps.pic_order_cnt_type == 1) {
throw new MuxingException("Have not yet imlemented pic_order_cnt_type 1");
}
}
public long getTimescale() {
return timescale;
}
public String getHandler() {
return "vide";
}
public String getLanguage() {
return "\u0060\u0060\u0060"; // 0 in Iso639
}
public SampleDescriptionBox getSampleDescriptionBox() {
return stsd;
}
public void close() {
}
private static H264NalUnitHeader getNalUnitHeader(@NonNull final ByteBuffer nal) {
final H264NalUnitHeader nalUnitHeader = new H264NalUnitHeader();
final int type = nal.get(0);
nalUnitHeader.nal_ref_idc = (type >> 5) & 3;
nalUnitHeader.nal_unit_type = type & 0x1f;
return nalUnitHeader;
}
void consumeNal(@NonNull final ByteBuffer nal, final long presentationTimeUs) throws IOException {
final H264NalUnitHeader nalUnitHeader = getNalUnitHeader(nal);
switch (nalUnitHeader.nal_unit_type) {
case H264NalUnitTypes.CODED_SLICE_NON_IDR:
case H264NalUnitTypes.CODED_SLICE_DATA_PART_A:
case H264NalUnitTypes.CODED_SLICE_DATA_PART_B:
case H264NalUnitTypes.CODED_SLICE_DATA_PART_C:
case H264NalUnitTypes.CODED_SLICE_IDR:
final FirstVclNalDetector current = new FirstVclNalDetector(nal, nalUnitHeader.nal_ref_idc, nalUnitHeader.nal_unit_type);
if (fvnd != null && fvnd.isFirstInNew(current)) {
pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false);
bufferedNals.clear();
}
currentPresentationTimeUs = Math.max(currentPresentationTimeUs, presentationTimeUs);
sliceNalUnitHeader = nalUnitHeader;
fvnd = current;
bufferedNals.add(nal);
break;
case H264NalUnitTypes.SEI:
case H264NalUnitTypes.AU_UNIT_DELIMITER:
if (fvnd != null) {
pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false);
bufferedNals.clear();
fvnd = null;
}
bufferedNals.add(nal);
break;
case H264NalUnitTypes.SEQ_PARAMETER_SET:
if (fvnd != null) {
pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false);
bufferedNals.clear();
fvnd = null;
}
handleSPS(nal);
break;
case H264NalUnitTypes.PIC_PARAMETER_SET:
if (fvnd != null) {
pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false);
bufferedNals.clear();
fvnd = null;
}
handlePPS(nal);
break;
case H264NalUnitTypes.END_OF_SEQUENCE:
case H264NalUnitTypes.END_OF_STREAM:
return;
case H264NalUnitTypes.SEQ_PARAMETER_SET_EXT:
throw new IOException("Sequence parameter set extension is not yet handled. Needs TLC.");
default:
Log.w(TAG, "Unknown NAL unit type: " + nalUnitHeader.nal_unit_type);
}
}
void consumeLastNal() throws IOException {
pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, 0), true, true);
}
private void pushSample(final StreamingSample ss, final boolean all, final boolean force) throws IOException {
if (ss != null) {
decFrameBuffer.add(ss);
}
if (all) {
while (decFrameBuffer.size() > 0) {
pushSample(null, false, true);
}
} else {
if ((decFrameBuffer.size() - 1 > maxDecFrameBuffering) || force) {
final StreamingSample first = decFrameBuffer.remove(0);
final PictureOrderCountType0SampleExtension poct0se = first.getSampleExtension(PictureOrderCountType0SampleExtension.class);
if (poct0se == null) {
sampleSink.acceptSample(first, this);
} else {
int delay = 0;
for (StreamingSample streamingSample : decFrameBuffer) {
if (poct0se.getPoc() > streamingSample.getSampleExtension(PictureOrderCountType0SampleExtension.class).getPoc()) {
delay++;
}
}
for (StreamingSample streamingSample : decFrameBuffer2) {
if (poct0se.getPoc() < streamingSample.getSampleExtension(PictureOrderCountType0SampleExtension.class).getPoc()) {
delay--;
}
}
decFrameBuffer2.add(first);
if (decFrameBuffer2.size() > maxDecFrameBuffering) {
decFrameBuffer2.remove(0).removeSampleExtension(PictureOrderCountType0SampleExtension.class);
}
first.addSampleExtension(CompositionTimeSampleExtension.create(delay * frametick));
sampleSink.acceptSample(first, this);
}
}
}
}
private SampleFlagsSampleExtension createSampleFlagsSampleExtension(H264NalUnitHeader nu, SliceHeader sliceHeader) {
final SampleFlagsSampleExtension sampleFlagsSampleExtension = new SampleFlagsSampleExtension();
if (nu.nal_ref_idc == 0) {
sampleFlagsSampleExtension.setSampleIsDependedOn(2);
} else {
sampleFlagsSampleExtension.setSampleIsDependedOn(1);
}
if ((sliceHeader.slice_type == SliceHeader.SliceType.I) || (sliceHeader.slice_type == SliceHeader.SliceType.SI)) {
sampleFlagsSampleExtension.setSampleDependsOn(2);
} else {
sampleFlagsSampleExtension.setSampleDependsOn(1);
}
sampleFlagsSampleExtension.setSampleIsNonSyncSample(H264NalUnitTypes.CODED_SLICE_IDR != nu.nal_unit_type);
return sampleFlagsSampleExtension;
}
private PictureOrderCountType0SampleExtension createPictureOrderCountType0SampleExtension(SliceHeader sliceHeader) {
if (sliceHeader.sps.pic_order_cnt_type == 0) {
return new PictureOrderCountType0SampleExtension(
sliceHeader, decFrameBuffer.size() > 0 ?
decFrameBuffer.get(decFrameBuffer.size() - 1).getSampleExtension(PictureOrderCountType0SampleExtension.class) :
null);
/* decFrameBuffer.add(ssi);
if (decFrameBuffer.size() - 1 > maxDecFrameBuffering) { // just added one
drainDecPictureBuffer(false);
}*/
} else if (sliceHeader.sps.pic_order_cnt_type == 1) {
throw new MuxingException("pic_order_cnt_type == 1 needs to be implemented");
} else if (sliceHeader.sps.pic_order_cnt_type == 2) {
return null; // no ctts
}
throw new MuxingException("I don't know sliceHeader.sps.pic_order_cnt_type of " + sliceHeader.sps.pic_order_cnt_type);
}
private StreamingSample createSample(List<ByteBuffer> nals, SliceHeader sliceHeader, H264NalUnitHeader nu, long sampleDurationNs) {
final long sampleDuration = getTimescale() * Math.max(0, sampleDurationNs) / 1000000L;
final StreamingSample ss = new StreamingSampleImpl(nals, sampleDuration);
ss.addSampleExtension(createSampleFlagsSampleExtension(nu, sliceHeader));
final SampleExtension pictureOrderCountType0SampleExtension = createPictureOrderCountType0SampleExtension(sliceHeader);
if (pictureOrderCountType0SampleExtension != null) {
ss.addSampleExtension(pictureOrderCountType0SampleExtension);
}
return ss;
}
private void handlePPS(final @NonNull ByteBuffer nal) {
nal.position(1);
try {
final PictureParameterSet _pictureParameterSet = PictureParameterSet.read(nal);
final ByteBuffer oldPpsSameId = ppsIdToPpsBytes.get(_pictureParameterSet.pic_parameter_set_id);
if (oldPpsSameId != null && !oldPpsSameId.equals(nal)) {
throw new MuxingException("OMG - I got two SPS with same ID but different settings! (AVC3 is the solution)");
} else {
ppsIdToPpsBytes.put(_pictureParameterSet.pic_parameter_set_id, nal);
ppsIdToPps.put(_pictureParameterSet.pic_parameter_set_id, _pictureParameterSet);
}
} catch (IOException e) {
throw new MuxingException("That's surprising to get IOException when working on ByteArrayInputStream", e);
}
}
private @NonNull SeqParameterSet handleSPS(final @NonNull ByteBuffer nal) {
nal.position(1);
try {
final SeqParameterSet seqParameterSet = SeqParameterSet.read(nal);
final ByteBuffer oldSpsSameId = spsIdToSpsBytes.get(seqParameterSet.seq_parameter_set_id);
if (oldSpsSameId != null && !oldSpsSameId.equals(nal)) {
throw new MuxingException("OMG - I got two SPS with same ID but different settings!");
} else {
spsIdToSpsBytes.put(seqParameterSet.seq_parameter_set_id, nal);
spsIdToSps.put(seqParameterSet.seq_parameter_set_id, seqParameterSet);
}
return seqParameterSet;
} catch (IOException e) {
throw new MuxingException("That's surprising to get IOException when working on ByteArrayInputStream", e);
}
}
class FirstVclNalDetector {
final SliceHeader sliceHeader;
final int frame_num;
final int pic_parameter_set_id;
final boolean field_pic_flag;
final boolean bottom_field_flag;
final int nal_ref_idc;
final int pic_order_cnt_type;
final int delta_pic_order_cnt_bottom;
final int pic_order_cnt_lsb;
final int delta_pic_order_cnt_0;
final int delta_pic_order_cnt_1;
final int idr_pic_id;
FirstVclNalDetector(ByteBuffer nal, int nal_ref_idc, int nal_unit_type) {
SliceHeader sh = new SliceHeader(nal, spsIdToSps, ppsIdToPps, nal_unit_type == 5);
this.sliceHeader = sh;
this.frame_num = sh.frame_num;
this.pic_parameter_set_id = sh.pic_parameter_set_id;
this.field_pic_flag = sh.field_pic_flag;
this.bottom_field_flag = sh.bottom_field_flag;
this.nal_ref_idc = nal_ref_idc;
this.pic_order_cnt_type = spsIdToSps.get(ppsIdToPps.get(sh.pic_parameter_set_id).seq_parameter_set_id).pic_order_cnt_type;
this.delta_pic_order_cnt_bottom = sh.delta_pic_order_cnt_bottom;
this.pic_order_cnt_lsb = sh.pic_order_cnt_lsb;
this.delta_pic_order_cnt_0 = sh.delta_pic_order_cnt_0;
this.delta_pic_order_cnt_1 = sh.delta_pic_order_cnt_1;
this.idr_pic_id = sh.idr_pic_id;
}
boolean isFirstInNew(FirstVclNalDetector nu) {
if (nu.frame_num != frame_num) {
return true;
}
if (nu.pic_parameter_set_id != pic_parameter_set_id) {
return true;
}
if (nu.field_pic_flag != field_pic_flag) {
return true;
}
if (nu.field_pic_flag) {
if (nu.bottom_field_flag != bottom_field_flag) {
return true;
}
}
if (nu.nal_ref_idc != nal_ref_idc) {
return true;
}
if (nu.pic_order_cnt_type == 0 && pic_order_cnt_type == 0) {
if (nu.pic_order_cnt_lsb != pic_order_cnt_lsb) {
return true;
}
if (nu.delta_pic_order_cnt_bottom != delta_pic_order_cnt_bottom) {
return true;
}
}
if (nu.pic_order_cnt_type == 1 && pic_order_cnt_type == 1) {
if (nu.delta_pic_order_cnt_0 != delta_pic_order_cnt_0) {
return true;
}
if (nu.delta_pic_order_cnt_1 != delta_pic_order_cnt_1) {
return true;
}
}
return false;
}
}
static class PictureOrderCountType0SampleExtension implements SampleExtension {
int picOrderCntMsb;
int picOrderCountLsb;
PictureOrderCountType0SampleExtension(final @NonNull SliceHeader currentSlice, final @Nullable PictureOrderCountType0SampleExtension previous) {
int prevPicOrderCntLsb = 0;
int prevPicOrderCntMsb = 0;
if (previous != null) {
prevPicOrderCntLsb = previous.picOrderCountLsb;
prevPicOrderCntMsb = previous.picOrderCntMsb;
}
final int maxPicOrderCountLsb = (1 << (currentSlice.sps.log2_max_pic_order_cnt_lsb_minus4 + 4));
// System.out.print(" pic_order_cnt_lsb " + pic_order_cnt_lsb + " " + max_pic_order_count);
picOrderCountLsb = currentSlice.pic_order_cnt_lsb;
picOrderCntMsb = 0;
if ((picOrderCountLsb < prevPicOrderCntLsb) && ((prevPicOrderCntLsb - picOrderCountLsb) >= (maxPicOrderCountLsb / 2))) {
picOrderCntMsb = prevPicOrderCntMsb + maxPicOrderCountLsb;
} else if ((picOrderCountLsb > prevPicOrderCntLsb) && ((picOrderCountLsb - prevPicOrderCntLsb) > (maxPicOrderCountLsb / 2))) {
picOrderCntMsb = prevPicOrderCntMsb - maxPicOrderCountLsb;
} else {
picOrderCntMsb = prevPicOrderCntMsb;
}
}
int getPoc() {
return picOrderCntMsb + picOrderCountLsb;
}
@NonNull
@Override
public String toString() {
return "picOrderCntMsb=" + picOrderCntMsb + ", picOrderCountLsb=" + picOrderCountLsb;
}
}
}

99
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/H264Utils.java Normal file
View file

@ -0,0 +1,99 @@
/*
* Copyright 2008-2019 JCodecProject
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer. Redistributions in binary form
* must reproduce the above copyright notice, this list of conditions and the
* following disclaimer in the documentation and/or other materials provided with
* the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* https://github.com/jcodec/jcodec/blob/master/src/main/java/org/jcodec/codecs/h264/H264Utils.java
*
* This file has been modified by Signal.
*/
package org.thoughtcrime.securesms.video.videoconverter.muxer;
import androidx.annotation.NonNull;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.List;
final class H264Utils {
private H264Utils() {}
static @NonNull List<ByteBuffer> getNals(ByteBuffer buffer) {
final List<ByteBuffer> nals = new ArrayList<>();
ByteBuffer nal;
while ((nal = nextNALUnit(buffer)) != null) {
nals.add(nal);
}
return nals;
}
static ByteBuffer nextNALUnit(ByteBuffer buf) {
skipToNALUnit(buf);
return gotoNALUnit(buf);
}
static void skipToNALUnit(ByteBuffer buf) {
if (!buf.hasRemaining())
return;
int val = 0xffffffff;
while (buf.hasRemaining()) {
val <<= 8;
val |= (buf.get() & 0xff);
if ((val & 0xffffff) == 1) {
buf.position(buf.position());
break;
}
}
}
/**
* Finds next Nth H.264 bitstream NAL unit (0x00000001) and returns the data
* that preceeds it as a ByteBuffer slice
* <p>
* Segment byte order is always little endian
* <p>
* TODO: emulation prevention
*/
static ByteBuffer gotoNALUnit(ByteBuffer buf) {
if (!buf.hasRemaining())
return null;
int from = buf.position();
ByteBuffer result = buf.slice();
result.order(ByteOrder.BIG_ENDIAN);
int val = 0xffffffff;
while (buf.hasRemaining()) {
val <<= 8;
val |= (buf.get() & 0xff);
if ((val & 0xffffff) == 1) {
buf.position(buf.position() - (val == 1 ? 4 : 3));
result.limit(buf.position() - from);
break;
}
}
return result;
}
}

261
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/HevcTrack.java Normal file
View file

@ -0,0 +1,261 @@
package org.thoughtcrime.securesms.video.videoconverter.muxer;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.mp4parser.boxes.iso14496.part12.SampleDescriptionBox;
import org.mp4parser.boxes.iso14496.part15.HevcConfigurationBox;
import org.mp4parser.boxes.iso14496.part15.HevcDecoderConfigurationRecord;
import org.mp4parser.boxes.sampleentry.VisualSampleEntry;
import org.mp4parser.muxer.tracks.CleanInputStream;
import org.mp4parser.muxer.tracks.h265.H265NalUnitHeader;
import org.mp4parser.muxer.tracks.h265.H265NalUnitTypes;
import org.mp4parser.muxer.tracks.h265.SequenceParameterSetRbsp;
import org.mp4parser.streaming.StreamingSample;
import org.mp4parser.streaming.extensions.DimensionTrackExtension;
import org.mp4parser.streaming.extensions.SampleFlagsSampleExtension;
import org.mp4parser.streaming.input.AbstractStreamingTrack;
import org.mp4parser.streaming.input.StreamingSampleImpl;
import org.mp4parser.tools.ByteBufferByteChannel;
import org.mp4parser.tools.IsoTypeReader;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
abstract class HevcTrack extends AbstractStreamingTrack implements H265NalUnitTypes {
private final ArrayList<ByteBuffer> bufferedNals = new ArrayList<>();
private boolean vclNalUnitSeenInAU;
private boolean isIdr = true;
private long currentPresentationTimeUs;
private final SampleDescriptionBox stsd;
HevcTrack(final @NonNull List<ByteBuffer> csd) throws IOException {
final ArrayList<ByteBuffer> sps = new ArrayList<>();
final ArrayList<ByteBuffer> pps = new ArrayList<>();
final ArrayList<ByteBuffer> vps = new ArrayList<>();
SequenceParameterSetRbsp spsStruct = null;
for (ByteBuffer nal : csd) {
final H265NalUnitHeader unitHeader = getNalUnitHeader(nal);
nal.position(0);
// collect sps/vps/pps
switch (unitHeader.nalUnitType) {
case NAL_TYPE_PPS_NUT:
pps.add(nal.duplicate());
break;
case NAL_TYPE_VPS_NUT:
vps.add(nal.duplicate());
break;
case NAL_TYPE_SPS_NUT:
sps.add(nal.duplicate());
nal.position(2);
spsStruct = new SequenceParameterSetRbsp(new CleanInputStream(Channels.newInputStream(new ByteBufferByteChannel(nal.slice()))));
break;
case NAL_TYPE_PREFIX_SEI_NUT:
//new SEIMessage(new BitReaderBuffer(nal.slice()));
break;
}
}
stsd = new SampleDescriptionBox();
stsd.addBox(createSampleEntry(sps, pps, vps, spsStruct));
}
@Override
public long getTimescale() {
return 90000;
}
@Override
public String getHandler() {
return "vide";
}
@Override
public String getLanguage() {
return "\u0060\u0060\u0060"; // 0 in Iso639
}
@Override
public SampleDescriptionBox getSampleDescriptionBox() {
return stsd;
}
@Override
public void close() {
}
void consumeLastNal() throws IOException {
wrapUp(bufferedNals, currentPresentationTimeUs);
}
void consumeNal(final @NonNull ByteBuffer nal, final long presentationTimeUs) throws IOException {
final H265NalUnitHeader unitHeader = getNalUnitHeader(nal);
final boolean isVcl = isVcl(unitHeader);
//
if (vclNalUnitSeenInAU) { // we need at least 1 VCL per AU
// This branch checks if we encountered the start of a samples/AU
if (isVcl) {
if ((nal.get(2) & -128) != 0) { // this is: first_slice_segment_in_pic_flag u(1)
wrapUp(bufferedNals, presentationTimeUs);
}
} else {
switch (unitHeader.nalUnitType) {
case NAL_TYPE_PREFIX_SEI_NUT:
case NAL_TYPE_AUD_NUT:
case NAL_TYPE_PPS_NUT:
case NAL_TYPE_VPS_NUT:
case NAL_TYPE_SPS_NUT:
case NAL_TYPE_RSV_NVCL41:
case NAL_TYPE_RSV_NVCL42:
case NAL_TYPE_RSV_NVCL43:
case NAL_TYPE_RSV_NVCL44:
case NAL_TYPE_UNSPEC48:
case NAL_TYPE_UNSPEC49:
case NAL_TYPE_UNSPEC50:
case NAL_TYPE_UNSPEC51:
case NAL_TYPE_UNSPEC52:
case NAL_TYPE_UNSPEC53:
case NAL_TYPE_UNSPEC54:
case NAL_TYPE_UNSPEC55:
case NAL_TYPE_EOB_NUT: // a bit special but also causes a sample to be formed
case NAL_TYPE_EOS_NUT:
wrapUp(bufferedNals, presentationTimeUs);
break;
}
}
}
switch (unitHeader.nalUnitType) {
case NAL_TYPE_SPS_NUT:
case NAL_TYPE_VPS_NUT:
case NAL_TYPE_PPS_NUT:
case NAL_TYPE_EOB_NUT:
case NAL_TYPE_EOS_NUT:
case NAL_TYPE_AUD_NUT:
case NAL_TYPE_FD_NUT:
// ignore these
break;
default:
bufferedNals.add(nal);
break;
}
if (isVcl) {
isIdr = unitHeader.nalUnitType == NAL_TYPE_IDR_W_RADL || unitHeader.nalUnitType == NAL_TYPE_IDR_N_LP;
vclNalUnitSeenInAU = true;
}
}
private void wrapUp(final @NonNull List<ByteBuffer> nals, final long presentationTimeUs) throws IOException {
final long duration = presentationTimeUs - currentPresentationTimeUs;
currentPresentationTimeUs = presentationTimeUs;
final StreamingSample sample = new StreamingSampleImpl(
nals, getTimescale() * Math.max(0, duration) / 1000000L);
final SampleFlagsSampleExtension sampleFlagsSampleExtension = new SampleFlagsSampleExtension();
sampleFlagsSampleExtension.setSampleIsNonSyncSample(!isIdr);
sample.addSampleExtension(sampleFlagsSampleExtension);
sampleSink.acceptSample(sample, this);
vclNalUnitSeenInAU = false;
isIdr = true;
nals.clear();
}
private static @NonNull H265NalUnitHeader getNalUnitHeader(final @NonNull ByteBuffer nal) {
nal.position(0);
final int nalUnitHeaderValue = IsoTypeReader.readUInt16(nal);
final H265NalUnitHeader nalUnitHeader = new H265NalUnitHeader();
nalUnitHeader.forbiddenZeroFlag = (nalUnitHeaderValue & 0x8000) >> 15;
nalUnitHeader.nalUnitType = (nalUnitHeaderValue & 0x7E00) >> 9;
nalUnitHeader.nuhLayerId = (nalUnitHeaderValue & 0x1F8) >> 3;
nalUnitHeader.nuhTemporalIdPlusOne = (nalUnitHeaderValue & 0x7);
return nalUnitHeader;
}
private @NonNull VisualSampleEntry createSampleEntry(
final @NonNull ArrayList<ByteBuffer> sps,
final @NonNull ArrayList<ByteBuffer> pps,
final @NonNull ArrayList<ByteBuffer> vps,
final @Nullable SequenceParameterSetRbsp spsStruct)
{
final VisualSampleEntry visualSampleEntry = new VisualSampleEntry("hvc1");
visualSampleEntry.setDataReferenceIndex(1);
visualSampleEntry.setDepth(24);
visualSampleEntry.setFrameCount(1);
visualSampleEntry.setHorizresolution(72);
visualSampleEntry.setVertresolution(72);
visualSampleEntry.setCompressorname("HEVC Coding");
final HevcConfigurationBox hevcConfigurationBox = new HevcConfigurationBox();
hevcConfigurationBox.getHevcDecoderConfigurationRecord().setConfigurationVersion(1);
if (spsStruct != null) {
visualSampleEntry.setWidth(spsStruct.pic_width_in_luma_samples);
visualSampleEntry.setHeight(spsStruct.pic_height_in_luma_samples);
final DimensionTrackExtension dte = this.getTrackExtension(DimensionTrackExtension.class);
if (dte == null) {
this.addTrackExtension(new DimensionTrackExtension(spsStruct.pic_width_in_luma_samples, spsStruct.pic_height_in_luma_samples));
}
final HevcDecoderConfigurationRecord hevcDecoderConfigurationRecord = hevcConfigurationBox.getHevcDecoderConfigurationRecord();
hevcDecoderConfigurationRecord.setChromaFormat(spsStruct.chroma_format_idc);
hevcDecoderConfigurationRecord.setGeneral_profile_idc(spsStruct.general_profile_idc);
hevcDecoderConfigurationRecord.setGeneral_profile_compatibility_flags(spsStruct.general_profile_compatibility_flags);
hevcDecoderConfigurationRecord.setGeneral_constraint_indicator_flags(spsStruct.general_constraint_indicator_flags);
hevcDecoderConfigurationRecord.setGeneral_level_idc(spsStruct.general_level_idc);
hevcDecoderConfigurationRecord.setGeneral_tier_flag(spsStruct.general_tier_flag);
hevcDecoderConfigurationRecord.setGeneral_profile_space(spsStruct.general_profile_space);
hevcDecoderConfigurationRecord.setBitDepthChromaMinus8(spsStruct.bit_depth_chroma_minus8);
hevcDecoderConfigurationRecord.setBitDepthLumaMinus8(spsStruct.bit_depth_luma_minus8);
hevcDecoderConfigurationRecord.setTemporalIdNested(spsStruct.sps_temporal_id_nesting_flag);
}
hevcConfigurationBox.getHevcDecoderConfigurationRecord().setLengthSizeMinusOne(3);
final HevcDecoderConfigurationRecord.Array vpsArray = new HevcDecoderConfigurationRecord.Array();
vpsArray.array_completeness = false;
vpsArray.nal_unit_type = NAL_TYPE_VPS_NUT;
vpsArray.nalUnits = new ArrayList<>();
for (ByteBuffer vp : vps) {
vpsArray.nalUnits.add(Utils.toArray(vp));
}
final HevcDecoderConfigurationRecord.Array spsArray = new HevcDecoderConfigurationRecord.Array();
spsArray.array_completeness = false;
spsArray.nal_unit_type = NAL_TYPE_SPS_NUT;
spsArray.nalUnits = new ArrayList<>();
for (ByteBuffer sp : sps) {
spsArray.nalUnits.add(Utils.toArray(sp));
}
final HevcDecoderConfigurationRecord.Array ppsArray = new HevcDecoderConfigurationRecord.Array();
ppsArray.array_completeness = false;
ppsArray.nal_unit_type = NAL_TYPE_PPS_NUT;
ppsArray.nalUnits = new ArrayList<>();
for (ByteBuffer pp : pps) {
ppsArray.nalUnits.add(Utils.toArray(pp));
}
hevcConfigurationBox.getArrays().addAll(Arrays.asList(spsArray, vpsArray, ppsArray));
visualSampleEntry.addBox(hevcConfigurationBox);
return visualSampleEntry;
}
private boolean isVcl(final @NonNull H265NalUnitHeader nalUnitHeader) {
return nalUnitHeader.nalUnitType >= 0 && nalUnitHeader.nalUnitType <= 31;
}
}

523
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/Mp4Writer.java Normal file
View file

@ -0,0 +1,523 @@
/*
* Copyright (C) https://github.com/sannies/mp4parser/blob/master/LICENSE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* https://github.com/sannies/mp4parser/blob/4ed724754cde751c3f27fdda51f288df4f4c5db5/streaming/src/main/java/org/mp4parser/streaming/output/mp4/StandardMp4Writer.java
*
* This file has been modified by Signal.
*/
package org.thoughtcrime.securesms.video.videoconverter.muxer;
import androidx.annotation.NonNull;
import org.mp4parser.Box;
import org.mp4parser.boxes.iso14496.part12.ChunkOffsetBox;
import org.mp4parser.boxes.iso14496.part12.CompositionTimeToSample;
import org.mp4parser.boxes.iso14496.part12.FileTypeBox;
import org.mp4parser.boxes.iso14496.part12.MediaHeaderBox;
import org.mp4parser.boxes.iso14496.part12.MovieBox;
import org.mp4parser.boxes.iso14496.part12.MovieHeaderBox;
import org.mp4parser.boxes.iso14496.part12.SampleSizeBox;
import org.mp4parser.boxes.iso14496.part12.SampleTableBox;
import org.mp4parser.boxes.iso14496.part12.SampleToChunkBox;
import org.mp4parser.boxes.iso14496.part12.SyncSampleBox;
import org.mp4parser.boxes.iso14496.part12.TimeToSampleBox;
import org.mp4parser.boxes.iso14496.part12.TrackBox;
import org.mp4parser.boxes.iso14496.part12.TrackHeaderBox;
import org.mp4parser.streaming.StreamingSample;
import org.mp4parser.streaming.StreamingTrack;
import org.mp4parser.streaming.extensions.CompositionTimeSampleExtension;
import org.mp4parser.streaming.extensions.CompositionTimeTrackExtension;
import org.mp4parser.streaming.extensions.SampleFlagsSampleExtension;
import org.mp4parser.streaming.extensions.TrackIdTrackExtension;
import org.mp4parser.streaming.output.SampleSink;
import org.mp4parser.streaming.output.mp4.DefaultBoxes;
import org.mp4parser.tools.Mp4Arrays;
import org.mp4parser.tools.Mp4Math;
import org.mp4parser.tools.Path;
import org.signal.core.util.logging.Log;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.WritableByteChannel;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Queue;
import java.util.concurrent.ConcurrentHashMap;
import static org.mp4parser.tools.CastUtils.l2i;
/**
* Creates an MP4 file with ftyp, mdat+, moov order.
* A very special property of this variant is that it written sequentially. You can start transferring the
* data while the <code>sink</code> receives it. (in contrast to typical implementations which need random
* access to write length fields at the beginning of the file)
*/
final class Mp4Writer extends DefaultBoxes implements SampleSink {
private static final String TAG = "Mp4Writer";
private static final Long UInt32_MAX = (1L << 32) - 1;
private final WritableByteChannel sink;
private final List<StreamingTrack> source;
private final Date creationTime = new Date();
private boolean hasWrittenMdat = false;
/**
* Contains the start time of the next segment in line that will be created.
*/
private final Map<StreamingTrack, Long> nextChunkCreateStartTime = new ConcurrentHashMap<>();
/**
* Contains the start time of the next segment in line that will be written.
*/
private final Map<StreamingTrack, Long> nextChunkWriteStartTime = new ConcurrentHashMap<>();
/**
* Contains the next sample's start time.
*/
private final Map<StreamingTrack, Long> nextSampleStartTime = new HashMap<>();
/**
* Buffers the samples per track until there are enough samples to form a Segment.
*/
private final Map<StreamingTrack, List<StreamingSample>> sampleBuffers = new HashMap<>();
private final Map<StreamingTrack, TrackBox> trackBoxes = new HashMap<>();
/**
* Buffers segments until it's time for a segment to be written.
*/
private final Map<StreamingTrack, Queue<ChunkContainer>> chunkBuffers = new ConcurrentHashMap<>();
private final Map<StreamingTrack, Long> chunkNumbers = new HashMap<>();
private final Map<StreamingTrack, Long> sampleNumbers = new HashMap<>();
private long bytesWritten = 0;
private long mMDatTotalContentLength = 0;
Mp4Writer(final @NonNull List<StreamingTrack> source, final @NonNull WritableByteChannel sink) throws IOException {
this.source = new ArrayList<>(source);
this.sink = sink;
final HashSet<Long> trackIds = new HashSet<>();
for (StreamingTrack streamingTrack : source) {
streamingTrack.setSampleSink(this);
chunkNumbers.put(streamingTrack, 1L);
sampleNumbers.put(streamingTrack, 1L);
nextSampleStartTime.put(streamingTrack, 0L);
nextChunkCreateStartTime.put(streamingTrack, 0L);
nextChunkWriteStartTime.put(streamingTrack, 0L);
sampleBuffers.put(streamingTrack, new ArrayList<>());
chunkBuffers.put(streamingTrack, new LinkedList<>());
if (streamingTrack.getTrackExtension(TrackIdTrackExtension.class) != null) {
final TrackIdTrackExtension trackIdTrackExtension = streamingTrack.getTrackExtension(TrackIdTrackExtension.class);
if (trackIds.contains(trackIdTrackExtension.getTrackId())) {
throw new MuxingException("There may not be two tracks with the same trackID within one file");
}
trackIds.add(trackIdTrackExtension.getTrackId());
}
}
for (StreamingTrack streamingTrack : source) {
if (streamingTrack.getTrackExtension(TrackIdTrackExtension.class) == null) {
long maxTrackId = 0;
for (Long trackId : trackIds) {
maxTrackId = Math.max(trackId, maxTrackId);
}
final TrackIdTrackExtension tiExt = new TrackIdTrackExtension(maxTrackId + 1);
trackIds.add(tiExt.getTrackId());
streamingTrack.addTrackExtension(tiExt);
}
}
final List<String> minorBrands = new LinkedList<>();
minorBrands.add("isom");
minorBrands.add("mp42");
write(sink, new FileTypeBox("mp42", 0, minorBrands));
}
public void close() throws IOException {
for (StreamingTrack streamingTrack : source) {
writeChunkContainer(createChunkContainer(streamingTrack));
streamingTrack.close();
}
write(sink, createMoov());
hasWrittenMdat = false;
}
public long getTotalMdatContentLength() {
return mMDatTotalContentLength;
}
private Box createMoov() {
final MovieBox movieBox = new MovieBox();
final MovieHeaderBox mvhd = createMvhd();
movieBox.addBox(mvhd);
// update durations
for (StreamingTrack streamingTrack : source) {
final TrackBox tb = trackBoxes.get(streamingTrack);
final MediaHeaderBox mdhd = Path.getPath(tb, "mdia[0]/mdhd[0]");
mdhd.setCreationTime(creationTime);
mdhd.setModificationTime(creationTime);
final Long mediaHeaderDuration = Objects.requireNonNull(nextSampleStartTime.get(streamingTrack));
if (mediaHeaderDuration >= UInt32_MAX) {
mdhd.setVersion(1);
}
mdhd.setDuration(mediaHeaderDuration);
mdhd.setTimescale(streamingTrack.getTimescale());
mdhd.setLanguage(streamingTrack.getLanguage());
movieBox.addBox(tb);
final TrackHeaderBox tkhd = Path.getPath(tb, "tkhd[0]");
final double duration = (double) mediaHeaderDuration / streamingTrack.getTimescale();
tkhd.setCreationTime(creationTime);
tkhd.setModificationTime(creationTime);
final long trackHeaderDuration = (long) (mvhd.getTimescale() * duration);
if (trackHeaderDuration >= UInt32_MAX) {
tkhd.setVersion(1);
}
tkhd.setDuration(trackHeaderDuration);
}
// metadata here
return movieBox;
}
private void sortTracks() {
Collections.sort(source, (o1, o2) -> {
// compare times and account for timestamps!
final long a = Objects.requireNonNull(nextChunkWriteStartTime.get(o1)) * o2.getTimescale();
final long b = Objects.requireNonNull(nextChunkWriteStartTime.get(o2)) * o1.getTimescale();
return (int) Math.signum(a - b);
});
}
@Override
protected MovieHeaderBox createMvhd() {
final MovieHeaderBox mvhd = new MovieHeaderBox();
mvhd.setVersion(1);
mvhd.setCreationTime(creationTime);
mvhd.setModificationTime(creationTime);
long[] timescales = new long[0];
long maxTrackId = 0;
double duration = 0;
for (StreamingTrack streamingTrack : source) {
duration = Math.max((double) Objects.requireNonNull(nextSampleStartTime.get(streamingTrack)) / streamingTrack.getTimescale(), duration);
timescales = Mp4Arrays.copyOfAndAppend(timescales, streamingTrack.getTimescale());
maxTrackId = Math.max(streamingTrack.getTrackExtension(TrackIdTrackExtension.class).getTrackId(), maxTrackId);
}
long chosenTimescale = Mp4Math.lcm(timescales);
Log.d(TAG, "chosenTimescale = " + chosenTimescale);
final long MAX_UNSIGNED_INT = 0xFFFFFFFFL;
if (chosenTimescale > MAX_UNSIGNED_INT) {
int nRatio = (int)(chosenTimescale / MAX_UNSIGNED_INT);
Log.d(TAG, "chosenTimescale exceeds 32-bit range " + nRatio + " times !");
int nDownscaleFactor = 1;
if (nRatio < 10) {
nDownscaleFactor = 10;
} else if (nRatio < 100) {
nDownscaleFactor = 100;
} else if (nRatio < 1000) {
nDownscaleFactor = 1000;
} else if (nRatio < 10000) {
nDownscaleFactor = 10000;
}
chosenTimescale /= nDownscaleFactor;
Log.d(TAG, "chosenTimescale is scaled down by factor of " + nDownscaleFactor + " to value " + chosenTimescale);
}
double fDurationTicks = chosenTimescale * duration;
Log.d(TAG, "fDurationTicks = chosenTimescale * duration = " + fDurationTicks);
final double MAX_UNSIGNED_64_BIT_VALUE = 18446744073709551615.0;
if (fDurationTicks > MAX_UNSIGNED_64_BIT_VALUE) {
// Highly unlikely, as duration (number of seconds)
// would need to be larger than MAX_UNSIGNED_INT
// to produce fDuration = chosenTimescale * duration
// which whould exceed 64-bit storage
Log.d(TAG, "Numeric overflow !!!");
}
mvhd.setTimescale(chosenTimescale);
mvhd.setDuration((long) (fDurationTicks));
// find the next available trackId
mvhd.setNextTrackId(maxTrackId + 1);
return mvhd;
}
private void write(final @NonNull WritableByteChannel out, Box... boxes) throws IOException {
for (Box box1 : boxes) {
box1.getBox(out);
bytesWritten += box1.getSize();
}
}
/**
* Tests if the currently received samples for a given track
* are already a 'chunk' as we want to have it. The next
* sample will not be part of the chunk
* will be added to the fragment buffer later.
*
* @param streamingTrack track to test
* @param next the lastest samples
* @return true if a chunk is to b e created.
*/
private boolean isChunkReady(StreamingTrack streamingTrack, StreamingSample next) {
final long ts = Objects.requireNonNull(nextSampleStartTime.get(streamingTrack));
final long cfst = Objects.requireNonNull(nextChunkCreateStartTime.get(streamingTrack));
return (ts >= cfst + 2 * streamingTrack.getTimescale());
// chunk interleave of 2 seconds
}
private void writeChunkContainer(ChunkContainer chunkContainer) throws IOException {
final TrackBox tb = trackBoxes.get(chunkContainer.streamingTrack);
final ChunkOffsetBox stco = Objects.requireNonNull(Path.getPath(tb, "mdia[0]/minf[0]/stbl[0]/stco[0]"));
final int extraChunkOffset = hasWrittenMdat ? 0 : 8;
stco.setChunkOffsets(Mp4Arrays.copyOfAndAppend(stco.getChunkOffsets(), bytesWritten + extraChunkOffset));
chunkContainer.mdat.includeHeader = !hasWrittenMdat;
write(sink, chunkContainer.mdat);
mMDatTotalContentLength += chunkContainer.mdat.getSize();
if (!hasWrittenMdat) {
hasWrittenMdat = true;
}
}
public void acceptSample(
final @NonNull StreamingSample streamingSample,
final @NonNull StreamingTrack streamingTrack) throws IOException
{
if (streamingSample.getContent().limit() == 0) {
//
// For currently unknown reason, the STSZ table of AAC audio stream
// related to the very last chunk comes with the extra table elements
// whose value is zero.
//
// The ISO MP4 spec does not absolutely prohibit such a case, but strongly
// stipulates that the stream has to have the inner logic to support
// the zero length audio frames (QCELP happens to be one such example).
//
// Spec excerpt:
// ----------------------------------------------------------------------
// 8.7.3 Sample Size Boxes
// 8.7.3.1 Definition
// ...
// NOTE A sample size of zero is not prohibited in general, but it
// must be valid and defined for the coding system, as defined by
// the sample entry, that the sample belongs to
// ----------------------------------------------------------------------
//
// In all other cases, having zero STSZ table values is very illogical
// and may pose the problems down the road. Here we will eliminate such
// samples from all the related bookkeeping
//
Log.d(TAG, "skipping zero-sized sample");
return;
}
TrackBox tb = trackBoxes.get(streamingTrack);
if (tb == null) {
tb = new TrackBox();
tb.addBox(createTkhd(streamingTrack));
tb.addBox(createMdia(streamingTrack));
trackBoxes.put(streamingTrack, tb);
}
if (isChunkReady(streamingTrack, streamingSample)) {
final ChunkContainer chunkContainer = createChunkContainer(streamingTrack);
//System.err.println("Creating fragment for " + streamingTrack);
Objects.requireNonNull(sampleBuffers.get(streamingTrack)).clear();
nextChunkCreateStartTime.put(streamingTrack, Objects.requireNonNull(nextChunkCreateStartTime.get(streamingTrack)) + chunkContainer.duration);
final Queue<ChunkContainer> chunkQueue = Objects.requireNonNull(chunkBuffers.get(streamingTrack));
chunkQueue.add(chunkContainer);
if (source.get(0) == streamingTrack) {
Queue<ChunkContainer> tracksFragmentQueue;
StreamingTrack currentStreamingTrack;
// This will write AT LEAST the currently created fragment and possibly a few more
while (!(tracksFragmentQueue = chunkBuffers.get((currentStreamingTrack = this.source.get(0)))).isEmpty()) {
final ChunkContainer currentFragmentContainer = tracksFragmentQueue.remove();
writeChunkContainer(currentFragmentContainer);
Log.d(TAG, "write chunk " + currentStreamingTrack.getHandler() + ". duration " + (double) currentFragmentContainer.duration / currentStreamingTrack.getTimescale());
final long ts = Objects.requireNonNull(nextChunkWriteStartTime.get(currentStreamingTrack)) + currentFragmentContainer.duration;
nextChunkWriteStartTime.put(currentStreamingTrack, ts);
Log.d(TAG, currentStreamingTrack.getHandler() + " track advanced to " + (double) ts / currentStreamingTrack.getTimescale());
sortTracks();
}
} else {
Log.d(TAG, streamingTrack.getHandler() + " track delayed, queue size is " + chunkQueue.size());
}
}
Objects.requireNonNull(sampleBuffers.get(streamingTrack)).add(streamingSample);
nextSampleStartTime.put(streamingTrack, Objects.requireNonNull(nextSampleStartTime.get(streamingTrack)) + streamingSample.getDuration());
}
private ChunkContainer createChunkContainer(final @NonNull StreamingTrack streamingTrack) {
final List<StreamingSample> samples = Objects.requireNonNull(sampleBuffers.get(streamingTrack));
final long chunkNumber = Objects.requireNonNull(chunkNumbers.get(streamingTrack));
chunkNumbers.put(streamingTrack, chunkNumber + 1);
final ChunkContainer cc = new ChunkContainer();
cc.streamingTrack = streamingTrack;
cc.mdat = new Mdat(samples);
cc.duration = Objects.requireNonNull(nextSampleStartTime.get(streamingTrack)) - Objects.requireNonNull(nextChunkCreateStartTime.get(streamingTrack));
final TrackBox tb = trackBoxes.get(streamingTrack);
final SampleTableBox stbl = Objects.requireNonNull(Path.getPath(tb, "mdia[0]/minf[0]/stbl[0]"));
final SampleToChunkBox stsc = Objects.requireNonNull(Path.getPath(stbl, "stsc[0]"));
if (stsc.getEntries().isEmpty()) {
final List<SampleToChunkBox.Entry> entries = new ArrayList<>();
stsc.setEntries(entries);
entries.add(new SampleToChunkBox.Entry(chunkNumber, samples.size(), 1));
} else {
final SampleToChunkBox.Entry e = stsc.getEntries().get(stsc.getEntries().size() - 1);
if (e.getSamplesPerChunk() != samples.size()) {
stsc.getEntries().add(new SampleToChunkBox.Entry(chunkNumber, samples.size(), 1));
}
}
long sampleNumber = Objects.requireNonNull(sampleNumbers.get(streamingTrack));
final SampleSizeBox stsz = Objects.requireNonNull(Path.getPath(stbl, "stsz[0]"));
final TimeToSampleBox stts = Objects.requireNonNull(Path.getPath(stbl, "stts[0]"));
SyncSampleBox stss = Path.getPath(stbl, "stss[0]");
CompositionTimeToSample ctts = Path.getPath(stbl, "ctts[0]");
if (streamingTrack.getTrackExtension(CompositionTimeTrackExtension.class) != null) {
if (ctts == null) {
ctts = new CompositionTimeToSample();
ctts.setEntries(new ArrayList<>());
final ArrayList<Box> bs = new ArrayList<>(stbl.getBoxes());
bs.add(bs.indexOf(stts), ctts);
}
}
final long[] sampleSizes = new long[samples.size()];
int i = 0;
for (StreamingSample sample : samples) {
sampleSizes[i++] = sample.getContent().limit();
if (ctts != null) {
ctts.getEntries().add(new CompositionTimeToSample.Entry(1, l2i(sample.getSampleExtension(CompositionTimeSampleExtension.class).getCompositionTimeOffset())));
}
if (stts.getEntries().isEmpty()) {
final ArrayList<TimeToSampleBox.Entry> entries = new ArrayList<>(stts.getEntries());
entries.add(new TimeToSampleBox.Entry(1, sample.getDuration()));
stts.setEntries(entries);
} else {
final TimeToSampleBox.Entry sttsEntry = stts.getEntries().get(stts.getEntries().size() - 1);
if (sttsEntry.getDelta() == sample.getDuration()) {
sttsEntry.setCount(sttsEntry.getCount() + 1);
} else {
stts.getEntries().add(new TimeToSampleBox.Entry(1, sample.getDuration()));
}
}
final SampleFlagsSampleExtension sampleFlagsSampleExtension = sample.getSampleExtension(SampleFlagsSampleExtension.class);
if (sampleFlagsSampleExtension != null && sampleFlagsSampleExtension.isSyncSample()) {
if (stss == null) {
stss = new SyncSampleBox();
stbl.addBox(stss);
}
stss.setSampleNumber(Mp4Arrays.copyOfAndAppend(stss.getSampleNumber(), sampleNumber));
}
sampleNumber++;
}
stsz.setSampleSizes(Mp4Arrays.copyOfAndAppend(stsz.getSampleSizes(), sampleSizes));
sampleNumbers.put(streamingTrack, sampleNumber);
samples.clear();
Log.d(TAG, "chunk container created for " + streamingTrack.getHandler() + ". mdat size: " + cc.mdat.size + ". chunk duration is " + (double) cc.duration / streamingTrack.getTimescale());
return cc;
}
protected @NonNull Box createMdhd(final @NonNull StreamingTrack streamingTrack) {
final MediaHeaderBox mdhd = new MediaHeaderBox();
mdhd.setCreationTime(creationTime);
mdhd.setModificationTime(creationTime);
//mdhd.setDuration(nextSampleStartTime.get(streamingTrack)); will update at the end, in createMoov
mdhd.setTimescale(streamingTrack.getTimescale());
mdhd.setLanguage(streamingTrack.getLanguage());
return mdhd;
}
@Override
protected Box createTkhd(StreamingTrack streamingTrack) {
TrackHeaderBox tkhd = (TrackHeaderBox) super.createTkhd(streamingTrack);
tkhd.setEnabled(true);
tkhd.setInMovie(true);
return tkhd;
}
private class Mdat implements Box {
final ArrayList<StreamingSample> samples;
long size;
boolean includeHeader;
Mdat(final @NonNull List<StreamingSample> samples) {
this.samples = new ArrayList<>(samples);
size = 8;
for (StreamingSample sample : samples) {
size += sample.getContent().limit();
}
}
@Override
public String getType() {
return "mdat";
}
@Override
public long getSize() {
if (includeHeader) {
return size;
} else {
return size - 8;
}
}
@Override
public void getBox(WritableByteChannel writableByteChannel) throws IOException {
if (includeHeader) {
// When we include the header, we specify the declared size as 1, indicating the size is from here until the end of the file
writableByteChannel.write(ByteBuffer.wrap(new byte[] {
0, 0, 0, 0, // size (4 bytes)
109, 100, 97, 116, // 'm' 'd' 'a' 't'
}));
}
for (StreamingSample sample : samples) {
writableByteChannel.write((ByteBuffer) sample.getContent().rewind());
}
}
}
private class ChunkContainer {
Mdat mdat;
StreamingTrack streamingTrack;
long duration;
}
}

12
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/MuxingException.java Normal file
View file

@ -0,0 +1,12 @@
package org.thoughtcrime.securesms.video.videoconverter.muxer;
final class MuxingException extends RuntimeException {
public MuxingException(String message) {
super(message);
}
public MuxingException(String message, Throwable cause) {
super(message, cause);
}
}

190
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/StreamingMuxer.java Normal file
View file

@ -0,0 +1,190 @@
package org.thoughtcrime.securesms.video.videoconverter.muxer;
import android.media.MediaCodec;
import android.media.MediaFormat;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.mp4parser.boxes.iso14496.part1.objectdescriptors.DecoderSpecificInfo;
import org.mp4parser.streaming.StreamingTrack;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.video.interfaces.Muxer;
import org.thoughtcrime.securesms.video.videoconverter.utils.MediaCodecCompat;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.util.ArrayList;
import java.util.List;
public final class StreamingMuxer implements Muxer {
private static final String TAG = Log.tag(StreamingMuxer.class);
private final OutputStream outputStream;
private final List<MediaCodecTrack> tracks = new ArrayList<>();
private Mp4Writer mp4Writer;
public StreamingMuxer(OutputStream outputStream) {
this.outputStream = outputStream;
}
@Override
public void start() throws IOException {
final List<StreamingTrack> source = new ArrayList<>();
for (MediaCodecTrack track : tracks) {
source.add((StreamingTrack) track);
}
mp4Writer = new Mp4Writer(source, Channels.newChannel(outputStream));
}
@Override
public long stop() throws IOException {
if (mp4Writer == null) {
throw new IllegalStateException("calling stop prior to start");
}
for (MediaCodecTrack track : tracks) {
track.finish();
}
mp4Writer.close();
long mdatLength = mp4Writer.getTotalMdatContentLength();
mp4Writer = null;
return mdatLength;
}
@Override
public int addTrack(@NonNull MediaFormat format) throws IOException {
final String mime = format.getString(MediaFormat.KEY_MIME);
switch (mime) {
case "video/avc":
tracks.add(new MediaCodecAvcTrack(format));
break;
case "audio/mp4a-latm":
tracks.add(MediaCodecAacTrack.create(format));
break;
case "video/hevc":
tracks.add(new MediaCodecHevcTrack(format));
break;
default:
throw new IllegalArgumentException("unknown track format");
}
return tracks.size() - 1;
}
@Override
public void writeSampleData(int trackIndex, @NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException {
tracks.get(trackIndex).writeSampleData(byteBuf, bufferInfo);
}
@Override
public void release() {
}
@Override
public boolean supportsAudioRemux() {
return true;
}
interface MediaCodecTrack {
void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException;
void finish() throws IOException;
}
static class MediaCodecAvcTrack extends AvcTrack implements MediaCodecTrack {
MediaCodecAvcTrack(@NonNull MediaFormat format) {
super(Utils.subBuffer(format.getByteBuffer("csd-0"), 4), Utils.subBuffer(format.getByteBuffer("csd-1"), 4));
}
@Override
public void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException {
final List<ByteBuffer> nals = H264Utils.getNals(byteBuf);
for (ByteBuffer nal : nals) {
consumeNal(Utils.clone(nal), bufferInfo.presentationTimeUs);
}
}
@Override
public void finish() throws IOException {
consumeLastNal();
}
}
static class MediaCodecHevcTrack extends HevcTrack implements MediaCodecTrack {
MediaCodecHevcTrack(@NonNull MediaFormat format) throws IOException {
super(H264Utils.getNals(format.getByteBuffer("csd-0")));
}
@Override
public void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException {
final List<ByteBuffer> nals = H264Utils.getNals(byteBuf);
for (ByteBuffer nal : nals) {
consumeNal(Utils.clone(nal), bufferInfo.presentationTimeUs);
}
}
@Override
public void finish() throws IOException {
consumeLastNal();
}
}
static class MediaCodecAacTrack extends AacTrack implements MediaCodecTrack {
private MediaCodecAacTrack(long avgBitrate, long maxBitrate, int sampleRate, int channelCount, int aacProfile, @Nullable DecoderSpecificInfo decoderSpecificInfo) {
super(avgBitrate, maxBitrate, sampleRate, channelCount, aacProfile, decoderSpecificInfo);
}
public static MediaCodecAacTrack create(@NonNull MediaFormat format) {
final int bitrate = format.getInteger(MediaFormat.KEY_BIT_RATE);
final int maxBitrate;
if (format.containsKey(MediaCodecCompat.MEDIA_FORMAT_KEY_MAX_BIT_RATE)) {
maxBitrate = format.getInteger(MediaCodecCompat.MEDIA_FORMAT_KEY_MAX_BIT_RATE);
} else {
maxBitrate = bitrate;
}
final DecoderSpecificInfo filledDecoderSpecificInfo;
if (format.containsKey(MediaCodecCompat.MEDIA_FORMAT_KEY_MAX_BIT_RATE)) {
final ByteBuffer csd = format.getByteBuffer(MediaCodecCompat.MEDIA_FORMAT_KEY_CODEC_SPECIFIC_DATA_0);
DecoderSpecificInfo decoderSpecificInfo = new DecoderSpecificInfo();
boolean parseSuccess = false;
try {
decoderSpecificInfo.parseDetail(csd);
parseSuccess = true;
} catch (IOException e) {
Log.w(TAG, "Could not parse AAC codec-specific data!", e);
}
if (parseSuccess) {
filledDecoderSpecificInfo = decoderSpecificInfo;
} else {
filledDecoderSpecificInfo = null;
}
} else {
filledDecoderSpecificInfo = null;
}
return new MediaCodecAacTrack(bitrate, maxBitrate,
format.getInteger(MediaFormat.KEY_SAMPLE_RATE), format.getInteger(MediaFormat.KEY_CHANNEL_COUNT),
format.getInteger(MediaFormat.KEY_AAC_PROFILE), filledDecoderSpecificInfo);
}
@Override
public void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException {
final byte[] buffer = new byte[bufferInfo.size];
byteBuf.position(bufferInfo.offset);
byteBuf.get(buffer, 0, bufferInfo.size);
processSample(ByteBuffer.wrap(buffer));
}
@Override
public void finish() {
}
}
}

44
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/Utils.java Normal file
View file

@ -0,0 +1,44 @@
package org.thoughtcrime.securesms.video.videoconverter.muxer;
import androidx.annotation.NonNull;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.List;
/**
* Based on https://github.com/jcodec/jcodec/blob/master/src/main/java/org/jcodec/codecs/h264/H264Utils.java
*/
final class Utils {
private Utils() {}
static byte[] toArray(final @NonNull ByteBuffer buf) {
final ByteBuffer newBuf = buf.duplicate();
byte[] bytes = new byte[newBuf.remaining()];
newBuf.get(bytes, 0, bytes.length);
return bytes;
}
public static ByteBuffer clone(final @NonNull ByteBuffer original) {
final ByteBuffer clone = ByteBuffer.allocate(original.capacity());
original.rewind();
clone.put(original);
original.rewind();
clone.flip();
return clone;
}
static @NonNull ByteBuffer subBuffer(final @NonNull ByteBuffer buf, final int start) {
return subBuffer(buf, start, buf.limit() - start);
}
static @NonNull ByteBuffer subBuffer(final @NonNull ByteBuffer buf, final int start, final int count) {
final ByteBuffer newBuf = buf.duplicate();
byte[] bytes = new byte[count];
newBuf.position(start);
newBuf.get(bytes, 0, bytes.length);
return ByteBuffer.wrap(bytes);
}
}

19
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/utils/DeviceCapabilities.kt Normal file
View file

@ -0,0 +1,19 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.video.videoconverter.utils
import android.media.MediaCodecList
import android.media.MediaFormat
import org.signal.core.util.isNotNullOrBlank
object DeviceCapabilities {
@JvmStatic
fun canEncodeHevc(): Boolean {
val mediaCodecList = MediaCodecList(MediaCodecList.REGULAR_CODECS)
val encoder = mediaCodecList.findEncoderForFormat(MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_HEVC, VideoConstants.VIDEO_LONG_EDGE_HD, VideoConstants.VIDEO_SHORT_EDGE_HD))
return encoder.isNotNullOrBlank()
}
}

24
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/utils/Extensions.kt Normal file
View file

@ -0,0 +1,24 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.video.videoconverter.utils
import kotlin.math.ceil
import kotlin.math.floor
import kotlin.math.roundToLong
object Extensions {
/**
* Determines if the [actual] value is close enough to the [expected] value within the [tolerance]
*
* @param tolerance a float value, where 0f defines an exact match, 0.1f defines a 10% tolerance, etc.
*/
@JvmStatic
fun isWithin(expected: Long, actual: Long, tolerance: Float): Boolean {
val floor = floor(expected * (1 - tolerance)).roundToLong()
val ceiling = ceil(expected * (1 + tolerance)).roundToLong()
return actual in floor..ceiling
}
}

152
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/utils/MediaCodecCompat.kt Normal file
View file

@ -0,0 +1,152 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.video.videoconverter.utils
import android.media.MediaCodec
import android.media.MediaCodecInfo.CodecProfileLevel
import android.media.MediaCodecList
import android.media.MediaFormat
import android.os.Build
import org.signal.core.util.logging.Log
import java.io.IOException
object MediaCodecCompat {
private const val TAG = "MediaDataSourceCompat"
const val MEDIA_FORMAT_KEY_MAX_BIT_RATE = "max-bitrate"
// https://developer.android.com/reference/android/media/MediaCodec#CSD
const val MEDIA_FORMAT_KEY_CODEC_SPECIFIC_DATA_0 = "csd-0"
const val MEDIA_FORMAT_KEY_CODEC_SPECIFIC_DATA_1 = "csd-1"
const val MEDIA_FORMAT_KEY_CODEC_SPECIFIC_DATA_2 = "csd-2"
@JvmStatic
fun findDecoder(inputFormat: MediaFormat): Pair<MediaCodec, MediaFormat> {
val codecs = MediaCodecList(MediaCodecList.REGULAR_CODECS)
val decoderName: String? = codecs.findDecoderForFormat(inputFormat)
if (decoderName != null) {
return Pair(MediaCodec.createByCodecName(decoderName), inputFormat)
}
val mimeType = inputFormat.getString(MediaFormat.KEY_MIME)
if (MediaFormat.MIMETYPE_VIDEO_DOLBY_VISION == mimeType) {
return if (Build.VERSION.SDK_INT >= 29) {
findBackupDecoderForDolbyVision(MediaFormat(inputFormat)) ?: throw IOException("Can't create decoder for $mimeType!")
} else {
findBackupDecoderForDolbyVision(inputFormat) ?: throw IOException("Can't create decoder for $mimeType!")
}
} else if (mimeType != null) {
try {
val decoder = MediaCodec.createDecoderByType(mimeType)
return Pair(decoder, inputFormat)
} catch (iae: IllegalArgumentException) {
throw IOException("Can't create decoder for $mimeType, which is not a valid MIME type.", iae)
}
}
throw IOException("Can't create decoder for $mimeType!")
}
/**
* Find backup decoder for a [MediaFormat] object with a MIME type of Dolby Vision.
*
* Dolby Vision is implemented as a two-layer stream in a video file: a "base layer" and an "enhancement layer".
* Both are (usually) standards-compliant video bitstreams that proprietary decoders combine to form the high-quality Dolby Vision stream.
* On devices where Dolby Vision is not supported, they should still be able to read the base layer stream if they can send it to the appropriate decoder.
*
* This function mutates the input [MediaFormat] so that the appropriate decoder is selected for the base layer.
*
* More information can be found here: [Dolby Vision Knowledge Base](https://professionalsupport.dolby.com/s/article/What-is-Dolby-Vision-Profile?language=en_US)
*
* @param mediaFormat
* @return the mutated [MediaFormat] to signal to the decoder to read only the base layer.
*/
private fun findBackupDecoderForDolbyVision(mediaFormat: MediaFormat): Pair<MediaCodec, MediaFormat>? {
if (MediaFormat.MIMETYPE_VIDEO_DOLBY_VISION != mediaFormat.getString(MediaFormat.KEY_MIME)) {
throw IllegalStateException("Must supply Dolby Vision MediaFormat!")
}
return try {
when (mediaFormat.getInteger(MediaFormat.KEY_PROFILE)) {
CodecProfileLevel.DolbyVisionProfileDvheDtr,
CodecProfileLevel.DolbyVisionProfileDvheSt -> {
// dolby vision profile 04/08: Base layer is H.265 Main10 High Profile, Rec709/HLG/HDR10
mediaFormat.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_VIDEO_HEVC)
mediaFormat.setInteger(MediaFormat.KEY_PROFILE, CodecProfileLevel.HEVCProfileMain10)
mediaFormat.setBaseCodecLevelFromDolbyVisionLevel()
return findDecoder(mediaFormat)
}
CodecProfileLevel.DolbyVisionProfileDvavSe -> {
// dolby vision profile 09: Base layer is H.264 High/Progressive/Constrained Profile, Rec 709
mediaFormat.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_VIDEO_AVC)
mediaFormat.setInteger(MediaFormat.KEY_PROFILE, CodecProfileLevel.AVCProfileHigh)
mediaFormat.setBaseCodecLevelFromDolbyVisionLevel()
return findDecoder(mediaFormat)
}
else -> return null
}
} catch (npe: NullPointerException) {
null
}
}
private fun MediaFormat.setBaseCodecLevelFromDolbyVisionLevel(): Boolean {
val mimeType = this.getString(MediaFormat.KEY_MIME) ?: return false
try {
val codecLevel = this.getInteger(MediaFormat.KEY_LEVEL)
when (mimeType) {
MediaFormat.MIMETYPE_VIDEO_AVC -> {
val mapDvLevelToAvcLevel = mapDvLevelToAvcLevel(codecLevel) ?: return false
this.setInteger(MediaFormat.KEY_LEVEL, mapDvLevelToAvcLevel)
}
MediaFormat.MIMETYPE_VIDEO_HEVC -> {
val mapDvLevelToAvcLevel = mapDvLevelToHevcLevel(codecLevel) ?: return false
this.setInteger(MediaFormat.KEY_LEVEL, mapDvLevelToAvcLevel)
}
}
} catch (npe: NullPointerException) {
Log.d(TAG, "Could not update codec level in media format.")
return false
}
return true
}
private fun mapDvLevelToHevcLevel(level: Int) = when (level) {
CodecProfileLevel.DolbyVisionLevelHd24,
CodecProfileLevel.DolbyVisionLevelHd30 -> CodecProfileLevel.HEVCHighTierLevel31
CodecProfileLevel.DolbyVisionLevelFhd24,
CodecProfileLevel.DolbyVisionLevelFhd30 -> CodecProfileLevel.HEVCHighTierLevel4
CodecProfileLevel.DolbyVisionLevelFhd60 -> CodecProfileLevel.HEVCHighTierLevel41
CodecProfileLevel.DolbyVisionLevelUhd24,
CodecProfileLevel.DolbyVisionLevelUhd30 -> CodecProfileLevel.HEVCHighTierLevel5
CodecProfileLevel.DolbyVisionLevelUhd48,
CodecProfileLevel.DolbyVisionLevelUhd60 -> CodecProfileLevel.HEVCHighTierLevel51
CodecProfileLevel.DolbyVisionLevel8k60 -> CodecProfileLevel.HEVCHighTierLevel61
else -> null
}
private fun mapDvLevelToAvcLevel(level: Int) = when (level) {
CodecProfileLevel.DolbyVisionLevelHd24,
CodecProfileLevel.DolbyVisionLevelHd30 -> CodecProfileLevel.AVCLevel31
CodecProfileLevel.DolbyVisionLevelFhd24,
CodecProfileLevel.DolbyVisionLevelFhd30 -> CodecProfileLevel.AVCLevel4
CodecProfileLevel.DolbyVisionLevelFhd60 -> CodecProfileLevel.AVCLevel42
CodecProfileLevel.DolbyVisionLevelUhd24 -> CodecProfileLevel.AVCLevel51
CodecProfileLevel.DolbyVisionLevelUhd30 -> CodecProfileLevel.AVCLevel52
CodecProfileLevel.DolbyVisionLevelUhd48,
CodecProfileLevel.DolbyVisionLevelUhd60,
CodecProfileLevel.DolbyVisionLevel8k60 -> avcLevel6()
else -> null
}
private fun avcLevel6() = if (Build.VERSION.SDK_INT >= 29) {
CodecProfileLevel.AVCLevel62
} else {
null
}
}

8
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/utils/Preconditions.kt Normal file
View file

@ -0,0 +1,8 @@
package org.thoughtcrime.securesms.video.videoconverter.utils
object Preconditions {
@JvmStatic
fun checkState(errorMessage: String, expression: Boolean) {
check(expression) { errorMessage }
}
}

22
video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/utils/VideoConstants.kt Normal file
View file

@ -0,0 +1,22 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.video.videoconverter.utils
import android.media.MediaFormat
object VideoConstants {
const val AUDIO_BITRATE = 128_000
const val VIDEO_BITRATE_L1 = 1_250_000
const val VIDEO_BITRATE_L2 = 1_250_000
const val VIDEO_BITRATE_L3 = 2_500_000
const val VIDEO_SHORT_EDGE_SD = 480
const val VIDEO_SHORT_EDGE_HD = 720
const val VIDEO_LONG_EDGE_HD = 1280
const val VIDEO_MAX_RECORD_LENGTH_S = 60
const val MAX_ALLOWED_BYTES_PER_SECOND = VIDEO_BITRATE_L3 / 8 + AUDIO_BITRATE / 8
const val VIDEO_MIME_TYPE = MediaFormat.MIMETYPE_VIDEO_AVC
const val AUDIO_MIME_TYPE = MediaFormat.MIMETYPE_AUDIO_AAC
const val RECORDED_VIDEO_CONTENT_TYPE: String = "video/mp4"
}