nomad/ananas
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: scrubbing preview (#295)

Browse source 
* Scrubbing Preview

Add Jellyscrub plugin support

* Fix syntax

* Some adjustments

Rounded corners
Fix switch

* refactor: switch to `StateFlow`

* refactor: remove `FrameLayout`

* refactor: move trick play retrieval to `onMediaItemTransition`

Only load trick play data for current item
Make it async

---------

Co-authored-by: Jarne Demeulemeester <jarnedemeulemeester@gmail.com>
This commit is contained in:
Faywyrr 2023-02-21 19:46:00 +01:00 committed by GitHub
parent c5df381a80
commit 01d8c11a2c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 318 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
app/phone/src/main/java/dev/jdtech/jellyfin/PlayerActivity.kt
View file

@ -7,11 +7,13 @@ import android.view.View
import android.view.WindowManager
import android.widget.Button
import android.widget.ImageButton
import android.widget.ImageView
import android.widget.TextView
import androidx.activity.viewModels
import androidx.core.view.isVisible
import androidx.media3.common.C
import androidx.media3.exoplayer.ExoPlayer
import androidx.media3.ui.DefaultTimeBar
import androidx.media3.ui.TrackSelectionDialogBuilder
import androidx.navigation.navArgs
import dagger.hilt.android.AndroidEntryPoint
@ -21,6 +23,7 @@ import dev.jdtech.jellyfin.dialogs.TrackSelectionDialogFragment
import dev.jdtech.jellyfin.mpv.MPVPlayer
import dev.jdtech.jellyfin.mpv.TrackType
import dev.jdtech.jellyfin.utils.PlayerGestureHelper
import dev.jdtech.jellyfin.utils.PreviewScrubListener
import dev.jdtech.jellyfin.viewmodels.PlayerActivityViewModel
import javax.inject.Inject
import timber.log.Timber
@ -168,6 +171,19 @@ class PlayerActivity : BasePlayerActivity() {
}
}
if (appPreferences.playerTrickPlay) {
val imagePreview = binding.playerView.findViewById<ImageView>(R.id.image_preview)
val timeBar = binding.playerView.findViewById<DefaultTimeBar>(R.id.exo_progress)
val previewScrubListener = PreviewScrubListener(
imagePreview,
timeBar,
viewModel.player,
viewModel.currentTrickPlay
)
timeBar.addListener(previewScrubListener)
}
viewModel.fileLoaded.observe(this) {
if (it) {
audioButton.isEnabled = true

68
app/phone/src/main/java/dev/jdtech/jellyfin/utils/PreviewScrubListener.kt Normal file
View file

@ -0,0 +1,68 @@
package dev.jdtech.jellyfin.utils
import android.view.View
import android.view.ViewGroup
import android.widget.ImageView
import androidx.media3.common.Player
import androidx.media3.ui.TimeBar
import com.bumptech.glide.Glide
import com.bumptech.glide.load.resource.bitmap.RoundedCorners
import dev.jdtech.jellyfin.utils.bif.BifData
import dev.jdtech.jellyfin.utils.bif.BifUtil
import kotlinx.coroutines.flow.StateFlow
import timber.log.Timber
class PreviewScrubListener(
private val scrubbingPreview: ImageView,
private val timeBarView: View,
private val player: Player,
private val currentTrickPlay: StateFlow<BifData?>
) : TimeBar.OnScrubListener {
private val roundedCorners = RoundedCorners(10)
override fun onScrubStart(timeBar: TimeBar, position: Long) {
Timber.d("Scrubbing started at $position")
if (currentTrickPlay.value == null)
return
scrubbingPreview.visibility = View.VISIBLE
onScrubMove(timeBar, position)
}
override fun onScrubMove(timeBar: TimeBar, position: Long) {
Timber.d("Scrubbing to $position")
val currentBifData = currentTrickPlay.value ?: return
val image = BifUtil.getTrickPlayFrame(position.toInt(), currentBifData) ?: return
val parent = scrubbingPreview.parent as ViewGroup
val offset = position.toFloat() / player.duration
val minX = scrubbingPreview.left
val maxX = parent.width - parent.paddingRight
val startX = timeBarView.left + (timeBarView.right - timeBarView.left) * offset - scrubbingPreview.width / 2
val endX = startX + scrubbingPreview.width
val layoutX = when {
startX >= minX && endX <= maxX -> startX
startX < minX -> minX
else -> maxX - scrubbingPreview.width
}.toFloat()
scrubbingPreview.x = layoutX
Glide.with(scrubbingPreview)
.load(image)
.transform(roundedCorners)
.into(scrubbingPreview)
}
override fun onScrubStop(timeBar: TimeBar, position: Long, canceled: Boolean) {
Timber.d("Scrubbing stopped at $position")
scrubbingPreview.visibility = View.GONE
}
}

9
app/phone/src/main/res/layout/exo_player_control_view.xml
View file

@ -173,6 +173,15 @@
android:orientation="vertical"
android:padding="16dp">
<ImageView
android:id="@+id/image_preview"
android:layout_width="160dp"
android:layout_height="90dp"
android:layout_gravity="start"
android:background="@android:color/transparent"
android:visibility="gone"
tools:visibility="visible" />
<LinearLayout
android:layout_width="wrap_content"
android:layout_height="wrap_content"

2
core/src/main/res/values/strings.xml
View file

@ -150,6 +150,8 @@
<string name="pref_player_mpv_gpu_api" translatable="false">GPU API</string>
<string name="pref_player_intro_skipper">Intro Skipper</string>
<string name="pref_player_intro_skipper_summary">Requires ConfusedPolarBear\'s Intro Skipper plugin to be installed on the server</string>
<string name="pref_player_trick_play">Trick Play</string>
<string name="pref_player_trick_play_summary">Requires nicknsy\'s Jellyscrub plugin to be installed on the server</string>
<string name="addresses">Addresses</string>
<string name="add_address">Add address</string>
<string name="add_server_address">Add server address</string>

7
core/src/main/res/xml/fragment_settings_player.xml
View file

@ -106,4 +106,11 @@
app:title="@string/pref_player_intro_skipper"
app:widgetLayout="@layout/preference_material3_switch" />
<SwitchPreferenceCompat
app:defaultValue="true"
app:key="pref_player_trick_play"
app:summary="@string/pref_player_trick_play_summary"
app:title="@string/pref_player_trick_play"
app:widgetLayout="@layout/preference_material3_switch" />
</PreferenceScreen>

12
data/src/main/java/dev/jdtech/jellyfin/models/TrickPlayManifest.kt Normal file
View file

@ -0,0 +1,12 @@
package dev.jdtech.jellyfin.models
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
@Serializable
data class TrickPlayManifest(
@SerialName("Version")
val version: String,
@SerialName("WidthResolutions")
val widthResolutions: List<Int>
)

5
data/src/main/java/dev/jdtech/jellyfin/repository/JellyfinRepository.kt
View file

@ -3,6 +3,7 @@ package dev.jdtech.jellyfin.repository
import androidx.paging.PagingData
import dev.jdtech.jellyfin.models.Intro
import dev.jdtech.jellyfin.models.SortBy
import dev.jdtech.jellyfin.models.TrickPlayManifest
import java.util.UUID
import kotlinx.coroutines.flow.Flow
import org.jellyfin.sdk.model.api.BaseItemDto
@ -65,6 +66,10 @@ interface JellyfinRepository {
suspend fun getIntroTimestamps(itemId: UUID): Intro?
suspend fun getTrickPlayManifest(itemId: UUID): TrickPlayManifest?
suspend fun getTrickPlayData(itemId: UUID, width: Int): ByteArray?
suspend fun postCapabilities()
suspend fun postPlaybackStart(itemId: UUID)

30
data/src/main/java/dev/jdtech/jellyfin/repository/JellyfinRepositoryImpl.kt
View file

@ -6,6 +6,9 @@ import androidx.paging.PagingData
import dev.jdtech.jellyfin.api.JellyfinApi
import dev.jdtech.jellyfin.models.Intro
import dev.jdtech.jellyfin.models.SortBy
import dev.jdtech.jellyfin.models.TrickPlayManifest
import io.ktor.util.cio.toByteArray
import io.ktor.utils.io.ByteReadChannel
import java.util.UUID
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.Flow
@ -233,6 +236,33 @@ class JellyfinRepositoryImpl(private val jellyfinApi: JellyfinApi) : JellyfinRep
}
}
override suspend fun getTrickPlayManifest(itemId: UUID): TrickPlayManifest? =
withContext(Dispatchers.IO) {
// https://github.com/nicknsy/jellyscrub/blob/main/Nick.Plugin.Jellyscrub/Api/TrickplayController.cs
val pathParameters = mutableMapOf<String, UUID>()
pathParameters["itemId"] = itemId
try {
return@withContext jellyfinApi.api.get<TrickPlayManifest>("/Trickplay/{itemId}/GetManifest", pathParameters).content
} catch (e: Exception) {
return@withContext null
}
}
override suspend fun getTrickPlayData(itemId: UUID, width: Int): ByteArray? =
withContext(Dispatchers.IO) {
// https://github.com/nicknsy/jellyscrub/blob/main/Nick.Plugin.Jellyscrub/Api/TrickplayController.cs
val pathParameters = mutableMapOf<String, Any>()
pathParameters["itemId"] = itemId
pathParameters["width"] = width
try {
return@withContext jellyfinApi.api.get<ByteReadChannel>("/Trickplay/{itemId}/{width}/GetBIF", pathParameters).content.toByteArray()
} catch (e: Exception) {
return@withContext null
}
}
override suspend fun postCapabilities() {
Timber.d("Sending capabilities")
withContext(Dispatchers.IO) {

11
player/core/src/main/java/dev/jdtech/jellyfin/utils/bif/BifData.kt Normal file
View file

@ -0,0 +1,11 @@
package dev.jdtech.jellyfin.utils.bif
import android.graphics.Bitmap
data class BifData(
val version: Int,
val timestampMultiplier: Int,
val imageCount: Int,
val images: Map<Int, Bitmap>,
val imageWidth: Int
)

3
player/core/src/main/java/dev/jdtech/jellyfin/utils/bif/BifIndexEntry.kt Normal file
View file

@ -0,0 +1,3 @@
package dev.jdtech.jellyfin.utils.bif
data class BifIndexEntry(val timestamp: Int, val offset: Int)

77
player/core/src/main/java/dev/jdtech/jellyfin/utils/bif/BifUtil.kt Normal file
View file

@ -0,0 +1,77 @@
package dev.jdtech.jellyfin.utils.bif
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import java.nio.ByteBuffer
import timber.log.Timber
object BifUtil {
/* https://github.com/nicknsy/jellyscrub/blob/main/Nick.Plugin.Jellyscrub/Api/trickplay.js */
private val BIF_MAGIC_NUMBERS = byteArrayOf(0x89.toByte(), 0x42.toByte(), 0x49.toByte(), 0x46.toByte(), 0x0D.toByte(), 0x0A.toByte(), 0x1A.toByte(), 0x0A.toByte())
private const val SUPPORTED_BIF_VERSION = 0
fun trickPlayDecode(array: ByteArray, width: Int): BifData? {
val data = Indexed8Array(array)
Timber.d("BIF file size: ${data.limit()}")
for (b in BIF_MAGIC_NUMBERS) {
if (data.read() != b) {
Timber.d("Attempted to read invalid bif file.")
return null
}
}
val bifVersion = data.readInt32()
if (bifVersion != SUPPORTED_BIF_VERSION) {
Timber.d("Client only supports BIF v$SUPPORTED_BIF_VERSION but file is v$bifVersion")
return null
}
Timber.d("BIF version: $bifVersion")
val bifImgCount = data.readInt32()
if (bifImgCount <= 0) {
Timber.d("BIF file contains no images.")
return null
}
Timber.d("BIF image count: $bifImgCount")
var timestampMultiplier = data.readInt32()
if (timestampMultiplier == 0) timestampMultiplier = 1000
data.addPosition(44) // Reserved
val bifIndex = mutableListOf<BifIndexEntry>()
for (i in 0 until bifImgCount) {
bifIndex.add(BifIndexEntry(data.readInt32(), data.readInt32()))
}
val bifImages = mutableMapOf<Int, Bitmap>()
for (i in bifIndex.indices) {
val indexEntry = bifIndex[i]
val timestamp = indexEntry.timestamp
val offset = indexEntry.offset
val nextOffset = bifIndex.getOrNull(i + 1)?.offset ?: data.limit()
val imageBuffer = ByteBuffer.wrap(data.array(), offset, nextOffset - offset).order(data.order())
val imageBytes = ByteArray(imageBuffer.remaining())
imageBuffer.get(imageBytes)
val bmp = BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.size)
bifImages[timestamp] = bmp
}
return BifData(bifVersion, timestampMultiplier, bifImgCount, bifImages, width)
}
fun getTrickPlayFrame(playerTimestamp: Int, data: BifData): Bitmap? {
val multiplier = data.timestampMultiplier
val images = data.images
val frame = playerTimestamp / multiplier
return images[frame]
}
}

37
player/core/src/main/java/dev/jdtech/jellyfin/utils/bif/Indexed8Array.kt Normal file
View file

@ -0,0 +1,37 @@
package dev.jdtech.jellyfin.utils.bif
import java.nio.ByteOrder
class Indexed8Array(private val array: ByteArray) {
private var readIndex = 0
fun read(): Byte {
return array[readIndex++]
}
fun addPosition(amount: Int) {
readIndex += amount
}
fun readInt32(): Int {
val b1 = read().toInt().and(0xFF)
val b2 = read().toInt().and(0xFF)
val b3 = read().toInt().and(0xFF)
val b4 = read().toInt().and(0xFF)
return b1 or (b2 shl 8) or (b3 shl 16) or (b4 shl 24)
}
fun array(): ByteArray {
return array
}
fun limit(): Int {
return array.size
}
fun order(): ByteOrder {
return ByteOrder.BIG_ENDIAN
}
}

40
player/video/src/main/java/dev/jdtech/jellyfin/viewmodels/PlayerActivityViewModel.kt
View file

@ -23,6 +23,8 @@ import dev.jdtech.jellyfin.models.PlayerItem
import dev.jdtech.jellyfin.mpv.MPVPlayer
import dev.jdtech.jellyfin.mpv.TrackType
import dev.jdtech.jellyfin.repository.JellyfinRepository
import dev.jdtech.jellyfin.utils.bif.BifData
import dev.jdtech.jellyfin.utils.bif.BifUtil
import dev.jdtech.jellyfin.utils.postDownloadPlaybackProgress
import java.util.UUID
import javax.inject.Inject
@ -30,6 +32,8 @@ import kotlinx.coroutines.DelicateCoroutinesApi
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import timber.log.Timber
@ -55,6 +59,10 @@ constructor(
private val _currentIntro = MutableLiveData<Intro?>(null)
val currentIntro: LiveData<Intro?> = _currentIntro
private val trickPlays: MutableMap<UUID, BifData> = mutableMapOf()
private val _currentTrickPlay = MutableStateFlow<BifData?>(null)
val currentTrickPlay = _currentTrickPlay.asStateFlow()
var currentAudioTracks: MutableList<MPVPlayer.Companion.Track> = mutableListOf()
var currentSubtitleTracks: MutableList<MPVPlayer.Companion.Track> = mutableListOf()
@ -181,6 +189,7 @@ constructor(
}
}
_currentTrickPlay.value = null
playWhenReady = player.playWhenReady
playbackPosition = player.currentPosition
currentMediaItemIndex = player.currentMediaItemIndex
@ -246,9 +255,14 @@ constructor(
"S${item.parentIndexNumber}:E${item.indexNumber} - ${item.name}"
else
_currentItemTitle.value = item.name.orEmpty()
jellyfinRepository.postPlaybackStart(item.itemId)
if (appPreferences.playerTrickPlay) {
getTrickPlay(item.itemId)
}
}
}
jellyfinRepository.postPlaybackStart(UUID.fromString(mediaItem?.mediaId))
} catch (e: Exception) {
Timber.e(e)
}
@ -311,4 +325,28 @@ constructor(
player.setPlaybackSpeed(speed)
playbackSpeed = speed
}
private suspend fun getTrickPlay(itemId: UUID) {
if (trickPlays[itemId] != null) return
jellyfinRepository.getTrickPlayManifest(itemId)
?.let { trickPlayManifest ->
val widthResolution =
trickPlayManifest.widthResolutions.max()
Timber.d("Trickplay Resolution: $widthResolution")
jellyfinRepository.getTrickPlayData(
itemId,
widthResolution
)?.let { byteArray ->
val trickPlayData =
BifUtil.trickPlayDecode(byteArray, widthResolution)
trickPlayData?.let {
Timber.d("Trickplay Images: ${it.imageCount}")
trickPlays[itemId] = it
_currentTrickPlay.value = trickPlays[itemId]
}
}
}
}
}

1
preferences/src/main/java/dev/jdtech/jellyfin/AppPreferences.kt
View file

@ -72,6 +72,7 @@ constructor(
val playerMpvAo get() = sharedPreferences.getString(Constants.PREF_PLAYER_MPV_AO, "audiotrack")!!
val playerMpvGpuApi get() = sharedPreferences.getString(Constants.PREF_PLAYER_MPV_GPU_API, "opengl")!!
val playerIntroSkipper get() = sharedPreferences.getBoolean(Constants.PREF_PLAYER_INTRO_SKIPPER, true)
val playerTrickPlay get() = sharedPreferences.getBoolean(Constants.PREF_PLAYER_TRICK_PLAY, true)
// Language
val preferredAudioLanguage get() = sharedPreferences.getString(Constants.PREF_AUDIO_LANGUAGE, "")!!

1
preferences/src/main/java/dev/jdtech/jellyfin/Constants.kt
View file

@ -26,6 +26,7 @@ object Constants {
const val PREF_PLAYER_MPV_AO = "pref_player_mpv_ao"
const val PREF_PLAYER_MPV_GPU_API = "pref_player_mpv_gpu_api"
const val PREF_PLAYER_INTRO_SKIPPER = "pref_player_intro_skipper"
const val PREF_PLAYER_TRICK_PLAY = "pref_player_trick_play"
const val PREF_AUDIO_LANGUAGE = "pref_audio_language"
const val PREF_SUBTITLE_LANGUAGE = "pref_subtitle_language"
const val PREF_IMAGE_CACHE = "pref_image_cache"