feat: scrubbing preview (#295)

* Scrubbing Preview

Add Jellyscrub plugin support

* Fix syntax

* Some adjustments

Rounded corners
Fix switch

* refactor: switch to `StateFlow`

* refactor: remove `FrameLayout`

* refactor: move trick play retrieval to `onMediaItemTransition`

Only load trick play data for current item
Make it async

---------

Co-authored-by: Jarne Demeulemeester <jarnedemeulemeester@gmail.com>

app/phone/src/main/java/dev/jdtech/jellyfin/PlayerActivity.kt

@@ -7,11 +7,13 @@ import android.view.View
 import android.view.WindowManager
 import android.widget.Button
 import android.widget.ImageButton
+import android.widget.ImageView
 import android.widget.TextView
 import androidx.activity.viewModels
 import androidx.core.view.isVisible
 import androidx.media3.common.C
 import androidx.media3.exoplayer.ExoPlayer
+import androidx.media3.ui.DefaultTimeBar
 import androidx.media3.ui.TrackSelectionDialogBuilder
 import androidx.navigation.navArgs
 import dagger.hilt.android.AndroidEntryPoint
@@ -21,6 +23,7 @@ import dev.jdtech.jellyfin.dialogs.TrackSelectionDialogFragment
 import dev.jdtech.jellyfin.mpv.MPVPlayer
 import dev.jdtech.jellyfin.mpv.TrackType
 import dev.jdtech.jellyfin.utils.PlayerGestureHelper
+import dev.jdtech.jellyfin.utils.PreviewScrubListener
 import dev.jdtech.jellyfin.viewmodels.PlayerActivityViewModel
 import javax.inject.Inject
 import timber.log.Timber
@@ -168,6 +171,19 @@ class PlayerActivity : BasePlayerActivity() {
             }
         }
 
+        if (appPreferences.playerTrickPlay) {
+            val imagePreview = binding.playerView.findViewById<ImageView>(R.id.image_preview)
+            val timeBar = binding.playerView.findViewById<DefaultTimeBar>(R.id.exo_progress)
+            val previewScrubListener = PreviewScrubListener(
+                imagePreview,
+                timeBar,
+                viewModel.player,
+                viewModel.currentTrickPlay
+            )
+
+            timeBar.addListener(previewScrubListener)
+        }
+
         viewModel.fileLoaded.observe(this) {
             if (it) {
                 audioButton.isEnabled = true

app/phone/src/main/java/dev/jdtech/jellyfin/utils/PreviewScrubListener.kt

@@ -0,0 +1,68 @@
+package dev.jdtech.jellyfin.utils
+
+import android.view.View
+import android.view.ViewGroup
+import android.widget.ImageView
+import androidx.media3.common.Player
+import androidx.media3.ui.TimeBar
+import com.bumptech.glide.Glide
+import com.bumptech.glide.load.resource.bitmap.RoundedCorners
+import dev.jdtech.jellyfin.utils.bif.BifData
+import dev.jdtech.jellyfin.utils.bif.BifUtil
+import kotlinx.coroutines.flow.StateFlow
+import timber.log.Timber
+
+class PreviewScrubListener(
+    private val scrubbingPreview: ImageView,
+    private val timeBarView: View,
+    private val player: Player,
+    private val currentTrickPlay: StateFlow<BifData?>
+) : TimeBar.OnScrubListener {
+
+    private val roundedCorners = RoundedCorners(10)
+
+    override fun onScrubStart(timeBar: TimeBar, position: Long) {
+        Timber.d("Scrubbing started at $position")
+
+        if (currentTrickPlay.value == null)
+            return
+
+        scrubbingPreview.visibility = View.VISIBLE
+        onScrubMove(timeBar, position)
+    }
+
+    override fun onScrubMove(timeBar: TimeBar, position: Long) {
+        Timber.d("Scrubbing to $position")
+
+        val currentBifData = currentTrickPlay.value ?: return
+        val image = BifUtil.getTrickPlayFrame(position.toInt(), currentBifData) ?: return
+
+        val parent = scrubbingPreview.parent as ViewGroup
+
+        val offset = position.toFloat() / player.duration
+        val minX = scrubbingPreview.left
+        val maxX = parent.width - parent.paddingRight
+
+        val startX = timeBarView.left + (timeBarView.right - timeBarView.left) * offset - scrubbingPreview.width / 2
+        val endX = startX + scrubbingPreview.width
+
+        val layoutX = when {
+            startX >= minX && endX <= maxX -> startX
+            startX < minX -> minX
+            else -> maxX - scrubbingPreview.width
+        }.toFloat()
+
+        scrubbingPreview.x = layoutX
+
+        Glide.with(scrubbingPreview)
+            .load(image)
+            .transform(roundedCorners)
+            .into(scrubbingPreview)
+    }
+
+    override fun onScrubStop(timeBar: TimeBar, position: Long, canceled: Boolean) {
+        Timber.d("Scrubbing stopped at $position")
+
+        scrubbingPreview.visibility = View.GONE
+    }
+}

app/phone/src/main/res/layout/exo_player_control_view.xml

@@ -173,6 +173,15 @@
         android:orientation="vertical"
         android:padding="16dp">
 
+        <ImageView
+            android:id="@+id/image_preview"
+            android:layout_width="160dp"
+            android:layout_height="90dp"
+            android:layout_gravity="start"
+            android:background="@android:color/transparent"
+            android:visibility="gone"
+            tools:visibility="visible" />
+
         <LinearLayout
             android:layout_width="wrap_content"
             android:layout_height="wrap_content"

core/src/main/res/values/strings.xml

@@ -150,6 +150,8 @@
     <string name="pref_player_mpv_gpu_api" translatable="false">GPU API</string>
     <string name="pref_player_intro_skipper">Intro Skipper</string>
     <string name="pref_player_intro_skipper_summary">Requires ConfusedPolarBear\'s Intro Skipper plugin to be installed on the server</string>
+    <string name="pref_player_trick_play">Trick Play</string>
+    <string name="pref_player_trick_play_summary">Requires nicknsy\'s Jellyscrub plugin to be installed on the server</string>
     <string name="addresses">Addresses</string>
     <string name="add_address">Add address</string>
     <string name="add_server_address">Add server address</string>

core/src/main/res/xml/fragment_settings_player.xml

@@ -106,4 +106,11 @@
         app:title="@string/pref_player_intro_skipper"
         app:widgetLayout="@layout/preference_material3_switch" />
 
+    <SwitchPreferenceCompat
+        app:defaultValue="true"
+        app:key="pref_player_trick_play"
+        app:summary="@string/pref_player_trick_play_summary"
+        app:title="@string/pref_player_trick_play"
+        app:widgetLayout="@layout/preference_material3_switch" />
+
 </PreferenceScreen>

data/src/main/java/dev/jdtech/jellyfin/models/TrickPlayManifest.kt

@@ -0,0 +1,12 @@
+package dev.jdtech.jellyfin.models
+
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class TrickPlayManifest(
+    @SerialName("Version")
+    val version: String,
+    @SerialName("WidthResolutions")
+    val widthResolutions: List<Int>
+)

data/src/main/java/dev/jdtech/jellyfin/repository/JellyfinRepository.kt

@@ -3,6 +3,7 @@ package dev.jdtech.jellyfin.repository
 import androidx.paging.PagingData
 import dev.jdtech.jellyfin.models.Intro
 import dev.jdtech.jellyfin.models.SortBy
+import dev.jdtech.jellyfin.models.TrickPlayManifest
 import java.util.UUID
 import kotlinx.coroutines.flow.Flow
 import org.jellyfin.sdk.model.api.BaseItemDto
@@ -65,6 +66,10 @@ interface JellyfinRepository {
 
     suspend fun getIntroTimestamps(itemId: UUID): Intro?
 
+    suspend fun getTrickPlayManifest(itemId: UUID): TrickPlayManifest?
+
+    suspend fun getTrickPlayData(itemId: UUID, width: Int): ByteArray?
+
     suspend fun postCapabilities()
 
     suspend fun postPlaybackStart(itemId: UUID)

data/src/main/java/dev/jdtech/jellyfin/repository/JellyfinRepositoryImpl.kt

@@ -6,6 +6,9 @@ import androidx.paging.PagingData
 import dev.jdtech.jellyfin.api.JellyfinApi
 import dev.jdtech.jellyfin.models.Intro
 import dev.jdtech.jellyfin.models.SortBy
+import dev.jdtech.jellyfin.models.TrickPlayManifest
+import io.ktor.util.cio.toByteArray
+import io.ktor.utils.io.ByteReadChannel
 import java.util.UUID
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.flow.Flow
@@ -233,6 +236,33 @@ class JellyfinRepositoryImpl(private val jellyfinApi: JellyfinApi) : JellyfinRep
             }
         }
 
+    override suspend fun getTrickPlayManifest(itemId: UUID): TrickPlayManifest? =
+        withContext(Dispatchers.IO) {
+            // https://github.com/nicknsy/jellyscrub/blob/main/Nick.Plugin.Jellyscrub/Api/TrickplayController.cs
+            val pathParameters = mutableMapOf<String, UUID>()
+            pathParameters["itemId"] = itemId
+
+            try {
+                return@withContext jellyfinApi.api.get<TrickPlayManifest>("/Trickplay/{itemId}/GetManifest", pathParameters).content
+            } catch (e: Exception) {
+                return@withContext null
+            }
+        }
+
+    override suspend fun getTrickPlayData(itemId: UUID, width: Int): ByteArray? =
+        withContext(Dispatchers.IO) {
+            // https://github.com/nicknsy/jellyscrub/blob/main/Nick.Plugin.Jellyscrub/Api/TrickplayController.cs
+            val pathParameters = mutableMapOf<String, Any>()
+            pathParameters["itemId"] = itemId
+            pathParameters["width"] = width
+
+            try {
+                return@withContext jellyfinApi.api.get<ByteReadChannel>("/Trickplay/{itemId}/{width}/GetBIF", pathParameters).content.toByteArray()
+            } catch (e: Exception) {
+                return@withContext null
+            }
+        }
+
     override suspend fun postCapabilities() {
         Timber.d("Sending capabilities")
         withContext(Dispatchers.IO) {

player/core/src/main/java/dev/jdtech/jellyfin/utils/bif/BifData.kt

@@ -0,0 +1,11 @@
+package dev.jdtech.jellyfin.utils.bif
+
+import android.graphics.Bitmap
+
+data class BifData(
+    val version: Int,
+    val timestampMultiplier: Int,
+    val imageCount: Int,
+    val images: Map<Int, Bitmap>,
+    val imageWidth: Int
+)

player/core/src/main/java/dev/jdtech/jellyfin/utils/bif/BifIndexEntry.kt

@@ -0,0 +1,3 @@
+package dev.jdtech.jellyfin.utils.bif
+
+data class BifIndexEntry(val timestamp: Int, val offset: Int)

player/core/src/main/java/dev/jdtech/jellyfin/utils/bif/BifUtil.kt

@@ -0,0 +1,77 @@
+package dev.jdtech.jellyfin.utils.bif
+
+import android.graphics.Bitmap
+import android.graphics.BitmapFactory
+import java.nio.ByteBuffer
+import timber.log.Timber
+
+object BifUtil {
+
+    /* https://github.com/nicknsy/jellyscrub/blob/main/Nick.Plugin.Jellyscrub/Api/trickplay.js */
+    private val BIF_MAGIC_NUMBERS = byteArrayOf(0x89.toByte(), 0x42.toByte(), 0x49.toByte(), 0x46.toByte(), 0x0D.toByte(), 0x0A.toByte(), 0x1A.toByte(), 0x0A.toByte())
+    private const val SUPPORTED_BIF_VERSION = 0
+
+    fun trickPlayDecode(array: ByteArray, width: Int): BifData? {
+        val data = Indexed8Array(array)
+
+        Timber.d("BIF file size: ${data.limit()}")
+
+        for (b in BIF_MAGIC_NUMBERS) {
+            if (data.read() != b) {
+                Timber.d("Attempted to read invalid bif file.")
+                return null
+            }
+        }
+
+        val bifVersion = data.readInt32()
+        if (bifVersion != SUPPORTED_BIF_VERSION) {
+            Timber.d("Client only supports BIF v$SUPPORTED_BIF_VERSION but file is v$bifVersion")
+            return null
+        }
+
+        Timber.d("BIF version: $bifVersion")
+
+        val bifImgCount = data.readInt32()
+        if (bifImgCount <= 0) {
+            Timber.d("BIF file contains no images.")
+            return null
+        }
+
+        Timber.d("BIF image count: $bifImgCount")
+
+        var timestampMultiplier = data.readInt32()
+        if (timestampMultiplier == 0) timestampMultiplier = 1000
+
+        data.addPosition(44) // Reserved
+
+        val bifIndex = mutableListOf<BifIndexEntry>()
+        for (i in 0 until bifImgCount) {
+            bifIndex.add(BifIndexEntry(data.readInt32(), data.readInt32()))
+        }
+
+        val bifImages = mutableMapOf<Int, Bitmap>()
+        for (i in bifIndex.indices) {
+            val indexEntry = bifIndex[i]
+            val timestamp = indexEntry.timestamp
+            val offset = indexEntry.offset
+            val nextOffset = bifIndex.getOrNull(i + 1)?.offset ?: data.limit()
+
+            val imageBuffer = ByteBuffer.wrap(data.array(), offset, nextOffset - offset).order(data.order())
+            val imageBytes = ByteArray(imageBuffer.remaining())
+            imageBuffer.get(imageBytes)
+
+            val bmp = BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.size)
+            bifImages[timestamp] = bmp
+        }
+
+        return BifData(bifVersion, timestampMultiplier, bifImgCount, bifImages, width)
+    }
+
+    fun getTrickPlayFrame(playerTimestamp: Int, data: BifData): Bitmap? {
+        val multiplier = data.timestampMultiplier
+        val images = data.images
+
+        val frame = playerTimestamp / multiplier
+        return images[frame]
+    }
+}

player/core/src/main/java/dev/jdtech/jellyfin/utils/bif/Indexed8Array.kt

@@ -0,0 +1,37 @@
+package dev.jdtech.jellyfin.utils.bif
+
+import java.nio.ByteOrder
+
+class Indexed8Array(private val array: ByteArray) {
+
+    private var readIndex = 0
+
+    fun read(): Byte {
+        return array[readIndex++]
+    }
+
+    fun addPosition(amount: Int) {
+        readIndex += amount
+    }
+
+    fun readInt32(): Int {
+        val b1 = read().toInt().and(0xFF)
+        val b2 = read().toInt().and(0xFF)
+        val b3 = read().toInt().and(0xFF)
+        val b4 = read().toInt().and(0xFF)
+
+        return b1 or (b2 shl 8) or (b3 shl 16) or (b4 shl 24)
+    }
+
+    fun array(): ByteArray {
+        return array
+    }
+
+    fun limit(): Int {
+        return array.size
+    }
+
+    fun order(): ByteOrder {
+        return ByteOrder.BIG_ENDIAN
+    }
+}

player/video/src/main/java/dev/jdtech/jellyfin/viewmodels/PlayerActivityViewModel.kt

@@ -23,6 +23,8 @@ import dev.jdtech.jellyfin.models.PlayerItem
 import dev.jdtech.jellyfin.mpv.MPVPlayer
 import dev.jdtech.jellyfin.mpv.TrackType
 import dev.jdtech.jellyfin.repository.JellyfinRepository
+import dev.jdtech.jellyfin.utils.bif.BifData
+import dev.jdtech.jellyfin.utils.bif.BifUtil
 import dev.jdtech.jellyfin.utils.postDownloadPlaybackProgress
 import java.util.UUID
 import javax.inject.Inject
@@ -30,6 +32,8 @@ import kotlinx.coroutines.DelicateCoroutinesApi
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.GlobalScope
 import kotlinx.coroutines.delay
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.asStateFlow
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.withContext
 import timber.log.Timber
@@ -55,6 +59,10 @@ constructor(
     private val _currentIntro = MutableLiveData<Intro?>(null)
     val currentIntro: LiveData<Intro?> = _currentIntro
 
+    private val trickPlays: MutableMap<UUID, BifData> = mutableMapOf()
+    private val _currentTrickPlay = MutableStateFlow<BifData?>(null)
+    val currentTrickPlay = _currentTrickPlay.asStateFlow()
+
     var currentAudioTracks: MutableList<MPVPlayer.Companion.Track> = mutableListOf()
     var currentSubtitleTracks: MutableList<MPVPlayer.Companion.Track> = mutableListOf()
 
@@ -181,6 +189,7 @@ constructor(
             }
         }
 
+        _currentTrickPlay.value = null
         playWhenReady = player.playWhenReady
         playbackPosition = player.currentPosition
         currentMediaItemIndex = player.currentMediaItemIndex
@@ -246,9 +255,14 @@ constructor(
                                 "S${item.parentIndexNumber}:E${item.indexNumber} - ${item.name}"
                         else
                             _currentItemTitle.value = item.name.orEmpty()
+
+                        jellyfinRepository.postPlaybackStart(item.itemId)
+
+                        if (appPreferences.playerTrickPlay) {
+                            getTrickPlay(item.itemId)
+                        }
                     }
                 }
-                jellyfinRepository.postPlaybackStart(UUID.fromString(mediaItem?.mediaId))
             } catch (e: Exception) {
                 Timber.e(e)
             }
@@ -311,4 +325,28 @@ constructor(
         player.setPlaybackSpeed(speed)
         playbackSpeed = speed
     }
+
+    private suspend fun getTrickPlay(itemId: UUID) {
+        if (trickPlays[itemId] != null) return
+        jellyfinRepository.getTrickPlayManifest(itemId)
+            ?.let { trickPlayManifest ->
+                val widthResolution =
+                    trickPlayManifest.widthResolutions.max()
+                Timber.d("Trickplay Resolution: $widthResolution")
+
+                jellyfinRepository.getTrickPlayData(
+                    itemId,
+                    widthResolution
+                )?.let { byteArray ->
+                    val trickPlayData =
+                        BifUtil.trickPlayDecode(byteArray, widthResolution)
+
+                    trickPlayData?.let {
+                        Timber.d("Trickplay Images: ${it.imageCount}")
+                        trickPlays[itemId] = it
+                        _currentTrickPlay.value = trickPlays[itemId]
+                    }
+                }
+            }
+    }
 }

preferences/src/main/java/dev/jdtech/jellyfin/AppPreferences.kt

@@ -72,6 +72,7 @@ constructor(
     val playerMpvAo get() = sharedPreferences.getString(Constants.PREF_PLAYER_MPV_AO, "audiotrack")!!
     val playerMpvGpuApi get() = sharedPreferences.getString(Constants.PREF_PLAYER_MPV_GPU_API, "opengl")!!
     val playerIntroSkipper get() = sharedPreferences.getBoolean(Constants.PREF_PLAYER_INTRO_SKIPPER, true)
+    val playerTrickPlay get() = sharedPreferences.getBoolean(Constants.PREF_PLAYER_TRICK_PLAY, true)
 
     // Language
     val preferredAudioLanguage get() = sharedPreferences.getString(Constants.PREF_AUDIO_LANGUAGE, "")!!

preferences/src/main/java/dev/jdtech/jellyfin/Constants.kt

@@ -26,6 +26,7 @@ object Constants {
     const val PREF_PLAYER_MPV_AO = "pref_player_mpv_ao"
     const val PREF_PLAYER_MPV_GPU_API = "pref_player_mpv_gpu_api"
     const val PREF_PLAYER_INTRO_SKIPPER = "pref_player_intro_skipper"
+    const val PREF_PLAYER_TRICK_PLAY = "pref_player_trick_play"
     const val PREF_AUDIO_LANGUAGE = "pref_audio_language"
     const val PREF_SUBTITLE_LANGUAGE = "pref_subtitle_language"
     const val PREF_IMAGE_CACHE = "pref_image_cache"