Support remux/transcode fallback in new music player

This commit is contained in:
Niels van Velzen 2023-10-10 14:46:26 +02:00 committed by Niels van Velzen
parent d18922f7df
commit 655b9ce736
13 changed files with 196 additions and 127 deletions

View File

@ -7,9 +7,9 @@ import kotlinx.coroutines.flow.asStateFlow
import org.jellyfin.playback.core.backend.BackendService
import org.jellyfin.playback.core.backend.PlayerBackendEventListener
import org.jellyfin.playback.core.mediastream.DefaultMediaStreamState
import org.jellyfin.playback.core.mediastream.MediaStream
import org.jellyfin.playback.core.mediastream.MediaStreamResolver
import org.jellyfin.playback.core.mediastream.MediaStreamState
import org.jellyfin.playback.core.mediastream.PlayableMediaStream
import org.jellyfin.playback.core.model.PlayState
import org.jellyfin.playback.core.model.PlaybackOrder
import org.jellyfin.playback.core.model.PositionInfo
@ -100,7 +100,7 @@ class MutablePlayerState(
_videoSize.value = VideoSize(width, height)
}
override fun onMediaStreamEnd(mediaStream: MediaStream) = Unit
override fun onMediaStreamEnd(mediaStream: PlayableMediaStream) = Unit
})
queue = DefaultPlayerQueueState(this, scope, backendService)

View File

@ -1,6 +1,6 @@
package org.jellyfin.playback.core.backend
import org.jellyfin.playback.core.mediastream.MediaStream
import org.jellyfin.playback.core.mediastream.PlayableMediaStream
import org.jellyfin.playback.core.model.PlayState
/**
@ -42,7 +42,7 @@ class BackendService {
callListeners { onVideoSizeChange(width, height) }
}
override fun onMediaStreamEnd(mediaStream: MediaStream) {
override fun onMediaStreamEnd(mediaStream: PlayableMediaStream) {
callListeners { onMediaStreamEnd(mediaStream) }
}
}

View File

@ -1,6 +1,7 @@
package org.jellyfin.playback.core.backend
import org.jellyfin.playback.core.mediastream.MediaStream
import org.jellyfin.playback.core.mediastream.PlayableMediaStream
import org.jellyfin.playback.core.model.PositionInfo
import org.jellyfin.playback.core.support.PlaySupportReport
import kotlin.time.Duration
@ -20,8 +21,8 @@ interface PlayerBackend {
// Mutation
fun prepareStream(stream: MediaStream)
fun playStream(stream: MediaStream)
fun prepareStream(stream: PlayableMediaStream)
fun playStream(stream: PlayableMediaStream)
fun play()
fun pause()

View File

@ -1,10 +1,10 @@
package org.jellyfin.playback.core.backend
import org.jellyfin.playback.core.mediastream.MediaStream
import org.jellyfin.playback.core.mediastream.PlayableMediaStream
import org.jellyfin.playback.core.model.PlayState
interface PlayerBackendEventListener {
fun onPlayStateChange(state: PlayState)
fun onVideoSizeChange(width: Int, height: Int)
fun onMediaStreamEnd(mediaStream: MediaStream)
fun onMediaStreamEnd(mediaStream: PlayableMediaStream)
}

View File

@ -2,14 +2,40 @@ package org.jellyfin.playback.core.mediastream
import org.jellyfin.playback.core.queue.item.QueueEntry
data class MediaStream(
val identifier: String,
interface MediaStream {
val identifier: String
val conversionMethod: MediaConversionMethod
val container: MediaStreamContainer
val tracks: Collection<MediaStreamTrack>
}
data class BasicMediaStream(
override val identifier: String,
override val conversionMethod: MediaConversionMethod,
override val container: MediaStreamContainer,
override val tracks: Collection<MediaStreamTrack>,
) : MediaStream {
fun toPlayableMediaStream(
queueEntry: QueueEntry,
url: String,
) = PlayableMediaStream(
identifier = identifier,
conversionMethod = conversionMethod,
container = container,
tracks = tracks,
queueEntry = queueEntry,
url = url,
)
}
data class PlayableMediaStream(
override val identifier: String,
override val conversionMethod: MediaConversionMethod,
override val container: MediaStreamContainer,
override val tracks: Collection<MediaStreamTrack>,
val queueEntry: QueueEntry,
val conversionMethod: MediaConversionMethod,
val url: String,
val container: MediaStreamContainer,
val tracks: Collection<MediaStreamTrack>,
)
) : MediaStream
data class MediaStreamContainer(
val format: String,

View File

@ -1,13 +1,17 @@
package org.jellyfin.playback.core.mediastream
import org.jellyfin.playback.core.queue.item.QueueEntry
import org.jellyfin.playback.core.support.PlaySupportReport
/**
* Determine the media stream for a given queue item.
*/
interface MediaStreamResolver {
/**
* @return [MediaStream] or null if no stream can be determined by this resolver
* @return [PlayableMediaStream] or null if no stream can be determined by this resolver
*/
suspend fun getStream(queueEntry: QueueEntry): MediaStream?
suspend fun getStream(
queueEntry: QueueEntry,
testStream: (stream: MediaStream) -> PlaySupportReport,
): PlayableMediaStream?
}

View File

@ -7,14 +7,15 @@ import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.flow.launchIn
import kotlinx.coroutines.flow.onEach
import kotlinx.coroutines.withContext
import kotlinx.coroutines.plus
import org.jellyfin.playback.core.PlayerState
import org.jellyfin.playback.core.backend.BackendService
import org.jellyfin.playback.core.backend.PlayerBackend
import timber.log.Timber
interface MediaStreamState {
val current: StateFlow<MediaStream?>
val next: StateFlow<MediaStream?>
val current: StateFlow<PlayableMediaStream?>
val next: StateFlow<PlayableMediaStream?>
}
class DefaultMediaStreamState(
@ -23,63 +24,56 @@ class DefaultMediaStreamState(
private val mediaStreamResolvers: Collection<MediaStreamResolver>,
private val backendService: BackendService,
) : MediaStreamState {
private val _current = MutableStateFlow<MediaStream?>(null)
override val current: StateFlow<MediaStream?> get() = _current.asStateFlow()
private val _current = MutableStateFlow<PlayableMediaStream?>(null)
override val current: StateFlow<PlayableMediaStream?> get() = _current.asStateFlow()
private val _next = MutableStateFlow<MediaStream?>(null)
override val next: StateFlow<MediaStream?> get() = _next.asStateFlow()
private val _next = MutableStateFlow<PlayableMediaStream?>(null)
override val next: StateFlow<PlayableMediaStream?> get() = _next.asStateFlow()
init {
state.queue.entry.onEach { entry ->
Timber.d("Queue entry changed to $entry")
val backend = requireNotNull(backendService.backend)
if (entry == null) {
setCurrent(null)
backend.setCurrent(null)
} else {
val streamResult = runCatching {
mediaStreamResolvers.firstNotNullOfOrNull { resolver -> resolver.getStream(entry) }
val stream = mediaStreamResolvers.firstNotNullOfOrNull { resolver ->
runCatching {
resolver.getStream(entry, backend::supportsStream)
}.onFailure {
Timber.e(it, "Media stream resolver failed for $entry")
}.getOrNull()
}
val stream = streamResult.getOrNull()
when {
streamResult.isFailure -> Timber.e(streamResult.exceptionOrNull(), "Media stream resolver failed for $entry")
stream == null -> Timber.e("Unable to resolve stream for entry $entry")
else -> {
if (!canPlayStream(stream)) {
Timber.w("Playback of the received media stream for $entry is not supported")
}
setCurrent(stream)
if (stream == null) {
Timber.e("Unable to resolve stream for entry $entry")
// TODO: Somehow notify the user that we skipped an unplayable entry
if (state.queue.peekNext() != null) {
state.queue.next(usePlaybackOrder = true, useRepeatMode = false)
} else {
backend.setCurrent(null)
}
} else {
backend.setCurrent(stream)
}
}
}.launchIn(coroutineScope)
}.launchIn(coroutineScope + Dispatchers.Main)
// TODO Register some kind of event when $current item is at -30 seconds to setNext()
}
private suspend fun canPlayStream(stream: MediaStream) = withContext(Dispatchers.Main) {
backendService.backend?.supportsStream(stream)?.canPlay == true
}
private suspend fun setCurrent(stream: MediaStream?) {
private fun PlayerBackend.setCurrent(stream: PlayableMediaStream?) {
Timber.d("Current stream changed to $stream")
val backend = requireNotNull(backendService.backend)
_current.value = stream
withContext(Dispatchers.Main) {
if (stream == null) backend.stop()
else backend.playStream(stream)
}
if (stream == null) stop()
else playStream(stream)
}
private suspend fun setNext(stream: MediaStream) {
val backend = requireNotNull(backendService.backend)
private fun PlayerBackend.setNext(stream: PlayableMediaStream) {
_current.value = stream
withContext(Dispatchers.Main) {
backend.prepareStream(stream)
}
prepareStream(stream)
}
}

View File

@ -10,7 +10,7 @@ import kotlinx.coroutines.launch
import org.jellyfin.playback.core.PlayerState
import org.jellyfin.playback.core.backend.BackendService
import org.jellyfin.playback.core.backend.PlayerBackendEventListener
import org.jellyfin.playback.core.mediastream.MediaStream
import org.jellyfin.playback.core.mediastream.PlayableMediaStream
import org.jellyfin.playback.core.model.PlayState
import org.jellyfin.playback.core.model.PlaybackOrder
import org.jellyfin.playback.core.model.RepeatMode
@ -80,7 +80,7 @@ class DefaultPlayerQueueState(
override fun onPlayStateChange(state: PlayState) = Unit
override fun onVideoSizeChange(width: Int, height: Int) = Unit
override fun onMediaStreamEnd(mediaStream: MediaStream) {
override fun onMediaStreamEnd(mediaStream: PlayableMediaStream) {
// TODO: Find position based on $mediaStream instead
// TODO: This doesn't work as expected
coroutineScope.launch { next(usePlaybackOrder = true, useRepeatMode = true) }

View File

@ -10,6 +10,7 @@ import com.google.android.exoplayer2.Player
import com.google.android.exoplayer2.video.VideoSize
import org.jellyfin.playback.core.backend.BasePlayerBackend
import org.jellyfin.playback.core.mediastream.MediaStream
import org.jellyfin.playback.core.mediastream.PlayableMediaStream
import org.jellyfin.playback.core.model.PlayState
import org.jellyfin.playback.core.model.PositionInfo
import org.jellyfin.playback.core.support.PlaySupportReport
@ -23,7 +24,7 @@ import kotlin.time.Duration.Companion.milliseconds
class ExoPlayerBackend(
private val context: Context,
) : BasePlayerBackend() {
private var currentStream: MediaStream? = null
private var currentStream: PlayableMediaStream? = null
private val exoPlayer by lazy {
val renderersFactory = DefaultRenderersFactory(context).apply {
@ -74,7 +75,7 @@ class ExoPlayerBackend(
stream: MediaStream
): PlaySupportReport = exoPlayer.getPlaySupportReport(stream.toFormat())
override fun prepareStream(stream: MediaStream) {
override fun prepareStream(stream: PlayableMediaStream) {
val mediaItem = MediaItem.Builder().apply {
setTag(stream)
setMediaId(stream.hashCode().toString())
@ -89,7 +90,7 @@ class ExoPlayerBackend(
exoPlayer.prepare()
}
override fun playStream(stream: MediaStream) {
override fun playStream(stream: PlayableMediaStream) {
if (currentStream == stream) return
currentStream = stream

View File

@ -7,6 +7,9 @@ import org.jellyfin.playback.jellyfin.playsession.PlaySessionSocketService
import org.jellyfin.sdk.api.client.ApiClient
import org.jellyfin.sdk.api.sockets.SocketInstance
import org.jellyfin.sdk.model.api.DeviceProfile
import org.jellyfin.sdk.model.api.DlnaProfileType
import org.jellyfin.sdk.model.api.EncodingContext
import org.jellyfin.sdk.model.api.TranscodingProfile
fun jellyfinPlugin(
api: ApiClient,
@ -20,13 +23,23 @@ fun jellyfinPlugin(
responseProfiles = emptyList(),
subtitleProfiles = emptyList(),
supportedMediaTypes = "",
transcodingProfiles = emptyList(),
// Add at least one transcoding profile so the server returns a value
// for "SupportsTranscoding" based on the user policy
// We don't actually use this profile in the client
transcodingProfiles = listOf(
TranscodingProfile(
type = DlnaProfileType.AUDIO,
context = EncodingContext.STREAMING,
protocol = "hls",
container = "mp3",
audioCodec = "mp3",
videoCodec = "",
conditions = emptyList()
)
),
xmlRootAttributes = emptyList(),
)
provide(AudioMediaStreamResolver(api, profile).apply {
// TODO: Remove once we have a proper device profile
forceDirectPlay = true
})
provide(AudioMediaStreamResolver(api, profile))
val playSessionService = PlaySessionService(api)
provide(playSessionService)

View File

@ -1,13 +1,16 @@
package org.jellyfin.playback.jellyfin.mediastream
import org.jellyfin.playback.core.mediastream.BasicMediaStream
import org.jellyfin.playback.core.mediastream.MediaConversionMethod
import org.jellyfin.playback.core.mediastream.MediaStream
import org.jellyfin.playback.core.mediastream.MediaStreamContainer
import org.jellyfin.playback.core.mediastream.PlayableMediaStream
import org.jellyfin.playback.core.queue.item.QueueEntry
import org.jellyfin.playback.core.support.PlaySupportReport
import org.jellyfin.playback.jellyfin.queue.item.BaseItemDtoUserQueueEntry
import org.jellyfin.sdk.api.client.ApiClient
import org.jellyfin.sdk.api.client.exception.MissingBaseUrlException
import org.jellyfin.sdk.api.client.extensions.audioApi
import org.jellyfin.sdk.api.client.util.UrlBuilder
import org.jellyfin.sdk.api.client.extensions.dynamicHlsApi
import org.jellyfin.sdk.model.api.BaseItemKind
import org.jellyfin.sdk.model.api.DeviceProfile
@ -15,72 +18,94 @@ class AudioMediaStreamResolver(
val api: ApiClient,
val profile: DeviceProfile,
) : JellyfinStreamResolver(api, profile) {
/**
* Force direct play when enabled, even when we know it will fail.
*/
var forceDirectPlay = false
companion object {
private val REMUX_CONTAINERS = arrayOf("mp3", "ogg", "mkv")
private const val REMUX_SEGMENT_CONTAINER = "mp3"
}
override suspend fun getStream(queueEntry: QueueEntry): MediaStream? {
private fun MediaInfo.getDirectPlayStream() = BasicMediaStream(
identifier = playSessionId,
conversionMethod = MediaConversionMethod.None,
container = getMediaStreamContainer(),
tracks = getTracks()
)
private fun MediaInfo.getRemuxStream(container: String) = BasicMediaStream(
identifier = playSessionId,
conversionMethod = MediaConversionMethod.Remux,
container = MediaStreamContainer(
format = container
),
tracks = getTracks()
)
private fun MediaInfo.getTranscodeStream() = BasicMediaStream(
identifier = playSessionId,
conversionMethod = MediaConversionMethod.Transcode,
// The server doesn't provide us with the transcode information os we return mock data
container = MediaStreamContainer(format = "unknown"),
tracks = emptyList()
)
override suspend fun getStream(
queueEntry: QueueEntry,
testStream: (stream: MediaStream) -> PlaySupportReport,
): PlayableMediaStream? {
if (queueEntry !is BaseItemDtoUserQueueEntry) return null
if (queueEntry.baseItem.type != BaseItemKind.AUDIO) return null
val mediaInfo = getPlaybackInfo(queueEntry.baseItem)
val conversionMethod = when {
// Direct play
mediaInfo.mediaSource.supportsDirectPlay || forceDirectPlay -> MediaConversionMethod.None
// Remux (Direct stream)
mediaInfo.mediaSource.supportsDirectStream -> MediaConversionMethod.Remux
// Transcode
mediaInfo.mediaSource.supportsTranscoding -> MediaConversionMethod.Transcode
else -> error("Unable to find a suitable playback method for media")
}
val url = when (conversionMethod) {
// Direct play
is MediaConversionMethod.None -> {
api.audioApi.getAudioStreamUrl(
// Test for direct play support
val directPlayStream = mediaInfo.getDirectPlayStream()
if (testStream(directPlayStream).canPlay) {
return directPlayStream.toPlayableMediaStream(
queueEntry = queueEntry,
url = api.audioApi.getAudioStreamUrl(
itemId = queueEntry.baseItem.id,
mediaSourceId = mediaInfo.mediaSource.id,
playSessionId = mediaInfo.playSessionId,
static = true,
)
}
// Remux (Direct stream)
is MediaConversionMethod.Remux -> {
val container = requireNotNull(mediaInfo.mediaSource.container) {
"MediaSource supports direct stream but container is null"
}
)
}
api.audioApi.getAudioStreamByContainerUrl(
itemId = queueEntry.baseItem.id,
mediaSourceId = mediaInfo.mediaSource.id,
playSessionId = mediaInfo.playSessionId,
container = container,
)
}
// Transcode
is MediaConversionMethod.Transcode -> {
val url = requireNotNull(mediaInfo.mediaSource.transcodingUrl) {
"MediaSource supports transcoding but transcodingUrl is null"
// Try remuxing
if (mediaInfo.mediaSource.supportsDirectStream) {
for (container in REMUX_CONTAINERS) {
val remuxStream = mediaInfo.getRemuxStream(container)
if (testStream(remuxStream).canPlay) {
return remuxStream.toPlayableMediaStream(
queueEntry = queueEntry,
url = api.audioApi.getAudioStreamByContainerUrl(
itemId = queueEntry.baseItem.id,
mediaSourceId = mediaInfo.mediaSource.id,
playSessionId = mediaInfo.playSessionId,
container = container,
)
)
}
// TODO Use api.createUrl() with SDK 1.5
UrlBuilder.buildUrl(
api.baseUrl ?: throw MissingBaseUrlException(),
url,
ignorePathParameters = true,
)
}
}
return MediaStream(
identifier = mediaInfo.playSessionId,
queueEntry = queueEntry,
conversionMethod = conversionMethod,
url = url,
container = mediaInfo.getMediaStreamContainer(),
tracks = mediaInfo.getTracks()
)
// Fallback to provided transcode
if (mediaInfo.mediaSource.supportsTranscoding) {
val transcodeStream = mediaInfo.getTranscodeStream()
// Skip testing transcode stream because we lack the information to do so
return transcodeStream.toPlayableMediaStream(
queueEntry = queueEntry,
url = api.dynamicHlsApi.getMasterHlsAudioPlaylistUrl(
itemId = queueEntry.baseItem.id,
mediaSourceId = requireNotNull(mediaInfo.mediaSource.id),
playSessionId = mediaInfo.playSessionId,
tag = mediaInfo.mediaSource.eTag,
segmentContainer = REMUX_SEGMENT_CONTAINER,
)
)
}
// Unable to find a suitable stream, return
return null
}
}

View File

@ -2,7 +2,9 @@ package org.jellyfin.playback.jellyfin.mediastream
import org.jellyfin.playback.core.mediastream.MediaConversionMethod
import org.jellyfin.playback.core.mediastream.MediaStream
import org.jellyfin.playback.core.mediastream.PlayableMediaStream
import org.jellyfin.playback.core.queue.item.QueueEntry
import org.jellyfin.playback.core.support.PlaySupportReport
import org.jellyfin.playback.jellyfin.queue.item.BaseItemDtoUserQueueEntry
import org.jellyfin.sdk.api.client.ApiClient
import org.jellyfin.sdk.api.client.extensions.universalAudioApi
@ -13,7 +15,10 @@ class UniversalAudioMediaStreamResolver(
val api: ApiClient,
val profile: DeviceProfile,
) : JellyfinStreamResolver(api, profile) {
override suspend fun getStream(queueEntry: QueueEntry): MediaStream? {
override suspend fun getStream(
queueEntry: QueueEntry,
testStream: (stream: MediaStream) -> PlaySupportReport,
): PlayableMediaStream? {
if (queueEntry !is BaseItemDtoUserQueueEntry) return null
if (queueEntry.baseItem.type != BaseItemKind.AUDIO) return null
@ -35,13 +40,13 @@ class UniversalAudioMediaStreamResolver(
audioCodec = "mp3",
)
return MediaStream(
return PlayableMediaStream(
identifier = mediaInfo.playSessionId,
queueEntry = queueEntry,
conversionMethod = MediaConversionMethod.None,
url = url,
container = mediaInfo.getMediaStreamContainer(),
tracks = mediaInfo.getTracks()
)
).takeIf { stream -> testStream(stream).canPlay }
}
}

View File

@ -6,7 +6,7 @@ import kotlinx.coroutines.flow.onEach
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import org.jellyfin.playback.core.mediastream.MediaConversionMethod
import org.jellyfin.playback.core.mediastream.MediaStream
import org.jellyfin.playback.core.mediastream.PlayableMediaStream
import org.jellyfin.playback.core.model.PlayState
import org.jellyfin.playback.core.model.RepeatMode
import org.jellyfin.playback.core.plugin.PlayerService
@ -25,7 +25,7 @@ import org.jellyfin.sdk.model.api.RepeatMode as SdkRepeatMode
class PlaySessionService(
private val api: ApiClient,
) : PlayerService() {
private var reportedStream: MediaStream? = null
private var reportedStream: PlayableMediaStream? = null
override suspend fun onInitialize() {
state.streams.current.onEach { stream -> onMediaStreamChange(stream) }.launchIn(coroutineScope)
@ -40,7 +40,7 @@ class PlaySessionService(
}.launchIn(coroutineScope)
}
private val MediaStream.baseItem
private val PlayableMediaStream.baseItem
get() = when (val entry = queueEntry) {
is BaseItemDtoUserQueueEntry -> entry.baseItem
else -> null
@ -66,7 +66,7 @@ class PlaySessionService(
}
}
private fun onMediaStreamChange(stream: MediaStream?) {
private fun onMediaStreamChange(stream: PlayableMediaStream?) {
reportedStream = stream
onStart()
}
@ -98,7 +98,7 @@ class PlaySessionService(
.map { QueueItem(id = it.baseItem.id, playlistItemId = it.baseItem.playlistItemId) }
}
private suspend fun sendStreamStart(stream: MediaStream) {
private suspend fun sendStreamStart(stream: PlayableMediaStream) {
val item = stream.baseItem ?: return
api.playStateApi.reportPlaybackStart(PlaybackStartInfo(
itemId = item.id,
@ -116,7 +116,7 @@ class PlaySessionService(
))
}
private suspend fun sendStreamUpdate(stream: MediaStream) {
private suspend fun sendStreamUpdate(stream: PlayableMediaStream) {
val item = stream.baseItem ?: return
api.playStateApi.reportPlaybackProgress(PlaybackProgressInfo(
itemId = item.id,
@ -134,7 +134,7 @@ class PlaySessionService(
))
}
private suspend fun sendStreamStop(stream: MediaStream) {
private suspend fun sendStreamStop(stream: PlayableMediaStream) {
val item = stream.baseItem ?: return
api.playStateApi.reportPlaybackStopped(PlaybackStopInfo(
itemId = item.id,