Compare commits

...
6 Commits
Author SHA1 Message Date
zoriya 59218e8fa0 Fix git installs 2026-07-27 22:19:47 +02:00
zoriyaandClaude Opus 4.8 f98c787b15 feat(cast): unify ass/pgs subtitles through cast tracks
ASS/PGS subtitles are now declared as regular cast text tracks; the custom
receiver draws the ones it can't render natively, keyed off the active track
id. Removes the custom message channel, the customData readback, and the
per-platform overlay-selection state so both senders treat every subtitle the
same.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WQ5PxMcMRmAMVxP8ZnYKnm
2026-07-27 22:11:35 +02:00
zoriya 12be950ea0 fix(pip): disable pip when casting 2026-07-27 00:29:05 +02:00
zoriya 6ab5743f94 fix(pip): fix wrong player source 2026-07-26 23:26:17 +02:00
zoriya 9dae350c73 fix(pip): fix black screen 2026-07-26 22:36:05 +02:00
zoriya 074b821e06 fix(pip): prevent pip without a view mounted 2026-07-26 21:07:00 +02:00
11 changed files with 210 additions and 170 deletions
@@ -56,6 +56,12 @@ class OmniCastTrackSelector : CastTrackSelector() {
indices.forEach { selections.remove(groups[it]) }
if (disabled) return
// An explicit override selects an exact track by identity; this wins over
// language/label matching, which can't disambiguate same-language tracks.
val overrides = request.trackSelectionParameters.overrides
val overrideMatch = indices.firstOrNull { i ->
overrides.keys.any { it.id == groups[i].id }
}
val match = indices.firstOrNull { i ->
val name: String? = tracks[i].name
val language: String? = tracks[i].language
@@ -64,6 +70,7 @@ class OmniCastTrackSelector : CastTrackSelector() {
Util.normalizeLanguageCode(language) in languages)
}
val chosen = when {
overrideMatch != null -> overrideMatch
match != null -> match
previous.isNotEmpty() -> previous.first()
selectDefault -> indices.first()
@@ -8,7 +8,6 @@ import android.app.PendingIntent
import android.content.Intent
import android.os.Build
import android.os.Bundle
import android.view.SurfaceHolder
import androidx.core.app.NotificationCompat
import androidx.media3.common.AudioAttributes
import androidx.media3.common.C
@@ -38,6 +37,9 @@ import androidx.media3.session.DefaultMediaNotificationProvider
import androidx.media3.session.MediaSession
import androidx.media3.session.MediaSessionService
import androidx.mediarouter.app.MediaRouteChooserDialog
import androidx.mediarouter.media.MediaRouteSelector
import androidx.mediarouter.media.MediaRouter
import com.google.android.gms.cast.CastMediaControlIntent
import com.google.android.gms.cast.framework.CastContext
import com.google.android.gms.cast.framework.CastState
import com.google.android.gms.cast.framework.CastStateListener
@@ -78,6 +80,14 @@ class OmniPlayer(
private val castStateListener =
CastStateListener { eventMap.emitCastStatus(computeCastStatus()) }
// Without a MediaRouteButton the Cast SDK never actively scans, so
// CastContext.castState stays NO_DEVICES_AVAILABLE (reported as
// "unavailable") even when receivers are on the network. Hold an active
// discovery request so castState reflects real availability. MediaRouter
// must only be touched on the main thread.
private var mediaRouter: MediaRouter? = null
private val mediaRouterCallback = object : MediaRouter.Callback() {}
val player: Player = runOnMainThreadSync {
castOptions?.receiverApplicationId?.let { receiverApplicationId = it }
val cc = try {
@@ -91,6 +101,21 @@ class OmniPlayer(
localPlayer
} else {
cc.addCastStateListener(castStateListener)
// Build the discovery selector from the receiver app id directly
// (cc.mergedSelector can be null this early), then keep an active
// discovery request so castState reflects real availability.
val appId = receiverApplicationId
?: CastMediaControlIntent.DEFAULT_MEDIA_RECEIVER_APPLICATION_ID
val selector = cc.mergedSelector ?: MediaRouteSelector.Builder()
.addControlCategory(CastMediaControlIntent.categoryForCast(appId))
.build()
val router = MediaRouter.getInstance(ctx)
router.addCallback(
selector,
mediaRouterCallback,
MediaRouter.CALLBACK_FLAG_REQUEST_DISCOVERY,
)
mediaRouter = router
val remote = RemoteCastPlayer.Builder(ctx)
.setMediaItemConverter(OmniMediaItemConverter())
.setTrackSelector(OmniCastTrackSelector())
@@ -125,6 +150,9 @@ class OmniPlayer(
override val castStatus: CastStatus
get() = runOnMainThreadSync { computeCastStatus() }
val isCasting: Boolean
get() = castContext?.sessionManager?.currentCastSession?.isConnected == true
private fun computeCastStatus(): CastStatus {
val cc = castContext ?: return CastStatus.UNSUPPORTED
return when (cc.castState) {
@@ -157,6 +185,7 @@ class OmniPlayer(
eventMap.dispose()
runOnMainThread {
castContext?.removeCastStateListener(castStateListener)
mediaRouter?.removeCallback(mediaRouterCallback)
// release both cast and local players.
player.release()
}
@@ -208,16 +237,29 @@ class OmniPlayer(
.build()
}
fun setSurface(holder: SurfaceHolder?) {
fun setVideoView(surfaceView: android.view.SurfaceView?) {
runOnMainThread {
if (holder == null) {
if (surfaceView == null) {
localPlayer.clearVideoSurface()
} else {
localPlayer.setVideoSurfaceHolder(holder)
localPlayer.setVideoSurfaceView(surfaceView)
}
}
}
// rebuild video pipeline after pip, noop for exoplayer that recover on it's own.
fun rebuildVideoOutput() {
runOnMainThread {
(localPlayer as? VlcPlayer)?.rebuildVideoOutput()
}
}
fun updateVideoLayout(width: Int, height: Int) {
runOnMainThread {
(localPlayer as? VlcPlayer)?.updateVideoLayout(width, height)
}
}
override val hasPrev: Boolean get() = player.hasPreviousMediaItem()
override val hasNext: Boolean get() = player.hasNextMediaItem()
override val status by mainThreadProperty {
@@ -346,57 +388,45 @@ class OmniPlayer(
}
}
override fun selectAudio(audio: Track) {
override fun selectAudio(audio: Track) = selectTrack(C.TRACK_TYPE_AUDIO, audio)
// Select the exact track by its group id (stable identity) rather than by
// preferred language/label: two same-language tracks, or tracks whose labels
// don't survive the cast round-trip, can't be told apart by preference. Falls
// back to language/label only when the group can't be found yet.
private fun selectTrack(type: Int, track: Track?) {
runOnMainThreadSync {
player.trackSelectionParameters = player.trackSelectionParameters
.buildUpon()
.setPreferredAudioLanguage(audio.language)
.setPreferredAudioLabels(*(audio.label?.let { arrayOf(it) } ?: emptyArray()))
.build()
val params = player.trackSelectionParameters.buildUpon()
.clearOverridesOfType(type)
if (track == null) {
params.setTrackTypeDisabled(type, true)
} else {
params.setTrackTypeDisabled(type, false)
val group = player.currentTracks.groups.firstOrNull {
it.type == type && it.mediaTrackGroup.id == track.id
}?.mediaTrackGroup
if (group != null) {
params.setOverrideForType(TrackSelectionOverride(group, 0))
} else {
val labels = track.label?.let { arrayOf(it) } ?: emptyArray()
if (type == C.TRACK_TYPE_AUDIO) {
params.setPreferredAudioLanguage(track.language)
.setPreferredAudioLabels(*labels)
} else {
params.setPreferredTextLanguage(track.language)
.setPreferredTextLabels(*labels)
}
}
}
player.trackSelectionParameters = params.build()
}
}
override fun selectSubtitle(subtitle: Track?) {
// Custom (ASS/PGS) subtitles can't be rendered by the cast receiver's
// native pipeline, so while casting they are forwarded over omni's
// message channel (the receiver draws them as an overlay). This mirrors
// the web behavior.
val custom = subtitle?.let { track ->
source?.subtitles?.firstOrNull { it.id == track.id }?.let { sub ->
val mime = sub.mimeType?.lowercase() ?: ""
val ext = sub.link.substringBefore('?').substringBefore('#')
.substringAfterLast('.', "").lowercase()
mime.contains("ass") || mime.contains("ssa") || mime.contains("pgs") ||
ext == "ass" || ext == "ssa" || ext == "sup"
}
} ?: false
runOnMainThreadSync {
val session = castContext?.sessionManager?.currentCastSession
val casting = session?.isConnected == true
if (subtitle == null || (custom && casting)) {
player.trackSelectionParameters = player.trackSelectionParameters
.buildUpon()
.setTrackTypeDisabled(C.TRACK_TYPE_TEXT, true)
.build()
} else {
player.trackSelectionParameters = player.trackSelectionParameters
.buildUpon()
.setTrackTypeDisabled(C.TRACK_TYPE_TEXT, false)
.setPreferredTextLanguage(subtitle.language)
.setPreferredTextLabels(*(subtitle.label?.let { arrayOf(it) } ?: emptyArray()))
.build()
}
if (casting) {
val id = if (custom) subtitle?.id else null
val payload = JSONObject().apply { put("subtitle", id ?: JSONObject.NULL) }
try {
session?.sendMessage(CAST_MESSAGE_NAMESPACE, payload.toString())
} catch (_: Throwable) {
// best effort; receiver may not support the namespace.
}
}
}
}
// ASS/PGS subtitles are declared as regular cast tracks; the receiver draws
// the ones it can't render natively, keyed off the active track id. Selection
// is uniform (see selectTrack): pick the exact track, which maps to
// activeTrackIds while casting and to native tracks locally.
override fun selectSubtitle(subtitle: Track?) = selectTrack(C.TRACK_TYPE_TEXT, subtitle)
private fun tracksByType(trackType: Int): Array<Track> {
val groups = player.currentTracks.groups.filter { it.type == trackType }
@@ -477,10 +507,6 @@ class OmniPlayer(
companion object {
var notificationPlayer: Player? = null
// Cast custom-message channel (shared with the web receiver) used to
// forward overlay (ASS/PGS) subtitle selection to the receiver.
const val CAST_MESSAGE_NAMESPACE = "urn:x-cast:dev.zoriya.omni"
// MediaItem RequestMetadata extras keys carrying cast-only data.
const val CAST_ID_EXTRA = "dev.zoriya.omni.castId"
const val CAST_DATA_EXTRA = "dev.zoriya.omni.castData"
@@ -528,6 +554,15 @@ class OmniPlayerService : MediaSessionService() {
triggerNotificationUpdate()
}
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
val current = OmniPlayer.notificationPlayer
if (current != null && ::mediaSession.isInitialized && mediaSession.player !== current) {
player = current
mediaSession.player = current
}
return super.onStartCommand(intent, flags, startId)
}
private fun createImmediateNotification(): Notification {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
val channel = NotificationChannel(
@@ -125,6 +125,7 @@ class OmniView(val context: ThemedReactContext) :
autoPip == false && curPip == this -> {
activeView = WeakReference(null)
clearPictureInPictureParams()
}
}
@@ -139,12 +140,12 @@ class OmniView(val context: ThemedReactContext) :
}
boundPlayer?.localPlayer?.removeListener(this)
boundPlayer?.setSurface(null)
boundPlayer?.setVideoView(null)
boundPlayer = omniPlayer
omniPlayer.localPlayer.addListener(this)
if (surfaceReady) {
omniPlayer.setSurface(surfaceView.holder)
omniPlayer.setVideoView(surfaceView)
}
if (autoplay == true && !omniPlayer.isPlaying) {
@@ -156,6 +157,7 @@ class OmniView(val context: ThemedReactContext) :
restoreUiAfterPip()
if (activeView.get() == this) {
activeView = WeakReference(null)
clearPictureInPictureParams()
}
surfaceView.removeOnLayoutChangeListener(this)
context.removeLifecycleEventListener(this)
@@ -164,7 +166,7 @@ class OmniView(val context: ThemedReactContext) :
val omniPlayer = player as? OmniPlayer ?: return
boundPlayer?.localPlayer?.removeListener(this)
omniPlayer.setSurface(null)
omniPlayer.setVideoView(null)
boundPlayer = null
}
@@ -183,8 +185,10 @@ class OmniView(val context: ThemedReactContext) :
return
}
val omniPlayer = boundPlayer ?: return
if (!omniPlayer.isPlaying) {
if (boundPlayer == null ||
boundPlayer?.localPlayer?.isPlaying != true ||
boundPlayer?.isCasting == true
) {
return
}
@@ -218,10 +222,26 @@ class OmniView(val context: ThemedReactContext) :
val autoEnterEnabled =
Build.VERSION.SDK_INT >= Build.VERSION_CODES.S &&
autoPip == true &&
boundPlayer?.isPlaying == true
boundPlayer?.localPlayer?.isPlaying == true &&
boundPlayer?.isCasting != true
activity.setPictureInPictureParams(buildPipParams(autoEnterEnabled))
}
private fun clearPictureInPictureParams() {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) {
return
}
val activity = context.currentActivity ?: return
if (activity.isFinishing || activity.isDestroyed) {
return
}
activity.setPictureInPictureParams(
PictureInPictureParams.Builder()
.setAutoEnterEnabled(false)
.build()
)
}
private fun buildPipParams(autoEnterEnabled: Boolean): PictureInPictureParams {
val aspectRatio = if (surfaceView.width > 0 && surfaceView.height > 0) {
Rational(surfaceView.width, surfaceView.height)
@@ -253,7 +273,7 @@ class OmniView(val context: ThemedReactContext) :
if (context.currentActivity !== activity || autoPip != true || boundPlayer == null) {
return
}
if (!afterEnteredPip && boundPlayer?.isPlaying != true) return
if (!afterEnteredPip && boundPlayer?.localPlayer?.isPlaying != true) return
isolateUiForPip(activity)
}
@@ -305,9 +325,22 @@ class OmniView(val context: ThemedReactContext) :
movedSurfaceToRootForPip = false
}
// A surface that is destroyed and recreated while playback runs (the PIP
// reparent) makes VLC drop its video pipeline; it won't rebuild it for the
// new surface on its own, so we must ask it to. The rebuild only succeeds
// once the (PIP) window has settled at its final size — firing it mid-resize
// leaves the vout stopped — so it is debounced off surfaceChanged.
private var pendingVideoRebuild = false
private val rebuildRunnable = Runnable {
if (!pendingVideoRebuild) return@Runnable
pendingVideoRebuild = false
boundPlayer?.rebuildVideoOutput()
}
override fun surfaceCreated(holder: SurfaceHolder) {
surfaceReady = true
boundPlayer?.setSurface(holder)
boundPlayer?.setVideoView(surfaceView)
if (boundPlayer?.localPlayer?.isPlaying == true) pendingVideoRebuild = true
updatePictureInPictureParams()
}
@@ -316,10 +349,17 @@ class OmniView(val context: ThemedReactContext) :
format: Int,
width: Int,
height: Int
) { }
) {
boundPlayer?.updateVideoLayout(width, height)
if (pendingVideoRebuild) {
surfaceView.removeCallbacks(rebuildRunnable)
surfaceView.postDelayed(rebuildRunnable, 200)
}
}
override fun surfaceDestroyed(holder: SurfaceHolder) {
surfaceReady = false
boundPlayer?.setSurface(null)
surfaceView.removeCallbacks(rebuildRunnable)
boundPlayer?.setVideoView(null)
}
}
@@ -101,6 +101,7 @@ class VlcPlayer(ctx: Context) :
private var playlistMetadata: MediaMetadata = MediaMetadata.EMPTY
private var userInitiatedTransition: Boolean = false
private var cachedBufferedPosition: Long = 0L
private var boundSurfaceView: SurfaceView? = null
private val availableCommands: Player.Commands = Player.Commands.Builder()
.add(COMMAND_PLAY_PAUSE)
@@ -795,6 +796,7 @@ class VlcPlayer(ctx: Context) :
}
override fun clearVideoSurface() {
boundSurfaceView = null
vlcVout.detachViews()
}
@@ -825,10 +827,39 @@ class VlcPlayer(ctx: Context) :
}
override fun setVideoSurfaceView(surfaceView: SurfaceView?) {
vlcVout.setVideoView(surfaceView ?: return clearVideoSurface())
if (surfaceView == null) return clearVideoSurface()
if (vlcVout.areViewsAttached() && boundSurfaceView === surfaceView) return
boundSurfaceView = surfaceView
// Hand VLC the SurfaceView (not a raw Surface) so it reads the real view
// size and keeps the video layout correct as the surface resizes (e.g. the
// PIP window shrinking/growing); see updateVideoLayout for size updates.
vlcVout.setVideoView(surfaceView)
vlcVout.attachViews()
}
/**
* VLC tears down its video decoder + output whenever the output Surface is
* destroyed (e.g. the SurfaceView being reparented for PIP) and does not
* rebuild them when a new Surface arrives — the picture stays black. Toggling
* the video track forces VLC to spin up a fresh decoder/vout against the
* currently-attached Surface. Safe no-op when nothing is attached/playing.
*/
fun rebuildVideoOutput() {
if (!vlcVout.areViewsAttached() || !player.isPlaying) return
player.setVideoTrackEnabled(false)
player.setVideoTrackEnabled(true)
player.updateVideoSurfaces()
}
/** Recompute the video layout for the current surface size. Must run whenever
* the surface resizes (VLC latches a stale geometry otherwise -> the picture
* renders at the wrong size, anchored in a corner). */
fun updateVideoLayout(width: Int, height: Int) {
if (!vlcVout.areViewsAttached()) return
if (width > 0 && height > 0) vlcVout.setWindowSize(width, height)
player.updateVideoSurfaces()
}
override fun clearVideoSurfaceView(surfaceView: SurfaceView?) {
clearVideoSurface()
}
-1
View File
@@ -1 +0,0 @@
module.exports = require("./plugin/build").default;
+6
View File
@@ -0,0 +1,6 @@
export {
default,
withCast,
withMediaNotifications,
withPip,
} from "./plugin/src/index";
+4 -8
View File
@@ -12,9 +12,8 @@
"android": " cd example && bunx react-native run-android --no-packager --port 8082",
"web": "cd example && bun web",
"typecheck": "tsc --noEmit && tsc -p plugin/tsconfig.json --noEmit",
"build:plugin": "tsc -p plugin/tsconfig.json",
"build": "bun run typecheck && bob build && bun run build:plugin",
"prepare": "bun run build",
"build": "bun run typecheck && bob build",
"prepare": "bob build",
"codegen": "nitrogen --logLevel=\"debug\" && node post-script.js",
"release": "semantic-release"
},
@@ -41,8 +40,8 @@
"ios/**/*.mm",
"ios/**/*.cpp",
"ios/**/*.swift",
"app.plugin.js",
"plugin/build",
"app.plugin.ts",
"plugin/src",
"*.podspec",
"README.md"
],
@@ -98,8 +97,5 @@
}
]
]
},
"patchedDependencies": {
"jassub@2.5.7": "patches/jassub@2.5.7.patch"
}
}
-21
View File
@@ -126,27 +126,6 @@ export const useEvent = <Event extends keyof OmniEvents>(
}
return undefined;
}, [event, player]);
// ASS/PGS (overlay) subtitle selection is not a videojs text-track change, so
// re-fire subtitleChange when the overlay subtitle changes too — consumers
// only need useEvent("subtitleChange").
useEffect(() => {
if (event !== "subtitleChange") return;
return player.subscribeOverlaySubtitle(() => {
const sel = player.subtitles.find((s) => s.selected);
const cb = callbackRef.current as (s: unknown) => void;
cb(
sel
? {
id: sel.id,
label: sel.label,
language: sel.language,
selected: true,
}
: undefined,
);
});
}, [event, player]);
};
function createMapper<Key extends keyof OmniPlayerState, State, Result>(
+10 -27
View File
@@ -216,8 +216,11 @@ export class WebOmniPlayer implements OmniPlayer {
}
get subtitles(): Track[] {
// Every subtitle (incl. ass/pgs) is a real text track now; the cast
// receiver draws the ones it can't render natively, keyed off the active
// track. So the text-track list already contains all of them.
const textTracks = selectTextTrack(this._store.state)?.textTrackList ?? [];
const native = textTracks
return textTracks
.filter((x) => x.kind === "subtitles" || x.kind === "captions")
.map((track) => ({
id: track.id!,
@@ -225,39 +228,19 @@ export class WebOmniPlayer implements OmniPlayer {
language: track.language,
selected: track.mode === "showing",
}));
const overlay = this.overlaySubtitles.map((sub) => ({
id: sub.id,
label: sub.label,
language: sub.language,
selected: this.overlaySubtitle?.id === sub.id,
}));
return [...native, ...overlay];
}
selectSubtitle(subtitle?: Track): void {
// Selecting a text track drives local playback and, while casting,
// video.js forwards it to the receiver as an active cast track. ass/pgs
// additionally need our local overlay since the browser can't render them.
const tracks = selectTextTrack(this._store.state);
tracks?.selectSubtitlesTrack(subtitle ? subtitle.id : "off");
const overlay = subtitle
? this.overlaySubtitles.find((s) => s.id === subtitle.id)
: undefined;
const tracks = selectTextTrack(this._store.state);
tracks?.selectSubtitlesTrack(overlay || !subtitle ? "off" : subtitle.id);
this.setOverlaySubtitle(overlay ?? null);
if (this.castStatus === "connected") {
window.cast.framework.CastContext.getInstance()
.getCurrentSession()
?.sendMessage("urn:x-cast:dev.zoriya.omni", {
subtitle: overlay?.id ?? null,
})
.catch(() => {});
}
}
// To be used on a callback of a chromecast session, it only changes the local state
applyRemoteSubtitle(id: string | null): void {
this.setOverlaySubtitle(
id ? (this.overlaySubtitles.find((s) => s.id === id) ?? null) : null,
);
}
private setOverlaySubtitle(sub: Subtitle | null): void {
-30
View File
@@ -75,36 +75,6 @@ const PlayerInitializer = ({
player.showNotification = showNotification;
}, [showNotification]);
useEffect(() => {
let media: chrome.cast.media.Media | null = null;
const onMediaUpdate = () => {
const data = media?.customData as { subtitle?: unknown } | undefined;
player.applyRemoteSubtitle(
typeof data?.subtitle === "string" ? data.subtitle : null,
);
};
const syncMedia = () => {
const next =
window.cast.framework.CastContext.getInstance()
.getCurrentSession()
?.getMediaSession() ?? null;
if (next === media) return;
media?.removeUpdateListener(onMediaUpdate);
media = next;
media?.addUpdateListener(onMediaUpdate);
onMediaUpdate();
};
const unsubscribe = store.subscribe(syncMedia);
syncMedia();
return () => {
unsubscribe();
media?.removeUpdateListener(onMediaUpdate);
};
}, [store]);
return <PlayerCtx.Provider value={player}>{children}</PlayerCtx.Provider>;
};
+11 -17
View File
@@ -10,11 +10,7 @@ import {
useSyncExternalStore,
} from "react";
import { usePlayerState } from "./events";
import {
getSubtitleFormat,
isCustomSubtitle,
type WebOmniPlayer,
} from "./player.web";
import { getSubtitleFormat, type WebOmniPlayer } from "./player.web";
import { usePlayer, VideoPlayer } from "./provider.web";
import type { SubtitleAssets } from "./types/subtitles";
import type { OmniViewProps } from "./types/view";
@@ -165,18 +161,16 @@ export const OmniView = ({
crossOrigin="anonymous"
style={{ width: "100%", height: "100%", objectFit: "contain" }}
>
{(player.source?.subtitles ?? [])
.filter((subtitle) => !isCustomSubtitle(subtitle))
.map((subtitle) => (
<track
key={subtitle.id}
id={subtitle.id}
kind="subtitles"
src={subtitle.link}
srcLang={subtitle.language}
label={subtitle.label ?? subtitle.language ?? subtitle.id}
/>
))}
{(player.source?.subtitles ?? []).map((subtitle) => (
<track
key={subtitle.id}
id={subtitle.id}
kind="subtitles"
src={subtitle.link}
srcLang={subtitle.language}
label={subtitle.label ?? subtitle.language ?? subtitle.id}
/>
))}
</Tech>
)}
{castStatus !== "connected" && castStatus !== "connecting" && (