diff --git a/src/pgs.ts b/src/pgs.ts index b00e110..79c1de8 100644 --- a/src/pgs.ts +++ b/src/pgs.ts @@ -13,11 +13,11 @@ import {WindowDefinition} from "./pgs/windowDefinitionSegment"; import {PgsRendererHelper} from "./pgsRendererHelper"; export interface PgsLoadOptions { - /** - * Async pgs streams can return partial updates. When invoked, the `displaySets` and `updateTimestamps` are updated - * to the last available subtitle. There is a minimum threshold of one-second to prevent to many updates. - */ - onProgress?: () => void; + /** + * Async pgs streams can return partial updates. When invoked, the `displaySets` and `updateTimestamps` are updated + * to the last available subtitle. There is a minimum threshold of one-second to prevent to many updates. + */ + onProgress?: () => void; } /** @@ -26,200 +26,227 @@ export interface PgsLoadOptions { */ export class Pgs { - /** - * The currently loaded display sets. - */ - public displaySets: DisplaySet[] = []; + /** + * The currently loaded display sets. + */ + public displaySets: DisplaySet[] = []; - /** - * The PGS timestamps when a display set with the same index is presented. - */ - public updateTimestamps: number[] = []; + /** + * The PGS timestamps when a display set with the same index is presented. + */ + public updateTimestamps: number[] = []; - /** - * Loads the subtitle file from the given url. - * @param url The url to the PGS file. - * @param options Optional loading options. Use `onProgress` as callback for partial update while loading. - */ - public async loadFromUrl(url: string, options?: PgsLoadOptions): Promise { - const response = await fetch(url); - if (!response.ok) { - throw new Error(`HTTP error: ${response.status}`); - } - const stream = response.body?.getReader()!; - const reader = new StreamBinaryReader(stream) - - await this.loadFromReader(reader, options); - } - - /** - * Loads the subtitle file from the given buffer. - * @param buffer The PGS data. - * @param options Optional loading options. Use `onProgress` as callback for partial update while loading. - */ - public async loadFromBuffer(buffer: ArrayBuffer, options?: PgsLoadOptions): Promise { - await this.loadFromReader(new ArrayBinaryReader(new Uint8Array(buffer)), options); - } - - /** - * Loads the subtitle file from the given buffer. - * @param reader The PGS data reader. - * @param options Optional loading options. Use `onProgress` as callback for partial update while loading. - */ - public async loadFromReader(reader: BinaryReader, options?: PgsLoadOptions): Promise { - this.displaySets = []; - this.updateTimestamps = []; - this.cachedSubtitleData = undefined; - - let lastUpdateTime = performance.now(); - - const bigEndianReader = new BigEndianBinaryReader(reader); - while (!reader.eof) { - const displaySet = new DisplaySet(); - await displaySet.read(bigEndianReader, true); - this.displaySets.push(displaySet); - this.updateTimestamps.push(displaySet.presentationTimestamp); - - // For async loading, we support frequent progress updates. Sending one update for every new display set - // would be too much. Instead, we use a one-second threshold. - if (options?.onProgress) { - let now = performance.now(); - if (now > lastUpdateTime + 1000) { - lastUpdateTime = now; - options.onProgress(); + /** + * Loads the subtitle file from the given url. + * @param url The url to the PGS file. + * @param options Optional loading options. Use `onProgress` as callback for partial update while loading. + */ + public async loadFromUrl(url: string, options?: PgsLoadOptions): Promise { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`HTTP error: ${response.status}`); } - } + // The `body` and therefore readable streams are only available since Chrome 105. With this available we can utilize + // partial reading while downloading. As a fallback we wait for the whole file to download before reading. + const stream = response.body?.getReader(); + let reader: BinaryReader; + if (stream) { + reader = new StreamBinaryReader(stream); + } else { + const buffer = await response.arrayBuffer(); + reader = new ArrayBinaryReader(new Uint8Array(buffer)); + } + + await this.loadFromReader(reader, options); } - // Call final update. - if (options?.onProgress) { - options.onProgress(); - } - } - - // Information about the next compiled subtitle data. This is calculated after the current subtitle is rendered. - // So by the time the next subtitle change is requested, this should already be completed. - private cachedSubtitleData?: { index: number, data: SubtitleData | undefined }; - - - /** - * Pre-compiles and caches the subtitle data for the given index. - * This will speed up the next call to `buildSubtitleDataAtIndex` with the same index. - * @param index The index of the display set to cache. - */ - public cacheSubtitleAtIndex(index: number) { - // Pre-calculating the next subtitle, so it is ready whenever the next subtitle change is requested. - const nextSubtitleData = this.getSubtitleAtIndex(index); - this.cachedSubtitleData = { index: index, data: nextSubtitleData }; - } - - /** - * Renders the subtitle at the given timestamp. - * @param time The timestamp in seconds. - */ - public getSubtitleAtTimestamp(time: number): SubtitleData | undefined { - const index = PgsRendererHelper.getIndexFromTimestamps(time, this.updateTimestamps); - return this.getSubtitleAtIndex(index); - } - - /** - * Pre-compiles the required subtitle data (windows and pixel data) for the frame at the given index. - * @param index The index of the display set to render. - */ - public getSubtitleAtIndex(index: number): SubtitleData | undefined { - // Check if this index was already cached. - if (this.cachedSubtitleData && this.cachedSubtitleData.index === index) { - return this.cachedSubtitleData.data; + /** + * Loads the subtitle file from the given buffer. + * @param buffer The PGS data. + * @param options Optional loading options. Use `onProgress` as callback for partial update while loading. + */ + public async loadFromBuffer(buffer: ArrayBuffer, options?: PgsLoadOptions): Promise { + await this.loadFromReader(new ArrayBinaryReader(new Uint8Array(buffer)), options); } - if (index < 0 || index >= this.displaySets.length) { - return; + /** + * Loads the subtitle file from the given buffer. + * @param reader The PGS data reader. + * @param options Optional loading options. Use `onProgress` as callback for partial update while loading. + */ + public async loadFromReader(reader: BinaryReader, options?: PgsLoadOptions): Promise { + this.displaySets = []; + this.updateTimestamps = []; + this.cachedSubtitleData = undefined; + + let lastUpdateTime = performance.now(); + + const bigEndianReader = new BigEndianBinaryReader(reader); + while (!reader.eof) { + const displaySet = new DisplaySet(); + await displaySet.read(bigEndianReader, true); + this.displaySets.push(displaySet); + this.updateTimestamps.push(displaySet.presentationTimestamp); + + // For async loading, we support frequent progress updates. Sending one update for every new display set + // would be too much. Instead, we use a one-second threshold. + if (options?.onProgress) { + let now = performance.now(); + if (now > lastUpdateTime + 1000) { + lastUpdateTime = now; + options.onProgress(); + } + } + } + + // Call final update. + if (options?.onProgress) { + options.onProgress(); + } } - const displaySet = this.displaySets[index]; - if (!displaySet.presentationComposition) return; + // Information about the next compiled subtitle data. This is calculated after the current subtitle is rendered. + // So by the time the next subtitle change is requested, this should already be completed. + private cachedSubtitleData?: { index: number, data: SubtitleData | undefined }; - // We need to collect all valid objects and palettes up to this point. PGS can update and reuse elements from - // previous display sets. The `compositionState` defines if the previous elements should be cleared. - // If it is `0` previous elements should be kept. Because the user can seek through the file, we can not store - // previous elements, and we should collect these elements for every new render. - const ctxObjects: ObjectDefinitionSegment[] = []; - const ctxPalettes: PaletteDefinitionSegment[] = []; - const ctxWindows: WindowDefinition[] = []; - let curIndex = index; - while (curIndex >= 0) { - // Because we are moving backwards, we would end up with the inverted array order. - // We'll use `unshift` to add these elements to the front of the array. - ctxObjects.unshift(...this.displaySets[curIndex].objectDefinitions); - ctxPalettes.unshift(...this.displaySets[curIndex].paletteDefinitions); - ctxWindows.unshift(...this.displaySets[curIndex].windowDefinitions - .flatMap(w => w.windows)); - // Any other state that `0` frees all previous segments, so we can stop here. - if (this.displaySets[curIndex].presentationComposition?.compositionState !== 0) { - break; - } - - curIndex--; + /** + * Pre-compiles and caches the subtitle data for the given index. + * This will speed up the next call to `buildSubtitleDataAtIndex` with the same index. + * @param index The index of the display set to cache. + */ + public cacheSubtitleAtIndex(index: number) { + // Pre-calculating the next subtitle, so it is ready whenever the next subtitle change is requested. + const nextSubtitleData = this.getSubtitleAtIndex(index); + this.cachedSubtitleData = { index: index, data: nextSubtitleData }; } - // Find the used palette for this composition. - let palette = ctxPalettes - .find(w => w.id === displaySet.presentationComposition?.paletteId); - if (!palette) return; - - const compositionData: SubtitleCompositionData[] = []; - for (const compositionObject of displaySet.presentationComposition.compositionObjects) { - // Find the window to draw on. - let window = ctxWindows.find(w => w.id === compositionObject.windowId); - if (!window) continue; - - // Builds the subtitle. - const pixelData = this.getPixelDataFromComposition(compositionObject, palette, ctxObjects); - if (pixelData) { - compositionData.push(new SubtitleCompositionData(window, pixelData)); - } + /** + * Renders the subtitle at the given timestamp. + * @param time The timestamp in seconds. + */ + public getSubtitleAtTimestamp(time: number): SubtitleData | undefined { + const index = PgsRendererHelper.getIndexFromTimestamps(time, this.updateTimestamps); + return this.getSubtitleAtIndex(index); } - if (compositionData.length === 0) return; + /** + * Pre-compiles the required subtitle data (windows and pixel data) for the frame at the given index. + * @param index The index of the display set to render. + */ + public getSubtitleAtIndex(index: number): SubtitleData | undefined { + // Check if this index was already cached. + if (this.cachedSubtitleData && this.cachedSubtitleData.index === index) { + return this.cachedSubtitleData.data; + } - return new SubtitleData(displaySet.presentationComposition.width, displaySet.presentationComposition.height, - compositionData); - } + if (index < 0 || index >= this.displaySets.length) { + return; + } + const displaySet = this.displaySets[index]; + if (!displaySet.presentationComposition) return; - private getPixelDataFromComposition(composition: CompositionObject, palette: PaletteDefinitionSegment, - ctxObjects: ObjectDefinitionSegment[]): ImageData | undefined { - // Multiple object definition can define a single subtitle image. - // However, only the first element in sequence hold the image size. - let width: number = 0; - let height: number = 0; - const dataChunks: Uint8Array[] = []; - for (const ods of ctxObjects) { - if (ods.id != composition.id) continue; - if (ods.isFirstInSequence) { - width = ods.width; - height = ods.height; - } + // We need to collect all valid objects and palettes up to this point. PGS can update and reuse elements from + // previous display sets. The `compositionState` defines if the previous elements should be cleared. + // If it is `0` previous elements should be kept. Because the user can seek through the file, we can not store + // previous elements, and we should collect these elements for every new render. + const ctxObjects: ObjectDefinitionSegment[] = []; + const ctxPalettes: PaletteDefinitionSegment[] = []; + const ctxWindows: WindowDefinition[] = []; + let curIndex = index; + while (curIndex >= 0) { + const displaySet = this.displaySets[curIndex]; + // Because we are moving backwards, we would end up with the inverted array order. + // We'll use `unshift` to add these elements to the front of the array. + ctxObjects.unshift(...displaySet.objectDefinitions); + ctxPalettes.unshift(...displaySet.paletteDefinitions); + // `flatMap` is available since Chrome 69. + for (const windowDefinition of displaySet.windowDefinitions) { + ctxWindows.unshift(...windowDefinition.windows); + } - if (ods.data) { - dataChunks.push(ods.data); - } - } - if (dataChunks.length == 0) { - return undefined; + // Any other state that `0` frees all previous segments, so we can stop here. + if (this.displaySets[curIndex].presentationComposition?.compositionState !== 0) { + break; + } + + curIndex--; + } + + // Find the used palette for this composition. + let palette = ctxPalettes + .find(w => w.id === displaySet.presentationComposition?.paletteId); + if (!palette) return; + + const compositionData: SubtitleCompositionData[] = []; + for (const compositionObject of displaySet.presentationComposition.compositionObjects) { + // Find the window to draw on. + let window = ctxWindows.find(w => w.id === compositionObject.windowId); + if (!window) continue; + + // Builds the subtitle. + const pixelData = this.getPixelDataFromComposition(compositionObject, palette, ctxObjects); + if (pixelData) { + compositionData.push(new SubtitleCompositionData(window, pixelData)); + } + } + + if (compositionData.length === 0) return; + + return new SubtitleData(displaySet.presentationComposition.width, displaySet.presentationComposition.height, + compositionData); } - // Using a combined reader instead of stitching the data together. - // This hopefully avoids a larger memory allocation. - const data = new CombinedBinaryReader(dataChunks); - // Building a canvas element with the subtitle image data. - const imageBuffer = new Uint32Array(width * height); + private getPixelDataFromComposition(composition: CompositionObject, palette: PaletteDefinitionSegment, + ctxObjects: ObjectDefinitionSegment[]): ImageData | undefined { + // Multiple object definition can define a single subtitle image. + // However, only the first element in sequence hold the image size. + let width: number = 0; + let height: number = 0; + const dataChunks: Uint8Array[] = []; + for (const ods of ctxObjects) { + if (ods.id != composition.id) continue; + if (ods.isFirstInSequence) { + width = ods.width; + height = ods.height; + } - // The pixel data is run-length encoded. The decoded value is the palette entry index. - RunLengthEncoding.decode(data, palette.rgba, imageBuffer); + if (ods.data) { + dataChunks.push(ods.data); + } + } + if (dataChunks.length == 0) { + return undefined; + } + + // Using a combined reader instead of stitching the data together. + // This hopefully avoids a larger memory allocation. + const data = new CombinedBinaryReader(dataChunks); + + // Detect if we are running in a web-worker or in main browser + if (typeof document !== 'undefined') { + // We have to use the canvas api to create image data. + const canvas = document.createElement("canvas"); + const context = canvas.getContext("2d")!; + const imageData = context.createImageData(width, height); + const imageBuffer = new Uint32Array(imageData.data.buffer); + + // The pixel data is run-length encoded. The decoded value is the palette entry index. + RunLengthEncoding.decode(data, palette.rgba, imageBuffer); + + return imageData; + } else { + // Building a canvas element with the subtitle image data. + const imageBuffer = new Uint32Array(width * height); + + // The pixel data is run-length encoded. The decoded value is the palette entry index. + RunLengthEncoding.decode(data, palette.rgba, imageBuffer); + + // We can use the image data constructor in web-workers. + return new ImageData(new Uint8ClampedArray(imageBuffer.buffer), width, height); + } + } - return new ImageData(new Uint8ClampedArray(imageBuffer.buffer), width, height); - } } diff --git a/src/pgsRenderer.ts b/src/pgsRenderer.ts index 0dc48c2..7f8e7f3 100644 --- a/src/pgsRenderer.ts +++ b/src/pgsRenderer.ts @@ -1,7 +1,7 @@ import {PgsRendererOptions} from "./pgsRendererOptions"; import {PgsRendererImpl} from "./pgsRendererImpl"; import {PgsRendererInWorkerWithOffscreenCanvas} from "./pgsRendererInWorkerWithOffscreenCanvas"; -import {PgsRendererInWorkerWithoutOffscreenCanvas} from "./pgsRendererInWorkerWithoutOffscreenCanvas"; +import {PgsRendererInMainThread} from "./pgsRendererInMainThread"; /** * Renders PGS subtitle on-top of a video element using a canvas element. This also handles timestamp updates if a @@ -31,21 +31,10 @@ export class PgsRenderer { throw new Error('No canvas or video element was provided!'); } - // Jellyfin still supports webOS 5 aka Chrome 69. There `transferControlToOffscreen` is not available. - // In that case we will render the canvas on the main thread. - // If required we could add a non-worker implementation in the future. Also, we could add an option to force a - // certain implementation when needed. - const isOffscreenCanvasSupported = this.isOffscreenCanvasSupported(); - console.log(`isOffscreenCanvasSupported: ${isOffscreenCanvasSupported}`); - if (isOffscreenCanvasSupported) { - this.implementation = new PgsRendererInWorkerWithOffscreenCanvas(options, this.canvas); - } else { - this.implementation = new PgsRendererInWorkerWithoutOffscreenCanvas(options, this.canvas); - } - - // Re-render the current subtitle if the timestamps were updates (e.g. through partial load). + this.implementation = this.createPgsRendererFormPlatform(options); this.implementation.onTimestampsUpdated = () => { + // Re-render the current subtitle if the timestamps were updates (e.g. through partial load). this.renderAtVideoTimestamp(); } @@ -58,6 +47,13 @@ export class PgsRenderer { this.registerVideoEvents(); } + /** + * Checks if the web worker is supported in the current environment. + */ + private isWorkerSupported(): boolean { + return !!window.Worker; + } + /** * Checks if the offscreen-canvas and `transferControlToOffscreen` are supported in the current environment. */ @@ -65,6 +61,28 @@ export class PgsRenderer { return !!HTMLCanvasElement.prototype.transferControlToOffscreen; } + /** + * Performs a platform check and returns the optimal implementation for subtitle rendering. + * @param options The PGS renderer options. + */ + private createPgsRendererFormPlatform(options: PgsRendererOptions): PgsRendererImpl { + // Jellyfin still supports webOS 1.2 and Tizen 2.3 with real old WebKit and Chromium version. + // We can run a different fallback implementation if the offscreen canvas is not supported. + + const isWorkerSupported = this.isWorkerSupported(); + const isOffscreenCanvasSupported = this.isOffscreenCanvasSupported(); + console.log(`isWebWorkerSupported: ${isWorkerSupported}, isOffscreenCanvasSupported: ${isOffscreenCanvasSupported}`); + + // FIXME: I would like to use `PgsRendererInWorkerWithoutOffscreenCanvas` if the offscreen canvas is not + // available, but even the web worker thread won't start on webOS and I can't figure out why. + if (isWorkerSupported && isOffscreenCanvasSupported) { + return new PgsRendererInWorkerWithOffscreenCanvas(options, this.canvas); + } else { + + return new PgsRendererInMainThread(options, this.canvas); + } + } + private implementation: PgsRendererImpl; /** diff --git a/src/pgsRendererImpl.ts b/src/pgsRendererImpl.ts index 53dabba..3cddda0 100644 --- a/src/pgsRendererImpl.ts +++ b/src/pgsRendererImpl.ts @@ -7,7 +7,7 @@ import {PgsRendererHelper} from "./pgsRendererHelper"; */ export abstract class PgsRendererImpl { - protected updateTimestamps: number[] = []; + private updateTimestamps: number[] = []; private previousTimestampIndex: number = 0; /** @@ -15,6 +15,21 @@ export abstract class PgsRendererImpl { */ public onTimestampsUpdated?: () => void; + /** + * Sets the update timestamps and invokes an update event. + * @param updateTimestamps The new array of update timestamps. + */ + protected setUpdateTimestamps(updateTimestamps: number[]): void { + // Stores the update timestamps, so we don't need to push the timestamp to the worker on every tick. + // Instead, we push the timestamp index if it was changed. + this.updateTimestamps = updateTimestamps; + + // Notify timestamp updates. + if (this.onTimestampsUpdated) { + this.onTimestampsUpdated(); + } + } + /** * Renders the subtitle for the given timestamp. * @param time The timestamp in seconds. diff --git a/src/pgsRendererInMainThread.ts b/src/pgsRendererInMainThread.ts new file mode 100644 index 0000000..30f72f4 --- /dev/null +++ b/src/pgsRendererInMainThread.ts @@ -0,0 +1,67 @@ +import {PgsRendererImpl} from "./pgsRendererImpl"; +import {PgsRendererOptions} from "./pgsRendererOptions"; +import {Renderer} from "./renderer"; +import {Pgs} from "./pgs"; + +/** + * The implementation without web workers. This loads and renders the subtitle in the main thread. + * This is meant as a compatibility fallback if advanced workers are not supported. + */ +export class PgsRendererInMainThread extends PgsRendererImpl { + + public constructor(options: PgsRendererOptions, canvas: HTMLCanvasElement) { + super(); + + this.pgs = new Pgs(); + this.renderer = new Renderer(canvas); + } + + /** + * The PGS loader. + * @private + */ + private readonly pgs: Pgs; + + /** + * The subtitle renderer, running in the main thread. + */ + private readonly renderer: Renderer; + + protected render(index: number): void { + const subtitleData = this.pgs.getSubtitleAtIndex(index); + requestAnimationFrame(() => { + this.renderer.draw(subtitleData); + }); + this.pgs.cacheSubtitleAtIndex(index + 1); + } + + public loadFromUrl(url: string): void { + this.pgs.loadFromUrl(url, { + onProgress: () => { + this.invokeTimestampsUpdate(); + } + }).then(() => { + this.invokeTimestampsUpdate(); + }); + } + + public loadFromBuffer(buffer: ArrayBuffer): void { + this.pgs.loadFromBuffer(buffer).then(() => { + this.invokeTimestampsUpdate(); + }); + } + + /** + * Submits the update timestamps from the pgs loader and invokes events. + */ + private invokeTimestampsUpdate(): void { + this.setUpdateTimestamps(this.pgs.updateTimestamps); + } + + /** + * Disposes the renderer. + */ + public dispose(): void { + // Nothing to do + } +} diff --git a/src/pgsRendererInWorker.ts b/src/pgsRendererInWorker.ts index 681ef95..5ebf7fb 100644 --- a/src/pgsRendererInWorker.ts +++ b/src/pgsRendererInWorker.ts @@ -15,10 +15,6 @@ export abstract class PgsRendererInWorker extends PgsRendererImpl { this.worker.onmessage = this.$onWorkerMessage; } - /** - * Tells the worker to load the subtitle file from the given url. - * @param url The url to the PGS file. - */ public loadFromUrl(url: string): void { this.worker.postMessage({ op: 'loadFromUrl', @@ -54,14 +50,7 @@ export abstract class PgsRendererInWorker extends PgsRendererImpl { switch (e.data.op) { // Is called once a subtitle file was loaded. case 'updateTimestamps': { - // Stores the update timestamps, so we don't need to push the timestamp to the worker on every tick. - // Instead, we push the timestamp index if it was changed. - this.updateTimestamps = e.data.updateTimestamps; - - // Notify timestamp updates. - if (this.onTimestampsUpdated) { - this.onTimestampsUpdated(); - } + this.setUpdateTimestamps(e.data.updateTimestamps); break; } } diff --git a/src/pgsRendererInWorkerWithOffscreenCanvas.ts b/src/pgsRendererInWorkerWithOffscreenCanvas.ts index 0e8c192..06c4d53 100644 --- a/src/pgsRendererInWorkerWithOffscreenCanvas.ts +++ b/src/pgsRendererInWorkerWithOffscreenCanvas.ts @@ -7,23 +7,23 @@ import {PgsRendererInWorker} from "./pgsRendererInWorker"; */ export class PgsRendererInWorkerWithOffscreenCanvas extends PgsRendererInWorker { - public constructor(options: PgsRendererOptions, canvas: HTMLCanvasElement) { - super(options); + public constructor(options: PgsRendererOptions, canvas: HTMLCanvasElement) { + super(options); - // Initialize the worker with an offscreen-canvas. Rendering will occur in the worker thread. - const offscreenCanvas = canvas.transferControlToOffscreen(); - this.worker.postMessage({ - op: 'init', - canvas: offscreenCanvas, - }, [offscreenCanvas]); - } + // Initialize the worker with an offscreen-canvas. Rendering will occur in the worker thread. + const offscreenCanvas = canvas.transferControlToOffscreen(); + this.worker.postMessage({ + op: 'init', + canvas: offscreenCanvas, + }, [offscreenCanvas]); + } - protected render(index: number): void { - // Tells the worker to render the subtitle at this timestamp index. - this.worker.postMessage({ - op: 'render', - index: index - }); - } + protected render(index: number): void { + // Tells the worker to render the subtitle at this timestamp index. + this.worker.postMessage({ + op: 'render', + index: index + }); + } } diff --git a/src/worker.ts b/src/worker.ts index 8d48be7..9320938 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -20,7 +20,7 @@ onmessage = (e: MessageEvent) => { case 'init': { const canvas: OffscreenCanvas = e.data.canvas; - // The canvas is optional. If + // The canvas is optional. If provided, the web-worker can use it to render the subtitles. if (canvas) { renderer = new Renderer(canvas); } @@ -70,6 +70,7 @@ onmessage = (e: MessageEvent) => { // Returns the data to the main thread. postMessage({ op: 'subtitleData', + index: index, subtitleData: subtitleData })