feat(subtitles): switch to assjs subtitle renderer (#2500)

Advanced SSA or ASS subs should now be rendered in DOM using assjs. This should greatly improve the performance of subtitle rendering.
Still have to "figure out" the "Precise Rendering" option to enable transcoding of subtitles on the server.

Co-authored-by: Fernando Fernández <ferferga@hotmail.com>
Signed-off-by: Fernando Fernández <ferferga@hotmail.com>

frontend/package.json

@@ -25,6 +25,7 @@
     "@skirtle/vue-vnode-utils": "0.2.0",
     "@vueuse/components": "11.3.0",
     "@vueuse/core": "11.3.0",
+    "assjs": "0.1.3",
     "audiomotion-analyzer": "4.5.0",
     "axios": "1.7.8",
     "blurhash": "2.0.5",
@@ -35,7 +36,6 @@
     "dompurify": "3.2.1",
     "fast-equals": "5.0.1",
     "hls.js": "1.5.17",
-    "jassub": "1.7.17",
     "libpgs": "0.8.1",
     "marked": "14.1.4",
     "sortablejs": "1.15.4",

frontend/src/components/Playback/PlayerElement.vue

@@ -4,7 +4,7 @@
       :to="videoContainerRef"
       :disabled="!videoContainerRef"
       defer>
-      <div class="uno-my-auto">
+      <div class="uno-relative">
         <Component
           :is="mediaElementType"
           v-show="mediaElementType === 'video' && videoContainerRef"
@@ -14,7 +14,11 @@
           crossorigin
           playsinline
           :loop="playbackManager.isRepeatingOnce"
-          :class="{ 'uno-object-fill': playerElement.isStretched.value, 'uno-max-h-100vh': true}"
+          class="uno-h-full uno-max-h-100vh"
+          :class="{
+            'uno-object-fill': playerElement.isStretched.value,
+            'uno-w-screen': playerElement.isStretched.value
+          }"
           @loadeddata="onLoadedData">
           <track
             v-for="sub in playbackManager.currentItemVttParsedSubtitleTracks"

frontend/src/store/player-element.ts

@@ -4,17 +4,15 @@
  * In the other part, playbackManager is suited to handle the playback state in
  * an agnostic way, regardless of where the media is being played (remotely or locally)
  */
-import JASSUB from 'jassub';
+import ASSSUB, { type IASSSUB } from 'assjs';
-import jassubWorker from 'jassub/dist/jassub-worker.js?url';
-import jassubWasmUrl from 'jassub/dist/jassub-worker.wasm?url';
 import { PgsRenderer } from 'libpgs';
 import pgssubWorker from 'libpgs/dist/libpgs.worker.js?url';
 import { computed, nextTick, shallowRef, watch } from 'vue';
 import { SubtitleDeliveryMethod } from '@jellyfin/sdk/lib/generated-client/models/subtitle-delivery-method';
 import { useFullscreen } from '@vueuse/core';
 import { playbackManager, type PlaybackExternalTrack } from './playback-manager';
-import { isArray, isNil, sealed } from '@/utils/validation';
+import { isNil, sealed } from '@/utils/validation';
-import { DEFAULT_TYPOGRAPHY, mediaElementRef } from '@/store';
+import { mediaElementRef } from '@/store';
 import { CommonStore } from '@/store/super/common-store';
 import { router } from '@/plugins/router';
 import { remote } from '@/plugins/remote';
@@ -22,6 +20,11 @@ import type { ParsedSubtitleTrack } from '@/plugins/workers/generic/subtitles';
 import { genericWorker } from '@/plugins/workers';
 import { subtitleSettings } from '@/store/client-settings/subtitle-settings';
 
+/**
+ * TODO: Provide option for the user to select if subtitles should be rendered by the browser or transcoded in server
+ * That option must take into account if transcoding is enabled in server (and user has permission to use it as well)
+ */
+
 interface SubtitleExternalTrack extends PlaybackExternalTrack {
   parsed?: ParsedSubtitleTrack;
 }
@@ -46,7 +49,8 @@ class PlayerElementStore extends CommonStore<PlayerElementState> {
    * Reactive state is defined in the super() constructor
    */
   private readonly _fullscreenVideoRoute = '/playback/video';
-  private _jassub: JASSUB | undefined;
+  private readonly _cleanups = new Set<() => void>();
+  private _asssub: IASSSUB | undefined;
   private _pgssub: PgsRenderer | undefined;
   protected _storeKey = 'playerElement';
 
@@ -118,98 +122,62 @@ class PlayerElementStore extends CommonStore<PlayerElementState> {
     }
   };
 
-  private readonly _setSsaTrack = (trackSrc: string, attachedFonts?: string[]): void => {
+  private readonly _fetchSubtitleTrack = async (trackSrc: string) => {
-    if (
+    const axios = remote.sdk.api?.axiosInstance;
-      !this._jassub
-      && mediaElementRef.value
-      && mediaElementRef.value instanceof HTMLVideoElement
-    ) {
-      const hasAttachedFonts = !isNil(attachedFonts) && attachedFonts.length !== 0;
 
-      this._jassub = new JASSUB({
+    return {
-        video: mediaElementRef.value,
+      [trackSrc]: (await axios!.get(trackSrc)).data as string
-        subUrl: trackSrc,
+    };
-        ...(hasAttachedFonts
+  };
-          ? {
+
-              fonts: attachedFonts
+  private readonly _setSsaTrack = async (trackSrc: string): Promise<void> => {
-            }
+    if (!mediaElementRef.value || !(mediaElementRef.value instanceof HTMLVideoElement)) {
-          : {
+      return;
-              useLocalFonts: true
+    }
-            }),
+
-        fallbackFont: DEFAULT_TYPOGRAPHY,
+    this._clear();
-        workerUrl: jassubWorker,
+
-        wasmUrl: jassubWasmUrl,
+    const subtitleTrackPayload = await this._fetchSubtitleTrack(trackSrc);
-        // Both parameters needed for subs to work on iOS
+
-        prescaleFactor: 0.8,
+    if (this.currentExternalSubtitleTrack && subtitleTrackPayload[this.currentExternalSubtitleTrack.src]) {
-        onDemandRender: false,
+      /**
-        // OffscreenCanvas doesn't work perfectly on Workers: https://github.com/ThaUnknown/jassub/issues/33
+       * video_width works better with ultrawide monitors
-        offscreenRender: false
+       */
+      this._asssub = new ASSSUB(
+        subtitleTrackPayload[this.currentExternalSubtitleTrack.src],
+        mediaElementRef.value,
+        { resampling: 'video_width' }
+      );
+
+      this._cleanups.add(() => {
+        this._asssub?.destroy();
+        this._asssub = undefined;
       });
-    } else if (this._jassub) {
-      if (isArray(attachedFonts)) {
-        for (const font of attachedFonts) {
-          this._jassub.addFont(font);
-        }
-      }
-
-      this._jassub.setTrackByUrl(trackSrc);
     }
   };
 
-  private readonly _freeSsaTrack = (): void => {
-    if (this._jassub) {
-      try {
-        this._jassub.destroy();
-      } catch {}
-
-      this._jassub = undefined;
-    }
-  };
-
-  private readonly _isSupportedFont = (mimeType: string | undefined | null): boolean => {
-    return (
-      !isNil(mimeType)
-      && mimeType.startsWith('font/')
-      && (mimeType.includes('ttf')
-        || mimeType.includes('otf')
-        || mimeType.includes('woff'))
-    );
-  };
-
-  private readonly _setPgsTrack = (trackSrc: string): void => {
-    if (
-      !this._pgssub
-      && mediaElementRef.value instanceof HTMLVideoElement
-    ) {
-      this._pgssub = new PgsRenderer({
-        video: mediaElementRef.value,
-        subUrl: trackSrc,
-        workerUrl: pgssubWorker
-      });
-    } else if (this._pgssub) {
-      this._pgssub.loadFromUrl(trackSrc);
-    }
-  };
-
-  private readonly _freePgsTrack = (): void => {
-    if (this._pgssub) {
-      this._pgssub.dispose();
-    }
-
-    this._pgssub = undefined;
-  };
-
-  /**
-   * Applies PGS subtitles to the media element.
-   */
   private readonly _applyPgsSubtitles = (): void => {
-    if (
+    const trackSrc = this.currentExternalSubtitleTrack?.src;
-      mediaElementRef.value
-      && this.currentExternalSubtitleTrack
-    ) {
-      const subtitleTrack = this.currentExternalSubtitleTrack;
 
-      this._setPgsTrack(subtitleTrack.src);
+    if (trackSrc) {
+      if (
+        !this._pgssub
+        && mediaElementRef.value instanceof HTMLVideoElement
+        && this.currentExternalSubtitleTrack
+      ) {
+        this._pgssub = new PgsRenderer({
+          video: mediaElementRef.value,
+          subUrl: trackSrc,
+          workerUrl: pgssubWorker
+        });
+
+        this._cleanups.add(() => {
+          this._pgssub?.dispose();
+          this._pgssub = undefined;
+        });
+      } else if (this._pgssub) {
+        this._pgssub.loadFromUrl(trackSrc);
+      }
     }
   };
 
@@ -241,49 +209,29 @@ class PlayerElementStore extends CommonStore<PlayerElementState> {
    * Applies SSA (SubStation Alpha) subtitles to the media element.
    */
   private readonly _applySsaSubtitles = async (): Promise<void> => {
-    if (
+    if (!this.currentExternalSubtitleTrack || !mediaElementRef) {
-      mediaElementRef.value
+      return;
-      && this.currentExternalSubtitleTrack
+    }
-    ) {
+
-      const subtitleTrack = this.currentExternalSubtitleTrack;
+    const subtitleTrack = this.currentExternalSubtitleTrack;
+
+    /**
+     * Check if client is able to display custom subtitle track
+     */
+    if (this._useCustomSubtitleTrack) {
+      const data = await genericWorker.parseSsaFile(subtitleTrack.src);
 
       /**
-       * Check if client is able to display custom subtitle track
+       * Check if worker returned that the sub data is 'basic', when true use basic renderer method
-       * otherwise use JASSUB to render subtitles
        */
-      let applyJASSUB = !this._useCustomSubtitleTrack;
+      if (data?.isBasic) {
+        this.currentExternalSubtitleTrack.parsed = data;
 
-      if (this._useCustomSubtitleTrack) {
+        return;
-        const data = await genericWorker.parseSsaFile(subtitleTrack.src);
-
-        /**
-         * If style isn't basic (animations, custom typographics, etc.)
-         * fallback to rendering subtitles with JASSUB
-         */
-        if (data?.isBasic) {
-          this.currentExternalSubtitleTrack.parsed = data;
-        } else {
-          applyJASSUB = true;
-        }
-      }
-
-      if (applyJASSUB) {
-        const serverAddress = remote.sdk.api?.basePath;
-
-        const attachedFonts
-        = playbackManager.currentMediaSource?.MediaAttachments?.filter(a =>
-          this._isSupportedFont(a.MimeType)
-        )
-          .map((a) => {
-            if (a.DeliveryUrl && serverAddress) {
-              return `${serverAddress}${a.DeliveryUrl}`;
-            }
-          })
-          .filter((a): a is string => a !== undefined) ?? [];
-
-        this._setSsaTrack(subtitleTrack.src, attachedFonts);
       }
     }
+
+    this._setSsaTrack(subtitleTrack.src);
   };
 
   /**
@@ -307,8 +255,7 @@ class PlayerElementStore extends CommonStore<PlayerElementState> {
       }
     }
 
-    this._freeSsaTrack();
+    this._clear();
-    this._freePgsTrack();
     this.currentExternalSubtitleTrack = undefined;
 
     await nextTick();
@@ -333,6 +280,17 @@ class PlayerElementStore extends CommonStore<PlayerElementState> {
     }
   };
 
+  /**
+   * Disposes all the subtitle resources
+   */
+  private readonly _clear = () => {
+    for (const func of this._cleanups) {
+      func();
+    }
+
+    this._cleanups.clear();
+  };
+
   public constructor() {
     super('playerElement', () => ({
       isStretched: false,
@@ -360,8 +318,7 @@ class PlayerElementStore extends CommonStore<PlayerElementState> {
      */
     watch(videoContainerRef, () => {
       if (!videoContainerRef.value) {
-        this._freeSsaTrack();
+        this._clear();
-        this._freePgsTrack();
       }
     }, { flush: 'sync' });
 

package-lock.json

@@ -34,6 +34,7 @@
         "@skirtle/vue-vnode-utils": "0.2.0",
         "@vueuse/components": "11.3.0",
         "@vueuse/core": "11.3.0",
+        "assjs": "0.1.3",
         "audiomotion-analyzer": "4.5.0",
         "axios": "1.7.8",
         "blurhash": "2.0.5",
@@ -44,7 +45,6 @@
         "dompurify": "3.2.1",
         "fast-equals": "5.0.1",
         "hls.js": "1.5.17",
-        "jassub": "1.7.17",
         "libpgs": "0.8.1",
         "marked": "14.1.4",
         "sortablejs": "1.15.4",
@@ -4056,6 +4056,12 @@
       "dev": true,
       "license": "Python-2.0"
     },
+    "node_modules/assjs": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/assjs/-/assjs-0.1.3.tgz",
+      "integrity": "sha512-EfztH3E42c5r31l9Q0CaWAxYBlER03e6mjMQ/K4GBP5yOfJDv0dWdwhadb4r3Mo65uOiju3Z7i6LJbjN1aqFdw==",
+      "license": "MIT"
+    },
     "node_modules/ast-kit": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/ast-kit/-/ast-kit-1.0.1.tgz",
@@ -6503,15 +6509,6 @@
         "@pkgjs/parseargs": "^0.11.0"
       }
     },
-    "node_modules/jassub": {
-      "version": "1.7.17",
-      "resolved": "https://registry.npmjs.org/jassub/-/jassub-1.7.17.tgz",
-      "integrity": "sha512-573efPTIYLh9YaauuSX2mgPrrYedqdeu6KYNNjsJXbLbBsxPijcUHrMK4zpXCeBv955VTUlMX6l0Afa8AXf33A==",
-      "license": "LGPL-2.1-or-later AND (FTL OR GPL-2.0-or-later) AND MIT AND MIT-Modern-Variant AND ISC AND NTP AND Zlib AND BSL-1.0",
-      "dependencies": {
-        "rvfc-polyfill": "^1.0.7"
-      }
-    },
     "node_modules/jiti": {
       "version": "2.4.0",
       "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.4.0.tgz",
@@ -8192,12 +8189,6 @@
         "queue-microtask": "^1.2.2"
       }
     },
-    "node_modules/rvfc-polyfill": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/rvfc-polyfill/-/rvfc-polyfill-1.0.7.tgz",
-      "integrity": "sha512-seBl7J1J3/k0LuzW2T9fG6JIOpni5AbU+/87LA+zTYKgTVhsfShmS8K/yOo1eeEjGJHnAdkVAUUM+PEjN9Mpkw==",
-      "license": "GPL-3.0"
-    },
     "node_modules/safer-buffer": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",