feat(subtitles): switch to assjs subtitle renderer (#2500)

Advanced SSA or ASS subs should now be rendered in DOM using assjs. This should greatly improve the performance of subtitle rendering. Still have to "figure out" the "Precise Rendering" option to enable transcoding of subtitles on the server. Co-authored-by: Fernando Fernández <ferferga@hotmail.com> Signed-off-by: Fernando Fernández <ferferga@hotmail.com>
2024-12-04 04:01:26 +00:00 · 2024-11-26 06:07:42 -06:00 · 2024-11-26 06:07:42 -06:00 · b7a2952917
commit b7a2952917
parent 6c46e7e9c0
4 changed files with 103 additions and 151 deletions
--- a/frontend/package.json
+++ b/frontend/package.json
@ -25,6 +25,7 @@
    "@skirtle/vue-vnode-utils": "0.2.0",
    "@vueuse/components": "11.3.0",
    "@vueuse/core": "11.3.0",
+    "assjs": "0.1.3",
    "audiomotion-analyzer": "4.5.0",
    "axios": "1.7.8",
    "blurhash": "2.0.5",
@ -35,7 +36,6 @@
    "dompurify": "3.2.1",
    "fast-equals": "5.0.1",
    "hls.js": "1.5.17",
-    "jassub": "1.7.17",
    "libpgs": "0.8.1",
    "marked": "14.1.4",
    "sortablejs": "1.15.4",
--- a/frontend/src/components/Playback/PlayerElement.vue
+++ b/frontend/src/components/Playback/PlayerElement.vue
@ -4,7 +4,7 @@
      :to="videoContainerRef"
      :disabled="!videoContainerRef"
      defer>
-      <div class="uno-my-auto">
+      <div class="uno-relative">
        <Component
          :is="mediaElementType"
          v-show="mediaElementType === 'video' && videoContainerRef"
@ -14,7 +14,11 @@
          crossorigin
          playsinline
          :loop="playbackManager.isRepeatingOnce"
-          :class="{ 'uno-object-fill': playerElement.isStretched.value, 'uno-max-h-100vh': true}"
+          class="uno-h-full uno-max-h-100vh"
+          :class="{
+            'uno-object-fill': playerElement.isStretched.value,
+            'uno-w-screen': playerElement.isStretched.value
+          }"
          @loadeddata="onLoadedData">
          <track
            v-for="sub in playbackManager.currentItemVttParsedSubtitleTracks"
--- a/frontend/src/store/player-element.ts
+++ b/frontend/src/store/player-element.ts
@ -4,17 +4,15 @@
 * In the other part, playbackManager is suited to handle the playback state in
 * an agnostic way, regardless of where the media is being played (remotely or locally)
 */
-import JASSUB from 'jassub';
-import jassubWorker from 'jassub/dist/jassub-worker.js?url';
-import jassubWasmUrl from 'jassub/dist/jassub-worker.wasm?url';
+import ASSSUB, { type IASSSUB } from 'assjs';
 import { PgsRenderer } from 'libpgs';
 import pgssubWorker from 'libpgs/dist/libpgs.worker.js?url';
 import { computed, nextTick, shallowRef, watch } from 'vue';
 import { SubtitleDeliveryMethod } from '@jellyfin/sdk/lib/generated-client/models/subtitle-delivery-method';
 import { useFullscreen } from '@vueuse/core';
 import { playbackManager, type PlaybackExternalTrack } from './playback-manager';
-import { isArray, isNil, sealed } from '@/utils/validation';
-import { DEFAULT_TYPOGRAPHY, mediaElementRef } from '@/store';
+import { isNil, sealed } from '@/utils/validation';
+import { mediaElementRef } from '@/store';
 import { CommonStore } from '@/store/super/common-store';
 import { router } from '@/plugins/router';
 import { remote } from '@/plugins/remote';
@ -22,6 +20,11 @@ import type { ParsedSubtitleTrack } from '@/plugins/workers/generic/subtitles';
 import { genericWorker } from '@/plugins/workers';
 import { subtitleSettings } from '@/store/client-settings/subtitle-settings';

+/**
+ * TODO: Provide option for the user to select if subtitles should be rendered by the browser or transcoded in server
+ * That option must take into account if transcoding is enabled in server (and user has permission to use it as well)
+ */
+
 interface SubtitleExternalTrack extends PlaybackExternalTrack {
  parsed?: ParsedSubtitleTrack;
 }
@ -46,7 +49,8 @@ class PlayerElementStore extends CommonStore<PlayerElementState> {
   * Reactive state is defined in the super() constructor
   */
  private readonly _fullscreenVideoRoute = '/playback/video';
-  private _jassub: JASSUB | undefined;
+  private readonly _cleanups = new Set<() => void>();
+  private _asssub: IASSSUB | undefined;
  private _pgssub: PgsRenderer | undefined;
  protected _storeKey = 'playerElement';

@ -118,98 +122,62 @@ class PlayerElementStore extends CommonStore<PlayerElementState> {
    }
  };

-  private readonly _setSsaTrack = (trackSrc: string, attachedFonts?: string[]): void => {
-    if (
-      !this._jassub
-      && mediaElementRef.value
-      && mediaElementRef.value instanceof HTMLVideoElement
-    ) {
-      const hasAttachedFonts = !isNil(attachedFonts) && attachedFonts.length !== 0;
+  private readonly _fetchSubtitleTrack = async (trackSrc: string) => {
+    const axios = remote.sdk.api?.axiosInstance;

-      this._jassub = new JASSUB({
-        video: mediaElementRef.value,
-        subUrl: trackSrc,
-        ...(hasAttachedFonts
-          ? {
-              fonts: attachedFonts
-            }
-          : {
-              useLocalFonts: true
-            }),
-        fallbackFont: DEFAULT_TYPOGRAPHY,
-        workerUrl: jassubWorker,
-        wasmUrl: jassubWasmUrl,
-        // Both parameters needed for subs to work on iOS
-        prescaleFactor: 0.8,
-        onDemandRender: false,
-        // OffscreenCanvas doesn't work perfectly on Workers: https://github.com/ThaUnknown/jassub/issues/33
-        offscreenRender: false
+    return {
+      [trackSrc]: (await axios!.get(trackSrc)).data as string
+    };
+  };
+
+  private readonly _setSsaTrack = async (trackSrc: string): Promise<void> => {
+    if (!mediaElementRef.value || !(mediaElementRef.value instanceof HTMLVideoElement)) {
+      return;
+    }
+
+    this._clear();
+
+    const subtitleTrackPayload = await this._fetchSubtitleTrack(trackSrc);
+
+    if (this.currentExternalSubtitleTrack && subtitleTrackPayload[this.currentExternalSubtitleTrack.src]) {
+      /**
+       * video_width works better with ultrawide monitors
+       */
+      this._asssub = new ASSSUB(
+        subtitleTrackPayload[this.currentExternalSubtitleTrack.src],
+        mediaElementRef.value,
+        { resampling: 'video_width' }
+      );
+
+      this._cleanups.add(() => {
+        this._asssub?.destroy();
+        this._asssub = undefined;
      });
-    } else if (this._jassub) {
-      if (isArray(attachedFonts)) {
-        for (const font of attachedFonts) {
-          this._jassub.addFont(font);
-        }
-      }
-
-      this._jassub.setTrackByUrl(trackSrc);
    }
  };

-  private readonly _freeSsaTrack = (): void => {
-    if (this._jassub) {
-      try {
-        this._jassub.destroy();
-      } catch {}
-
-      this._jassub = undefined;
-    }
-  };
-
-  private readonly _isSupportedFont = (mimeType: string | undefined | null): boolean => {
-    return (
-      !isNil(mimeType)
-      && mimeType.startsWith('font/')
-      && (mimeType.includes('ttf')
-        || mimeType.includes('otf')
-        || mimeType.includes('woff'))
-    );
-  };
-
-  private readonly _setPgsTrack = (trackSrc: string): void => {
-    if (
-      !this._pgssub
-      && mediaElementRef.value instanceof HTMLVideoElement
-    ) {
-      this._pgssub = new PgsRenderer({
-        video: mediaElementRef.value,
-        subUrl: trackSrc,
-        workerUrl: pgssubWorker
-      });
-    } else if (this._pgssub) {
-      this._pgssub.loadFromUrl(trackSrc);
-    }
-  };
-
-  private readonly _freePgsTrack = (): void => {
-    if (this._pgssub) {
-      this._pgssub.dispose();
-    }
-
-    this._pgssub = undefined;
-  };
-
-  /**
-   * Applies PGS subtitles to the media element.
-   */
  private readonly _applyPgsSubtitles = (): void => {
-    if (
-      mediaElementRef.value
-      && this.currentExternalSubtitleTrack
-    ) {
-      const subtitleTrack = this.currentExternalSubtitleTrack;
+    const trackSrc = this.currentExternalSubtitleTrack?.src;

-      this._setPgsTrack(subtitleTrack.src);
+    if (trackSrc) {
+      if (
+        !this._pgssub
+        && mediaElementRef.value instanceof HTMLVideoElement
+        && this.currentExternalSubtitleTrack
+      ) {
+        this._pgssub = new PgsRenderer({
+          video: mediaElementRef.value,
+          subUrl: trackSrc,
+          workerUrl: pgssubWorker
+        });
+
+        this._cleanups.add(() => {
+          this._pgssub?.dispose();
+          this._pgssub = undefined;
+        });
+      } else if (this._pgssub) {
+        this._pgssub.loadFromUrl(trackSrc);
+      }
    }
  };

@ -241,49 +209,29 @@ class PlayerElementStore extends CommonStore<PlayerElementState> {
   * Applies SSA (SubStation Alpha) subtitles to the media element.
   */
  private readonly _applySsaSubtitles = async (): Promise<void> => {
-    if (
-      mediaElementRef.value
-      && this.currentExternalSubtitleTrack
-    ) {
-      const subtitleTrack = this.currentExternalSubtitleTrack;
+    if (!this.currentExternalSubtitleTrack || !mediaElementRef) {
+      return;
+    }
+
+    const subtitleTrack = this.currentExternalSubtitleTrack;
+
+    /**
+     * Check if client is able to display custom subtitle track
+     */
+    if (this._useCustomSubtitleTrack) {
+      const data = await genericWorker.parseSsaFile(subtitleTrack.src);

      /**
-       * Check if client is able to display custom subtitle track
-       * otherwise use JASSUB to render subtitles
+       * Check if worker returned that the sub data is 'basic', when true use basic renderer method
       */
-      let applyJASSUB = !this._useCustomSubtitleTrack;
+      if (data?.isBasic) {
+        this.currentExternalSubtitleTrack.parsed = data;

-      if (this._useCustomSubtitleTrack) {
-        const data = await genericWorker.parseSsaFile(subtitleTrack.src);
-
-        /**
-         * If style isn't basic (animations, custom typographics, etc.)
-         * fallback to rendering subtitles with JASSUB
-         */
-        if (data?.isBasic) {
-          this.currentExternalSubtitleTrack.parsed = data;
-        } else {
-          applyJASSUB = true;
-        }
-      }
-
-      if (applyJASSUB) {
-        const serverAddress = remote.sdk.api?.basePath;
-
-        const attachedFonts
-        = playbackManager.currentMediaSource?.MediaAttachments?.filter(a =>
-          this._isSupportedFont(a.MimeType)
-        )
-          .map((a) => {
-            if (a.DeliveryUrl && serverAddress) {
-              return `${serverAddress}${a.DeliveryUrl}`;
-            }
-          })
-          .filter((a): a is string => a !== undefined) ?? [];
-
-        this._setSsaTrack(subtitleTrack.src, attachedFonts);
+        return;
      }
    }
+
+    this._setSsaTrack(subtitleTrack.src);
  };

  /**
@ -307,8 +255,7 @@ class PlayerElementStore extends CommonStore<PlayerElementState> {
      }
    }

-    this._freeSsaTrack();
-    this._freePgsTrack();
+    this._clear();
    this.currentExternalSubtitleTrack = undefined;

    await nextTick();
@ -333,6 +280,17 @@ class PlayerElementStore extends CommonStore<PlayerElementState> {
    }
  };

+  /**
+   * Disposes all the subtitle resources
+   */
+  private readonly _clear = () => {
+    for (const func of this._cleanups) {
+      func();
+    }
+
+    this._cleanups.clear();
+  };
+
  public constructor() {
    super('playerElement', () => ({
      isStretched: false,
@ -360,8 +318,7 @@ class PlayerElementStore extends CommonStore<PlayerElementState> {
     */
    watch(videoContainerRef, () => {
      if (!videoContainerRef.value) {
-        this._freeSsaTrack();
-        this._freePgsTrack();
+        this._clear();
      }
    }, { flush: 'sync' });

--- a/package-lock.json
+++ b/package-lock.json
@ -34,6 +34,7 @@
        "@skirtle/vue-vnode-utils": "0.2.0",
        "@vueuse/components": "11.3.0",
        "@vueuse/core": "11.3.0",
+        "assjs": "0.1.3",
        "audiomotion-analyzer": "4.5.0",
        "axios": "1.7.8",
        "blurhash": "2.0.5",
@ -44,7 +45,6 @@
        "dompurify": "3.2.1",
        "fast-equals": "5.0.1",
        "hls.js": "1.5.17",
-        "jassub": "1.7.17",
        "libpgs": "0.8.1",
        "marked": "14.1.4",
        "sortablejs": "1.15.4",
@ -4056,6 +4056,12 @@
      "dev": true,
      "license": "Python-2.0"
    },
+    "node_modules/assjs": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/assjs/-/assjs-0.1.3.tgz",
+      "integrity": "sha512-EfztH3E42c5r31l9Q0CaWAxYBlER03e6mjMQ/K4GBP5yOfJDv0dWdwhadb4r3Mo65uOiju3Z7i6LJbjN1aqFdw==",
+      "license": "MIT"
+    },
    "node_modules/ast-kit": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/ast-kit/-/ast-kit-1.0.1.tgz",
@ -6503,15 +6509,6 @@
        "@pkgjs/parseargs": "^0.11.0"
      }
    },
-    "node_modules/jassub": {
-      "version": "1.7.17",
-      "resolved": "https://registry.npmjs.org/jassub/-/jassub-1.7.17.tgz",
-      "integrity": "sha512-573efPTIYLh9YaauuSX2mgPrrYedqdeu6KYNNjsJXbLbBsxPijcUHrMK4zpXCeBv955VTUlMX6l0Afa8AXf33A==",
-      "license": "LGPL-2.1-or-later AND (FTL OR GPL-2.0-or-later) AND MIT AND MIT-Modern-Variant AND ISC AND NTP AND Zlib AND BSL-1.0",
-      "dependencies": {
-        "rvfc-polyfill": "^1.0.7"
-      }
-    },
    "node_modules/jiti": {
      "version": "2.4.0",
      "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.4.0.tgz",
@ -8192,12 +8189,6 @@
        "queue-microtask": "^1.2.2"
      }
    },
-    "node_modules/rvfc-polyfill": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/rvfc-polyfill/-/rvfc-polyfill-1.0.7.tgz",
-      "integrity": "sha512-seBl7J1J3/k0LuzW2T9fG6JIOpni5AbU+/87LA+zTYKgTVhsfShmS8K/yOo1eeEjGJHnAdkVAUUM+PEjN9Mpkw==",
-      "license": "GPL-3.0"
-    },
    "node_modules/safer-buffer": {
      "version": "2.1.2",
      "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",