feat: add generated video transcriptions to dash preview

feat: Refactor dashviewer transcription handling to await

Author: kronosmichall

src/lib/__tests__/Preview-test.js

@@ -2724,7 +2724,7 @@ describe('lib/Preview', () => {
 
         test('should add dash hints if the browser supports dash', () => {
             stubs.canPlayDash.mockReturnValue(true);
-            stubs.headers['X-Rep-Hints'] += '[dash,mp4][filmstrip]';
+            stubs.headers['X-Rep-Hints'] += '[dash,mp4][filmstrip][extracted_text]';
 
             preview.getRequestHeaders();
             expect(stubs.getHeaders).toHaveBeenCalledWith(stubs.headers, 'previewtoken', 'link', 'Passw0rd!');

src/lib/constants.js

@@ -97,7 +97,7 @@ export const STATUS_VIEWABLE = 'viewable';
 export const X_REP_HINT_BASE = '[3d][pdf][text][mp3][json]';
 export const X_REP_HINT_DOC_THUMBNAIL = '[jpg?dimensions=1024x1024&paged=false]';
 export const X_REP_HINT_IMAGE = '[jpg?dimensions=2048x2048,png?dimensions=2048x2048]';
-export const X_REP_HINT_VIDEO_DASH = '[dash,mp4][filmstrip]';
+export const X_REP_HINT_VIDEO_DASH = '[dash,mp4][filmstrip][extracted_text]';
 export const X_REP_HINT_VIDEO_MP4 = '[mp4]';
 
 export const PDFJS_CSS_UNITS = 96.0 / 72.0; // Should match CSS_UNITS in pdf_viewer.js

src/lib/viewers/media/DashViewer.js

@@ -46,6 +46,9 @@ class DashViewer extends VideoBaseViewer {
     /** @property {Object} - Status of the filmstrip representation */
     filmstripStatus;
 
+    /** @property {Object} - Status of the extracted_text (transcription) representation */
+    transcriptionStatus;
+
     /** @property {string} - URL for the filmstrip image */
     filmstripUrl;
 
@@ -164,6 +167,10 @@ class DashViewer extends VideoBaseViewer {
             this.filmstripStatus.destroy();
         }
 
+        if (this.transcriptionStatus) {
+            this.transcriptionStatus.destroy();
+        }
+
         clearInterval(this.statsIntervalId);
         if (this.player) {
             this.player.destroy();
@@ -687,6 +694,17 @@ class DashViewer extends VideoBaseViewer {
         }
     }
 
+    /**
+     * Returns the display-friendly name for a text track's language.
+     * Maps the undetermined language code 'und' to a localized "Auto-Generated" label.
+     *
+     * @param {Object} track - A Shaka text track object
+     * @return {string} Localized language name or the raw language code
+     */
+    getTrackDisplayLanguage(track) {
+        return track.language === 'und' ? __('auto_generated') : getLanguageName(track.language) || track.language;
+    }
+
     /**
      * Loads captions/subtitles into the settings menu
      *
@@ -701,7 +719,7 @@ class DashViewer extends VideoBaseViewer {
                 this.initSubtitles();
             } else {
                 this.mediaControls.initSubtitles(
-                    this.textTracks.map(track => getLanguageName(track.language) || track.language),
+                    this.textTracks.map(track => this.getTrackDisplayLanguage(track)),
                     getLanguageName(this.options.location.locale.substring(0, 2)),
                 );
             }
@@ -741,7 +759,7 @@ class DashViewer extends VideoBaseViewer {
 
         this.textTracks = this.textTracks.map(track => ({
             ...track,
-            displayLanguage: getLanguageName(track.language) || track.language,
+            displayLanguage: this.getTrackDisplayLanguage(track),
         }));
 
         // Do intelligent selection: Prefer user's language, fallback to English, then first subtitle in list
@@ -895,6 +913,7 @@ class DashViewer extends VideoBaseViewer {
         this.startBandwidthTracking();
         this.loadFilmStrip();
         this.loadSubtitles();
+        this.loadTranscription();
         this.loadAlternateAudio();
         this.showPlayButton();
 
@@ -967,6 +986,69 @@ class DashViewer extends VideoBaseViewer {
         }
     }
 
+    /**
+     * Loads the extracted_text transcription (.vtt) as a text track when available
+     *
+     * @private
+     * @return {void}
+     */
+    async loadTranscription() {
+        const extractedText = getRepresentation(this.options.file, 'extracted_text');
+        if (!extractedText?.content?.url_template) {
+            return;
+        }
+
+        const transcriptionUrl = this.createContentUrlWithAuthParams(extractedText.content.url_template);
+        this.transcriptionStatus = this.getRepStatus(extractedText);
+
+        try {
+            await this.transcriptionStatus.getPromise();
+
+            if (this.isDestroyed() || !this.player) {
+                return;
+            }
+
+            await this.player.addTextTrackAsync(
+                transcriptionUrl,
+                'und',
+                'subtitles',
+                'text/vtt',
+                undefined,
+                __('auto_generated'),
+            );
+
+            if (this.isDestroyed()) {
+                return;
+            }
+
+            if (this.textTracks.length > 0) {
+                // Subtitles were already initialized — find and append only
+                // the new track(s) without disturbing the user's selection.
+                const prevTracks = this.textTracks;
+                const prevIds = new Set(prevTracks.map(t => t.id));
+                this.textTracks = this.player.getTextTracks().sort((track1, track2) => track1.id - track2.id);
+
+                if (this.useReactControls()) {
+                    this.textTracks = this.textTracks.map(track => ({
+                        ...track,
+                        displayLanguage: this.getTrackDisplayLanguage(track),
+                    }));
+                    this.renderUI();
+                } else {
+                    this.textTracks.forEach((track, idx) => {
+                        if (!prevIds.has(track.id)) {
+                            this.mediaControls.settings.addSubtitle(__('auto_generated'), idx);
+                        }
+                    });
+                }
+            } else {
+                this.loadSubtitles();
+            }
+        } catch {
+            // Transcription is non-critical; allow the viewer to continue without it
+        }
+    }
+
     /**
      * Calculates the video dimension based on representations
      *

src/lib/viewers/media/Settings.js

@@ -692,6 +692,27 @@ class Settings extends EventEmitter {
         this.reset();
     }
 
+    /**
+     * Appends a single subtitle entry to the settings menu without disturbing
+     * existing items or the user's current selection.
+     *
+     * @param {string} subtitle - Display name for the subtitle track
+     * @param {number} idx - data-value index for the new menu item
+     * @return {void}
+     */
+    addSubtitle(subtitle, idx) {
+        const subtitlesSubMenu = this.settingsEl.querySelector('.bp-media-settings-menu-subtitles');
+        insertTemplate(
+            subtitlesSubMenu,
+            SUBMENU_SUBITEM_TEMPLATE.replace(/{{dataType}}/g, 'subtitles').replace(/{{dataValue}}/g, idx),
+        );
+        const languageNode = subtitlesSubMenu.lastChild.querySelector('.bp-media-settings-value');
+        languageNode.textContent = subtitle;
+
+        this.subtitles.push(subtitle);
+        this.reset();
+    }
+
     /**
      * Takes an ordered list of audio languages and populates the settings menu
      *