API enhancements for audio and subtitle selection

Resolves #5532
video-dev · Nov 16, 2023 · a22b718 · a22b718
1 parent a99be72
commit a22b718
Show file tree

Hide file tree

Showing 18 changed files with 929 additions and 175 deletions.
diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md
@@ -13,7 +13,13 @@ export interface AbrComponentAPI extends ComponentAPI {
     // (undocumented)
     readonly bwEstimator?: EwmaBandWidthEstimator;
     // (undocumented)
+    firstAutoLevel: number;
+    // (undocumented)
+    forcedAutoLevel: number;
+    // (undocumented)
     nextAutoLevel: number;
+    // (undocumented)
+    resetEstimator(abrEwmaDefaultEstimate: number): any;
 }
 
 // Warning: (ae-missing-release-tag) "AbrController" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -112,6 +118,19 @@ export class AttrList {
 // @public (undocumented)
 export type AudioPlaylistType = 'AUDIO';
 
+// Warning: (ae-missing-release-tag) "AudioSelectionOption" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export type AudioSelectionOption = {
+    lang?: string;
+    assocLang?: string;
+    characteristics?: string;
+    channels?: string;
+    name?: string;
+    audioCodec?: string;
+    groupId?: string;
+};
+
 // Warning: (ae-missing-release-tag) "AudioStreamController" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -189,6 +208,8 @@ export class AudioTrackController extends BasePlaylistController {
     protected onManifestLoading(): void;
     // (undocumented)
     protected onManifestParsed(event: Events.MANIFEST_PARSED, data: ManifestParsedData): void;
+    // (undocumented)
+    setAudioOption(audioOption: MediaPlaylist | AudioSelectionOption | undefined): MediaPlaylist | null;
 }
 
 // Warning: (ae-missing-release-tag) "AudioTrackLoadedData" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -1627,6 +1648,8 @@ class Hls implements HlsEventEmitter {
     // (undocumented)
     removeLevel(levelIndex: number): void;
     resumeBuffering(): void;
+    setAudioOption(audioOption: MediaPlaylist | AudioSelectionOption | undefined): MediaPlaylist | null;
+    setSubtitleOption(subtitleOption: MediaPlaylist | SubtitleSelectionOption | undefined): MediaPlaylist | null;
     get startLevel(): number;
     // Warning: (ae-setter-with-docs) The doc comment for the property "startLevel" must appear on the getter, not the setter.
     set startLevel(newLevel: number);
@@ -1694,7 +1717,7 @@ export type HlsConfig = {
     fpsController: typeof FPSController;
     progressive: boolean;
     lowLatencyMode: boolean;
-} & ABRControllerConfig & BufferControllerConfig & CapLevelControllerConfig & EMEControllerConfig & FPSControllerConfig & LevelControllerConfig & MP4RemuxerConfig & StreamControllerConfig & LatencyControllerConfig & MetadataControllerConfig & TimelineControllerConfig & TSDemuxerConfig & HlsLoadPolicies & FragmentLoaderConfig & PlaylistLoaderConfig;
+} & ABRControllerConfig & BufferControllerConfig & CapLevelControllerConfig & EMEControllerConfig & FPSControllerConfig & LevelControllerConfig & MP4RemuxerConfig & StreamControllerConfig & SelectionPreferences & LatencyControllerConfig & MetadataControllerConfig & TimelineControllerConfig & TSDemuxerConfig & HlsLoadPolicies & FragmentLoaderConfig & PlaylistLoaderConfig;
 
 // Warning: (ae-missing-release-tag) "HlsEventEmitter" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
@@ -2052,7 +2075,7 @@ export class Level {
     // (undocumented)
     get maxBitrate(): number;
     // (undocumented)
-    readonly name: string | undefined;
+    readonly name: string;
     // (undocumented)
     get pathwayId(): string;
     // (undocumented)
@@ -2135,7 +2158,7 @@ export type LevelControllerConfig = {
 //
 // @public
 export class LevelDetails {
-    constructor(baseUrl: any);
+    constructor(baseUrl: string);
     // (undocumented)
     advanced: boolean;
     // (undocumented)
@@ -2791,6 +2814,8 @@ export interface MediaKeySessionContext {
 //
 // @public (undocumented)
 export interface MediaPlaylist {
+    // (undocumented)
+    assocLang?: string;
     // (undocumented)
     attrs: MediaAttributes;
     // (undocumented)
@@ -3058,6 +3083,14 @@ export type RetryConfig = {
     shouldRetry?: (retryConfig: RetryConfig | null | undefined, retryCount: number, isTimeout: boolean, loaderResponse: LoaderResponse | undefined, retry: boolean) => boolean;
 };
 
+// Warning: (ae-missing-release-tag) "SelectionPreferences" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export type SelectionPreferences = {
+    audioPreference?: AudioSelectionOption;
+    subtitlePreference?: SubtitleSelectionOption;
+};
+
 // Warning: (ae-missing-release-tag) "SourceBufferName" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -3121,6 +3154,17 @@ export interface SubtitleFragProcessedData {
 // @public (undocumented)
 export type SubtitlePlaylistType = 'SUBTITLES' | 'CLOSED-CAPTIONS';
 
+// Warning: (ae-missing-release-tag) "SubtitleSelectionOption" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export type SubtitleSelectionOption = {
+    lang?: string;
+    assocLang?: string;
+    characteristics?: string;
+    name?: string;
+    groupId?: string;
+};
+
 // Warning: (ae-missing-release-tag) "SubtitleStreamController" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -3194,6 +3238,8 @@ export class SubtitleTrackController extends BasePlaylistController {
     // (undocumented)
     protected onSubtitleTrackLoaded(event: Events.SUBTITLE_TRACK_LOADED, data: TrackLoadedData): void;
     // (undocumented)
+    setSubtitleOption(subtitleOption: MediaPlaylist | SubtitleSelectionOption | undefined): MediaPlaylist | null;
+    // (undocumented)
     get subtitleDisplay(): boolean;
     set subtitleDisplay(value: boolean);
     get subtitleTrack(): number;

diff --git a/docs/API.md b/docs/API.md
@@ -79,6 +79,8 @@ See [API Reference](https://hlsjs-dev.video-dev.org/api-docs/) for a complete li
   - [`pLoader`](#ploader)
   - [`xhrSetup`](#xhrsetup)
   - [`fetchSetup`](#fetchsetup)
+  - [`audioPreference`](#audiopreference)
+  - [`subtitlePreference`](#subtitlepreference)
   - [`abrController`](#abrcontroller)
   - [`bufferController`](#buffercontroller)
   - [`capLevelController`](#caplevelcontroller)
@@ -146,9 +148,13 @@ See [API Reference](https://hlsjs-dev.video-dev.org/api-docs/) for a complete li
   - [`hls.startLoad(startPosition=-1)`](#hlsstartloadstartposition-1)
   - [`hls.stopLoad()`](#hlsstopload)
 - [Audio Tracks Control API](#audio-tracks-control-api)
+  - [`hls.setAudioOption(audioOption)`](#hlssetaudiooptionaudiooption)
+  - [`hls.allAudioTracks`](#hlsallaudiotracks)
   - [`hls.audioTracks`](#hlsaudiotracks)
   - [`hls.audioTrack`](#hlsaudiotrack)
 - [Subtitle Tracks Control API](#subtitle-tracks-control-api)
+  - [`hls.setSubtitleOption(subtitleOption)`](#hlssetsubtitleoptionsubtitleoption)
+  - [`hls.allSubtitleTracks`](#hlsallsubtitletracks)
   - [`hls.subtitleTracks`](#hlssubtitletracks)
   - [`hls.subtitleTrack`](#hlssubtitletrack)
   - [`hls.subtitleDisplay`](#hlssubtitledisplay)
@@ -379,19 +385,30 @@ var config = {
   preferManagedMediaSource: false,
   enableWorker: true,
   enableSoftwareAES: true,
-  manifestLoadingTimeOut: 10000,
-  manifestLoadingMaxRetry: 1,
-  manifestLoadingRetryDelay: 1000,
-  manifestLoadingMaxRetryTimeout: 64000,
+  fragLoadPolicy: {
+    default: {
+      maxTimeToFirstByteMs: 9000,
+      maxLoadTimeMs: 100000,
+      timeoutRetry: {
+        maxNumRetry: 2,
+        retryDelayMs: 0,
+        maxRetryDelayMs: 0,
+      },
+      errorRetry: {
+        maxNumRetry: 5,
+        retryDelayMs: 3000,
+        maxRetryDelayMs: 15000,
+        backoff: 'linear',
+      },
+    },
+  },
   startLevel: undefined,
-  levelLoadingTimeOut: 10000,
-  levelLoadingMaxRetry: 4,
-  levelLoadingRetryDelay: 1000,
-  levelLoadingMaxRetryTimeout: 64000,
-  fragLoadingTimeOut: 20000,
-  fragLoadingMaxRetry: 6,
-  fragLoadingRetryDelay: 1000,
-  fragLoadingMaxRetryTimeout: 64000,
+  audioPreference: {
+    characteristics: 'public.accessibility.describes-video',
+  },
+  subtitlePreference: {
+    lang: 'en-US',
+  },
   startFragPrefetch: false,
   testBandwidth: true,
   progressive: false,
@@ -435,7 +452,11 @@ var config = {
   drmSystems: {},
   drmSystemOptions: {},
   requestMediaKeySystemAccessFunc: requestMediaKeySystemAccess,
-  cmcd: undefined,
+  cmcd: {
+    sessionId: uuid(),
+    contentId: hash(contentURL),
+    useHeaders: false,
+  },
 };
 
 var hls = new Hls(config);
@@ -1124,6 +1145,18 @@ var config = {
 };
 ```
 
+### `audioPreference`
+
+(default: `undefined`)
+
+Set a preference used to find and select the best matching audio track on start. The selection can influence starting level selection based on the audio group(s) available to match the preference. `audioPreference` accepts a value of an audio track object (MediaPlaylist), AudioSelectionOption (track fields to match), or undefined. If not set or set to a value of `undefined`, HLS.js will auto select a default track on start.
+
+### `subtitlePreference`
+
+(default: `undefined`)
+
+Set a preference used to find and select the best matching subtitle track on start. `subtitlePreference` accepts a value of a subtitle track object (MediaPlaylist), SubtitleSelectionOption (track fields to match), or undefined. If not set or set to a value of `undefined`, HLS.js will not enable subtitles unless there is a default or forced option.
+
 ### `abrController`
 
 (default: internal ABR controller)
@@ -1701,23 +1734,39 @@ stop playlist/fragment loading. could be resumed later on by calling `hls.startL
 
 ## Audio Tracks Control API
 
+### `hls.setAudioOption(audioOption)`
+
+Find and select the best matching audio track, making a level switch when a Group change is necessary. Updates `hls.config.audioPreference`. Returns the selected track or null when no matching track is found.
+
+### `hls.allAudioTracks`
+
+get : array of all supported audio tracks found in the Multivariant Playlist
+
 ### `hls.audioTracks`
 
-get : array of audio tracks exposed in manifest
+get : array of supported audio tracks in the active audio group ID
 
 ### `hls.audioTrack`
 
-get/set : audio track id (returned by)
+get/set : index of selected audio track in `hls.audioTracks`
 
 ## Subtitle Tracks Control API
 
+### `hls.setSubtitleOption(subtitleOption)`
+
+Find and select the best matching subtitle track, making a level switch when a Group change is necessary. Updates `hls.config.subtitlePreference`. Returns the selected track or null when no matching track is found.
+
+### `hls.allSubtitleTracks`
+
+get : array of all subtitle tracks found in the Multivariant Playlist
+
 ### `hls.subtitleTracks`
 
-get : array of subtitle tracks exposed in manifest
+get : array of subtitle tracks in the active subtitle group ID
 
 ### `hls.subtitleTrack`
 
-get/set : subtitle track id (returned by). Returns -1 if no track is visible. Set to -1 to disable all subtitle tracks.
+get/set : index of selected subtitle track in `hls.subtitleTracks`. Returns -1 if no track is visible. Set to -1 to disable all subtitle tracks.
 
 ### `hls.subtitleDisplay`
 

diff --git a/src/config.ts b/src/config.ts
@@ -29,6 +29,10 @@ import type {
   LoaderResponse,
   PlaylistLoaderContext,
 } from './types/loader';
+import type {
+  AudioSelectionOption,
+  SubtitleSelectionOption,
+} from './types/media-playlist';
 
 export type ABRControllerConfig = {
   abrEwmaFastLive: number;
@@ -217,6 +221,11 @@ export type StreamControllerConfig = {
   testBandwidth: boolean;
 };
 
+export type SelectionPreferences = {
+  audioPreference?: AudioSelectionOption;
+  subtitlePreference?: SubtitleSelectionOption;
+};
+
 export type LatencyControllerConfig = {
   liveSyncDurationCount: number;
   liveMaxLatencyDurationCount: number;
@@ -298,6 +307,7 @@ export type HlsConfig = {
   LevelControllerConfig &
   MP4RemuxerConfig &
   StreamControllerConfig &
+  SelectionPreferences &
   LatencyControllerConfig &
   MetadataControllerConfig &
   TimelineControllerConfig &