From 2480e0a47709ef594780040ba38daa5ecfe315e4 Mon Sep 17 00:00:00 2001 From: tonihei Date: Thu, 7 Jan 2021 16:29:23 +0000 Subject: [PATCH] Add option to align durations of MergingMediaSource. Without this feature it's impossible to nicely merge multiple sources with different durations if these durations are not known exactly before the start of playback. Issue: #8422 PiperOrigin-RevId: 350567625 --- RELEASENOTES.md | 3 + .../exoplayer2/source/MergingMediaSource.java | 149 +++++++++++++++++- .../source/MergingMediaSourceTest.java | 88 +++++++---- 3 files changed, 204 insertions(+), 36 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 8a35bde4c0d..f12907b9728 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -42,6 +42,9 @@ * Populate codecs string for H.264/AVC in MP4, Matroska and FLV streams to allow decoder capability checks based on codec profile/level ([#8393](https://github.com/google/ExoPlayer/issues/8393)). + * Add option to `MergingMediaSource` to clip the durations of all sources + to have the same length + ([#8422](https://github.com/google/ExoPlayer/issues/8422)). * Track selection: * Allow parallel adaptation for video and audio ([#5111](https://github.com/google/ExoPlayer/issues/5111)). diff --git a/library/core/src/main/java/com/google/android/exoplayer2/source/MergingMediaSource.java b/library/core/src/main/java/com/google/android/exoplayer2/source/MergingMediaSource.java index 8df7a639c6f..3a9bfe4f918 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/source/MergingMediaSource.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/source/MergingMediaSource.java @@ -15,12 +15,18 @@ */ package com.google.android.exoplayer2.source; +import static com.google.android.exoplayer2.util.Assertions.checkNotNull; +import static java.lang.Math.min; + import androidx.annotation.IntDef; import androidx.annotation.Nullable; +import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.MediaItem; import com.google.android.exoplayer2.Timeline; import com.google.android.exoplayer2.upstream.Allocator; import com.google.android.exoplayer2.upstream.TransferListener; +import com.google.common.collect.Multimap; +import com.google.common.collect.MultimapBuilder; import java.io.IOException; import java.lang.annotation.Documented; import java.lang.annotation.Retention; @@ -28,6 +34,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; /** * Merges multiple {@link MediaSource}s. @@ -70,21 +78,26 @@ public IllegalMergeException(@Reason int reason) { new MediaItem.Builder().setMediaId("MergingMediaSource").build(); private final boolean adjustPeriodTimeOffsets; + private final boolean clipDurations; private final MediaSource[] mediaSources; private final Timeline[] timelines; private final ArrayList pendingTimelineSources; private final CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory; + private final Map clippedDurationsUs; + private final Multimap clippedMediaPeriods; private int periodCount; private long[][] periodTimeOffsetsUs; + @Nullable private IllegalMergeException mergeError; /** * Creates a merging media source. * - *

Offsets between the timestamps in the media sources will not be adjusted. + *

Neither offsets between the timestamps in the media sources nor the durations of the media + * sources will be adjusted. * - * @param mediaSources The {@link MediaSource}s to merge. + * @param mediaSources The {@link MediaSource MediaSources} to merge. */ public MergingMediaSource(MediaSource... mediaSources) { this(/* adjustPeriodTimeOffsets= */ false, mediaSources); @@ -93,12 +106,14 @@ public MergingMediaSource(MediaSource... mediaSources) { /** * Creates a merging media source. * + *

Durations of the media sources will not be adjusted. + * * @param adjustPeriodTimeOffsets Whether to adjust timestamps of the merged media sources to all * start at the same time. - * @param mediaSources The {@link MediaSource}s to merge. + * @param mediaSources The {@link MediaSource MediaSources} to merge. */ public MergingMediaSource(boolean adjustPeriodTimeOffsets, MediaSource... mediaSources) { - this(adjustPeriodTimeOffsets, new DefaultCompositeSequenceableLoaderFactory(), mediaSources); + this(adjustPeriodTimeOffsets, /* clipDurations= */ false, mediaSources); } /** @@ -106,22 +121,46 @@ public MergingMediaSource(boolean adjustPeriodTimeOffsets, MediaSource... mediaS * * @param adjustPeriodTimeOffsets Whether to adjust timestamps of the merged media sources to all * start at the same time. + * @param clipDurations Whether to clip the durations of the media sources to match the shortest + * duration. + * @param mediaSources The {@link MediaSource MediaSources} to merge. + */ + public MergingMediaSource( + boolean adjustPeriodTimeOffsets, boolean clipDurations, MediaSource... mediaSources) { + this( + adjustPeriodTimeOffsets, + clipDurations, + new DefaultCompositeSequenceableLoaderFactory(), + mediaSources); + } + + /** + * Creates a merging media source. + * + * @param adjustPeriodTimeOffsets Whether to adjust timestamps of the merged media sources to all + * start at the same time. + * @param clipDurations Whether to clip the durations of the media sources to match the shortest + * duration. * @param compositeSequenceableLoaderFactory A factory to create composite {@link * SequenceableLoader}s for when this media source loads data from multiple streams (video, * audio etc...). - * @param mediaSources The {@link MediaSource}s to merge. + * @param mediaSources The {@link MediaSource MediaSources} to merge. */ public MergingMediaSource( boolean adjustPeriodTimeOffsets, + boolean clipDurations, CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory, MediaSource... mediaSources) { this.adjustPeriodTimeOffsets = adjustPeriodTimeOffsets; + this.clipDurations = clipDurations; this.mediaSources = mediaSources; this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory; pendingTimelineSources = new ArrayList<>(Arrays.asList(mediaSources)); periodCount = PERIOD_COUNT_UNSET; timelines = new Timeline[mediaSources.length]; periodTimeOffsetsUs = new long[0][]; + clippedDurationsUs = new HashMap<>(); + clippedMediaPeriods = MultimapBuilder.hashKeys().arrayListValues().build(); } /** @@ -167,12 +206,33 @@ public MediaPeriod createPeriod(MediaPeriodId id, Allocator allocator, long star mediaSources[i].createPeriod( childMediaPeriodId, allocator, startPositionUs - periodTimeOffsetsUs[periodIndex][i]); } - return new MergingMediaPeriod( - compositeSequenceableLoaderFactory, periodTimeOffsetsUs[periodIndex], periods); + MediaPeriod mediaPeriod = + new MergingMediaPeriod( + compositeSequenceableLoaderFactory, periodTimeOffsetsUs[periodIndex], periods); + if (clipDurations) { + mediaPeriod = + new ClippingMediaPeriod( + mediaPeriod, + /* enableInitialDiscontinuity= */ true, + /* startUs= */ 0, + /* endUs= */ checkNotNull(clippedDurationsUs.get(id.periodUid))); + clippedMediaPeriods.put(id.periodUid, (ClippingMediaPeriod) mediaPeriod); + } + return mediaPeriod; } @Override public void releasePeriod(MediaPeriod mediaPeriod) { + if (clipDurations) { + ClippingMediaPeriod clippingMediaPeriod = (ClippingMediaPeriod) mediaPeriod; + for (Map.Entry entry : clippedMediaPeriods.entries()) { + if (entry.getValue().equals(clippingMediaPeriod)) { + clippedMediaPeriods.remove(entry.getKey(), entry.getValue()); + break; + } + } + mediaPeriod = clippingMediaPeriod.mediaPeriod; + } MergingMediaPeriod mergingPeriod = (MergingMediaPeriod) mediaPeriod; for (int i = 0; i < mediaSources.length; i++) { mediaSources[i].releasePeriod(mergingPeriod.getChildPeriod(i)); @@ -210,7 +270,12 @@ protected void onChildSourceInfoRefreshed( if (adjustPeriodTimeOffsets) { computePeriodTimeOffsets(); } - refreshSourceInfo(timelines[0]); + Timeline mergedTimeline = timelines[0]; + if (clipDurations) { + updateClippedDuration(); + mergedTimeline = new ClippedTimeline(mergedTimeline, clippedDurationsUs); + } + refreshSourceInfo(mergedTimeline); } } @@ -234,4 +299,72 @@ private void computePeriodTimeOffsets() { } } } + + private void updateClippedDuration() { + Timeline.Period period = new Timeline.Period(); + for (int periodIndex = 0; periodIndex < periodCount; periodIndex++) { + long minDurationUs = C.TIME_END_OF_SOURCE; + for (int timelineIndex = 0; timelineIndex < timelines.length; timelineIndex++) { + long durationUs = timelines[timelineIndex].getPeriod(periodIndex, period).getDurationUs(); + if (durationUs == C.TIME_UNSET) { + continue; + } + long adjustedDurationUs = durationUs + periodTimeOffsetsUs[periodIndex][timelineIndex]; + if (minDurationUs == C.TIME_END_OF_SOURCE || adjustedDurationUs < minDurationUs) { + minDurationUs = adjustedDurationUs; + } + } + Object periodUid = timelines[0].getUidOfPeriod(periodIndex); + clippedDurationsUs.put(periodUid, minDurationUs); + for (ClippingMediaPeriod clippingMediaPeriod : clippedMediaPeriods.get(periodUid)) { + clippingMediaPeriod.updateClipping(/* startUs= */ 0, /* endUs= */ minDurationUs); + } + } + } + + private static final class ClippedTimeline extends ForwardingTimeline { + + private final long[] periodDurationsUs; + private final long[] windowDurationsUs; + + public ClippedTimeline(Timeline timeline, Map clippedDurationsUs) { + super(timeline); + int windowCount = timeline.getWindowCount(); + windowDurationsUs = new long[timeline.getWindowCount()]; + Window window = new Window(); + for (int i = 0; i < windowCount; i++) { + windowDurationsUs[i] = timeline.getWindow(i, window).durationUs; + } + int periodCount = timeline.getPeriodCount(); + periodDurationsUs = new long[periodCount]; + Period period = new Period(); + for (int i = 0; i < periodCount; i++) { + timeline.getPeriod(i, period, /* setIds= */ true); + long clippedDurationUs = checkNotNull(clippedDurationsUs.get(period.uid)); + periodDurationsUs[i] = + clippedDurationUs != C.TIME_END_OF_SOURCE ? clippedDurationUs : period.durationUs; + if (period.durationUs != C.TIME_UNSET) { + windowDurationsUs[period.windowIndex] -= period.durationUs - periodDurationsUs[i]; + } + } + } + + @Override + public Window getWindow(int windowIndex, Window window, long defaultPositionProjectionUs) { + super.getWindow(windowIndex, window, defaultPositionProjectionUs); + window.durationUs = windowDurationsUs[windowIndex]; + window.defaultPositionUs = + window.durationUs == C.TIME_UNSET || window.defaultPositionUs == C.TIME_UNSET + ? window.defaultPositionUs + : min(window.defaultPositionUs, window.durationUs); + return window; + } + + @Override + public Period getPeriod(int periodIndex, Period period, boolean setIds) { + super.getPeriod(periodIndex, period, setIds); + period.durationUs = periodDurationsUs[periodIndex]; + return period; + } + } } diff --git a/library/core/src/test/java/com/google/android/exoplayer2/source/MergingMediaSourceTest.java b/library/core/src/test/java/com/google/android/exoplayer2/source/MergingMediaSourceTest.java index c66a5cff741..b0f54dd8606 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/source/MergingMediaSourceTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/source/MergingMediaSourceTest.java @@ -16,7 +16,7 @@ package com.google.android.exoplayer2.source; import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.fail; +import static org.junit.Assert.assertThrows; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.android.exoplayer2.C; @@ -35,35 +35,65 @@ public class MergingMediaSourceTest { @Test - public void mergingDynamicTimelines() throws IOException { - FakeTimeline firstTimeline = - new FakeTimeline(new TimelineWindowDefinition(true, true, C.TIME_UNSET)); - FakeTimeline secondTimeline = - new FakeTimeline(new TimelineWindowDefinition(true, true, C.TIME_UNSET)); - testMergingMediaSourcePrepare(firstTimeline, secondTimeline); + public void prepare_withoutDurationClipping_usesTimelineOfFirstSource() throws IOException { + FakeTimeline timeline1 = + new FakeTimeline( + new TimelineWindowDefinition( + /* isSeekable= */ true, /* isDynamic= */ true, /* durationUs= */ 30)); + FakeTimeline timeline2 = + new FakeTimeline( + new TimelineWindowDefinition( + /* isSeekable= */ true, /* isDynamic= */ true, /* durationUs= */ C.TIME_UNSET)); + FakeTimeline timeline3 = + new FakeTimeline( + new TimelineWindowDefinition( + /* isSeekable= */ true, /* isDynamic= */ true, /* durationUs= */ 10)); + + Timeline mergedTimeline = + prepareMergingMediaSource(/* clipDurations= */ false, timeline1, timeline2, timeline3); + + assertThat(mergedTimeline).isEqualTo(timeline1); } @Test - public void mergingStaticTimelines() throws IOException { - FakeTimeline firstTimeline = new FakeTimeline(new TimelineWindowDefinition(true, false, 20)); - FakeTimeline secondTimeline = new FakeTimeline(new TimelineWindowDefinition(true, false, 10)); - testMergingMediaSourcePrepare(firstTimeline, secondTimeline); + public void prepare_withDurationClipping_usesDurationOfShortestSource() throws IOException { + FakeTimeline timeline1 = + new FakeTimeline( + new TimelineWindowDefinition( + /* isSeekable= */ true, /* isDynamic= */ true, /* durationUs= */ 30)); + FakeTimeline timeline2 = + new FakeTimeline( + new TimelineWindowDefinition( + /* isSeekable= */ true, /* isDynamic= */ true, /* durationUs= */ C.TIME_UNSET)); + FakeTimeline timeline3 = + new FakeTimeline( + new TimelineWindowDefinition( + /* isSeekable= */ true, /* isDynamic= */ true, /* durationUs= */ 10)); + + Timeline mergedTimeline = + prepareMergingMediaSource(/* clipDurations= */ true, timeline1, timeline2, timeline3); + + assertThat(mergedTimeline).isEqualTo(timeline3); } @Test - public void mergingTimelinesWithDifferentPeriodCounts() throws IOException { - FakeTimeline firstTimeline = new FakeTimeline(new TimelineWindowDefinition(1, null)); - FakeTimeline secondTimeline = new FakeTimeline(new TimelineWindowDefinition(2, null)); - try { - testMergingMediaSourcePrepare(firstTimeline, secondTimeline); - fail("Expected merging to fail."); - } catch (IllegalMergeException e) { - assertThat(e.reason).isEqualTo(IllegalMergeException.REASON_PERIOD_COUNT_MISMATCH); - } + public void prepare_differentPeriodCounts_fails() throws IOException { + FakeTimeline firstTimeline = + new FakeTimeline(new TimelineWindowDefinition(/* periodCount= */ 1, /* id= */ 1)); + FakeTimeline secondTimeline = + new FakeTimeline(new TimelineWindowDefinition(/* periodCount= */ 2, /* id= */ 2)); + + IllegalMergeException exception = + assertThrows( + IllegalMergeException.class, + () -> + prepareMergingMediaSource( + /* clipDurations= */ false, firstTimeline, secondTimeline)); + assertThat(exception.reason).isEqualTo(IllegalMergeException.REASON_PERIOD_COUNT_MISMATCH); } @Test - public void mergingMediaSourcePeriodCreation() throws Exception { + public void createPeriod_createsChildPeriods() throws Exception { FakeMediaSource[] mediaSources = new FakeMediaSource[2]; for (int i = 0; i < mediaSources.length; i++) { mediaSources[i] = new FakeMediaSource(new FakeTimeline(/* windowCount= */ 2)); @@ -83,24 +113,26 @@ public void mergingMediaSourcePeriodCreation() throws Exception { } /** - * Wraps the specified timelines in a {@link MergingMediaSource}, prepares it and checks that it - * forwards the first of the wrapped timelines. + * Wraps the specified timelines in a {@link MergingMediaSource}, prepares it and returns the + * merged timeline. */ - private static void testMergingMediaSourcePrepare(Timeline... timelines) throws IOException { + private static Timeline prepareMergingMediaSource(boolean clipDurations, Timeline... timelines) + throws IOException { FakeMediaSource[] mediaSources = new FakeMediaSource[timelines.length]; for (int i = 0; i < timelines.length; i++) { mediaSources[i] = new FakeMediaSource(timelines[i]); } - MergingMediaSource mergingMediaSource = new MergingMediaSource(mediaSources); - MediaSourceTestRunner testRunner = new MediaSourceTestRunner(mergingMediaSource, null); + MergingMediaSource mergingMediaSource = + new MergingMediaSource(/* adjustPeriodTimeOffsets= */ false, clipDurations, mediaSources); + MediaSourceTestRunner testRunner = + new MediaSourceTestRunner(mergingMediaSource, /* allocator= */ null); try { Timeline timeline = testRunner.prepareSource(); - // The merged timeline should always be the one from the first child. - assertThat(timeline).isEqualTo(timelines[0]); testRunner.releaseSource(); for (FakeMediaSource mediaSource : mediaSources) { mediaSource.assertReleased(); } + return timeline; } finally { testRunner.release(); }