Skip to content

Commit

Permalink
Add option to align durations of MergingMediaSource.
Browse files Browse the repository at this point in the history 
Without this feature it's impossible to nicely merge multiple sources
with different durations if these durations are not known exactly
before the start of playback.

Issue: #8422
PiperOrigin-RevId: 350567625
  • Loading branch information
@tonihei @icbaker
tonihei authored and icbaker committed Jan 8, 2021
1 parent 1ef06b8 commit 2480e0a
Show file tree
Hide file tree
Showing 3 changed files with 204 additions and 36 deletions.
3 changes: 3 additions & 0 deletions RELEASENOTES.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@
* Populate codecs string for H.264/AVC in MP4, Matroska and FLV streams to
allow decoder capability checks based on codec profile/level
([#8393](https://github.com/google/ExoPlayer/issues/8393)).
* Add option to `MergingMediaSource` to clip the durations of all sources
to have the same length
([#8422](https://github.com/google/ExoPlayer/issues/8422)).
* Track selection:
* Allow parallel adaptation for video and audio
([#5111](https://github.com/google/ExoPlayer/issues/5111)).
Expand Down
149 changes: 141 additions & 8 deletions library/core/src/main/java/com/google/android/exoplayer2/source/MergingMediaSource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,19 +15,27 @@
*/
package com.google.android.exoplayer2.source;

import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static java.lang.Math.min;

import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.MediaItem;
import com.google.android.exoplayer2.Timeline;
import com.google.android.exoplayer2.upstream.Allocator;
import com.google.android.exoplayer2.upstream.TransferListener;
import com.google.common.collect.Multimap;
import com.google.common.collect.MultimapBuilder;
import java.io.IOException;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

/**
* Merges multiple {@link MediaSource}s.
Expand Down Expand Up @@ -70,21 +78,26 @@ public IllegalMergeException(@Reason int reason) {
new MediaItem.Builder().setMediaId("MergingMediaSource").build();

private final boolean adjustPeriodTimeOffsets;
private final boolean clipDurations;
private final MediaSource[] mediaSources;
private final Timeline[] timelines;
private final ArrayList<MediaSource> pendingTimelineSources;
private final CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory;
private final Map<Object, Long> clippedDurationsUs;
private final Multimap<Object, ClippingMediaPeriod> clippedMediaPeriods;

private int periodCount;
private long[][] periodTimeOffsetsUs;

@Nullable private IllegalMergeException mergeError;

/**
* Creates a merging media source.
*
* <p>Offsets between the timestamps in the media sources will not be adjusted.
* <p>Neither offsets between the timestamps in the media sources nor the durations of the media
* sources will be adjusted.
*
* @param mediaSources The {@link MediaSource}s to merge.
* @param mediaSources The {@link MediaSource MediaSources} to merge.
*/
public MergingMediaSource(MediaSource... mediaSources) {
this(/* adjustPeriodTimeOffsets= */ false, mediaSources);
Expand All @@ -93,35 +106,61 @@ public MergingMediaSource(MediaSource... mediaSources) {
/**
* Creates a merging media source.
*
* <p>Durations of the media sources will not be adjusted.
*
* @param adjustPeriodTimeOffsets Whether to adjust timestamps of the merged media sources to all
* start at the same time.
* @param mediaSources The {@link MediaSource}s to merge.
* @param mediaSources The {@link MediaSource MediaSources} to merge.
*/
public MergingMediaSource(boolean adjustPeriodTimeOffsets, MediaSource... mediaSources) {
this(adjustPeriodTimeOffsets, new DefaultCompositeSequenceableLoaderFactory(), mediaSources);
this(adjustPeriodTimeOffsets, /* clipDurations= */ false, mediaSources);
}

/**
* Creates a merging media source.
*
* @param adjustPeriodTimeOffsets Whether to adjust timestamps of the merged media sources to all
* start at the same time.
* @param clipDurations Whether to clip the durations of the media sources to match the shortest
* duration.
* @param mediaSources The {@link MediaSource MediaSources} to merge.
*/
public MergingMediaSource(
boolean adjustPeriodTimeOffsets, boolean clipDurations, MediaSource... mediaSources) {
this(
adjustPeriodTimeOffsets,
clipDurations,
new DefaultCompositeSequenceableLoaderFactory(),
mediaSources);
}

/**
* Creates a merging media source.
*
* @param adjustPeriodTimeOffsets Whether to adjust timestamps of the merged media sources to all
* start at the same time.
* @param clipDurations Whether to clip the durations of the media sources to match the shortest
* duration.
* @param compositeSequenceableLoaderFactory A factory to create composite {@link
* SequenceableLoader}s for when this media source loads data from multiple streams (video,
* audio etc...).
* @param mediaSources The {@link MediaSource}s to merge.
* @param mediaSources The {@link MediaSource MediaSources} to merge.
*/
public MergingMediaSource(
boolean adjustPeriodTimeOffsets,
boolean clipDurations,
CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory,
MediaSource... mediaSources) {
this.adjustPeriodTimeOffsets = adjustPeriodTimeOffsets;
this.clipDurations = clipDurations;
this.mediaSources = mediaSources;
this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory;
pendingTimelineSources = new ArrayList<>(Arrays.asList(mediaSources));
periodCount = PERIOD_COUNT_UNSET;
timelines = new Timeline[mediaSources.length];
periodTimeOffsetsUs = new long[0][];
clippedDurationsUs = new HashMap<>();
clippedMediaPeriods = MultimapBuilder.hashKeys().arrayListValues().build();
}

/**
Expand Down Expand Up @@ -167,12 +206,33 @@ public MediaPeriod createPeriod(MediaPeriodId id, Allocator allocator, long star
mediaSources[i].createPeriod(
childMediaPeriodId, allocator, startPositionUs - periodTimeOffsetsUs[periodIndex][i]);
}
return new MergingMediaPeriod(
compositeSequenceableLoaderFactory, periodTimeOffsetsUs[periodIndex], periods);
MediaPeriod mediaPeriod =
new MergingMediaPeriod(
compositeSequenceableLoaderFactory, periodTimeOffsetsUs[periodIndex], periods);
if (clipDurations) {
mediaPeriod =
new ClippingMediaPeriod(
mediaPeriod,
/* enableInitialDiscontinuity= */ true,
/* startUs= */ 0,
/* endUs= */ checkNotNull(clippedDurationsUs.get(id.periodUid)));
clippedMediaPeriods.put(id.periodUid, (ClippingMediaPeriod) mediaPeriod);
}
return mediaPeriod;
}

@Override
public void releasePeriod(MediaPeriod mediaPeriod) {
if (clipDurations) {
ClippingMediaPeriod clippingMediaPeriod = (ClippingMediaPeriod) mediaPeriod;
for (Map.Entry<Object, ClippingMediaPeriod> entry : clippedMediaPeriods.entries()) {
if (entry.getValue().equals(clippingMediaPeriod)) {
clippedMediaPeriods.remove(entry.getKey(), entry.getValue());
break;
}
}
mediaPeriod = clippingMediaPeriod.mediaPeriod;
}
MergingMediaPeriod mergingPeriod = (MergingMediaPeriod) mediaPeriod;
for (int i = 0; i < mediaSources.length; i++) {
mediaSources[i].releasePeriod(mergingPeriod.getChildPeriod(i));
Expand Down Expand Up @@ -210,7 +270,12 @@ protected void onChildSourceInfoRefreshed(
if (adjustPeriodTimeOffsets) {
computePeriodTimeOffsets();
}
refreshSourceInfo(timelines[0]);
Timeline mergedTimeline = timelines[0];
if (clipDurations) {
updateClippedDuration();
mergedTimeline = new ClippedTimeline(mergedTimeline, clippedDurationsUs);
}
refreshSourceInfo(mergedTimeline);
}
}

Expand All @@ -234,4 +299,72 @@ private void computePeriodTimeOffsets() {
}
}
}

private void updateClippedDuration() {
Timeline.Period period = new Timeline.Period();
for (int periodIndex = 0; periodIndex < periodCount; periodIndex++) {
long minDurationUs = C.TIME_END_OF_SOURCE;
for (int timelineIndex = 0; timelineIndex < timelines.length; timelineIndex++) {
long durationUs = timelines[timelineIndex].getPeriod(periodIndex, period).getDurationUs();
if (durationUs == C.TIME_UNSET) {
continue;
}
long adjustedDurationUs = durationUs + periodTimeOffsetsUs[periodIndex][timelineIndex];
if (minDurationUs == C.TIME_END_OF_SOURCE || adjustedDurationUs < minDurationUs) {
minDurationUs = adjustedDurationUs;
}
}
Object periodUid = timelines[0].getUidOfPeriod(periodIndex);
clippedDurationsUs.put(periodUid, minDurationUs);
for (ClippingMediaPeriod clippingMediaPeriod : clippedMediaPeriods.get(periodUid)) {
clippingMediaPeriod.updateClipping(/* startUs= */ 0, /* endUs= */ minDurationUs);
}
}
}

private static final class ClippedTimeline extends ForwardingTimeline {

private final long[] periodDurationsUs;
private final long[] windowDurationsUs;

public ClippedTimeline(Timeline timeline, Map<Object, Long> clippedDurationsUs) {
super(timeline);
int windowCount = timeline.getWindowCount();
windowDurationsUs = new long[timeline.getWindowCount()];
Window window = new Window();
for (int i = 0; i < windowCount; i++) {
windowDurationsUs[i] = timeline.getWindow(i, window).durationUs;
}
int periodCount = timeline.getPeriodCount();
periodDurationsUs = new long[periodCount];
Period period = new Period();
for (int i = 0; i < periodCount; i++) {
timeline.getPeriod(i, period, /* setIds= */ true);
long clippedDurationUs = checkNotNull(clippedDurationsUs.get(period.uid));
periodDurationsUs[i] =
clippedDurationUs != C.TIME_END_OF_SOURCE ? clippedDurationUs : period.durationUs;
if (period.durationUs != C.TIME_UNSET) {
windowDurationsUs[period.windowIndex] -= period.durationUs - periodDurationsUs[i];
}
}
}

@Override
public Window getWindow(int windowIndex, Window window, long defaultPositionProjectionUs) {
super.getWindow(windowIndex, window, defaultPositionProjectionUs);
window.durationUs = windowDurationsUs[windowIndex];
window.defaultPositionUs =
window.durationUs == C.TIME_UNSET || window.defaultPositionUs == C.TIME_UNSET
? window.defaultPositionUs
: min(window.defaultPositionUs, window.durationUs);
return window;
}

@Override
public Period getPeriod(int periodIndex, Period period, boolean setIds) {
super.getPeriod(periodIndex, period, setIds);
period.durationUs = periodDurationsUs[periodIndex];
return period;
}
}
}
88 changes: 60 additions & 28 deletions library/core/src/test/java/com/google/android/exoplayer2/source/MergingMediaSourceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
package com.google.android.exoplayer2.source;

import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.fail;
import static org.junit.Assert.assertThrows;

import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.android.exoplayer2.C;
Expand All @@ -35,35 +35,65 @@
public class MergingMediaSourceTest {

@Test
public void mergingDynamicTimelines() throws IOException {
FakeTimeline firstTimeline =
new FakeTimeline(new TimelineWindowDefinition(true, true, C.TIME_UNSET));
FakeTimeline secondTimeline =
new FakeTimeline(new TimelineWindowDefinition(true, true, C.TIME_UNSET));
testMergingMediaSourcePrepare(firstTimeline, secondTimeline);
public void prepare_withoutDurationClipping_usesTimelineOfFirstSource() throws IOException {
FakeTimeline timeline1 =
new FakeTimeline(
new TimelineWindowDefinition(
/* isSeekable= */ true, /* isDynamic= */ true, /* durationUs= */ 30));
FakeTimeline timeline2 =
new FakeTimeline(
new TimelineWindowDefinition(
/* isSeekable= */ true, /* isDynamic= */ true, /* durationUs= */ C.TIME_UNSET));
FakeTimeline timeline3 =
new FakeTimeline(
new TimelineWindowDefinition(
/* isSeekable= */ true, /* isDynamic= */ true, /* durationUs= */ 10));

Timeline mergedTimeline =
prepareMergingMediaSource(/* clipDurations= */ false, timeline1, timeline2, timeline3);

assertThat(mergedTimeline).isEqualTo(timeline1);
}

@Test
public void mergingStaticTimelines() throws IOException {
FakeTimeline firstTimeline = new FakeTimeline(new TimelineWindowDefinition(true, false, 20));
FakeTimeline secondTimeline = new FakeTimeline(new TimelineWindowDefinition(true, false, 10));
testMergingMediaSourcePrepare(firstTimeline, secondTimeline);
public void prepare_withDurationClipping_usesDurationOfShortestSource() throws IOException {
FakeTimeline timeline1 =
new FakeTimeline(
new TimelineWindowDefinition(
/* isSeekable= */ true, /* isDynamic= */ true, /* durationUs= */ 30));
FakeTimeline timeline2 =
new FakeTimeline(
new TimelineWindowDefinition(
/* isSeekable= */ true, /* isDynamic= */ true, /* durationUs= */ C.TIME_UNSET));
FakeTimeline timeline3 =
new FakeTimeline(
new TimelineWindowDefinition(
/* isSeekable= */ true, /* isDynamic= */ true, /* durationUs= */ 10));

Timeline mergedTimeline =
prepareMergingMediaSource(/* clipDurations= */ true, timeline1, timeline2, timeline3);

assertThat(mergedTimeline).isEqualTo(timeline3);
}

@Test
public void mergingTimelinesWithDifferentPeriodCounts() throws IOException {
FakeTimeline firstTimeline = new FakeTimeline(new TimelineWindowDefinition(1, null));
FakeTimeline secondTimeline = new FakeTimeline(new TimelineWindowDefinition(2, null));
try {
testMergingMediaSourcePrepare(firstTimeline, secondTimeline);
fail("Expected merging to fail.");
} catch (IllegalMergeException e) {
assertThat(e.reason).isEqualTo(IllegalMergeException.REASON_PERIOD_COUNT_MISMATCH);
}
public void prepare_differentPeriodCounts_fails() throws IOException {
FakeTimeline firstTimeline =
new FakeTimeline(new TimelineWindowDefinition(/* periodCount= */ 1, /* id= */ 1));
FakeTimeline secondTimeline =
new FakeTimeline(new TimelineWindowDefinition(/* periodCount= */ 2, /* id= */ 2));

IllegalMergeException exception =
assertThrows(
IllegalMergeException.class,
() ->
prepareMergingMediaSource(
/* clipDurations= */ false, firstTimeline, secondTimeline));
assertThat(exception.reason).isEqualTo(IllegalMergeException.REASON_PERIOD_COUNT_MISMATCH);
}

@Test
public void mergingMediaSourcePeriodCreation() throws Exception {
public void createPeriod_createsChildPeriods() throws Exception {
FakeMediaSource[] mediaSources = new FakeMediaSource[2];
for (int i = 0; i < mediaSources.length; i++) {
mediaSources[i] = new FakeMediaSource(new FakeTimeline(/* windowCount= */ 2));
Expand All @@ -83,24 +113,26 @@ public void mergingMediaSourcePeriodCreation() throws Exception {
}

/**
* Wraps the specified timelines in a {@link MergingMediaSource}, prepares it and checks that it
* forwards the first of the wrapped timelines.
* Wraps the specified timelines in a {@link MergingMediaSource}, prepares it and returns the
* merged timeline.
*/
private static void testMergingMediaSourcePrepare(Timeline... timelines) throws IOException {
private static Timeline prepareMergingMediaSource(boolean clipDurations, Timeline... timelines)
throws IOException {
FakeMediaSource[] mediaSources = new FakeMediaSource[timelines.length];
for (int i = 0; i < timelines.length; i++) {
mediaSources[i] = new FakeMediaSource(timelines[i]);
}
MergingMediaSource mergingMediaSource = new MergingMediaSource(mediaSources);
MediaSourceTestRunner testRunner = new MediaSourceTestRunner(mergingMediaSource, null);
MergingMediaSource mergingMediaSource =
new MergingMediaSource(/* adjustPeriodTimeOffsets= */ false, clipDurations, mediaSources);
MediaSourceTestRunner testRunner =
new MediaSourceTestRunner(mergingMediaSource, /* allocator= */ null);
try {
Timeline timeline = testRunner.prepareSource();
// The merged timeline should always be the one from the first child.
assertThat(timeline).isEqualTo(timelines[0]);
testRunner.releaseSource();
for (FakeMediaSource mediaSource : mediaSources) {
mediaSource.assertReleased();
}
return timeline;
} finally {
testRunner.release();
}
Expand Down

0 comments on commit 2480e0a

Please sign in to comment.