From bd7be1b5e7cc41a59ebbc348d394820fc857db92 Mon Sep 17 00:00:00 2001 From: eguven Date: Tue, 30 Aug 2016 04:58:44 -0700 Subject: [PATCH] Cache support unbounded requests. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=131696858 --- .../upstream/cache/CacheDataSourceTest.java | 190 ++++++++++++++++++ .../upstream/cache/SimpleCacheTest.java | 123 ++++++++++++ .../exoplayer2/upstream/cache/Cache.java | 24 ++- .../upstream/cache/CacheDataSink.java | 16 +- .../upstream/cache/CacheDataSource.java | 166 +++++++++++---- .../upstream/cache/CacheEvictor.java | 4 +- .../cache/LeastRecentlyUsedCacheEvictor.java | 4 +- .../upstream/cache/NoOpCacheEvictor.java | 2 +- .../upstream/cache/SimpleCache.java | 107 +++++++--- .../exoplayer2/testutil/FakeDataSource.java | 18 +- .../android/exoplayer2/testutil/TestUtil.java | 10 + 11 files changed, 579 insertions(+), 85 deletions(-) create mode 100644 library/src/androidTest/java/com/google/android/exoplayer2/upstream/cache/CacheDataSourceTest.java create mode 100644 library/src/androidTest/java/com/google/android/exoplayer2/upstream/cache/SimpleCacheTest.java diff --git a/library/src/androidTest/java/com/google/android/exoplayer2/upstream/cache/CacheDataSourceTest.java b/library/src/androidTest/java/com/google/android/exoplayer2/upstream/cache/CacheDataSourceTest.java new file mode 100644 index 00000000000..789c8bb4133 --- /dev/null +++ b/library/src/androidTest/java/com/google/android/exoplayer2/upstream/cache/CacheDataSourceTest.java @@ -0,0 +1,190 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.upstream.cache; + +import android.net.Uri; +import android.test.InstrumentationTestCase; +import android.test.MoreAsserts; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.testutil.FakeDataSource; +import com.google.android.exoplayer2.testutil.FakeDataSource.Builder; +import com.google.android.exoplayer2.testutil.TestUtil; +import com.google.android.exoplayer2.upstream.DataSpec; +import com.google.android.exoplayer2.upstream.HttpDataSource.InvalidResponseCodeException; +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +/** Unit tests for {@link CacheDataSource}. */ +public class CacheDataSourceTest extends InstrumentationTestCase { + + private static final byte[] TEST_DATA = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + private static final int MAX_CACHE_FILE_SIZE = 3; + private static final String KEY_1 = "key 1"; + private static final String KEY_2 = "key 2"; + + private File cacheDir; + private SimpleCache simpleCache; + + @Override + protected void setUp() throws Exception { + // Create a temporary folder + cacheDir = File.createTempFile("CacheDataSourceTest", null); + assertTrue(cacheDir.delete()); + assertTrue(cacheDir.mkdir()); + + simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor()); + } + + @Override + protected void tearDown() throws Exception { + TestUtil.recursiveDelete(cacheDir); + } + + public void testMaxCacheFileSize() throws Exception { + CacheDataSource cacheDataSource = createCacheDataSource(false, false, false); + assertReadDataContentLength(cacheDataSource, false, false); + assertEquals((int) Math.ceil((double) TEST_DATA.length / MAX_CACHE_FILE_SIZE), + cacheDir.listFiles().length); + } + + public void testCacheAndRead() throws Exception { + assertCacheAndRead(false, false); + } + + public void testCacheAndReadUnboundedRequest() throws Exception { + assertCacheAndRead(true, false); + } + + public void testCacheAndReadUnknownLength() throws Exception { + assertCacheAndRead(false, true); + } + + // Disabled test as we don't support caching of definitely unknown length content + public void disabledTestCacheAndReadUnboundedRequestUnknownLength() throws Exception { + assertCacheAndRead(true, true); + } + + public void testUnsatisfiableRange() throws Exception { + // Bounded request but the content length is unknown. This forces all data to be cached but not + // the length + assertCacheAndRead(false, true); + + // Now do an unbounded request. This will read all of the data from cache and then try to read + // more from upstream which will cause to a 416 so CDS will store the length. + CacheDataSource cacheDataSource = createCacheDataSource(true, true, true); + assertReadDataContentLength(cacheDataSource, true, true); + + // If the user try to access off range then it should throw an IOException + try { + cacheDataSource = createCacheDataSource(false, false, false); + cacheDataSource.open(new DataSpec(Uri.EMPTY, TEST_DATA.length, 5, KEY_1)); + fail(); + } catch (TestIOException e) { + // success + } + } + + public void testContentLengthEdgeCases() throws Exception { + // Read partial at EOS but don't cross it so length is unknown + CacheDataSource cacheDataSource = createCacheDataSource(false, false, true); + assertReadData(cacheDataSource, true, TEST_DATA.length - 2, 2); + assertEquals(C.LENGTH_UNSET, simpleCache.getContentLength(KEY_1)); + + // Now do an unbounded request for whole data. This will cause a bounded request from upstream. + // End of data from upstream shouldn't be mixed up with EOS and cause length set wrong. + cacheDataSource = createCacheDataSource(true, false, true); + assertReadDataContentLength(cacheDataSource, true, true); + + // Now the length set correctly do an unbounded request with offset + assertEquals(2, cacheDataSource.open(new DataSpec(Uri.EMPTY, TEST_DATA.length - 2, + C.LENGTH_UNSET, KEY_1))); + + // An unbounded request with offset for not cached content + assertEquals(C.LENGTH_UNSET, cacheDataSource.open(new DataSpec(Uri.EMPTY, TEST_DATA.length - 2, + C.LENGTH_UNSET, KEY_2))); + } + + private void assertCacheAndRead(boolean unboundedRequest, boolean simulateUnknownLength) + throws IOException { + // Read all data from upstream and cache + CacheDataSource cacheDataSource = createCacheDataSource(false, false, simulateUnknownLength); + assertReadDataContentLength(cacheDataSource, unboundedRequest, simulateUnknownLength); + + // Just read from cache + cacheDataSource = createCacheDataSource(false, true, simulateUnknownLength); + assertReadDataContentLength(cacheDataSource, unboundedRequest, + false /*length is already cached*/); + } + + /** + * Reads data until EOI and compares it to {@link #TEST_DATA}. Also checks content length returned + * from open() call and the cached content length. + */ + private void assertReadDataContentLength(CacheDataSource cacheDataSource, + boolean unboundedRequest, boolean unknownLength) throws IOException { + int length = unboundedRequest ? C.LENGTH_UNSET : TEST_DATA.length; + assertReadData(cacheDataSource, unknownLength, 0, length); + assertEquals("When the range specified, CacheDataSource doesn't reach EOS so shouldn't cache " + + "content length", !unboundedRequest ? C.LENGTH_UNSET : TEST_DATA.length, + simpleCache.getContentLength(KEY_1)); + } + + private void assertReadData(CacheDataSource cacheDataSource, boolean unknownLength, int position, + int length) throws IOException { + int actualLength = TEST_DATA.length - position; + if (length != C.LENGTH_UNSET) { + actualLength = Math.min(actualLength, length); + } + assertEquals(unknownLength ? length : actualLength, + cacheDataSource.open(new DataSpec(Uri.EMPTY, position, length, KEY_1))); + + byte[] buffer = new byte[100]; + int index = 0; + while (true) { + int read = cacheDataSource.read(buffer, index, buffer.length - index); + if (read == C.RESULT_END_OF_INPUT) { + break; + } + index += read; + } + assertEquals(actualLength, index); + MoreAsserts.assertEquals(Arrays.copyOfRange(TEST_DATA, position, position + actualLength), + Arrays.copyOf(buffer, index)); + + cacheDataSource.close(); + } + + private CacheDataSource createCacheDataSource(boolean set416exception, boolean setReadException, + boolean simulateUnknownLength) { + Builder builder = new Builder(); + if (setReadException) { + builder.appendReadError(new IOException("Shouldn't read from upstream")); + } + builder.setSimulateUnknownLength(simulateUnknownLength); + builder.appendReadData(TEST_DATA); + FakeDataSource upstream = builder.build(); + upstream.setUnsatisfiableRangeException(set416exception + ? new InvalidResponseCodeException(416, null, null) + : new TestIOException()); + return new CacheDataSource(simpleCache, upstream, + CacheDataSource.FLAG_BLOCK_ON_CACHE | CacheDataSource.FLAG_CACHE_UNBOUNDED_REQUESTS, + MAX_CACHE_FILE_SIZE); + } + + private static class TestIOException extends IOException {} + +} diff --git a/library/src/androidTest/java/com/google/android/exoplayer2/upstream/cache/SimpleCacheTest.java b/library/src/androidTest/java/com/google/android/exoplayer2/upstream/cache/SimpleCacheTest.java new file mode 100644 index 00000000000..2c8ea912fb6 --- /dev/null +++ b/library/src/androidTest/java/com/google/android/exoplayer2/upstream/cache/SimpleCacheTest.java @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.upstream.cache; + +import android.test.InstrumentationTestCase; + +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.testutil.TestUtil; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.NavigableSet; +import java.util.Set; + +/** + * Unit tests for {@link SimpleCache}. + */ +public class SimpleCacheTest extends InstrumentationTestCase { + + private static final String KEY_1 = "key1"; + + private File cacheDir; + + @Override + protected void setUp() throws Exception { + // Create a temporary folder + cacheDir = File.createTempFile("SimpleCacheTest", null); + assertTrue(cacheDir.delete()); + assertTrue(cacheDir.mkdir()); + } + + @Override + protected void tearDown() throws Exception { + TestUtil.recursiveDelete(cacheDir); + } + + public void testCommittingOneFile() throws Exception { + SimpleCache simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor()); + + CacheSpan cacheSpan = simpleCache.startReadWrite(KEY_1, 0); + assertFalse(cacheSpan.isCached); + assertTrue(cacheSpan.isOpenEnded()); + + assertNull(simpleCache.startReadWriteNonBlocking(KEY_1, 0)); + + assertEquals(0, simpleCache.getKeys().size()); + NavigableSet cachedSpans = simpleCache.getCachedSpans(KEY_1); + assertTrue(cachedSpans == null || cachedSpans.size() == 0); + assertEquals(0, simpleCache.getCacheSpace()); + assertEquals(0, cacheDir.listFiles().length); + + addCache(simpleCache, 0, 15); + + Set cachedKeys = simpleCache.getKeys(); + assertEquals(1, cachedKeys.size()); + assertTrue(cachedKeys.contains(KEY_1)); + cachedSpans = simpleCache.getCachedSpans(KEY_1); + assertEquals(1, cachedSpans.size()); + assertTrue(cachedSpans.contains(cacheSpan)); + assertEquals(15, simpleCache.getCacheSpace()); + + cacheSpan = simpleCache.startReadWrite(KEY_1, 0); + assertTrue(cacheSpan.isCached); + assertFalse(cacheSpan.isOpenEnded()); + assertEquals(15, cacheSpan.length); + } + + public void testSetGetLength() throws Exception { + SimpleCache simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor()); + + assertEquals(C.LENGTH_UNSET, simpleCache.getContentLength(KEY_1)); + assertTrue(simpleCache.setContentLength(KEY_1, 15)); + assertEquals(15, simpleCache.getContentLength(KEY_1)); + + simpleCache.startReadWrite(KEY_1, 0); + + addCache(simpleCache, 0, 15); + + assertTrue(simpleCache.setContentLength(KEY_1, 150)); + assertEquals(150, simpleCache.getContentLength(KEY_1)); + + addCache(simpleCache, 140, 10); + + // Try to set length shorter then the content + assertFalse(simpleCache.setContentLength(KEY_1, 15)); + assertEquals("Content length should be unchanged.", + 150, simpleCache.getContentLength(KEY_1)); + + /* TODO Enable when the length persistance is fixed + // Check if values are kept after cache is reloaded. + simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor()); + assertEquals(150, simpleCache.getContentLength(KEY_1)); + CacheSpan lastSpan = simpleCache.startReadWrite(KEY_1, 145); + + // Removing the last span shouldn't cause the length be change next time cache loaded + simpleCache.removeSpan(lastSpan); + simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor()); + assertEquals(150, simpleCache.getContentLength(KEY_1)); + */ + } + + private void addCache(SimpleCache simpleCache, int position, int length) throws IOException { + File file = simpleCache.startFile(KEY_1, position, length); + FileOutputStream fos = new FileOutputStream(file); + fos.write(new byte[length]); + fos.close(); + simpleCache.commitFile(file); + } + +} diff --git a/library/src/main/java/com/google/android/exoplayer2/upstream/cache/Cache.java b/library/src/main/java/com/google/android/exoplayer2/upstream/cache/Cache.java index 7f84e97a6e7..a8a8de43611 100644 --- a/library/src/main/java/com/google/android/exoplayer2/upstream/cache/Cache.java +++ b/library/src/main/java/com/google/android/exoplayer2/upstream/cache/Cache.java @@ -143,11 +143,11 @@ interface Listener { * * @param key The cache key for the data. * @param position The starting position of the data. - * @param length The length of the data to be written. Used only to ensure that there is enough - * space in the cache. + * @param maxLength The maximum length of the data to be written. Used only to ensure that there + * is enough space in the cache. * @return The file into which data should be written. */ - File startFile(String key, long position, long length); + File startFile(String key, long position, long maxLength); /** * Commits a file into the cache. Must only be called when holding a corresponding hole @@ -182,4 +182,22 @@ interface Listener { */ boolean isCached(String key, long position, long length); + /** + * Sets the content length for the given key. + * + * @param key The cache key for the data. + * @param length The length of the data. + * @return Whether the length was set successfully. Returns false if the length conflicts with the + * existing contents of the cache. + */ + boolean setContentLength(String key, long length); + + /** + * Returns the content length for the given key if one set, or {@link + * com.google.android.exoplayer2.C#LENGTH_UNSET} otherwise. + * + * @param key The cache key for the data. + */ + long getContentLength(String key); + } diff --git a/library/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheDataSink.java b/library/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheDataSink.java index 1f4abd47415..96c198b4c9f 100644 --- a/library/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheDataSink.java +++ b/library/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheDataSink.java @@ -64,12 +64,12 @@ public CacheDataSink(Cache cache, long maxCacheFileSize) { @Override public void open(DataSpec dataSpec) throws CacheDataSinkException { - // TODO: Support caching for unbounded requests. See TODO in {@link CacheDataSource} for - // more details. - Assertions.checkState(dataSpec.length != C.LENGTH_UNSET); + this.dataSpec = dataSpec; + if (dataSpec.length == C.LENGTH_UNSET) { + return; + } + dataSpecBytesWritten = 0; try { - this.dataSpec = dataSpec; - dataSpecBytesWritten = 0; openNextOutputStream(); } catch (FileNotFoundException e) { throw new CacheDataSinkException(e); @@ -78,6 +78,9 @@ public void open(DataSpec dataSpec) throws CacheDataSinkException { @Override public void write(byte[] buffer, int offset, int length) throws CacheDataSinkException { + if (dataSpec.length == C.LENGTH_UNSET) { + return; + } try { int bytesWritten = 0; while (bytesWritten < length) { @@ -99,6 +102,9 @@ public void write(byte[] buffer, int offset, int length) throws CacheDataSinkExc @Override public void close() throws CacheDataSinkException { + if (dataSpec == null || dataSpec.length == C.LENGTH_UNSET) { + return; + } try { closeCurrentOutputStream(); } catch (IOException e) { diff --git a/library/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheDataSource.java b/library/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheDataSource.java index 84d300f5c20..72e8df3ce3a 100644 --- a/library/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheDataSource.java +++ b/library/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheDataSource.java @@ -22,6 +22,7 @@ import com.google.android.exoplayer2.upstream.DataSource; import com.google.android.exoplayer2.upstream.DataSpec; import com.google.android.exoplayer2.upstream.FileDataSource; +import com.google.android.exoplayer2.upstream.HttpDataSource.InvalidResponseCodeException; import com.google.android.exoplayer2.upstream.TeeDataSource; import com.google.android.exoplayer2.upstream.cache.CacheDataSink.CacheDataSinkException; import java.io.IOException; @@ -34,6 +35,34 @@ */ public final class CacheDataSource implements DataSource { + /** + * Default maximum single cache file size. + * + * @see #CacheDataSource(Cache, DataSource, int) + * @see #CacheDataSource(Cache, DataSource, int, long) + */ + public static final long DEFAULT_MAX_CACHE_FILE_SIZE = 2 * 1024 * 1024; + + /** + * A flag indicating whether we will block reads if the cache key is locked. If this flag is + * set, then we will read from upstream if the cache key is locked. + */ + public static final int FLAG_BLOCK_ON_CACHE = 1 << 0; + + /** + * A flag indicating whether the cache is bypassed following any cache related error. If set + * then cache related exceptions may be thrown for one cycle of open, read and close calls. + * Subsequent cycles of these calls will then bypass the cache. + */ + public static final int FLAG_IGNORE_CACHE_ON_ERROR = 1 << 1; + + /** + * A flag indicating whether the response is cached if the range of the request is unbounded. + * Disabled by default because, as a side effect, this may allow streams with every chunk from a + * separate URL cached which is broken currently. + */ + public static final int FLAG_CACHE_UNBOUNDED_REQUESTS = 1 << 2; + /** * Listener of {@link CacheDataSource} events. */ @@ -59,35 +88,44 @@ public interface EventListener { private final boolean blockOnCache; private final boolean ignoreCacheOnError; + private final boolean bypassUnboundedRequests; private DataSource currentDataSource; + private boolean currentRequestUnbounded; private Uri uri; private int flags; private String key; private long readPosition; private long bytesRemaining; private CacheSpan lockedSpan; - private boolean ignoreCache; + private boolean seenCacheError; + private boolean currentRequestIgnoresCache; private long totalCachedBytesRead; /** * Constructs an instance with default {@link DataSource} and {@link DataSink} instances for - * reading and writing the cache. + * reading and writing the cache and with {@link #DEFAULT_MAX_CACHE_FILE_SIZE}. */ - public CacheDataSource(Cache cache, DataSource upstream, boolean blockOnCache, - boolean ignoreCacheOnError) { - this(cache, upstream, blockOnCache, ignoreCacheOnError, Long.MAX_VALUE); + public CacheDataSource(Cache cache, DataSource upstream, int flags) { + this(cache, upstream, flags, DEFAULT_MAX_CACHE_FILE_SIZE); } /** * Constructs an instance with default {@link DataSource} and {@link DataSink} instances for * reading and writing the cache. The sink is configured to fragment data such that no single * cache file is greater than maxCacheFileSize bytes. + * + * @param cache The cache. + * @param upstream A {@link DataSource} for reading data not in the cache. + * @param flags A combination of {@link #FLAG_BLOCK_ON_CACHE}, {@link #FLAG_IGNORE_CACHE_ON_ERROR} + * and {@link #FLAG_CACHE_UNBOUNDED_REQUESTS} or 0. + * @param maxCacheFileSize The maximum size of a cache file, in bytes. If the cached data size + * exceeds this value, then the data will be fragmented into multiple cache files. The + * finer-grained this is the finer-grained the eviction policy can be. */ - public CacheDataSource(Cache cache, DataSource upstream, boolean blockOnCache, - boolean ignoreCacheOnError, long maxCacheFileSize) { + public CacheDataSource(Cache cache, DataSource upstream, int flags, long maxCacheFileSize) { this(cache, upstream, new FileDataSource(), new CacheDataSink(cache, maxCacheFileSize), - blockOnCache, ignoreCacheOnError, null); + flags, null); } /** @@ -99,20 +137,17 @@ public CacheDataSource(Cache cache, DataSource upstream, boolean blockOnCache, * @param upstream A {@link DataSource} for reading data not in the cache. * @param cacheReadDataSource A {@link DataSource} for reading data from the cache. * @param cacheWriteDataSink A {@link DataSink} for writing data to the cache. - * @param blockOnCache A flag indicating whether we will block reads if the cache key is locked. - * If this flag is false, then we will read from upstream if the cache key is locked. - * @param ignoreCacheOnError Whether the cache is bypassed following any cache related error. If - * true, then cache related exceptions may be thrown for one cycle of open, read and close - * calls. Subsequent cycles of these calls will then bypass the cache. + * @param flags A combination of {@link #FLAG_BLOCK_ON_CACHE}, {@link #FLAG_IGNORE_CACHE_ON_ERROR} + * and {@link #FLAG_CACHE_UNBOUNDED_REQUESTS} or 0. * @param eventListener An optional {@link EventListener} to receive events. */ public CacheDataSource(Cache cache, DataSource upstream, DataSource cacheReadDataSource, - DataSink cacheWriteDataSink, boolean blockOnCache, boolean ignoreCacheOnError, - EventListener eventListener) { + DataSink cacheWriteDataSink, int flags, EventListener eventListener) { this.cache = cache; this.cacheReadDataSource = cacheReadDataSource; - this.blockOnCache = blockOnCache; - this.ignoreCacheOnError = ignoreCacheOnError; + this.blockOnCache = (flags & FLAG_BLOCK_ON_CACHE) != 0; + this.ignoreCacheOnError = (flags & FLAG_IGNORE_CACHE_ON_ERROR) != 0; + this.bypassUnboundedRequests = (flags & FLAG_CACHE_UNBOUNDED_REQUESTS) == 0; this.upstreamDataSource = upstream; if (cacheWriteDataSink != null) { this.cacheWriteDataSource = new TeeDataSource(upstream, cacheWriteDataSink); @@ -129,9 +164,18 @@ public long open(DataSpec dataSpec) throws IOException { flags = dataSpec.flags; key = dataSpec.key; readPosition = dataSpec.position; - bytesRemaining = dataSpec.length; - openNextSource(); - return dataSpec.length; + currentRequestIgnoresCache = (ignoreCacheOnError && seenCacheError) + || (bypassUnboundedRequests && dataSpec.length == C.LENGTH_UNSET); + if (dataSpec.length != C.LENGTH_UNSET || currentRequestIgnoresCache) { + bytesRemaining = dataSpec.length; + } else { + bytesRemaining = cache.getContentLength(key); + if (bytesRemaining != C.LENGTH_UNSET) { + bytesRemaining -= dataSpec.position; + } + } + openNextSource(true); + return bytesRemaining; } catch (IOException e) { handleBeforeThrow(e); throw e; @@ -151,10 +195,17 @@ public int read(byte[] buffer, int offset, int max) throws IOException { bytesRemaining -= bytesRead; } } else { + if (currentRequestUnbounded) { + // We only do unbounded requests to upstream and only when we don't know the actual stream + // length. So we reached the end of stream. + setContentLength(readPosition); + bytesRemaining = 0; + } closeCurrentSource(); - if (bytesRemaining > 0 && bytesRemaining != C.LENGTH_UNSET) { - openNextSource(); - return read(buffer, offset, max); + if (bytesRemaining > 0 || bytesRemaining == C.LENGTH_UNSET) { + if (openNextSource(false)) { + return read(buffer, offset, max); + } } } return bytesRead; @@ -185,16 +236,12 @@ public void close() throws IOException { * Opens the next source. If the cache contains data spanning the current read position then * {@link #cacheReadDataSource} is opened to read from it. Else {@link #upstreamDataSource} is * opened to read from the upstream source and write into the cache. + * @param initial Whether it is the initial open call. */ - private void openNextSource() throws IOException { + private boolean openNextSource(boolean initial) throws IOException { DataSpec dataSpec; CacheSpan span; - if (ignoreCache) { - span = null; - } else if (bytesRemaining == C.LENGTH_UNSET) { - // TODO: Support caching for unbounded requests. This requires storing the source length - // into the cache (the simplest approach is to incorporate it into each cache file's name). - Log.w(TAG, "Cache bypassed due to unbounded length."); + if (currentRequestIgnoresCache) { span = null; } else if (blockOnCache) { try { @@ -205,6 +252,7 @@ private void openNextSource() throws IOException { } else { span = cache.startReadWriteNonBlocking(key, readPosition); } + if (span == null) { // The data is locked in the cache, or we're ignoring the cache. Bypass the cache and read // from upstream. @@ -214,18 +262,63 @@ private void openNextSource() throws IOException { // Data is cached, read from cache. Uri fileUri = Uri.fromFile(span.file); long filePosition = readPosition - span.position; - long length = Math.min(span.length - filePosition, bytesRemaining); + long length = span.length - filePosition; + if (bytesRemaining != C.LENGTH_UNSET) { + length = Math.min(length, bytesRemaining); + } dataSpec = new DataSpec(fileUri, readPosition, filePosition, length, key, flags); currentDataSource = cacheReadDataSource; } else { // Data is not cached, and data is not locked, read from upstream with cache backing. lockedSpan = span; - long length = span.isOpenEnded() ? bytesRemaining : Math.min(span.length, bytesRemaining); + long length; + if (span.isOpenEnded()) { + length = bytesRemaining; + } else { + length = span.length; + if (bytesRemaining != C.LENGTH_UNSET) { + length = Math.min(length, bytesRemaining); + } + } dataSpec = new DataSpec(uri, readPosition, length, key, flags); currentDataSource = cacheWriteDataSource != null ? cacheWriteDataSource : upstreamDataSource; } - currentDataSource.open(dataSpec); + + currentRequestUnbounded = dataSpec.length == C.LENGTH_UNSET; + boolean successful = false; + long currentBytesRemaining; + try { + currentBytesRemaining = currentDataSource.open(dataSpec); + successful = true; + } catch (InvalidResponseCodeException e) { + // if this isn't the initial open call (we had read some bytes) and got an 'unsatisfiable + // byte-range' (416) response for an unbounded range request then mute the exception. We are + // trying to find the stream end. + if (!initial && e.responseCode == 416 && currentRequestUnbounded) { + currentBytesRemaining = 0; + } else { + throw e; + } + } + + // If we did an unbounded request (which means bytesRemaining == C.LENGTH_UNSET) and got a + // resolved length from open() request + if (currentRequestUnbounded && currentBytesRemaining != C.LENGTH_UNSET) { + bytesRemaining = currentBytesRemaining; + // If writing into cache + if (lockedSpan != null) { + setContentLength(dataSpec.position + bytesRemaining); + } + } + return successful; + } + + private void setContentLength(long length) { + if (!cache.setContentLength(key, length)) { + Log.e(TAG, "cache.setContentLength(" + length + ") failed. cache.getContentLength() = " + + cache.getContentLength(key)); + } } private void closeCurrentSource() throws IOException { @@ -235,6 +328,7 @@ private void closeCurrentSource() throws IOException { try { currentDataSource.close(); currentDataSource = null; + currentRequestUnbounded = false; } finally { if (lockedSpan != null) { cache.releaseHoleSpan(lockedSpan); @@ -244,10 +338,8 @@ private void closeCurrentSource() throws IOException { } private void handleBeforeThrow(IOException exception) { - if (ignoreCacheOnError && (currentDataSource == cacheReadDataSource - || exception instanceof CacheDataSinkException)) { - // Ignore the cache from now on. - ignoreCache = true; + if (currentDataSource == cacheReadDataSource || exception instanceof CacheDataSinkException) { + seenCacheError = true; } } diff --git a/library/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheEvictor.java b/library/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheEvictor.java index 665039f9393..627bb7e2f45 100644 --- a/library/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheEvictor.java +++ b/library/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheEvictor.java @@ -32,8 +32,8 @@ public interface CacheEvictor extends Cache.Listener { * @param cache The source of the event. * @param key The key being written. * @param position The starting position of the data being written. - * @param length The maximum length of the data being written. + * @param maxLength The maximum length of the data being written. */ - void onStartFile(Cache cache, String key, long position, long length); + void onStartFile(Cache cache, String key, long position, long maxLength); } diff --git a/library/src/main/java/com/google/android/exoplayer2/upstream/cache/LeastRecentlyUsedCacheEvictor.java b/library/src/main/java/com/google/android/exoplayer2/upstream/cache/LeastRecentlyUsedCacheEvictor.java index 392df970ed3..791fb677f13 100644 --- a/library/src/main/java/com/google/android/exoplayer2/upstream/cache/LeastRecentlyUsedCacheEvictor.java +++ b/library/src/main/java/com/google/android/exoplayer2/upstream/cache/LeastRecentlyUsedCacheEvictor.java @@ -39,8 +39,8 @@ public void onCacheInitialized() { } @Override - public void onStartFile(Cache cache, String key, long position, long length) { - evictCache(cache, length); + public void onStartFile(Cache cache, String key, long position, long maxLength) { + evictCache(cache, maxLength); } @Override diff --git a/library/src/main/java/com/google/android/exoplayer2/upstream/cache/NoOpCacheEvictor.java b/library/src/main/java/com/google/android/exoplayer2/upstream/cache/NoOpCacheEvictor.java index 9d5520fb58b..b0c8c7e0876 100644 --- a/library/src/main/java/com/google/android/exoplayer2/upstream/cache/NoOpCacheEvictor.java +++ b/library/src/main/java/com/google/android/exoplayer2/upstream/cache/NoOpCacheEvictor.java @@ -30,7 +30,7 @@ public void onCacheInitialized() { } @Override - public void onStartFile(Cache cache, String key, long position, long length) { + public void onStartFile(Cache cache, String key, long position, long maxLength) { // Do nothing. } diff --git a/library/src/main/java/com/google/android/exoplayer2/upstream/cache/SimpleCache.java b/library/src/main/java/com/google/android/exoplayer2/upstream/cache/SimpleCache.java index 0def07c5375..9128ba99ca6 100644 --- a/library/src/main/java/com/google/android/exoplayer2/upstream/cache/SimpleCache.java +++ b/library/src/main/java/com/google/android/exoplayer2/upstream/cache/SimpleCache.java @@ -16,6 +16,9 @@ package com.google.android.exoplayer2.upstream.cache; import android.os.ConditionVariable; + +import android.util.Pair; +import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.util.Assertions; import java.io.File; import java.util.ArrayList; @@ -35,7 +38,7 @@ public final class SimpleCache implements Cache { private final File cacheDir; private final CacheEvictor evictor; private final HashMap lockedSpans; - private final HashMap> cachedSpans; + private final HashMap>> cachedSpans; private final HashMap> listeners; private long totalSpace = 0; @@ -89,7 +92,7 @@ public synchronized void removeListener(String key, Listener listener) { @Override public synchronized NavigableSet getCachedSpans(String key) { - TreeSet spansForKey = cachedSpans.get(key); + TreeSet spansForKey = getSpansForKey(key); return spansForKey == null ? null : new TreeSet<>(spansForKey); } @@ -127,26 +130,21 @@ public synchronized CacheSpan startReadWriteNonBlocking(String key, long positio } private synchronized CacheSpan startReadWriteNonBlocking(CacheSpan lookupSpan) { - CacheSpan spanningRegion = getSpan(lookupSpan); + CacheSpan cacheSpan = getSpan(lookupSpan); // Read case. - if (spanningRegion.isCached) { - CacheSpan oldCacheSpan = spanningRegion; - // Remove the old span from the in-memory representation. - TreeSet spansForKey = cachedSpans.get(oldCacheSpan.key); - Assertions.checkState(spansForKey.remove(oldCacheSpan)); + if (cacheSpan.isCached) { // Obtain a new span with updated last access timestamp. - spanningRegion = oldCacheSpan.touch(); - // Add the updated span back into the in-memory representation. - spansForKey.add(spanningRegion); - notifySpanTouched(oldCacheSpan, spanningRegion); - return spanningRegion; + CacheSpan newCacheSpan = cacheSpan.touch(); + replaceSpan(cacheSpan, newCacheSpan); + notifySpanTouched(cacheSpan, newCacheSpan); + return newCacheSpan; } // Write case, lock available. if (!lockedSpans.containsKey(lookupSpan.key)) { - lockedSpans.put(lookupSpan.key, spanningRegion); - return spanningRegion; + lockedSpans.put(lookupSpan.key, cacheSpan); + return cacheSpan; } // Write case, lock not available. @@ -154,14 +152,14 @@ private synchronized CacheSpan startReadWriteNonBlocking(CacheSpan lookupSpan) { } @Override - public synchronized File startFile(String key, long position, long length) { + public synchronized File startFile(String key, long position, long maxLength) { Assertions.checkState(lockedSpans.containsKey(key)); if (!cacheDir.exists()) { // For some reason the cache directory doesn't exist. Make a best effort to create it. removeStaleSpans(); cacheDir.mkdirs(); } - evictor.onStartFile(this, key, position, length); + evictor.onStartFile(this, key, position, maxLength); return CacheSpan.getCacheFileName(cacheDir, key, position, System.currentTimeMillis(), false); } @@ -175,11 +173,15 @@ public synchronized void commitFile(File file) { return; } // If the file has length 0, delete it and don't add it to the in-memory representation. - long length = file.length(); - if (length == 0) { + if (file.length() == 0) { file.delete(); return; } + // Check if the span conflicts with the set content length + Long length = getContentLength(span.key); + if (length != C.LENGTH_UNSET) { + Assertions.checkState((span.position + span.length) <= length); + } addSpan(span); notifyAll(); } @@ -204,7 +206,7 @@ public synchronized void releaseHoleSpan(CacheSpan holeSpan) { private CacheSpan getSpan(CacheSpan lookupSpan) { String key = lookupSpan.key; long offset = lookupSpan.position; - TreeSet entries = cachedSpans.get(key); + TreeSet entries = getSpansForKey(key); if (entries == null) { return CacheSpan.createOpenHole(key, lookupSpan.position); } @@ -260,10 +262,13 @@ private void initialize() { * @param span The span to be added. */ private void addSpan(CacheSpan span) { - TreeSet spansForKey = cachedSpans.get(span.key); - if (spansForKey == null) { + Pair> entryForKey = cachedSpans.get(span.key); + TreeSet spansForKey; + if (entryForKey == null) { spansForKey = new TreeSet<>(); - cachedSpans.put(span.key, spansForKey); + setKeyValue(span.key, C.LENGTH_UNSET, spansForKey); + } else { + spansForKey = entryForKey.second; } spansForKey.add(span); totalSpace += span.length; @@ -272,7 +277,7 @@ private void addSpan(CacheSpan span) { @Override public synchronized void removeSpan(CacheSpan span) { - TreeSet spansForKey = cachedSpans.get(span.key); + TreeSet spansForKey = getSpansForKey(span.key); totalSpace -= span.length; Assertions.checkState(spansForKey.remove(span)); span.file.delete(); @@ -287,10 +292,11 @@ public synchronized void removeSpan(CacheSpan span) { * no longer exist. */ private void removeStaleSpans() { - Iterator>> iterator = cachedSpans.entrySet().iterator(); + Iterator>>> iterator = + cachedSpans.entrySet().iterator(); while (iterator.hasNext()) { - Entry> next = iterator.next(); - Iterator spanIterator = next.getValue().iterator(); + Entry>> next = iterator.next(); + Iterator spanIterator = next.getValue().second.iterator(); boolean isEmpty = true; while (spanIterator.hasNext()) { CacheSpan span = spanIterator.next(); @@ -342,7 +348,7 @@ private void notifySpanTouched(CacheSpan oldSpan, CacheSpan newSpan) { @Override public synchronized boolean isCached(String key, long position, long length) { - TreeSet entries = cachedSpans.get(key); + TreeSet entries = getSpansForKey(key); if (entries == null) { return false; } @@ -375,4 +381,49 @@ public synchronized boolean isCached(String key, long position, long length) { return false; } + @Override + public synchronized boolean setContentLength(String key, long length) { + Pair> entryForKey = cachedSpans.get(key); + TreeSet entries; + if (entryForKey != null) { + entries = entryForKey.second; + if (entries != null && !entries.isEmpty()) { + CacheSpan last = entries.last(); + long end = last.position + last.length; + if (end > length) { + return false; + } + } + } else { + entries = new TreeSet<>(); + } + // TODO persist the length value + setKeyValue(key, length, entries); + return true; + } + + @Override + public synchronized long getContentLength(String key) { + Pair> entryForKey = cachedSpans.get(key); + return entryForKey == null ? C.LENGTH_UNSET : entryForKey.first; + } + + + private TreeSet getSpansForKey(String key) { + Pair> entryForKey = cachedSpans.get(key); + return entryForKey != null ? entryForKey.second : null; + } + + private void setKeyValue(String key, long length, TreeSet entries) { + cachedSpans.put(key, Pair.create(length, entries)); + } + + private void replaceSpan(CacheSpan oldSpan, CacheSpan newSpan) { + // Remove the old span from the in-memory representation. + TreeSet spansForKey = getSpansForKey(oldSpan.key); + Assertions.checkState(spansForKey.remove(oldSpan)); + // Add the updated span back into the in-memory representation. + spansForKey.add(newSpan); + } + } diff --git a/testutils/src/main/java/com/google/android/exoplayer2/testutil/FakeDataSource.java b/testutils/src/main/java/com/google/android/exoplayer2/testutil/FakeDataSource.java index d6c64916b46..7a5a0d3773a 100644 --- a/testutils/src/main/java/com/google/android/exoplayer2/testutil/FakeDataSource.java +++ b/testutils/src/main/java/com/google/android/exoplayer2/testutil/FakeDataSource.java @@ -49,6 +49,7 @@ public final class FakeDataSource implements DataSource { private boolean opened; private int currentSegmentIndex; private long bytesRemaining; + private IOException unsatisfiableRangeException; private FakeDataSource(boolean simulateUnknownLength, ArrayList segments) { this.simulateUnknownLength = simulateUnknownLength; @@ -59,6 +60,7 @@ private FakeDataSource(boolean simulateUnknownLength, ArrayList segment } this.totalLength = totalLength; openedDataSpecs = new ArrayList<>(); + unsatisfiableRangeException = new IOException("Unsatisfiable range"); } @Override @@ -69,11 +71,9 @@ public long open(DataSpec dataSpec) throws IOException { uri = dataSpec.uri; openedDataSpecs.add(dataSpec); // If the source knows that the request is unsatisfiable then fail. - if (dataSpec.position >= totalLength) { - throw new IOException("Unsatisfiable position"); - } else if (dataSpec.length != C.LENGTH_UNSET - && dataSpec.position + dataSpec.length > totalLength) { - throw new IOException("Unsatisfiable range"); + if (dataSpec.position >= totalLength || (dataSpec.length != C.LENGTH_UNSET + && (dataSpec.position + dataSpec.length > totalLength))) { + throw (IOException) unsatisfiableRangeException.fillInStackTrace(); } // Scan through the segments, configuring them for the current read. boolean findingCurrentSegmentIndex = true; @@ -107,10 +107,10 @@ public int read(byte[] buffer, int offset, int readLength) throws IOException { return C.RESULT_END_OF_INPUT; } Segment current = segments.get(currentSegmentIndex); - if (current.exception != null) { + if (current.isErrorSegment()) { if (!current.exceptionCleared) { current.exceptionThrown = true; - throw current.exception; + throw (IOException) current.exception.fillInStackTrace(); } else { currentSegmentIndex++; } @@ -160,6 +160,10 @@ public DataSpec[] getAndClearOpenedDataSpecs() { return dataSpecs; } + public void setUnsatisfiableRangeException(IOException unsatisfiableRangeException) { + this.unsatisfiableRangeException = unsatisfiableRangeException; + } + private static class Segment { public final IOException exception; diff --git a/testutils/src/main/java/com/google/android/exoplayer2/testutil/TestUtil.java b/testutils/src/main/java/com/google/android/exoplayer2/testutil/TestUtil.java index 110e5f7a98b..7b880627188 100644 --- a/testutils/src/main/java/com/google/android/exoplayer2/testutil/TestUtil.java +++ b/testutils/src/main/java/com/google/android/exoplayer2/testutil/TestUtil.java @@ -24,6 +24,7 @@ import com.google.android.exoplayer2.testutil.FakeExtractorInput.SimulatedIOException; import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.Util; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.Arrays; @@ -303,4 +304,13 @@ && assetExists(instrumentation, sampleFile + UNKNOWN_LENGTH_EXTENSION)) { return extractorOutput; } + public static void recursiveDelete(File fileOrDirectory) { + if (fileOrDirectory.isDirectory()) { + for (File child : fileOrDirectory.listFiles()) { + recursiveDelete(child); + } + } + fileOrDirectory.delete(); + } + }