First checkin for the implementation of Byte Ranges in Readium SDK (W…

…ORK IN PROGRESS)

This is the first checkin for the implementation of Byte Ranges in Readium SDK. In short, the objective of adding Byte Ranges
to Readium SDK is so that there is a way to deal with resources of great size inside an ePUB. For example, you may have an ePUB
file with a 1 GB video file in it, and in that case you want to allow the client app to be able to read just chunks of bytes out of that video,
and not have to put the entire 1 GB video in memory.

This is still a work in progress. There are still code changes that we need to make, plus some more code cleanup that we want to do,
before marking this as concluded. Following checkins will take care of that.

Platform/Apple/RDServices/Main/RDPackage.mm

@@ -302,13 +302,32 @@ - (RDPackageResource *)resourceAtRelativePath:(NSString *)relativePath {
 	}
 
 	ePub3::string s = ePub3::string(relativePath.UTF8String);
-	std::unique_ptr<ePub3::ByteStream> byteStream = m_package->ReadStreamForRelativePath(s);
+	ePub3::ConstManifestItemPtr manifestItem = m_package->ManifestItemAtRelativePath(s);
 
-	if (byteStream == nullptr) {
-		NSLog(@"Relative path '%@' does not have a byte stream!", relativePath);
+	if (manifestItem == nullptr) {
+		NSLog(@"Relative path '%@' does not have a manifest item!", relativePath);
 		return nil;
 	}
 
+	ePub3::ManifestItemPtr m = std::const_pointer_cast<ePub3::ManifestItem>(manifestItem);
+    ePub3::ByteStreamPtr byteStream = m_package->SyncContentStreamForItem(m);
+    if (byteStream == nullptr)
+    {
+        NSLog(@"Relative path '%@' does not have a byte stream!", relativePath);
+        return nil;
+    }
+    /*
+    TODO: uncomment and enable the following block of code. This piece of code will allow
+          the use of Byte Ranges. For resources above a given size, a ByteRangeFilterSyncStream
+          will be returned, which allows reading just ranges of bytes from a given resource.
+          However, currently this block of code is currently disabled because we are seeing
+          crashes when playing a Quicktime video using byte ranges.
+    */
+    /* if (byteStream->BytesAvailable() > 1000000)
+    {
+        byteStream = m_package->SyncByteRangeForItem(m);
+    } */
+
 	RDPackageResource *resource = [[RDPackageResource alloc]
 		initWithDelegate:self
 		byteStream:byteStream.get()
@@ -317,12 +336,8 @@ - (RDPackageResource *)resourceAtRelativePath:(NSString *)relativePath {
 
 	if (resource != nil) {
 		m_byteStreamVector.push_back(std::move(byteStream));
-		ePub3::ConstManifestItemPtr item = m_package->ManifestItemAtRelativePath(s);
-
-		if (item) {
-			const ePub3::ManifestItem::MimeType &mediaType = item->MediaType();
-			resource.mimeType = [NSString stringWithUTF8String:mediaType.c_str()];
-		}
+		const ePub3::ManifestItem::MimeType &mediaType = manifestItem->MediaType();
+		resource.mimeType = [NSString stringWithUTF8String:mediaType.c_str()];
 	}
 
 	return resource;

Platform/Apple/RDServices/Main/RDPackageResource.h

@@ -63,5 +63,6 @@
 
 - (NSData *)readDataOfLength:(NSUInteger)length;
 - (void)setOffset:(UInt64)offset;
+- (BOOL)isByteRangeResource;
 
 @end

Platform/Apple/RDServices/Main/RDPackageResource.mm

@@ -29,6 +29,8 @@
 
 #import "RDPackageResource.h"
 #import <ePub3/archive.h>
+#import <ePub3/filter.h>
+#import <ePub3/filter_chain.h>
 #import <ePub3/package.h>
 #import <ePub3/utilities/byte_stream.h>
 #import "RDPackage.h"
@@ -37,6 +39,7 @@
 @interface RDPackageResource() {
 	@private ePub3::ByteStream *m_byteStream;
 	@private NSUInteger m_contentLength;
+	@private UInt64 m_offset;
 }
 
 @end
@@ -105,19 +108,31 @@ - (void)dealloc {
 
 
 - (NSData *)readDataOfLength:(NSUInteger)length {
-	NSMutableData *md = [[NSMutableData alloc] initWithCapacity:length == 0 ? 1 : length];
-	NSUInteger totalRead = 0;
-
-	while (totalRead < length) {
-		NSUInteger thisLength = MIN(sizeof(m_buffer), length - totalRead);
-		std::size_t count = m_byteStream->ReadBytes(m_buffer, thisLength);
-		totalRead += count;
-		[md appendBytes:m_buffer length:count];
-
-		if (count != thisLength) {
-			NSLog(@"Did not read the expected number of bytes! (%lu %lu)",
-				count, (unsigned long)thisLength);
-			break;
+	NSMutableData *md = [[NSMutableData alloc] initWithCapacity:length];
+
+	ePub3::ByteRangeFilterSyncStream *filterStream =
+		dynamic_cast<ePub3::ByteRangeFilterSyncStream *>(m_byteStream);
+
+	if (filterStream == nullptr) {
+		NSLog(@"The byte stream is not a FilterChainSyncStream!");
+	}
+	else {
+		ePub3::ByteRange range;
+		range.Location(m_offset);
+		NSUInteger totalRead = 0;
+
+		while (totalRead < length) {
+			range.Length(MIN(sizeof(m_buffer), length - totalRead));
+			std::size_t count = filterStream->ReadBytes(m_buffer, sizeof(m_buffer), range);
+			[md appendBytes:m_buffer length:count];
+			totalRead += count;
+            range.Location(range.Location() + count);
+
+			if (count != range.Length()) {
+				NSLog(@"Did not read the expected number of bytes! (%lu %lu)",
+					count, (unsigned long)range.Length());
+				break;
+			}
 		}
 	}
 
@@ -126,12 +141,13 @@ - (NSData *)readDataOfLength:(NSUInteger)length {
 
 
 - (void)setOffset:(UInt64)offset {
-	ePub3::SeekableByteStream* seekStream = dynamic_cast<ePub3::SeekableByteStream*>(m_byteStream);
-	ePub3::ByteStream::size_type pos = seekStream->Seek(offset, std::ios::beg);
+	m_offset = offset;
+}
 
-	if (pos != offset) {
-		NSLog(@"Setting the byte stream offset failed! pos = %lu, offset = %llu", pos, offset);
-	}
+- (BOOL)isByteRangeResource
+{
+    ePub3::ByteRangeFilterSyncStream *filterStream = dynamic_cast<ePub3::ByteRangeFilterSyncStream *>(m_byteStream);
+    return (filterStream != nullptr);
 }
 
 

Platform/Apple/RDServices/Main/RDPackageResourceConnection.m

@@ -69,7 +69,7 @@ @implementation RDPackageResourceConnection
 		if (resource == nil) {
 			NSLog(@"No resource found! (%@)", path);
 		}
-		else if (resource.contentLength < 1000000) {
+        else if (!resource.isByteRangeResource) {
 
 			// This resource is small enough that we can just fetch the entire thing in memory,
 			// which simplifies access into the byte stream.  Adjust the threshold to taste.

ePub3/ePub/filter.h

@@ -39,6 +39,50 @@ typedef std::shared_ptr<Package>    PackagePtr;
 class ContentFilter;
 typedef std::shared_ptr<ContentFilter>  ContentFilterPtr;
 
+// -------------------------------------------------------------------------------------------
+
+class ByteRange
+{
+public:
+    ByteRange()
+    {
+        Reset();
+    }
+
+    uint32_t Location() const { return m_location; }
+    void Location(uint32_t location) { m_isFullRange = false; m_location = location; }
+    uint32_t Length() const { return m_length; }
+    void Length(uint32_t length) { m_isFullRange = false; m_length = length; }
+    bool IsFullRange() const { return m_isFullRange; }
+    void Reset() { m_location = 0; m_length = 0; m_isFullRange = true; }
+
+    ByteRange &operator=(const ByteRange &b)
+    {
+        if (b.m_isFullRange)
+        {
+            Reset();
+        }
+        else
+        {
+            m_location = b.m_location;
+            m_length = b.m_length;
+            m_isFullRange = false;
+        }
+        return (*this);
+    }
+
+private:
+    ByteRange(const ByteRange &b) _DELETED_; // Delete copy constructor
+    ByteRange(ByteRange &&b) _DELETED_; // Delete move constructor
+    ByteRange &operator=(ByteRange &&b) _DELETED_; // Delete move assignment operator
+
+    uint32_t m_location;
+    uint32_t m_length;
+    bool m_isFullRange;
+};
+
+// -------------------------------------------------------------------------------------------
+
 /**
  The FilterContext abstract class can be extended by individual filters to hold
  data unique to each pass across a single stream of data.
@@ -57,10 +101,44 @@ typedef std::shared_ptr<ContentFilter>  ContentFilterPtr;
 class FilterContext
 {
 public:
-    FilterContext() {}
-    virtual ~FilterContext() {}
+    FilterContext() { }
+    virtual ~FilterContext() { }
+};
+
+// -------------------------------------------------------------------------------------------
+
+class SeekableByteStream;
+
+/**
+ The RangeFilterContext abstract class is an extension of the FilterContext class, and it is
+ used to pass data needed for extracting ranges of bytes from a given resource.
+ 
+ As you may imagine, one piece of data that we need to pass is the range of bytes that is being
+ requested. That is passed by the ByteRange object that it is included in this class. In
+ addition to that, the FilterContext object will need direct access to the ZIP file so that it
+ can read only the pieces that it needs (after all, the whole idea was that we didn't want to
+ cram a whole 1 GB video file in memory). That reference is kept the m_byteStream member
+ variable.
+ */
+
+class RangeFilterContext : public FilterContext
+{
+public:
+    RangeFilterContext() : FilterContext(), m_byteStream(nullptr) { }
+    virtual ~RangeFilterContext() { }
+
+    ByteRange &GetByteRange() { return m_byteRange; }
+    void SetSeekableByteStream(SeekableByteStream *byteStream) { m_byteStream = byteStream; }
+    SeekableByteStream *GetSeekableByteStream() const { return m_byteStream; }
+    void ResetSeekableByteStream() { m_byteStream = nullptr; }
+
+private:
+    ByteRange m_byteRange;
+    SeekableByteStream *m_byteStream;
 };
 
+// -------------------------------------------------------------------------------------------
+
 /**
  ContentFilter is an abstract base class from which all content filters must be
  derived.
@@ -173,6 +251,9 @@ class ContentFilter
     /// Subclasses can return `true` if they need all data in one chunk.
     virtual bool RequiresCompleteData() const { return false; }
 
+    /// Subclasses can return `true` if this filter supports filtering of ranges instead of full resource
+    virtual bool SupportsByteRanges() const { return false; }
+
     ///
     /// Obtains the type-sniffer for this filter.
     virtual TypeSnifferFn TypeSniffer() const { return _sniffer; }