Support reading or downloading a GCS object range. (#1748)

With this change applications can choose to download a small portion of
an object.

This fixes #1732.

Author: coryan

google/cloud/storage/CMakeLists.txt

@@ -76,6 +76,7 @@ add_library(storage_client
             client.cc
             client_options.h
             client_options.cc
+            download_options.h
             hashing_options.h
             hashing_options.cc
             idempotency_policy.h

google/cloud/storage/client.h

@@ -792,7 +792,7 @@ class Client {
    *     Valid types for this operation include `DisableCrc32cChecksum`,
    *     `DisableMD5Hash`, `IfGenerationMatch`, `EncryptionKey`, `Generation`,
    *     `IfGenerationMatch`, `IfGenerationNotMatch`, `IfMetagenerationMatch`,
-   *     `IfMetagenerationNotMatch`, and `UserProject`.
+   *     `IfMetagenerationNotMatch`, `ReadRange`, and `UserProject`.
    *
    * @throw std::runtime_error if there is a permanent failure, or if there were
    *     more transient failures than allowed by the current retry policy.
@@ -943,7 +943,7 @@ class Client {
    * @param options a list of optional query parameters and/or request headers.
    *   Valid types for this operation include `IfGenerationMatch`,
    *   `IfGenerationNotMatch`, `IfMetagenerationMatch`,
-   *   `IfMetagenerationNotMatch`, `Generation`, and `UserProject`.
+   *   `IfMetagenerationNotMatch`, `Generation`, `ReadRange`, and `UserProject`.
    *
    * @throw std::runtime_error if there is a permanent failure, or if there were
    *     more transient failures than allowed by the current retry policy.

google/cloud/storage/download_options.h

@@ -0,0 +1,55 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_DOWNLOAD_OPTIONS_H_
+#define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_DOWNLOAD_OPTIONS_H_
+
+#include "google/cloud/storage/internal/complex_option.h"
+#include <cstdint>
+#include <iostream>
+#include <string>
+
+namespace google {
+namespace cloud {
+namespace storage {
+inline namespace STORAGE_CLIENT_NS {
+struct ReadRangeData {
+  std::int64_t begin;
+  std::int64_t end;
+};
+
+/**
+ * Request only a portion of the GCS object in a ReadObject operation.
+ *
+ * Note that the range is right-open, as it is customary in C++. That is, it
+ * excludes the `end` byte.
+ */
+struct ReadRange : public internal::ComplexOption<ReadRange, ReadRangeData> {
+  ReadRange() : ComplexOption() {}
+  explicit ReadRange(std::int64_t begin, std::int64_t end)
+      : ComplexOption(ReadRangeData{begin, end}) {}
+  static char const* name() { return "read-range"; }
+};
+
+inline std::ostream& operator<<(std::ostream& os, ReadRangeData const& rhs) {
+  return os << "ReadRangeData={begin=" << rhs.begin << ", end=" << rhs.end
+            << "}";
+}
+
+}  // namespace STORAGE_CLIENT_NS
+}  // namespace storage
+}  // namespace cloud
+}  // namespace google
+
+#endif  // GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_DOWNLOAD_OPTIONS_H_

google/cloud/storage/internal/curl_client.cc

@@ -68,6 +68,9 @@ std::unique_ptr<HashValidator> CreateHashValidator(bool disable_md5,
 /// Create a HashValidator for a download request.
 std::unique_ptr<HashValidator> CreateHashValidator(
     ReadObjectRangeRequest const& request) {
+  if (request.HasOption<ReadRange>()) {
+    return google::cloud::internal::make_unique<NullHashValidator>();
+  }
   return CreateHashValidator(request.HasOption<DisableMD5Hash>(),
                              request.HasOption<DisableCrc32cChecksum>());
 }
@@ -541,6 +544,18 @@ StatusOr<std::unique_ptr<ObjectReadStreambuf>> CurlClient::ReadObject(
     return status;
   }
   builder.AddQueryParameter("alt", "media");
+  if (request.HasOption<ReadRange>()) {
+    auto range = request.GetOption<ReadRange>().value();
+    std::string header = "Range: bytes=" + std::to_string(range.begin) + "-" +
+                         std::to_string(range.end - 1);
+    builder.AddHeader(header);
+    // When doing a range read we need to disable decompression because range
+    // reads do not work in that case:
+    //   https://cloud.google.com/storage/docs/transcoding#range
+    // and
+    //   https://cloud.google.com/storage/docs/transcoding#decompressive_transcoding
+    builder.AddHeader("Cache-Control: no-transform");
+  }
 
   std::unique_ptr<CurlReadStreambuf> buf(new CurlReadStreambuf(
       builder.BuildDownloadRequest(std::string{}),
@@ -1171,6 +1186,19 @@ StatusOr<std::unique_ptr<ObjectReadStreambuf>> CurlClient::ReadObjectXml(
   // QuotaUser cannot be set, checked by the caller.
   // UserIp cannot be set, checked by the caller.
 
+  if (request.HasOption<ReadRange>()) {
+    auto range = request.GetOption<ReadRange>().value();
+    std::string header = "Range: bytes=" + std::to_string(range.begin) + "-" +
+                         std::to_string(range.end - 1);
+    builder.AddHeader(header);
+    // When doing a range read we need to disable decompression because range
+    // reads do not work in that case:
+    //   https://cloud.google.com/storage/docs/transcoding#range
+    // and
+    //   https://cloud.google.com/storage/docs/transcoding#decompressive_transcoding
+    builder.AddHeader("Cache-Control: no-transform");
+  }
+
   std::unique_ptr<CurlReadStreambuf> buf(new CurlReadStreambuf(
       builder.BuildDownloadRequest(std::string{}),
       client_options().download_buffer_size(), CreateHashValidator(request)));

google/cloud/storage/internal/object_requests.cc

@@ -95,8 +95,7 @@ std::ostream& operator<<(std::ostream& os, CopyObjectRequest const& r) {
 
 std::ostream& operator<<(std::ostream& os, ReadObjectRangeRequest const& r) {
   os << "ReadObjectRangeRequest={bucket_name=" << r.bucket_name()
-     << ", object_name=" << r.object_name() << ", begin=" << r.begin()
-     << ", end=" << r.end();
+     << ", object_name=" << r.object_name();
   r.DumpOptions(os, ", ");
   return os << "}";
 }

google/cloud/storage/internal/object_requests.h

@@ -15,6 +15,7 @@
 #ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_INTERNAL_OBJECT_REQUESTS_H_
 #define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_INTERNAL_OBJECT_REQUESTS_H_
 
+#include "google/cloud/storage/download_options.h"
 #include "google/cloud/storage/hashing_options.h"
 #include "google/cloud/storage/internal/generic_object_request.h"
 #include "google/cloud/storage/internal/http_response.h"
@@ -178,29 +179,10 @@ class ReadObjectRangeRequest
     : public GenericObjectRequest<
           ReadObjectRangeRequest, DisableCrc32cChecksum, DisableMD5Hash,
           EncryptionKey, Generation, IfGenerationMatch, IfGenerationNotMatch,
-          IfMetagenerationMatch, IfMetagenerationNotMatch, UserProject> {
+          IfMetagenerationMatch, IfMetagenerationNotMatch, ReadRange,
+          UserProject> {
  public:
-  ReadObjectRangeRequest() : GenericObjectRequest(), begin_(0), end_(0) {}
-
-  explicit ReadObjectRangeRequest(std::string bucket_name,
-                                  std::string object_name, std::int64_t begin,
-                                  std::int64_t end)
-      : GenericObjectRequest(std::move(bucket_name), std::move(object_name)),
-        begin_(begin),
-        end_(end) {}
-
-  explicit ReadObjectRangeRequest(std::string bucket_name,
-                                  std::string object_name)
-      : GenericObjectRequest(std::move(bucket_name), std::move(object_name)),
-        begin_(0),
-        end_(0) {}
-
-  std::int64_t begin() const { return begin_; }
-  std::int64_t end() const { return end_; }
-
- private:
-  std::int64_t begin_;
-  std::int64_t end_;
+  using GenericObjectRequest::GenericObjectRequest;
 };
 
 std::ostream& operator<<(std::ostream& os, ReadObjectRangeRequest const& r);

google/cloud/storage/internal/object_requests_test.cc

@@ -198,26 +198,25 @@ HttpResponse CreateRangeRequestResponse(
 }
 
 TEST(ObjectRequestsTest, ReadObjectRange) {
-  ReadObjectRangeRequest request("my-bucket", "my-object", 0, 1024);
+  ReadObjectRangeRequest request("my-bucket", "my-object");
 
   EXPECT_EQ("my-bucket", request.bucket_name());
   EXPECT_EQ("my-object", request.object_name());
-  EXPECT_EQ(0, request.begin());
-  EXPECT_EQ(1024, request.end());
 
   request.set_option(storage::UserProject("my-project"));
   request.set_multiple_options(storage::IfGenerationMatch(7),
-                               storage::UserProject("my-project"));
+                               storage::UserProject("my-project"),
+                               storage::ReadRange(0, 1024));
 
   std::ostringstream os;
   os << request;
   std::string actual = os.str();
   EXPECT_THAT(actual, HasSubstr("my-bucket"));
   EXPECT_THAT(actual, HasSubstr("my-object"));
-  EXPECT_THAT(actual, HasSubstr("begin=0"));
-  EXPECT_THAT(actual, HasSubstr("end=1024"));
   EXPECT_THAT(actual, HasSubstr("ifGenerationMatch=7"));
   EXPECT_THAT(actual, HasSubstr("my-project"));
+  EXPECT_THAT(actual, HasSubstr("begin=0"));
+  EXPECT_THAT(actual, HasSubstr("end=1024"));
 }
 
 TEST(ObjectRequestsTest, RangeResponseParse) {

google/cloud/storage/storage_client.bzl

@@ -21,6 +21,7 @@ storage_client_hdrs = [
     "bucket_metadata.h",
     "client.h",
     "client_options.h",
+    "download_options.h",
     "hashing_options.h",
     "idempotency_policy.h",
     "internal/access_control_common.h",

google/cloud/storage/testbench/testbench.py

@@ -23,6 +23,7 @@
 import httpbin
 import json
 import os
+import re
 import testbench_utils
 from werkzeug import serving
 from werkzeug import wsgi
@@ -445,14 +446,27 @@ def objects_get(bucket_name, object_name):
     if media != 'media':
         raise error_response.ErrorResponse('Invalid alt=%s parameter' % media)
     revision.validate_encryption_for_read(flask.request)
+    # Respect the Range: header, if present.
+    range_header = flask.request.headers.get('range')
+    response_payload = revision.media
+    begin = 0
+    end = len(response_payload)
+    if range_header is not None:
+        print("\n\n\nrange_header = %s\n\n" % range_header)
+        m = re.match('bytes=([0-9]+)-([0-9]+)', range_header)
+        if m:
+            print("\n\n\nmatch = %s\n\n" % m)
+            begin = int(m.group(1))
+            end = int(m.group(2))
+            response_payload = response_payload[begin:end + 1]
+    # Process custome headers to test error conditions.
     instructions = flask.request.headers.get('x-goog-testbench-instructions')
     if instructions == 'return-corrupted-data':
-        response_payload = testbench_utils.corrupt_media(revision.media)
-    else:
-        response_payload = revision.media
+        response_payload = testbench_utils.corrupt_media(response_payload)
     response = flask.make_response(response_payload)
     length = len(response_payload)
-    response.headers['Content-Range'] = 'bytes 0-%d/%d' % (length - 1, length)
+    content_range = 'bytes %d-%d/%d' % (begin, end - 1, length)
+    response.headers['Content-Range'] = content_range
     response.headers['x-goog-hash'] = revision.x_goog_hash_header()
     return response
 
@@ -693,14 +707,27 @@ def xmlapi_get_object(bucket_name, object_name):
     blob.check_preconditions_by_value(generation_match, None,
                                       metageneration_match, None)
     revision = blob.get_revision(flask.request)
+    # Respect the Range: header, if present.
+    range_header = flask.request.headers.get('range')
+    response_payload = revision.media
+    begin = 0
+    end = len(response_payload)
+    if range_header is not None:
+        print("\n\n\nrange_header = %s\n\n" % range_header)
+        m = re.match('bytes=([0-9]+)-([0-9]+)', range_header)
+        if m:
+            print("\n\n\nmatch = %s\n\n" % m)
+            begin = int(m.group(1))
+            end = int(m.group(2))
+            response_payload = response_payload[begin:end + 1]
+    # Process custome headers to test error conditions.
     instructions = flask.request.headers.get('x-goog-testbench-instructions')
     if instructions == 'return-corrupted-data':
-        response_payload = testbench_utils.corrupt_media(revision.media)
-    else:
-        response_payload = revision.media
+        response_payload = testbench_utils.corrupt_media(response_payload)
     response = flask.make_response(response_payload)
     length = len(response_payload)
-    response.headers['Content-Range'] = 'bytes 0-%d/%d' % (length - 1, length)
+    content_range = 'bytes %d-%d/%d' % (begin, end - 1, length)
+    response.headers['Content-Range'] = content_range
     response.headers['x-goog-hash'] = revision.x_goog_hash_header()
     return response