Add automatic content hash for uploads (#385)

Compute and send a Dropbox content_hash for seekable upload streams so the
server verifies upload integrity and rejects corrupted uploads. Applies to
all upload routes whose request carries a ContentHash (direct upload and
upload sessions).

- ContentHasher: public helper computing the Dropbox block-SHA256 content
  hash (sync + async), plus WithoutAutoContentHash to opt a stream out.
- Handler: hash the body before sending, restore the stream position, and
  set the hash on a cloned request; retry now rewinds to the original
  upload position rather than 0.
- Property access is resolved once per request type and cached.
- Config: DropboxClientConfig.AutoContentHash, on by default; opt out
  per-client or per-upload. Non-seekable streams and manually supplied
  hashes are left untouched.
- Tests: unit tests for the hasher and upload wiring, and live e2e tests
  covering direct upload round-trip and a chunked upload session.
- README: document the behavior, the 2x read cost, and opt-out.

Author: AndreyVMarkelov

README.md

@@ -37,6 +37,38 @@ We provide [Examples][examples] to help get you started with a lot of the basic
     - [Simple Business Dashboard](https://github.com/dropbox/dropbox-sdk-dotnet/tree/main/dropbox-sdk-dotnet/Examples/SimpleBusinessDashboard) - This is a demo of a business dashboard
     - [Universal Demo](https://github.com/dropbox/dropbox-sdk-dotnet/tree/main/dropbox-sdk-dotnet/Examples/UniversalDemo/UniversalDemo) - This is an example of how to use the SDK across multiple platforms
 
+## Upload integrity
+
+For file upload calls whose request type includes `ContentHash`, the SDK automatically computes and sends a Dropbox `content_hash` when the upload stream is seekable. The server rejects the upload if the hash does not match the bytes it receives.
+
+Because `content_hash` is part of the request header, this default-on validation reads the stream once to compute the hash, seeks back to the original position, and then reads it again for the upload body. That is 2x read I/O for seekable uploads. For local files the second read is often served from OS cache; for slow, network-backed, or encrypted streams it can materially increase upload time. Non-seekable streams are uploaded without automatic hashing. A manually supplied `contentHash` argument always wins. If a seekable stream has unusual read or rewind behavior, disable automatic hashing for that upload or client.
+
+To compute a Dropbox content hash manually:
+
+```csharp
+using (var stream = File.OpenRead("local-file.txt"))
+{
+    var contentHash = ContentHasher.ComputeHash(stream);
+}
+```
+
+To disable automatic hashing for one upload, wrap the stream:
+
+```csharp
+await client.Files.UploadAsync(
+    new UploadArg("/remote-file.txt"),
+    ContentHasher.WithoutAutoContentHash(stream));
+```
+
+To disable automatic hashing for a client:
+
+```csharp
+var config = new DropboxClientConfig
+{
+    AutoContentHash = false,
+};
+```
+
 ## Getting Help
 
 If you find a bug, please see [CONTRIBUTING.md][contributing] for information on how to report it.

dropbox-sdk-dotnet/Dropbox.Api.Integration.Tests/DropboxApiTests.cs

@@ -285,6 +285,60 @@ public async Task TestUploadKeepsStreamOpenWhenConfigured()
             Assert.AreEqual(3, stream.Length);
         }
 
+        /// <summary>
+        /// Tests that an auto-hashed upload is accepted by the server and round-trips.
+        /// </summary>
+        /// <returns>The <see cref="Task" />.</returns>
+        [TestMethod]
+        public async Task TestAutoContentHashUploadRoundTrip()
+        {
+            var content = "auto-content-hash-roundtrip";
+            var expectedHash = ContentHasher.ComputeHash(GetStream(content));
+
+            // Upload with auto content hash enabled (the default). The server
+            // rejects the upload if the hash the SDK sent does not match.
+            var uploaded = await client.Files.UploadAsync(testingPath + "/Hash.txt", body: GetStream(content));
+            Assert.AreEqual(expectedHash, uploaded.ContentHash);
+
+            // Metadata and downloaded content should match too.
+            var metadata = await client.Files.GetMetadataAsync(testingPath + "/Hash.txt");
+            Assert.AreEqual(expectedHash, metadata.AsFile.ContentHash);
+
+            var download = await client.Files.DownloadAsync(testingPath + "/Hash.txt");
+            var downloaded = await download.GetContentAsStringAsync();
+            Assert.AreEqual(content, downloaded);
+        }
+
+        /// <summary>
+        /// Tests that a chunked upload session with auto content hashes round-trips.
+        /// </summary>
+        /// <returns>The <see cref="Task" />.</returns>
+        [TestMethod]
+        public async Task TestAutoContentHashUploadSession()
+        {
+            var chunk1 = "chunk-one-";
+            var chunk2 = "chunk-two-";
+            var chunk3 = "chunk-three";
+            var full = chunk1 + chunk2 + chunk3;
+            var path = testingPath + "/Session.txt";
+
+            // Each session call hashes its own chunk; the server rejects a mismatch.
+            var start = await client.Files.UploadSessionStartAsync(body: GetStream(chunk1));
+            var appendCursor = new UploadSessionCursor(start.SessionId, (ulong)chunk1.Length);
+
+            await client.Files.UploadSessionAppendV2Async(appendCursor, body: GetStream(chunk2));
+            var finishCursor = new UploadSessionCursor(start.SessionId, (ulong)(chunk1.Length + chunk2.Length));
+
+            var uploaded = await client.Files.UploadSessionFinishAsync(
+                finishCursor, new CommitInfo(path), body: GetStream(chunk3));
+
+            Assert.AreEqual(ContentHasher.ComputeHash(GetStream(full)), uploaded.ContentHash);
+
+            var download = await client.Files.DownloadAsync(path);
+            var downloaded = await download.GetContentAsStringAsync();
+            Assert.AreEqual(full, downloaded);
+        }
+
         /// <summary>
         /// Test get metadata.
         /// </summary>

dropbox-sdk-dotnet/Dropbox.Api.Unit.Tests/AutoContentHashUploadTests.cs

@@ -0,0 +1,202 @@
+//-----------------------------------------------------------------------------
+// <copyright file="AutoContentHashUploadTests.cs" company="Dropbox Inc">
+//  Copyright (c) Dropbox Inc. All rights reserved.
+// </copyright>
+//-----------------------------------------------------------------------------
+
+namespace Dropbox.Api.Unit.Tests
+{
+    using System.Collections.Generic;
+    using System.IO;
+    using System.Net;
+    using System.Net.Http;
+    using System.Text;
+    using System.Threading;
+    using System.Threading.Tasks;
+    using Dropbox.Api.Files;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+    using Newtonsoft.Json.Linq;
+
+    /// <summary>
+    /// Tests for automatic upload content hash injection.
+    /// </summary>
+    [TestClass]
+    public class AutoContentHashUploadTests
+    {
+        /// <summary>
+        /// Verifies seekable upload streams get a computed content hash.
+        /// </summary>
+        /// <returns>The asynchronous test task.</returns>
+        [TestMethod]
+        public async Task TestAutoContentHashUpload()
+        {
+            var handler = new CaptureUploadHandler();
+            using (var client = CreateClient(handler))
+            {
+                var uploadArg = new UploadArg("/auto.txt");
+                var content = new MemoryStream(Encoding.UTF8.GetBytes("payload"));
+
+                await client.Files.UploadAsync(uploadArg, content).ConfigureAwait(false);
+
+                var request = handler.Requests[0];
+                var arg = JObject.Parse(request.Arg);
+                Assert.AreEqual(ContentHasher.ComputeHash(new MemoryStream(Encoding.UTF8.GetBytes("payload"))), (string)arg["content_hash"]);
+                Assert.AreEqual("payload", request.Body);
+                Assert.IsNull(uploadArg.ContentHash);
+                Assert.AreEqual(Encoding.UTF8.GetByteCount("payload"), content.Position);
+            }
+        }
+
+        /// <summary>
+        /// Verifies manual content hash values are preserved.
+        /// </summary>
+        /// <returns>The asynchronous test task.</returns>
+        [TestMethod]
+        public async Task TestManualContentHashWins()
+        {
+            const string ManualHash = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+            var handler = new CaptureUploadHandler();
+            using (var client = CreateClient(handler))
+            {
+                await client.Files.UploadAsync(
+                    new UploadArg("/manual.txt", contentHash: ManualHash),
+                    new MemoryStream(Encoding.UTF8.GetBytes("payload"))).ConfigureAwait(false);
+
+                var arg = JObject.Parse(handler.Requests[0].Arg);
+                Assert.AreEqual(ManualHash, (string)arg["content_hash"]);
+            }
+        }
+
+        /// <summary>
+        /// Verifies the per-stream opt-out wrapper suppresses auto hash population.
+        /// </summary>
+        /// <returns>The asynchronous test task.</returns>
+        [TestMethod]
+        public async Task TestWithoutAutoContentHash()
+        {
+            var handler = new CaptureUploadHandler();
+            using (var client = CreateClient(handler))
+            {
+                await client.Files.UploadAsync(
+                    new UploadArg("/opt-out.txt"),
+                    ContentHasher.WithoutAutoContentHash(new MemoryStream(Encoding.UTF8.GetBytes("payload")))).ConfigureAwait(false);
+
+                var arg = JObject.Parse(handler.Requests[0].Arg);
+                Assert.IsNull(arg["content_hash"]);
+                Assert.AreEqual("payload", handler.Requests[0].Body);
+            }
+        }
+
+        /// <summary>
+        /// Verifies non-seekable streams are uploaded without automatic content hashes.
+        /// </summary>
+        /// <returns>The asynchronous test task.</returns>
+        [TestMethod]
+        public async Task TestNonSeekableStreamSkipsAutoContentHash()
+        {
+            var handler = new CaptureUploadHandler();
+            using (var client = CreateClient(handler))
+            {
+                await client.Files.UploadAsync(
+                    new UploadArg("/non-seekable.txt"),
+                    new NonSeekableStream(Encoding.UTF8.GetBytes("payload"))).ConfigureAwait(false);
+
+                var arg = JObject.Parse(handler.Requests[0].Arg);
+                Assert.IsNull(arg["content_hash"]);
+                Assert.AreEqual("payload", handler.Requests[0].Body);
+            }
+        }
+
+        /// <summary>
+        /// Verifies auto hashing uses the stream's current position as the upload start.
+        /// </summary>
+        /// <returns>The asynchronous test task.</returns>
+        [TestMethod]
+        public async Task TestAutoContentHashPreservesCurrentPosition()
+        {
+            var handler = new CaptureUploadHandler();
+            using (var client = CreateClient(handler))
+            {
+                var content = new MemoryStream(Encoding.UTF8.GetBytes("prefix:payload"));
+                content.Position = Encoding.UTF8.GetByteCount("prefix:");
+
+                await client.Files.UploadAsync(new UploadArg("/offset.txt"), content).ConfigureAwait(false);
+
+                var arg = JObject.Parse(handler.Requests[0].Arg);
+                Assert.AreEqual(ContentHasher.ComputeHash(new MemoryStream(Encoding.UTF8.GetBytes("payload"))), (string)arg["content_hash"]);
+                Assert.AreEqual("payload", handler.Requests[0].Body);
+            }
+        }
+
+        /// <summary>
+        /// Verifies the client-level opt-out suppresses auto hash population.
+        /// </summary>
+        /// <returns>The asynchronous test task.</returns>
+        [TestMethod]
+        public async Task TestConfigDisablesAutoContentHash()
+        {
+            var handler = new CaptureUploadHandler();
+            using (var client = CreateClient(handler, autoContentHash: false))
+            {
+                await client.Files.UploadAsync(
+                    new UploadArg("/disabled.txt"),
+                    new MemoryStream(Encoding.UTF8.GetBytes("payload"))).ConfigureAwait(false);
+
+                var arg = JObject.Parse(handler.Requests[0].Arg);
+                Assert.IsNull(arg["content_hash"]);
+            }
+        }
+
+        private static DropboxClient CreateClient(CaptureUploadHandler handler, bool autoContentHash = true)
+        {
+            return new DropboxClient(
+                "access-token",
+                new DropboxClientConfig
+                {
+                    HttpClient = new HttpClient(handler),
+                    DisposeUploadStream = false,
+                    AutoContentHash = autoContentHash,
+                });
+        }
+
+        private sealed class CaptureUploadHandler : HttpMessageHandler
+        {
+            public IList<CapturedUploadRequest> Requests { get; } = new List<CapturedUploadRequest>();
+
+            protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+            {
+                var body = await request.Content.ReadAsStringAsync().ConfigureAwait(false);
+                this.Requests.Add(new CapturedUploadRequest
+                {
+                    Arg = string.Join(string.Empty, request.Headers.GetValues("Dropbox-API-Arg")),
+                    Body = body,
+                });
+
+                return new HttpResponseMessage(HttpStatusCode.OK)
+                {
+                    Content = new StringContent(
+                        "{\"name\":\"file.txt\",\"id\":\"id:test\",\"client_modified\":\"2020-01-02T03:04:05Z\",\"server_modified\":\"2020-01-02T03:04:06Z\",\"rev\":\"abcdef123\",\"size\":1,\"is_downloadable\":true}",
+                        Encoding.UTF8,
+                        "application/json"),
+                };
+            }
+        }
+
+        private sealed class CapturedUploadRequest
+        {
+            public string Arg { get; set; }
+
+            public string Body { get; set; }
+        }
+
+        private sealed class NonSeekableStream : MemoryStream
+        {
+            public NonSeekableStream(byte[] buffer)
+                : base(buffer)
+            {
+            }
+
+            public override bool CanSeek => false;
+        }
+    }
+}

dropbox-sdk-dotnet/Dropbox.Api.Unit.Tests/ContentHasherTests.cs

@@ -0,0 +1,70 @@
+//-----------------------------------------------------------------------------
+// <copyright file="ContentHasherTests.cs" company="Dropbox Inc">
+//  Copyright (c) Dropbox Inc. All rights reserved.
+// </copyright>
+//-----------------------------------------------------------------------------
+
+namespace Dropbox.Api.Unit.Tests
+{
+    using System.IO;
+    using System.Text;
+    using System.Threading.Tasks;
+    using Dropbox.Api.Files;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    /// <summary>
+    /// Tests for Dropbox content hash computation.
+    /// </summary>
+    [TestClass]
+    public class ContentHasherTests
+    {
+        /// <summary>
+        /// Verifies known Dropbox content hash vectors.
+        /// </summary>
+        [TestMethod]
+        public void TestComputeHashKnownVectors()
+        {
+            Assert.AreEqual(
+                "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+                ContentHasher.ComputeHash(new MemoryStream()));
+
+            Assert.AreEqual(
+                "bf5d3affb73efd2ec6c36ad3112dd933efed63c4e1cbffcfa88e2759c144f2d8",
+                ContentHasher.ComputeHash(new MemoryStream(Encoding.UTF8.GetBytes("a"))));
+
+            Assert.AreEqual(
+                "4f8b42c22dd3729b519ba6f68d2da7cc5b2d606d05daed5ad5128cc03e6c6358",
+                ContentHasher.ComputeHash(new MemoryStream(Encoding.UTF8.GetBytes("abc"))));
+        }
+
+        /// <summary>
+        /// Verifies block boundary handling.
+        /// </summary>
+        [TestMethod]
+        public void TestComputeHashBlockBoundary()
+        {
+            var content = new byte[ContentHasher.BlockSize + 1];
+            for (var i = 0; i < content.Length; i++)
+            {
+                content[i] = (byte)'a';
+            }
+
+            Assert.AreEqual(
+                "5f858b62ccd88447586305aec6fd53c96747cfebf527cbba129a6dfed47d9624",
+                ContentHasher.ComputeHash(new MemoryStream(content)));
+        }
+
+        /// <summary>
+        /// Verifies asynchronous hash computation.
+        /// </summary>
+        /// <returns>The asynchronous test task.</returns>
+        [TestMethod]
+        public async Task TestComputeHashAsync()
+        {
+            var content = new MemoryStream(Encoding.UTF8.GetBytes("abc"));
+            var contentHash = await ContentHasher.ComputeHashAsync(content).ConfigureAwait(false);
+
+            Assert.AreEqual("4f8b42c22dd3729b519ba6f68d2da7cc5b2d606d05daed5ad5128cc03e6c6358", contentHash);
+        }
+    }
+}