diff --git a/src/FileDownloader.Tests/BreakpointResumptionTransmissionManagerTest.cs b/src/FileDownloader.Tests/BreakpointResumptionTransmissionManagerTest.cs index 5c6cdef..25a23b0 100644 --- a/src/FileDownloader.Tests/BreakpointResumptionTransmissionManagerTest.cs +++ b/src/FileDownloader.Tests/BreakpointResumptionTransmissionManagerTest.cs @@ -1,13 +1,16 @@ - +#nullable enable + +using System; using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; using dotnetCampus.FileDownloader; using dotnetCampus.FileDownloader.Utils.BreakpointResumptionTransmissions; using Microsoft.VisualStudio.TestTools.UnitTesting; -using Moq; - using MSTest.Extensions.Contracts; namespace FileDownloader.Tests @@ -18,60 +21,283 @@ public class BreakpointResumptionTransmissionManagerTest [ContractTestCase] public void GetDownloadSegmentList() { - "传入下载长度 100 分段分别为 10-20 和 20-30 和 50-55 到 GetDownloadSegmentList 方法,可以获取需要下载的段".Test(() => + "传入的断点续传记录的校验信息中,有一半与下载文件不匹配,返回有一半已经下载成功".Test(async () => { - const int DownloadLength = 100; - var mock = new Mock(); - var manager = new BreakpointResumptionTransmissionManager(new System.IO.FileInfo("Foo"), mock.Object, DownloadLength); + DebugRange[] dataRanges = new DebugRange[] + { + new(10, 10), // 10-20 + new(25, 10), // 25-35 + new(50, 10), // 50-60 + new(90, 5) // 90-95 + }; + + ISharedArrayPool sharedArrayPool = new SharedArrayPool(); + var fileWriter = new FakeRandomFileWriter(); + var breakpointResumptionTransmissionRecordFile = + new System.IO.FileInfo($"BreakpointResumption_{Path.GetRandomFileName()}"); + var manager = new BreakpointResumptionTransmissionManager(breakpointResumptionTransmissionRecordFile, + fileWriter, sharedArrayPool, contentLength: DownloadLength); + + var downloadFile = new System.IO.FileInfo($"FooDownloadFile_{Path.GetRandomFileName()}"); + await using var fileStream = downloadFile.Open(FileMode.Create, FileAccess.ReadWrite); + + var segmentManager = await manager.CreateSegmentManagerAsync(fileStream); + Assert.IsTrue(DownloadLength == segmentManager.FileLength); + var currentDownloadSegmentList = segmentManager.GetCurrentDownloadSegmentList(); + Assert.AreEqual(0, currentDownloadSegmentList.Count); + + // 通过 FakeRandomFileWriter 写入数据 + var buffer = new byte[DownloadLength]; + Random.Shared.NextBytes(buffer); + + foreach (var (start, length) in dataRanges) + { + fileWriter.QueueWrite(start, buffer, start, length); + } + + // 关闭文件流,模拟文件写入完成。准备再次新建一个 + manager.Dispose(); + + // 对下载文件写入不一样的内容,模拟下载文件和校验内容不匹配 + // 对 1 2 项填充错误的内容 + FillErrorBuffer(dataRanges[1]); + FillErrorBuffer(dataRanges[2]); + await fileStream.WriteAsync(buffer, 0, buffer.Length); + + manager = new BreakpointResumptionTransmissionManager(breakpointResumptionTransmissionRecordFile, + fileWriter, sharedArrayPool, contentLength: DownloadLength); + // 再次读取新的列表,此时预期能够读取到具备已下载内容的列表 + var segmentManager2 = await manager.CreateSegmentManagerAsync(fileStream); + + IReadOnlyList downloadSegmentList = segmentManager2.GetCurrentDownloadSegmentList(); + + AssertDownloadSegmentList(DownloadLength, downloadSegmentList); + + // 预期读取到一半任何一个已经下载完成的列表内容 + Assert.AreEqual(dataRanges.Length / 2, downloadSegmentList.Count(t => t.LoadingState == DownloadingState.Finished)); + Assert.AreEqual(dataRanges.Length / 2, downloadSegmentList.Count(t => t.Finished)); + + AssertDownloadedList(downloadSegmentList, new DebugRange[2] + { + dataRanges[0], + dataRanges[3], + }); + + void FillErrorBuffer(DebugRange range) + { + Random.Shared.NextBytes(buffer.AsSpan(range.Start,range.Length)); + } + }); - List list = new List() + "传入的断点续传记录的校验信息全部与下载文件不匹配,返回空列表".Test(async () => + { + DebugRange[] dataRanges = new DebugRange[] { - new DataRange(10,10),// 10-20 - new DataRange(20,10),// 20-30 - new DataRange(50,5),// 50-55 + new(10, 10), + new(20, 10), + new(30, 20) }; - var downloadSegmentList = manager.GetDownloadSegmentList(list); + + ISharedArrayPool sharedArrayPool = new SharedArrayPool(); + var fileWriter = new FakeRandomFileWriter(); + var breakpointResumptionTransmissionRecordFile = + new System.IO.FileInfo($"BreakpointResumption_{Path.GetRandomFileName()}"); + var manager = new BreakpointResumptionTransmissionManager(breakpointResumptionTransmissionRecordFile, + fileWriter, sharedArrayPool, contentLength: DownloadLength); + + var downloadFile = new System.IO.FileInfo($"FooDownloadFile_{Path.GetRandomFileName()}"); + await using var fileStream = downloadFile.Open(FileMode.Create, FileAccess.ReadWrite); + + var segmentManager = await manager.CreateSegmentManagerAsync(fileStream); + Assert.IsTrue(DownloadLength == segmentManager.FileLength); + var currentDownloadSegmentList = segmentManager.GetCurrentDownloadSegmentList(); + Assert.AreEqual(0, currentDownloadSegmentList.Count); + + // 通过 FakeRandomFileWriter 写入数据 + var buffer = new byte[DownloadLength]; + Random.Shared.NextBytes(buffer); + + foreach (var (start, length) in dataRanges) + { + fileWriter.QueueWrite(start, buffer, start, length); + } + + // 关闭文件流,模拟文件写入完成。准备再次新建一个 + manager.Dispose(); + + // 对下载文件写入完全不一样的内容,模拟下载文件和校验内容不匹配 + Random.Shared.NextBytes(buffer); + await fileStream.WriteAsync(buffer, 0, buffer.Length); + + manager = new BreakpointResumptionTransmissionManager(breakpointResumptionTransmissionRecordFile, + fileWriter, sharedArrayPool, contentLength: DownloadLength); + // 再次读取新的列表,此时预期能够读取到具备已下载内容的列表 + var segmentManager2 = await manager.CreateSegmentManagerAsync(fileStream); + + IReadOnlyList downloadSegmentList = segmentManager2.GetCurrentDownloadSegmentList(); AssertDownloadSegmentList(DownloadLength, downloadSegmentList); + + // 预期读取不到任何一个已经下载完成的列表内容 + Assert.AreEqual(0, downloadSegmentList.Count(t => t.LoadingState == DownloadingState.Finished)); + Assert.AreEqual(0, downloadSegmentList.Count(t => t.Finished)); }); - "传入下载长度 100 分段分别为 10-20 和 20-30 和 30-50 到 GetDownloadSegmentList 方法,可以获取需要下载的段".Test(() => + "传入下载长度 100 分段分别为 10-20 和 20-30 和 50-55 到 GetDownloadSegmentList 方法,可以获取需要下载的段".Test(async () => { - const int DownloadLength = 100; - var mock = new Mock(); - var manager = new BreakpointResumptionTransmissionManager(new System.IO.FileInfo("Foo"), mock.Object, DownloadLength); + ISharedArrayPool sharedArrayPool = new SharedArrayPool(); + var fileWriter = new FakeRandomFileWriter(); + var breakpointResumptionTransmissionRecordFile = + new System.IO.FileInfo($"BreakpointResumption_{Path.GetRandomFileName()}"); + var manager = new BreakpointResumptionTransmissionManager(breakpointResumptionTransmissionRecordFile, + fileWriter, sharedArrayPool, contentLength: DownloadLength); + + var downloadFile = new System.IO.FileInfo($"FooDownloadFile_{Path.GetRandomFileName()}"); + await using var fileStream = downloadFile.Open(FileMode.Create, FileAccess.ReadWrite); - List list = new List() + var segmentManager = await manager.CreateSegmentManagerAsync(fileStream); + Assert.IsTrue(DownloadLength == segmentManager.FileLength); + var currentDownloadSegmentList = segmentManager.GetCurrentDownloadSegmentList(); + Assert.AreEqual(0, currentDownloadSegmentList.Count); + + // 通过 FakeRandomFileWriter 写入数据 + var buffer = new byte[DownloadLength]; + Random.Shared.NextBytes(buffer); + await fileStream.WriteAsync(buffer, 0, buffer.Length); + + // 10-20 + // 20-30 + // 50-55 + DebugRange[] dataRanges = new DebugRange[] { - new DataRange(10,10),// 10-20 - new DataRange(20,10),// 20-30 - new DataRange(30,20),// 30-50 + new(10, 10), + new(20, 10), + new(50, 5) }; - var downloadSegmentList = manager.GetDownloadSegmentList(list); + foreach (var (start, length) in dataRanges) + { + fileWriter.QueueWrite(start, buffer, start, length); + } + + // 关闭文件流,模拟文件写入完成。准备再次新建一个 + manager.Dispose(); + + manager = new BreakpointResumptionTransmissionManager(breakpointResumptionTransmissionRecordFile, + fileWriter, sharedArrayPool, contentLength: DownloadLength); + // 再次读取新的列表,此时预期能够读取到具备已下载内容的列表 + var segmentManager2 = await manager.CreateSegmentManagerAsync(fileStream); + + IReadOnlyList downloadSegmentList = segmentManager2.GetCurrentDownloadSegmentList(); AssertDownloadSegmentList(DownloadLength, downloadSegmentList); + AssertDownloadedList(downloadSegmentList, dataRanges); }); - "传入下载长度 100 分段分别为 10-20 和 30-50 到 GetDownloadSegmentList 方法,可以获取需要下载的段".Test(() => + "传入下载长度 100 分段分别为 10-20 和 20-30 和 30-50 到 GetDownloadSegmentList 方法,可以获取需要下载的段".Test(async () => { - const int DownloadLength = 100; - var mock = new Mock(); - var manager = new BreakpointResumptionTransmissionManager(new System.IO.FileInfo("Foo"), mock.Object, DownloadLength); + // 10-20 + // 20-30 + // 30-50 + DebugRange[] dataRanges = new DebugRange[] + { + new(10, 10), + new(20, 10), + new(30, 20) + }; + + ISharedArrayPool sharedArrayPool = new SharedArrayPool(); + var fileWriter = new FakeRandomFileWriter(); + var breakpointResumptionTransmissionRecordFile = + new System.IO.FileInfo($"BreakpointResumption_{Path.GetRandomFileName()}"); + var manager = new BreakpointResumptionTransmissionManager(breakpointResumptionTransmissionRecordFile, + fileWriter, sharedArrayPool, contentLength: DownloadLength); + + var downloadFile = new System.IO.FileInfo($"FooDownloadFile_{Path.GetRandomFileName()}"); + await using var fileStream = downloadFile.Open(FileMode.Create, FileAccess.ReadWrite); - List list = new List() + var segmentManager = await manager.CreateSegmentManagerAsync(fileStream); + Assert.IsTrue(DownloadLength == segmentManager.FileLength); + var currentDownloadSegmentList = segmentManager.GetCurrentDownloadSegmentList(); + Assert.AreEqual(0, currentDownloadSegmentList.Count); + + // 通过 FakeRandomFileWriter 写入数据 + var buffer = new byte[DownloadLength]; + Random.Shared.NextBytes(buffer); + await fileStream.WriteAsync(buffer, 0, buffer.Length); + + foreach (var (start, length) in dataRanges) { - new DataRange(10,10),// 10-20 - new DataRange(30,20),// 30-50 + fileWriter.QueueWrite(start, buffer, start, length); + } + + // 关闭文件流,模拟文件写入完成。准备再次新建一个 + manager.Dispose(); + + manager = new BreakpointResumptionTransmissionManager(breakpointResumptionTransmissionRecordFile, + fileWriter, sharedArrayPool, contentLength: DownloadLength); + // 再次读取新的列表,此时预期能够读取到具备已下载内容的列表 + var segmentManager2 = await manager.CreateSegmentManagerAsync(fileStream); + + IReadOnlyList downloadSegmentList = segmentManager2.GetCurrentDownloadSegmentList(); + + AssertDownloadSegmentList(DownloadLength, downloadSegmentList); + AssertDownloadedList(downloadSegmentList, dataRanges); + }); + + "传入下载长度 100 分段分别为 10-20 和 30-50 到 GetDownloadSegmentList 方法,可以获取需要下载的段".Test(async () => + { + // 10-20 + // 30-50 + DebugRange[] dataRanges = new DebugRange[] + { + new(10, 20), + new(30, 20) }; - var downloadSegmentList = manager.GetDownloadSegmentList(list); - Assert.IsNotNull(downloadSegmentList); - Assert.AreEqual(5, downloadSegmentList.Count); + ISharedArrayPool sharedArrayPool = new SharedArrayPool(); + var fileWriter = new FakeRandomFileWriter(); + var breakpointResumptionTransmissionRecordFile = + new System.IO.FileInfo($"BreakpointResumption_{Path.GetRandomFileName()}"); + var manager = new BreakpointResumptionTransmissionManager(breakpointResumptionTransmissionRecordFile, + fileWriter, sharedArrayPool, contentLength: DownloadLength); + + var downloadFile = new System.IO.FileInfo($"FooDownloadFile_{Path.GetRandomFileName()}"); + await using var fileStream = downloadFile.Open(FileMode.Create, FileAccess.ReadWrite); + + var segmentManager = await manager.CreateSegmentManagerAsync(fileStream); + Assert.IsTrue(DownloadLength == segmentManager.FileLength); + var currentDownloadSegmentList = segmentManager.GetCurrentDownloadSegmentList(); + Assert.AreEqual(0, currentDownloadSegmentList.Count); + + // 通过 FakeRandomFileWriter 写入数据 + var buffer = new byte[DownloadLength]; + Random.Shared.NextBytes(buffer); + await fileStream.WriteAsync(buffer, 0, buffer.Length); + + foreach (var (start, length) in dataRanges) + { + fileWriter.QueueWrite(start, buffer, start, length); + } + + // 关闭文件流,模拟文件写入完成。准备再次新建一个 + manager.Dispose(); + + manager = new BreakpointResumptionTransmissionManager(breakpointResumptionTransmissionRecordFile, + fileWriter, sharedArrayPool, contentLength: DownloadLength); + // 再次读取新的列表,此时预期能够读取到具备已下载内容的列表 + var segmentManager2 = await manager.CreateSegmentManagerAsync(fileStream); + + IReadOnlyList downloadSegmentList = segmentManager2.GetCurrentDownloadSegmentList(); + AssertDownloadSegmentList(DownloadLength, downloadSegmentList); + AssertDownloadedList(downloadSegmentList, dataRanges); }); } - private static void AssertDownloadSegmentList(int downloadLength, List downloadSegmentList) + private const int DownloadLength = 100; + + private static void AssertDownloadSegmentList(int downloadLength, + IReadOnlyList downloadSegmentList) { for (var i = 0; i < downloadSegmentList.Count; i++) { @@ -93,5 +319,43 @@ private static void AssertDownloadSegmentList(int downloadLength, List downloadSegmentList, DebugRange[] dataRanges) + { + var list = dataRanges.ToList(); + foreach (DownloadSegment downloadSegment in downloadSegmentList) + { + if (downloadSegment.LoadingState == DownloadingState.Finished) + { + // 下载完成的,必定是在列表里面 + DebugRange? range = list.FirstOrDefault(t => + t.Start == downloadSegment.StartPoint && t.Length == downloadSegment.DownloadedLength); + Assert.IsNotNull(range); + + list.Remove(range!); + } + } + + // 全部列表记录的,都能从 downloadSegmentList 找到,即被删除 + Assert.AreEqual(0, list.Count); + } + + record DebugRange(int Start, int Length); + + class FakeRandomFileWriter : IRandomFileWriter + { + public void QueueWrite(long fileStartPoint, byte[] data, int dataOffset, int dataLength) + { + StepWriteFinished?.Invoke(this, + new StepWriteFinishedArgs(fileStartPoint, dataOffset, data, dataLength)); + } + + public event EventHandler? StepWriteFinished; + + public ValueTask DisposeAsync() + { + return ValueTask.CompletedTask; + } + } } } diff --git a/src/FileDownloader.Tests/BreakpointResumptionTransmissionRecordFileFormatterTest.cs b/src/FileDownloader.Tests/BreakpointResumptionTransmissionRecordFileFormatterTest.cs index 1b2bb37..5520af2 100644 --- a/src/FileDownloader.Tests/BreakpointResumptionTransmissionRecordFileFormatterTest.cs +++ b/src/FileDownloader.Tests/BreakpointResumptionTransmissionRecordFileFormatterTest.cs @@ -15,7 +15,7 @@ public class BreakpointResumptionTransmissionRecordFileFormatterTest [ContractTestCase] public void Format() { - "写入断点续传信息之后,可以读取数据".Test(() => + "写入断点续传信息之后,可以读取数据".Test(async () => { var formatter = new BreakpointResumptionTransmissionRecordFileFormatter(); @@ -25,9 +25,9 @@ public void Format() var downloadLength = 1024; var downloadedInfo = new List() { - new DataRange(0, 10), - new DataRange(10,20), - new DataRange(100,2) + new DataRange(0, 10,0), + new DataRange(10,20,0), + new DataRange(100,2,0) }; var info = new BreakpointResumptionTransmissionInfo(downloadLength, downloadedInfo); @@ -35,11 +35,12 @@ public void Format() memoryStream.Seek(0, SeekOrigin.Begin); - var result = formatter.Read(memoryStream); + var result = await formatter.ReadAsync(memoryStream); Assert.IsNotNull(result); Assert.AreEqual(downloadLength, result.DownloadLength); + Assert.IsNotNull(result.DownloadedInfo); Assert.AreEqual(downloadedInfo.Count, result.DownloadedInfo.Count); for (int i = 0; i < downloadedInfo.Count; i++) diff --git a/src/FileDownloader.Tests/DataRangeTest.cs b/src/FileDownloader.Tests/DataRangeTest.cs index 25a53c3..5a8eed1 100644 --- a/src/FileDownloader.Tests/DataRangeTest.cs +++ b/src/FileDownloader.Tests/DataRangeTest.cs @@ -10,39 +10,39 @@ namespace FileDownloader.Tests [TestClass] public class DataRangeTest { - [ContractTestCase] - public void TryMerge() - { - "传入两个不相邻的 DataRange 对象,返回不可合并".Test(() => - { - var a = new DataRange(0, 1); - var b = new DataRange(3, 2); - var result = DataRange.TryMerge(a, b, out var data); - Assert.AreEqual(false, result); - }); + //[ContractTestCase] + //public void TryMerge() + //{ + // "传入两个不相邻的 DataRange 对象,返回不可合并".Test(() => + // { + // var a = new DataRange(0, 1); + // var b = new DataRange(3, 2); + // var result = DataRange.TryMerge(a, b, out var data); + // Assert.AreEqual(false, result); + // }); - "给定两个相邻的 DataRange 对象,传入顺序是先将传入起点较大的,再传入起点较小的,可以进行合并".Test(() => - { - var a = new DataRange(0, 3); - var b = new DataRange(3, 2); + // "给定两个相邻的 DataRange 对象,传入顺序是先将传入起点较大的,再传入起点较小的,可以进行合并".Test(() => + // { + // var a = new DataRange(0, 3); + // var b = new DataRange(3, 2); - // 传入顺序是先将传入起点较大的,再传入起点较小的 - var result = DataRange.TryMerge(b, a, out var data); - Assert.IsTrue(result); - Assert.AreEqual(a.StartPoint, data.StartPoint); - Assert.AreEqual(a.Length + b.Length, data.Length); - }); + // // 传入顺序是先将传入起点较大的,再传入起点较小的 + // var result = DataRange.TryMerge(b, a, out var data); + // Assert.IsTrue(result); + // Assert.AreEqual(a.StartPoint, data.StartPoint); + // Assert.AreEqual(a.Length + b.Length, data.Length); + // }); - "给定两个相邻的 DataRange 对象,可以进行合并".Test(() => - { - var a = new DataRange(0, 3); - var b = new DataRange(3, 2); + // "给定两个相邻的 DataRange 对象,可以进行合并".Test(() => + // { + // var a = new DataRange(0, 3); + // var b = new DataRange(3, 2); - var result = DataRange.TryMerge(a, b, out var data); - Assert.IsTrue(result); - Assert.AreEqual(a.StartPoint, data.StartPoint); - Assert.AreEqual(a.Length + b.Length, data.Length); - }); - } + // var result = DataRange.TryMerge(a, b, out var data); + // Assert.IsTrue(result); + // Assert.AreEqual(a.StartPoint, data.StartPoint); + // Assert.AreEqual(a.Length + b.Length, data.Length); + // }); + //} } } diff --git a/src/FileDownloader.Tests/SegmentFileDownloaderTest.cs b/src/FileDownloader.Tests/SegmentFileDownloaderTest.cs index b23ca3e..b7beb62 100644 --- a/src/FileDownloader.Tests/SegmentFileDownloaderTest.cs +++ b/src/FileDownloader.Tests/SegmentFileDownloaderTest.cs @@ -150,6 +150,8 @@ public interface IMockSegmentFileDownloader } } +#pragma warning disable SYSLIB0014 // warning SYSLIB0014: “WebRequest.WebRequest()”已过时:“WebRequest, HttpWebRequest, ServicePoint, and WebClient are obsolete. Use HttpClient instead.” + // 这里就是专门测试旧代码的,忽略即可 class FakeHttpWebRequest : WebRequest { public FakeHttpWebRequest(FakeWebResponse fakeWebResponse) @@ -192,4 +194,6 @@ public override Stream GetResponseStream() return Stream; } } +#pragma warning restore SYSLIB0014 + } diff --git a/src/dotnetCampus.FileDownloader.Tool/Program.cs b/src/dotnetCampus.FileDownloader.Tool/Program.cs index be611e8..6aa4746 100644 --- a/src/dotnetCampus.FileDownloader.Tool/Program.cs +++ b/src/dotnetCampus.FileDownloader.Tool/Program.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Net.Http; using System.Runtime.InteropServices.ComTypes; using System.Threading.Tasks; using CommandLine; @@ -21,19 +22,29 @@ static async Task Main(string[] args) // https://www.speedtest.cn/ var url = - "https://speedtest1.gd.chinamobile.com.prod.hosts.ooklaserver.net:8080/download?size=25000000&r=0.2978374611691549"; + "https://node-103-27-27-20.speedtest.cn:51090/download?size=25000000&r=0.625866791098137"; url = - "https://download.jetbrains.8686c.com/resharper/ReSharperUltimate.2020.1.3/JetBrains.ReSharperUltimate.2020.1.3.exe"; + "https://download.jetbrains.com/resharper/dotUltimate.2025.1/JetBrains.dotUltimate.2025.1.exe"; //var md5 = "7d6bbeb6617a7c0b7e615098fca1b167";// resharper - url = "http://localhost:5000"; + //url = "http://localhost:5000"; - var file = new FileInfo(@"File.txt"); + // 这里的 gitdl.cn 是 iFileProxy 离线下载工具的地址,这是一个非常好的工具。开源地址: https://git.linxi.info/xianglin_admin/iFileProxy + url = "https://gitdl.cn/https://github.com/srwi/EverythingToolbar/releases/download/1.5.2/EverythingToolbar-1.5.2.msi"; + + //url = + // "https://down.pc.yyb.qq.com/pcyyb/packing/14e1e37f997f49a58d560ab97fa335aa/pcyyb_2702800040_installer.exe"; + //url = "https://pm.myapp.com/invc/xfspeed/qqpcmgr/download/QQPCDownload320001.exe"; + //// 这个地址带了 Content-Disposition 头,文件名是从这个头中获取的 + url = + "https://sw.pcmgr.qq.com/2f472366ca30d8ac1ad4acb64c77d2ad/680c8f51/spcmgr/download/BaiduNetdisk_txgj1_7.50.0.132.exe"; + //url = "https://pc-package.wpscdn.cn/wps/download/W.P.S.60.1955.exe"; + + var downloadFolder = new DirectoryInfo(@"DownloadFolder"); var progress = new Progress(); - var segmentFileDownloader = new SegmentFileDownloader(url, file, logger, progress); - await segmentFileDownloader.DownloadFileAsync(); + await FileDownloaderHelper.DownloadFileToFolderAsync(url, downloadFolder, progress:progress); #endif await Task.Delay(100); }); @@ -76,7 +87,7 @@ private static async Task DownloadFileAsync(DownloadOption option) var file = new FileInfo(output); var url = option.Url; - var segmentFileDownloader = new SegmentFileDownloader(url, file, logger, progress); + using var segmentFileDownloader = new SegmentFileDownloaderByHttpClient(url, file, httpClient:null, logger, progress); await segmentFileDownloader.DownloadFileAsync(); diff --git a/src/dotnetCampus.FileDownloader.WPF/Business/DownloadFileListManager.cs b/src/dotnetCampus.FileDownloader.WPF/Business/DownloadFileListManager.cs index 4eea677..b7cb043 100644 --- a/src/dotnetCampus.FileDownloader.WPF/Business/DownloadFileListManager.cs +++ b/src/dotnetCampus.FileDownloader.WPF/Business/DownloadFileListManager.cs @@ -17,7 +17,7 @@ public class DownloadFileListManager /// 读取本地存储的下载列表 /// /// - public async Task> ReadDownloadedFileListAsync() + public async Task?> ReadDownloadedFileListAsync() { var file = GetStorageFilePath(); diff --git a/src/dotnetCampus.FileDownloader/DownloadByHttpClient/SegmentFileDownloaderByHttpClient.cs b/src/dotnetCampus.FileDownloader/DownloadByHttpClient/SegmentFileDownloaderByHttpClient.cs index 46ca054..e1830f3 100644 --- a/src/dotnetCampus.FileDownloader/DownloadByHttpClient/SegmentFileDownloaderByHttpClient.cs +++ b/src/dotnetCampus.FileDownloader/DownloadByHttpClient/SegmentFileDownloaderByHttpClient.cs @@ -16,6 +16,7 @@ using Microsoft.Extensions.Logging; +// ReSharper disable once CheckNamespace namespace dotnetCampus.FileDownloader; /// @@ -61,7 +62,7 @@ public SegmentFileDownloaderByHttpClient(string url, FileInfo file, _shouldDisposeHttpClient = httpClient is null; HttpClient = httpClient ?? CreateDefaultHttpClient(); - DownloadDataList = Channel.CreateUnbounded(new UnboundedChannelOptions() + DownloadDataChannel = Channel.CreateUnbounded(new UnboundedChannelOptions() { SingleReader = false, AllowSynchronousContinuations = true, @@ -135,6 +136,12 @@ private static SocketsHttpHandler CreateDefaultSocketsHttpHandler() //{ // return ValueTask.FromResult(context.PlaintextStream); //} + + // 忽略证书错误 + //SslOptions = + //{ + // RemoteCertificateValidationCallback = (sender, certificate, chain, errors) => true + //} }; return socketsHttpHandler; @@ -163,12 +170,12 @@ private static SocketsHttpHandler CreateDefaultSocketsHttpHandler() /// public string Url { get; } - private Channel DownloadDataList { get; } + private Channel DownloadDataChannel { get; } /// - /// 被加入到 的下载数量 + /// 被加入到 的下载数量 /// - private int _workTaskCount; + private int _workingTaskCount; /// /// 下载的文件 @@ -233,7 +240,7 @@ private async void ControlSwitch() { LogDebugInternal("Start ControlSwitch"); var (segment, runCount, maxReportTime) = SegmentManager.GetMaxWaitReportTimeDownloadSegmentStatus(); - var waitCount = _workTaskCount; + var waitCount = _workingTaskCount; LogDebugInternal("ControlSwitch 当前等待数量:{0},待命最大响应时间:{1},运行数量:{2},运行线程{3}", waitCount, maxReportTime, runCount, _threadCount); @@ -320,10 +327,13 @@ public async ValueTask DownloadFileAsync() return; } - FileStream = File.Create(); - FileStream.SetLength(contentLength); + FileStream = File.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read/*允许边下边播*/); + if (FileStream.Length == 0) + { + // 如果是刚刚创建的,则预先分配空间。实际测试下载器预先分配空间能够获取更好的机械硬盘性能 + FileStream.SetLength(contentLength); + } FileWriter = new RandomFileWriterWithOrderFirst(FileStream); - FileWriter.StepWriteFinished += (sender, args) => SharedArrayPool.Return(args.Data); if (BreakpointResumptionTransmissionRecordFile is null) { @@ -333,11 +343,13 @@ public async ValueTask DownloadFileAsync() else { // 有断点续传 - var manager = new BreakpointResumptionTransmissionManager(BreakpointResumptionTransmissionRecordFile, FileWriter, contentLength); + var manager = new BreakpointResumptionTransmissionManager(BreakpointResumptionTransmissionRecordFile, FileWriter, SharedArrayPool, contentLength, BufferLength); // 有断点续传情况下,先读取断点续传文件,通过此文件获取到需要下载的内容 - SegmentManager = manager.CreateSegmentManager(); + SegmentManager = await manager.CreateSegmentManagerAsync(FileStream); BreakpointResumptionTransmissionManager = manager; } + // 由于 BreakpointResumptionTransmissionManager 也在监控 StepWriteFinished 事件,如果这里的事件加等更快执行,则会导致数据已经还给了池,被其他地方使用,在 BreakpointResumptionTransmissionManager 存在线程安全 + FileWriter.StepWriteFinished += (sender, args) => SharedArrayPool.Return(args.Data); _progress.Report(new DownloadProgress($"file length = {contentLength}", SegmentManager)); @@ -559,15 +571,20 @@ private async ValueTask DownloadTask() DownloadData data; try { - var canRead = await DownloadDataList.Reader.WaitToReadAsync(); + var canRead = await DownloadDataChannel.Reader.WaitToReadAsync(); if (!canRead) { // 不能读取了,那就返回吧 return; } - data = await DownloadDataList.Reader.ReadAsync().ConfigureAwait(false); - Interlocked.Decrement(ref _workTaskCount); + if (!DownloadDataChannel.Reader.TryRead(out data)) + { + // 居然读取不到数据,那就再次进入循环吧 + continue; + } + + Interlocked.Decrement(ref _workingTaskCount); } catch (ChannelClosedException) { @@ -712,8 +729,8 @@ private void LogDownloadSegment(DownloadSegment downloadSegment) private async void Download(HttpResponseMessage? httpResponseMessage, DownloadSegment downloadSegment) { LogDebugInternal("[Download] Enqueue Download. {0}", downloadSegment); - await DownloadDataList.Writer.WriteAsync(new DownloadData(httpResponseMessage, downloadSegment)).ConfigureAwait(false); - Interlocked.Increment(ref _workTaskCount); + await DownloadDataChannel.Writer.WriteAsync(new DownloadData(httpResponseMessage, downloadSegment)).ConfigureAwait(false); + Interlocked.Increment(ref _workingTaskCount); } private void Download(DownloadSegment? downloadSegment) @@ -748,7 +765,7 @@ private async ValueTask FinishDownload() await FileWriter.DisposeAsync().ConfigureAwait(false); await FileStream.DisposeAsync().ConfigureAwait(false); - DownloadDataList.Writer.Complete(); + DownloadDataChannel.Writer.Complete(); BreakpointResumptionTransmissionManager?.Dispose(); // 默认下载完成删除断点续传文件 diff --git a/src/dotnetCampus.FileDownloader/DownloadSegment.cs b/src/dotnetCampus.FileDownloader/DownloadSegment.cs index b329a8e..3bc672c 100644 --- a/src/dotnetCampus.FileDownloader/DownloadSegment.cs +++ b/src/dotnetCampus.FileDownloader/DownloadSegment.cs @@ -48,8 +48,19 @@ public DownloadSegment(long startPoint, long requirementDownloadPoint) /// 当前的信息,仅用于调试 /// public string? Message { get; set; } + + /// + /// 最后的下载状态更新时间 + /// + /// todo 改名 LastDownloadTime 或 LastUpdateTime public DateTime LastDownTime { get; set; } = DateTime.Now; + + /// + /// 下载状态 + /// + /// todo 改名 DownloadingState public DownloadingState LoadingState { get; set; } = DownloadingState.Pause; + /// /// 需要下载到的点 /// @@ -98,13 +109,24 @@ internal set /// public SegmentManager? SegmentManager { set; get; } } + /// /// 下载状态 /// public enum DownloadingState { + /// + /// 运行中 + /// + /// todo 改名 Running Runing = 1, + /// + /// 暂停 + /// Pause = 0, + /// + /// 完成 + /// Finished = -1 } } diff --git a/src/dotnetCampus.FileDownloader/FileDownloaderHelper.cs b/src/dotnetCampus.FileDownloader/FileDownloaderHelper.cs index c71279e..627ddbb 100644 --- a/src/dotnetCampus.FileDownloader/FileDownloaderHelper.cs +++ b/src/dotnetCampus.FileDownloader/FileDownloaderHelper.cs @@ -4,11 +4,19 @@ using System.IO; using System.Linq; using System.Net; + +#if NETCOREAPP3_1_OR_GREATER +using System.Net.Http; +using System.Text.RegularExpressions; +using System.Net.Http.Headers; +#endif using System.Runtime.InteropServices; using System.Security.AccessControl; using System.Threading; using System.Threading.Tasks; + using dotnetCampus.FileDownloader.Utils; + using Microsoft.Extensions.Logging; namespace dotnetCampus.FileDownloader @@ -30,14 +38,20 @@ public static class FileDownloaderHelper /// 缓存的数组长度,默认是 65535 的长度 /// 每一步 每一分段下载超时时间 默认 10 秒 /// - public static Task DownloadFileAsync(string url, FileInfo file, + public static async Task DownloadFileAsync(string url, FileInfo file, ILogger? logger = null, IProgress? progress = null, ISharedArrayPool? sharedArrayPool = null, int bufferLength = ushort.MaxValue, TimeSpan? stepTimeOut = null) { - var segmentFileDownloader = new SegmentFileDownloader(url, file, logger, progress, sharedArrayPool, bufferLength, stepTimeOut); +#if NETCOREAPP3_1_OR_GREATER + using var segmentFileDownloaderByHttpClient = new SegmentFileDownloaderByHttpClient(url, file, httpClient: null, logger, progress, sharedArrayPool, bufferLength, stepTimeOut); + await segmentFileDownloaderByHttpClient.DownloadFileAsync(); +#else + var segmentFileDownloader = + new SegmentFileDownloader(url, file, logger, progress, sharedArrayPool, bufferLength, stepTimeOut); - return segmentFileDownloader.DownloadFileAsync(); + await segmentFileDownloader.DownloadFileAsync(); +#endif } /// @@ -58,7 +72,7 @@ public static async Task DownloadFileToFolderAsync(string url, Directo int bufferLength = ushort.MaxValue, TimeSpan? stepTimeOut = null) { #if !NETFRAMEWORK - // 也许会在非 Windows 下系统使用 + // 也许会在非 Windows 下系统使用。但本方法没有测试过非 Windows 的情况,且文件命名规范都按照 Windows 的来,于是就决定不支持非 Windows 上跑的情况 if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { throw new PlatformNotSupportedException($"当前方法仅供 Windows 系统使用"); @@ -73,12 +87,19 @@ public static async Task DownloadFileToFolderAsync(string url, Directo var fileName = FileNameHelper.GuessFileNameFromUrl(url, fallbackName: Path.GetRandomFileName()); var downloadFile = new FileInfo(Path.Combine(tempFolder.FullName, fileName)); + +#if NETCOREAPP3_1_OR_GREATER + using var segmentFileDownloader = new InnerSegmentFileDownloaderByHttpClient(url, downloadFile, httpClient: null, + logger, progress, sharedArrayPool, bufferLength, stepTimeOut); +#else var segmentFileDownloader = new InnerSegmentFileDownloader(url, downloadFile, logger, progress, sharedArrayPool, bufferLength, stepTimeOut); +#endif await segmentFileDownloader.DownloadFileAsync(); // 下载完成了之后,尝试移动文件夹 // 优先使用服务器返回的文件名 var finallyFileName = segmentFileDownloader.ServerSuggestionFileName; + if (string.IsNullOrEmpty(finallyFileName)) { finallyFileName = fileName; @@ -96,7 +117,7 @@ public static async Task DownloadFileToFolderAsync(string url, Directo if (finallyFile.Exists) { // 重新加个名字,理论上这个名字不会重叠 - finallyFile = new FileInfo(Path.Combine(finallyFile.Directory.FullName, + finallyFile = new FileInfo(Path.Combine(finallyFile.Directory!.FullName, Path.GetFileNameWithoutExtension(finallyFile.FullName) + Path.GetRandomFileName() + finallyFile.Extension)); } @@ -117,6 +138,97 @@ public static async Task DownloadFileToFolderAsync(string url, Directo return finallyFile; } +#if NETCOREAPP3_1_OR_GREATER + + class InnerSegmentFileDownloaderByHttpClient + ( + string url, + FileInfo file, + HttpClient? httpClient = null, + ILogger? logger = null, + IProgress? progress = null, + ISharedArrayPool? sharedArrayPool = null, + int bufferLength = UInt16.MaxValue, + TimeSpan? stepTimeOut = null, + FileInfo? breakpointResumptionTransmissionRecordFile = null + ) + : SegmentFileDownloaderByHttpClient(url, file, httpClient, logger, progress, sharedArrayPool, bufferLength, + stepTimeOut, breakpointResumptionTransmissionRecordFile) + { + /// + /// 服务器端返回的文件名 + /// + public string? ServerSuggestionFileName { get; private set; } + + protected override async Task GetResponseAsync(HttpRequestMessage request) + { + var response = await base.GetResponseAsync(request); + if (string.IsNullOrEmpty(ServerSuggestionFileName)) + { + ServerSuggestionFileName = TryGetServerSuggestionFileName(); + } + + return response; + + string? TryGetServerSuggestionFileName() + { + var httpContentHeaders = response.Content.Headers; + + if (httpContentHeaders.ContentDisposition == null) + { + return null; + } + + // {Last-Modified: Mon, 30 Dec 2024 09:27:04 GMT + // Content-Type: application/octet-stream + // Content-Disposition: attachment; filename*="UTF-8''BaiduNetdisk_txgj1_7.50.0.132.exe" + // Content-Length: 403338840 + // } + // 这里拿到的 nameValueHeaderValue 可能就取出 + NameValueHeaderValue? nameValueHeaderValue = + httpContentHeaders.ContentDisposition.Parameters.FirstOrDefault(t => t.Name == "filename*"); + var fileNameValue = nameValueHeaderValue?.Value; + if (!string.IsNullOrEmpty(fileNameValue)) + { + // 额外判断一下 UTF-8 存在的情况 + fileNameValue = fileNameValue.Trim('"'); + var match = Regex.Match(fileNameValue, @"([\S\s]*)''([\S\s]*)"); + if (match.Success) + { + var encodingText = match.Groups[1].Value; + var fileNameText = match.Groups[2].Value; + if (string.Equals("utf-8", encodingText, StringComparison.OrdinalIgnoreCase)) + { + // 以下转码用于防止中文名乱码 + var unescapeDataString = Uri.UnescapeDataString(fileNameText); + return unescapeDataString; + } + } + else + { + // 匹配不上,那就应该是整个都是文件名了 + return fileNameValue; + } + } + + var fileName = httpContentHeaders.ContentDisposition.FileName; + if (!string.IsNullOrEmpty(fileName)) + { + return fileName; + } + + if (response.Headers.TryGetValues("Content-Disposition", out var contentDispositionTextEnumerable) && contentDispositionTextEnumerable.FirstOrDefault() is { } contentDispositionText) + { + // 正常不会放在这里的,都是在 Content 的 Header 里面的 + return + WebResponseHelper.GetFileNameFromContentDispositionText(contentDispositionText); + } + + return null; + } + } + } +#else class InnerSegmentFileDownloader : SegmentFileDownloader { /// 下载链接,不对下载链接是否有效进行校对 @@ -145,6 +257,7 @@ protected override async Task GetResponseAsync(WebRequest request) return response; } } +#endif /// /// 为文件名提供辅助方法。 diff --git a/src/dotnetCampus.FileDownloader/Utils/BreakpointResumptionTransmissions/BreakpointResumptionTransmissionManager.Crc64.cs b/src/dotnetCampus.FileDownloader/Utils/BreakpointResumptionTransmissions/BreakpointResumptionTransmissionManager.Crc64.cs new file mode 100644 index 0000000..66737ef --- /dev/null +++ b/src/dotnetCampus.FileDownloader/Utils/BreakpointResumptionTransmissions/BreakpointResumptionTransmissionManager.Crc64.cs @@ -0,0 +1,222 @@ +#if NETCOREAPP3_1_OR_GREATER + +using System; +using System.Buffers.Binary; +using System.IO; +using System.Threading.Tasks; + +namespace dotnetCampus.FileDownloader.Utils.BreakpointResumptionTransmissions; +internal partial class BreakpointResumptionTransmissionManager +{ + // Copy From dotnet runtime: \src\libraries\System.IO.Hashing\src\System\IO\Hashing\Crc64.cs + // Licensed to the .NET Foundation under one or more agreements. + // The .NET Foundation licenses this file to you under the MIT license. + class Crc64 + { + private const ulong InitialState = 0UL; + private const int Size = sizeof(ulong); + + private ulong _crc = InitialState; + + /// + /// Initializes a new instance of the class. + /// + public Crc64() + { + _crcLookup = GenerateTable(0x42F0E1EBA9EA3693); + } + + /// + /// Appends the contents of to the data already + /// processed for the current hash computation. + /// + /// The data to process. + public ulong Append(ReadOnlySpan source) + { + _crc = Update(_crc, source); + return _crc; + } + + /// + /// Resets the hash computation to the initial state. + /// + public void Reset() + { + _crc = InitialState; + } + + /// + /// Writes the computed hash value to + /// without modifying accumulated state. + /// + /// The buffer that receives the computed hash value. + protected void GetCurrentHashCore(Span destination) + { + // The finalization step of the CRC is to perform the ones' complement. + BinaryPrimitives.WriteUInt64BigEndian(destination, _crc); + } + + /// + /// Writes the computed hash value to + /// then clears the accumulated state. + /// + protected void GetHashAndResetCore(Span destination) + { + BinaryPrimitives.WriteUInt64BigEndian(destination, _crc); + _crc = InitialState; + } + + /// + /// Computes the CRC-32 hash of the provided data. + /// + /// The data to hash. + /// The CRC-32 hash of the provided data. + /// + /// is . + /// + public byte[] Hash(byte[] source) + { + if (source is null) + throw new ArgumentNullException(nameof(source)); + + return Hash(new ReadOnlySpan(source)); + } + + /// + /// Computes the CRC-32 hash of the provided data. + /// + /// The data to hash. + /// The CRC-32 hash of the provided data. + public byte[] Hash(ReadOnlySpan source) + { + byte[] ret = new byte[Size]; + StaticHash(source, ret); + return ret; + } + + /// + /// Attempts to compute the CRC-32 hash of the provided data into the provided destination. + /// + /// The data to hash. + /// The buffer that receives the computed hash value. + /// + /// On success, receives the number of bytes written to . + /// + /// + /// if is long enough to receive + /// the computed hash value (4 bytes); otherwise, . + /// + public bool TryHash(ReadOnlySpan source, Span destination, out int bytesWritten) + { + if (destination.Length < Size) + { + bytesWritten = 0; + return false; + } + + bytesWritten = StaticHash(source, destination); + return true; + } + + /// + /// Computes the CRC-32 hash of the provided data into the provided destination. + /// + /// The data to hash. + /// The buffer that receives the computed hash value. + /// + /// The number of bytes written to . + /// + public int Hash(ReadOnlySpan source, Span destination) + { + if (destination.Length < Size) + throw new ArgumentException("Argument_DestinationTooShort", nameof(destination)); + + return StaticHash(source, destination); + } + + private int StaticHash(ReadOnlySpan source, Span destination) + { + ulong crc = InitialState; + crc = Update(crc, source); + BinaryPrimitives.WriteUInt64BigEndian(destination, crc); + return Size; + } + + private ulong Update(ulong crc, ReadOnlySpan source) + { + for (int i = 0; i < source.Length; i++) + { + ulong idx = (crc >> 56); + idx ^= source[i]; + crc = _crcLookup[idx] ^ (crc << 8); + } + + return crc; + } + + // Pre-computed CRC-32 transition table. + // While this implementation is based on the standard CRC-32 polynomial, + // x32 + x26 + x23 + x22 + x16 + x12 + x11 + x10 + x8 + x7 + x5 + x4 + x2 + x1 + x0, + // this version uses reflected bit ordering, so 0x04C11DB7 becomes 0xEDB88320 + private readonly ulong[] _crcLookup; + + private static ulong[] GenerateTable(ulong polynomial) + { + ulong[] table = new ulong[256]; + + for (int i = 0; i < 256; i++) + { + ulong val = (ulong) i << 56; + + for (int j = 0; j < 8; j++) + { + if ((val & 0x8000_0000_0000_0000) == 0) + { + val <<= 1; + } + else + { + val = (val << 1) ^ polynomial; + } + } + + table[i] = val; + } + + return table; + } + } + + static class CrcHelper + { + public static async Task CheckCrcAsync(Stream stream, ulong expectedCrc, long checkLength, ISharedArrayPool sharedArrayPool, int bufferLength) + { + var buffer = sharedArrayPool.Rent(bufferLength); + try + { + var crc64 = new Crc64(); + ulong checksum = 0; + var remainLength = checkLength; + while (remainLength > 0) + { + var readLength = (int) Math.Min(bufferLength, remainLength); + var read = await stream.ReadAsync(buffer, 0, readLength); + if (read != readLength) + { + return false; + } + + checksum = crc64.Append(buffer.AsSpan(0, read)); + remainLength -= readLength; + } + + return checksum == expectedCrc; + } + finally + { + sharedArrayPool.Return(buffer); + } + } + } +} +#endif diff --git a/src/dotnetCampus.FileDownloader/Utils/BreakpointResumptionTransmissions/BreakpointResumptionTransmissionManager.cs b/src/dotnetCampus.FileDownloader/Utils/BreakpointResumptionTransmissions/BreakpointResumptionTransmissionManager.cs index 8c6bde5..dd069be 100644 --- a/src/dotnetCampus.FileDownloader/Utils/BreakpointResumptionTransmissions/BreakpointResumptionTransmissionManager.cs +++ b/src/dotnetCampus.FileDownloader/Utils/BreakpointResumptionTransmissions/BreakpointResumptionTransmissionManager.cs @@ -1,23 +1,30 @@ -using System; +#if NETCOREAPP3_1_OR_GREATER + +using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Threading.Tasks; namespace dotnetCampus.FileDownloader.Utils.BreakpointResumptionTransmissions; /// /// 断点续传管理 /// -internal class BreakpointResumptionTransmissionManager : IDisposable +internal partial class BreakpointResumptionTransmissionManager : IDisposable { - public BreakpointResumptionTransmissionManager(FileInfo breakpointResumptionTransmissionRecordFile, IRandomFileWriter fileWriter, long contentLength) + public BreakpointResumptionTransmissionManager(FileInfo breakpointResumptionTransmissionRecordFile, IRandomFileWriter fileWriter, ISharedArrayPool sharedArrayPool, long contentLength, int bufferLength = ushort.MaxValue) { BreakpointResumptionTransmissionRecordFile = breakpointResumptionTransmissionRecordFile; DownloadLength = contentLength; + BufferLength = bufferLength; + SharedArrayPool = sharedArrayPool; fileWriter.StepWriteFinished += (sender, args) => RecordDownloaded(args); } + public ISharedArrayPool SharedArrayPool { get; } + public FileInfo BreakpointResumptionTransmissionRecordFile { get; } /// @@ -25,29 +32,34 @@ public BreakpointResumptionTransmissionManager(FileInfo breakpointResumptionTran /// public long DownloadLength { get; } + public int BufferLength { get; } + /// /// 创建分段下载数据 /// + /// 正在被下载的文件的 FileStream 内容,用于断点续传校验内容 /// - /// - public SegmentManager CreateSegmentManager() + public async Task CreateSegmentManagerAsync(FileStream downloadFileStream) { // 如果存在断点续传记录文件,那将从此文件读取断点续传信息 // 如果读取的信息有误,或者是校验失败等 // 那就重新下载 // 还没有准备去释放 - FileStream = new FileStream(BreakpointResumptionTransmissionRecordFile.FullName, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read); + FileStream = new FileStream(BreakpointResumptionTransmissionRecordFile.FullName, FileMode.OpenOrCreate, + FileAccess.ReadWrite, FileShare.Read, bufferSize: 4096, + // 配置 WriteThrough 要求立即写入磁盘 + FileOptions.WriteThrough); BinaryWriter = new BinaryWriter(FileStream); // 进行一些初始化逻辑 Formatter = new BreakpointResumptionTransmissionRecordFileFormatter(); - var info = Formatter.Read(FileStream); + var info = await Formatter.ReadAsync(FileStream); if (info is not null && info.DownloadLength == DownloadLength && info.DownloadedInfo is not null && info.DownloadedInfo.Count > 0) { - var downloadSegmentList = GetDownloadSegmentList(info.DownloadedInfo); + var downloadSegmentList = await GetDownloadSegmentList(info.DownloadedInfo, downloadFileStream); var segmentManager = new SegmentManager(downloadSegmentList); for (var i = 0; i < downloadSegmentList.Count; i++) @@ -81,6 +93,8 @@ public SegmentManager CreateSegmentManager() item.SegmentManager = segmentManager; item.Number = i; } + + return segmentManager; } else { @@ -91,16 +105,15 @@ public SegmentManager CreateSegmentManager() Formatter.Write(BinaryWriter, new BreakpointResumptionTransmissionInfo(DownloadLength)); return new SegmentManager(DownloadLength); } - - return new SegmentManager(DownloadLength); } /// /// 通过断点续传的信息获取下载的内容 /// /// + /// /// - internal List GetDownloadSegmentList(List downloadedInfo) + private async Task> GetDownloadSegmentList(List downloadedInfo, FileStream downloadFileStream) { downloadedInfo.Sort(new DataRangeComparer()); var list = downloadedInfo; @@ -108,7 +121,7 @@ internal List GetDownloadSegmentList(List downloaded var downloadSegmentList = new List(); for (var i = 0; i < list.Count; i++) { - var current = list[i]; + DataRange current = list[i]; if (i == 0) { @@ -126,10 +139,29 @@ internal List GetDownloadSegmentList(List downloaded } } + // 需要对原文件进行校验,确保下载下去的和断点续传记录的相同 + async Task IsDownloaded() + { + var endPoint = current.StartPoint + current.Length; + if (downloadFileStream.Length < endPoint) + { + return false; + } + + downloadFileStream.Seek(current.StartPoint, SeekOrigin.Begin); + + return await CrcHelper.CheckCrcAsync(downloadFileStream, current.Checksum, current.Length, SharedArrayPool, + BufferLength); + } + + // 如果判断当前不是下载完成的内容,则配置状态不是 Finished 而是需要下载 + var isDownloaded = await IsDownloaded(); + var currentDownloadSegment = new DownloadSegment(current.StartPoint, current.StartPoint + current.Length) { - DownloadedLength = current.Length, - LoadingState = DownloadingState.Finished, + // 已经下载的长度。如果校验下载失败,则长度是 0 否则为记录的长度 + DownloadedLength = isDownloaded ? current.Length : 0, + LoadingState = isDownloaded ? DownloadingState.Finished : DownloadingState.Pause, }; downloadSegmentList.Add(currentDownloadSegment); @@ -197,10 +229,13 @@ private void RecordDownloaded(StepWriteFinishedArgs args) if (Formatter is null || FileStream is null || BinaryWriter is null) { - throw new InvalidOperationException("必须在调用 CreateSegmentManager 完成之后才能进入 RecordDownloaded 方法"); + throw new InvalidOperationException("必须在调用 CreateSegmentManagerAsync 完成之后才能进入 RecordDownloaded 方法"); } - Formatter.AppendDataRange(BinaryWriter, new DataRange(args.FileStartPoint, args.DataLength)); + var crc32 = new Crc64(); + var checksum = crc32.Append(args.Data.AsSpan(args.DataOffset, args.DataLength)); + + Formatter.AppendDataRange(BinaryWriter, new DataRange(args.FileStartPoint, args.DataLength, checksum)); } public void Dispose() @@ -213,3 +248,4 @@ public void Dispose() private bool _isDisposed; } +#endif diff --git a/src/dotnetCampus.FileDownloader/Utils/BreakpointResumptionTransmissions/BreakpointResumptionTransmissionRecordFileFormatter.cs b/src/dotnetCampus.FileDownloader/Utils/BreakpointResumptionTransmissions/BreakpointResumptionTransmissionRecordFileFormatter.cs index 9e6002b..ce09b42 100644 --- a/src/dotnetCampus.FileDownloader/Utils/BreakpointResumptionTransmissions/BreakpointResumptionTransmissionRecordFileFormatter.cs +++ b/src/dotnetCampus.FileDownloader/Utils/BreakpointResumptionTransmissions/BreakpointResumptionTransmissionRecordFileFormatter.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.IO; +using System.Runtime.InteropServices; +using System.Threading.Tasks; namespace dotnetCampus.FileDownloader.Utils.BreakpointResumptionTransmissions; @@ -10,12 +12,12 @@ namespace dotnetCampus.FileDownloader.Utils.BreakpointResumptionTransmissions; /// 文件格式:【文件头】【下载文件的下载长度】【各个已下载的数据段信息】 class BreakpointResumptionTransmissionRecordFileFormatter { - public BreakpointResumptionTransmissionInfo? Read(Stream stream) + public async Task ReadAsync(Stream stream) { var header = GetHeader(); // 设计上刚好可以复用 buffer 的值去进行读取 var buffer = new byte[sizeof(long)]; - (var success, var data) = Read(); + var (success, data) = await ReadInner(); if (!success || data != header) { // 如果读取不到 Header 的长度的内容,那返回空即可,让上层业务处理 @@ -23,21 +25,24 @@ class BreakpointResumptionTransmissionRecordFileFormatter return null; } - // 预期在 Header 之后是下载文件的长度 - (success, data) = Read(); + // 预期在 Header 之后是下载文件的长度。下载文件的长度包含两个部分内容:1. 文件长度标识(DataType.DownloadFileLength) 2. 文件长度 + // 读取文件长度标识 + (success, data) = await ReadInner(); if (!success || data != (long) DataType.DownloadFileLength) { - // 证明文件组织形式错误了,没有读取到下载文件的长度 + // 证明文件组织形式错误了,没有读取到下载文件的长度的标识 return null; } // 获取需要下载的文件长度 - (success, data) = Read(); + (success, data) = await ReadInner(); if (!success) { - // 没有读取到下载的文件长度,返回空即可 + // 没有读取到下载的文件长度,返回空即可,证明此记录内容不正确 return null; } + + // 文件长度之后的内容是分块下载的内容。分块下载的内容是一个个的 DataRange 结构体 var downloadLength = data; List downloadedInfo = new(); @@ -45,12 +50,13 @@ class BreakpointResumptionTransmissionRecordFileFormatter // 后续的信息就需要循环读取 while (success) { - // 后续的信息一个信息由三个 Int64 组成 + // 后续的信息一个信息由四个 Int64 组成 // 第一个是 DataType // 第二个是 起始点 // 第三个是 长度 + // 第四个是 校验信息 // 每段下载完成写入文件,将会记录写入的起始点和长度,通过起始点和长度 的列表可以算出当前还有哪些内容还没下载完成。如此即可实现断点续传功能 - (success, data) = Read(); + (success, data) = await ReadInner(); if (!success) { // 读取完成 @@ -59,43 +65,53 @@ class BreakpointResumptionTransmissionRecordFileFormatter if (data != (long) DataType.DownloadedInfo) { // 记录里面包含错误的数据,立刻返回 - // 如果在有错误的数据情况下,还不重新建立记录文件,那将会导致后续下载记录的内容被无效 - return null; + // 如果在有错误的数据情况下,如果还不重新建立记录文件,那将会导致后续下载记录的内容被无效 + break; } - (success, data) = Read(); + (success, data) = await ReadInner(); if (!success) { // 数据错误,没有记录全一条信息,重新建立记录文件 - return null; + break; } - var startPoint = data; - (success, data) = Read(); + + (success, data) = await ReadInner(); if (!success) { // 数据错误,没有记录全一条信息,重新建立记录文件 - return null; + break; } var length = data; - downloadedInfo.Add(new DataRange(startPoint, length)); + + (success, data) = await ReadInner(); + if (!success) + { + // 数据错误,没有记录全一条信息,重新建立记录文件 + break; + } + var checksum = (ulong) data; + + downloadedInfo.Add(new DataRange(startPoint, length, checksum)); } return new BreakpointResumptionTransmissionInfo(downloadLength, downloadedInfo); - (bool success, long data) Read() + async Task<(bool success, long data)> ReadInner() { // 用于调试读取失败时,读取到哪个内容 var originPosition = stream.Position; + _ = originPosition; - var readCount = stream.Read(buffer, 0, buffer.Length); + var readCount = await stream.ReadAsync(buffer, 0, buffer.Length); if (readCount != buffer.Length) { return (false, default(long)); } - var data = BitConverter.ToInt64(buffer, 0); - return (true, data); + var value = BitConverter.ToInt64(buffer, 0); + return (true, value); } } @@ -123,6 +139,7 @@ public void AppendDataRange(BinaryWriter binaryWriter, DataRange dataRange) binaryWriter.Write((long) DataType.DownloadedInfo); binaryWriter.Write(dataRange.StartPoint); binaryWriter.Write(dataRange.Length); + binaryWriter.Write(dataRange.Checksum); } private static long GetHeader() @@ -132,6 +149,8 @@ private static long GetHeader() //var headerByteList = System.Text.Encoding.ASCII.GetBytes("DCFBPRTI"); // var headerByteList = new byte[] { 68, 67, 70, 66, 80, 82, 84, 73 }; //return BitConverter.ToInt64(headerByteList) + // 由于这个类还想支持 NET45 等,就不用 MemoryMarshal 了 + //return MemoryMarshal.Read("DCFBPRTI"u8); return 5283938767475196740; } diff --git a/src/dotnetCampus.FileDownloader/Utils/BreakpointResumptionTransmissions/DataRange.cs b/src/dotnetCampus.FileDownloader/Utils/BreakpointResumptionTransmissions/DataRange.cs index 4529ae1..981a043 100644 --- a/src/dotnetCampus.FileDownloader/Utils/BreakpointResumptionTransmissions/DataRange.cs +++ b/src/dotnetCampus.FileDownloader/Utils/BreakpointResumptionTransmissions/DataRange.cs @@ -5,76 +5,82 @@ namespace dotnetCampus.FileDownloader.Utils.BreakpointResumptionTransmissions; readonly struct DataRange : IComparer, IEquatable { - public DataRange(long startPoint, long length) + public DataRange(long startPoint, long length, ulong checksum) { StartPoint = startPoint; Length = length; + Checksum = checksum; } public long StartPoint { get; } public long Length { get; } + public ulong Checksum { get; } + public long LastPoint => StartPoint + Length; public int Compare(DataRange x, DataRange y) { - if (ReferenceEquals(x, y)) - { - return 0; - } - - if (ReferenceEquals(null, y)) - { - return 1; - } - - if (ReferenceEquals(null, x)) - { - return -1; - } + // 由于 DataRange 从引用类型修改为值类型,这就导致原本调用 ReferenceEquals 的代码为傻逼代码,注释掉,避免无用的装箱判断不相等 + //if (ReferenceEquals(x, y)) + //{ + // return 0; + //} + + //if (ReferenceEquals(null, y)) + //{ + // return 1; + //} + + //if (ReferenceEquals(null, x)) + //{ + // return -1; + //} return x.StartPoint.CompareTo(y.StartPoint); } - public static bool TryMerge(DataRange a, DataRange b, out DataRange newDataRange) - { - newDataRange = default; - if (a.StartPoint > b.StartPoint) - { - var t = a; - a = b; - b = t; - } - - if (a.Equals(b)) - { - newDataRange = a; - return true; - } - - if (a.StartPoint <= b.StartPoint && a.LastPoint >= b.StartPoint) - { - var lastPoint = Math.Max(a.LastPoint, b.LastPoint); - var length = lastPoint - a.StartPoint; - newDataRange = new DataRange(a.StartPoint, length); - return true; - } - - return false; - } + // 由于加入了 Checksum 属性,因此无法执行合并逻辑 + //public static bool TryMerge(DataRange a, DataRange b, out DataRange newDataRange) + //{ + // newDataRange = default; + // if (a.StartPoint > b.StartPoint) + // { + // var t = a; + // a = b; + // b = t; + // } + + // if (a.Equals(b)) + // { + // newDataRange = a; + // return true; + // } + + // if (a.StartPoint <= b.StartPoint && a.LastPoint >= b.StartPoint) + // { + // var lastPoint = Math.Max(a.LastPoint, b.LastPoint); + // var length = lastPoint - a.StartPoint; + // newDataRange = new DataRange(a.StartPoint, length); + // return true; + // } + + // return false; + //} public bool Equals(DataRange other) { - if (ReferenceEquals(null, other)) - { - return false; - } + // 由于 DataRange 从引用类型修改为值类型,这就导致原本调用 ReferenceEquals 的代码为傻逼代码,注释掉,避免无用的装箱判断不相等 + //if (ReferenceEquals(null, other)) + //{ + // return false; + //} - if (ReferenceEquals(this, other)) - { - return true; - } + //if (ReferenceEquals(this, other)) + //{ + // return true; + //} return StartPoint == other.StartPoint && Length == other.Length; } @@ -86,11 +92,6 @@ public override bool Equals(object? obj) return false; } - if (ReferenceEquals(this, obj)) - { - return true; - } - if (obj.GetType() != GetType()) { return false; @@ -103,7 +104,11 @@ public override int GetHashCode() { unchecked { +#if NETCOREAPP3_1_OR_GREATER + return HashCode.Combine(StartPoint, Length); +#else return (StartPoint.GetHashCode() * 397) ^ Length.GetHashCode(); +#endif } }