Merge pull request #62 from dotnet-campus/t/lindexi/SegmentFileDownloaderByHttpClient

日常维护代码翻新 优化断点下载

Author: walterlv

src/FileDownloader.Tests/BreakpointResumptionTransmissionManagerTest.cs

@@ -15,7 +15,7 @@ public class BreakpointResumptionTransmissionRecordFileFormatterTest
         [ContractTestCase]
         public void Format()
         {
-            "写入断点续传信息之后，可以读取数据".Test(() =>
+            "写入断点续传信息之后，可以读取数据".Test(async () =>
             {
                 var formatter = new BreakpointResumptionTransmissionRecordFileFormatter();
 
@@ -25,21 +25,22 @@ public void Format()
                 var downloadLength = 1024;
                 var downloadedInfo = new List<DataRange>()
                 {
-                    new DataRange(0, 10),
-                    new DataRange(10,20),
-                    new DataRange(100,2)
+                    new DataRange(0, 10,0),
+                    new DataRange(10,20,0),
+                    new DataRange(100,2,0)
                 };
 
                 var info = new BreakpointResumptionTransmissionInfo(downloadLength, downloadedInfo);
                 formatter.Write(writer, info);
 
                 memoryStream.Seek(0, SeekOrigin.Begin);
 
-                var result = formatter.Read(memoryStream);
+                var result = await formatter.ReadAsync(memoryStream);
 
                 Assert.IsNotNull(result);
 
                 Assert.AreEqual(downloadLength, result.DownloadLength);
+                Assert.IsNotNull(result.DownloadedInfo);
                 Assert.AreEqual(downloadedInfo.Count, result.DownloadedInfo.Count);
 
                 for (int i = 0; i < downloadedInfo.Count; i++)

src/FileDownloader.Tests/BreakpointResumptionTransmissionRecordFileFormatterTest.cs

@@ -10,39 +10,39 @@ namespace FileDownloader.Tests
     [TestClass]
     public class DataRangeTest
     {
-        [ContractTestCase]
-        public void TryMerge()
-        {
-            "传入两个不相邻的 DataRange 对象，返回不可合并".Test(() =>
-            {
-                var a = new DataRange(0, 1);
-                var b = new DataRange(3, 2);
-                var result = DataRange.TryMerge(a, b, out var data);
-                Assert.AreEqual(false, result);
-            });
+        //[ContractTestCase]
+        //public void TryMerge()
+        //{
+        //    "传入两个不相邻的 DataRange 对象，返回不可合并".Test(() =>
+        //    {
+        //        var a = new DataRange(0, 1);
+        //        var b = new DataRange(3, 2);
+        //        var result = DataRange.TryMerge(a, b, out var data);
+        //        Assert.AreEqual(false, result);
+        //    });
 
-            "给定两个相邻的 DataRange 对象，传入顺序是先将传入起点较大的，再传入起点较小的，可以进行合并".Test(() =>
-            {
-                var a = new DataRange(0, 3);
-                var b = new DataRange(3, 2);
+        //    "给定两个相邻的 DataRange 对象，传入顺序是先将传入起点较大的，再传入起点较小的，可以进行合并".Test(() =>
+        //    {
+        //        var a = new DataRange(0, 3);
+        //        var b = new DataRange(3, 2);
 
-                // 传入顺序是先将传入起点较大的，再传入起点较小的
-                var result = DataRange.TryMerge(b, a, out var data);
-                Assert.IsTrue(result);
-                Assert.AreEqual(a.StartPoint, data.StartPoint);
-                Assert.AreEqual(a.Length + b.Length, data.Length);
-            });
+        //        // 传入顺序是先将传入起点较大的，再传入起点较小的
+        //        var result = DataRange.TryMerge(b, a, out var data);
+        //        Assert.IsTrue(result);
+        //        Assert.AreEqual(a.StartPoint, data.StartPoint);
+        //        Assert.AreEqual(a.Length + b.Length, data.Length);
+        //    });
 
-            "给定两个相邻的 DataRange 对象，可以进行合并".Test(() =>
-            {
-                var a = new DataRange(0, 3);
-                var b = new DataRange(3, 2);
+        //    "给定两个相邻的 DataRange 对象，可以进行合并".Test(() =>
+        //    {
+        //        var a = new DataRange(0, 3);
+        //        var b = new DataRange(3, 2);
 
-                var result = DataRange.TryMerge(a, b, out var data);
-                Assert.IsTrue(result);
-                Assert.AreEqual(a.StartPoint, data.StartPoint);
-                Assert.AreEqual(a.Length + b.Length, data.Length);
-            });
-        }
+        //        var result = DataRange.TryMerge(a, b, out var data);
+        //        Assert.IsTrue(result);
+        //        Assert.AreEqual(a.StartPoint, data.StartPoint);
+        //        Assert.AreEqual(a.Length + b.Length, data.Length);
+        //    });
+        //}
     }
 }

src/FileDownloader.Tests/DataRangeTest.cs

@@ -150,6 +150,8 @@ public interface IMockSegmentFileDownloader
         }
     }
 
+#pragma warning disable SYSLIB0014 //  warning SYSLIB0014: “WebRequest.WebRequest()”已过时:“WebRequest, HttpWebRequest, ServicePoint, and WebClient are obsolete. Use HttpClient instead.”
+    // 这里就是专门测试旧代码的，忽略即可
     class FakeHttpWebRequest : WebRequest
     {
         public FakeHttpWebRequest(FakeWebResponse fakeWebResponse)
@@ -192,4 +194,6 @@ public override Stream GetResponseStream()
             return Stream;
         }
     }
+#pragma warning restore SYSLIB0014
+
 }

src/FileDownloader.Tests/SegmentFileDownloaderTest.cs

@@ -1,5 +1,6 @@
 using System;
 using System.IO;
+using System.Net.Http;
 using System.Runtime.InteropServices.ComTypes;
 using System.Threading.Tasks;
 using CommandLine;
@@ -21,19 +22,29 @@ static async Task Main(string[] args)
 
                      // https://www.speedtest.cn/
                      var url =
-                          "https://speedtest1.gd.chinamobile.com.prod.hosts.ooklaserver.net:8080/download?size=25000000&r=0.2978374611691549";
+                         "https://node-103-27-27-20.speedtest.cn:51090/download?size=25000000&r=0.625866791098137";
                      url =
-                         "https://download.jetbrains.8686c.com/resharper/ReSharperUltimate.2020.1.3/JetBrains.ReSharperUltimate.2020.1.3.exe";
+                         "https://download.jetbrains.com/resharper/dotUltimate.2025.1/JetBrains.dotUltimate.2025.1.exe";
                      //var md5 = "7d6bbeb6617a7c0b7e615098fca1b167";// resharper
 
-                     url = "http://localhost:5000";
+                     //url = "http://localhost:5000";
 
-                     var file = new FileInfo(@"File.txt");
+                     // 这里的 gitdl.cn 是 iFileProxy 离线下载工具的地址，这是一个非常好的工具。开源地址： https://git.linxi.info/xianglin_admin/iFileProxy
+                     url = "https://gitdl.cn/https://github.com/srwi/EverythingToolbar/releases/download/1.5.2/EverythingToolbar-1.5.2.msi";
+
+                     //url =
+                     //    "https://down.pc.yyb.qq.com/pcyyb/packing/14e1e37f997f49a58d560ab97fa335aa/pcyyb_2702800040_installer.exe";
+                     //url = "https://pm.myapp.com/invc/xfspeed/qqpcmgr/download/QQPCDownload320001.exe";
+                     //// 这个地址带了 Content-Disposition 头，文件名是从这个头中获取的
+                     url =
+                         "https://sw.pcmgr.qq.com/2f472366ca30d8ac1ad4acb64c77d2ad/680c8f51/spcmgr/download/BaiduNetdisk_txgj1_7.50.0.132.exe";
+                     //url = "https://pc-package.wpscdn.cn/wps/download/W.P.S.60.1955.exe";
+
+                     var downloadFolder = new DirectoryInfo(@"DownloadFolder");
 
                      var progress = new Progress<DownloadProgress>();
 
-                     var segmentFileDownloader = new SegmentFileDownloader(url, file, logger, progress);
-                     await segmentFileDownloader.DownloadFileAsync();
+                     await FileDownloaderHelper.DownloadFileToFolderAsync(url, downloadFolder, progress:progress);
 #endif
                      await Task.Delay(100);
                  });
@@ -76,7 +87,7 @@ private static async Task DownloadFileAsync(DownloadOption option)
 
                 var file = new FileInfo(output);
                 var url = option.Url;
-                var segmentFileDownloader = new SegmentFileDownloader(url, file, logger, progress);
+                using var segmentFileDownloader = new SegmentFileDownloaderByHttpClient(url, file, httpClient:null, logger, progress);
 
                 await segmentFileDownloader.DownloadFileAsync();
 

src/dotnetCampus.FileDownloader.Tool/Program.cs

@@ -17,7 +17,7 @@ public class DownloadFileListManager
         /// 读取本地存储的下载列表
         /// </summary>
         /// <returns></returns>
-        public async Task<List<DownloadFileInfo>> ReadDownloadedFileListAsync()
+        public async Task<List<DownloadFileInfo>?> ReadDownloadedFileListAsync()
         {
             var file = GetStorageFilePath();
 

src/dotnetCampus.FileDownloader.WPF/Business/DownloadFileListManager.cs

@@ -16,6 +16,7 @@
 
 using Microsoft.Extensions.Logging;
 
+// ReSharper disable once CheckNamespace
 namespace dotnetCampus.FileDownloader;
 
 /// <summary>
@@ -61,7 +62,7 @@ public SegmentFileDownloaderByHttpClient(string url, FileInfo file,
         _shouldDisposeHttpClient = httpClient is null;
         HttpClient = httpClient ?? CreateDefaultHttpClient();
 
-        DownloadDataList = Channel.CreateUnbounded<DownloadData>(new UnboundedChannelOptions()
+        DownloadDataChannel = Channel.CreateUnbounded<DownloadData>(new UnboundedChannelOptions()
         {
             SingleReader = false,
             AllowSynchronousContinuations = true,
@@ -135,6 +136,12 @@ private static SocketsHttpHandler CreateDefaultSocketsHttpHandler()
             //{
             //    return ValueTask.FromResult(context.PlaintextStream);
             //}
+
+            // 忽略证书错误
+            //SslOptions =
+            //{
+            //    RemoteCertificateValidationCallback = (sender, certificate, chain, errors) => true
+            //}
         };
 
         return socketsHttpHandler;
@@ -163,12 +170,12 @@ private static SocketsHttpHandler CreateDefaultSocketsHttpHandler()
     /// </summary>
     public string Url { get; }
 
-    private Channel<DownloadData> DownloadDataList { get; }
+    private Channel<DownloadData> DownloadDataChannel { get; }
 
     /// <summary>
-    /// 被加入到 <see cref="DownloadDataList"/> 的下载数量
+    /// 被加入到 <see cref="DownloadDataChannel"/> 的下载数量
     /// </summary>
-    private int _workTaskCount;
+    private int _workingTaskCount;
 
     /// <summary>
     /// 下载的文件
@@ -233,7 +240,7 @@ private async void ControlSwitch()
         {
             LogDebugInternal("Start ControlSwitch");
             var (segment, runCount, maxReportTime) = SegmentManager.GetMaxWaitReportTimeDownloadSegmentStatus();
-            var waitCount = _workTaskCount;
+            var waitCount = _workingTaskCount;
 
             LogDebugInternal("ControlSwitch 当前等待数量：{0},待命最大响应时间：{1},运行数量：{2},运行线程{3}", waitCount, maxReportTime, runCount, _threadCount);
 
@@ -320,10 +327,13 @@ public async ValueTask DownloadFileAsync()
             return;
         }
 
-        FileStream = File.Create();
-        FileStream.SetLength(contentLength);
+        FileStream = File.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read/*允许边下边播*/);
+        if (FileStream.Length == 0)
+        {
+            // 如果是刚刚创建的，则预先分配空间。实际测试下载器预先分配空间能够获取更好的机械硬盘性能
+            FileStream.SetLength(contentLength);
+        }
         FileWriter = new RandomFileWriterWithOrderFirst(FileStream);
-        FileWriter.StepWriteFinished += (sender, args) => SharedArrayPool.Return(args.Data);
 
         if (BreakpointResumptionTransmissionRecordFile is null)
         {
@@ -333,11 +343,13 @@ public async ValueTask DownloadFileAsync()
         else
         {
             // 有断点续传
-            var manager = new BreakpointResumptionTransmissionManager(BreakpointResumptionTransmissionRecordFile, FileWriter, contentLength);
+            var manager = new BreakpointResumptionTransmissionManager(BreakpointResumptionTransmissionRecordFile, FileWriter, SharedArrayPool, contentLength, BufferLength);
             // 有断点续传情况下，先读取断点续传文件，通过此文件获取到需要下载的内容
-            SegmentManager = manager.CreateSegmentManager();
+            SegmentManager = await manager.CreateSegmentManagerAsync(FileStream);
             BreakpointResumptionTransmissionManager = manager;
         }
+        // 由于 BreakpointResumptionTransmissionManager 也在监控 StepWriteFinished 事件，如果这里的事件加等更快执行，则会导致数据已经还给了池，被其他地方使用，在 BreakpointResumptionTransmissionManager 存在线程安全
+        FileWriter.StepWriteFinished += (sender, args) => SharedArrayPool.Return(args.Data);
 
         _progress.Report(new DownloadProgress($"file length = {contentLength}", SegmentManager));
 
@@ -559,15 +571,20 @@ private async ValueTask DownloadTask()
             DownloadData data;
             try
             {
-                var canRead = await DownloadDataList.Reader.WaitToReadAsync();
+                var canRead = await DownloadDataChannel.Reader.WaitToReadAsync();
                 if (!canRead)
                 {
                     // 不能读取了，那就返回吧
                     return;
                 }
 
-                data = await DownloadDataList.Reader.ReadAsync().ConfigureAwait(false);
-                Interlocked.Decrement(ref _workTaskCount);
+                if (!DownloadDataChannel.Reader.TryRead(out data))
+                {
+                    // 居然读取不到数据，那就再次进入循环吧
+                    continue;
+                }
+
+                Interlocked.Decrement(ref _workingTaskCount);
             }
             catch (ChannelClosedException)
             {
@@ -712,8 +729,8 @@ private void LogDownloadSegment(DownloadSegment downloadSegment)
     private async void Download(HttpResponseMessage? httpResponseMessage, DownloadSegment downloadSegment)
     {
         LogDebugInternal("[Download] Enqueue Download. {0}", downloadSegment);
-        await DownloadDataList.Writer.WriteAsync(new DownloadData(httpResponseMessage, downloadSegment)).ConfigureAwait(false);
-        Interlocked.Increment(ref _workTaskCount);
+        await DownloadDataChannel.Writer.WriteAsync(new DownloadData(httpResponseMessage, downloadSegment)).ConfigureAwait(false);
+        Interlocked.Increment(ref _workingTaskCount);
     }
 
     private void Download(DownloadSegment? downloadSegment)
@@ -748,7 +765,7 @@ private async ValueTask FinishDownload()
         await FileWriter.DisposeAsync().ConfigureAwait(false);
         await FileStream.DisposeAsync().ConfigureAwait(false);
 
-        DownloadDataList.Writer.Complete();
+        DownloadDataChannel.Writer.Complete();
 
         BreakpointResumptionTransmissionManager?.Dispose();
         // 默认下载完成删除断点续传文件

src/dotnetCampus.FileDownloader/DownloadByHttpClient/SegmentFileDownloaderByHttpClient.cs

@@ -48,8 +48,19 @@ public DownloadSegment(long startPoint, long requirementDownloadPoint)
         /// 当前的信息，仅用于调试
         /// </summary>
         public string? Message { get; set; }
+
+        /// <summary>
+        /// 最后的下载状态更新时间
+        /// </summary>
+        /// todo 改名 LastDownloadTime 或 LastUpdateTime
         public DateTime LastDownTime { get; set; } = DateTime.Now;
+
+        /// <summary>
+        /// 下载状态
+        /// </summary>
+        /// todo 改名 DownloadingState
         public DownloadingState LoadingState { get; set; } = DownloadingState.Pause;
+
         /// <summary>
         /// 需要下载到的点
         /// </summary>
@@ -98,13 +109,24 @@ internal set
         /// </summary>
         public SegmentManager? SegmentManager { set; get; }
     }
+
     /// <summary>
     /// 下载状态
     /// </summary>
     public enum DownloadingState
     {
+        /// <summary>
+        /// 运行中
+        /// </summary>
+        /// todo 改名 Running 
         Runing = 1,
+        /// <summary>
+        /// 暂停
+        /// </summary>
         Pause = 0,
+        /// <summary>
+        /// 完成
+        /// </summary>
         Finished = -1
     }
 }