How to use httpclient task async to download multiple files with a progressbar report and to be able to cancel/resume ?

Question

The Downloader class :

using System;  
using System.Buffers;  
using System.Collections.Generic;  
using System.IO;  
using System.Linq;  
using System.Net.Http;  
using System.Text;  
using System.Threading.Tasks;  
  
namespace Download_Http  
{  
    class Downloader  
    {  
        public delegate void DownloadProgressHandler(long? totalFileSize, long totalBytesDownloaded, double? progressPercentage);  
  
        public static class DownloadWithProgress  
        {  
            public static async Task ExecuteAsync(HttpClient httpClient, string downloadPath, string destinationPath, DownloadProgressHandler progress, Func<HttpRequestMessage> requestMessageBuilder = null)  
            {  
                if (requestMessageBuilder != null)  
                    GetDefaultRequestBuilder(downloadPath);  
                var download = new HttpClientDownloadWithProgress(httpClient, destinationPath, requestMessageBuilder);  
                download.ProgressChanged += progress;  
                await download.StartDownload();  
                download.ProgressChanged -= progress;  
            }  
  
            private static Func<HttpRequestMessage> GetDefaultRequestBuilder(string downloadPath)  
            {  
                return () => new HttpRequestMessage(HttpMethod.Get, downloadPath);  
            }  
        }  
  
        internal class HttpClientDownloadWithProgress  
        {  
            private readonly HttpClient _httpClient;  
            private readonly string _destinationFilePath;  
            private readonly Func<HttpRequestMessage> _requestMessageBuilder;  
            private int _bufferSize = 8192;  
  
            public event DownloadProgressHandler ProgressChanged;  
  
            public HttpClientDownloadWithProgress(HttpClient httpClient, string destinationFilePath, Func<HttpRequestMessage> requestMessageBuilder)  
            {  
                _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));  
                _destinationFilePath = destinationFilePath ?? throw new ArgumentNullException(nameof(destinationFilePath));  
                _requestMessageBuilder = requestMessageBuilder ?? throw new ArgumentNullException(nameof(requestMessageBuilder));  
            }  
  
            public async Task StartDownload()  
            {  
                var requestMessage = _requestMessageBuilder.Invoke();  
                var response = await _httpClient.SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead);  
                await DownloadAsync(response);  
            }  
  
            private async Task DownloadAsync(HttpResponseMessage response)  
            {  
                response.EnsureSuccessStatusCode();  
  
                var totalBytes = response.Content.Headers.ContentLength;  
  
                using (var contentStream = await response.Content.ReadAsStreamAsync())  
                    await ProcessContentStream(totalBytes, contentStream);  
            }  
  
            private async Task ProcessContentStream(long? totalDownloadSize, Stream contentStream)  
            {  
                var totalBytesRead = 0L;  
                var readCount = 0L;  
                var buffer = ArrayPool<byte>.Shared.Rent(_bufferSize);  
                var isMoreToRead = true;  
  
                using (var fileStream = new FileStream(_destinationFilePath, FileMode.Create, FileAccess.Write, FileShare.None, _bufferSize, true))  
                {  
                    do  
                    {  
                        var bytesRead = await contentStream.ReadAsync(buffer, 0, buffer.Length);  
                        if (bytesRead == 0)  
                        {  
                            isMoreToRead = false;  
                            ReportProgress(totalDownloadSize, totalBytesRead);  
                            continue;  
                        }  
  
                        await fileStream.WriteAsync(buffer, 0, bytesRead);  
  
                        totalBytesRead += bytesRead;  
                        readCount += 1;  
  
                        if (readCount % 100 == 0)  
                            ReportProgress(totalDownloadSize, totalBytesRead);  
                    }  
                    while (isMoreToRead);  
                }  
  
                ArrayPool<byte>.Shared.Return(buffer);  
            }  
  
            private void ReportProgress(long? totalDownloadSize, long totalBytesRead)  
            {  
                double? progressPercentage = null;  
                if (totalDownloadSize.HasValue)  
                    progressPercentage = Math.Round((double)totalBytesRead / totalDownloadSize.Value * 100, 2);  
  
                ProgressChanged?.Invoke(totalDownloadSize, totalBytesRead, progressPercentage);  
            }  
        }  
    }  
}  

And form1 trying to use it without success :

using System;  
using System.Collections.Generic;  
using System.ComponentModel;  
using System.Data;  
using System.Drawing;  
using System.IO;  
using System.Linq;  
using System.Net.Http;  
using System.Text;  
using System.Threading.Tasks;  
using System.Windows.Forms;  
  
namespace Download_Http  
{  
    public partial class Form1 : Form  
    {  
  
        public Form1()  
        {  
            InitializeComponent();  
        }  
  
        private void Form1_Load(object sender, EventArgs e)  
        {  
  
        }  
  
        private async void button1_Click(object sender, EventArgs e)  
        {  
            HttpClient client = new HttpClient();  
            const string url = "https://speed.hetzner.de/100MB.bin";  
  
            Downloader.DownloadProgressHandler progressHandler = null;  
            await Downloader.DownloadWithProgress.ExecuteAsync(client,url, @"d:\Videos\", progressHandler, () =>  
            {  
                var requestMessage = new HttpRequestMessage(HttpMethod.Get, url);  
                requestMessage.Headers.Accept.TryParseAdd("application/octet-stream");  
                return requestMessage;  
            });  
        }  
    }  
}  

The second problem is how to report progress to progressBar1 or labels ? how to add events like progress changed completed ? how to download multiple files from a list<string> or url's ?

I have in the form1 designer progressBar1 control.

Answer 1

HttpClient doesn't support progress. Your code isn't actually doing what you think it is in my opinion. You're tracking the progress of reading the stream but to get that stream you've already used the client to send the request and get the response. Whether the stream you're reading has already read all the contents or not is an implementation detail partially tied to the client and server impl. For large files it may behave the way you want but for small downloads it likely has already gotten all the content.

Here's a library I've never used that supposedly adds progress reporting to HttpClient. I would recommend you use it.

The second problem is how to report progress to progressBar1 or labels ? how to add events like progress changed completed ?

Use the library I mentioned. Their readme shows you how to handle progress notifications. However be aware that the code is async so you need to marshal the event back to the UI thread. Something like this might work.

   progress.ProgressChanged += OnDownloadProgress;  
     
   void OnDownloadProgress ( object sender, float progress )  
   {  
      if (_progressBar1.InvokeRequired)  
         _progressBar1.Invoke(OnDownloadProgress, new object[] { progress });  
      else  
         _progressBar1.Value = (int)progress;  //Assuming the progress is between 0 and 100  
   }  

When the download call returns the download is complete so do any post-processing at that point.

how to download multiple files from a list<string> or url's ?

To download more than 1 file at a time you'll need to start the downloads and not wait for the results until all downloads are started. For example assume that you are getting the list of files to download then you'd do something like this.

   async Task DownloadFilesAsync ( string[] files )  
   {  
      var tasks = new List<Task>();    
      foreach (var file in files)  
      {  
          tasks.Add(DownloadFileAsync(file));  
      };  
     
      await Task.WhenAll(tasks);  
   }  
     
   Task DownloadFIleAsync ( string file )  
   {  
      //Your download code for a single file goes here  
   }  

Note that the above is just starter code. You'll need to flush it out with any data you might want to return. You would also need to adjust your progress bar for downloading multiple files. Also be aware that you really shouldn't be creating HttpClient instances in your code haphazardly. They are shared resources and need to be cleaned up otherwise you'll run out of ports. The best option is to create a single HttpClient instance for your app (or per base URL) and reuse it. HttpClient is thread safe provided you don't modify the base address or default request headers. If you need to send per-request headers then use the SendAsync method that accepts the HttpRequestMessage that you can provide.

If you don't particularly care about HttpClient then you might find WebClient to be easier to use and it supports progress notifications automatically.