Download 10 Million files from SharePoint Online - C# Graph API

Question

I want to download and process files from my SharePoint Online environment. For now ignore Processing Files. I have around 10Millions files in my SP Online Environment. Can anyone suggest a way to download all these files in a fastest way possible.

I want to use C#, Graph API with .NET6 and above. I have all SP admin, Global Admin Privileges.

Thank you in advance

Answer 1

Hi @Ahmednasr Nasr

language C# and .NET

suppose that you have a list of file IDs in the format "file-id-{i}", which you should replace with your actual file IDs.

using System;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading.Tasks;
using Microsoft.Graph;
using Microsoft.Identity.Client;

class Program
{
    static async Task Main(string[] args)
    {
        string clientId = "your-client-id";
        string clientSecret = "your-client-secret";
        string tenantId = "your-tenant-id";
        string siteId = "your-site-id";
        string driveId = "your-drive-id";
        int batchSize = 100;
        
        var confidentialClientApplication = ConfidentialClientApplicationBuilder
            .Create(clientId)
            .WithClientSecret(clientSecret)
            .WithAuthority(new Uri($"https://login.microsoftonline.com/{tenantId}"))
            .Build();

        string[] scopes = new[] { "https://graph.microsoft.com/.default" };
        var authenticationResult = await confidentialClientApplication
            .AcquireTokenForClient(scopes)
            .ExecuteAsync();

        var graphServiceClient = new GraphServiceClient(new DelegateAuthenticationProvider(requestMessage =>
        {
            requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", authenticationResult.AccessToken);
            return Task.FromResult(0);
        }));

        int totalFiles = 10000000;
        int totalPages = (int)Math.Ceiling((double)totalFiles / batchSize);

        for (int currentPage = 0; currentPage < totalPages; currentPage++)
        {
            int skip = currentPage * batchSize;

            var batchRequestContent = new BatchRequestContent();

            for (int i = skip; i < skip + batchSize; i++)
            {
                if (i >= totalFiles)
                    break;

                var request = graphServiceClient
                    .Sites[siteId]
                    .Drives[driveId]
                    .Items[$"file-id-{i}"] // Replace with actual file ID
                    .Content
                    .Request()
                    .GetHttpRequestMessage();

                batchRequestContent.AddBatchRequestStep(new BatchRequestStep(i.ToString(), request, null));
            }

            var batchResponse = await graphServiceClient.Batch.Request().PostAsync(batchRequestContent);

            foreach (var response in batchResponse)
            {
                if (response.Value.IsSuccessStatusCode)
                {
                    var fileStream = await response.Value.Content.ReadAsStreamAsync();
                    // Process fileStream (save to disk, etc.)
                }
                else
                {
                    var errorResponse = await response.Value.Content.ReadAsStringAsync();
                    // Handle error
                    Console.WriteLine($"Error downloading file: {errorResponse}");
                }
            }
        }
    }
}