ASP.NET Core Web API - How to call a 3rd party web service faster?

Question

Hi there,

I upgraded my legacy asp.net web API to the CORE with some adjustments. There is a 3rd party web service I am calling inside of my API. This 3rd party has a limitation of a maximum of 10 results/codes inside of a single request. There are times it is needed to get 20.000 results/codes from this service. I implemented an action method like this in order to get the codes, made some selects and inserts into the database, etc. I wonder If I can get those results/codes faster with parallel service calls or something else. In this current solution, a user triggers my web API from a web application first, then my web API calls the 3rd party web service, and the results/codes are displayed on the web application.

Here is the web API calling 3rd party service:

public async Task<HttpResponseMessage> CallServiceMultiple(RequestDto requestDto)  
        {  
            HttpResponseMessage response = null;  
            var customerId = int.Parse(_http.HttpContext.User.Claims.FirstOrDefault(x => x.Type == "UserId")?.Value);  
            var coupons = new List<Coupon>();  
            GameConfirmResponse gameConfirmResponse = null;  
  
            var requestedAmount = requestDto.quantity;  
            var requestClientCode = requestDto.productCode;  
  
            //ProductCode Conversion  
            var productCode = _unitOfWork.ProductCodeRepository.GetByCode(p => p.clientCode == requestDto.productCode);  
            if (productCode != null)  
            {  
                requestDto.productCode = productCode.gameCode;  
            }  
              
            while (requestedAmount > 0)  
            {  
                var gameRequest = _mapper.Map<RequestOyunPalasDto, GameRequest>(requestDto);  
  
                var count = Math.Min(requestedAmount, 10);  
                gameRequest.quantity = count;  
  
                //Unique reference ID  
                gameRequest.referenceId = Guid.NewGuid();  
                  
                var gameRequestDto = _mapper.Map<GameRequest, GameRequestDto>(gameRequest);  
  
                //Create signature  
                gameRequest = Utilities.Utilities.CreateSignature(gameRequestDto, RequestType.Initiate,_configuration);  
  
                //Set service  
                gameRequest.service = "ABC";  
                gameRequest.customerID = customerId; //Get from token  
  
                //Add initiation request into database  
                _unitOfWork.GameRepository.Insert(gameRequest);  
  
                #region Call Razer initiate/confirm  
  
                //Call Razer for initiation  
                response = await ABCApiCaller.CallRazer(gameRequest, "purchaseinitiation",_configuration);  
  
                //Read response  
                var htmlResponse = await response.Content.ReadAsStringAsync();  
  
                var gameResponse = JsonConvert.DeserializeObject<GameResponse>(htmlResponse);  
  
                //Adding initiation response into database  
                _unitOfWork.GameResponseRepository.Insert(gameResponse);  
  
                if (gameResponse.initiationResultCode == "00")  
                {  
                    gameRequestDto = _mapper.Map<GameRequest, GameRequestDto>(gameRequest);  
  
                    gameRequestDto.validatedToken = gameResponse.validatedToken;  
                    //Create signature  
                    var gameConfirmRequest = Utilities.Utilities.CreateSignature(gameRequestDto, RequestType.Confirm,_configuration);  
  
                    //Transform DTO into GameRequest for calling Razer Initiate  
  
                    var gameConfirmRequests = _mapper.Map<GameRequest, GameConfirmRequest>(gameConfirmRequest);  
  
                    //Add confirm request into database  
                    _unitOfWork.GameConfirmRequestRepository.Insert(gameConfirmRequests);  
  
                    //Call Razer for confirm  
                    response = await ABCApiCaller.CallRazer(gameConfirmRequest, "purchaseconfirmation",_configuration);  
  
                    //Read response  
                    htmlResponse = await response.Content.ReadAsStringAsync();  
                    //Convert UTC to Local Time  
                    var settings = new JsonSerializerSettings { DateTimeZoneHandling = DateTimeZoneHandling.Local };  
  
                    gameConfirmResponse = JsonConvert.DeserializeObject<GameConfirmResponse>(htmlResponse, settings);  
  
                    //add coupons  
                    coupons.AddRange(gameConfirmResponse.coupons);  
  
                    //Set service and client code  
                    gameConfirmResponse.service = "ABC";  
                    gameConfirmResponse.productCode = requestClientCode;  
                    gameConfirmResponse.status = 1; //Default Confirmed!  
  
                    //Add confirm response into database  
                    _unitOfWork.GameConfirmResponseRepository.Insert(gameConfirmResponse);  
  
                    //Add Confirm Cancel request into database - Confirmed!  
                    var confirmCancel = new ConfirmCancel { referenceId = gameConfirmResponse.referenceId, status = 1};  
                    _unitOfWork.ConfirmCancelRepository.Insert(confirmCancel);  
                }  
                  
                requestedAmount -= count;  
  
            }  
            #endregion Call Razer initiate/confirm  
  
            await _unitOfWork.SaveAsync();  
  
            if (gameConfirmResponse == null) return response;  
            gameConfirmResponse.coupons = coupons;  
            gameConfirmResponse.quantity = requestDto.quantity;  
            gameConfirmResponse.totalPrice = gameConfirmResponse.unitPrice * requestDto.quantity;  
  
            //Manipulate Response in order to send Client Product Code  
            var resultResponse = JsonConvert.SerializeObject(gameConfirmResponse, Formatting.Indented,  
                new JsonSerializerSettings()  
                {  
                    ReferenceLoopHandling = ReferenceLoopHandling.Ignore  
                });  
            response = new HttpResponseMessage  
            {  
                StatusCode = System.Net.HttpStatusCode.OK,  
                Content = new StringContent(resultResponse, System.Text.Encoding.UTF8, "application/json"),  
            };  
  
            return response;  
  
  
        }  

Calling API:

 public static class ABCApiCaller  
        {  
            public static async Task<HttpResponseMessage> CallABC(GameRequest gameRequest, string url, IConfiguration iConfiguration)  
            {  
                  
                HttpResponseMessage response;  
      
                //Transform GameRequest into ProductDTO  
                var config = new MapperConfiguration(cfg => { cfg.CreateMap<GameRequest, ProductRequestDto>(); });  
                var iMapper = config.CreateMapper();  
                var productRequest = iMapper.Map<GameRequest, ProductRequestDto>(gameRequest);  
      
                if (url == "Product/")  
                {  
                    var request = (HttpWebRequest)WebRequest.Create(iConfiguration.GetSection("ABC:Url").Value + url);  
      
                    request.ContentType = "application/x-www-form-urlencoded";  
                    request.Method = "POST";  
                    request.KeepAlive = false;  
                    var keyValueContent = productRequest.ToKeyValues();  
                    var formUrlEncodedContent = new FormUrlEncodedContent(keyValueContent);  
                    var urlEncodedString = await formUrlEncodedContent.ReadAsStringAsync();  
      
                    await using (var streamWriter = new StreamWriter(await request.GetRequestStreamAsync()))  
                    {  
                        await streamWriter.WriteAsync(urlEncodedString);  
                    }  
      
                    var httpResponse = (HttpWebResponse)(await request.GetResponseAsync());  
      
                    response = new HttpResponseMessage  
                    {  
                        StatusCode = httpResponse.StatusCode,  
                        Content = new StreamContent(httpResponse.GetResponseStream()),  
                    };  
      
                    return response;  
      
      
                }  
                else  
                {  
      
                    var request = (HttpWebRequest)WebRequest.Create(iConfiguration.GetSection("ABC:Url").Value + url);  
                    request.ContentType = "application/x-www-form-urlencoded";  
                    request.Method = "POST";  
                    request.KeepAlive = false;  
      
                    var keyValueContent = gameRequest.ToKeyValues();  
                    var formUrlEncodedContent = new FormUrlEncodedContent(keyValueContent);  
                    var urlEncodedString = await formUrlEncodedContent.ReadAsStringAsync();  
      
                    await using (var streamWriter = new StreamWriter(await request.GetRequestStreamAsync()))  
                    {  
                        await streamWriter.WriteAsync(urlEncodedString);  
                    }  
      
                    var httpResponse = (HttpWebResponse)(await request.GetResponseAsync());  
      
                    response = new HttpResponseMessage  
                    {  
                        StatusCode = httpResponse.StatusCode,  
                        Content = new StreamContent(httpResponse.GetResponseStream()),  
                    };  
      
                    return response;  
      
                }  
            }  
        }  
      
        public static class ObjectExtension  
        {  
            public static IDictionary<string, string?> ToKeyValues(this object? metaToken)  
            {  
                while (true)  
                {  
                    if (metaToken == null)  
                    {  
                        return null;  
                    }  
      
                    if (metaToken is not JToken token)  
                    {  
                        metaToken = JObject.FromObject(metaToken);  
                        continue;  
                    }  
      
                    if (token.HasValues)  
                    {  
                        var contentData = new Dictionary<string, string>();  
      
                        return token.Children().ToList().Select(child => child.ToKeyValues())  
                            .Where(childContent => childContent != null).Aggregate(contentData,  
                                (current, childContent) =>  
                                    current.Concat(childContent).ToDictionary(k => k.Key, v => v.Value));  
                    }  
      
                    var jValue = token as JValue;  
                    if (jValue?.Value == null)  
                    {  
                        return null;  
                    }  
      
                    var value = jValue?.Type == JTokenType.Date  
                        ? jValue?.ToString("o", CultureInfo.InvariantCulture)  
                        : jValue?.ToString(CultureInfo.InvariantCulture);  
      
                    return new Dictionary<string, string?> { { token.Path, value } };  
                    break;  
                }  
            }  
        }  

Answer 1

The parallel ForEach is probably the best. Before starting the parallel, I’d map the list to a collect of list the max size. Then the ForEach can process the mapped list.

https://learn.microsoft.com/en-us/dotnet/standard/parallel-programming/how-to-write-a-simple-parallel-foreach-loop

Answer 2

How can I implement Parallel.ForEachAsync for my solution?