Duplicate sources are comming while exporting the QnA pairs

Question

Hi,

I'm getting duplicate QnA pairs which are coming from 2 sources(one is updated and other is old one). I'm using blob url as source url which will gets expired after 60min (appending sas token as well). I created an automation to update the kb's frequently. I'm using the below code to get resources, delete them and add new resources and I'm deploying them.

This is the first time I'm facing this issue. This automation ran multiple times from the past few days but never got this kind of issue.

Can you please help in investigating & resolving this issue?

public async Task UpdateKnowledgeBaseAsync(string sourceUrl, string projectName)
        {
            var credential = new AzureKeyCredential(Configuration["Ocp-Apim-Subscription-Key"]);
            var endpoint = new Uri(Configuration["CognitiveServiceEndpoint"]);
            var client = new QuestionAnsweringAuthoringClient(endpoint, credential);

            //Get sources
            List<string> sources = new List<string>();
            await foreach (var data in client.GetSourcesAsync(projectName))
            {
                JsonElement result = JsonDocument.Parse(data.ToStream()).RootElement;
                sources.Add(result.GetProperty("sourceUri").ToString());
            }


            // delete knowledge base
            foreach (var item in sources)
            {
                if (item != null)
                {
                    var deleteData = new[] {
                new {
                    op = "delete",
                    value = new {
                        source = item,
                        sourceUri = item,
                        sourceKind = "url"
                    },
                }
            };

                    //Cognitive Service Call

                    var deleteOperation = await client.UpdateSourcesAsync(WaitUntil.Started, projectName, RequestContent.Create(deleteData));

                    var deleteResponse = await deleteOperation.WaitForCompletionAsync();
                }
            }

            //add knowledge base

            var addData = new[] {
                new {
                    op = "add",
                    value = new {
                        source = sourceUrl,
                        sourceUri = sourceUrl,
                        sourceKind = "url"
                    },
                }
            };

            //Cognitive Service Call

            var addOperation = await client.UpdateSourcesAsync(WaitUntil.Started, projectName, RequestContent.Create(addData));

            var addResponse = await addOperation.WaitForCompletionAsync();

            //Deploy project name

            var deployOperartion = await client.DeployProjectAsync(WaitUntil.Started, projectName, "production");

            var deployResponse = await deployOperartion.WaitForCompletionAsync();

        }