From 86f7da48ff754d835c7e0630864b6ef7e726c7c6 Mon Sep 17 00:00:00 2001 From: Henry van der Vegte Date: Thu, 10 Nov 2022 10:10:44 +0100 Subject: [PATCH] [IngestionClient] Retry HttpStatusCodeException and do not fail whole transcription on transcript download failure (#1734) --- .../ingestion-client/Connector/BatchClient.cs | 2 +- .../TranscriptionProcessor.cs | 40 ++++++++++++------- .../Setup/ArmTemplateBatch.json | 2 +- .../Setup/ArmTemplateRealtime.json | 2 +- 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/samples/ingestion/ingestion-client/Connector/BatchClient.cs b/samples/ingestion/ingestion-client/Connector/BatchClient.cs index ba24eb318..88a91b378 100644 --- a/samples/ingestion/ingestion-client/Connector/BatchClient.cs +++ b/samples/ingestion/ingestion-client/Connector/BatchClient.cs @@ -33,7 +33,7 @@ public static class BatchClient private static readonly AsyncRetryPolicy RetryPolicy = Policy - .Handle() + .Handle(e => e is HttpStatusCodeException || e is HttpRequestException) .WaitAndRetryAsync(MaxNumberOfRetries, retryAttempt => TimeSpan.FromSeconds(2)); public static Task GetTranscriptionReportFileFromSasAsync(string sasUri) diff --git a/samples/ingestion/ingestion-client/FetchTranscription/TranscriptionProcessor.cs b/samples/ingestion/ingestion-client/FetchTranscription/TranscriptionProcessor.cs index 657fc9c50..39993e964 100644 --- a/samples/ingestion/ingestion-client/FetchTranscription/TranscriptionProcessor.cs +++ b/samples/ingestion/ingestion-client/FetchTranscription/TranscriptionProcessor.cs @@ -9,6 +9,7 @@ namespace FetchTranscriptionFunction using System.Collections.Generic; using System.Linq; using System.Net; + using System.Net.Http; using System.Text; using System.Threading.Tasks; using System.Xml; @@ -355,29 +356,40 @@ private async Task ProcessSucceededTranscriptionAsync(string transcriptionLocati foreach (var resultFile in resultFiles) { log.LogInformation($"Getting result for file {resultFile.Name}"); - var transcriptionResult = await BatchClient.GetSpeechTranscriptFromSasAsync(resultFile.Links.ContentUrl).ConfigureAwait(false); - if (string.IsNullOrEmpty(transcriptionResult.Source)) + try { - var errorMessage = "Transcription source is unknown, skipping evaluation."; - log.LogError(errorMessage); + var transcriptionResult = await BatchClient.GetSpeechTranscriptFromSasAsync(resultFile.Links.ContentUrl).ConfigureAwait(false); - generalErrorsStringBuilder.AppendLine(errorMessage); - continue; - } + if (string.IsNullOrEmpty(transcriptionResult.Source)) + { + var errorMessage = "Transcription source is unknown, skipping evaluation."; + log.LogError(errorMessage); + + generalErrorsStringBuilder.AppendLine(errorMessage); + continue; + } - var audioFileName = StorageConnector.GetFileNameFromUri(new Uri(transcriptionResult.Source)); - var audioFileInfo = serviceBusMessage.AudioFileInfos.Where(a => a.FileName == audioFileName).First(); + var audioFileName = StorageConnector.GetFileNameFromUri(new Uri(transcriptionResult.Source)); + var audioFileInfo = serviceBusMessage.AudioFileInfos.Where(a => a.FileName == audioFileName).First(); - if (speechTranscriptMappings.ContainsKey(audioFileInfo)) + if (speechTranscriptMappings.ContainsKey(audioFileInfo)) + { + var errorMessage = $"Duplicate audio file in job, skipping: {audioFileInfo.FileName}."; + log.LogError(errorMessage); + generalErrorsStringBuilder.AppendLine(errorMessage); + continue; + } + + speechTranscriptMappings.Add(audioFileInfo, transcriptionResult); + } + catch (Exception e) when (e is HttpStatusCodeException || e is HttpRequestException) { - var errorMessage = $"Duplicate audio file in job, skipping: {audioFileInfo.FileName}."; + var errorMessage = $"Failed getting speech transcript from content url: {e.Message}"; log.LogError(errorMessage); + generalErrorsStringBuilder.AppendLine(errorMessage); - continue; } - - speechTranscriptMappings.Add(audioFileInfo, transcriptionResult); } if (textAnalyticsProvider != null && (FetchTranscriptionEnvironmentVariables.SentimentAnalysisSetting != SentimentAnalysisSetting.None diff --git a/samples/ingestion/ingestion-client/Setup/ArmTemplateBatch.json b/samples/ingestion/ingestion-client/Setup/ArmTemplateBatch.json index f54de0bbd..2e55afa06 100644 --- a/samples/ingestion/ingestion-client/Setup/ArmTemplateBatch.json +++ b/samples/ingestion/ingestion-client/Setup/ArmTemplateBatch.json @@ -227,7 +227,7 @@ } }, "variables": { - "Version": "v2.0.0", + "Version": "v2.0.1", "AudioInputContainer": "audio-input", "AudioProcessedContainer": "audio-processed", "ErrorFilesOutputContainer": "audio-failed", diff --git a/samples/ingestion/ingestion-client/Setup/ArmTemplateRealtime.json b/samples/ingestion/ingestion-client/Setup/ArmTemplateRealtime.json index 8e1aba06a..daed694d5 100644 --- a/samples/ingestion/ingestion-client/Setup/ArmTemplateRealtime.json +++ b/samples/ingestion/ingestion-client/Setup/ArmTemplateRealtime.json @@ -123,7 +123,7 @@ } }, "variables": { - "Version": "v2.0.0", + "Version": "v2.0.1", "AudioInputContainer": "audio-input", "AudioProcessedContainer": "audio-processed", "ErrorFilesOutputContainer": "audio-failed",