diff --git a/samples/ingestion/ingestion-client/Connector/Constants.cs b/samples/ingestion/ingestion-client/Connector/Constants.cs index e2b3012dc..466a52f4d 100644 --- a/samples/ingestion/ingestion-client/Connector/Constants.cs +++ b/samples/ingestion/ingestion-client/Connector/Constants.cs @@ -28,5 +28,7 @@ public static class Constants public const int DefaultFilesPerTranscriptionJob = 100; public const int DefaultConversationAnalysisMaxChunkSize = 5000; + + public const string SummarizationSupportedLocalePrefix = "en"; } } \ No newline at end of file diff --git a/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/AnalyzeConversationSummarizationResults.cs b/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/AnalyzeConversationSummarizationResults.cs index 227f8c5d1..333416c31 100644 --- a/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/AnalyzeConversationSummarizationResults.cs +++ b/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/AnalyzeConversationSummarizationResults.cs @@ -12,6 +12,6 @@ namespace Connector.Serializable.Language.Conversations public class AnalyzeConversationSummarizationResults : AnalyzeConversationResultsBase { [JsonProperty("conversations")] - public IEnumerable Conversations { get; set; } + public IEnumerable Conversations { get; set; } } } \ No newline at end of file diff --git a/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/Aspect.cs b/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/Aspect.cs new file mode 100644 index 000000000..c42095a5d --- /dev/null +++ b/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/Aspect.cs @@ -0,0 +1,19 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// + +namespace Connector.Serializable.Language.Conversations +{ + using System.Text.Json.Serialization; + + [JsonConverter(typeof(JsonStringEnumConverter))] + public enum Aspect + { + None = 0, + Issue = 1, + Resolution = 2, + ChapterTitle = 3, + Narrative = 4, + } +} diff --git a/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/ConversationItem.cs b/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/ConversationItem.cs index 9fa82cc2d..efb76c4ad 100644 --- a/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/ConversationItem.cs +++ b/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/ConversationItem.cs @@ -28,7 +28,13 @@ public class ConversationItem [JsonProperty("maskedItn")] public string MaskedItn { get; set; } - [JsonProperty("audioTimings")] + [JsonProperty("wordLevelTimings")] public IEnumerable AudioTimings { get; set; } + + [JsonProperty("conversationItemLevelTiming")] + public AudioTiming ConversationItemLevelTiming { get; set; } + + [JsonProperty("role")] + public string Role { get; set; } } } \ No newline at end of file diff --git a/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/ConversationSummarizationOptions.cs b/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/ConversationSummarizationOptions.cs new file mode 100644 index 000000000..a455818db --- /dev/null +++ b/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/ConversationSummarizationOptions.cs @@ -0,0 +1,20 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// + +namespace Connector.Serializable.Language.Conversations +{ + using System.Collections.Generic; + + public class ConversationSummarizationOptions + { + public bool Enabled { get; init; } + + public int InputLengthLimit { get; init; } + + public RoleAssignmentStratergy Stratergy { get; init; } + + public IEnumerable Aspects { get; init; } + } +} diff --git a/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/IssueResolutionSummary.cs b/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/ConversationsSummaryResult.cs similarity index 73% rename from samples/ingestion/ingestion-client/Connector/Serializable/Conversations/IssueResolutionSummary.cs rename to samples/ingestion/ingestion-client/Connector/Serializable/Conversations/ConversationsSummaryResult.cs index 10988d0a5..90f867570 100644 --- a/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/IssueResolutionSummary.cs +++ b/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/ConversationsSummaryResult.cs @@ -1,4 +1,4 @@ -// +// // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE.md file in the project root for full license information. // @@ -8,13 +8,13 @@ namespace Connector.Serializable.Language.Conversations using System.Collections.Generic; using Newtonsoft.Json; - public class IssueResolutionSummary + public class ConversationsSummaryResult { [JsonProperty(PropertyName = "id")] public string Id { get; set; } [JsonProperty(PropertyName = "summaries")] - public IEnumerable Summaries { get; set; } + public IEnumerable Summaries { get; set; } [JsonProperty("warnings")] public IEnumerable Warnings { get; set; } diff --git a/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/ItemizedSummaryContext.cs b/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/ItemizedSummaryContext.cs new file mode 100644 index 000000000..b73cb9471 --- /dev/null +++ b/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/ItemizedSummaryContext.cs @@ -0,0 +1,21 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// + +namespace Connector +{ + using Newtonsoft.Json; + + public class ItemizedSummaryContext + { + [JsonProperty(PropertyName = "conversationItemId")] + public string ConversationItemId { get; set; } + + [JsonProperty(PropertyName = "offset")] + public int Offset { get; set; } + + [JsonProperty(PropertyName = "length")] + public int Length { get; set; } + } +} diff --git a/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/Role.cs b/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/Role.cs new file mode 100644 index 000000000..846a20a2e --- /dev/null +++ b/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/Role.cs @@ -0,0 +1,17 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// + +namespace Connector.Serializable.Language.Conversations +{ + using System.Text.Json.Serialization; + + [JsonConverter(typeof(JsonStringEnumConverter))] + public enum Role + { + None = 0, + Agent = 1, + Customer = 2, + } +} diff --git a/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/RoleAssignmentMappingKey.cs b/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/RoleAssignmentMappingKey.cs new file mode 100644 index 000000000..e33a805f1 --- /dev/null +++ b/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/RoleAssignmentMappingKey.cs @@ -0,0 +1,17 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// + +namespace Connector.Serializable.Language.Conversations +{ + using System.Text.Json.Serialization; + + [JsonConverter(typeof(JsonStringEnumConverter))] + public enum RoleAssignmentMappingKey + { + None = 0, + Channel = 1, + Speaker = 2, + } +} diff --git a/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/RoleAssignmentStratergy.cs b/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/RoleAssignmentStratergy.cs new file mode 100644 index 000000000..9abc9e60f --- /dev/null +++ b/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/RoleAssignmentStratergy.cs @@ -0,0 +1,18 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// + +namespace Connector.Serializable.Language.Conversations +{ + using System.Collections.Generic; + + public class RoleAssignmentStratergy + { + public RoleAssignmentMappingKey Key { get; init; } + + public Dictionary Mapping { get; init; } + + public Role FallbackRole { get; init; } + } +} diff --git a/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/Summary.cs b/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/Summary.cs deleted file mode 100644 index 0372ab434..000000000 --- a/samples/ingestion/ingestion-client/Connector/Serializable/Conversations/Summary.cs +++ /dev/null @@ -1,25 +0,0 @@ -// -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. -// - -namespace Connector.Serializable.Language.Conversations -{ - using Newtonsoft.Json; - - public enum SummaryAspect - { - Unknown = 0, - Issue, - Resolution, - } - - public class Summary - { - [JsonProperty(PropertyName = "aspect")] - public SummaryAspect Aspect { get; set; } - - [JsonProperty(PropertyName = "text")] - public string Text { get; set; } - } -} diff --git a/samples/ingestion/ingestion-client/Connector/Serializable/TranscriptionResult/SummaryResultItem.cs b/samples/ingestion/ingestion-client/Connector/Serializable/TranscriptionResult/SummaryResultItem.cs new file mode 100644 index 000000000..023d9e5de --- /dev/null +++ b/samples/ingestion/ingestion-client/Connector/Serializable/TranscriptionResult/SummaryResultItem.cs @@ -0,0 +1,23 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// + +namespace Connector +{ + using System.Collections.Generic; + + using Newtonsoft.Json; + + public class SummaryResultItem + { + [JsonProperty("aspect")] + public string Aspect { get; set; } + + [JsonProperty("text")] + public string Text { get; set; } + + [JsonProperty("contexts")] + public IEnumerable Contexts { get; set; } + } +} diff --git a/samples/ingestion/ingestion-client/FetchTranscription/FetchTranscription.csproj b/samples/ingestion/ingestion-client/FetchTranscription/FetchTranscription.csproj index d6936db77..0dc8c5b06 100644 --- a/samples/ingestion/ingestion-client/FetchTranscription/FetchTranscription.csproj +++ b/samples/ingestion/ingestion-client/FetchTranscription/FetchTranscription.csproj @@ -4,7 +4,7 @@ v4 - + diff --git a/samples/ingestion/ingestion-client/FetchTranscription/FetchTranscriptionEnvironmentVariables.cs b/samples/ingestion/ingestion-client/FetchTranscription/FetchTranscriptionEnvironmentVariables.cs index c85867ca5..523538454 100644 --- a/samples/ingestion/ingestion-client/FetchTranscription/FetchTranscriptionEnvironmentVariables.cs +++ b/samples/ingestion/ingestion-client/FetchTranscription/FetchTranscriptionEnvironmentVariables.cs @@ -9,6 +9,7 @@ namespace FetchTranscriptionFunction using Connector; using Connector.Constants; using Connector.Enums; + using Connector.Serializable.Language.Conversations; public static class FetchTranscriptionEnvironmentVariables { @@ -26,6 +27,8 @@ public static class FetchTranscriptionEnvironmentVariables public static readonly int ConversationPiiMaxChunkSize = int.TryParse(Environment.GetEnvironmentVariable(nameof(ConversationPiiMaxChunkSize), EnvironmentVariableTarget.Process), out ConversationPiiMaxChunkSize) ? ConversationPiiMaxChunkSize : Constants.DefaultConversationAnalysisMaxChunkSize; + public static readonly ConversationSummarizationOptions ConversationSummarizationOptions = System.Text.Json.JsonSerializer.Deserialize(Environment.GetEnvironmentVariable(nameof(ConversationSummarizationOptions), EnvironmentVariableTarget.Process)); + public static readonly bool UseSqlDatabase = bool.TryParse(Environment.GetEnvironmentVariable(nameof(UseSqlDatabase), EnvironmentVariableTarget.Process), out UseSqlDatabase) && UseSqlDatabase; public static readonly int InitialPollingDelayInMinutes = int.TryParse(Environment.GetEnvironmentVariable(nameof(InitialPollingDelayInMinutes), EnvironmentVariableTarget.Process), out InitialPollingDelayInMinutes) ? InitialPollingDelayInMinutes.ClampInt(2, Constants.MaxInitialPollingDelayInMinutes) : Constants.DefaultInitialPollingDelayInMinutes; diff --git a/samples/ingestion/ingestion-client/FetchTranscription/Language/AnalyzeConversationsProvider.cs b/samples/ingestion/ingestion-client/FetchTranscription/Language/AnalyzeConversationsProvider.cs index 36742c53c..8f31a475b 100644 --- a/samples/ingestion/ingestion-client/FetchTranscription/Language/AnalyzeConversationsProvider.cs +++ b/samples/ingestion/ingestion-client/FetchTranscription/Language/AnalyzeConversationsProvider.cs @@ -16,6 +16,7 @@ namespace Language using Azure.Core; using Connector; + using Connector.Constants; using Connector.Serializable.Language.Conversations; using Connector.Serializable.TranscriptionStartedServiceBusMessage; @@ -51,6 +52,9 @@ public static bool IsConversationalPiiEnabled() return FetchTranscriptionEnvironmentVariables.ConversationPiiSetting != Connector.Enums.ConversationPiiSetting.None; } + public static bool IsConversationalSummarizationEnabled() + => FetchTranscriptionEnvironmentVariables.ConversationSummarizationOptions.Enabled; + /// /// API to submit an analyzeConversations async Request. /// @@ -61,9 +65,39 @@ public static bool IsConversationalPiiEnabled() speechTranscript = speechTranscript ?? throw new ArgumentNullException(nameof(speechTranscript)); var data = new List(); + var summarizationData = new AnalyzeConversationsRequest + { + DisplayName = "IngestionClient - Summarization", + AnalysisInput = new AnalysisInput(new[] + { + new Conversation + { + Id = $"whole transcript", + Modality = Modality.transcript, + ConversationItems = new List() + } + }), + Tasks = new List(), + }; + var count = -1; var jobCount = 0; var turnCount = 0; + foreach (var aspect in FetchTranscriptionEnvironmentVariables.ConversationSummarizationOptions.Aspects) + { + summarizationData.Tasks.Add(new AnalyzeConversationsTask + { + TaskName = "Conversation Summarization task - " + aspect, + Kind = AnalyzeConversationsTaskKind.ConversationalSummarizationTask, + Parameters = new Dictionary + { + { + "summaryAspects", new[] { aspect.ToString() } + }, + } + }); + } + foreach (var recognizedPhrase in speechTranscript.RecognizedPhrases) { var topResult = recognizedPhrase.NBest.First(); @@ -109,7 +143,7 @@ public static bool IsConversationalPiiEnabled() }); } - data.Last().AnalysisInput.Conversations[0].ConversationItems.Add(new ConversationItem + var utterance = new ConversationItem { Text = topResult.Display, Lexical = topResult.Lexical, @@ -117,6 +151,11 @@ public static bool IsConversationalPiiEnabled() MaskedItn = topResult.MaskedITN, Id = $"{turnCount}__{recognizedPhrase.Offset}__{recognizedPhrase.Channel}", ParticipantId = $"{recognizedPhrase.Channel}", + ConversationItemLevelTiming = new AudioTiming + { + Offset = recognizedPhrase.OffsetInTicks, + Duration = recognizedPhrase.DurationInTicks, + }, AudioTimings = topResult.Words ?.Select(word => new WordLevelAudioTiming { @@ -124,11 +163,43 @@ public static bool IsConversationalPiiEnabled() Duration = (long)word.DurationInTicks, Offset = (long)word.OffsetInTicks }) - }); + }; + data.Last().AnalysisInput.Conversations[0].ConversationItems.Add(utterance); + + // for summarization + var stratergy = FetchTranscriptionEnvironmentVariables.ConversationSummarizationOptions.Stratergy; + var roleKey = stratergy.Key switch + { + RoleAssignmentMappingKey.Channel => recognizedPhrase.Channel, + RoleAssignmentMappingKey.Speaker => recognizedPhrase.Speaker, + _ => throw new ArgumentOutOfRangeException($"Unknown stratergy.Key: {stratergy.Key}"), + }; + if (!stratergy.Mapping.TryGetValue(roleKey, out var role)) + { + role = stratergy.FallbackRole; + } + + if (role != Role.None && count + textCount < FetchTranscriptionEnvironmentVariables.ConversationSummarizationOptions.InputLengthLimit) + { + utterance.Role = utterance.ParticipantId = role.ToString(); + summarizationData.AnalysisInput.Conversations[0].ConversationItems.Add(utterance); + } + count += textCount; turnCount++; } + this.log.LogInformation($"{summarizationData.Tasks.Count} Summarization Tasks Prepared. Locale = {this.locale}. chars = {count}. total turns = {turnCount}. turns for summarization = {summarizationData.AnalysisInput.Conversations[0].ConversationItems.Count}"); + + if (this.locale != null + && this.locale.StartsWith(Constants.SummarizationSupportedLocalePrefix) + && summarizationData.AnalysisInput.Conversations[0].ConversationItems.Count > 0) + { + summarizationData.AnalysisInput.Conversations[0].Language = Constants.SummarizationSupportedLocalePrefix; + data.Add(summarizationData); + jobCount++; + } + this.log.LogInformation($"Submitting {jobCount} jobs to Conversations..."); return await this.SubmitConversationsAsync(data).ConfigureAwait(false); @@ -139,22 +210,22 @@ public static bool IsConversationalPiiEnabled() /// /// Enumerable of conversational jobIds. /// Enumerable of results of conversation PII redaction and errors encountered if any. - public async Task<(AnalyzeConversationPiiResults piiResults, IEnumerable errors)> GetConversationsOperationsResult(IEnumerable jobIds) + public async Task<(AnalyzeConversationPiiResults piiResults, AnalyzeConversationSummarizationResults summarizationResults, IEnumerable errors)> GetConversationsOperationsResult(IEnumerable jobIds) { var errors = new List(); if (!jobIds.Any()) { - return (null, errors); + return (null, null, errors); } var tasks = jobIds.Select(async jobId => await this.GetConversationsOperationResults(jobId).ConfigureAwait(false)); var results = await Task.WhenAll(tasks).ConfigureAwait(false); - var piiErrors = results.SelectMany(result => result.piiResults).SelectMany(s => s.Errors); - if (piiErrors.Any()) + var resultsErrors = results.SelectMany(result => result.piiResults).SelectMany(s => s.Errors).Concat(results.SelectMany(result => result.summarizationResults).SelectMany(s => s.Errors)); + if (resultsErrors.Any()) { - errors.AddRange(piiErrors.Select(s => $"Error thrown for conversation : {s.Id}")); - return (null, errors); + errors.AddRange(resultsErrors.Select(s => $"Error thrown for conversation : {s.Id}")); + return (null, null, errors); } var warnings = results.SelectMany(result => result.piiResults).SelectMany(s => s.Conversations).SelectMany(s => s.Warnings); @@ -190,7 +261,21 @@ public static bool IsConversationalPiiEnabled() CombinedRedactedContent = combinedRedactedContent }; - return (piiResults, errors); + var summarizationResults = new AnalyzeConversationSummarizationResults + { + Conversations = new List + { + new ConversationsSummaryResult + { + Summaries = results.SelectMany( + r => r.summarizationResults.SelectMany( + s => s.Conversations.SelectMany( + c => c.Summaries))) + }, + }, + }; + + return (piiResults, summarizationResults, errors); } /// @@ -200,7 +285,7 @@ public static bool IsConversationalPiiEnabled() /// True if all requests completed, else false. public async Task ConversationalRequestsCompleted(IEnumerable audioFileInfos) { - if (!IsConversationalPiiEnabled() || !audioFileInfos.Where(audioFileInfo => audioFileInfo.TextAnalyticsRequests.ConversationRequests != null).Any()) + if (!(IsConversationalPiiEnabled() || IsConversationalSummarizationEnabled()) || !audioFileInfos.Where(audioFileInfo => audioFileInfo.TextAnalyticsRequests.ConversationRequests != null).Any()) { return true; } @@ -243,8 +328,7 @@ public async Task> AddConversationalEntitiesAsync( speechTranscript = speechTranscript ?? throw new ArgumentNullException(nameof(speechTranscript)); var errors = new List(); - var isConversationalPiiEnabled = IsConversationalPiiEnabled(); - if (!isConversationalPiiEnabled) + if (!(IsConversationalPiiEnabled() || IsConversationalSummarizationEnabled())) { return new List(); } @@ -254,16 +338,17 @@ public async Task> AddConversationalEntitiesAsync( return errors; } - var conversationsPiiResults = await this.GetConversationsOperationsResult(conversationJobIds).ConfigureAwait(false); + var conversationsResults = await this.GetConversationsOperationsResult(conversationJobIds).ConfigureAwait(false); - if (conversationsPiiResults.errors.Any()) + if (conversationsResults.errors.Any()) { - errors.AddRange(conversationsPiiResults.errors); + errors.AddRange(conversationsResults.errors); } speechTranscript.ConversationAnalyticsResults = new ConversationAnalyticsResults { - AnalyzeConversationPiiResults = conversationsPiiResults.piiResults, + AnalyzeConversationPiiResults = conversationsResults.piiResults, + AnalyzeConversationSummarizationResults = conversationsResults.summarizationResults, }; return errors; @@ -306,7 +391,7 @@ public async Task> AddConversationalEntitiesAsync( return (jobs, errors); } - private async Task<(IEnumerable piiResults, IEnumerable errors)> GetConversationsOperationResults(string jobId) + private async Task<(IEnumerable piiResults, IEnumerable summarizationResults, IEnumerable errors)> GetConversationsOperationResults(string jobId) { var errors = new List(); try @@ -325,10 +410,15 @@ public async Task> AddConversationalEntitiesAsync( if (analysisResult.Tasks.InProgress == 0) { // all tasks completed. - return (analysisResult.Tasks + var piiResults = analysisResult.Tasks .Items.Where(item => item.Kind == AnalyzeConversationsTaskResultKind.conversationalPIIResults) .Select(s => s as ConversationPiiItem) - .Select(s => s.Results), errors); + .Select(s => s.Results); + var summarizationResults = analysisResult.Tasks + .Items.Where(item => item.Kind == AnalyzeConversationsTaskResultKind.conversationalSummarizationResults) + .Select(s => s as ConversationSummarizationItem) + .Select(s => s.Results); + return (piiResults, summarizationResults, errors); } } catch (OperationCanceledException) @@ -342,7 +432,7 @@ public async Task> AddConversationalEntitiesAsync( errors.Add($"Conversation analysis request failed with error: {e.Message}"); } - return (null, errors); + return (null, null, errors); } } } diff --git a/samples/ingestion/ingestion-client/FetchTranscription/TranscriptionProcessor.cs b/samples/ingestion/ingestion-client/FetchTranscription/TranscriptionProcessor.cs index 39993e964..91d831dd1 100644 --- a/samples/ingestion/ingestion-client/FetchTranscription/TranscriptionProcessor.cs +++ b/samples/ingestion/ingestion-client/FetchTranscription/TranscriptionProcessor.cs @@ -392,8 +392,11 @@ private async Task ProcessSucceededTranscriptionAsync(string transcriptionLocati } } - if (textAnalyticsProvider != null && (FetchTranscriptionEnvironmentVariables.SentimentAnalysisSetting != SentimentAnalysisSetting.None - || FetchTranscriptionEnvironmentVariables.PiiRedactionSetting != PiiRedactionSetting.None || AnalyzeConversationsProvider.IsConversationalPiiEnabled())) + if (textAnalyticsProvider != null && + (FetchTranscriptionEnvironmentVariables.SentimentAnalysisSetting != SentimentAnalysisSetting.None + || FetchTranscriptionEnvironmentVariables.PiiRedactionSetting != PiiRedactionSetting.None + || AnalyzeConversationsProvider.IsConversationalPiiEnabled() + || AnalyzeConversationsProvider.IsConversationalSummarizationEnabled())) { // If we already got text analytics requests in the transcript (containsTextAnalyticsRequest), add the results to the transcript. // Otherwise, submit new text analytics requests. diff --git a/samples/ingestion/ingestion-client/Setup/ArmTemplateBatch.json b/samples/ingestion/ingestion-client/Setup/ArmTemplateBatch.json index c306e9259..936c99af0 100644 --- a/samples/ingestion/ingestion-client/Setup/ArmTemplateBatch.json +++ b/samples/ingestion/ingestion-client/Setup/ArmTemplateBatch.json @@ -194,15 +194,15 @@ } }, "PiiRedaction": { - "defaultValue": "None", - "type": "String", - "allowedValues": [ - "None", - "UtteranceAndAudioLevel" - ], - "metadata": { - "description": "A value indicating whether personally identifiable information (PII) redaction is requested. Will only be performed if a Text Analytics Key and Region is provided." - } + "defaultValue": "None", + "type": "String", + "allowedValues": [ + "None", + "UtteranceAndAudioLevel" + ], + "metadata": { + "description": "A value indicating whether personally identifiable information (PII) redaction is requested. Will only be performed if a Text Analytics Key and Region is provided." + } }, "SqlAdministratorLogin": { "type": "string", @@ -227,7 +227,7 @@ } }, "variables": { - "Version": "v2.0.2", + "Version": "v2.0.3", "AudioInputContainer": "audio-input", "AudioProcessedContainer": "audio-processed", "ErrorFilesOutputContainer": "audio-failed", @@ -269,6 +269,7 @@ "ConversationPiiCategories": "", "ConversationPiiRedaction": "None", "ConversationPiiInferenceSource": "text", + "ConversationSummarizationOptions": "{\"Stratergy\":{\"Key\":\"Channel\",\"Mapping\":{\"0\":\"Agent\",\"1\":\"Customer\"},\"FallbackRole\":\"None\"},\"Aspects\":[\"Issue\",\"Resolution\",\"ChapterTitle\",\"Narrative\"],\"Enabled\":false,\"InputLengthLimit\":125000}", "IsAzureGovDeployment": "[or(equals(parameters('AzureSpeechServicesRegion'),'usgovarizona'), equals(parameters('AzureSpeechServicesRegion'),'usgovvirginia'))]", "AzureSpeechServicesEndpointUri": "[if(not(equals(parameters('CustomEndpoint'),'')), parameters('CustomEndpoint'), if(variables('IsAzureGovDeployment'), concat('https://', parameters('AzureSpeechServicesRegion') ,'.api.cognitive.microsoft.us/'), concat('https://', parameters('AzureSpeechServicesRegion') ,'.api.cognitive.microsoft.com/')))]", "EndpointSuffix": "[if(variables('IsAzureGovDeployment'),'core.usgovcloudapi.net','core.windows.net')]", @@ -941,7 +942,8 @@ "PiiCategories": "[variables('PiiCategories')]", "ConversationPiiCategories": "[variables('ConversationPiiCategories')]", "ConversationPiiInferenceSource": "[variables('ConversationPiiInferenceSource')]", - "ConversationPiiSetting":"[variables('ConversationPiiRedaction')]" + "ConversationPiiSetting":"[variables('ConversationPiiRedaction')]", + "ConversationSummarizationOptions":"[variables('ConversationSummarizationOptions')]" } } ], diff --git a/samples/ingestion/ingestion-client/Setup/ArmTemplateRealtime.json b/samples/ingestion/ingestion-client/Setup/ArmTemplateRealtime.json index 28760691c..ff4d74bf8 100644 --- a/samples/ingestion/ingestion-client/Setup/ArmTemplateRealtime.json +++ b/samples/ingestion/ingestion-client/Setup/ArmTemplateRealtime.json @@ -123,7 +123,7 @@ } }, "variables": { - "Version": "v2.0.2", + "Version": "v2.0.3", "AudioInputContainer": "audio-input", "AudioProcessedContainer": "audio-processed", "ErrorFilesOutputContainer": "audio-failed", diff --git a/samples/ingestion/ingestion-client/Tests/EndToEndTests.cs b/samples/ingestion/ingestion-client/Tests/EndToEndTests.cs index a4d512702..12521c5ff 100644 --- a/samples/ingestion/ingestion-client/Tests/EndToEndTests.cs +++ b/samples/ingestion/ingestion-client/Tests/EndToEndTests.cs @@ -10,10 +10,21 @@ namespace Tests using System.IO; using System.Linq; using System.Threading.Tasks; + + using Connector; + using Connector.Serializable.Language.Conversations; + using Connector.Serializable.TranscriptionStartedServiceBusMessage; + + using Language; + using Microsoft.CognitiveServices.Speech; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; + using Moq; + + using Newtonsoft.Json; + using RealtimeTranscription; [TestClass] @@ -52,5 +63,44 @@ public async Task TestMultiChannelFromSasTestAsync() var firstNBest = jsonResults.First().NBest.First(); Assert.AreEqual(firstNBest.Lexical, "hello"); } + + [TestMethod] + [TestCategory(TestCategories.EndToEndTest)] + public async Task AnalyzeConversationTestAsync() + { + Console.WriteLine(System.Text.Json.JsonSerializer.Serialize(new ConversationSummarizationOptions + { + Aspects = new HashSet { Aspect.Issue, Aspect.Resolution, Aspect.ChapterTitle, Aspect.Narrative }, + Stratergy = new RoleAssignmentStratergy + { + Key = RoleAssignmentMappingKey.Channel, + Mapping = new Dictionary { { 0, Role.Agent }, { 1, Role.Customer } }, + FallbackRole = Role.None, + } + })); + var region = testProperties["LanguageServiceRegion"].ToString(); + var subscriptionKey = testProperties["LanguageServiceSubscriptionKey"].ToString(); + var provider = new AnalyzeConversationsProvider("en-US", subscriptionKey, region, Logger.Object); + var body = File.ReadAllText(@"testFiles/summarizationInputSample.json"); + var transcription = JsonConvert.DeserializeObject(body); + var jobIds = await provider.SubmitAnalyzeConversationsRequestAsync(transcription).ConfigureAwait(false); + Console.WriteLine("Submit"); + Console.WriteLine(JsonConvert.SerializeObject(jobIds)); + Assert.AreEqual(0, jobIds.errors.Count()); + var req = jobIds.jobIds.Select(jobId => new AudioFileInfo(default, default, new TextAnalyticsRequests(default, default, new[] { new TextAnalyticsRequest(jobId, TextAnalyticsRequest.TextAnalyticsRequestStatus.Running) }))); + + while (!await provider.ConversationalRequestsCompleted(req).ConfigureAwait(false)) + { + await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false); + Console.WriteLine($"[{DateTime.Now}]jobs are running..."); + } + + Console.WriteLine($"[{DateTime.Now}]jobs done."); + + var err = await provider.AddConversationalEntitiesAsync(jobIds.jobIds, transcription); + Console.WriteLine($"annotation result: {JsonConvert.SerializeObject(transcription)}"); + Assert.AreEqual(0, err.Count()); + Assert.AreEqual(4, transcription.ConversationAnalyticsResults.AnalyzeConversationSummarizationResults.Conversations.First().Summaries.Count()); + } } } diff --git a/samples/ingestion/ingestion-client/Tests/EndToEndTests.runsettings b/samples/ingestion/ingestion-client/Tests/EndToEndTests.runsettings index 93a27a746..46e20896c 100644 --- a/samples/ingestion/ingestion-client/Tests/EndToEndTests.runsettings +++ b/samples/ingestion/ingestion-client/Tests/EndToEndTests.runsettings @@ -4,6 +4,8 @@ + + diff --git a/samples/ingestion/ingestion-client/Tests/TestFiles/summarizationInputSample.json b/samples/ingestion/ingestion-client/Tests/TestFiles/summarizationInputSample.json new file mode 100644 index 000000000..ea6884f4b --- /dev/null +++ b/samples/ingestion/ingestion-client/Tests/TestFiles/summarizationInputSample.json @@ -0,0 +1,224 @@ +{ + "source": "https://www.contoso.com", + "timeStamp": "2023-01-05T22:44:06Z", + "durationInTicks": 130300000, + "duration": "PT13.03S", + "combinedRecognizedPhrases": [ + { + "channel": 0, + "lexical": "hi hello how can i help you i am having trouble issuing a return of a game on my xbox call of duty you can bring the disk to the nearest microsoft store to return it", + "itn": "hi hello how can i help you i am having trouble issuing a return of a game on my xbox call of duty you can bring the disk to the nearest microsoft store to return it", + "maskedITN": "", + "display": "Hi. Hello how can I help you? I am having trouble issuing a return of a game on my Xbox Call of Duty. You can bring the disk to the nearest Microsoft Store to return it." + } + ], + "recognizedPhrases": [ + { + "recognitionStatus": "Success", + "channel": 0, + "speaker": 1, + "offset": "PT0.13S", + "duration": "PT0.43S", + "offsetInTicks": 1300000, + "durationInTicks": 4300000, + "nBest": [ + { + "confidence": 0.87210596, + "lexical": "hi", + "itn": "hi", + "maskedITN": "", + "display": "Hi." + } + ] + }, + { + "recognitionStatus": "Success", + "channel": 0, + "speaker": 2, + "offset": "PT1.41S", + "duration": "PT1.58S", + "offsetInTicks": 14100000, + "durationInTicks": 15800000, + "nBest": [ + { + "confidence": 0.9614907, + "lexical": "hello how can i help you", + "itn": "hello how can i help you", + "maskedITN": "", + "display": "Hello how can I help you?" + } + ] + }, + { + "recognitionStatus": "Success", + "channel": 0, + "speaker": 1, + "offset": "PT3.87S", + "duration": "PT4.23S", + "offsetInTicks": 38700000, + "durationInTicks": 42300000, + "nBest": [ + { + "confidence": 0.8985182, + "lexical": "i am having trouble issuing a return of a game on my xbox call of duty", + "itn": "i am having trouble issuing a return of a game on my xbox call of duty", + "maskedITN": "", + "display": "I am having trouble issuing a return of a game on my Xbox Call of Duty." + } + ] + }, + { + "recognitionStatus": "Success", + "channel": 0, + "speaker": 2, + "offset": "PT8.98S", + "duration": "PT3.33S", + "offsetInTicks": 89800000, + "durationInTicks": 33300000, + "nBest": [ + { + "confidence": 0.93731016, + "lexical": "you can bring the disk to the nearest microsoft store to return it", + "itn": "you can bring the disk to the nearest microsoft store to return it", + "maskedITN": "", + "display": "You can bring the disk to the nearest Microsoft Store to return it." + } + ] + } + ], + "conversationAnalyticsResults": { + "conversationPiiResults": { + "combinedRedactedContent": [ + { + "channel": "0", + "lexical": "hi hello how can i help you i am having trouble issuing a return of a game on my xbox call of duty you can bring the disk to the nearest microsoft store to return it", + "itn": "hi hello how can i help you i am having trouble issuing a return of a game on my xbox call of duty you can bring the disk to the nearest microsoft store to return it", + "display": "Hi. Hello how can I help you? I am having trouble issuing a return of a game on my Xbox Call of Duty. You can bring the disk to the nearest Microsoft Store to return it." + } + ], + "conversations": [ + { + "conversationItems": [ + { + "id": "0", + "channel": "0", + "offset": "PT0.13S", + "redactedContent": { + "itn": "hi", + "lexical": "hi", + "text": "Hi.", + "redactedAudioTimings": [] + }, + "entities": [] + }, + { + "id": "1", + "channel": "0", + "offset": "PT1.41S", + "redactedContent": { + "itn": "hello how can i help you", + "lexical": "hello how can i help you", + "text": "Hello how can I help you?", + "redactedAudioTimings": [] + }, + "entities": [] + }, + { + "id": "2", + "channel": "0", + "offset": "PT3.87S", + "redactedContent": { + "itn": "i am having trouble issuing a return of a game on my xbox call of duty", + "lexical": "i am having trouble issuing a return of a game on my xbox call of duty", + "text": "I am having trouble issuing a return of a game on my Xbox Call of Duty.", + "redactedAudioTimings": [] + }, + "entities": [] + }, + { + "id": "3", + "channel": "0", + "offset": "PT8.98S", + "redactedContent": { + "itn": "you can bring the disk to the nearest microsoft store to return it", + "lexical": "you can bring the disk to the nearest microsoft store to return it", + "text": "You can bring the disk to the nearest Microsoft Store to return it.", + "redactedAudioTimings": [] + }, + "entities": [] + } + ], + "warnings": [] + } + ] + }, + "conversationSummarizationResults": { + "conversations": [ + { + "summaries": [ + { + "aspect": "issue", + "text": "Customer wants to return a game. | Customer can't return the game." + }, + { + "aspect": "resolution", + "text": "Advised customer to bring the disk to the nearest Microsoft Store to return it." + }, + { + "aspect": "chapterTitle", + "text": "Trouble Issuing Return of Game", + "contexts": [ + { + "conversationItemId": "0__PT0.13S__0", + "offset": 0, + "length": 3 + }, + { + "conversationItemId": "1__PT1.41S__0", + "offset": 0, + "length": 25 + }, + { + "conversationItemId": "2__PT3.87S__0", + "offset": 0, + "length": 71 + }, + { + "conversationItemId": "3__PT8.98S__0", + "offset": 0, + "length": 67 + } + ] + }, + { + "aspect": "narrative", + "text": "Agent helps the customer to return a game on Xbox.", + "contexts": [ + { + "conversationItemId": "0__PT0.13S__0", + "offset": 0, + "length": 3 + }, + { + "conversationItemId": "1__PT1.41S__0", + "offset": 0, + "length": 25 + }, + { + "conversationItemId": "2__PT3.87S__0", + "offset": 0, + "length": 71 + }, + { + "conversationItemId": "3__PT8.98S__0", + "offset": 0, + "length": 67 + } + ] + } + ] + } + ] + } + } +} \ No newline at end of file