-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[IngestionClient] Fix database deployment issue - move database deplo…
…yment to code (EF) (#1728) * move DB migration to code * Update Connector.csproj * continue with DB migration * start with db connector update * continue * fixes * fix migration * fixes * fix SilenceBetweenCurrentAndPreviousSegmentInMs value * small fixes * catch db exceptions * update templates and GH action * remove BOMs * Update guide.md * Update IngestionClientDbContextExtensions.cs * Update DesignTimeSpeechServicesDbContextFactory.cs * Update Program.cs * remove unused references * Delete profile.arm.json
- Loading branch information
1 parent
0c96ba4
commit 54ddf15
Showing
28 changed files
with
1,504 additions
and
388 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
28 changes: 28 additions & 0 deletions
28
samples/ingestion/ingestion-client/Connector/Database/IngestionClientDbContext.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| // <copyright file="IngestionClientDbContext.cs" company="Microsoft Corporation"> | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT license. See LICENSE.md file in the project root for full license information. | ||
| // </copyright> | ||
|
|
||
| namespace Connector.Database | ||
| { | ||
| using Connector.Database.Models; | ||
| using Microsoft.EntityFrameworkCore; | ||
|
|
||
| public class IngestionClientDbContext : DbContext | ||
| { | ||
| public IngestionClientDbContext(DbContextOptions<IngestionClientDbContext> options) | ||
| : base(options) | ||
| { | ||
| } | ||
|
|
||
| public DbSet<Transcription> Transcriptions { get; set; } | ||
|
|
||
| public DbSet<CombinedRecognizedPhrase> CombinedRecognizedPhrases { get; set; } | ||
|
|
||
| public DbSet<NBest> NBests { get; set; } | ||
|
|
||
| public DbSet<RecognizedPhrase> RecognizedPhrases { get; set; } | ||
|
|
||
| public DbSet<Word> Words { get; set; } | ||
| } | ||
| } |
154 changes: 154 additions & 0 deletions
154
samples/ingestion/ingestion-client/Connector/Database/IngestionClientDbContextExtensions.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,154 @@ | ||
| // <copyright file="IngestionClientDbContextExtensions.cs" company="Microsoft Corporation"> | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT license. See LICENSE.md file in the project root for full license information. | ||
| // </copyright> | ||
|
|
||
| namespace Connector.Database | ||
| { | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Globalization; | ||
| using System.Linq; | ||
| using System.Threading.Tasks; | ||
|
|
||
| using Connector.Database.Models; | ||
|
|
||
| public static class IngestionClientDbContextExtensions | ||
| { | ||
| private const int MaxNBestsPerRecognizedPhrase = 1; | ||
|
|
||
| public static async Task StoreTranscriptionAsync( | ||
| this IngestionClientDbContext ingestionClientDbContext, | ||
| Guid transcriptionId, | ||
| string locale, | ||
| string fileName, | ||
| float approximateCost, | ||
| SpeechTranscript speechTranscript) | ||
| { | ||
| _ = ingestionClientDbContext ?? throw new ArgumentNullException(nameof(ingestionClientDbContext)); | ||
| _ = speechTranscript ?? throw new ArgumentNullException(nameof(speechTranscript)); | ||
|
|
||
| var transcription = new Transcription( | ||
| id: transcriptionId, | ||
| locale: locale, | ||
| name: fileName, | ||
| source: speechTranscript.Source, | ||
| timestamp: DateTime.Parse(speechTranscript.Timestamp, CultureInfo.InvariantCulture), | ||
| duration: speechTranscript.Duration ?? string.Empty, | ||
| durationInSeconds: TimeSpan.FromTicks(speechTranscript.DurationInTicks).TotalSeconds, | ||
| numberOfChannels: speechTranscript.CombinedRecognizedPhrases.Count(), | ||
| approximateCost: approximateCost); | ||
| var combinedRecognizedPhrases = new List<CombinedRecognizedPhrase>(); | ||
|
|
||
| var phrasesByChannel = speechTranscript.RecognizedPhrases.GroupBy(t => t.Channel); | ||
|
|
||
| foreach (var phrases in phrasesByChannel) | ||
| { | ||
| var channel = phrases.Key; | ||
| var combinedPhrase = speechTranscript.CombinedRecognizedPhrases.Where(t => t.Channel == channel).FirstOrDefault(); | ||
| var combinedRecognizedPhraseDb = AddCombinedRecognizedPhrase(combinedPhrase, channel, phrases); | ||
| combinedRecognizedPhrases.Add(combinedRecognizedPhraseDb); | ||
| } | ||
|
|
||
| transcription = transcription.WithCombinedRecognizedPhrases(combinedRecognizedPhrases); | ||
|
|
||
| ingestionClientDbContext.Add(transcription); | ||
| var entitiesAdded = await ingestionClientDbContext.SaveChangesAsync().ConfigureAwait(false); | ||
| } | ||
|
|
||
| private static CombinedRecognizedPhrase AddCombinedRecognizedPhrase(Connector.CombinedRecognizedPhrase combinedRecognizedPhrase, int channel, IEnumerable<Connector.RecognizedPhrase> recognizedPhrases) | ||
| { | ||
| var combinedRecognizedPhraseDb = new CombinedRecognizedPhrase( | ||
| id: Guid.NewGuid(), | ||
| channel: channel, | ||
| lexical: combinedRecognizedPhrase?.Lexical ?? string.Empty, | ||
| itn: combinedRecognizedPhrase?.Lexical ?? string.Empty, | ||
| maskedItn: combinedRecognizedPhrase?.Lexical ?? string.Empty, | ||
| display: combinedRecognizedPhrase?.Lexical ?? string.Empty, | ||
| sentimentNegative: combinedRecognizedPhrase?.Sentiment?.Negative ?? 0d, | ||
| sentimentNeutral: combinedRecognizedPhrase?.Sentiment?.Neutral ?? 0d, | ||
| sentimentPositive: combinedRecognizedPhrase?.Sentiment?.Positive ?? 0d); | ||
|
|
||
| var recognizedPhrasesDb = new List<RecognizedPhrase>(); | ||
|
|
||
| var orderedPhrases = recognizedPhrases.OrderBy(p => p.OffsetInTicks); | ||
| var previousEndInMs = 0.0; | ||
| foreach (var phrase in orderedPhrases) | ||
| { | ||
| var silenceBetweenCurrentAndPreviousSegmentInMs = Convert.ToInt32(Math.Max(0, TimeSpan.FromTicks(phrase.OffsetInTicks).TotalMilliseconds - previousEndInMs)); | ||
|
|
||
| var recognizedPhraseDb = AddRecognizedPhrase(phrase, silenceBetweenCurrentAndPreviousSegmentInMs); | ||
| previousEndInMs = (TimeSpan.FromTicks(phrase.OffsetInTicks) + TimeSpan.FromTicks(phrase.DurationInTicks)).TotalMilliseconds; | ||
|
|
||
| recognizedPhrasesDb.Add(recognizedPhraseDb); | ||
| } | ||
|
|
||
| combinedRecognizedPhraseDb = combinedRecognizedPhraseDb.WithRecognizedPhrases(recognizedPhrasesDb); | ||
| return combinedRecognizedPhraseDb; | ||
| } | ||
|
|
||
| private static RecognizedPhrase AddRecognizedPhrase(Connector.RecognizedPhrase recognizedPhrase, int silenceBetweenCurrentAndPreviousSegmentInMs) | ||
| { | ||
| var recognizedPhraseDb = new RecognizedPhrase( | ||
| id: Guid.NewGuid(), | ||
| recognitionStatus: recognizedPhrase.RecognitionStatus, | ||
| speaker: recognizedPhrase.Speaker, | ||
| channel: recognizedPhrase.Channel, | ||
| offset: recognizedPhrase.Offset, | ||
| duration: recognizedPhrase.Duration, | ||
| silenceBetweenCurrentAndPreviousSegmentInMs: silenceBetweenCurrentAndPreviousSegmentInMs); | ||
|
|
||
| var nbestsDb = new List<NBest>(); | ||
|
|
||
| foreach (var nbestResult in recognizedPhrase.NBest.Take(MaxNBestsPerRecognizedPhrase)) | ||
| { | ||
| var nbestDb = AddNBestResult(nbestResult); | ||
| nbestsDb.Add(nbestDb); | ||
| } | ||
|
|
||
| recognizedPhraseDb = recognizedPhraseDb.WithNBests(nbestsDb); | ||
| return recognizedPhraseDb; | ||
| } | ||
|
|
||
| private static NBest AddNBestResult(Connector.NBest nbest) | ||
| { | ||
| var nbestDb = new NBest( | ||
| id: Guid.NewGuid(), | ||
| confidence: nbest.Confidence, | ||
| lexical: nbest.Lexical, | ||
| itn: nbest.ITN, | ||
| maskedItn: nbest.MaskedITN, | ||
| display: nbest.Display, | ||
| sentimentNegative: nbest.Sentiment?.Negative ?? 0d, | ||
| sentimentNeutral: nbest.Sentiment?.Neutral ?? 0d, | ||
| sentimentPositive: nbest.Sentiment?.Positive ?? 0d); | ||
|
|
||
| if (nbest.Words != null) | ||
| { | ||
| var wordsDb = new List<Word>(); | ||
|
|
||
| foreach (var word in nbest.Words) | ||
| { | ||
| var wordDb = CreateWord(word); | ||
| wordsDb.Add(wordDb); | ||
| } | ||
|
|
||
| nbestDb = nbestDb.WithWords(wordsDb); | ||
| } | ||
|
|
||
| return nbestDb; | ||
| } | ||
|
|
||
| private static Word CreateWord(Connector.Words word) | ||
| { | ||
| var wordDb = new Word( | ||
| id: Guid.NewGuid(), | ||
| wordText: word.Word, | ||
| offset: word.Offset, | ||
| duration: word.Duration, | ||
| confidence: word.Confidence); | ||
|
|
||
| return wordDb; | ||
| } | ||
| } | ||
| } |
58 changes: 58 additions & 0 deletions
58
samples/ingestion/ingestion-client/Connector/Database/Models/CombinedRecognizedPhrase.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| // <copyright file="CombinedRecognizedPhrase.cs" company="Microsoft Corporation"> | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT license. See LICENSE.md file in the project root for full license information. | ||
| // </copyright> | ||
|
|
||
| namespace Connector.Database.Models | ||
| { | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.ComponentModel.DataAnnotations; | ||
| using System.ComponentModel.DataAnnotations.Schema; | ||
|
|
||
| [System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "CA2227:Collection properties should be read only", Justification = "Used by Entity Framework")] | ||
| public class CombinedRecognizedPhrase : DbModelBase | ||
| { | ||
| public CombinedRecognizedPhrase(Guid id, int channel, string lexical, string itn, string maskedItn, string display, double sentimentNegative, double sentimentNeutral, double sentimentPositive) | ||
| { | ||
| this.Id = id; | ||
| this.Channel = channel; | ||
| this.Lexical = lexical; | ||
| this.Itn = itn; | ||
| this.MaskedItn = maskedItn; | ||
| this.Display = display; | ||
| this.SentimentNegative = sentimentNegative; | ||
| this.SentimentNeutral = sentimentNeutral; | ||
| this.SentimentPositive = sentimentPositive; | ||
| } | ||
|
|
||
| [Column("ID")] | ||
| [Key] | ||
| public Guid Id { get; set; } | ||
|
|
||
| public int Channel { get; private set; } | ||
|
|
||
| public string Lexical { get; private set; } | ||
|
|
||
| public string Itn { get; private set; } | ||
|
|
||
| public string MaskedItn { get; private set; } | ||
|
|
||
| public string Display { get; private set; } | ||
|
|
||
| public double SentimentNegative { get; private set; } | ||
|
|
||
| public double SentimentNeutral { get; private set; } | ||
|
|
||
| public double SentimentPositive { get; private set; } | ||
|
|
||
| [ForeignKey("CombinedRecognizedPhraseID")] | ||
| public ICollection<RecognizedPhrase> RecognizedPhrases { get; set; } | ||
|
|
||
| public CombinedRecognizedPhrase WithRecognizedPhrases(ICollection<RecognizedPhrase> recognizedPhrases) | ||
| { | ||
| this.RecognizedPhrases = recognizedPhrases; | ||
| return this; | ||
| } | ||
| } | ||
| } |
20 changes: 20 additions & 0 deletions
20
samples/ingestion/ingestion-client/Connector/Database/Models/DbModelBase.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| // <copyright file="DbModelBase.cs" company="Microsoft Corporation"> | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT license. See LICENSE.md file in the project root for full license information. | ||
| // </copyright> | ||
|
|
||
| namespace Connector.Database.Models | ||
| { | ||
| public abstract class DbModelBase | ||
| { | ||
| public const int MaxTimeSpanColumnLength = 255; | ||
|
|
||
| public const int MaxLocaleLength = 255; | ||
|
|
||
| public const int MaxDefaultStringLength = 500; | ||
|
|
||
| public const int MaxWordLength = 511; | ||
|
|
||
| public const int MaxStateLength = 32; | ||
| } | ||
| } |
58 changes: 58 additions & 0 deletions
58
samples/ingestion/ingestion-client/Connector/Database/Models/NBest.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| // <copyright file="NBest.cs" company="Microsoft Corporation"> | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT license. See LICENSE.md file in the project root for full license information. | ||
| // </copyright> | ||
|
|
||
| namespace Connector.Database.Models | ||
| { | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.ComponentModel.DataAnnotations; | ||
| using System.ComponentModel.DataAnnotations.Schema; | ||
|
|
||
| [System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "CA2227:Collection properties should be read only", Justification = "Used by Entity Framework")] | ||
| public class NBest : DbModelBase | ||
| { | ||
| public NBest(Guid id, double confidence, string lexical, string itn, string maskedItn, string display, double sentimentNegative, double sentimentNeutral, double sentimentPositive) | ||
| { | ||
| this.Id = id; | ||
| this.Confidence = confidence; | ||
| this.Lexical = lexical; | ||
| this.Itn = itn; | ||
| this.MaskedItn = maskedItn; | ||
| this.Display = display; | ||
| this.SentimentNegative = sentimentNegative; | ||
| this.SentimentNeutral = sentimentNeutral; | ||
| this.SentimentPositive = sentimentPositive; | ||
| } | ||
|
|
||
| [Column("ID")] | ||
| [Key] | ||
| public Guid Id { get; set; } | ||
|
|
||
| public double Confidence { get; private set; } | ||
|
|
||
| public string Lexical { get; private set; } | ||
|
|
||
| public string Itn { get; private set; } | ||
|
|
||
| public string MaskedItn { get; private set; } | ||
|
|
||
| public string Display { get; private set; } | ||
|
|
||
| public double SentimentNegative { get; private set; } | ||
|
|
||
| public double SentimentNeutral { get; private set; } | ||
|
|
||
| public double SentimentPositive { get; private set; } | ||
|
|
||
| [ForeignKey("NBestID")] | ||
| public ICollection<Word> Words { get; set; } | ||
|
|
||
| public NBest WithWords(ICollection<Word> words) | ||
| { | ||
| this.Words = words; | ||
| return this; | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.