[TA] Add encoding support and length and offset

sdk/textanalytics/Azure.AI.TextAnalytics/CHANGELOG.md

@@ -6,6 +6,7 @@
 - It defaults to the latest supported API version, which currently is `3.1-preview.2`.
 - `ErrorCode` value returned from the service is now surfaced in `RequestFailedException`.
 - Support added for Opinion Mining. This feature is available in the Text Analytics service v3.1-preview.1 and above.
+- Added `Offset` and `Length` properties for `CategorizedEntity`, `SentenceSentiment`, and `LinkedEntityMatch`. The default encoding is UTF16 code units. For additional information see https://aka.ms/text-analytics-offsets
 - `TextAnalyticsError` and `TextAnalyticsWarning` now are marked as immutable.
 
 ## 5.0.0 (2020-07-27)

sdk/textanalytics/Azure.AI.TextAnalytics/README.md

@@ -190,7 +190,8 @@ CategorizedEntityCollection entities = client.RecognizeEntities(document);
 Console.WriteLine($"Recognized {entities.Count} entities:");
 foreach (CategorizedEntity entity in entities)
 {
-    Console.WriteLine($"Text: {entity.Text}, Category: {entity.Category}, SubCategory: {entity.SubCategory}, Confidence score: {entity.ConfidenceScore}");
+    Console.WriteLine($"Text: {entity.Text}, Offset (in UTF-16  code units): {entity.Offset}, Length (in UTF-16  code units): {entity.Length}");
+    Console.WriteLine($"Category: {entity.Category}, SubCategory: {entity.SubCategory}, Confidence score: {entity.ConfidenceScore}");
 }
 ```
 For samples on using the production recommended option `RecognizeEntitiesBatch` see [here][recognize_entities_sample].
@@ -211,7 +212,8 @@ foreach (LinkedEntity linkedEntity in linkedEntities)
     Console.WriteLine($"Name: {linkedEntity.Name}, Language: {linkedEntity.Language}, Data Source: {linkedEntity.DataSource}, Url: {linkedEntity.Url.ToString()}, Entity Id in Data Source: {linkedEntity.DataSourceEntityId}");
     foreach (LinkedEntityMatch match in linkedEntity.Matches)
     {
-        Console.WriteLine($"    Match Text: {match.Text}, Confidence score: {match.ConfidenceScore}");
+        Console.WriteLine($"    Match Text: {match.Text}, Offset (in UTF-16  code units): {match.Offset}, Length (in UTF-16 code units): {match.Length}");
+        Console.WriteLine($"    Confidence score: {match.ConfidenceScore}");
     }
 }
 ```
@@ -241,7 +243,8 @@ CategorizedEntityCollection entities = await client.RecognizeEntitiesAsync(docum
 Console.WriteLine($"Recognized {entities.Count} entities:");
 foreach (CategorizedEntity entity in entities)
 {
-    Console.WriteLine($"Text: {entity.Text}, Category: {entity.Category}, SubCategory: {entity.SubCategory}, Confidence score: {entity.ConfidenceScore}");
+    Console.WriteLine($"Text: {entity.Text}, Offset (in UTF-16  code units): {entity.Offset}, Length (in UTF-16  code units): {entity.Length}");
+    Console.WriteLine($"Category: {entity.Category}, SubCategory: {entity.SubCategory}, Confidence score: {entity.ConfidenceScore}");
 }
 ```
 

sdk/textanalytics/Azure.AI.TextAnalytics/samples/Sample4_RecognizeEntities.md

@@ -23,7 +23,8 @@ CategorizedEntityCollection entities = client.RecognizeEntities(document);
 Console.WriteLine($"Recognized {entities.Count} entities:");
 foreach (CategorizedEntity entity in entities)
 {
-    Console.WriteLine($"Text: {entity.Text}, Category: {entity.Category}, SubCategory: {entity.SubCategory}, Confidence score: {entity.ConfidenceScore}");
+    Console.WriteLine($"Text: {entity.Text}, Offset (in UTF-16  code units): {entity.Offset}, Length (in UTF-16  code units): {entity.Length}");
+    Console.WriteLine($"Category: {entity.Category}, SubCategory: {entity.SubCategory}, Confidence score: {entity.ConfidenceScore}");
 }
 ```
 

sdk/textanalytics/Azure.AI.TextAnalytics/samples/Sample6_RecognizeLinkedEntities.md

@@ -26,7 +26,8 @@ foreach (LinkedEntity linkedEntity in linkedEntities)
     Console.WriteLine($"Name: {linkedEntity.Name}, Language: {linkedEntity.Language}, Data Source: {linkedEntity.DataSource}, Url: {linkedEntity.Url.ToString()}, Entity Id in Data Source: {linkedEntity.DataSourceEntityId}");
     foreach (LinkedEntityMatch match in linkedEntity.Matches)
     {
-        Console.WriteLine($"    Match Text: {match.Text}, Confidence score: {match.ConfidenceScore}");
+        Console.WriteLine($"    Match Text: {match.Text}, Offset (in UTF-16  code units): {match.Offset}, Length (in UTF-16 code units): {match.Length}");
+        Console.WriteLine($"    Confidence score: {match.ConfidenceScore}");
     }
 }
 ```

sdk/textanalytics/Azure.AI.TextAnalytics/src/AspectSentiment.cs

@@ -17,11 +17,13 @@ public readonly struct AspectSentiment
     {
         private const double _neutralValue = 0d;
 
-        internal AspectSentiment(TextSentiment sentiment, string text, double positiveScore, double negativeScore)
+        internal AspectSentiment(TextSentiment sentiment, string text, double positiveScore, double negativeScore, int offset, int length)
         {
             Sentiment = sentiment;
             Text = text;
             ConfidenceScores = new SentimentConfidenceScores(positiveScore, _neutralValue, negativeScore);
+            Offset = offset;
+            Length = length;
         }
 
         internal AspectSentiment(SentenceAspect sentenceAspect)
@@ -31,6 +33,8 @@ internal AspectSentiment(SentenceAspect sentenceAspect)
             Text = sentenceAspect.Text;
             ConfidenceScores = new SentimentConfidenceScores(sentenceAspect.ConfidenceScores.Positive, _neutralValue, sentenceAspect.ConfidenceScores.Negative);
             Sentiment = (TextSentiment)Enum.Parse(typeof(TextSentiment), sentenceAspect.Sentiment, ignoreCase: true);
+            Offset = sentenceAspect.Offset;
+            Length = sentenceAspect.Length;
         }
 
         /// <summary>
@@ -50,5 +54,15 @@ internal AspectSentiment(SentenceAspect sentenceAspect)
         /// Higher values signify higher confidence.
         /// </summary>
         public SentimentConfidenceScores ConfidenceScores { get; }
+
+        /// <summary>
+        /// Gets the starting position (in UTF16 code units) for the aspect text.
+        /// </summary>
+        public int Offset { get; }
+
+        /// <summary>
+        /// Gets the length (in UTF16 code units) of the aspect text.
+        /// </summary>
+        public int Length { get; }
     }
 }

sdk/textanalytics/Azure.AI.TextAnalytics/src/CategorizedEntity.cs

@@ -21,6 +21,8 @@ internal CategorizedEntity(Entity entity)
             Text = entity.Text;
             SubCategory = entity.Subcategory;
             ConfidenceScore = entity.ConfidenceScore;
+            Offset = entity.Offset;
+            Length = entity.Length;
         }
 
         /// <summary>
@@ -50,5 +52,15 @@ internal CategorizedEntity(Entity entity)
         /// text substring matches this inferred entity.
         /// </summary>
         public double ConfidenceScore { get; }
+
+        /// <summary>
+        /// Gets the starting position (in UTF16 code units) for the matching text in the input document.
+        /// </summary>
+        public int Offset { get; }
+
+        /// <summary>
+        /// Gets the length (in UTF16 code units) of the matching text in the input document.
+        /// </summary>
+        public int Length { get; }
     }
 }

sdk/textanalytics/Azure.AI.TextAnalytics/src/LinkedEntityMatch.cs

@@ -34,9 +34,14 @@ internal LinkedEntityMatch(double confidenceScore, string text, int offset, int
         /// </summary>
         public double ConfidenceScore { get; }
 
-        /// <summary> Start position for the entity match text. </summary>
-        private int Offset { get; }
-        /// <summary> Length for the entity match text. </summary>
-        private int Length { get; }
-}
+        /// <summary>
+        /// Gets the starting position (in UTF16 code units) for the matching text in the document.
+        /// </summary>
+        public int Offset { get; }
+
+        /// <summary>
+        /// Gets the length (in UTF16 code units) of the matching text in the document.
+        /// </summary>
+        public int Length { get; }
+    }
 }

sdk/textanalytics/Azure.AI.TextAnalytics/src/OpinionSentiment.cs

@@ -15,12 +15,14 @@ public readonly struct OpinionSentiment
     {
         private const double _neutralValue = 0d;
 
-        internal OpinionSentiment(TextSentiment sentiment, double positiveScore, double negativeScore, string text, bool isNegated)
+        internal OpinionSentiment(TextSentiment sentiment, double positiveScore, double negativeScore, string text, bool isNegated, int offset, int length)
         {
             Sentiment = sentiment;
             ConfidenceScores = new SentimentConfidenceScores(positiveScore, _neutralValue, negativeScore);
             Text = text;
             IsNegated = isNegated;
+            Offset = offset;
+            Length = length;
         }
 
         internal OpinionSentiment(SentenceOpinion opinion)
@@ -31,6 +33,8 @@ internal OpinionSentiment(SentenceOpinion opinion)
             ConfidenceScores = new SentimentConfidenceScores(opinion.ConfidenceScores.Positive, _neutralValue, opinion.ConfidenceScores.Negative);
             Sentiment = (TextSentiment)Enum.Parse(typeof(TextSentiment), opinion.Sentiment, ignoreCase: true);
             IsNegated = opinion.IsNegated;
+            Offset = opinion.Offset;
+            Length = opinion.Length;
         }
 
         /// <summary>
@@ -57,5 +61,15 @@ internal OpinionSentiment(SentenceOpinion opinion)
         /// "The food is not good", the opinion "good" is negated.
         /// </summary>
         public bool IsNegated { get; }
+
+        /// <summary>
+        /// Gets the starting position (in UTF16 code units) for the opinion text.
+        /// </summary>
+        public int Offset { get; }
+
+        /// <summary>
+        /// Gets the length (in UTF16 code units) of the opinion text.
+        /// </summary>
+        public int Length { get; }
     }
 }

sdk/textanalytics/Azure.AI.TextAnalytics/src/SentenceSentiment.cs

@@ -16,11 +16,13 @@ namespace Azure.AI.TextAnalytics
     /// </summary>
     public readonly struct SentenceSentiment
     {
-        internal SentenceSentiment(TextSentiment sentiment, string text, double positiveScore, double neutralScore, double negativeScore, IReadOnlyList<MinedOpinion> minedOpinions)
+        internal SentenceSentiment(TextSentiment sentiment, string text, double positiveScore, double neutralScore, double negativeScore, int offset, int length, IReadOnlyList<MinedOpinion> minedOpinions)
         {
             Sentiment = sentiment;
             Text = text;
             ConfidenceScores = new SentimentConfidenceScores(positiveScore, neutralScore, negativeScore);
+            Offset = offset;
+            Length = length;
             MinedOpinions = new List<MinedOpinion>(minedOpinions);
         }
 
@@ -33,6 +35,8 @@ internal SentenceSentiment(SentenceSentimentInternal sentenceSentiment, IReadOnl
             ConfidenceScores = sentenceSentiment.ConfidenceScores;
             Sentiment = (TextSentiment)Enum.Parse(typeof(TextSentiment), sentenceSentiment.Sentiment, ignoreCase: true);
             MinedOpinions = ConvertToMinedOpinions(sentenceSentiment, allSentences);
+            Offset = sentenceSentiment.Offset;
+            Length = sentenceSentiment.Length;
         }
 
         /// <summary>
@@ -57,6 +61,16 @@ internal SentenceSentiment(SentenceSentimentInternal sentenceSentiment, IReadOnl
         /// </summary>
         public IReadOnlyCollection<MinedOpinion> MinedOpinions { get; }
 
+        /// <summary>
+        /// Gets the starting position (in UTF16 code units) for the matching text in the sentence.
+        /// </summary>
+        public int Offset { get; }
+
+        /// <summary>
+        /// Gets the length (in UTF16 code units) of the matching text in the sentence.
+        /// </summary>
+        public int Length { get; }
+
         private static IReadOnlyCollection<MinedOpinion> ConvertToMinedOpinions(SentenceSentimentInternal sentence, IReadOnlyList<SentenceSentimentInternal> allSentences)
         {
             var minedOpinions = new List<MinedOpinion>();