Implement IsUnicode API

Issue #3420

I think everything in the runtime and migrations areas is implemented. The ScaffoldingTypeMapper already supports unicode, but scaffolding will not make use of this until updated as described in issue #5410.

src/Microsoft.EntityFrameworkCore.Relational/Migrations/Internal/MigrationsModelDiffer.cs

@@ -552,7 +552,6 @@ protected virtual IEnumerable<MigrationOperation> Diff([NotNull] IProperty sourc
                                           // TODO: Detect type narrowing
                                           || columnTypeChanged;
 
-                // TODO: Set IsUnicode with #3420
                 var alterColumnOperation = new AlterColumnOperation
                 {
                     Schema = sourceEntityTypeAnnotations.Schema,
@@ -561,6 +560,7 @@ protected virtual IEnumerable<MigrationOperation> Diff([NotNull] IProperty sourc
                     ClrType = target.ClrType.UnwrapNullableType().UnwrapEnumType(),
                     ColumnType = targetAnnotations.ColumnType,
                     MaxLength = target.GetMaxLength(),
+                    IsUnicode = target.IsUnicode(),
                     IsRowVersion = target.ClrType == typeof(byte[])
                         && target.IsConcurrencyToken
                         && target.ValueGenerated == ValueGenerated.OnAddOrUpdate,
@@ -589,7 +589,6 @@ protected virtual IEnumerable<MigrationOperation> Add(
             var targetEntityTypeAnnotations = Annotations.For(
                 diffContext.FindSource(target.DeclaringEntityType.RootType()));
 
-            // TODO: Set IsUnicode with #3420
             var operation = new AddColumnOperation
             {
                 Schema = targetEntityTypeAnnotations.Schema,
@@ -598,6 +597,7 @@ protected virtual IEnumerable<MigrationOperation> Add(
                 ClrType = target.ClrType.UnwrapNullableType().UnwrapEnumType(),
                 ColumnType = targetAnnotations.ColumnType,
                 MaxLength = target.GetMaxLength(),
+                IsUnicode = target.IsUnicode(),
                 IsRowVersion = target.ClrType == typeof(byte[])
                     && target.IsConcurrencyToken
                     && target.ValueGenerated == ValueGenerated.OnAddOrUpdate,

src/Microsoft.EntityFrameworkCore.Relational/Migrations/MigrationsSqlGenerator.cs

@@ -677,8 +677,8 @@ protected virtual string GetColumnType(
             var property = FindProperty(model, schema, table, name);
             if (property != null)
             {
-                // TODO: Allow unicode to be overridden with #3420
-                if (maxLength == property.GetMaxLength()
+                if (unicode == property.IsUnicode()
+                    && maxLength == property.GetMaxLength()
                     && rowVersion == (property.IsConcurrencyToken && (property.ValueGenerated == ValueGenerated.OnAddOrUpdate)))
                 {
                     return TypeMapper.GetMapping(property).StoreType;

src/Microsoft.EntityFrameworkCore.Relational/Storage/RelationalTypeMapper.cs

@@ -197,11 +197,12 @@ protected virtual RelationalTypeMapping GetStringMapping([NotNull] IProperty pro
         {
             Check.NotNull(property, nameof(property));
 
-            // TODO: Use unicode-ness defined in property metadata
+            var principal = property.FindPrincipal();
+
             return StringMapper?.FindMapping(
-                true,
+                property.IsUnicode() ?? principal?.IsUnicode() ?? true,
                 RequiresKeyMapping(property),
-                property.GetMaxLength() ?? property.FindPrincipal()?.GetMaxLength());
+                property.GetMaxLength() ?? principal?.GetMaxLength());
         }
 
         /// <summary>

src/Microsoft.EntityFrameworkCore.Specification.Tests/BuiltInDataTypesFixtureBase.cs

@@ -33,6 +33,15 @@ public virtual void OnModelCreating(ModelBuilder modelBuilder)
                     b.Property(e => e.ByteArray9000).HasMaxLength(9000);
                     b.Property(e => e.String9000).HasMaxLength(9000);
                 });
+
+            modelBuilder.Entity<UnicodeDataTypes>(b =>
+            {
+                b.Property(e => e.Id).ValueGeneratedNever();
+                b.Property(e => e.StringAnsi).IsUnicode(false);
+                b.Property(e => e.StringAnsi3).HasMaxLength(3).IsUnicode(false);
+                b.Property(e => e.StringAnsi9000).IsUnicode(false).HasMaxLength(9000);
+                b.Property(e => e.StringUnicode).IsUnicode();
+            });
         }
 
         protected static void MakeRequired<TEntity>(ModelBuilder modelBuilder) where TEntity : class
@@ -107,6 +116,16 @@ public class MaxLengthDataTypes
         public byte[] ByteArray9000 { get; set; }
     }
 
+    public class UnicodeDataTypes
+    {
+        public int Id { get; set; }
+        public string StringDefault { get; set; }
+        public string StringAnsi { get; set; }
+        public string StringAnsi3 { get; set; }
+        public string StringAnsi9000 { get; set; }
+        public string StringUnicode { get; set; }
+    }
+
     public class BinaryKeyDataType
     {
         public byte[] Id { get; set; }

src/Microsoft.EntityFrameworkCore.Specification.Tests/BuiltInDataTypesTestBase.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -58,6 +58,55 @@ public virtual void Can_perform_query_with_max_length()
             }
         }
 
+        public virtual void Can_perform_query_with_ansi_strings(bool supportsAnsi)
+        {
+            var shortString = "Ϩky";
+            var longString = new string('Ϩ', 9000);
+
+            using (var context = CreateContext())
+            {
+                context.Set<UnicodeDataTypes>().Add(
+                    new UnicodeDataTypes
+                    {
+                        Id = 799,
+                        StringDefault = shortString,
+                        StringAnsi = shortString,
+                        StringAnsi3 = shortString,
+                        StringAnsi9000 = longString,
+                        StringUnicode = shortString
+                    });
+
+                Assert.Equal(1, context.SaveChanges());
+            }
+
+            using (var context = CreateContext())
+            {
+                Assert.NotNull(context.Set<UnicodeDataTypes>().SingleOrDefault(e => e.Id == 799 && e.StringDefault == shortString));
+                Assert.NotNull(context.Set<UnicodeDataTypes>().SingleOrDefault(e => e.Id == 799 && e.StringAnsi == shortString));
+                Assert.NotNull(context.Set<UnicodeDataTypes>().SingleOrDefault(e => e.Id == 799 && e.StringAnsi3 == shortString));
+                Assert.NotNull(context.Set<UnicodeDataTypes>().SingleOrDefault(e => e.Id == 799 && e.StringAnsi9000 == longString));
+                Assert.NotNull(context.Set<UnicodeDataTypes>().SingleOrDefault(e => e.Id == 799 && e.StringUnicode == shortString));
+
+                var entity = context.Set<UnicodeDataTypes>().SingleOrDefault(e => e.Id == 799);
+
+                Assert.Equal(shortString, entity.StringDefault);
+                Assert.Equal(shortString, entity.StringUnicode);
+
+                if (supportsAnsi)
+                {
+                    Assert.NotEqual(shortString, entity.StringAnsi);
+                    Assert.NotEqual(shortString, entity.StringAnsi3);
+                    Assert.NotEqual(longString, entity.StringAnsi9000);
+                }
+                else
+                {
+                    Assert.Equal(shortString, entity.StringAnsi);
+                    Assert.Equal(shortString, entity.StringAnsi3);
+                    Assert.Equal(longString, entity.StringAnsi9000);
+                }
+            }
+        }
+
         [Fact]
         public virtual void Can_query_using_any_data_type()
         {

src/Microsoft.EntityFrameworkCore/Extensions/MutablePropertyExtensions.cs

@@ -36,6 +36,18 @@ public static void SetMaxLength([NotNull] this IMutableProperty property, int? m
             property[CoreAnnotationNames.MaxLengthAnnotation] = maxLength;
         }
 
+        /// <summary>
+        ///     Sets a value indicating whether or not this property can persist unicode characters.
+        /// </summary>
+        /// <param name="property"> The property to set the value for. </param>
+        /// <param name="unicode"> True if the property accepts unicode characters, false if it does not, null to clear the setting. </param>
+        public static void IsUnicode([NotNull] this IMutableProperty property, bool? unicode)
+        {
+            Check.NotNull(property, nameof(property));
+
+            property[CoreAnnotationNames.UnicodeAnnotation] = unicode;
+        }
+
         /// <summary>
         ///     Gets all foreign keys that use this property (including composite foreign keys in which this property
         ///     is included).

src/Microsoft.EntityFrameworkCore/Extensions/PropertyExtensions.cs

@@ -29,6 +29,18 @@ public static class PropertyExtensions
             return (int?)property[CoreAnnotationNames.MaxLengthAnnotation];
         }
 
+        /// <summary>
+        ///     Gets a value indicating whether or not the property can persist unicode characters.
+        /// </summary>
+        /// <param name="property"> The property to get the unicode setting for. </param>
+        /// <returns> The unicode setting, or null if none if defined. </returns>
+        public static bool? IsUnicode([NotNull] this IProperty property)
+        {
+            Check.NotNull(property, nameof(property));
+
+            return (bool?)property[CoreAnnotationNames.UnicodeAnnotation];
+        }
+
         /// <summary>
         ///     Gets a value indicating whether this property is used as a foreign key (or part of a composite foreign key).
         /// </summary>

src/Microsoft.EntityFrameworkCore/Metadata/Builders/PropertyBuilder.cs

@@ -89,6 +89,19 @@ public virtual PropertyBuilder HasMaxLength(int maxLength)
             return this;
         }
 
+        /// <summary>
+        ///     Configures the property as capable of persisting unicode characters or not.
+        ///     Can only be set on <see cref="string" /> properties.
+        /// </summary>
+        /// <param name="unicode"> A value indicating whether the property can contain unicode characters or not. </param>
+        /// <returns> The same builder instance so that multiple configuration calls can be chained. </returns>
+        public virtual PropertyBuilder IsUnicode(bool unicode = true)
+        {
+            Builder.IsUnicode(unicode, ConfigurationSource.Explicit);
+
+            return this;
+        }
+
         /// <summary>
         ///     Configures whether this property should be used as a concurrency token. When a property is configured
         ///     as a concurrency token the value in the database will be checked when an instance of this entity type

src/Microsoft.EntityFrameworkCore/Metadata/Builders/PropertyBuilder`.cs

@@ -55,6 +55,15 @@ public PropertyBuilder([NotNull] InternalPropertyBuilder builder)
         public new virtual PropertyBuilder<TProperty> HasMaxLength(int maxLength)
             => (PropertyBuilder<TProperty>)base.HasMaxLength(maxLength);
 
+        /// <summary>
+        ///     Configures the property as capable of persisting unicode characters or not.
+        ///     Can only be set on <see cref="string" /> properties.
+        /// </summary>
+        /// <param name="unicode"> A value indicating whether the property can contain unicode characters or not. </param>
+        /// <returns> The same builder instance so that multiple configuration calls can be chained. </returns>
+        public new virtual PropertyBuilder<TProperty> IsUnicode(bool unicode = true)
+            => (PropertyBuilder<TProperty>)base.IsUnicode(unicode);
+
         /// <summary>
         ///     Configures whether this property should be used as a concurrency token. When a property is configured
         ///     as a concurrency token the value in the database will be checked when an instance of this entity type

src/Microsoft.EntityFrameworkCore/Metadata/Internal/CoreAnnotationNames.cs

@@ -14,7 +14,13 @@ public static class CoreAnnotationNames
         ///     directly from your code. This API may change or be removed in future releases.
         /// </summary>
         public const string MaxLengthAnnotation = "MaxLength";
-
+
+        /// <summary>
+        ///     This API supports the Entity Framework Core infrastructure and is not intended to be used 
+        ///     directly from your code. This API may change or be removed in future releases.
+        /// </summary>
+        public const string UnicodeAnnotation = "Unicode";
+
         /// <summary>
         ///     This API supports the Entity Framework Core infrastructure and is not intended to be used 
         ///     directly from your code. This API may change or be removed in future releases.

src/Microsoft.EntityFrameworkCore/Metadata/Internal/InternalPropertyBuilder.cs

@@ -67,6 +67,13 @@ public virtual bool CanSetRequired(bool isRequired, ConfigurationSource? configu
         public virtual bool HasMaxLength(int maxLength, ConfigurationSource configurationSource)
             => HasAnnotation(CoreAnnotationNames.MaxLengthAnnotation, maxLength, configurationSource);
 
+        /// <summary>
+        ///     This API supports the Entity Framework Core infrastructure and is not intended to be used 
+        ///     directly from your code. This API may change or be removed in future releases.
+        /// </summary>
+        public virtual bool IsUnicode(bool unicode, ConfigurationSource configurationSource)
+            => HasAnnotation(CoreAnnotationNames.UnicodeAnnotation, unicode, configurationSource);
+
         /// <summary>
         ///     This API supports the Entity Framework Core infrastructure and is not intended to be used 
         ///     directly from your code. This API may change or be removed in future releases.

test/Microsoft.EntityFrameworkCore.InMemory.FunctionalTests/BuiltInDataTypesInMemoryTest.cs

@@ -2,6 +2,7 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using Microsoft.EntityFrameworkCore.Specification.Tests;
+using Xunit;
 
 namespace Microsoft.EntityFrameworkCore.InMemory.FunctionalTests
 {
@@ -11,5 +12,11 @@ public BuiltInDataTypesInMemoryTest(BuiltInDataTypesInMemoryFixture fixture)
             : base(fixture)
         {
         }
+
+        [Fact]
+        public virtual void Can_perform_query_with_ansi_strings()
+        {
+            Can_perform_query_with_ansi_strings(supportsAnsi: false);
+        }
     }
 }

test/Microsoft.EntityFrameworkCore.Relational.Tests/Migrations/Internal/MigrationsModelDifferTest.cs

@@ -603,6 +603,37 @@ public void Alter_column_max_length()
                     });
         }
 
+        [Fact]
+        public void Alter_column_unicode()
+        {
+            Execute(
+                source => source.Entity(
+                    "Toad",
+                    x =>
+                    {
+                        x.Property<int>("Id");
+                        x.Property<string>("Name");
+                    }),
+                target => target.Entity(
+                    "Toad",
+                    x =>
+                    {
+                        x.Property<int>("Id");
+                        x.Property<string>("Name")
+                            .IsUnicode(false);
+                    }),
+                operations =>
+                {
+                    Assert.Equal(1, operations.Count);
+
+                    var operation = Assert.IsType<AlterColumnOperation>(operations[0]);
+                    Assert.Equal("Toad", operation.Table);
+                    Assert.Equal("Name", operation.Name);
+                    Assert.False(operation.IsUnicode);
+                    Assert.True(operation.IsDestructiveChange);
+                });
+        }
+
         [Fact]
         public void Alter_column_default()
         {

test/Microsoft.EntityFrameworkCore.Relational.Tests/Migrations/Internal/MigrationsModelDifferTestBase.cs

@@ -50,8 +50,8 @@ public override RelationalTypeMapping FindMapping(Type clrType)
                     : base.FindMapping(clrType);
 
             protected override RelationalTypeMapping FindCustomMapping(IProperty property)
-                => property.ClrType == typeof(string) && property.GetMaxLength().HasValue
-                    ? new RelationalTypeMapping("nvarchar(" + property.GetMaxLength() + ")", typeof(string), dbType: null, unicode: false, size: property.GetMaxLength())
+                => property.ClrType == typeof(string) && (property.GetMaxLength().HasValue || property.IsUnicode().HasValue)
+                    ? new RelationalTypeMapping(((property.IsUnicode() ?? true) ? "n" : "") + "varchar(" + (property.GetMaxLength() ?? 767) + ")", typeof(string), dbType: null, unicode: false, size: property.GetMaxLength())
                     : base.FindCustomMapping(property);
 
             private readonly IReadOnlyDictionary<Type, RelationalTypeMapping> _simpleMappings

test/Microsoft.EntityFrameworkCore.Relational.Tests/Migrations/MigrationSqlGeneratorTestBase.cs

@@ -69,9 +69,9 @@ public virtual void AddColumnOperation_without_column_type()
                 });
 
         [Fact]
-        public virtual void AddColumnOperation_with_unicode_overridden()
+        public virtual void AddColumnOperation_with_unicode()
             => Generate(
-                modelBuilder => modelBuilder.Entity("Person").Property<string>("Name"),
+                modelBuilder => modelBuilder.Entity("Person").Property<string>("Name").IsUnicode(false),
                 new AddColumnOperation
                 {
                     Table = "Person",
@@ -81,6 +81,19 @@ public virtual void AddColumnOperation_with_unicode_overridden()
                     IsNullable = true
                 });
 
+        [Fact]
+        public virtual void AddColumnOperation_with_unicode_overridden()
+            => Generate(
+                modelBuilder => modelBuilder.Entity("Person").Property<string>("Name").IsUnicode(false),
+                new AddColumnOperation
+                {
+                    Table = "Person",
+                    Name = "Name",
+                    ClrType = typeof(string),
+                    IsUnicode = true,
+                    IsNullable = true
+                });
+
         [Fact]
         public virtual void AddColumnOperation_with_unicode_no_model()
             => Generate(