Json Columns initial work

Finished work:
- basic metadata,
- property/navigation access translation,
- materialization (entity, collection, with nesting)

Remaining work:
- validation,
- migration,
- update,
- collections of primitives,
- array access/operations (force non-tracking!)
- some metadata fixes (e.g. using actual type mapping for properties mapped to json, rather than default mappings), flowing json type info (npgsql),
- honoring json property name attribute for custom naming,
- custom serialization (???),
- lots of fixes, cleanups, documentation,
- testing

src/EFCore.Relational/Extensions/RelationalEntityTypeBuilderExtensions.cs

@@ -1903,4 +1903,36 @@ public static bool CanHaveTrigger(
             tableName,
             tableSchema,
             fromDataAnnotation ? ConfigurationSource.DataAnnotation : ConfigurationSource.Convention);
+
+    /// <summary>
+    /// TODO
+    /// </summary>
+    public static EntityTypeBuilder<TEntity> MapReferenceToJson<TEntity, TRelatedEntity>(
+        this EntityTypeBuilder<TEntity> entityTypeBuilder,
+        Expression<Func<TEntity, TRelatedEntity?>> navigationExpression,
+        string jsonColumnName)
+        where TEntity : class
+        where TRelatedEntity : class
+    {
+        var ownedNavigationBuilder = entityTypeBuilder.OwnsOne(navigationExpression);
+        ownedNavigationBuilder.OwnedEntityType.SetAnnotation(RelationalAnnotationNames.MapToJsonColumnName, jsonColumnName);
+
+        return entityTypeBuilder;
+    }
+
+    /// <summary>
+    /// TODO
+    /// </summary>
+    public static EntityTypeBuilder<TEntity> MapCollectionToJson<TEntity, TRelatedEntity>(
+        this EntityTypeBuilder<TEntity> entityTypeBuilder,
+        Expression<Func<TEntity, IEnumerable<TRelatedEntity>?>> navigationExpression,
+        string jsonColumnName)
+        where TEntity : class
+        where TRelatedEntity : class
+    {
+        var ownedNavigationBuilder = entityTypeBuilder.OwnsMany(navigationExpression);
+        ownedNavigationBuilder.OwnedEntityType.SetAnnotation(RelationalAnnotationNames.MapToJsonColumnName, jsonColumnName);
+
+        return entityTypeBuilder;
+    }
 }

src/EFCore.Relational/Extensions/RelationalEntityTypeExtensions.cs

@@ -62,7 +62,7 @@ public static class RelationalEntityTypeExtensions
 
         var ownership = entityType.FindOwnership();
         if (ownership != null
-            && ownership.IsUnique)
+            && (ownership.IsUnique || entityType.MappedToJson()))
         {
             return ownership.PrincipalEntityType.GetTableName();
         }
@@ -846,12 +846,18 @@ public static IEnumerable<IReadOnlyForeignKey> FindRowInternalForeignKeys(
 
         foreach (var foreignKey in entityType.GetForeignKeys())
         {
+            var mappedToJson = entityType.MappedToJson();
             var principalEntityType = foreignKey.PrincipalEntityType;
+
+            var pkPropertiesToMatch = mappedToJson
+                ? primaryKey.Properties.Take(foreignKey.Properties.Count).ToList().AsReadOnly()
+                : primaryKey.Properties;
+
             if (!foreignKey.PrincipalKey.IsPrimaryKey()
                 || principalEntityType == foreignKey.DeclaringEntityType
-                || !foreignKey.IsUnique
+                || (!foreignKey.IsUnique && !mappedToJson)
 #pragma warning disable EF1001 // Internal EF Core API usage.
-                || !PropertyListComparer.Instance.Equals(foreignKey.Properties, primaryKey.Properties))
+                || !PropertyListComparer.Instance.Equals(foreignKey.Properties, pkPropertiesToMatch))
 #pragma warning restore EF1001 // Internal EF Core API usage.
             {
                 continue;
@@ -1263,4 +1269,34 @@ public static IEnumerable<ITrigger> GetDeclaredTriggers(this IEntityType entityT
         => Trigger.GetDeclaredTriggers(entityType).Cast<ITrigger>();
 
     #endregion Trigger
+
+    /// <summary>
+    ///     TODO
+    /// </summary>
+    public static bool MappedToJson(this IConventionEntityType entityType)
+        => !string.IsNullOrEmpty(entityType.MappedToJsonColumnName());
+
+    /// <summary>
+    ///     TODO
+    /// </summary>
+    public static bool MappedToJson(this IReadOnlyEntityType entityType)
+        => !string.IsNullOrEmpty(entityType.MappedToJsonColumnName());
+
+    /// <summary>
+    ///     TODO
+    /// </summary>
+    public static string? MappedToJsonColumnName(this IConventionEntityType entityType)
+        => entityType.FindAnnotation(RelationalAnnotationNames.MapToJsonColumnName)?.Value as string;
+
+    /// <summary>
+    ///     TODO
+    /// </summary>
+    public static string? MappedToJsonColumnName(this IReadOnlyEntityType entityType)
+        => entityType.FindAnnotation(RelationalAnnotationNames.MapToJsonColumnName)?.Value as string;
+
+    /// <summary>
+    ///     TODO
+    /// </summary>
+    public static string? MappedToJsonColumnTypeName(this IReadOnlyEntityType entityType)
+        => entityType.FindAnnotation(RelationalAnnotationNames.MapToJsonColumnTypeName)?.Value as string;
 }

src/EFCore.Relational/Extensions/RelationalPropertyExtensions.cs

@@ -113,7 +113,13 @@ public static string GetColumnBaseName(this IReadOnlyProperty property)
             return (string?)columnAnnotation.Value;
         }
 
-        return GetDefaultColumnName(property, storeObject);
+        // GetDefaultColumnName returns non-nullable, so for json collection entity ordinal keys (which are not mapped)
+        // just make it return emtpty string and convert to null here
+        var result = GetDefaultColumnName(property, storeObject);
+
+        return result == string.Empty && property.DeclaringEntityType.MappedToJson()
+            ? null
+            : result;
     }
 
     /// <summary>
@@ -144,6 +150,14 @@ public static string GetDefaultColumnName(this IReadOnlyProperty property, in St
             return sharedTablePrincipalConcurrencyProperty.GetColumnName(storeObject)!;
         }
 
+        if (property.DeclaringEntityType.MappedToJson())
+        {
+            // only PKs that are unhandled at this point should be ordinal keys - not mapped to anything
+            return property.IsPrimaryKey()
+                ? string.Empty
+                : property.GetDefaultColumnBaseName();
+        }
+
         var entityType = property.DeclaringEntityType;
         StringBuilder? builder = null;
         while (true)
@@ -1323,6 +1337,14 @@ public static RelationalTypeMapping GetRelationalTypeMapping(this IReadOnlyPrope
         var column = property.GetColumnName(storeObject);
         if (column == null)
         {
+            // for entities mapped to json, if column name was not returned the property is either contained in json,
+            // or, in case of collection, a synthesized key based on ordinal
+            // so just skip this, there is no shared objct root to speak of
+            if (property.DeclaringEntityType.MappedToJson())
+            {
+                return null;
+            }
+
             throw new InvalidOperationException(
                 RelationalStrings.PropertyNotMappedToTable(
                     property.Name, property.DeclaringEntityType.DisplayName(), storeObject.DisplayName()));
@@ -1379,7 +1401,21 @@ public static RelationalTypeMapping GetRelationalTypeMapping(this IReadOnlyPrope
                 break;
             }
 
-            principalProperty = linkingRelationship.PrincipalKey.Properties[linkingRelationship.Properties.IndexOf(principalProperty)];
+            if (property.DeclaringEntityType.MappedToJson())
+            {
+                // for json collection entities don't match the ordinal key
+                var index = linkingRelationship.Properties.IndexOf(principalProperty);
+                if (index == -1)
+                {
+                    return null;
+                }
+
+                principalProperty = linkingRelationship.PrincipalKey.Properties[index];
+            }
+            else
+            {
+                principalProperty = linkingRelationship.PrincipalKey.Properties[linkingRelationship.Properties.IndexOf(principalProperty)];
+            }
         }
 
         return principalProperty == property ? null : principalProperty;

src/EFCore.Relational/Infrastructure/RelationalModelRuntimeInitializer.cs

@@ -62,7 +62,11 @@ protected override void InitializeModel(IModel model, bool designTime, bool prev
         }
         else
         {
-            RelationalModel.Add(model, RelationalDependencies.RelationalAnnotationProvider, designTime);
+            RelationalModel.Add(
+                model,
+                RelationalDependencies.RelationalAnnotationProvider,
+                RelationalDependencies.RelationalTypeMappingSource,
+                designTime);
         }
     }
 }

src/EFCore.Relational/Infrastructure/RelationalModelRuntimeInitializerDependencies.cs

@@ -47,10 +47,12 @@ public sealed record RelationalModelRuntimeInitializerDependencies
     [EntityFrameworkInternal]
     public RelationalModelRuntimeInitializerDependencies(
         RelationalModelDependencies relationalModelDependencies,
-        IRelationalAnnotationProvider relationalAnnotationProvider)
+        IRelationalAnnotationProvider relationalAnnotationProvider,
+        IRelationalTypeMappingSource relationalTypeMappingSource)
     {
         RelationalModelDependencies = relationalModelDependencies;
         RelationalAnnotationProvider = relationalAnnotationProvider;
+        RelationalTypeMappingSource = relationalTypeMappingSource;
     }
 
     /// <summary>
@@ -62,4 +64,9 @@ public RelationalModelRuntimeInitializerDependencies(
     ///     The relational annotation provider.
     /// </summary>
     public IRelationalAnnotationProvider RelationalAnnotationProvider { get; init; }
+
+    /// <summary>
+    ///     The relational type mapping source.
+    /// </summary>
+    public IRelationalTypeMappingSource RelationalTypeMappingSource { get; init; }
 }

src/EFCore.Relational/Infrastructure/RelationalModelValidator.cs

@@ -310,6 +310,12 @@ protected virtual void ValidateSharedTableCompatibility(
                     continue;
                 }
 
+                // no need to check json-mapped types (or is there, just different?)
+                if (entityType.MappedToJson())
+                {
+                    continue;
+                }
+
                 var (principalEntityTypes, optional) = GetPrincipalEntityTypes(entityType);
                 if (!optional)
                 {
@@ -401,7 +407,8 @@ protected virtual void ValidateSharedTableCompatibility(
         }
 
         var storeObject = StoreObjectIdentifier.Table(tableName, schema);
-        var unvalidatedTypes = new HashSet<IEntityType>(mappedTypes);
+        // TODO: add proper validation for json-mapped types instead of filtering them out
+        var unvalidatedTypes = new HashSet<IEntityType>(mappedTypes.Where(x => !x.MappedToJson()));
         IEntityType? root = null;
         foreach (var mappedType in mappedTypes)
         {
@@ -410,6 +417,13 @@ protected virtual void ValidateSharedTableCompatibility(
                 continue;
             }
 
+            // HACK (do it properly, i.e. add some validation to this scenario, not everything should be allowed presumably)
+            // owned types mapped to json can share table with parent even in 1-many scenario
+            if (mappedType.MappedToJsonColumnName() != null)
+            {
+                continue;
+            }
+
             var primaryKey = mappedType.FindPrimaryKey();
             if (primaryKey != null
                 && (mappedType.FindForeignKeys(primaryKey.Properties)
@@ -710,9 +724,20 @@ protected virtual void ValidateSharedColumnsCompatibility(
                 }
             }
 
+            // TODO: add validation for json columns (JsonColumn has mappings for individual properties for the entire owned structure)
+            if (entityType.MappedToJson())
+            {
+                continue;
+            }
+
             foreach (var property in entityType.GetDeclaredProperties())
             {
                 var columnName = property.GetColumnName(storeObject)!;
+                if (columnName == null)
+                {
+                    continue;
+                }
+
                 missingConcurrencyTokens?.Remove(columnName);
                 if (!propertyMappings.TryGetValue(columnName, out var duplicateProperty))
                 {

src/EFCore.Relational/Infrastructure/RelationalPropertyExtensions.cs

@@ -33,12 +33,24 @@ public static string FormatColumns(
         foreach (var property in properties)
         {
             var columnName = property.GetColumnName(storeObject);
-            if (columnName == null)
+            // entity mapped to json can have part of it's key mapped to a column (i.e. part that is FK pointing to the PK of the owner)
+            // and part that is not (i.e. synthesized key property based on ordinal in the collection)
+            if (property.DeclaringEntityType.MappedToJson())
             {
-                return null;
+                if (columnName != null)
+                {
+                    propertyNames.Add(columnName);
+                }
             }
+            else
+            {
+                if (columnName == null)
+                {
+                    return null;
+                }
 
-            propertyNames.Add(columnName);
+                propertyNames.Add(columnName);
+            }
         }
 
         return propertyNames;

src/EFCore.Relational/Metadata/Conventions/Infrastructure/RelationalConventionSetBuilder.cs

@@ -81,6 +81,9 @@ public override ConventionSet CreateConventionSet()
         conventionSet.EntityTypeBaseTypeChangedConventions.Add(checkConstraintConvention);
         conventionSet.EntityTypeBaseTypeChangedConventions.Add(triggerConvention);
 
+        var mapToJsonConvention = new RelationalMapToJsonConvention();
+        conventionSet.EntityTypeAnnotationChangedConventions.Add(mapToJsonConvention);
+
         ReplaceConvention(conventionSet.ForeignKeyPropertiesChangedConventions, valueGenerationConvention);
 
         ReplaceConvention(conventionSet.ForeignKeyOwnershipChangedConventions, valueGenerationConvention);

src/EFCore.Relational/Metadata/Conventions/Infrastructure/RelationalConventionSetBuilderDependencies.cs

@@ -45,13 +45,21 @@ public sealed record RelationalConventionSetBuilderDependencies
     ///     the constructor at any point in this process.
     /// </remarks>
     [EntityFrameworkInternal]
-    public RelationalConventionSetBuilderDependencies(IRelationalAnnotationProvider relationalAnnotationProvider)
+    public RelationalConventionSetBuilderDependencies(
+        IRelationalAnnotationProvider relationalAnnotationProvider,
+        IRelationalTypeMappingSource relationalTypeMappingSource)
     {
         RelationalAnnotationProvider = relationalAnnotationProvider;
+        RelationalTypeMappingSource = relationalTypeMappingSource;
     }
 
     /// <summary>
     ///     The relational annotation provider.
     /// </summary>
     public IRelationalAnnotationProvider RelationalAnnotationProvider { get; init; }
+
+    /// <summary>
+    ///     The relational type mapping source.
+    /// </summary>
+    public IRelationalTypeMappingSource RelationalTypeMappingSource { get; init; }
 }

src/EFCore.Relational/Metadata/Conventions/RelationalMapToJsonConvention.cs

@@ -0,0 +1,53 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Microsoft.EntityFrameworkCore.Metadata.Conventions
+{
+    /// <summary>
+    /// TODO
+    /// </summary>
+    public class RelationalMapToJsonConvention : IEntityTypeAnnotationChangedConvention
+    {
+        /// <summary>
+        /// TODO
+        /// </summary>
+        public void ProcessEntityTypeAnnotationChanged(
+            IConventionEntityTypeBuilder entityTypeBuilder,
+            string name,
+            IConventionAnnotation? annotation,
+            IConventionAnnotation? oldAnnotation,
+            IConventionContext<IConventionAnnotation> context)
+        {
+            if (name == RelationalAnnotationNames.MapToJsonColumnName)
+            {
+                // TODO: if json type name was specified (via attribute) propagate it here and add it as annotation on the entity itself
+                // needed for postgres, since it has two json types
+                var tableName = entityTypeBuilder.Metadata.GetTableName();
+                if (tableName == null)
+                {
+                    throw new InvalidOperationException("need table name");
+                }
+
+                var jsonColumnName = annotation?.Value as string;
+                if (!string.IsNullOrEmpty(jsonColumnName))
+                {
+                    foreach (var navigation in entityTypeBuilder.Metadata.GetNavigations()
+                        .Where(n => n.ForeignKey.IsOwnership
+                            && n.DeclaringEntityType == entityTypeBuilder.Metadata
+                            && n.TargetEntityType.IsOwned()))
+                    {
+                        var currentJsonColumnName = navigation.TargetEntityType.MappedToJsonColumnName();
+                        if (currentJsonColumnName == null || currentJsonColumnName != jsonColumnName)
+                        {
+                            navigation.TargetEntityType.SetAnnotation(RelationalAnnotationNames.MapToJsonColumnName, jsonColumnName);
+                        }
+                    }
+                }
+                else
+                {
+                    // TODO: unwind everything
+                }
+            }
+        }
+    }
+}

src/EFCore.Relational/Metadata/Conventions/RelationalRuntimeModelConvention.cs

@@ -52,7 +52,11 @@ protected override void ProcessModelAnnotations(
         if (runtime)
         {
             annotations[RelationalAnnotationNames.RelationalModel] =
-                RelationalModel.Create(runtimeModel, RelationalDependencies.RelationalAnnotationProvider, designTime: false);
+                RelationalModel.Create(
+                    runtimeModel,
+                    RelationalDependencies.RelationalAnnotationProvider,
+                    RelationalDependencies.RelationalTypeMappingSource,
+                    designTime: false);
         }
         else
         {