Test JSON serialization of EF object graphs

Fixes #20548

Added tests that serialize graphs with both the BCL Json support and Newtonsoft.Json.

Findings:
*  BCL Json
  * Preserving references working fine; needed to increase max-depth for heavily nested graph, but exception message was clear
  * Lack of an Ignore option may be missed because output is harder to understand when it contains many internal references. Will follow up on this.
* Newtonsoft.JSON
  * Found it easy to get out-of-memory errors even when ignoring cycles
  * Got stack overflow errors when attempting to serialize cycles instead of ignoring them
  * I'm probably not configuring things correctly here, but this isn't high priority
* Net Topology Suite types are not JSON-serialization-friendly; will follow up on this

eng/Versions.props

@@ -32,6 +32,7 @@
     <BenchmarkDotNetVersion>0.12.0</BenchmarkDotNetVersion>
     <MicrosoftDataSqlClientVersion>2.0.0-preview2.20084.1</MicrosoftDataSqlClientVersion>
     <MicrosoftCSharpVersion>4.7.0</MicrosoftCSharpVersion>
+    <NewtonsoftJsonVersion>12.0.3</NewtonsoftJsonVersion>
   </PropertyGroup>
   <PropertyGroup Label="Dependencies from dotnet/extensions">
     <MicrosoftExtensionsCachingMemoryVersion>5.0.0-preview.3.20215.2</MicrosoftExtensionsCachingMemoryVersion>

test/EFCore.Specification.Tests/EFCore.Specification.Tests.csproj

@@ -23,6 +23,7 @@
   <ItemGroup>
     <PackageReference Include="NetTopologySuite" Version="$(NetTopologySuiteVersion)" />
     <PackageReference Include="xunit" Version="$(XUnitVersion)" />
+    <PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" />
   </ItemGroup>
 
 </Project>

test/EFCore.Specification.Tests/Query/NorthwindIncludeQueryTestBase.cs

@@ -6,6 +6,8 @@
 using System.Linq;
 using System.Linq.Expressions;
 using System.Reflection;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 using System.Threading.Tasks;
 using Microsoft.EntityFrameworkCore.Diagnostics;
 using Microsoft.EntityFrameworkCore.Internal;
@@ -80,6 +82,43 @@ var orders
             Assert.True(orders.Count > 0);
             Assert.True(orders.All(od => od.Customer != null));
             Assert.True(orders.All(od => od.Customer.Orders != null));
+
+            TestJsonSerialization(useNewtonsoft: false, ignoreLoops: false, writeIndented: false);
+            TestJsonSerialization(useNewtonsoft: false, ignoreLoops: false, writeIndented: true);
+            TestJsonSerialization(useNewtonsoft: true, ignoreLoops: true, writeIndented: false);
+            TestJsonSerialization(useNewtonsoft: true, ignoreLoops: true, writeIndented: true);
+            TestJsonSerialization(useNewtonsoft: true, ignoreLoops: false, writeIndented: false);
+            TestJsonSerialization(useNewtonsoft: true, ignoreLoops: false, writeIndented: true);
+
+            void TestJsonSerialization(bool useNewtonsoft, bool ignoreLoops, bool writeIndented)
+            {
+                var ordersAgain = useNewtonsoft
+                    ? ThroughNewtonsoftJson(orders, ignoreLoops, writeIndented)
+                    : ThroughBclJson(orders, ignoreLoops, writeIndented);
+
+                var ordersMap = ignoreLoops ? null : new Dictionary<int, Order>();
+                var customersMap = ignoreLoops ? null : new Dictionary<string, Customer>();
+
+                foreach (var order in ordersAgain)
+                {
+                    VerifyOrder(context, order, ordersMap);
+
+                    var customer = order.Customer;
+                    Assert.Equal(order.CustomerID, customer.CustomerID);
+
+                    VerifyCustomer(context, customer, customersMap);
+
+                    foreach (var orderAgain in customer.Orders)
+                    {
+                        VerifyOrder(context, orderAgain, ordersMap);
+
+                        if (!ignoreLoops)
+                        {
+                            Assert.Same(customer, orderAgain.Customer);
+                        }
+                    }
+                }
+            }
         }
 
         [ConditionalFact(Skip = "issue #15312")]
@@ -2135,6 +2174,44 @@ var orders
                     customerLoaded: true,
                     ordersLoaded: false);
             }
+
+            TestJsonSerialization(useNewtonsoft: false, ignoreLoops: false, writeIndented: false);
+            TestJsonSerialization(useNewtonsoft: false, ignoreLoops: false, writeIndented: true);
+            TestJsonSerialization(useNewtonsoft: true, ignoreLoops: true, writeIndented: false);
+            TestJsonSerialization(useNewtonsoft: true, ignoreLoops: true, writeIndented: true);
+            TestJsonSerialization(useNewtonsoft: true, ignoreLoops: false, writeIndented: false);
+            TestJsonSerialization(useNewtonsoft: true, ignoreLoops: false, writeIndented: true);
+
+            void TestJsonSerialization(bool useNewtonsoft, bool ignoreLoops, bool writeIndented)
+            {
+                var ordersAgain = useNewtonsoft
+                    ? ThroughNewtonsoftJson(orders, ignoreLoops, writeIndented)
+                    : ThroughBclJson(orders, ignoreLoops, writeIndented);
+
+                var ordersMap = ignoreLoops ? null : new Dictionary<int, Order>();
+                var ordersDetailsMap = ignoreLoops ? null : new Dictionary<(int, int), OrderDetail>();
+                var customersMap = ignoreLoops ? null : new Dictionary<string, Customer>();
+
+                foreach (var order in ordersAgain)
+                {
+                    VerifyOrder(context, order, ordersMap);
+
+                    var customer = order.Customer;
+                    Assert.Equal(order.CustomerID, customer.CustomerID);
+
+                    VerifyCustomer(context, customer, customersMap);
+
+                    foreach (var orderDetail in order.OrderDetails)
+                    {
+                        VerifyOrderDetails(context, orderDetail, ordersDetailsMap);
+
+                        if (!ignoreLoops)
+                        {
+                            Assert.Same(order, orderDetail.Order);
+                        }
+                    }
+                }
+            }
         }
 
         [ConditionalTheory]
@@ -2471,6 +2548,55 @@ var customers
                     orderDetailsLoaded: true,
                     productLoaded: true);
             }
+
+            TestJsonSerialization(useNewtonsoft: false, ignoreLoops: false, writeIndented: false);
+            TestJsonSerialization(useNewtonsoft: false, ignoreLoops: false, writeIndented: true);
+
+            // Newtonsoft runs out of memory trying to serialize this.
+            //TestJsonSerialization(useNewtonsoft: true, ignoreLoops: true, writeIndented: false);
+
+            TestJsonSerialization(useNewtonsoft: true, ignoreLoops: false, writeIndented: false);
+            TestJsonSerialization(useNewtonsoft: true, ignoreLoops: false, writeIndented: true);
+
+            void TestJsonSerialization(bool useNewtonsoft, bool ignoreLoops, bool writeIndented)
+            {
+                var customersAgain = useNewtonsoft
+                    ? ThroughNewtonsoftJson(customers, ignoreLoops, writeIndented)
+                    : ThroughBclJson(customers, ignoreLoops, writeIndented, maxDepth: 1024);
+
+                var ordersMap = ignoreLoops ? null : new Dictionary<int, Order>();
+                var ordersDetailsMap = ignoreLoops ? null : new Dictionary<(int, int), OrderDetail>();
+                var customersMap = ignoreLoops ? null : new Dictionary<string, Customer>();
+                var productsMap = ignoreLoops ? null : new Dictionary<int, Product>();
+
+                foreach (var customer in customersAgain)
+                {
+                    VerifyCustomer(context, customer, customersMap);
+
+                    foreach (var order in customer.Orders)
+                    {
+                        VerifyOrder(context, order, ordersMap);
+
+                        if (!ignoreLoops)
+                        {
+                            Assert.Same(customer, order.Customer);
+                        }
+
+                        foreach (var orderDetail in order.OrderDetails)
+                        {
+                            VerifyOrderDetails(context, orderDetail, ordersDetailsMap);
+
+                            if (!ignoreLoops)
+                            {
+                                Assert.Same(order, orderDetail.Order);
+                            }
+
+                            var product = orderDetail.Product;
+                            VerifyProduct(context, product, productsMap);
+                        }
+                    }
+                }
+            }
         }
 
         [ConditionalTheory]
@@ -3785,6 +3911,146 @@ where o.OrderID < 10800
             }
         }
 
+        private static T ThroughBclJson<T>(T collection, bool ignoreLoops, bool writeIndented, int maxDepth = 64)
+        {
+            Assert.False(ignoreLoops, "BCL doesn't support ignoring loops.");
+
+#if NETCOREAPP5_0
+            var options = new JsonSerializerOptions
+            {
+                ReferenceHandling = ReferenceHandling.Preserve,
+                WriteIndented = writeIndented,
+                MaxDepth = maxDepth
+            };
+
+            return JsonSerializer.Deserialize<T>(JsonSerializer.Serialize(collection, options), options);
+#else
+            return collection;
+#endif
+        }
+
+        private static T ThroughNewtonsoftJson<T>(T collection, bool ignoreLoops, bool writeIndented)
+        {
+            var options = new Newtonsoft.Json.JsonSerializerSettings
+            {
+                PreserveReferencesHandling = ignoreLoops
+                ? Newtonsoft.Json.PreserveReferencesHandling.None
+                : Newtonsoft.Json.PreserveReferencesHandling.All,
+                ReferenceLoopHandling = ignoreLoops
+                ? Newtonsoft.Json.ReferenceLoopHandling.Ignore
+                : Newtonsoft.Json.ReferenceLoopHandling.Error,
+                EqualityComparer = LegacyReferenceEqualityComparer.Instance,
+                Formatting = writeIndented
+                    ? Newtonsoft.Json.Formatting.Indented
+                    : Newtonsoft.Json.Formatting.None
+            };
+
+           var serializeObject = Newtonsoft.Json.JsonConvert.SerializeObject(collection, options);
+
+           return Newtonsoft.Json.JsonConvert.DeserializeObject<T>(serializeObject);
+        }
+
+        private static void VerifyCustomer(NorthwindContext context, Customer customer, Dictionary<string, Customer> customersMap)
+        {
+            var trackedCustomer = context.Customers.Find(customer.CustomerID);
+            Assert.Equal(trackedCustomer.Address, customer.Address);
+            Assert.Equal(trackedCustomer.City, customer.City);
+            Assert.Equal(trackedCustomer.Country, customer.Country);
+            Assert.Equal(trackedCustomer.Fax, customer.Fax);
+            Assert.Equal(trackedCustomer.Phone, customer.Phone);
+            Assert.Equal(trackedCustomer.Region, customer.Region);
+            Assert.Equal(trackedCustomer.CompanyName, customer.CompanyName);
+            Assert.Equal(trackedCustomer.ContactName, customer.ContactName);
+            Assert.Equal(trackedCustomer.ContactTitle, customer.ContactTitle);
+            Assert.Equal(trackedCustomer.IsLondon, customer.IsLondon);
+            Assert.Equal(trackedCustomer.PostalCode, customer.PostalCode);
+            Assert.Equal(trackedCustomer.CustomerID, customer.CustomerID);
+
+            if (customersMap != null)
+            {
+                if (customersMap.TryGetValue(customer.CustomerID, out var mappedCustomer))
+                {
+                    Assert.Same(customer, mappedCustomer);
+                }
+
+                customersMap[customer.CustomerID] = customer;
+            }
+        }
+
+        private static void VerifyOrder(NorthwindContext context, Order order, IDictionary<int, Order> ordersMap)
+        {
+            var trackedOrder = context.Orders.Find(order.OrderID);
+            Assert.Equal(trackedOrder.Freight, order.Freight);
+            Assert.Equal(trackedOrder.OrderDate, order.OrderDate);
+            Assert.Equal(trackedOrder.RequiredDate, order.RequiredDate);
+            Assert.Equal(trackedOrder.ShipAddress, order.ShipAddress);
+            Assert.Equal(trackedOrder.ShipCity, order.ShipCity);
+            Assert.Equal(trackedOrder.ShipCountry, order.ShipCountry);
+            Assert.Equal(trackedOrder.ShipName, order.ShipName);
+            Assert.Equal(trackedOrder.ShippedDate, order.ShippedDate);
+            Assert.Equal(trackedOrder.ShipRegion, order.ShipRegion);
+            Assert.Equal(trackedOrder.ShipVia, order.ShipVia);
+            Assert.Equal(trackedOrder.CustomerID, order.CustomerID);
+            Assert.Equal(trackedOrder.EmployeeID, order.EmployeeID);
+            Assert.Equal(trackedOrder.OrderID, order.OrderID);
+            Assert.Equal(trackedOrder.ShipPostalCode, order.ShipPostalCode);
+
+            if (ordersMap != null)
+            {
+                if (ordersMap.TryGetValue(order.OrderID, out var mappedOrder))
+                {
+                    Assert.Same(order, mappedOrder);
+                }
+
+                ordersMap[order.OrderID] = order;
+            }
+        }
+
+        private static void VerifyOrderDetails(NorthwindContext context, OrderDetail orderDetail, IDictionary<(int, int), OrderDetail> orderDetailsMap)
+        {
+            var trackedOrderDetail = context.OrderDetails.Find(orderDetail.OrderID, orderDetail.ProductID);
+            Assert.Equal(trackedOrderDetail.Discount, orderDetail.Discount);
+            Assert.Equal(trackedOrderDetail.Quantity, orderDetail.Quantity);
+            Assert.Equal(trackedOrderDetail.UnitPrice, orderDetail.UnitPrice);
+            Assert.Equal(trackedOrderDetail.OrderID, orderDetail.OrderID);
+            Assert.Equal(trackedOrderDetail.ProductID, orderDetail.ProductID);
+
+            if (orderDetailsMap != null)
+            {
+                if (orderDetailsMap.TryGetValue((orderDetail.OrderID, orderDetail.ProductID), out var mappedOrderDetail))
+                {
+                    Assert.Same(orderDetail, mappedOrderDetail);
+                }
+
+                orderDetailsMap[(orderDetail.OrderID, orderDetail.ProductID)] = orderDetail;
+            }
+        }
+
+        private static void VerifyProduct(NorthwindContext context, Product product, IDictionary<int, Product> productsMap)
+        {
+            var trackedProduct = context.Products.Find(product.ProductID);
+            Assert.Equal(trackedProduct.Discontinued, product.Discontinued);
+            Assert.Equal(trackedProduct.ProductName, product.ProductName);
+            Assert.Equal(trackedProduct.ReorderLevel, product.ReorderLevel);
+            Assert.Equal(trackedProduct.UnitPrice, product.UnitPrice);
+            Assert.Equal(trackedProduct.CategoryID, product.CategoryID);
+            Assert.Equal(trackedProduct.ProductID, product.ProductID);
+            Assert.Equal(trackedProduct.QuantityPerUnit, product.QuantityPerUnit);
+            Assert.Equal(trackedProduct.SupplierID, product.SupplierID);
+            Assert.Equal(trackedProduct.UnitsInStock, product.UnitsInStock);
+            Assert.Equal(trackedProduct.UnitsOnOrder, product.UnitsOnOrder);
+
+            if (productsMap != null)
+            {
+                if (productsMap.TryGetValue(product.ProductID, out var mappedProduct))
+                {
+                    Assert.Same(product, mappedProduct);
+                }
+
+                productsMap[product.ProductID] = product;
+            }
+        }
+
         private static void CheckIsLoaded(
             NorthwindContext context,
             Customer customer,

test/EFCore.Specification.Tests/TestModels/Northwind/Customer.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Generic;
 using System.ComponentModel.DataAnnotations.Schema;
+using System.Text.Json.Serialization;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 
 // ReSharper disable UnusedParameter.Local
@@ -36,6 +37,8 @@ public Customer(DbContext context, ILazyLoader lazyLoader, string customerID)
 
         public virtual List<Order> Orders { get; set; }
 
+        [JsonIgnore]
+        [Newtonsoft.Json.JsonIgnore]
         public NorthwindContext Context { get; set; }
 
         [NotMapped]