From fd315a1e12feb7e1494b3c527d89dc3946f839e6 Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Wed, 18 Sep 2024 19:50:36 +0200 Subject: [PATCH] WIP Closes #4808 Part of #4805 (new function translations for Cosmos) --- .openpublishing.redirection.json | 5 + .../core/providers/cosmos/functions.md | 83 ----- .../core/providers/cosmos/index.md | 127 ++++---- .../core/providers/cosmos/limitations.md | 4 +- .../core/providers/cosmos/querying.md | 272 +++++++++++++++++ .../core/providers/cosmos/vector-search.md | 58 ++++ entity-framework/core/querying/pagination.md | 5 +- .../ef-core-9.0/breaking-changes.md | 285 ++++++++++++++++-- .../core/what-is-new/ef-core-9.0/plan.md | 11 - .../core/what-is-new/ef-core-9.0/whatsnew.md | 140 +++++++-- entity-framework/toc.yml | 4 - 11 files changed, 775 insertions(+), 219 deletions(-) delete mode 100644 entity-framework/core/providers/cosmos/functions.md create mode 100644 entity-framework/core/providers/cosmos/querying.md create mode 100644 entity-framework/core/providers/cosmos/vector-search.md delete mode 100644 entity-framework/core/what-is-new/ef-core-9.0/plan.md diff --git a/.openpublishing.redirection.json b/.openpublishing.redirection.json index 83e84b0743..b81233e570 100644 --- a/.openpublishing.redirection.json +++ b/.openpublishing.redirection.json @@ -479,6 +479,11 @@ "source_path": "entity-framework/core/logging-events-diagnostics/event-counters.md", "redirect_url": "/ef/core/logging-events-diagnostics/metrics", "redirect_document_id": false + }, + { + "source_path": "entity-framework/core/providers/cosmos/functions.md", + "redirect_url": "/ef/core/providers/cosmos/querying", + "redirect_document_id": false } ] } diff --git a/entity-framework/core/providers/cosmos/functions.md b/entity-framework/core/providers/cosmos/functions.md deleted file mode 100644 index fb66960203..0000000000 --- a/entity-framework/core/providers/cosmos/functions.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Function Mappings - Azure Cosmos DB Provider - EF Core -description: Function Mappings of the Azure Cosmos DB EF Core Provider -author: bricelam -ms.date: 7/26/2023 -uid: core/providers/cosmos/functions ---- -# Function Mappings of the Azure Cosmos DB EF Core Provider - -This page shows which .NET members are translated into which SQL functions when using the Azure Cosmos DB provider. - -## Date and time functions - -.NET | SQL ---------------------- | --- -DateTime.UtcNow | GetCurrentDateTime() -DateTimeOffset.UtcNow | GetCurrentDateTime() - -## Numeric functions - -.NET | SQL | Added in --------------------------- | ----------------- | -------- -double.DegreesToRadians(x) | RADIANS(@x) | EF Core 8.0 -double.RadiansToDegrees(x) | DEGREES(@x) | EF Core 8.0 -EF.Functions.Random() | RAND() -Math.Abs(value) | ABS(@value) -Math.Acos(d) | ACOS(@d) -Math.Asin(d) | ASIN(@d) -Math.Atan(d) | ATAN(@d) -Math.Atan2(y, x) | ATN2(@y, @x) -Math.Ceiling(d) | CEILING(@d) -Math.Cos(d) | COS(@d) -Math.Exp(d) | EXP(@d) -Math.Floor(d) | FLOOR(@d) -Math.Log(a, newBase) | LOG(@a, @newBase) -Math.Log(d) | LOG(@d) -Math.Log10(d) | LOG10(@d) -Math.Pow(x, y) | POWER(@x, @y) -Math.Round(d) | ROUND(@d) -Math.Sign(value) | SIGN(@value) -Math.Sin(a) | SIN(@a) -Math.Sqrt(d) | SQRT(@d) -Math.Tan(a) | TAN(@a) -Math.Truncate(d) | TRUNC(@d) - -> [!TIP] -> In addition to the methods listed here, corresponding [generic math](/dotnet/standard/generics/math) implementations -> and [MathF](/dotnet/api/system.mathf) methods are also translated. For example, `Math.Sin`, `MathF.Sin`, `double.Sin`, -> and `float.Sin` all map to the `SIN` function in SQL. - -## String functions - -.NET | SQL | Added in -------------------------------------------------------------- | ---------------------------------------------------------- | -------- -Regex.IsMatch(input, pattern) | RegexMatch(@pattern, @input) | EF Core 7.0 -Regex.IsMatch(input, pattern, options) | RegexMatch(@input, @pattern, @options) | EF Core 7.0 -string.Concat(str0, str1) | @str0 + @str1 -string.Equals(a, b, StringComparison.Ordinal) | STRINGEQUALS(@a, @b) -string.Equals(a, b, StringComparison.OrdinalIgnoreCase) | STRINGEQUALS(@a, @b, true) -stringValue.Contains(value) | CONTAINS(@stringValue, @value) -stringValue.EndsWith(value) | ENDSWITH(@stringValue, @value) -stringValue.Equals(value, StringComparison.Ordinal) | STRINGEQUALS(@stringValue, @value) -stringValue.Equals(value, StringComparison.OrdinalIgnoreCase) | STRINGEQUALS(@stringValue, @value, true) -stringValue.FirstOrDefault() | LEFT(@stringValue, 1) -stringValue.IndexOf(value) | INDEX_OF(@stringValue, @value) -stringValue.IndexOf(value, startIndex) | INDEX_OF(@stringValue, @value, @startIndex) -stringValue.LastOrDefault() | RIGHT(@stringValue, 1) -stringValue.Length | LENGTH(@stringValue) -stringValue.Replace(oldValue, newValue) | REPLACE(@stringValue, @oldValue, @newValue) -stringValue.StartsWith(value) | STARTSWITH(@stringValue, @value) -stringValue.Substring(startIndex) | SUBSTRING(@stringValue, @startIndex, LENGTH(@stringValue)) -stringValue.Substring(startIndex, length) | SUBSTRING(@stringValue, @startIndex, @length) -stringValue.ToLower() | LOWER(@stringValue) -stringValue.ToUpper() | UPPER(@stringValue) -stringValue.Trim() | TRIM(@stringValue) -stringValue.TrimEnd() | RTRIM(@stringValue) -stringValue.TrimStart() | LTRIM(@stringValue) - -## Miscellaneous functions - -.NET | SQL ---------------------------|---- -collection.Contains(item) | @item IN @collection diff --git a/entity-framework/core/providers/cosmos/index.md b/entity-framework/core/providers/cosmos/index.md index 1e13188251..9d35852c45 100644 --- a/entity-framework/core/providers/cosmos/index.md +++ b/entity-framework/core/providers/cosmos/index.md @@ -70,106 +70,97 @@ The Azure Cosmos DB provider for EF Core has multiple overloads of the [UseCosmo | Account endpoint and token | `UseCosmos(accountEndpoint, tokenCredential, databaseName)` | [Resource tokens](/azure/cosmos-db/secure-access-to-data#primary-keys) | | Connection string | `UseCosmos(connectionString, databaseName)` | [Work with account keys and connection strings](/azure/cosmos-db/scripts/cli/common/keys) | -## Queries +## Azure Cosmos DB options -### LINQ queries +It is also possible to configure the Azure Cosmos DB provider with a single connection string and to specify other options to customize the connection: -[EF Core LINQ queries](xref:core/querying/index) can be executed against Azure Cosmos DB in the same way as for other database providers. For example: +[!code-csharp[Configuration](../../../../samples/core/Cosmos/ModelBuilding/OptionsContext.cs?name=Configuration)] - -[!code-csharp[StringTranslations](../../../../samples/core/Miscellaneous/NewInEFCore6.Cosmos/CosmosQueriesSample.cs?name=StringTranslations)] +The code above shows some possible options - these are not intended to be used at the same time. See the [Azure Cosmos DB Options documentation](/dotnet/api/microsoft.azure.cosmos.cosmosclientoptions) for a detailed description of the effect of each option mentioned above. -> [!NOTE] -> The Azure Cosmos DB provider does not translate the same set of LINQ queries as other providers. See [_Limitations_](xref:core/providers/cosmos/limitations) for more information. +## Cosmos-specific model customization -### SQL queries +### Containers and entity types -Queries can also be written [directly in SQL](xref:core/querying/sql-queries). For example: +In Azure Cosmos DB, JSON documents are stored in containers. Unlike tables in relational databases, Cosmos DB containers can contain documents with different shapes - a container does not impose a uniform schema on its documents. However, various configuration options are defined at the container level, and therefore affect all documents contained within it. See the [Cosmos DB documentation on containers](/azure/cosmos-db/resource-model) for more information. - -[!code-csharp[FromSql](../../../../samples/core/Miscellaneous/NewInEFCore6.Cosmos/CosmosQueriesSample.cs?name=FromSql)] +By default, EF maps all entity types to the same container; this is usually a good default in terms of performance and pricing. The default container is named after the your .NET context type (`OrderContext` in this case). To change the default container name, use : + +```c# +modelBuilder.HasDefaultContainer("Store"); +``` -This query results in the following query execution: +To map an entity type to a different container use : -```sql -SELECT c -FROM ( - SELECT * FROM root c WHERE c["Angle1"] <= @p0 OR c["Angle2"] <= @p0 -) c +```c# +modelBuilder.Entity().ToContainer("Orders"); ``` -Just like for relational `FromSql` queries, the hand written SQL can be further composed using LINQ operators. For example: +Before mapping entity types to different containers, make sure you understand the potential performance and pricing implications (e.g. with regards to dedicated and shared throughput); [see the Cosmos DB documentation to learn more](/azure/cosmos-db/resource-model). - -[!code-csharp[FromSqlComposed](../../../../samples/core/Miscellaneous/NewInEFCore6.Cosmos/CosmosQueriesSample.cs?name=FromSqlComposed)] +### IDs and keys -This combination of SQL and LINQ is translated to: +Cosmos DB requires all documents to have an `id` JSON property which uniquely identifies them. Like other EF providers, the EF Cosmos provider will attempt to find a property named `Id` or `Id`, and configure that property as the key of your entity type, mapping it to the `id` JSON property. You can configure any property to be the key property by using ; see [the general EF documentation on keys](xref:core/modeling/keys) for more information. -```sql -SELECT DISTINCT c["Angle1"] -FROM ( - SELECT * FROM root c WHERE c["Angle1"] <= @p0 OR c["Angle2"] <= @p0 -) c -WHERE (c["InsertedOn"] <= GetCurrentDateTime()) -``` +Developers coming to Cosmos DB from other database sometimes expect the key (`Id`) property to be generated automatically. For example, on SQL Server, EF configures numeric key properties to be IDENTITY columns, where auto-incrementing values are generated in the database. In contrast, Cosmos DB does not support automatic generation of properties, and so key properties must be explicitly set. Inserting an entity type with an unset key property will simply insert the CLR default value for that property (e.g. 0 for `int`), and a second insert will fail; EF issues a warning if you attempt to do this. -## Azure Cosmos DB options +If you'd like to have a GUID as your key property, you can configure EF to generate random values at the client: -It is also possible to configure the Azure Cosmos DB provider with a single connection string and to specify other options to customize the connection: +```c# +modelBuilder.Entity().Property(b => b.Id).HasValueGenerator(); +``` -[!code-csharp[Configuration](../../../../samples/core/Cosmos/ModelBuilding/OptionsContext.cs?name=Configuration)] +### Discriminators -> [!TIP] -> The code above shows possible options. It is not intended that these will all be used at the same time! See the [Azure Cosmos DB Options documentation](/dotnet/api/microsoft.azure.cosmos.cosmosclientoptions) for a detailed description of the effect of each option mentioned above. +Since multiple entity types may be mapped to the same container, EF Core always adds a `$type` discriminator property to all JSON documents you save (this property was called `Discriminator` before EF 9.0); this allows EF to recognize documents being loaded from the database, and materialize the right .NET type. Developers coming from relational databases may be familiar with discriminators in the context of [table-per-hierarchy inheritance (TPH)](xref/core/modeling/inheritance#table-per-hierarchy-and-discriminator-configuration); in Cosmos, discriminators are used not just in inheritance mapping scenarios, but also because the same container can contain completely different document types. -## Cosmos-specific model customization +The discriminator property name and values can be configured with the standard EF APIs, [see these docs for more information](xref:core/modeling/inheritance). If you're mapping a single entity type to a container, are confident that you'll never be mapping another one, and would like to get rid of the discriminator property, call [HasNoDiscriminator](/dotnet/api/Microsoft.EntityFrameworkCore.Metadata.Builders.EntityTypeBuilder.HasNoDiscriminator): -By default all entity types are mapped to the same container, named after the derived context (`"OrderContext"` in this case). To change the default container name use [HasDefaultContainer](/dotnet/api/Microsoft.EntityFrameworkCore.CosmosModelBuilderExtensions.HasDefaultContainer): +```c# +modelBuilder.Entity().HasNoDiscriminator(); +``` -[!code-csharp[DefaultContainer](../../../../samples/core/Cosmos/ModelBuilding/OrderContext.cs?name=DefaultContainer)] +Since the same container can contain entity types of different types, and the JSON `id` property must be unique within a container partition, you cannot have the same `id` value for entities of different types in the same container partition. Compare this to relational databases, where each entity type is mapped to a different table, and therefore has its own, separate key space. It is therefore your responsibility to ensure the `id` uniqueness of documents you insert into a container. If you need to have different entity types with the same `id` values, you can instruct EF to automatically insert the discriminator into the `id` property as follows: -To map an entity type to a different container use [ToContainer](/dotnet/api/Microsoft.EntityFrameworkCore.CosmosEntityTypeBuilderExtensions.ToContainer): +```c# +modelBuilder.Entity().HasDiscriminatorInJsonId(); +``` -[!code-csharp[Container](../../../../samples/core/Cosmos/ModelBuilding/OrderContext.cs?name=Container)] +While this may make it easier to work with `id` values, it may make it harder to interoperate with external applications working with your documents, as they now must be aware of EF's concatenated `id` format, as well as the discriminator values, which are by default derived from your .NET types. Note that this was the default behavior prior to EF 9.0. -To identify the entity type that a given item represent EF Core adds a discriminator value even if there are no derived entity types. The name and value of the discriminator [can be changed](xref:core/modeling/inheritance). +An additional option is to instruct EF to insert only the _root discriminator_, which is the discriminator of the root entity type of the hierarchy, into the `id` property: -If no other entity type will ever be stored in the same container the discriminator can be removed by calling [HasNoDiscriminator](/dotnet/api/Microsoft.EntityFrameworkCore.Metadata.Builders.EntityTypeBuilder.HasNoDiscriminator): +```c# +modelBuilder.Entity().HasRootDiscriminatorInJsonId(); +``` -[!code-csharp[NoDiscriminator](../../../../samples/core/Cosmos/ModelBuilding/OrderContext.cs?name=NoDiscriminator)] +This is similar, but allows EF to use efficient [point reads](xref:core/providers/cosmos/querying#point-reads) in more scenarios. If you need to insert a discriminator into the `id` property, consider inserting the root discriminator for better performance. ### Partition keys -By default, EF Core will create containers with the partition key set to `"__partitionKey"` without supplying any value for it when inserting items. But to fully leverage the performance capabilities of Azure Cosmos DB, a [carefully selected partition key](/azure/cosmos-db/partition-data) should be used. It can be configured by calling [HasPartitionKey](/dotnet/api/Microsoft.EntityFrameworkCore.CosmosEntityTypeBuilderExtensions.HasPartitionKey): +Azure Cosmos DB uses partitioning to achieve horizontal scaling; proper modeling and careful selection of the partition key is vital for achieving good performance and keeping costs down. It's highly recommended to read [the Cosmos DB documentation on partitioning](/azure/cosmos-db/partition-data) and to plan your partitioning strategy in advance. -[!code-csharp[PartitionKey](../../../../samples/core/Cosmos/ModelBuilding/OrderContext.cs?name=PartitionKey)] +To configure the partitioning key with EF, call [HasPartitionKey](/dotnet/api/Microsoft.EntityFrameworkCore.CosmosEntityTypeBuilderExtensions.HasPartitionKey), passing it a regular property on your entity type: -> [!NOTE] ->The partition key property can be of any type as long as it is [converted to string](xref:core/modeling/value-conversions). +```c# +modelBuilder.Entity().HasPartitionKey(o => o.PartitionKey); +``` + +Any property can be made into a partition key as long as it is [converted to string](xref:core/modeling/value-conversions). Once configured, the partition key property should always have a non-null value; trying to insert a new entity type with an unset partition key property will result in an error. + +Cosmos DB also supports _hierarchical_ partition keys to optimize data distribution even further; [see the documentation for more details](/azure/cosmos-db/hierarchical-partition-keys). EF 9.0 added support for hierarchical partition keys; to configure these, simply pass up to 3 properties to [HasPartitionKey](/dotnet/api/Microsoft.EntityFrameworkCore.CosmosEntityTypeBuilderExtensions.HasPartitionKey): + +```c# +modelBuilder.Entity().HasPartitionKey(o => new { e.TenantId, e.UserId, e.SessionId }); +``` + +With such a hierarchical partition key, queries can be easily sent only to partitions containing data for a given tenant. -Once configured the partition key property should always have a non-null value. A query can be made single-partition by adding a call. +Note that the partition key is defined at the container level. This notably means that it's not possible to map multiple entity types to the same container, and for those entity types to have different partition keys. If you need to define different partition keys, map the relevant entity types to different containers. -[!code-csharp[PartitionKey](../../../../samples/core/Cosmos/ModelBuilding/Sample.cs?name=PartitionKey&highlight=14)] +If you don't configure a partition key with EF, a warning will be logged at startup; EF Core will create containers with the partition key set to `__partitionKey`, and won't supply any value for it when inserting items. While this can work and may be a good way to start as you're exploring Cosmos DB and your data modeling, it is highly discouraged to deploy a production application without a well-configured partition key strategy. -It is generally recommended to add the partition key to the primary key as that best reflects the server semantics and allows some optimizations, for example in `FindAsync`. +Once your partition keys properties are properly configured, you can provide values for them in queries; see [Querying with partition keys](xref:core/providers/cosmos/querying#partition-keys) for more information. ### Provisioned throughput @@ -400,8 +391,8 @@ context.SaveChanges(); Limitations: -* Only dictionaries with string keys are supported -* Querying into the contents of primitive collections is not currently supported. Vote for [#16926](https://github.com/dotnet/efcore/issues/16926), [#25700](https://github.com/dotnet/efcore/issues/25700), and [#25701](https://github.com/dotnet/efcore/issues/25701) if these features are important to you. +* Only dictionaries with string keys are supported. +* Support for querying into primitive collections was added in EF Core 9.0. ## Working with disconnected entities diff --git a/entity-framework/core/providers/cosmos/limitations.md b/entity-framework/core/providers/cosmos/limitations.md index 446e0aee1e..bd1ad6187e 100644 --- a/entity-framework/core/providers/cosmos/limitations.md +++ b/entity-framework/core/providers/cosmos/limitations.md @@ -17,6 +17,8 @@ Common EF Core patterns that either do not apply, or are a pit-of-failure, when - Loading graphs of related entities from different documents is not supported. Document databases are not designed to perform joins across many documents; doing so would be very inefficient. Instead, it is more common to denormalize data so that everything needed is in one, or a small number, of documents. However, there are some forms of cross-document relationships that could be handled--see [Limited Include support for Cosmos](https://github.com/dotnet/efcore/issues/16920#issuecomment-989721078). > [!WARNING] -> Since there are no sync versions of the low level methods EF Core relies on, the corresponding functionality is currently implemented by calling `.Wait()` on the returned `Task`. This means that using methods like `SaveChanges`, or `ToList` instead of their async counterparts could lead to a deadlock in your application +> The Cosmos SDK, which the EF provider uses, does not support synchronous I/O. As a result, synchronous EF APIs such as `ToList` or `SaveChanges` throw in version 9.0 and above; always use asynchronous +> methods when using EF. +> Previous versions of EF supported the synchronous APIs by calling `.Wait()` on the returned `Task`; this is known as "sync over async", and is a highly discouraged technique that can lead to deadlocks. See the EF 9.0 [breaking change note](xref:core/what-is-new/ef-core-9.0/breaking-changes#cosmos-nosync) for more information. Beyond the differences in relational and document databases, and limitations in the SDK, the EF Core provider for Azure Cosmos DB NoSQL does not include everything that _could_ be implemented using the combination of EF Core and the Cosmos SDK. Potential enhancements in this area are tracked by [issues in the EF Core GitHub repo marked with the label `area-cosmos`](https://github.com/dotnet/efcore/issues?q=is%3Aopen+is%3Aissue+label%3Aarea-cosmos+sort%3Areactions-%2B1-desc+label%3Atype-enhancement) The best way to indicate the importance of an issue is to vote (👍) for it. This data will then feed into the [planning process](xref:core/what-is-new/release-planning) for the next release. diff --git a/entity-framework/core/providers/cosmos/querying.md b/entity-framework/core/providers/cosmos/querying.md new file mode 100644 index 0000000000..e48474d2ab --- /dev/null +++ b/entity-framework/core/providers/cosmos/querying.md @@ -0,0 +1,272 @@ +--- +title: Querying - Azure Cosmos DB Provider - EF Core +description: Querying with the Azure Cosmos DB EF Core Provider +author: roji +ms.date: 09/19/2024 +uid: core/providers/cosmos/querying +--- +# Querying with the EF Core Azure Cosmos DB Provider + +## Querying basics + +[EF Core LINQ queries](xref:core/querying/index) can be executed against Azure Cosmos DB in the same way as for other database providers. For example: + + +[!code-csharp[StringTranslations](../../../../samples/core/Miscellaneous/NewInEFCore6.Cosmos/CosmosQueriesSample.cs?name=StringTranslations)] + +> [!NOTE] +> The Azure Cosmos DB provider does not translate the same set of LINQ queries as other providers. +> For example, the EF `Include()` operator isn't supported on Cosmos, since cross-document queries aren't supported in the database. + +## Partition keys + +The advantage of partitioning is to have your queries execute only against the partition where the relevant data is found, both saving computation and costs, and ensuring faster result speed. Queries which don't specify partition keys are executed on all the partitions, which can be quite costly. + +Starting with EF 9.0, EF automatically detects and extracts partition key comparisons in your LINQ query's `Where` operators. For example: + +```c# +var tenantId = "Microsoft"; +var userId = new Guid("99A410D7-E467-4CC5-92DE-148F3FC53F4C"); +var username = "scott"; + +var sessions = await context.Sessions + .Where( + e => e.TenantId == tenantId + && e.UserId == userId + && e.SessionId > 0 + && e.Username == username) + .ToListAsync(); +``` + +If you examine the logs generated by EF, you'll see this query executed as follows: + +```sql +Executed ReadNext (166.6985 ms, 2.8 RU) ActivityId='312da0d2-095c-4e73-afab-27072b5ad33c', Container='test', Partition='["Microsoft","99a410d7-e467-4cc5-92de-148f3fc53f4c"]', Parameters=[] +SELECT VALUE c +FROM root c +WHERE ((c["SessionId"] > 0) AND CONTAINS(c["Username"], "a")) +``` + +Examining the logs, we notice the following: + +* The first two comparisons - on `TenantId` and `UserId` - have been lifted out, and appear in the ReadNext's "Partition" rather than in the `WHERE` clause; this means that query will only execute on the subpartitions for those values. +* `SessionId` is also part of the hierarchical partition key, but instead of an equality comparison it uses a greater-than operator (`>`), and therefore cannot be lifted out. It is part of the `WHERE` clause like any regular property. +* `Username` is a regular property - not part of the partition key - and therefore remains in the `WHERE` clause as well. + +Note that even though not all three partition key properties are provided, hierarchical partition keys still allow targeting only the subpartitions which correspond to the first two properties. While this isn't as efficient as targeting a single partition (as identified by all three properties), it's still much more efficient than targeting all partitions. + +Rather than referencing partition key properties in a `Where` operator, you can explicitly specify them by using the operator: + +```c# +var sessions = await context.Sessions + .WithPartitionKey(tenantId, userId) + .Where(e => e.SessionId > 0 && e.Username.Contains("a")) + .ToListAsync(); +``` + +This executes in the same way as the above query. Using may be necessary in versions of EF prior to 9.0 - always keep an eye on the logs to ensure that queries are using partition keys as expected. + +## Point reads + +While Azure Cosmos DB has rich support for SQL, allowing powerful querying, such SQL queries can be quite expensive. Cosmos DB also supports _point reads_, which can be used when both the `id` property and the entire partition key are known. Such point reads directly identify a specific document in a specific partition, and execute extremely efficiently and with reduced costs. If at all possible, it's worth designing your system in a way which leverages point reads as much as possible. To read more, see the [Cosmos DB documentation](/azure/cosmos-db/nosql/how-to-dotnet-read-item). + +In the previous section, we saw EF identifying and extracting partition key comparisons from the `Where` for more efficient querying, restricting processing only to the relevant partitions. It's possible to go a step further, and provide the `id` property in the query as well. Let's examine the following query: + +```c# +var session = await context.Sessions.SingleAsync( + e => e.Id == someId + && e.TenantId == tenantId + && e.UserId == userId + && e.SessionId == sessionId); +``` + +In this query, a value for the `Id` property is provided (this is mapped to the Cosmos DB `id` property), and well as values for all the partition key properties; and furthermore, there are no additional components to the query. In this specific conditions, EF is able to execute the query as follows: + +```console +Executed ReadItem (46 ms, 1 RU) ActivityId='d7391311-2266-4811-ae2d-535904c42c43', Container='test', Id='9', Partition='["Microsoft","99a410d7-e467-4cc5-92de-148f3fc53f4c",10.0]' +``` + +Note the `ReadItem`, which indicates that the query was executed as an efficient point read - no SQL query is involved. + +Note that as with partition key extraction, significant improvements have been made to this mechanism in EF 9.0; older versions do not reliably detect and use point reads. + +## Pagination + +> [!NOTE] +> This feature was introduced in EF Core 9.0. + +Pagination refers to retrieving results in pages, rather than all at once; this is typically done for large resultsets, where a user interface is shown that allows the user to navigate to the next or previous page of the results. + +A common way to implement pagination with databases is to use the `Skip` and `Take` LINQ operators (`OFFSET` and `LIMIT` in SQL). Given a page size of 10 results, the third page can be fetched with EF Core as follows: + +```c# +var position = 20; +var nextPage = context.Posts + .OrderBy(b => b.PostId) + .Skip(position) + .Take(10) + .ToList(); +``` + +Unfortunately, this technique is quite inefficient and can considerably increase querying costs. Cosmos DB provides a special mechanism for paginating through the result of a query, via the use of _continuation tokens_: + +```c# +var firstPage = await context.Posts + .OrderBy(p => p.Id) + .ToPageAsync(pageSize: 10, continuationToken: null); + +var continuationToken = page.ContinuationToken; +foreach (var post in page.Values) +{ + // Display/send the posts to the user +} +``` + +Rather than terminating the LINQ query with `ToListAsync` or similar, we use the `ToPageAsync` method, instructing it to get at most 10 items in every page (note that there may be fewer in the database). Since this is our first query, we'd like to get results from the beginning, and pass `null` as the continuation token. `ToPageAsync` returns a `CosmosPage`, which exposes a continuation token and the values in the page (up to 10 Posts). Your program will typically send those values to the client, along with the continuation token; this will allow resuming the query and fetching more results. + +Let's assume the user now clicks on the "Next" button in their UI, asking for the next 10 Posts. You can then execute the query as follows: + +```c# +var nextPage = await context.Sessions.OrderBy(s => s.Id).ToPageAsync(10, continuationToken); +var continuationToken = page.ContinuationToken; +foreach (var post in page.Values) +{ + // Display/send the post to the user +} +``` + +We execute the same query, but this time we pass in the continuation token received from the first execution; this instructs Cosmos DB to continue the query where it left off, and fetch the next 10 Posts. This method of paginating is extremely efficient and cost-effective compared to using `Skip` and `Take`. + +To learn more about pagination in Cosmos DB, [see this page](/azure/cosmos-db/nosql/query/pagination). + +> [!NOTE] +> Cosmos DB does not support backwards pagination. +> +> `ToPageAsync` is currently annotated as experimental, since it may be replaced with a more general EF pagination API that isn't Cosmos-specific. Although using it will generate a compilation warning (`EF9102`), doing so should be safe - future changes may require minor tweaks in the API shape. + +## SQL queries + +Queries can also be written [directly in SQL](xref:core/querying/sql-queries). For example: + +```c# +var rating = 3; +_ = await context.Blogs + .FromSql($"SELECT VALUE c FROM root c WHERE c.Rating > {rating}") + .ToListAsync(); +``` + +This query results in the following query execution: + +```sql +SELECT VALUE s +FROM ( + SELECT VALUE c FROM root c WHERE c.Angle1 <= @p0 +) s +``` + +Note that `FromSql` was introduced in EF 9.0. On previous versions, `FromSqlRaw` can be used instead, although note that that method is vulnerable to SQL injection attacks. + +For more information on SQL querying, see the [relational documentation on SQL queries](xref:core/querying/sql-queries); most of that content is relevant for the Cosmos provider as well. + +## Function mappings + +This section shows which .NET methods and members are translated into which SQL functions when querying with the Azure Cosmos DB provider. + +### Date and time functions + +.NET | SQL | Added in +---------------------------------- | --------------------------------------------------------------------------------- | -------- +DateTime.UtcNow | [GetCurrentDateTime()](/azure/cosmos-db/nosql/query/getcurrentdatetime) +DateTimeOffset.UtcNow | [GetCurrentDateTime()](/azure/cosmos-db/nosql/query/getcurrentdatetime) +dateTime.Year[^1] | [DateTimePart("yyyy", dateTime)](/azure/cosmos-db/nosql/query/datetimepart) | EF Core 9.0 +dateTimeOffset.Year[^1] | [DateTimePart("yyyy", dateTimeOffset)](/azure/cosmos-db/nosql/query/datetimepart) | EF Core 9.0 +dateTime.AddYears(years)[^1] | [DateTimeAdd("yyyy", dateTime)](/azure/cosmos-db/nosql/query/datetimeadd) | EF Core 9.0 +dateTimeOffset.AddYears(years)[^1] | [DateTimeAdd("yyyy", dateTimeOffset)](/azure/cosmos-db/nosql/query/datetimeadd) | EF Core 9.0 + +[^1]: The other component members are translated as well (Month, Day...). + +### Numeric functions + +.NET | SQL | Added in +-------------------------- | --------------------------------------------------- | -------- +double.DegreesToRadians(x) | [RADIANS(@x)](/azure/cosmos-db/nosql/query/radians) | EF Core 8.0 +double.RadiansToDegrees(x) | [DEGREES(@x)](/azure/cosmos-db/nosql/query/degrees) | EF Core 8.0 +EF.Functions.Random() | [RAND()](/azure/cosmos-db/nosql/query/rand) +Math.Abs(value) | [ABS(@value)](/azure/cosmos-db/nosql/query/abs) +Math.Acos(d) | [ACOS(@d)](/azure/cosmos-db/nosql/query/acos) +Math.Asin(d) | [ASIN(@d)](/azure/cosmos-db/nosql/query/asin) +Math.Atan(d) | [ATAN(@d)](/azure/cosmos-db/nosql/query/atan) +Math.Atan2(y, x) | [ATN2(@y, @x)](/azure/cosmos-db/nosql/query/atn2) +Math.Ceiling(d) | [CEILING(@d)](/azure/cosmos-db/nosql/query/ceiling) +Math.Cos(d) | [COS(@d)](/azure/cosmos-db/nosql/query/cos) +Math.Exp(d) | [EXP(@d)](/azure/cosmos-db/nosql/query/exp) +Math.Floor(d) | [FLOOR(@d)](/azure/cosmos-db/nosql/query/floor) +Math.Log(a, newBase) | [LOG(@a, @newBase)](/azure/cosmos-db/nosql/query/log) +Math.Log(d) | [LOG(@d)](/azure/cosmos-db/nosql/query/log) +Math.Log10(d) | [LOG10(@d)](/azure/cosmos-db/nosql/query/log10) +Math.Pow(x, y) | [POWER(@x, @y)](/azure/cosmos-db/nosql/query/power) +Math.Round(d) | [ROUND(@d)](/azure/cosmos-db/nosql/query/round) +Math.Sign(value) | [SIGN(@value)](/azure/cosmos-db/nosql/query/sign) +Math.Sin(a) | [SIN(@a)](/azure/cosmos-db/nosql/query/sin) +Math.Sqrt(d) | [SQRT(@d)](/azure/cosmos-db/nosql/query/sqrt) +Math.Tan(a) | [TAN(@a)](/azure/cosmos-db/nosql/query/atan) +Math.Truncate(d) | [TRUNC(@d)](/azure/cosmos-db/nosql/query/trunc) + +> [!TIP] +> In addition to the methods listed here, corresponding [generic math](/dotnet/standard/generics/math) implementations +> and [MathF](/dotnet/api/system.mathf) methods are also translated. For example, `Math.Sin`, `MathF.Sin`, `double.Sin`, +> and `float.Sin` all map to the `SIN` function in SQL. + +### String functions + +.NET | SQL | Added in +----------------------------------------------------------------- | ---------------------------------------------------------------------------------- | -------- +Regex.IsMatch(input, pattern) | [RegexMatch(@pattern, @input)](azure/cosmos-db/nosql/query/regexmatch) | EF Core 7.0 +Regex.IsMatch(input, pattern, options) | [RegexMatch(@input, @pattern, @options)](azure/cosmos-db/nosql/query/regexmatch) | EF Core 7.0 +string.Concat(str0, str1) | @str0 + @str1 +string.Equals(a, b, StringComparison.Ordinal) | [STRINGEQUALS(@a, @b)](/azure/cosmos-db/nosql/query/stringequals) +string.Equals(a, b, StringComparison.OrdinalIgnoreCase) | [STRINGEQUALS(@a, @b, true)](/azure/cosmos-db/nosql/query/stringequals) +stringValue.Contains(value) | [CONTAINS(@stringValue, @value)](/azure/cosmos-db/nosql/query/contains) +stringValue.Contains(value, StringComparison.Ordinal) | [CONTAINS(@stringValue, @value, false)](/azure/cosmos-db/nosql/query/contains) | EF Core 9.0 +stringValue.Contains(value, StringComparison.OrdinalIgnoreCase) | [CONTAINS(@stringValue, @value, true)](/azure/cosmos-db/nosql/query/contains) | EF Core 9.0 +stringValue.EndsWith(value) | [ENDSWITH(@stringValue, @value)](/azure/cosmos-db/nosql/query/endswith) +stringValue.EndsWith(value, StringComparison.Ordinal) | [ENDSWITH(@stringValue, @value, false)](/azure/cosmos-db/nosql/query/endswith) | EF Core 9.0 +stringValue.EndsWith(value, StringComparison.OrdinalIgnoreCase) | [ENDSWITH(@stringValue, @value, true)](/azure/cosmos-db/nosql/query/endswith) | EF Core 9.0 +stringValue.Equals(value, StringComparison.Ordinal) | [STRINGEQUALS(@stringValue, @value)](/azure/cosmos-db/nosql/query/stringequals) +stringValue.Equals(value, StringComparison.OrdinalIgnoreCase) | [STRINGEQUALS(@stringValue, @value, true)](/azure/cosmos-db/nosql/query/stringequals) +stringValue.FirstOrDefault() | [LEFT(@stringValue, 1)](/azure/cosmos-db/nosql/query/left) +stringValue.IndexOf(value) | [INDEX_OF(@stringValue, @value)](/azure/cosmos-db/nosql/query/index-of) +stringValue.IndexOf(value, startIndex) | [INDEX_OF(@stringValue, @value, @startIndex)](/azure/cosmos-db/nosql/query/index-of) +stringValue.LastOrDefault() | [RIGHT(@stringValue, 1)](azure/cosmos-db/nosql/query/right) +stringValue.Length | [LENGTH(@stringValue)](/azure/cosmos-db/nosql/query/length) +stringValue.Replace(oldValue, newValue) | [REPLACE(@stringValue, @oldValue, @newValue)](/azure/cosmos-db/nosql/query/replace) +stringValue.StartsWith(value) | [STARTSWITH(@stringValue, @value)](/azure/cosmos-db/nosql/query/startswith) +stringValue.StartsWith(value, StringComparison.Ordinal) | [STARTSWITH(@stringValue, @value, false)](/azure/cosmos-db/nosql/query/startswith) | EF Core 9.0 +stringValue.StartsWith(value, StringComparison.OrdinalIgnoreCase) | [STARTSWITH(@stringValue, @value, true)](/azure/cosmos-db/nosql/query/startswith) | EF Core 9.0 +stringValue.Substring(startIndex) | [SUBSTRING(@stringValue, @startIndex, LENGTH(@stringValue))](/azure/cosmos-db/nosql/query/substring) +stringValue.Substring(startIndex, length) | [SUBSTRING(@stringValue, @startIndex, @length)](/azure/cosmos-db/nosql/query/substring) +stringValue.ToLower() | [LOWER(@stringValue)](/azure/cosmos-db/nosql/query/lower) +stringValue.ToUpper() | [UPPER(@stringValue)](/azure/cosmos-db/nosql/query/upper) +stringValue.Trim() | [TRIM(@stringValue)](/azure/cosmos-db/nosql/query/trim) +stringValue.TrimEnd() | [RTRIM(@stringValue)](/azure/cosmos-db/nosql/query/rtrim) +stringValue.TrimStart() | [LTRIM(@stringValue)](/azure/cosmos-db/nosql/query/ltrim) + +### Miscellaneous functions + +.NET | SQL | Notes +------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | ----- +collection.Contains(item) | @item IN @collection +EF.Functions.CoalesceUndefined(x, y)[^2] | [x ?? y](/azure/cosmos-db/nosql/query/ternary-coalesce-operators#coalesce-operator) | Added in EF Core 9.0 +EF.Functions.IsDefined(x) | [IS_DEFINED(x)](/azure/cosmos-db/nosql/query/is-defined) | Added in EF Core 9.0 +EF.Functions.VectorDistance(vector1, vector2)[^3] | [VectorDistance(vector1, vector2)](/azure/cosmos-db/nosql/query/vectordistance) | Added in EF Core 9.0, Experimental +EF.Functions.VectorDistance(vector1, vector2, bruteForce)[^3] | [VectorDistance(vector1, vector2, bruteForce)](/azure/cosmos-db/nosql/query/vectordistance) | Added in EF Core 9.0, Experimental +EF.Functions.VectorDistance(vector1, vector2, bruteForce, distanceFunction)[^3] | [VectorDistance(vector1, vector2, bruteForce, distanceFunction)](/azure/cosmos-db/nosql/query/vectordistance) | Added in EF Core 9.0, Experimental + +[^2]: Note that `EF.Functions.CoalesceUndefined` coalesces `undefined`, not `null`. To coalesce `null`, use the regular C# `??` operator. +[^3]: [See the documentation](xref:TODO) for information on using vector search in Azure Cosmos DB. Cosmos DB vector searching is experimental and the APIs are subject to change. diff --git a/entity-framework/core/providers/cosmos/vector-search.md b/entity-framework/core/providers/cosmos/vector-search.md new file mode 100644 index 0000000000..9ebcdff656 --- /dev/null +++ b/entity-framework/core/providers/cosmos/vector-search.md @@ -0,0 +1,58 @@ +--- +title: Vector Search - Azure Cosmos DB Provider - EF Core +description: Vector search with the Azure Cosmos DB EF Core Provider +author: roji +ms.date: 09/20/2024 +uid: core/providers/cosmos/vector-search +--- +# Vector search + +> [!WARNING] +> Azure Cosmos DB vector search is currently in preview. As a result, using EF's vector search APIs will generate an "experimental API" warning (`EF9103`) which must be suppressed. The APIs and capabilities may change in breaking ways in the future. + +Azure Cosmos DB now offers preview support for vector similarity search. Vector search is a fundamental part of some application types, include AI, semantic search and others. The Cosmos DB support for vector search allows storing your data and vectors, and performing your queries in a single database, which can considerably simplify your architecture and remove the need for an additional, dedicated vector database solution in your stack. To learn more about Cosmos DB vector search, [see the documentation](/azure/cosmos-db/nosql/vector-search). + +To use vector search, you must first [enroll in the preview feature](/azure/cosmos-db/nosql/vector-search#enroll-in-the-vector-search-preview-feature). Then, [define the vector policies on your container](/vector-search#container-vector-policies), which determine which JSON property in your documents contain vectors, various vector-related information for those properties (dimensions, data type, distance function). + +Once your container is properly set up, add a vector property to your model in the path you defined in the container policy, and configure it with EF as a vector: + +```c# +public class Blog +{ + ... + + public float[] Vector { get; set; } +} + +public class BloggingContext +{ + ... + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity() + .Property(b => b.Embeddings) + .IsVector(DistanceFunction.Cosine, dimensions: 1536); + } +} +``` + +At this point your model is configured. Insertion of vector data is done just like any other data type with EF: + +```c# +float[] vector = /* generate vector data from text, image, etc. */ +context.Add(new Blog { Vector = vector }); +await context.SaveChangesAsync(); +``` + +Finally, use the `EF.Functions.VectorDistance()` function in LINQ queries to perform vector similarity search: + +```c# +float[] anotherVector = /* generate vector data from text, image, etc. */ +var blogs = await context.Blogs + .OrderBy(s => EF.Functions.VectorDistance(s.Vector, anotherVector)) + .Take(5) + .ToListAsync(); +``` + +This will returns the top five Blogs, based on the similarity of their `Vector` property and the externally-provided `anotherVector` data. diff --git a/entity-framework/core/querying/pagination.md b/entity-framework/core/querying/pagination.md index 13937c48c4..75a3a0821f 100644 --- a/entity-framework/core/querying/pagination.md +++ b/entity-framework/core/querying/pagination.md @@ -12,9 +12,12 @@ Pagination refers to retrieving results in pages, rather than all at once; this > [!WARNING] > Regardless of the pagination method used, always make sure that your ordering is fully unique. For example, if results are ordered only by date, but there can be multiple results with the same date, then results could be skipped when paginating as they're ordered differently across two paginating queries. Ordering by both date and ID (or any other unique property or combination of properties) makes the ordering fully unique and avoids this problem. Note that relational databases do not apply any ordering by default, even on the primary key. +> [!NOTE] +> Azure Cosmos DB has its own mechanism for pagination, [see the dedicated documentation page](xref:core/providers/cosmos/querying#pagination). + ## Offset pagination -A common way to implement pagination with databases is to use the `Skip` and `Take` (`OFFSET` and `LIMIT` in SQL). Given a page size of 10 results, the third page can be fetched with EF Core as follows: +A common way to implement pagination with databases is to use the `Skip` and `Take` LINQ operators (`OFFSET` and `LIMIT` in SQL). Given a page size of 10 results, the third page can be fetched with EF Core as follows: [!code-csharp[Main](../../../samples/core/Querying/Pagination/Program.cs?name=OffsetPagination&highlight=4)] diff --git a/entity-framework/core/what-is-new/ef-core-9.0/breaking-changes.md b/entity-framework/core/what-is-new/ef-core-9.0/breaking-changes.md index 810ee5f2e9..d017fa3e6f 100644 --- a/entity-framework/core/what-is-new/ef-core-9.0/breaking-changes.md +++ b/entity-framework/core/what-is-new/ef-core-9.0/breaking-changes.md @@ -20,33 +20,168 @@ EF Core 9 targets .NET 8. This means that existing applications that target .NET ## Summary +> [!NOTE] +> If you are using Azure Cosmos DB, please see the [separate section below on Cosmos DB breaking changes](#cosmos-breaking-changes). + | **Breaking change** | **Impact** | |:-----------------------------------------------------------------------------------------------------|------------| -| [Sync I/O via the Azure Cosmos DB provider is no longer supported](#cosmos-nosync) | Medium | | [EF.Functions.Unhex() now returns `byte[]?`](#unhex) | Low | | [SqlFunctionExpression's nullability arguments' arity validated](#sqlfunctionexpression-nullability) | Low | -## Medium-impact changes +## Low-impact changes + + + +### EF.Functions.Unhex() now returns `byte[]?` + +[Tracking Issue #33864](https://github.com/dotnet/efcore/issues/33864) + +#### Old behavior + +The EF.Functions.Unhex() function was previously annotated to return `byte[]`. + +#### New behavior + +Starting with EF Core 9.0, Unhex() is now annotated to return `byte[]?`. + +#### Why + +Unhex() is translated to the SQLite `unhex` function, which returns NULL for invalid inputs. As a result, Unhex() returned `null` for those cases, in violation of the annotation. + +#### Mitigations + +If you are sure that the text content passed to Unhex() represents a valid, hexadecimal string, you can simply add the null-forgiving operator as an assertion that the invocation will never return null: + +```c# +var binaryData = await context.Blogs.Select(b => EF.Functions.Unhex(b.HexString)!).ToListAsync(); +``` + +Otherwise, add runtime checks for null on the return value of Unhex(). + + + +### SqlFunctionExpression's nullability arguments' arity validated + +[Tracking Issue #33852](https://github.com/dotnet/efcore/issues/33852) + +#### Old behavior + +Previously it was possible to create a `SqlFunctionExpression` with a different number of arguments and nullability propagation arguments. + +#### New behavior + +Starting with EF Core 9.0, EF now throws if the number of arguments and nullability propagation arguments do not match. + +#### Why + +Not having matching number of arguments and nullability propagation arguments can lead to unexpected behavior. + +#### Mitigations + +Make sure the `argumentsPropagateNullability` has same number of elements as the `arguments`. When in doubt use `false` for nullability argument. + +## Cosmos breaking changes + +Extensive work has gone into making the Cosmos DB provider better in 9.0. The changes include a number of high-impact breaking changes; if you are upgrading an existing application, please read the following carefully. + +| **Breaking change** | **Impact** | +|:---------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | +| [The discriminator property is now named `$type` instead of `Discriminator`](#cosmos-discriminator-name-change) | High | +| [The `id` property no longer contqins the discriminator by default](#cosmos-id-property-changes) | High | +| [Sync I/O via the Azure Cosmos DB provider is no longer supported](#cosmos-nosync) | Medium | +| [SQL queries must now project JSON values directly](#cosmos-sql-queries-with-value) | Medium | +| [Undefined results are now automatically filtered from query results](#cosmos-undefined-filtering) | Medium | +| [`IncludeRootDiscriminatorInJsonId` was renamed to `HasRootDiscriminatorInJsonId` after 9.0.0-rc.2](#cosmos-IncludeRootDiscriminatorInJsonId-rename) | Low | + +### High-impact changes + + + +#### The discriminator property is now named `$type` instead of `Discriminator` + +[Tracking Issue #34269](https://github.com/dotnet/efcore/issues/34269) + +##### Old behavior + +EF automatically adds a discriminator property to JSON documents to identify the entity type that the document represents. In previous versions of EF, this JSON property used to be named `Discriminator` by default. + +##### New behavior + +Starting with EF Core 9.0, the discriminator property is now called `$type` by default. If you have existing documents in Cosmos DB from previous versions of EF, these use the old `Discriminator` naming, and after upgrading to EF 9.0, queries against those documents will fail. + +##### Why + +An emerging JSON practice uses a `$type` property in scenarios where a document's type needs to be identified. For example, .NET's System.Text.Json also supports polymorphism, using `$type` as its default discriminator property name ([docs](/dotnet/standard/serialization/system-text-json/polymorphism#customize-the-type-discriminator-name)). To align with the rest of the ecosystem and make it easier to interoperate with external tools, the default was changed. + +##### Mitigations + +The easiest mitigation is to simply configure the name of the discriminator property to be `Discriminator`, just as before: + +```c# +modelBuilder.Entity().HasDiscriminator("Discriminator"); +``` + +Doing this for all your top-level entity types will make EF behave just like before. + +At this point, if you wish, you can also update all your documents to use the new `$type` naming. + +TODO: Cosmos SDK code sample for updating the documents + + + +#### The `id` property now contains only the EF key property by default + +[Tracking Issue #34179](https://github.com/dotnet/efcore/issues/34179) + +##### Old behavior + +Previously, EF inserted the discriminator value of your entity type into the `id` property of the document. For example, if you saved a `Blog` entity type with an `Id` property containing 8, the JSON `id` property would contain `Blog|8`. + +##### New behavior + +Starting with EF Core 9.0, the JSON `id` property no longer contains the discriminator value, and only contains the avlue of your key property. For the above example, the JSON `id` property would simply be `8`. If you have existing documents in Cosmos DB from previous versions of EF, these have the discriminator value in the JSON `id` property, and after upgrading to EF 9.0, queries against those documents will fail. + +##### Why + +Since the JSON `id` property must be unique, the discriminator was previously added to it so as to allow different entities to exist with the same key value; for example, this allowed having both a `Blog` and a `Post` with an `Id` property containing 8 within the same container and partition. This was assumed to be expected by developers used to relational databases, where each entity type is mapped its own table, and therefore has its own key-space. + +EF 9.0 generally changed the mapping to be more aligned with common Cosmos DB practices and expectations, rather than to correspond to the expectations of users coming from relational databases. In addition, having the discriminator value in the `id` property made it more difficult for external tools and systems to interact with EF-generated JSON documents; such external systems aren't generally aware of the EF discriminator values, which are by default derived from .NET types. + +##### Mitigations + +The easiest mitigation is to simply configure EF to include the discriminator in the JSON `id` property, as before. A new configuration option has been introduced for this purpose: + +```c# +modelBuilder.Entity().HasDiscriminatorInJsonId(); +``` + +Doing this for all your top-level entity types will make EF behave just like before. + +At this point, if you wish, you can also update all your documents to rewrite their JSON `id` property. Note that this is only possible if entities of different types don't share the same id value within the same container. + +TODO: Cosmos SDK code sample for updating the documents + +### Medium-impact changes -### Sync I/O via the Azure Cosmos DB provider is no longer supported +#### Sync I/O via the Azure Cosmos DB provider is no longer supported [Tracking Issue #32563](https://github.com/dotnet/efcore/issues/32563) -#### Old behavior +##### Old behavior Previously, calling synchronous methods like `ToList` or `SaveChanges` would cause EF Core to block synchronously using `.GetAwaiter().GetResult()` when executing async calls against the Azure Cosmos DB SDK. This can result in deadlock. -#### New behavior +##### New behavior Starting with EF Core 9.0, EF now throws by default when attempting to use synchronous I/O. The exception message is "Azure Cosmos DB does not support synchronous I/O. Make sure to use and correctly await only async methods when using Entity Framework Core to access Azure Cosmos DB. See [https://aka.ms/ef-cosmos-nosync](https://aka.ms/ef-cosmos-nosync) for more information." -#### Why +##### Why Synchronous blocking on asynchronous methods can result in deadlock, and the Azure Cosmos DB SDK only supports async methods. -#### Mitigations +##### Mitigations In EF Core 9.0, the error can be suppressed with: @@ -59,54 +194,140 @@ protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) That being said, applications should stop using sync APIs with Azure Cosmos DB since this is not supported by the Azure Cosmos DB SDK. The ability to suppress the exception will be removed in a future release of EF Core, after which the only option will be to use async APIs. -## Low-impact changes + - +#### SQL queries must now project JSON values directly -### EF.Functions.Unhex() now returns `byte[]?` +[Tracking Issue #25527](https://github.com/dotnet/efcore/issues/25527) -[Tracking Issue #33864](https://github.com/dotnet/efcore/issues/33864) +##### Old behavior -#### Old behavior +Previously, EF generated queries such as the following: -The EF.Functions.Unhex() function was previously annotated to return `byte[]`. +```sql +SELECT c["City"] FROM root c +``` -#### New behavior +Such queries cause Cosmos DB to wrap each result in a JSON object, as follows: + +```json +[ + { + "City": "Berlin" + }, + { + "City": "México D.F." + } +] +``` -Starting with EF Core 9.0, Unhex() is now annotated to return `byte[]?`. +##### New behavior -#### Why +Starting with EF Core 9.0, EF now adds the `VALUE` modifier to queries as follows: -Unhex() is translated to the SQLite `unhex` function, which returns NULL for invalid inputs. As a result, Unhex() returned `null` for those cases, in violation of the annotation. +```sql +SELECT VALUE c["City"] FROM root c +``` -#### Mitigations +Such queries cause Cosmos DB to return the values directly, without being wrapped: -If you are sure that the text content passed to Unhex() represents a valid, hexadecimal string, you can simply add the null-forgiving operator as an assertion that the invocation will never return null: +```json +[ + "Berlin", + "México D.F." +] +``` + +If your application makes use of [SQL queries](xref:core/providers/cosmos/querying#sql-queries), such queries are likely broken after upgrading to EF 9.0, as they don't include the `VALUE` modifier. + +##### Why + +Wrapping each result in an additional JSON object can cause performance degradation in some scenarios, bloats the JSON result payload, and isn't the natural way to work with Cosmos DB. + +##### Mitigations + +To mitigate, simply add the `VALUE` modifier to the projections of your SQL queries, as shown above. + +#### Undefined results are now automatically filtered from query results + +[Tracking Issue #25527](https://github.com/dotnet/efcore/issues/25527) + +##### Old behavior + +Previously, EF generated queries such as the following: + +```sql +SELECT c["City"] FROM root c +``` + +Such queries cause Cosmos DB to wrap each result in a JSON object, as follows: + +```json +[ + { + "City": "Berlin" + }, + { + "City": "México D.F." + } +] +``` + +If any of the results were undefined (e.g. the `City` property was absent from the document), an empty document was returned, and EF would return `null` for that result. + +##### New behavior + +Starting with EF Core 9.0, EF now adds the `VALUE` modifier to queries as follows: + +```sql +SELECT VALUE c["City"] FROM root c +``` + +Such queries cause Cosmos DB to return the values directly, without being wrapped: + +```json +[ + "Berlin", + "México D.F." +] +``` + +The Cosmos DB behavior is to automatically filter `undefined` values out of results; this means that if one of the `City` properties is absent from the document, the query would return just a single result, rather than two results, with one being `null`. + +##### Why + +Wrapping each result in an additional JSON object can cause performance degradation in some scenarios, bloats the JSON result payload, and isn't the natural way to work with Cosmos DB. + +##### Mitigations + +If getting `null` values for undefined results is important for your application, coalesce the `undefined` values to `null` using the new `EF.Functions.Coalesce` operator: ```c# -var binaryData = await context.Blogs.Select(b => EF.Functions.Unhex(b.HexString)!).ToListAsync(); +var users = await context.Customer + .Select(c => EF.Functions.CoalesceUndefined(c.City, null)) + .ToListAsync(); ``` -Otherwise, add runtime checks for null on the return value of Unhex(). +### Low-impact changes - + -### SqlFunctionExpression's nullability arguments' arity validated +#### `IncludeRootDiscriminatorInJsonId` was renamed to `HasRootDiscriminatorInJsonId` after 9.0.0-rc.2 -[Tracking Issue #33852](https://github.com/dotnet/efcore/issues/33852) +[Tracking Issue #34717](https://github.com/dotnet/efcore/pull/34717) -#### Old behavior +##### Old behavior -Previously it was possible to create a `SqlFunctionExpression` with a different number of arguments and nullability propagation arguments. +The `IncludeRootDiscriminatorInJsonId` Cosmos API was introduced in 9.0.0 rc.1. -#### New behavior +##### New behavior -Starting with EF Core 9.0, EF now throws if the number of arguments and nullability propagation arguments do not match. +For the final release of EF Core 9.0, the API was renamed to `HasRootDiscriminatorInJsonId` -#### Why +##### Why -Not having matching number of arguments and nullability propagation arguments can lead to unexpected behavior. +Another related API was renamed to start with `Has` instead of `Include`, and so this one was renamed for consistency as well. -#### Mitigations +##### Mitigations -Make sure the `argumentsPropagateNullability` has same number of elements as the `arguments`. When in doubt use `false` for nullability argument. +If your code is using the `IncludeRootDiscriminatorInJsonId` API, simply change it to reference `HasRootDiscriminatorInJsonId` instead. diff --git a/entity-framework/core/what-is-new/ef-core-9.0/plan.md b/entity-framework/core/what-is-new/ef-core-9.0/plan.md deleted file mode 100644 index 4da73b27cf..0000000000 --- a/entity-framework/core/what-is-new/ef-core-9.0/plan.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -title: Plan for Entity Framework Core 9 -description: The themes and features planned for EF Core 9 -author: ajcvickers -ms.date: 12/01/2023 -uid: core/what-is-new/ef-core-9.0/plan ---- - -# Plan for Entity Framework Core 9 - -Coming soon... diff --git a/entity-framework/core/what-is-new/ef-core-9.0/whatsnew.md b/entity-framework/core/what-is-new/ef-core-9.0/whatsnew.md index 80105540bd..a838d56cd8 100644 --- a/entity-framework/core/what-is-new/ef-core-9.0/whatsnew.md +++ b/entity-framework/core/what-is-new/ef-core-9.0/whatsnew.md @@ -24,6 +24,9 @@ EF9 targets .NET 8, and can therefore be used with either [.NET 8 (LTS)](https:/ ## Azure Cosmos DB for NoSQL +> [!WARNING] +> Extensive work has gone into making the Cosmos DB provider better in 9.0. The changes include a number of high-impact breaking changes; if you are upgrading an existing application, please read the [breaking changes section](xref:xref:core/what-is-new/ef-core-9.0/breaking-changes#cosmos-breaking-changes) carefully. + We are working on significant updates in EF9 to the EF Core database provider for Azure Cosmos DB for NoSQL. ### Hierarchical partition keys @@ -281,38 +284,64 @@ info: 6/11/2024 09:30:42.553 CosmosEventId.ExecutingSqlQuery[30100] (Microsoft.E > [!NOTE] > [Issue #33960](https://github.com/dotnet/efcore/issues/33960) is tracking a bug in this translation. -### Role-based access +### Improved modeling aligned to JSON and Cosmos standards -Azure Cosmos DB for NoSQL includes a [built-in role-based access control (RBAC) system](/azure/cosmos-db/role-based-access-control). This is now supported by EF9 for both management and use of containers. No changes are required to application code. See [Issue #32197](https://github.com/dotnet/efcore/issues/32197) for more information. +EF 9.0 maps to Cosmos DB documents in ways which are more natural for a JSON-based document database, and help interoperate with other systems accessing your documents. Although this entails breaking changes, APIs exist which allow reverting back to the pre-9.0 behavior in all cases. -### Synchronous access blocked by default +#### Simplified `id` properties without discriminators -> [!TIP] -> The code shown here comes from [CosmosSyncApisSample.cs](https://github.com/dotnet/EntityFramework.Docs/tree/main/samples/core/Miscellaneous/NewInEFCore9.Cosmos/CosmosSyncApisSample.cs). +First, previous versions of EF inserted the discriminator value into the JSON `id` property, producing documents such as the following: -Azure Cosmos DB for NoSQL does not support synchronous (blocking) access from application code. Previously, EF masked this by default by blocking for you on async calls. However, this both encourages sync use, which is bad practice, and [may cause deadlocks](https://blog.stephencleary.com/2012/07/dont-block-on-async-code.html). Therefore, starting with EF9, an exception is thrown when synchronous access is attempted. For example: +```json +{ + "id": "Blog|1099", + ... +} +``` -```output -System.InvalidOperationException: An error was generated for warning 'Microsoft.EntityFrameworkCore.Database.SyncNotSupported': - Azure Cosmos DB does not support synchronous I/O. Make sure to use and correctly await only async methods when using - Entity Framework Core to access Azure Cosmos DB. See https://aka.ms/ef-cosmos-nosync for more information. - This exception can be suppressed or logged by passing event ID 'CosmosEventId.SyncNotSupported' to the 'ConfigureWarnings' - method in 'DbContext.OnConfiguring' or 'AddDbContext'. - at Microsoft.EntityFrameworkCore.Diagnostics.EventDefinition.Log[TLoggerCategory](IDiagnosticsLogger`1 logger, Exception exception) - at Microsoft.EntityFrameworkCore.Cosmos.Diagnostics.Internal.CosmosLoggerExtensions.SyncNotSupported(IDiagnosticsLogger`1 diagnostics) - at Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal.CosmosClientWrapper.DeleteDatabase() - at Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal.CosmosDatabaseCreator.EnsureDeleted() - at Microsoft.EntityFrameworkCore.Infrastructure.DatabaseFacade.EnsureDeleted() +This was done in order to allow for documents of different types (e.g. Blog and Post) and the same key value (1099) to exist within the same container property. Starting with EF 9.0, the `id` property contains contains only the key value: + +```json +{ + "id": 1099, + ... +} ``` -As the exception says, sync access can still be used for now by configuring the warning level appropriately. For example, in `OnConfiguring` on your `DbContext` type: +Note this is a breaking change, since EF will no longer be able to query existing documents with the old `id` format. An API has been introduced to revert to the previous behavior, see the [breaking change note](xref:core/what-is-new/ef-core-9.0/breaking-changes#cosmos-id-property-changes) and the [the documentation](xref:core/providers/cosmos/index#Discriminators) for more details. + +#### Discriminator property renamed to `$type` + +The default discriminator property was previously named `Discriminator`. EF 9.0 changes the default to `$type`: + +```json +{ + "id": 1099, + "$type": "Blog", + ... +} +``` + +This follows the emerging standard for JSON polymorphism, allowing better interoperability with other tools. For example, .NET's System.Text.Json also supports polymorphism, using `$type` as its default discriminator property name ([docs](/dotnet/standard/serialization/system-text-json/polymorphism#customize-the-type-discriminator-name)). + +Note this is a breaking change, since EF will no longer be able to query existing documents with the old discriminator property name. See the [breaking change note](xref:core/what-is-new/ef-core-9.0/breaking-changes#cosmos-discriminator-name-change) for details on how to revert to the previous naming. + +### Role-based access + +Azure Cosmos DB for NoSQL includes a [built-in role-based access control (RBAC) system](/azure/cosmos-db/role-based-access-control). This is now supported by EF9 for both management and use of containers. No changes are required to application code. See [Issue #32197](https://github.com/dotnet/efcore/issues/32197) for more information. + +### Synchronous I/O blocked by default + +Azure Cosmos DB for NoSQL does not support synchronous (blocking) APIs from application code. Previously, EF masked this by default by blocking for you on async calls. However, this both encourages synchronous I/O use, which is bad practice, and [may cause deadlocks](https://blog.stephencleary.com/2012/07/dont-block-on-async-code.html). Therefore, starting with EF 9, an exception is thrown when synchronous access is attempted. For example: + +Synchronous I/O can still be used for now by configuring the warning level appropriately. For example, in `OnConfiguring` on your `DbContext` type: ```csharp protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) => optionsBuilder.ConfigureWarnings(b => b.Ignore(CosmosEventId.SyncNotSupported)); ``` -Note, however, that we plan to fully remove sync support in EF11, so start updating to use async methods like `ToListAsync` and `SaveChangesAsync` as soon as possible! +Note, however, that we plan to fully remove sync support in EF 11, so start updating to use async methods like `ToListAsync` and `SaveChangesAsync` as soon as possible! ### Enhanced primitive collections @@ -330,6 +359,79 @@ The Cosmos DB provider has supported primitive collections in a limited form sin See [What's new in EF8: primitive collections](xref:core/what-is-new/ef-core-8.0/whatsnew#primitive-collections) for more information on the model building API. +### Vector similarity search (preview) + +Azure Cosmos DB now offers preview support for vector similarity search. Vector search is a fundamental part of some application types, include AI, semantic search and others. The Cosmos DB support for vector search allows storing your data and vectors, and performing your queries in a single database, which can considerably simplify your architecture and remove the need for an additional, dedicated vector database solution in your stack. To learn more about Cosmos DB vector search, [see the documentation](/azure/cosmos-db/nosql/vector-search). + +Using vector search via EF is a simple matter of adding a vector property and configuring it: + +```c# +public class Blog +{ + ... + + public float[] Vector { get; set; } +} + +public class BloggingContext +{ + ... + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity() + .Property(b => b.Embeddings) + .IsVector(DistanceFunction.Cosine, dimensions: 1536); + } +} +``` + +Once that's done, use the `EF.Functions.VectorDistance()` function in LINQ queries to perform vector similarity search: + +```c# +var blogs = await context.Blogs + .OrderBy(s => EF.Functions.VectorDistance(s.Vector, vector)) + .Take(5) + .ToListAsync(); +``` + +For more information, see the [documentation on vector search](xref:core/providers/cosmos/vector-search). + +### Pagination support + +The Cosmos DB provider now allows for paginating through query results via _continuation tokens_, which is far more efficient and cost-effective than the traditional use of `Skip` and `Take`. + +```c# +var firstPage = await context.Posts + .OrderBy(p => p.Id) + .ToPageAsync(pageSize: 10, continuationToken: null); + +var continuationToken = page.ContinuationToken; +foreach (var post in page.Values) +{ + // Display/send the posts to the user +} +``` + +The `ToPageAsync` operator returns a `CosmosPage`, which exposes a continuation token that can be used to efficiently resume the query at a later point, fetching the next 10 items: + +```c# +var nextPage = await context.Sessions.OrderBy(s => s.Id).ToPageAsync(10, continuationToken); +``` + +For more information, [see the documentation section on pagination](xref:core/providers/cosmos/querying#pagination). + +### FromSql for safer SQL querying + +The Cosmos DB provider has allowed SQL querying via . However, that API can be susceptible to SQL injection attacks when user-provided data is interpolated or concatenated into the SQL. In EF 9.0, you can now use the new `FromSql` method, which always integrates parameterized data as a parameter outside the SQL: + +```c# +var maxAngle = 8; +_ = await context.Blogs + .FromSql($"SELECT VALUE c FROM root c WHERE c.Angle1 <= {maxAngle}") + .ToListAsync(); +``` + ## AOT and pre-compiled queries As mentioned in the introduction, there is a lot of work going on behind the scenes to allow EF Core to run without just-in-time (JIT) compilation. Instead, EF compile ahead-of-time (AOT) everything needed to run queries in the application. This AOT compilation and related processing will happen as part of building and publishing the application. At this point in the EF9 release, there is not much available that can be used by you, the app developer. However, for those interested, the completed issues in EF9 that support AOT and pre-compiled queries are: diff --git a/entity-framework/toc.yml b/entity-framework/toc.yml index 4ac3399f45..cfbd060ae9 100644 --- a/entity-framework/toc.yml +++ b/entity-framework/toc.yml @@ -34,8 +34,6 @@ href: core/what-is-new/ef-core-8.0/whatsnew.md - name: "Breaking changes in EF Core 8.0" href: core/what-is-new/ef-core-8.0/breaking-changes.md - - name: "The plan for EF Core 9.0" - href: core/what-is-new/ef-core-9.0/plan.md - name: Getting started items: - name: EF Core Overview @@ -66,8 +64,6 @@ href: core/what-is-new/release-planning.md - name: EF Core 9.0 items: - - name: High-level plan - href: core/what-is-new/ef-core-9.0/plan.md - name: "What's new?" href: core/what-is-new/ef-core-9.0/whatsnew.md - name: Breaking changes