From 2200e00c403cb37dd08183439f0404472cd81170 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Thu, 5 Sep 2024 12:54:16 -0700 Subject: [PATCH] .Net: Added implementation of Weaviate connector for new memory design (#8403) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation and Context Related: https://github.com/microsoft/semantic-kernel/issues/6527 In this PR: - Implemented `IVectorStore` - Implemented `IVectorStoreRecordCollection` - Weaviate default record mapper - `Options` classes - Extension methods for DI - Integration tests   - Unit tests ### Contribution Checklist - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone :smile: --------- Co-authored-by: Mark Wallace <127216156+markwallace-microsoft@users.noreply.github.com> --- dotnet/SK-dotnet.sln | 15 +- .../Connectors.Memory.Weaviate.csproj | 4 + .../WeaviateDateTimeOffsetConverter.cs | 33 ++ ...WeaviateNullableDateTimeOffsetConverter.cs | 45 ++ .../Http/HttpRequest.cs | 15 +- .../WeaviateCreateCollectionSchemaRequest.cs | 36 ++ .../WeaviateDeleteCollectionSchemaRequest.cs | 19 + .../WeaviateDeleteObjectBatchRequest.cs | 27 + .../HttpV2/WeaviateDeleteObjectRequest.cs | 23 + .../WeaviateGetCollectionObjectRequest.cs | 35 ++ .../WeaviateGetCollectionSchemaRequest.cs | 19 + .../WeaviateGetCollectionSchemaResponse.cs | 11 + .../HttpV2/WeaviateGetCollectionsRequest.cs | 15 + .../HttpV2/WeaviateGetCollectionsResponse.cs | 12 + ...viateUpsertCollectionObjectBatchRequest.cs | 32 ++ ...iateUpsertCollectionObjectBatchResponse.cs | 15 + ...viateVectorStoreRecordCollectionFactory.cs | 28 ++ .../ModelV2/WeaviateCollectionSchema.cs | 24 + .../WeaviateCollectionSchemaProperty.cs | 21 + .../WeaviateCollectionSchemaVectorConfig.cs | 20 + ...aviateCollectionSchemaVectorIndexConfig.cs | 11 + .../ModelV2/WeaviateOperationResult.cs | 20 + .../ModelV2/WeaviateOperationResultError.cs | 11 + .../ModelV2/WeaviateOperationResultErrors.cs | 12 + .../ModelV2/WeaviateQueryMatch.cs | 14 + .../ModelV2/WeaviateQueryMatchWhereClause.cs | 18 + .../WeaviateConstants.cs | 18 + .../WeaviateKernelBuilderExtensions.cs | 35 ++ .../WeaviateServiceCollectionExtensions.cs | 45 ++ .../WeaviateVectorStore.cs | 93 ++++ ...viateVectorStoreCollectionCreateMapping.cs | 196 ++++++++ .../WeaviateVectorStoreOptions.cs | 29 ++ .../WeaviateVectorStoreRecordCollection.cs | 405 +++++++++++++++ ...viateVectorStoreRecordCollectionOptions.cs | 41 ++ .../WeaviateVectorStoreRecordMapper.cs | 126 +++++ .../.editorconfig | 6 + .../Connectors.Weaviate.UnitTests.csproj | 37 ++ .../WeaviateHotel.cs | 49 ++ .../WeaviateKernelBuilderExtensionsTests.cs | 36 ++ .../WeaviateMemoryBuilderExtensionsTests.cs | 7 +- .../WeaviateMemoryStoreTests.cs | 7 +- ...eaviateServiceCollectionExtensionsTests.cs | 36 ++ ...VectorStoreCollectionCreateMappingTests.cs | 184 +++++++ ...eaviateVectorStoreRecordCollectionTests.cs | 470 ++++++++++++++++++ .../WeaviateVectorStoreRecordMapperTests.cs | 108 ++++ .../WeaviateVectorStoreTests.cs | 111 +++++ .../Memory/Weaviate/WeaviateHotel.cs | 49 ++ .../WeaviateVectorStoreCollectionFixture.cs | 9 + .../Weaviate/WeaviateVectorStoreFixture.cs | 125 +++++ ...eaviateVectorStoreRecordCollectionTests.cs | 245 +++++++++ .../Weaviate/WeaviateVectorStoreTests.cs | 35 ++ .../Data/RecordDefinition/DistanceFunction.cs | 13 + .../Data/RecordDefinition/IndexKind.cs | 5 + 53 files changed, 3048 insertions(+), 7 deletions(-) create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/Converters/WeaviateDateTimeOffsetConverter.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/Converters/WeaviateNullableDateTimeOffsetConverter.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateCreateCollectionSchemaRequest.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateDeleteCollectionSchemaRequest.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateDeleteObjectBatchRequest.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateDeleteObjectRequest.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionObjectRequest.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionSchemaRequest.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionSchemaResponse.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionsRequest.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionsResponse.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateUpsertCollectionObjectBatchRequest.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateUpsertCollectionObjectBatchResponse.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/IWeaviateVectorStoreRecordCollectionFactory.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchema.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaProperty.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaVectorConfig.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaVectorIndexConfig.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateOperationResult.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateOperationResultError.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateOperationResultErrors.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateQueryMatch.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateQueryMatchWhereClause.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateConstants.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateKernelBuilderExtensions.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateServiceCollectionExtensions.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionCreateMapping.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreOptions.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionOptions.cs create mode 100644 dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordMapper.cs create mode 100644 dotnet/src/Connectors/Connectors.Weaviate.UnitTests/.editorconfig create mode 100644 dotnet/src/Connectors/Connectors.Weaviate.UnitTests/Connectors.Weaviate.UnitTests.csproj create mode 100644 dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateHotel.cs create mode 100644 dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateKernelBuilderExtensionsTests.cs rename dotnet/src/Connectors/{Connectors.UnitTests/Memory/Weaviate => Connectors.Weaviate.UnitTests}/WeaviateMemoryBuilderExtensionsTests.cs (90%) rename dotnet/src/Connectors/{Connectors.UnitTests/Memory/Weaviate => Connectors.Weaviate.UnitTests}/WeaviateMemoryStoreTests.cs (92%) create mode 100644 dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateServiceCollectionExtensionsTests.cs create mode 100644 dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs create mode 100644 dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionTests.cs create mode 100644 dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs create mode 100644 dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreTests.cs create mode 100644 dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateHotel.cs create mode 100644 dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreCollectionFixture.cs create mode 100644 dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreFixture.cs create mode 100644 dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreRecordCollectionTests.cs create mode 100644 dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreTests.cs diff --git a/dotnet/SK-dotnet.sln b/dotnet/SK-dotnet.sln index 8cbd7d2855b1..3358ed644325 100644 --- a/dotnet/SK-dotnet.sln +++ b/dotnet/SK-dotnet.sln @@ -316,11 +316,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Qdrant.UnitTests EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StepwisePlannerMigration", "samples\Demos\StepwisePlannerMigration\StepwisePlannerMigration.csproj", "{38374C62-0263-4FE8-A18C-70FC8132912B}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.AzureCosmosDBMongoDB.UnitTests", "src\Connectors\Connectors.AzureCosmosDBMongoDB.UnitTests\Connectors.AzureCosmosDBMongoDB.UnitTests.csproj", "{2918478E-BC86-4D53-9D01-9C318F80C14F}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.AzureCosmosDBMongoDB.UnitTests", "src\Connectors\Connectors.AzureCosmosDBMongoDB.UnitTests\Connectors.AzureCosmosDBMongoDB.UnitTests.csproj", "{2918478E-BC86-4D53-9D01-9C318F80C14F}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AIModelRouter", "samples\Demos\AIModelRouter\AIModelRouter.csproj", "{E06818E3-00A5-41AC-97ED-9491070CDEA1}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AIModelRouter", "samples\Demos\AIModelRouter\AIModelRouter.csproj", "{E06818E3-00A5-41AC-97ED-9491070CDEA1}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.AzureCosmosDBNoSQL.UnitTests", "src\Connectors\Connectors.AzureCosmosDBNoSQL.UnitTests\Connectors.AzureCosmosDBNoSQL.UnitTests.csproj", "{385A8FE5-87E2-4458-AE09-35E10BD2E67F}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.AzureCosmosDBNoSQL.UnitTests", "src\Connectors\Connectors.AzureCosmosDBNoSQL.UnitTests\Connectors.AzureCosmosDBNoSQL.UnitTests.csproj", "{385A8FE5-87E2-4458-AE09-35E10BD2E67F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.Weaviate.UnitTests", "src\Connectors\Connectors.Weaviate.UnitTests\Connectors.Weaviate.UnitTests.csproj", "{AD9ECE32-088A-49D8-8ACB-890E79F1E7B8}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.OpenAI.UnitTests", "src\Connectors\Connectors.OpenAI.UnitTests\Connectors.OpenAI.UnitTests.csproj", "{36DDC119-C030-407E-AC51-A877E9E0F660}" EndProject @@ -809,6 +811,12 @@ Global {385A8FE5-87E2-4458-AE09-35E10BD2E67F}.Publish|Any CPU.Build.0 = Debug|Any CPU {385A8FE5-87E2-4458-AE09-35E10BD2E67F}.Release|Any CPU.ActiveCfg = Release|Any CPU {385A8FE5-87E2-4458-AE09-35E10BD2E67F}.Release|Any CPU.Build.0 = Release|Any CPU + {AD9ECE32-088A-49D8-8ACB-890E79F1E7B8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AD9ECE32-088A-49D8-8ACB-890E79F1E7B8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AD9ECE32-088A-49D8-8ACB-890E79F1E7B8}.Publish|Any CPU.ActiveCfg = Debug|Any CPU + {AD9ECE32-088A-49D8-8ACB-890E79F1E7B8}.Publish|Any CPU.Build.0 = Debug|Any CPU + {AD9ECE32-088A-49D8-8ACB-890E79F1E7B8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AD9ECE32-088A-49D8-8ACB-890E79F1E7B8}.Release|Any CPU.Build.0 = Release|Any CPU {36DDC119-C030-407E-AC51-A877E9E0F660}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {36DDC119-C030-407E-AC51-A877E9E0F660}.Debug|Any CPU.Build.0 = Debug|Any CPU {36DDC119-C030-407E-AC51-A877E9E0F660}.Publish|Any CPU.ActiveCfg = Debug|Any CPU @@ -937,6 +945,7 @@ Global {2918478E-BC86-4D53-9D01-9C318F80C14F} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} {E06818E3-00A5-41AC-97ED-9491070CDEA1} = {5D4C0700-BBB5-418F-A7B2-F392B9A18263} {385A8FE5-87E2-4458-AE09-35E10BD2E67F} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} + {AD9ECE32-088A-49D8-8ACB-890E79F1E7B8} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} {36DDC119-C030-407E-AC51-A877E9E0F660} = {1B4CBDE0-10C2-4E7D-9CD0-FE7586C96ED1} {7AAD7388-307D-41FB-B80A-EF9E3A4E31F0} = {1B4CBDE0-10C2-4E7D-9CD0-FE7586C96ED1} {8CF06B22-50F3-4F71-A002-622DB49DF0F5} = {1B4CBDE0-10C2-4E7D-9CD0-FE7586C96ED1} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Connectors.Memory.Weaviate.csproj b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Connectors.Memory.Weaviate.csproj index 7f75b9c28864..06901c892c22 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Connectors.Memory.Weaviate.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Connectors.Memory.Weaviate.csproj @@ -26,4 +26,8 @@ + + + + \ No newline at end of file diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Converters/WeaviateDateTimeOffsetConverter.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Converters/WeaviateDateTimeOffsetConverter.cs new file mode 100644 index 000000000000..754b6b10cbdd --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Converters/WeaviateDateTimeOffsetConverter.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +/// +/// Converts datetime type to RFC 3339 formatted string. +/// +internal sealed class WeaviateDateTimeOffsetConverter : JsonConverter +{ + private const string DateTimeFormat = "yyyy-MM-dd'T'HH:mm:ss.fffK"; + + public override DateTimeOffset Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var dateString = reader.GetString(); + + if (string.IsNullOrWhiteSpace(dateString)) + { + return default; + } + + return DateTimeOffset.Parse(dateString, CultureInfo.InvariantCulture); + } + + public override void Write(Utf8JsonWriter writer, DateTimeOffset value, JsonSerializerOptions options) + { + writer.WriteStringValue(value.ToString(DateTimeFormat, CultureInfo.InvariantCulture)); + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Converters/WeaviateNullableDateTimeOffsetConverter.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Converters/WeaviateNullableDateTimeOffsetConverter.cs new file mode 100644 index 000000000000..8dde4702aac0 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Converters/WeaviateNullableDateTimeOffsetConverter.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +/// +/// Converts datetime type to RFC 3339 formatted string. +/// +internal sealed class WeaviateNullableDateTimeOffsetConverter : JsonConverter +{ + private const string DateTimeFormat = "yyyy-MM-dd'T'HH:mm:ss.fffK"; + + public override DateTimeOffset? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) + { + return null; + } + + var dateString = reader.GetString(); + + if (string.IsNullOrWhiteSpace(dateString)) + { + return null; + } + + return DateTimeOffset.Parse(dateString, CultureInfo.InvariantCulture); + } + + public override void Write(Utf8JsonWriter writer, DateTimeOffset? value, JsonSerializerOptions options) + { + if (value.HasValue) + { + writer.WriteStringValue(value.Value.ToString(DateTimeFormat, CultureInfo.InvariantCulture)); + } + else + { + writer.WriteNullValue(); + } + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Http/HttpRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Http/HttpRequest.cs index 255dcf91363d..6b5bdaac1158 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Http/HttpRequest.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Http/HttpRequest.cs @@ -33,9 +33,20 @@ public static HttpRequestMessage CreatePostRequest(string url, object? payload = }; } - public static HttpRequestMessage CreateDeleteRequest(string url) + public static HttpRequestMessage CreateDeleteRequest(string url, object? payload = null) { - return new(HttpMethod.Delete, url); + return new(HttpMethod.Delete, url) + { + Content = GetJsonContent(payload) + }; + } + + public static HttpRequestMessage CreatePutRequest(string url, object? payload = null) + { + return new(HttpMethod.Put, url) + { + Content = GetJsonContent(payload) + }; } private static StringContent? GetJsonContent(object? payload) diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateCreateCollectionSchemaRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateCreateCollectionSchemaRequest.cs new file mode 100644 index 000000000000..96015b5323b7 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateCreateCollectionSchemaRequest.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Net.Http; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateCreateCollectionSchemaRequest +{ + private const string ApiRoute = "schema"; + + [JsonConstructor] + public WeaviateCreateCollectionSchemaRequest() { } + + public WeaviateCreateCollectionSchemaRequest(WeaviateCollectionSchema collectionSchema) + { + this.CollectionName = collectionSchema.CollectionName; + this.VectorConfigurations = collectionSchema.VectorConfigurations; + this.Properties = collectionSchema.Properties; + } + + [JsonPropertyName("class")] + public string? CollectionName { get; set; } + + [JsonPropertyName("vectorConfig")] + public Dictionary? VectorConfigurations { get; set; } + + [JsonPropertyName("properties")] + public List? Properties { get; set; } + + public HttpRequestMessage Build() + { + return HttpRequest.CreatePostRequest(ApiRoute, this); + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateDeleteCollectionSchemaRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateDeleteCollectionSchemaRequest.cs new file mode 100644 index 000000000000..4cdbaa9fd8da --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateDeleteCollectionSchemaRequest.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateDeleteCollectionSchemaRequest(string collectionName) +{ + private const string ApiRoute = "schema"; + + [JsonIgnore] + public string CollectionName { get; set; } = collectionName; + + public HttpRequestMessage Build() + { + return HttpRequest.CreateDeleteRequest($"{ApiRoute}/{this.CollectionName}"); + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateDeleteObjectBatchRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateDeleteObjectBatchRequest.cs new file mode 100644 index 000000000000..c7948e1ae530 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateDeleteObjectBatchRequest.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateDeleteObjectBatchRequest +{ + private const string ApiRoute = "batch/objects"; + + [JsonConstructor] + public WeaviateDeleteObjectBatchRequest() { } + + public WeaviateDeleteObjectBatchRequest(WeaviateQueryMatch match) + { + this.Match = match; + } + + [JsonPropertyName("match")] + public WeaviateQueryMatch? Match { get; set; } + + public HttpRequestMessage Build() + { + return HttpRequest.CreateDeleteRequest(ApiRoute, this); + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateDeleteObjectRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateDeleteObjectRequest.cs new file mode 100644 index 000000000000..e88b64b1b3fe --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateDeleteObjectRequest.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net.Http; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateDeleteObjectRequest(string collectionName, Guid id) +{ + private const string ApiRoute = "objects"; + + [JsonIgnore] + public string CollectionName { get; set; } = collectionName; + + [JsonIgnore] + public Guid Id { get; set; } = id; + + public HttpRequestMessage Build() + { + return HttpRequest.CreateDeleteRequest($"{ApiRoute}/{this.CollectionName}/{this.Id}"); + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionObjectRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionObjectRequest.cs new file mode 100644 index 000000000000..5ddb40f438a1 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionObjectRequest.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net.Http; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateGetCollectionObjectRequest(string collectionName, Guid id, bool includeVectors) +{ + private const string ApiRoute = "objects"; + private const string IncludeQueryParameterName = "include"; + private const string IncludeVectorQueryParameterValue = "vector"; + + [JsonIgnore] + public string CollectionName { get; set; } = collectionName; + + [JsonIgnore] + public Guid Id { get; set; } = id; + + [JsonIgnore] + public bool IncludeVectors { get; set; } = includeVectors; + + public HttpRequestMessage Build() + { + var uri = $"{ApiRoute}/{this.CollectionName}/{this.Id}"; + + if (this.IncludeVectors) + { + uri += $"?{IncludeQueryParameterName}={IncludeVectorQueryParameterValue}"; + } + + return HttpRequest.CreateGetRequest(uri); + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionSchemaRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionSchemaRequest.cs new file mode 100644 index 000000000000..d863f4dc74d6 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionSchemaRequest.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateGetCollectionSchemaRequest(string collectionName) +{ + private const string ApiRoute = "schema"; + + [JsonIgnore] + public string CollectionName { get; set; } = collectionName; + + public HttpRequestMessage Build() + { + return HttpRequest.CreateGetRequest($"{ApiRoute}/{this.CollectionName}"); + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionSchemaResponse.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionSchemaResponse.cs new file mode 100644 index 000000000000..7277ac7d929f --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionSchemaResponse.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateGetCollectionSchemaResponse +{ + [JsonPropertyName("class")] + public string? CollectionName { get; set; } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionsRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionsRequest.cs new file mode 100644 index 000000000000..f31017ca8685 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionsRequest.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateGetCollectionsRequest +{ + private const string ApiRoute = "schema"; + + public HttpRequestMessage Build() + { + return HttpRequest.CreateGetRequest(ApiRoute, this); + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionsResponse.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionsResponse.cs new file mode 100644 index 000000000000..84c1b5d6611c --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionsResponse.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateGetCollectionsResponse +{ + [JsonPropertyName("classes")] + public List? Collections { get; set; } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateUpsertCollectionObjectBatchRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateUpsertCollectionObjectBatchRequest.cs new file mode 100644 index 000000000000..f19918655312 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateUpsertCollectionObjectBatchRequest.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Net.Http; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateUpsertCollectionObjectBatchRequest +{ + private const string ApiRoute = "batch/objects"; + + [JsonConstructor] + public WeaviateUpsertCollectionObjectBatchRequest() { } + + public WeaviateUpsertCollectionObjectBatchRequest(List collectionObjects) + { + this.CollectionObjects = collectionObjects; + } + + [JsonPropertyName("fields")] + public List Fields { get; set; } = [WeaviateConstants.ReservedKeyPropertyName]; + + [JsonPropertyName("objects")] + public List? CollectionObjects { get; set; } + + public HttpRequestMessage Build() + { + return HttpRequest.CreatePostRequest(ApiRoute, this); + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateUpsertCollectionObjectBatchResponse.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateUpsertCollectionObjectBatchResponse.cs new file mode 100644 index 000000000000..1e540cdc5872 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateUpsertCollectionObjectBatchResponse.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateUpsertCollectionObjectBatchResponse +{ + [JsonPropertyName("id")] + public Guid Id { get; set; } + + [JsonPropertyName("result")] + public WeaviateOperationResult? Result { get; set; } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/IWeaviateVectorStoreRecordCollectionFactory.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/IWeaviateVectorStoreRecordCollectionFactory.cs new file mode 100644 index 000000000000..66ca41ca4a54 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/IWeaviateVectorStoreRecordCollectionFactory.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.SemanticKernel.Data; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +/// +/// Interface for constructing Weaviate instances when using to retrieve these. +/// +public interface IWeaviateVectorStoreRecordCollectionFactory +{ + /// + /// Constructs a new instance of the . + /// + /// The data type of the record key. + /// The data model to use for adding, updating and retrieving data from storage. + /// that is used to interact with Weaviate API. + /// The name of the collection to connect to. + /// An optional record definition that defines the schema of the record type. If not present, attributes on will be used. + /// The new instance of . + IVectorStoreRecordCollection CreateVectorStoreRecordCollection( + HttpClient httpClient, + string name, + VectorStoreRecordDefinition? vectorStoreRecordDefinition) + where TKey : notnull + where TRecord : class; +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchema.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchema.cs new file mode 100644 index 000000000000..e0f403ddb0e8 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchema.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateCollectionSchema +{ + [JsonConstructor] + public WeaviateCollectionSchema(string collectionName) + { + this.CollectionName = collectionName; + } + + [JsonPropertyName("class")] + public string CollectionName { get; set; } + + [JsonPropertyName("vectorConfig")] + public Dictionary VectorConfigurations { get; set; } = []; + + [JsonPropertyName("properties")] + public List Properties { get; set; } = []; +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaProperty.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaProperty.cs new file mode 100644 index 000000000000..d8719fe66764 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaProperty.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateCollectionSchemaProperty +{ + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("dataType")] + public List DataType { get; set; } = []; + + [JsonPropertyName("indexFilterable")] + public bool IndexFilterable { get; set; } + + [JsonPropertyName("indexSearchable")] + public bool IndexSearchable { get; set; } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaVectorConfig.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaVectorConfig.cs new file mode 100644 index 000000000000..75bd33471eb7 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaVectorConfig.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateCollectionSchemaVectorConfig +{ + private const string DefaultVectorizer = "none"; + + [JsonPropertyName("vectorizer")] + public Dictionary Vectorizer { get; set; } = new() { [DefaultVectorizer] = null }; + + [JsonPropertyName("vectorIndexType")] + public string? VectorIndexType { get; set; } + + [JsonPropertyName("vectorIndexConfig")] + public WeaviateCollectionSchemaVectorIndexConfig? VectorIndexConfig { get; set; } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaVectorIndexConfig.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaVectorIndexConfig.cs new file mode 100644 index 000000000000..49d01896d395 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaVectorIndexConfig.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateCollectionSchemaVectorIndexConfig +{ + [JsonPropertyName("distance")] + public string? Distance { get; set; } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateOperationResult.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateOperationResult.cs new file mode 100644 index 000000000000..fc76ac8c2435 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateOperationResult.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateOperationResult +{ + private const string Success = nameof(Success); + + [JsonPropertyName("errors")] + public WeaviateOperationResultErrors? Errors { get; set; } + + [JsonPropertyName("status")] + public string? Status { get; set; } + + [JsonIgnore] + public bool? IsSuccess => this.Status?.Equals(Success, StringComparison.OrdinalIgnoreCase); +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateOperationResultError.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateOperationResultError.cs new file mode 100644 index 000000000000..51470a0af40a --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateOperationResultError.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateOperationResultError +{ + [JsonPropertyName("message")] + public string? Message { get; set; } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateOperationResultErrors.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateOperationResultErrors.cs new file mode 100644 index 000000000000..c76555f2914b --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateOperationResultErrors.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal class WeaviateOperationResultErrors +{ + [JsonPropertyName("error")] + public List? Errors { get; set; } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateQueryMatch.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateQueryMatch.cs new file mode 100644 index 000000000000..4bb6b431807d --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateQueryMatch.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateQueryMatch +{ + [JsonPropertyName("class")] + public string? CollectionName { get; set; } + + [JsonPropertyName("where")] + public WeaviateQueryMatchWhereClause? WhereClause { get; set; } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateQueryMatchWhereClause.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateQueryMatchWhereClause.cs new file mode 100644 index 000000000000..b2423ab868fc --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateQueryMatchWhereClause.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateQueryMatchWhereClause +{ + [JsonPropertyName("operator")] + public string? Operator { get; set; } + + [JsonPropertyName("path")] + public List Path { get; set; } = []; + + [JsonPropertyName("valueTextArray")] + public List Values { get; set; } = []; +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateConstants.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateConstants.cs new file mode 100644 index 000000000000..57fa4695b0bf --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateConstants.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateConstants +{ + /// Reserved key property name in Weaviate. + internal const string ReservedKeyPropertyName = "id"; + + /// Reserved data property name in Weaviate. + internal const string ReservedDataPropertyName = "properties"; + + /// Reserved vector property name in Weaviate. + internal const string ReservedVectorPropertyName = "vectors"; + + /// Reserved collection property name in Weaviate. + internal const string ReservedCollectionPropertyName = "class"; +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateKernelBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateKernelBuilderExtensions.cs new file mode 100644 index 000000000000..57d10183d7de --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateKernelBuilderExtensions.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.SemanticKernel.Connectors.Weaviate; +using Microsoft.SemanticKernel.Data; + +namespace Microsoft.SemanticKernel; + +/// +/// Extension methods to register Weaviate instances on the . +/// +public static class WeaviateKernelBuilderExtensions +{ + /// + /// Register a Weaviate with the specified service ID. + /// + /// The builder to register the on. + /// + /// that is used to interact with Weaviate API. + /// should point to remote or local cluster and API key can be configured via . + /// It's also possible to provide these parameters via . + /// + /// Optional options to further configure the . + /// An optional service id to use as the service key. + /// The kernel builder. + public static IKernelBuilder AddWeaviateVectorStore( + this IKernelBuilder builder, + HttpClient? httpClient = default, + WeaviateVectorStoreOptions? options = default, + string? serviceId = default) + { + builder.Services.AddWeaviateVectorStore(httpClient, options, serviceId); + return builder; + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateServiceCollectionExtensions.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateServiceCollectionExtensions.cs new file mode 100644 index 000000000000..cdaff1ddd070 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateServiceCollectionExtensions.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel.Connectors.Weaviate; +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.Http; + +namespace Microsoft.SemanticKernel; + +/// +/// Extension methods to register Weaviate instances on an +/// +public static class WeaviateServiceCollectionExtensions +{ + /// + /// Register a Weaviate with the specified service ID. + /// + /// The to register the on. + /// + /// that is used to interact with Weaviate API. + /// should point to remote or local cluster and API key can be configured via . + /// It's also possible to provide these parameters via . + /// + /// Optional options to further configure the . + /// An optional service id to use as the service key. + /// Service collection. + public static IServiceCollection AddWeaviateVectorStore( + this IServiceCollection services, + HttpClient? httpClient = default, + WeaviateVectorStoreOptions? options = default, + string? serviceId = default) + { + services.AddKeyedTransient( + serviceId, + (sp, obj) => + { + var selectedHttpClient = HttpClientProvider.GetHttpClient(httpClient, sp); + var selectedOptions = options ?? sp.GetService(); + return new WeaviateVectorStore(selectedHttpClient, options); + }); + + return services; + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs new file mode 100644 index 000000000000..ddd605236c27 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Threading; +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.Http; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +/// +/// Class for accessing the list of collections in a Weaviate vector store. +/// +/// +/// This class can be used with collections of any schema type, but requires you to provide schema information when getting a collection. +/// +public sealed class WeaviateVectorStore : IVectorStore +{ + /// that is used to interact with Weaviate API. + private readonly HttpClient _httpClient; + + /// Optional configuration options for this class. + private readonly WeaviateVectorStoreOptions _options; + + /// + /// Initializes a new instance of the class. + /// + /// + /// that is used to interact with Weaviate API. + /// should point to remote or local cluster and API key can be configured via . + /// It's also possible to provide these parameters via . + /// + /// Optional configuration options for this class. + public WeaviateVectorStore(HttpClient httpClient, WeaviateVectorStoreOptions? options = null) + { + Verify.NotNull(httpClient); + + this._httpClient = httpClient; + this._options = options ?? new(); + } + + /// + public IVectorStoreRecordCollection GetCollection(string name, VectorStoreRecordDefinition? vectorStoreRecordDefinition = null) + where TKey : notnull + where TRecord : class + { + if (typeof(TKey) != typeof(Guid)) + { + throw new NotSupportedException($"Only {nameof(Guid)} key is supported."); + } + + if (this._options.VectorStoreCollectionFactory is not null) + { + return this._options.VectorStoreCollectionFactory.CreateVectorStoreRecordCollection( + this._httpClient, + name, + vectorStoreRecordDefinition); + } + + var recordCollection = new WeaviateVectorStoreRecordCollection( + this._httpClient, + name, + new() + { + VectorStoreRecordDefinition = vectorStoreRecordDefinition, + Endpoint = this._options.Endpoint, + ApiKey = this._options.ApiKey + }) as IVectorStoreRecordCollection; + + return recordCollection!; + } + + /// + public async IAsyncEnumerable ListCollectionNamesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + using var request = new WeaviateGetCollectionsRequest().Build(); + + var response = await this._httpClient.SendWithSuccessCheckAsync(request, cancellationToken).ConfigureAwait(false); + var responseContent = await response.Content.ReadAsStringWithExceptionMappingAsync().ConfigureAwait(false); + var collectionResponse = JsonSerializer.Deserialize(responseContent); + + if (collectionResponse?.Collections is not null) + { + foreach (var collection in collectionResponse.Collections) + { + yield return collection.CollectionName; + } + } + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionCreateMapping.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionCreateMapping.cs new file mode 100644 index 000000000000..58331c66cd1c --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionCreateMapping.cs @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using Microsoft.SemanticKernel.Data; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +/// +/// Class to construct Weaviate collection schema with configuration for data and vector properties. +/// More information here: . +/// +internal static class WeaviateVectorStoreCollectionCreateMapping +{ + /// + /// Maps record type properties to Weaviate collection schema for collection creation. + /// + /// The name of the vector store collection. + /// Collection of record data properties. + /// Collection of record vector properties. + /// A dictionary that maps from a property name to the storage name that should be used when serializing it to JSON for data and vector properties. + /// Weaviate collection schema. + public static WeaviateCollectionSchema MapToSchema( + string collectionName, + IEnumerable dataProperties, + IEnumerable vectorProperties, + Dictionary storagePropertyNames) + { + var schema = new WeaviateCollectionSchema(collectionName); + + // Handle data properties. + foreach (var property in dataProperties) + { + schema.Properties.Add(new WeaviateCollectionSchemaProperty + { + Name = storagePropertyNames[property.DataModelPropertyName], + DataType = [MapType(property.PropertyType)], + IndexFilterable = property.IsFilterable, + IndexSearchable = property.IsFullTextSearchable + }); + } + + // Handle vector properties. + foreach (var property in vectorProperties) + { + var vectorPropertyName = storagePropertyNames[property.DataModelPropertyName]; + schema.VectorConfigurations.Add(vectorPropertyName, new WeaviateCollectionSchemaVectorConfig + { + VectorIndexType = MapIndexKind(property.IndexKind, vectorPropertyName), + VectorIndexConfig = new WeaviateCollectionSchemaVectorIndexConfig + { + Distance = MapDistanceFunction(property.DistanceFunction, vectorPropertyName) + } + }); + } + + return schema; + } + + #region private + + /// + /// Maps record vector property index kind to Weaviate index kind. + /// More information here: . + /// + private static string MapIndexKind(string? indexKind, string vectorPropertyName) + { + const string Hnsw = "hnsw"; + const string Flat = "flat"; + const string Dynamic = "dynamic"; + + // If index kind is not provided, use default one. + if (string.IsNullOrWhiteSpace(indexKind)) + { + return Hnsw; + } + + return indexKind switch + { + IndexKind.Hnsw => Hnsw, + IndexKind.Flat => Flat, + IndexKind.Dynamic => Dynamic, + _ => throw new InvalidOperationException( + $"Index kind '{indexKind}' on {nameof(VectorStoreRecordVectorProperty)} '{vectorPropertyName}' is not supported by the Weaviate VectorStore. " + + $"Supported index kinds: {string.Join(", ", + IndexKind.Hnsw, + IndexKind.Flat, + IndexKind.Dynamic)}") + }; + } + + /// + /// Maps record vector property distance function to Weaviate distance function. + /// More information here: . + /// + private static string MapDistanceFunction(string? distanceFunction, string vectorPropertyName) + { + const string Cosine = "cosine"; + const string Dot = "dot"; + const string EuclideanSquared = "l2-squared"; + const string Hamming = "hamming"; + const string Manhattan = "manhattan"; + + // If distance function is not provided, use default one. + if (string.IsNullOrWhiteSpace(distanceFunction)) + { + return Cosine; + } + + return distanceFunction switch + { + DistanceFunction.CosineDistance => Cosine, + DistanceFunction.DotProductSimilarity => Dot, + DistanceFunction.EuclideanSquaredDistance => EuclideanSquared, + DistanceFunction.Hamming => Hamming, + DistanceFunction.ManhattanDistance => Manhattan, + _ => throw new InvalidOperationException( + $"Distance function '{distanceFunction}' on {nameof(VectorStoreRecordVectorProperty)} '{vectorPropertyName}' is not supported by the Weaviate VectorStore. " + + $"Supported distance functions: {string.Join(", ", + DistanceFunction.CosineDistance, + DistanceFunction.DotProductSimilarity, + DistanceFunction.EuclideanSquaredDistance, + DistanceFunction.Hamming, + DistanceFunction.ManhattanDistance)}") + }; + } + + /// + /// Maps record property type to Weaviate data type taking into account if the type is a collection or single value. + /// + private static string MapType(Type type) + { + // Check if the type is a collection. + if (typeof(IEnumerable).IsAssignableFrom(type) && type != typeof(string)) + { + var elementType = GetCollectionElementType(type); + + // If type is a collection, handle collection element type. + return MapType(elementType, isCollection: true); + } + + // If type is not a collection, handle single type. + return MapType(type, isCollection: false); + } + + /// + /// Maps record property type to Weaviate data type. + /// More information here: . + /// + private static string MapType(Type type, bool isCollection) + { + return type switch + { + Type t when t == typeof(string) => isCollection ? "text[]" : "text", + Type t when t == typeof(int) || t == typeof(long) || t == typeof(short) || t == typeof(byte) || + t == typeof(int?) || t == typeof(long?) || t == typeof(short?) || t == typeof(byte?) => isCollection ? "int[]" : "int", + Type t when t == typeof(float) || t == typeof(double) || t == typeof(decimal) || + t == typeof(float?) || t == typeof(double?) || t == typeof(decimal?) => isCollection ? "number[]" : "number", + Type t when t == typeof(DateTime) || t == typeof(DateTime?) || + t == typeof(DateTimeOffset) || t == typeof(DateTimeOffset?) => isCollection ? "date[]" : "date", + Type t when t == typeof(Guid) || t == typeof(Guid?) => isCollection ? "uuid[]" : "uuid", + Type t when t == typeof(bool) || t == typeof(bool?) => isCollection ? "boolean[]" : "boolean", + _ => isCollection ? "object[]" : "object", + }; + } + + /// + /// Gets the element type of a collection. + /// + /// + /// For example, when is , returned type will be generic parameter . + /// + private static Type GetCollectionElementType(Type type) + { + if (type.IsArray) + { + var elementType = type.GetElementType(); + + if (elementType is not null) + { + return elementType; + } + } + + if (type.IsGenericType) + { + return type.GetGenericArguments().First(); + } + + return typeof(object); + } + + #endregion +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreOptions.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreOptions.cs new file mode 100644 index 000000000000..9feab8c9047d --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreOptions.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +/// +/// Options when creating a . +/// +public sealed class WeaviateVectorStoreOptions +{ + /// + /// An optional factory to use for constructing instances, if a custom record collection is required. + /// + public IWeaviateVectorStoreRecordCollectionFactory? VectorStoreCollectionFactory { get; init; } + + /// + /// Weaviate endpoint for remote or local cluster. + /// + public Uri? Endpoint { get; set; } = null; + + /// + /// Weaviate API key. + /// + /// + /// This parameter is optional because authentication may be disabled in local clusters for testing purposes. + /// + public string? ApiKey { get; set; } = null; +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs new file mode 100644 index 000000000000..8e471c3c5d8c --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs @@ -0,0 +1,405 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.Http; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +/// +/// Service for storing and retrieving vector records, that uses Weaviate as the underlying storage. +/// +/// The data model to use for adding, updating and retrieving data from storage. +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public sealed class WeaviateVectorStoreRecordCollection : IVectorStoreRecordCollection where TRecord : class +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// The name of this database for telemetry purposes. + private const string DatabaseName = "Weaviate"; + + /// A set of types that a key on the provided model may have. + private static readonly HashSet s_supportedKeyTypes = + [ + typeof(Guid) + ]; + + /// A set of types that vectors on the provided model may have. + private static readonly HashSet s_supportedVectorTypes = + [ + typeof(ReadOnlyMemory), + typeof(ReadOnlyMemory?), + typeof(ReadOnlyMemory), + typeof(ReadOnlyMemory?) + ]; + + /// A set of types that data properties on the provided model may have. + private static readonly HashSet s_supportedDataTypes = + [ + typeof(string), + typeof(int), + typeof(int?), + typeof(long), + typeof(long?), + typeof(short), + typeof(short?), + typeof(byte), + typeof(byte?), + typeof(float), + typeof(float?), + typeof(double), + typeof(double?), + typeof(decimal), + typeof(decimal?), + typeof(DateTime), + typeof(DateTime?), + typeof(DateTimeOffset), + typeof(DateTimeOffset?), + typeof(Guid), + typeof(Guid?), + typeof(bool), + typeof(bool?) + ]; + + /// Default JSON serializer options. + private static readonly JsonSerializerOptions s_jsonSerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + Converters = + { + new WeaviateDateTimeOffsetConverter(), + new WeaviateNullableDateTimeOffsetConverter() + } + }; + + /// that is used to interact with Weaviate API. + private readonly HttpClient _httpClient; + + /// Optional configuration options for this class. + private readonly WeaviateVectorStoreRecordCollectionOptions _options; + + /// A definition of the current storage model. + private readonly VectorStoreRecordDefinition _vectorStoreRecordDefinition; + + /// A dictionary that maps from a property name to the storage name that should be used when serializing it to JSON for data and vector properties. + private readonly Dictionary _storagePropertyNames; + + /// The key property of the current storage model. + private readonly VectorStoreRecordKeyProperty _keyProperty; + + /// The data properties of the current storage model. + private readonly List _dataProperties; + + /// The vector properties of the current storage model. + private readonly List _vectorProperties; + + /// The mapper to use when mapping between the consumer data model and the Weaviate record. + private readonly IVectorStoreRecordMapper _mapper; + + /// Weaviate endpoint. + private readonly Uri _endpoint; + + /// Weaviate API key. + private readonly string? _apiKey; + + /// + public string CollectionName { get; } + + /// + /// Initializes a new instance of the class. + /// + /// + /// that is used to interact with Weaviate API. + /// should point to remote or local cluster and API key can be configured via . + /// It's also possible to provide these parameters via . + /// + /// The name of the collection that this will access. + /// Optional configuration options for this class. + public WeaviateVectorStoreRecordCollection( + HttpClient httpClient, + string collectionName, + WeaviateVectorStoreRecordCollectionOptions? options = default) + { + // Verify. + Verify.NotNull(httpClient); + Verify.NotNullOrWhiteSpace(collectionName); + + var endpoint = (options?.Endpoint ?? httpClient.BaseAddress) ?? throw new ArgumentException($"Weaviate endpoint should be provided via HttpClient.BaseAddress property or {nameof(WeaviateVectorStoreRecordCollectionOptions)} options parameter."); + + // Assign. + this._httpClient = httpClient; + this._endpoint = endpoint; + this.CollectionName = collectionName; + this._options = options ?? new(); + this._vectorStoreRecordDefinition = this._options.VectorStoreRecordDefinition ?? VectorStoreRecordPropertyReader.CreateVectorStoreRecordDefinitionFromType(typeof(TRecord), supportsMultipleVectors: true); + this._apiKey = this._options.ApiKey; + + // Validate property types. + var properties = VectorStoreRecordPropertyReader.SplitDefinitionAndVerify(typeof(TRecord).Name, this._vectorStoreRecordDefinition, supportsMultipleVectors: true, requiresAtLeastOneVector: false); + VectorStoreRecordPropertyReader.VerifyPropertyTypes([properties.KeyProperty], s_supportedKeyTypes, "Key"); + VectorStoreRecordPropertyReader.VerifyPropertyTypes(properties.DataProperties, s_supportedDataTypes, "Data", supportEnumerable: true); + VectorStoreRecordPropertyReader.VerifyPropertyTypes(properties.VectorProperties, s_supportedVectorTypes, "Vector"); + + // Assign properties and names for later usage. + this._storagePropertyNames = VectorStoreRecordPropertyReader.BuildPropertyNameToJsonPropertyNameMap(properties, typeof(TRecord), s_jsonSerializerOptions); + this._keyProperty = properties.KeyProperty; + this._dataProperties = properties.DataProperties; + this._vectorProperties = properties.VectorProperties; + + // Assign mapper. + this._mapper = this._options.JsonNodeCustomMapper ?? + new WeaviateVectorStoreRecordMapper( + this.CollectionName, + this._keyProperty, + this._dataProperties, + this._vectorProperties, + this._storagePropertyNames, + s_jsonSerializerOptions); + } + + /// + public Task CollectionExistsAsync(CancellationToken cancellationToken = default) + { + const string OperationName = "GetCollectionSchema"; + + return this.RunOperationAsync(OperationName, async () => + { + var request = new WeaviateGetCollectionSchemaRequest(this.CollectionName).Build(); + + var response = await this + .ExecuteRequestWithNotFoundHandlingAsync(request, cancellationToken) + .ConfigureAwait(false); + + return response != null; + }); + } + + /// + public Task CreateCollectionAsync(CancellationToken cancellationToken = default) + { + const string OperationName = "CreateCollectionSchema"; + + return this.RunOperationAsync(OperationName, () => + { + var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema( + this.CollectionName, + this._dataProperties, + this._vectorProperties, + this._storagePropertyNames); + + var request = new WeaviateCreateCollectionSchemaRequest(schema).Build(); + + return this.ExecuteRequestAsync(request, cancellationToken); + }); + } + + /// + public async Task CreateCollectionIfNotExistsAsync(CancellationToken cancellationToken = default) + { + if (!await this.CollectionExistsAsync(cancellationToken).ConfigureAwait(false)) + { + await this.CreateCollectionAsync(cancellationToken).ConfigureAwait(false); + } + } + + /// + public Task DeleteCollectionAsync(CancellationToken cancellationToken = default) + { + const string OperationName = "DeleteCollectionSchema"; + + return this.RunOperationAsync(OperationName, () => + { + var request = new WeaviateDeleteCollectionSchemaRequest(this.CollectionName).Build(); + + return this.ExecuteRequestAsync(request, cancellationToken); + }); + } + + /// + public Task DeleteAsync(Guid key, DeleteRecordOptions? options = null, CancellationToken cancellationToken = default) + { + const string OperationName = "DeleteObject"; + + return this.RunOperationAsync(OperationName, () => + { + var request = new WeaviateDeleteObjectRequest(this.CollectionName, key).Build(); + + return this.ExecuteRequestAsync(request, cancellationToken); + }); + } + + /// + public Task DeleteBatchAsync(IEnumerable keys, DeleteRecordOptions? options = null, CancellationToken cancellationToken = default) + { + const string OperationName = "DeleteObjectBatch"; + const string ContainsAnyOperator = "ContainsAny"; + + return this.RunOperationAsync(OperationName, () => + { + var match = new WeaviateQueryMatch + { + CollectionName = this.CollectionName, + WhereClause = new WeaviateQueryMatchWhereClause + { + Operator = ContainsAnyOperator, + Path = [WeaviateConstants.ReservedKeyPropertyName], + Values = keys.Select(key => key.ToString()).ToList() + } + }; + + var request = new WeaviateDeleteObjectBatchRequest(match).Build(); + + return this.ExecuteRequestAsync(request, cancellationToken); + }); + } + + /// + public Task GetAsync(Guid key, GetRecordOptions? options = null, CancellationToken cancellationToken = default) + { + const string OperationName = "GetCollectionObject"; + + return this.RunOperationAsync(OperationName, async () => + { + var includeVectors = options?.IncludeVectors is true; + var request = new WeaviateGetCollectionObjectRequest(this.CollectionName, key, includeVectors).Build(); + + var jsonNode = await this.ExecuteRequestWithNotFoundHandlingAsync(request, cancellationToken).ConfigureAwait(false); + + if (jsonNode is null) + { + return null; + } + + return VectorStoreErrorHandler.RunModelConversion( + DatabaseName, + this.CollectionName, + OperationName, + () => this._mapper.MapFromStorageToDataModel(jsonNode!, new() { IncludeVectors = includeVectors })); + }); + } + + /// + public async IAsyncEnumerable GetBatchAsync( + IEnumerable keys, + GetRecordOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var tasks = keys.Select(key => this.GetAsync(key, options, cancellationToken)); + + var records = await Task.WhenAll(tasks).ConfigureAwait(false); + + foreach (var record in records) + { + if (record is not null) + { + yield return record; + } + } + } + + /// + public async Task UpsertAsync(TRecord record, UpsertRecordOptions? options = null, CancellationToken cancellationToken = default) + { + return await this.UpsertBatchAsync([record], options, cancellationToken) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + } + + /// + public async IAsyncEnumerable UpsertBatchAsync( + IEnumerable records, + UpsertRecordOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + const string OperationName = "UpsertCollectionObject"; + + var responses = await this.RunOperationAsync(OperationName, async () => + { + var jsonNodes = records.Select(record => VectorStoreErrorHandler.RunModelConversion( + DatabaseName, + this.CollectionName, + OperationName, + () => this._mapper.MapFromDataToStorageModel(record))).ToList(); + + var request = new WeaviateUpsertCollectionObjectBatchRequest(jsonNodes).Build(); + + return await this.ExecuteRequestAsync>(request, cancellationToken).ConfigureAwait(false); + }).ConfigureAwait(false); + + if (responses is not null) + { + foreach (var response in responses) + { + if (response?.Result?.IsSuccess is true) + { + yield return response.Id; + } + } + } + } + + #region private + + private Task ExecuteRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + request.RequestUri = new Uri(this._endpoint, request.RequestUri!); + + if (!string.IsNullOrWhiteSpace(this._apiKey)) + { + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", this._apiKey); + } + + return this._httpClient.SendWithSuccessCheckAsync(request, cancellationToken); + } + + private async Task ExecuteRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var response = await this.ExecuteRequestAsync(request, cancellationToken).ConfigureAwait(false); + + var responseContent = await response.Content.ReadAsStringWithExceptionMappingAsync().ConfigureAwait(false); + + return JsonSerializer.Deserialize(responseContent, s_jsonSerializerOptions); + } + + private async Task ExecuteRequestWithNotFoundHandlingAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + try + { + return await this.ExecuteRequestAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (HttpOperationException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + return default; + } + } + + private async Task RunOperationAsync(string operationName, Func> operation) + { + try + { + return await operation.Invoke().ConfigureAwait(false); + } + catch (Exception ex) + { + throw new VectorStoreOperationException("Call to vector store failed.", ex) + { + VectorStoreType = DatabaseName, + CollectionName = this.CollectionName, + OperationName = operationName + }; + } + } + + #endregion +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionOptions.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionOptions.cs new file mode 100644 index 000000000000..b3223f2d3192 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionOptions.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json.Nodes; +using Microsoft.SemanticKernel.Data; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +/// +/// Options when creating a . +/// +public sealed class WeaviateVectorStoreRecordCollectionOptions where TRecord : class +{ + /// + /// Gets or sets an optional custom mapper to use when converting between the data model and Weaviate record. + /// + public IVectorStoreRecordMapper? JsonNodeCustomMapper { get; init; } = null; + + /// + /// Gets or sets an optional record definition that defines the schema of the record type. + /// + /// + /// If not provided, the schema will be inferred from the record model class using reflection. + /// In this case, the record model properties must be annotated with the appropriate attributes to indicate their usage. + /// See , and . + /// + public VectorStoreRecordDefinition? VectorStoreRecordDefinition { get; init; } = null; + + /// + /// Weaviate endpoint for remote or local cluster. + /// + public Uri? Endpoint { get; set; } = null; + + /// + /// Weaviate API key. + /// + /// + /// This parameter is optional because authentication may be disabled in local clusters for testing purposes. + /// + public string? ApiKey { get; set; } = null; +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordMapper.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordMapper.cs new file mode 100644 index 000000000000..fe1527cdfd8d --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordMapper.cs @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.SemanticKernel.Data; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +internal sealed class WeaviateVectorStoreRecordMapper : IVectorStoreRecordMapper where TRecord : class +{ + private readonly string _collectionName; + + private readonly string _keyProperty; + + private readonly List _dataProperties; + + private readonly List _vectorProperties; + + private readonly Dictionary _storagePropertyNames; + + private readonly JsonSerializerOptions _jsonSerializerOptions; + + public WeaviateVectorStoreRecordMapper( + string collectionName, + VectorStoreRecordKeyProperty keyProperty, + List dataProperties, + List vectorProperties, + Dictionary storagePropertyNames, + JsonSerializerOptions jsonSerializerOptions) + { + Verify.NotNullOrWhiteSpace(collectionName); + Verify.NotNull(keyProperty); + Verify.NotNull(dataProperties); + Verify.NotNull(vectorProperties); + Verify.NotNull(storagePropertyNames); + Verify.NotNull(jsonSerializerOptions); + + this._collectionName = collectionName; + this._storagePropertyNames = storagePropertyNames; + this._jsonSerializerOptions = jsonSerializerOptions; + + this._keyProperty = this._storagePropertyNames[keyProperty.DataModelPropertyName]; + this._dataProperties = dataProperties.Select(property => this._storagePropertyNames[property.DataModelPropertyName]).ToList(); + this._vectorProperties = vectorProperties.Select(property => this._storagePropertyNames[property.DataModelPropertyName]).ToList(); + } + + public JsonNode MapFromDataToStorageModel(TRecord dataModel) + { + Verify.NotNull(dataModel); + + var jsonNodeDataModel = JsonSerializer.SerializeToNode(dataModel, this._jsonSerializerOptions)!; + + // Transform data model to Weaviate object model. + var weaviateObjectModel = new JsonObject + { + { WeaviateConstants.ReservedCollectionPropertyName, JsonValue.Create(this._collectionName) }, + { WeaviateConstants.ReservedKeyPropertyName, jsonNodeDataModel[this._keyProperty]!.DeepClone() }, + { WeaviateConstants.ReservedDataPropertyName, new JsonObject() }, + { WeaviateConstants.ReservedVectorPropertyName, new JsonObject() }, + }; + + // Populate data properties. + foreach (var property in this._dataProperties) + { + var node = jsonNodeDataModel[property]; + + if (node is not null) + { + weaviateObjectModel[WeaviateConstants.ReservedDataPropertyName]![property] = node.DeepClone(); + } + } + + // Populate vector properties. + foreach (var property in this._vectorProperties) + { + var node = jsonNodeDataModel[property]; + + if (node is not null) + { + weaviateObjectModel[WeaviateConstants.ReservedVectorPropertyName]![property] = node.DeepClone(); + } + } + + return weaviateObjectModel; + } + + public TRecord MapFromStorageToDataModel(JsonNode storageModel, StorageToDataModelMapperOptions options) + { + Verify.NotNull(storageModel); + + // Transform Weaviate object model to data model. + var jsonNodeDataModel = new JsonObject + { + { this._keyProperty, storageModel[WeaviateConstants.ReservedKeyPropertyName]?.DeepClone() }, + }; + + // Populate data properties. + foreach (var property in this._dataProperties) + { + var node = storageModel[WeaviateConstants.ReservedDataPropertyName]?[property]; + + if (node is not null) + { + jsonNodeDataModel[property] = node.DeepClone(); + } + } + + // Populate vector properties. + if (options.IncludeVectors) + { + foreach (var property in this._vectorProperties) + { + var node = storageModel[WeaviateConstants.ReservedVectorPropertyName]?[property]; + + if (node is not null) + { + jsonNodeDataModel[property] = node.DeepClone(); + } + } + } + + return jsonNodeDataModel.Deserialize(this._jsonSerializerOptions)!; + } +} diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/.editorconfig b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/.editorconfig new file mode 100644 index 000000000000..394eef685f21 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/.editorconfig @@ -0,0 +1,6 @@ +# Suppressing errors for Test projects under dotnet folder +[*.cs] +dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task +dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave +dotnet_diagnostic.CS1591.severity = none # Missing XML comment for publicly visible type or member +dotnet_diagnostic.IDE1006.severity = warning # Naming rule violations diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/Connectors.Weaviate.UnitTests.csproj b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/Connectors.Weaviate.UnitTests.csproj new file mode 100644 index 000000000000..ca442f3b3233 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/Connectors.Weaviate.UnitTests.csproj @@ -0,0 +1,37 @@ + + + + SemanticKernel.Connectors.Weaviate.UnitTests + SemanticKernel.Connectors.Weaviate.UnitTests + net8.0 + true + enable + disable + false + $(NoWarn);SKEXP0001,SKEXP0020,VSTHRD111,CA2007 + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateHotel.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateHotel.cs new file mode 100644 index 000000000000..6b38dbc507e9 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateHotel.cs @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; +using Microsoft.SemanticKernel.Data; + +namespace SemanticKernel.Connectors.Weaviate.UnitTests; + +#pragma warning disable CS8618 + +public sealed record WeaviateHotel +{ + /// The key of the record. + [VectorStoreRecordKey] + public Guid HotelId { get; init; } + + /// A string metadata field. + [VectorStoreRecordData(IsFilterable = true)] + public string? HotelName { get; set; } + + /// An int metadata field. + [VectorStoreRecordData] + public int HotelCode { get; set; } + + /// A float metadata field. + [VectorStoreRecordData] + public float? HotelRating { get; set; } + + /// A bool metadata field. + [JsonPropertyName("parking_is_included")] + [VectorStoreRecordData] + public bool ParkingIncluded { get; set; } + + /// An array metadata field. + [VectorStoreRecordData] + public List Tags { get; set; } = []; + + /// A data field. + [VectorStoreRecordData(IsFullTextSearchable = true)] + public string Description { get; set; } + + [VectorStoreRecordData] + public DateTimeOffset Timestamp { get; set; } + + /// A vector field. + [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.Hnsw, DistanceFunction: DistanceFunction.CosineDistance)] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } +} diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateKernelBuilderExtensionsTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateKernelBuilderExtensionsTests.cs new file mode 100644 index 000000000000..60c6525b797b --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateKernelBuilderExtensionsTests.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.Weaviate; +using Microsoft.SemanticKernel.Data; +using Moq; +using Xunit; + +namespace SemanticKernel.Connectors.Weaviate.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class WeaviateKernelBuilderExtensionsTests +{ + private readonly IKernelBuilder _kernelBuilder = Kernel.CreateBuilder(); + + [Fact] + public void AddVectorStoreRegistersClass() + { + // Arrange + this._kernelBuilder.Services.AddSingleton(Mock.Of()); + + // Act + this._kernelBuilder.AddWeaviateVectorStore(); + + var kernel = this._kernelBuilder.Build(); + var vectorStore = kernel.Services.GetRequiredService(); + + // Assert + Assert.NotNull(vectorStore); + Assert.IsType(vectorStore); + } +} diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Weaviate/WeaviateMemoryBuilderExtensionsTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateMemoryBuilderExtensionsTests.cs similarity index 90% rename from dotnet/src/Connectors/Connectors.UnitTests/Memory/Weaviate/WeaviateMemoryBuilderExtensionsTests.cs rename to dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateMemoryBuilderExtensionsTests.cs index 58fb5c23ee08..d3c4a2a0c92f 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Weaviate/WeaviateMemoryBuilderExtensionsTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateMemoryBuilderExtensionsTests.cs @@ -18,6 +18,11 @@ namespace SemanticKernel.Connectors.UnitTests.Weaviate; public sealed class WeaviateMemoryBuilderExtensionsTests : IDisposable { + private static readonly JsonSerializerOptions s_jsonSerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }; + private readonly HttpMessageHandlerStub _messageHandlerStub; private readonly HttpClient _httpClient; @@ -46,7 +51,7 @@ public async Task WeaviateMemoryStoreShouldBeProperlyInitializedAsync(string? ap } }; - this._messageHandlerStub.ResponseToReturn.Content = new StringContent(JsonSerializer.Serialize(getResponse, new JsonSerializerOptions() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }), Encoding.UTF8, MediaTypeNames.Application.Json); + this._messageHandlerStub.ResponseToReturn.Content = new StringContent(JsonSerializer.Serialize(getResponse, s_jsonSerializerOptions), Encoding.UTF8, MediaTypeNames.Application.Json); var builder = new MemoryBuilder(); builder.WithWeaviateMemoryStore(this._httpClient, "https://fake-random-test-weaviate-host", "fake-api-key", apiVersion); diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Weaviate/WeaviateMemoryStoreTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateMemoryStoreTests.cs similarity index 92% rename from dotnet/src/Connectors/Connectors.UnitTests/Memory/Weaviate/WeaviateMemoryStoreTests.cs rename to dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateMemoryStoreTests.cs index a19a7df73192..97134f46818a 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Weaviate/WeaviateMemoryStoreTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateMemoryStoreTests.cs @@ -18,6 +18,11 @@ namespace SemanticKernel.Connectors.UnitTests.Weaviate; /// public sealed class WeaviateMemoryStoreTests : IDisposable { + private static readonly JsonSerializerOptions s_jsonSerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }; + private readonly HttpMessageHandlerStub _messageHandlerStub; private readonly HttpClient _httpClient; @@ -35,7 +40,7 @@ public WeaviateMemoryStoreTests() } }; - this._messageHandlerStub.ResponseToReturn.Content = new StringContent(JsonSerializer.Serialize(getResponse, new JsonSerializerOptions() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }), Encoding.UTF8, MediaTypeNames.Application.Json); + this._messageHandlerStub.ResponseToReturn.Content = new StringContent(JsonSerializer.Serialize(getResponse, s_jsonSerializerOptions), Encoding.UTF8, MediaTypeNames.Application.Json); this._httpClient = new HttpClient(this._messageHandlerStub, false); } diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateServiceCollectionExtensionsTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateServiceCollectionExtensionsTests.cs new file mode 100644 index 000000000000..74ed9f185485 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateServiceCollectionExtensionsTests.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.Weaviate; +using Microsoft.SemanticKernel.Data; +using Moq; +using Xunit; + +namespace SemanticKernel.Connectors.Weaviate.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class WeaviateServiceCollectionExtensionsTests +{ + private readonly IServiceCollection _serviceCollection = new ServiceCollection(); + + [Fact] + public void AddVectorStoreRegistersClass() + { + // Arrange + this._serviceCollection.AddSingleton(Mock.Of()); + + // Act + this._serviceCollection.AddWeaviateVectorStore(); + + var serviceProvider = this._serviceCollection.BuildServiceProvider(); + var vectorStore = serviceProvider.GetRequiredService(); + + // Assert + Assert.NotNull(vectorStore); + Assert.IsType(vectorStore); + } +} diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs new file mode 100644 index 000000000000..e81bb12b97fa --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs @@ -0,0 +1,184 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Microsoft.SemanticKernel.Connectors.Weaviate; +using Microsoft.SemanticKernel.Data; +using Xunit; + +namespace SemanticKernel.Connectors.Weaviate.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class WeaviateVectorStoreCollectionCreateMappingTests +{ + [Fact] + public void ItThrowsExceptionWithInvalidIndexKind() + { + // Arrange + var vectorProperties = new List + { + new("PropertyName", typeof(ReadOnlyMemory)) { IndexKind = "non-existent-index-kind" } + }; + + var storagePropertyNames = new Dictionary { ["PropertyName"] = "propertyName" }; + + // Act & Assert + Assert.Throws(() => WeaviateVectorStoreCollectionCreateMapping.MapToSchema( + collectionName: "CollectionName", + dataProperties: [], + vectorProperties: vectorProperties, + storagePropertyNames: storagePropertyNames)); + } + + [Theory] + [InlineData(IndexKind.Hnsw, "hnsw")] + [InlineData(IndexKind.Flat, "flat")] + [InlineData(IndexKind.Dynamic, "dynamic")] + public void ItReturnsCorrectSchemaWithValidIndexKind(string indexKind, string expectedIndexKind) + { + // Arrange + var vectorProperties = new List + { + new("PropertyName", typeof(ReadOnlyMemory)) { IndexKind = indexKind } + }; + + var storagePropertyNames = new Dictionary { ["PropertyName"] = "propertyName" }; + + // Act + var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema( + collectionName: "CollectionName", + dataProperties: [], + vectorProperties: vectorProperties, + storagePropertyNames: storagePropertyNames); + + var actualIndexKind = schema.VectorConfigurations["propertyName"].VectorIndexType; + + // Assert + Assert.Equal(expectedIndexKind, actualIndexKind); + } + + [Fact] + public void ItThrowsExceptionWithInvalidDistanceFunction() + { + // Arrange + var vectorProperties = new List + { + new("PropertyName", typeof(ReadOnlyMemory)) { DistanceFunction = "non-existent-distance-function" } + }; + + var storagePropertyNames = new Dictionary { ["PropertyName"] = "propertyName" }; + + // Act & Assert + Assert.Throws(() => WeaviateVectorStoreCollectionCreateMapping.MapToSchema( + collectionName: "CollectionName", + dataProperties: [], + vectorProperties: vectorProperties, + storagePropertyNames: storagePropertyNames)); + } + + [Theory] + [InlineData(DistanceFunction.CosineDistance, "cosine")] + [InlineData(DistanceFunction.DotProductSimilarity, "dot")] + [InlineData(DistanceFunction.EuclideanSquaredDistance, "l2-squared")] + [InlineData(DistanceFunction.Hamming, "hamming")] + [InlineData(DistanceFunction.ManhattanDistance, "manhattan")] + public void ItReturnsCorrectSchemaWithValidDistanceFunction(string distanceFunction, string expectedDistanceFunction) + { + // Arrange + var vectorProperties = new List + { + new("PropertyName", typeof(ReadOnlyMemory)) { DistanceFunction = distanceFunction } + }; + + var storagePropertyNames = new Dictionary { ["PropertyName"] = "propertyName" }; + + // Act + var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema( + collectionName: "CollectionName", + dataProperties: [], + vectorProperties: vectorProperties, + storagePropertyNames: storagePropertyNames); + + var actualDistanceFunction = schema.VectorConfigurations["propertyName"].VectorIndexConfig?.Distance; + + // Assert + Assert.Equal(expectedDistanceFunction, actualDistanceFunction); + } + + [Theory] + [InlineData(typeof(string), "text")] + [InlineData(typeof(List), "text[]")] + [InlineData(typeof(int), "int")] + [InlineData(typeof(int?), "int")] + [InlineData(typeof(List), "int[]")] + [InlineData(typeof(List), "int[]")] + [InlineData(typeof(long), "int")] + [InlineData(typeof(long?), "int")] + [InlineData(typeof(List), "int[]")] + [InlineData(typeof(List), "int[]")] + [InlineData(typeof(short), "int")] + [InlineData(typeof(short?), "int")] + [InlineData(typeof(List), "int[]")] + [InlineData(typeof(List), "int[]")] + [InlineData(typeof(byte), "int")] + [InlineData(typeof(byte?), "int")] + [InlineData(typeof(List), "int[]")] + [InlineData(typeof(List), "int[]")] + [InlineData(typeof(float), "number")] + [InlineData(typeof(float?), "number")] + [InlineData(typeof(List), "number[]")] + [InlineData(typeof(List), "number[]")] + [InlineData(typeof(double), "number")] + [InlineData(typeof(double?), "number")] + [InlineData(typeof(List), "number[]")] + [InlineData(typeof(List), "number[]")] + [InlineData(typeof(decimal), "number")] + [InlineData(typeof(decimal?), "number")] + [InlineData(typeof(List), "number[]")] + [InlineData(typeof(List), "number[]")] + [InlineData(typeof(DateTime), "date")] + [InlineData(typeof(DateTime?), "date")] + [InlineData(typeof(List), "date[]")] + [InlineData(typeof(List), "date[]")] + [InlineData(typeof(DateTimeOffset), "date")] + [InlineData(typeof(DateTimeOffset?), "date")] + [InlineData(typeof(List), "date[]")] + [InlineData(typeof(List), "date[]")] + [InlineData(typeof(Guid), "uuid")] + [InlineData(typeof(Guid?), "uuid")] + [InlineData(typeof(List), "uuid[]")] + [InlineData(typeof(List), "uuid[]")] + [InlineData(typeof(bool), "boolean")] + [InlineData(typeof(bool?), "boolean")] + [InlineData(typeof(List), "boolean[]")] + [InlineData(typeof(List), "boolean[]")] + [InlineData(typeof(object), "object")] + [InlineData(typeof(List), "object[]")] + public void ItMapsPropertyCorrectly(Type propertyType, string expectedPropertyType) + { + // Arrange + var dataProperties = new List + { + new("PropertyName", propertyType) { IsFilterable = true, IsFullTextSearchable = true } + }; + + var storagePropertyNames = new Dictionary { ["PropertyName"] = "propertyName" }; + + // Act + var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema( + collectionName: "CollectionName", + dataProperties: dataProperties, + vectorProperties: [], + storagePropertyNames: storagePropertyNames); + + var property = schema.Properties[0]; + + // Assert + Assert.Equal("propertyName", property.Name); + Assert.Equal(expectedPropertyType, property.DataType[0]); + Assert.True(property.IndexSearchable); + Assert.True(property.IndexFilterable); + } +} diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionTests.cs new file mode 100644 index 000000000000..92fc374aec4d --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionTests.cs @@ -0,0 +1,470 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Connectors.Weaviate; +using Microsoft.SemanticKernel.Data; +using Moq; +using Xunit; + +namespace SemanticKernel.Connectors.Weaviate.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class WeaviateVectorStoreRecordCollectionTests : IDisposable +{ + private readonly HttpMessageHandlerStub _messageHandlerStub = new(); + private readonly HttpClient _mockHttpClient; + + public WeaviateVectorStoreRecordCollectionTests() + { + this._mockHttpClient = new(this._messageHandlerStub, false) { BaseAddress = new Uri("http://default-endpoint") }; + } + + [Fact] + public void ConstructorForModelWithoutKeyThrowsException() + { + // Act & Assert + var exception = Assert.Throws(() => new WeaviateVectorStoreRecordCollection(this._mockHttpClient, "collection")); + Assert.Contains("No key property found", exception.Message); + } + + [Fact] + public void ConstructorWithoutEndpointThrowsException() + { + // Arrange + using var httpClient = new HttpClient(); + + // Act & Assert + var exception = Assert.Throws(() => new WeaviateVectorStoreRecordCollection(httpClient, "collection")); + Assert.Contains("Weaviate endpoint should be provided", exception.Message); + } + + [Fact] + public void ConstructorWithDeclarativeModelInitializesCollection() + { + // Act & Assert + var collection = new WeaviateVectorStoreRecordCollection( + this._mockHttpClient, + "collection"); + + Assert.NotNull(collection); + } + + [Fact] + public void ConstructorWithImperativeModelInitializesCollection() + { + // Arrange + var definition = new VectorStoreRecordDefinition + { + Properties = [new VectorStoreRecordKeyProperty("Id", typeof(Guid))] + }; + + // Act + var collection = new WeaviateVectorStoreRecordCollection( + this._mockHttpClient, + "collection", + new() { VectorStoreRecordDefinition = definition }); + + // Assert + Assert.NotNull(collection); + } + + [Theory] + [MemberData(nameof(CollectionExistsData))] + public async Task CollectionExistsReturnsValidResultAsync(HttpResponseMessage responseMessage, bool expectedResult) + { + // Arrange + this._messageHandlerStub.ResponseToReturn = responseMessage; + + var sut = new WeaviateVectorStoreRecordCollection(this._mockHttpClient, "Collection"); + + // Act + var actualResult = await sut.CollectionExistsAsync(); + + // Assert + Assert.Equal(expectedResult, actualResult); + } + + [Fact] + public async Task CreateCollectionUsesValidCollectionSchemaAsync() + { + // Arrange + const string CollectionName = "Collection"; + var sut = new WeaviateVectorStoreRecordCollection(this._mockHttpClient, CollectionName); + + // Act + await sut.CreateCollectionAsync(); + + // Assert + var schemaRequest = JsonSerializer.Deserialize(this._messageHandlerStub.RequestContent); + + Assert.NotNull(schemaRequest); + + Assert.Equal(CollectionName, schemaRequest.CollectionName); + + Assert.NotNull(schemaRequest.VectorConfigurations); + Assert.Equal("descriptionEmbedding", schemaRequest.VectorConfigurations.Keys.First()); + + var vectorConfiguration = schemaRequest.VectorConfigurations["descriptionEmbedding"]; + + Assert.Equal("cosine", vectorConfiguration.VectorIndexConfig?.Distance); + Assert.Equal("hnsw", vectorConfiguration.VectorIndexType); + + Assert.NotNull(schemaRequest.Properties); + + this.AssertSchemaProperty(schemaRequest.Properties[0], "hotelName", "text", true, false); + this.AssertSchemaProperty(schemaRequest.Properties[1], "hotelCode", "int", false, false); + this.AssertSchemaProperty(schemaRequest.Properties[2], "hotelRating", "number", false, false); + this.AssertSchemaProperty(schemaRequest.Properties[3], "parking_is_included", "boolean", false, false); + this.AssertSchemaProperty(schemaRequest.Properties[4], "tags", "text[]", false, false); + this.AssertSchemaProperty(schemaRequest.Properties[5], "description", "text", false, true); + this.AssertSchemaProperty(schemaRequest.Properties[6], "timestamp", "date", false, false); + } + + [Fact] + public async Task DeleteCollectionSendsValidRequestAsync() + { + // Arrange + const string CollectionName = "Collection"; + var sut = new WeaviateVectorStoreRecordCollection(this._mockHttpClient, CollectionName); + + // Act + await sut.DeleteCollectionAsync(); + + // Assert + Assert.Equal("http://default-endpoint/schema/Collection", this._messageHandlerStub.RequestUri?.AbsoluteUri); + Assert.Equal(HttpMethod.Delete, this._messageHandlerStub.Method); + } + + [Fact] + public async Task DeleteSendsValidRequestAsync() + { + // Arrange + const string CollectionName = "Collection"; + var id = new Guid("55555555-5555-5555-5555-555555555555"); + + var sut = new WeaviateVectorStoreRecordCollection(this._mockHttpClient, CollectionName); + + // Act + await sut.DeleteAsync(id); + + // Assert + Assert.Equal("http://default-endpoint/objects/Collection/55555555-5555-5555-5555-555555555555", this._messageHandlerStub.RequestUri?.AbsoluteUri); + Assert.Equal(HttpMethod.Delete, this._messageHandlerStub.Method); + } + + [Fact] + public async Task DeleteBatchUsesValidQueryMatchAsync() + { + // Arrange + const string CollectionName = "Collection"; + List ids = [new Guid("11111111-1111-1111-1111-111111111111"), new Guid("22222222-2222-2222-2222-222222222222")]; + + var sut = new WeaviateVectorStoreRecordCollection(this._mockHttpClient, CollectionName); + + // Act + await sut.DeleteBatchAsync(ids); + + // Assert + var request = JsonSerializer.Deserialize(this._messageHandlerStub.RequestContent); + + Assert.NotNull(request?.Match); + + Assert.Equal(CollectionName, request.Match.CollectionName); + + Assert.NotNull(request.Match.WhereClause); + + var clause = request.Match.WhereClause; + + Assert.Equal("ContainsAny", clause.Operator); + Assert.Equal(["id"], clause.Path); + Assert.Equal(["11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222"], clause.Values); + } + + [Fact] + public async Task GetExistingRecordReturnsValidRecordAsync() + { + // Arrange + var id = new Guid("55555555-5555-5555-5555-555555555555"); + + var jsonObject = new JsonObject { ["id"] = id.ToString(), ["properties"] = new JsonObject() }; + + jsonObject["properties"]!["hotelName"] = "Test Name"; + + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(JsonSerializer.Serialize(jsonObject)) + }; + + var sut = new WeaviateVectorStoreRecordCollection(this._mockHttpClient, "Collection"); + + // Act + var result = await sut.GetAsync(id); + + // Assert + Assert.NotNull(result); + Assert.Equal(id, result.HotelId); + Assert.Equal("Test Name", result.HotelName); + } + + [Fact] + public async Task GetExistingBatchRecordsReturnsValidRecordsAsync() + { + // Arrange + var id1 = new Guid("11111111-1111-1111-1111-111111111111"); + var id2 = new Guid("22222222-2222-2222-2222-222222222222"); + + var jsonObject1 = new JsonObject { ["id"] = id1.ToString(), ["properties"] = new JsonObject() }; + var jsonObject2 = new JsonObject { ["id"] = id2.ToString(), ["properties"] = new JsonObject() }; + + jsonObject1["properties"]!["hotelName"] = "Test Name 1"; + jsonObject2["properties"]!["hotelName"] = "Test Name 2"; + + using var response1 = new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(JsonSerializer.Serialize(jsonObject1)) }; + using var response2 = new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(JsonSerializer.Serialize(jsonObject2)) }; + + this._messageHandlerStub.ResponseQueue.Enqueue(response1); + this._messageHandlerStub.ResponseQueue.Enqueue(response2); + + var sut = new WeaviateVectorStoreRecordCollection(this._mockHttpClient, "Collection"); + + // Act + var results = await sut.GetBatchAsync([id1, id2]).ToListAsync(); + + // Assert + Assert.NotNull(results[0]); + Assert.Equal(id1, results[0].HotelId); + Assert.Equal("Test Name 1", results[0].HotelName); + + Assert.NotNull(results[1]); + Assert.Equal(id2, results[1].HotelId); + Assert.Equal("Test Name 2", results[1].HotelName); + } + + [Fact] + public async Task UpsertReturnsRecordKeyAsync() + { + // Arrange + var id = new Guid("11111111-1111-1111-1111-111111111111"); + var hotel = new WeaviateHotel { HotelId = id, HotelName = "Test Name" }; + + var batchResponse = new List { new() { Id = id, Result = new() { Status = "Success" } } }; + + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(JsonSerializer.Serialize(batchResponse)), + }; + + var sut = new WeaviateVectorStoreRecordCollection(this._mockHttpClient, "Collection"); + + // Act + var result = await sut.UpsertAsync(hotel); + + // Assert + Assert.Equal(id, result); + + var request = JsonSerializer.Deserialize(this._messageHandlerStub.RequestContent); + + Assert.NotNull(request?.CollectionObjects); + + var jsonObject = request.CollectionObjects[0]; + + Assert.Equal("11111111-1111-1111-1111-111111111111", jsonObject["id"]?.GetValue()); + Assert.Equal("Test Name", jsonObject["properties"]?["hotelName"]?.GetValue()); + } + + [Fact] + public async Task UpsertReturnsRecordKeysAsync() + { + // Arrange + var id1 = new Guid("11111111-1111-1111-1111-111111111111"); + var id2 = new Guid("22222222-2222-2222-2222-222222222222"); + + var hotel1 = new WeaviateHotel { HotelId = id1, HotelName = "Test Name 1" }; + var hotel2 = new WeaviateHotel { HotelId = id2, HotelName = "Test Name 2" }; + + var batchResponse = new List + { + new() { Id = id1, Result = new() { Status = "Success" } }, + new() { Id = id2, Result = new() { Status = "Success" } } + }; + + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(JsonSerializer.Serialize(batchResponse)), + }; + + var sut = new WeaviateVectorStoreRecordCollection(this._mockHttpClient, "Collection"); + + // Act + var results = await sut.UpsertBatchAsync([hotel1, hotel2]).ToListAsync(); + + // Assert + Assert.Contains(id1, results); + Assert.Contains(id2, results); + + var request = JsonSerializer.Deserialize(this._messageHandlerStub.RequestContent); + + Assert.NotNull(request?.CollectionObjects); + + var jsonObject1 = request.CollectionObjects[0]; + var jsonObject2 = request.CollectionObjects[1]; + + Assert.Equal("11111111-1111-1111-1111-111111111111", jsonObject1["id"]?.GetValue()); + Assert.Equal("Test Name 1", jsonObject1["properties"]?["hotelName"]?.GetValue()); + + Assert.Equal("22222222-2222-2222-2222-222222222222", jsonObject2["id"]?.GetValue()); + Assert.Equal("Test Name 2", jsonObject2["properties"]?["hotelName"]?.GetValue()); + } + + [Fact] + public async Task UpsertWithCustomMapperWorksCorrectlyAsync() + { + // Arrange + var id = new Guid("11111111-1111-1111-1111-111111111111"); + var hotel = new WeaviateHotel { HotelId = id, HotelName = "Test Name" }; + + var jsonObject = new JsonObject { ["id"] = id.ToString(), ["properties"] = new JsonObject() }; + + jsonObject["properties"]!["hotel_name"] = "Test Name from Mapper"; + + var mockMapper = new Mock>(); + + mockMapper + .Setup(l => l.MapFromDataToStorageModel(It.IsAny())) + .Returns(jsonObject); + + var batchResponse = new List { new() { Id = id, Result = new() { Status = "Success" } } }; + + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(JsonSerializer.Serialize(batchResponse)), + }; + + var sut = new WeaviateVectorStoreRecordCollection( + this._mockHttpClient, + "Collection", + new() { JsonNodeCustomMapper = mockMapper.Object }); + + // Act + var result = await sut.UpsertAsync(hotel); + + // Assert + Assert.Equal(id, result); + + var request = JsonSerializer.Deserialize(this._messageHandlerStub.RequestContent); + + Assert.NotNull(request?.CollectionObjects); + + var requestObject = request.CollectionObjects[0]; + + Assert.Equal("11111111-1111-1111-1111-111111111111", requestObject["id"]?.GetValue()); + Assert.Equal("Test Name from Mapper", requestObject["properties"]?["hotel_name"]?.GetValue()); + } + + [Fact] + public async Task GetWithCustomMapperWorksCorrectlyAsync() + { + // Arrange + var id = new Guid("11111111-1111-1111-1111-111111111111"); + var jsonObject = new JsonObject { ["id"] = id.ToString(), ["properties"] = new JsonObject() }; + + jsonObject["properties"]!["hotelName"] = "Test Name"; + + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(JsonSerializer.Serialize(jsonObject)) + }; + + var mockMapper = new Mock>(); + + mockMapper + .Setup(l => l.MapFromStorageToDataModel(It.IsAny(), It.IsAny())) + .Returns(new WeaviateHotel { HotelId = id, HotelName = "Test Name from mapper" }); + + var sut = new WeaviateVectorStoreRecordCollection( + this._mockHttpClient, + "Collection", + new() { JsonNodeCustomMapper = mockMapper.Object }); + + // Act + var result = await sut.GetAsync(id); + + // Assert + Assert.NotNull(result); + Assert.Equal(id, result.HotelId); + Assert.Equal("Test Name from mapper", result.HotelName); + } + + [Theory] + [InlineData(true, "http://test-endpoint/schema", "Bearer fake-key")] + [InlineData(false, "http://default-endpoint/schema", null)] + public async Task ItUsesHttpClientParametersAsync(bool initializeOptions, string expectedEndpoint, string? expectedHeader) + { + // Arrange + const string CollectionName = "Collection"; + + var options = initializeOptions ? + new WeaviateVectorStoreRecordCollectionOptions() { Endpoint = new Uri("http://test-endpoint"), ApiKey = "fake-key" } : + null; + + var sut = new WeaviateVectorStoreRecordCollection(this._mockHttpClient, CollectionName, options); + + // Act + await sut.CreateCollectionAsync(); + + var headers = this._messageHandlerStub.RequestHeaders; + var endpoint = this._messageHandlerStub.RequestUri; + + // Assert + Assert.Equal(expectedEndpoint, endpoint?.AbsoluteUri); + Assert.Equal(expectedHeader, headers?.Authorization?.ToString()); + } + + public void Dispose() + { + this._mockHttpClient.Dispose(); + this._messageHandlerStub.Dispose(); + } + + public static TheoryData CollectionExistsData => new() + { + { new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(JsonSerializer.Serialize(new WeaviateGetCollectionSchemaResponse { CollectionName = "Collection" })) }, true }, + { new HttpResponseMessage(HttpStatusCode.NotFound), false } + }; + + #region private + + private void AssertSchemaProperty( + WeaviateCollectionSchemaProperty property, + string propertyName, + string dataType, + bool indexFilterable, + bool indexSearchable) + { + Assert.NotNull(property); + Assert.Equal(propertyName, property.Name); + Assert.Equal(dataType, property.DataType[0]); + Assert.Equal(indexFilterable, property.IndexFilterable); + Assert.Equal(indexSearchable, property.IndexSearchable); + } + +#pragma warning disable CA1812 + private sealed class TestModel + { + public Guid Id { get; set; } + + public string? HotelName { get; set; } + } +#pragma warning restore CA1812 + + #endregion +} diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs new file mode 100644 index 000000000000..326de1146cd9 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using Microsoft.SemanticKernel.Connectors.Weaviate; +using Microsoft.SemanticKernel.Data; +using Xunit; + +namespace SemanticKernel.Connectors.Weaviate.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class WeaviateVectorStoreRecordMapperTests +{ + private static readonly JsonSerializerOptions s_jsonSerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + private readonly WeaviateVectorStoreRecordMapper _sut; + + public WeaviateVectorStoreRecordMapperTests() + { + var storagePropertyNames = new Dictionary + { + ["HotelId"] = "hotelId", + ["HotelName"] = "hotelName", + ["Tags"] = "tags", + ["DescriptionEmbedding"] = "descriptionEmbedding", + }; + + var dataProperties = new List + { + new("HotelName", typeof(string)), + new("Tags", typeof(List)) + }; + + var vectorProperties = new List + { + new("DescriptionEmbedding", typeof(ReadOnlyMemory)) + }; + + this._sut = new WeaviateVectorStoreRecordMapper( + "CollectionName", + new VectorStoreRecordKeyProperty("HotelId", typeof(Guid)), + dataProperties, + vectorProperties, + storagePropertyNames, + s_jsonSerializerOptions); + } + + [Fact] + public void MapFromDataToStorageModelReturnsValidObject() + { + // Arrange + var hotel = new WeaviateHotel + { + HotelId = new Guid("55555555-5555-5555-5555-555555555555"), + HotelName = "Test Name", + Tags = ["tag1", "tag2"], + DescriptionEmbedding = new ReadOnlyMemory([1f, 2f, 3f]) + }; + + // Act + var document = this._sut.MapFromDataToStorageModel(hotel); + + // Assert + Assert.NotNull(document); + + Assert.Equal("55555555-5555-5555-5555-555555555555", document["id"]!.GetValue()); + Assert.Equal("Test Name", document["properties"]!["hotelName"]!.GetValue()); + Assert.Equal(["tag1", "tag2"], document["properties"]!["tags"]!.AsArray().Select(l => l!.GetValue())); + Assert.Equal([1f, 2f, 3f], document["vectors"]!["descriptionEmbedding"]!.AsArray().Select(l => l!.GetValue())); + } + + [Fact] + public void MapFromStorageToDataModelReturnsValidObject() + { + // Arrange + var document = new JsonObject + { + ["id"] = "55555555-5555-5555-5555-555555555555", + ["properties"] = new JsonObject(), + ["vectors"] = new JsonObject() + }; + + document["properties"]!["hotelName"] = "Test Name"; + document["properties"]!["tags"] = new JsonArray(new List { "tag1", "tag2" }.Select(l => JsonValue.Create(l)).ToArray()); + document["vectors"]!["descriptionEmbedding"] = new JsonArray(new List { 1f, 2f, 3f }.Select(l => JsonValue.Create(l)).ToArray()); + + // Act + var hotel = this._sut.MapFromStorageToDataModel(document, new() { IncludeVectors = true }); + + // Assert + Assert.NotNull(hotel); + + Assert.Equal(new Guid("55555555-5555-5555-5555-555555555555"), hotel.HotelId); + Assert.Equal("Test Name", hotel.HotelName); + Assert.Equal(["tag1", "tag2"], hotel.Tags); + Assert.True(new ReadOnlyMemory([1f, 2f, 3f]).Span.SequenceEqual(hotel.DescriptionEmbedding!.Value.Span)); + } +} diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreTests.cs new file mode 100644 index 000000000000..012d9c7b2369 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreTests.cs @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Connectors.Weaviate; +using Microsoft.SemanticKernel.Data; +using Moq; +using Xunit; + +namespace SemanticKernel.Connectors.Weaviate.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class WeaviateVectorStoreTests : IDisposable +{ + private readonly HttpMessageHandlerStub _messageHandlerStub = new(); + private readonly HttpClient _mockHttpClient; + + public WeaviateVectorStoreTests() + { + this._mockHttpClient = new(this._messageHandlerStub, false) { BaseAddress = new Uri("http://test") }; + } + + [Fact] + public void GetCollectionWithNotSupportedKeyThrowsException() + { + // Arrange + var sut = new WeaviateVectorStore(this._mockHttpClient); + + // Act & Assert + Assert.Throws(() => sut.GetCollection("collection")); + } + + [Fact] + public void GetCollectionWithSupportedKeyReturnsCollection() + { + // Arrange + var sut = new WeaviateVectorStore(this._mockHttpClient); + + // Act + var collection = sut.GetCollection("collection1"); + + // Assert + Assert.NotNull(collection); + } + + [Fact] + public void GetCollectionWithFactoryReturnsCustomCollection() + { + // Arrange + var mockFactory = new Mock(); + var mockRecordCollection = new Mock>(); + + mockFactory + .Setup(l => l.CreateVectorStoreRecordCollection( + this._mockHttpClient, + "collection", + It.IsAny())) + .Returns(mockRecordCollection.Object); + + var sut = new WeaviateVectorStore( + this._mockHttpClient, + new WeaviateVectorStoreOptions { VectorStoreCollectionFactory = mockFactory.Object }); + + // Act + var collection = sut.GetCollection("collection"); + + // Assert + Assert.Same(mockRecordCollection.Object, collection); + mockFactory.Verify(l => l.CreateVectorStoreRecordCollection( + this._mockHttpClient, + "collection", + It.IsAny()), Times.Once()); + } + + [Fact] + public async Task ListCollectionNamesReturnsCollectionNamesAsync() + { + // Arrange + var expectedCollectionNames = new List { "Collection1", "Collection2", "Collection3" }; + var response = new WeaviateGetCollectionsResponse + { + Collections = expectedCollectionNames.Select(name => new WeaviateCollectionSchema(name)).ToList() + }; + + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(JsonSerializer.Serialize(response)) + }; + + var sut = new WeaviateVectorStore(this._mockHttpClient); + + // Act + var actualCollectionNames = await sut.ListCollectionNamesAsync().ToListAsync(); + + // Assert + Assert.Equal(expectedCollectionNames, actualCollectionNames); + } + + public void Dispose() + { + this._mockHttpClient.Dispose(); + this._messageHandlerStub.Dispose(); + } +} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateHotel.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateHotel.cs new file mode 100644 index 000000000000..e50be0e3e89d --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateHotel.cs @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; +using Microsoft.SemanticKernel.Data; + +namespace SemanticKernel.IntegrationTests.Connectors.Memory.Weaviate; + +#pragma warning disable CS8618 + +public sealed record WeaviateHotel +{ + /// The key of the record. + [VectorStoreRecordKey] + public Guid HotelId { get; init; } + + /// A string metadata field. + [VectorStoreRecordData(IsFilterable = true)] + public string? HotelName { get; set; } + + /// An int metadata field. + [VectorStoreRecordData] + public int HotelCode { get; set; } + + /// A float metadata field. + [VectorStoreRecordData] + public float? HotelRating { get; set; } + + /// A bool metadata field. + [JsonPropertyName("parking_is_included")] + [VectorStoreRecordData] + public bool ParkingIncluded { get; set; } + + /// An array metadata field. + [VectorStoreRecordData] + public List Tags { get; set; } = []; + + /// A data field. + [VectorStoreRecordData(IsFullTextSearchable = true)] + public string Description { get; set; } + + [VectorStoreRecordData] + public DateTimeOffset Timestamp { get; set; } + + /// A vector field. + [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.Hnsw, DistanceFunction: DistanceFunction.CosineDistance)] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } +} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreCollectionFixture.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreCollectionFixture.cs new file mode 100644 index 000000000000..2f3f67f723a2 --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreCollectionFixture.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Xunit; + +namespace SemanticKernel.IntegrationTests.Connectors.Memory.Weaviate; + +[CollectionDefinition("WeaviateVectorStoreCollection")] +public class WeaviateVectorStoreCollectionFixture : ICollectionFixture +{ } diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreFixture.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreFixture.cs new file mode 100644 index 000000000000..7da5be1a85c7 --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreFixture.cs @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Docker.DotNet; +using Docker.DotNet.Models; +using Xunit; + +namespace SemanticKernel.IntegrationTests.Connectors.Memory.Weaviate; + +public class WeaviateVectorStoreFixture : IAsyncLifetime +{ + /// The Docker client we are using to create a Weaviate container with. + private readonly DockerClient _client; + + /// The id of the Weaviate container that we are testing with. + private string? _containerId = null; + + public HttpClient? HttpClient { get; private set; } + + public WeaviateVectorStoreFixture() + { + using var dockerClientConfiguration = new DockerClientConfiguration(); + this._client = dockerClientConfiguration.CreateClient(); + } + + public async Task InitializeAsync() + { + this._containerId = await SetupWeaviateContainerAsync(this._client); + + this.HttpClient = new HttpClient { BaseAddress = new Uri("http://localhost:8080/v1/") }; + + await WaitForInitializationAsync(this.HttpClient); + } + + public async Task DisposeAsync() + { + if (this._containerId != null) + { + await this._client.Containers.StopContainerAsync(this._containerId, new ContainerStopParameters()); + await this._client.Containers.RemoveContainerAsync(this._containerId, new ContainerRemoveParameters()); + } + } + + #region private + + private async static Task WaitForInitializationAsync(HttpClient httpClient) + { + const int MaxAttemptCount = 10; + const int DelayInterval = 1000; + + int attemptCount = 0; + bool clusterReady = false; + + do + { + await Task.Delay(DelayInterval); + attemptCount++; + clusterReady = await CheckIfClusterReadyAsync(httpClient); + } while (!clusterReady && attemptCount <= MaxAttemptCount); + + if (!clusterReady) + { + throw new InvalidOperationException("Weaviate cluster is not ready for usage."); + } + } + + private static async Task CheckIfClusterReadyAsync(HttpClient httpClient) + { + try + { + var response = await httpClient.GetAsync(new Uri("schema", UriKind.Relative)); + + return response.StatusCode == HttpStatusCode.OK; + } + catch (HttpRequestException) + { + return false; + } + } + + private static async Task SetupWeaviateContainerAsync(DockerClient client) + { + const string Image = "cr.weaviate.io/semitechnologies/weaviate"; + + await client.Images.CreateImageAsync( + new ImagesCreateParameters + { + FromImage = Image, + Tag = "latest", + }, + null, + new Progress()); + + var container = await client.Containers.CreateContainerAsync(new CreateContainerParameters() + { + Image = Image, + HostConfig = new HostConfig() + { + PortBindings = new Dictionary> + { + { "8080", new List { new() { HostPort = "8080" } } }, + { "50051", new List { new() { HostPort = "50051" } } } + }, + PublishAllPorts = true + }, + ExposedPorts = new Dictionary + { + { "8080", default }, + { "50051", default } + }, + }); + + await client.Containers.StartContainerAsync( + container.ID, + new ContainerStartParameters()); + + return container.ID; + } + + #endregion +} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreRecordCollectionTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreRecordCollectionTests.cs new file mode 100644 index 000000000000..3982dd0443c4 --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreRecordCollectionTests.cs @@ -0,0 +1,245 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Connectors.Weaviate; +using Microsoft.SemanticKernel.Data; +using Xunit; + +namespace SemanticKernel.IntegrationTests.Connectors.Memory.Weaviate; + +[Collection("WeaviateVectorStoreCollection")] +public sealed class WeaviateVectorStoreRecordCollectionTests(WeaviateVectorStoreFixture fixture) +{ + [Fact] + public async Task ItCanCreateCollectionAsync() + { + // Arrange + var sut = new WeaviateVectorStoreRecordCollection(fixture.HttpClient!, "TestCreateCollection"); + + // Act + await sut.CreateCollectionAsync(); + + // Assert + Assert.True(await sut.CollectionExistsAsync()); + } + + [Theory] + [InlineData("ExistingCollection", true)] + [InlineData("NonExistentCollection", false)] + public async Task ItCanCheckIfCollectionExistsAsync(string collectionName, bool collectionExists) + { + // Arrange + var sut = new WeaviateVectorStoreRecordCollection(fixture.HttpClient!, collectionName); + + if (collectionExists) + { + await sut.CreateCollectionAsync(); + } + + // Act + var result = await sut.CollectionExistsAsync(); + + // Assert + Assert.Equal(collectionExists, result); + } + + [Theory] + [InlineData("CollectionWithVectorAndDefinition", true, true)] + [InlineData("CollectionWithVector", true, false)] + [InlineData("CollectionWithDefinition", false, true)] + [InlineData("CollectionWithoutVectorAndDefinition", false, false)] + public async Task ItCanUpsertAndGetRecordAsync(string collectionName, bool includeVectors, bool useRecordDefinition) + { + // Arrange + var hotelId = new Guid("55555555-5555-5555-5555-555555555555"); + + var options = new WeaviateVectorStoreRecordCollectionOptions + { + VectorStoreRecordDefinition = useRecordDefinition ? this.GetTestHotelRecordDefinition() : null + }; + + var sut = new WeaviateVectorStoreRecordCollection(fixture.HttpClient!, collectionName, options); + + var record = this.CreateTestHotel(hotelId); + + // Act && Assert + await sut.CreateCollectionAsync(); + + var upsertResult = await sut.UpsertAsync(record); + + Assert.Equal(hotelId, upsertResult); + + var getResult = await sut.GetAsync(hotelId, new() { IncludeVectors = includeVectors }); + + Assert.NotNull(getResult); + + Assert.Equal(record.HotelId, getResult.HotelId); + Assert.Equal(record.HotelName, getResult.HotelName); + Assert.Equal(record.HotelCode, getResult.HotelCode); + Assert.Equal(record.HotelRating, getResult.HotelRating); + Assert.Equal(record.ParkingIncluded, getResult.ParkingIncluded); + Assert.Equal(record.Tags.ToArray(), getResult.Tags.ToArray()); + Assert.Equal(record.Description, getResult.Description); + Assert.Equal(record.Timestamp, getResult.Timestamp); + + if (includeVectors) + { + Assert.NotNull(getResult.DescriptionEmbedding); + Assert.Equal(record.DescriptionEmbedding!.Value.ToArray(), getResult.DescriptionEmbedding.Value.ToArray()); + } + else + { + Assert.Null(getResult.DescriptionEmbedding); + } + } + + [Fact] + public async Task ItCanDeleteCollectionAsync() + { + // Arrange + const string CollectionName = "TestDeleteCollection"; + + var sut = new WeaviateVectorStoreRecordCollection(fixture.HttpClient!, CollectionName); + + await sut.CreateCollectionAsync(); + + Assert.True(await sut.CollectionExistsAsync()); + + // Act + await sut.DeleteCollectionAsync(); + + // Assert + Assert.False(await sut.CollectionExistsAsync()); + } + + [Fact] + public async Task ItCanDeleteRecordAsync() + { + // Arrange + var hotelId = new Guid("55555555-5555-5555-5555-555555555555"); + + var sut = new WeaviateVectorStoreRecordCollection(fixture.HttpClient!, "TestDeleteRecord"); + + var record = this.CreateTestHotel(hotelId); + + var upsertResult = await sut.UpsertAsync(record); + var getResult = await sut.GetAsync(hotelId); + + Assert.Equal(hotelId, upsertResult); + Assert.NotNull(getResult); + + // Act + await sut.DeleteAsync(hotelId); + + getResult = await sut.GetAsync(hotelId); + + // Assert + Assert.Null(getResult); + } + + [Fact] + public async Task ItCanUpsertAndGetAndDeleteBatchAsync() + { + // Arrange + var hotelId1 = new Guid("11111111-1111-1111-1111-111111111111"); + var hotelId2 = new Guid("22222222-2222-2222-2222-222222222222"); + var hotelId3 = new Guid("33333333-3333-3333-3333-333333333333"); + + var sut = new WeaviateVectorStoreRecordCollection(fixture.HttpClient!, "TestBatch"); + + await sut.CreateCollectionAsync(); + + var record1 = this.CreateTestHotel(hotelId1); + var record2 = this.CreateTestHotel(hotelId2); + var record3 = this.CreateTestHotel(hotelId3); + + var upsertResults = await sut.UpsertBatchAsync([record1, record2, record3]).ToListAsync(); + var getResults = await sut.GetBatchAsync([hotelId1, hotelId2, hotelId3]).ToListAsync(); + + Assert.Equal([hotelId1, hotelId2, hotelId3], upsertResults); + + Assert.NotNull(getResults.First(l => l.HotelId == hotelId1)); + Assert.NotNull(getResults.First(l => l.HotelId == hotelId2)); + Assert.NotNull(getResults.First(l => l.HotelId == hotelId3)); + + // Act + await sut.DeleteBatchAsync([hotelId1, hotelId2, hotelId3]); + + getResults = await sut.GetBatchAsync([hotelId1, hotelId2, hotelId3]).ToListAsync(); + + // Assert + Assert.Empty(getResults); + } + + [Fact] + public async Task ItCanUpsertRecordAsync() + { + // Arrange + var hotelId = new Guid("55555555-5555-5555-5555-555555555555"); + var sut = new WeaviateVectorStoreRecordCollection(fixture.HttpClient!, "TestUpsert"); + + await sut.CreateCollectionAsync(); + + var record = this.CreateTestHotel(hotelId); + + var upsertResult = await sut.UpsertAsync(record); + var getResult = await sut.GetAsync(hotelId); + + Assert.Equal(hotelId, upsertResult); + Assert.NotNull(getResult); + + // Act + record.HotelName = "Updated name"; + record.HotelRating = 10; + + upsertResult = await sut.UpsertAsync(record); + getResult = await sut.GetAsync(hotelId); + + // Assert + Assert.NotNull(getResult); + Assert.Equal("Updated name", getResult.HotelName); + Assert.Equal(10, getResult.HotelRating); + } + + #region private + + private WeaviateHotel CreateTestHotel(Guid hotelId, string? hotelName = null) + { + return new WeaviateHotel + { + HotelId = hotelId, + HotelName = hotelName ?? $"My Hotel {hotelId}", + HotelCode = 42, + HotelRating = 4.5f, + ParkingIncluded = true, + Tags = { "t1", "t2" }, + Description = "This is a great hotel.", + DescriptionEmbedding = new[] { 30f, 31f, 32f, 33f }, + Timestamp = new DateTime(2024, 8, 28, 10, 11, 12) + }; + } + + private VectorStoreRecordDefinition GetTestHotelRecordDefinition() + { + return new() + { + Properties = + [ + new VectorStoreRecordKeyProperty("HotelId", typeof(Guid)), + new VectorStoreRecordDataProperty("HotelName", typeof(string)), + new VectorStoreRecordDataProperty("HotelCode", typeof(int)), + new VectorStoreRecordDataProperty("parking_is_included", typeof(bool)), + new VectorStoreRecordDataProperty("HotelRating", typeof(float)), + new VectorStoreRecordDataProperty("Tags", typeof(List)), + new VectorStoreRecordDataProperty("Description", typeof(string)), + new VectorStoreRecordDataProperty("Timestamp", typeof(DateTime)), + new VectorStoreRecordVectorProperty("DescriptionEmbedding", typeof(ReadOnlyMemory?)) { Dimensions = 4, IndexKind = IndexKind.Hnsw, DistanceFunction = DistanceFunction.CosineDistance } + ] + }; + } + + #endregion +} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreTests.cs new file mode 100644 index 000000000000..7de9413084ae --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreTests.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Linq; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Connectors.Weaviate; +using Xunit; + +namespace SemanticKernel.IntegrationTests.Connectors.Memory.Weaviate; + +[Collection("WeaviateVectorStoreCollection")] +public sealed class WeaviateVectorStoreTests(WeaviateVectorStoreFixture fixture) +{ + [Fact] + public async Task ItCanGetAListOfExistingCollectionNamesAsync() + { + // Arrange + var collection1 = new WeaviateVectorStoreRecordCollection(fixture.HttpClient!, "Collection1"); + var collection2 = new WeaviateVectorStoreRecordCollection(fixture.HttpClient!, "Collection2"); + var collection3 = new WeaviateVectorStoreRecordCollection(fixture.HttpClient!, "Collection3"); + + await collection1.CreateCollectionAsync(); + await collection2.CreateCollectionAsync(); + await collection3.CreateCollectionAsync(); + + var sut = new WeaviateVectorStore(fixture.HttpClient!); + + // Act + var collectionNames = await sut.ListCollectionNamesAsync().ToListAsync(); + + // Assert + Assert.Contains("Collection1", collectionNames); + Assert.Contains("Collection2", collectionNames); + Assert.Contains("Collection3", collectionNames); + } +} diff --git a/dotnet/src/SemanticKernel.Abstractions/Data/RecordDefinition/DistanceFunction.cs b/dotnet/src/SemanticKernel.Abstractions/Data/RecordDefinition/DistanceFunction.cs index 32601243966b..cf8b459b2472 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Data/RecordDefinition/DistanceFunction.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Data/RecordDefinition/DistanceFunction.cs @@ -54,6 +54,19 @@ public static class DistanceFunction /// public const string EuclideanDistance = nameof(EuclideanDistance); + /// + /// Measures the Euclidean squared distance between two vectors. + /// + /// + /// Also known as l2-squared. + /// + public const string EuclideanSquaredDistance = nameof(EuclideanSquaredDistance); + + /// + /// Number of differences between vectors at each dimensions. + /// + public const string Hamming = nameof(Hamming); + /// /// Measures the Manhattan distance between two vectors. /// diff --git a/dotnet/src/SemanticKernel.Abstractions/Data/RecordDefinition/IndexKind.cs b/dotnet/src/SemanticKernel.Abstractions/Data/RecordDefinition/IndexKind.cs index fc6a7ced0b89..88f14b7ef67c 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Data/RecordDefinition/IndexKind.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Data/RecordDefinition/IndexKind.cs @@ -50,4 +50,9 @@ public static class IndexKind /// Index that compresses vectors using DiskANN-based quantization methods for better efficiency in the kNN search. /// public const string QuantizedFlat = nameof(QuantizedFlat); + + /// + /// Dynamic index allows to automatically switch from to indexes. + /// + public const string Dynamic = nameof(Dynamic); }