.Net: Added implementation of Weaviate connector for new memory design (

#8403)

### Motivation and Context

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

Related: #6527

In this PR:
- Implemented `IVectorStore`
- Implemented `IVectorStoreRecordCollection<TKey, TRecord>`
- Weaviate default record mapper
- `Options` classes
- Extension methods for DI
- Integration tests  
- Unit tests

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [x] All unit tests pass, and I have added new tests where possible
- [x] I didn't break anyone 😄

---------

Co-authored-by: Mark Wallace <127216156+markwallace-microsoft@users.noreply.github.com>

dotnet/SK-dotnet.sln

@@ -316,11 +316,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Qdrant.UnitTests
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StepwisePlannerMigration", "samples\Demos\StepwisePlannerMigration\StepwisePlannerMigration.csproj", "{38374C62-0263-4FE8-A18C-70FC8132912B}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.AzureCosmosDBMongoDB.UnitTests", "src\Connectors\Connectors.AzureCosmosDBMongoDB.UnitTests\Connectors.AzureCosmosDBMongoDB.UnitTests.csproj", "{2918478E-BC86-4D53-9D01-9C318F80C14F}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.AzureCosmosDBMongoDB.UnitTests", "src\Connectors\Connectors.AzureCosmosDBMongoDB.UnitTests\Connectors.AzureCosmosDBMongoDB.UnitTests.csproj", "{2918478E-BC86-4D53-9D01-9C318F80C14F}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AIModelRouter", "samples\Demos\AIModelRouter\AIModelRouter.csproj", "{E06818E3-00A5-41AC-97ED-9491070CDEA1}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AIModelRouter", "samples\Demos\AIModelRouter\AIModelRouter.csproj", "{E06818E3-00A5-41AC-97ED-9491070CDEA1}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.AzureCosmosDBNoSQL.UnitTests", "src\Connectors\Connectors.AzureCosmosDBNoSQL.UnitTests\Connectors.AzureCosmosDBNoSQL.UnitTests.csproj", "{385A8FE5-87E2-4458-AE09-35E10BD2E67F}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.AzureCosmosDBNoSQL.UnitTests", "src\Connectors\Connectors.AzureCosmosDBNoSQL.UnitTests\Connectors.AzureCosmosDBNoSQL.UnitTests.csproj", "{385A8FE5-87E2-4458-AE09-35E10BD2E67F}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.Weaviate.UnitTests", "src\Connectors\Connectors.Weaviate.UnitTests\Connectors.Weaviate.UnitTests.csproj", "{AD9ECE32-088A-49D8-8ACB-890E79F1E7B8}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.OpenAI.UnitTests", "src\Connectors\Connectors.OpenAI.UnitTests\Connectors.OpenAI.UnitTests.csproj", "{36DDC119-C030-407E-AC51-A877E9E0F660}"
 EndProject
@@ -809,6 +811,12 @@ Global
 		{385A8FE5-87E2-4458-AE09-35E10BD2E67F}.Publish|Any CPU.Build.0 = Debug|Any CPU
 		{385A8FE5-87E2-4458-AE09-35E10BD2E67F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{385A8FE5-87E2-4458-AE09-35E10BD2E67F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AD9ECE32-088A-49D8-8ACB-890E79F1E7B8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AD9ECE32-088A-49D8-8ACB-890E79F1E7B8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AD9ECE32-088A-49D8-8ACB-890E79F1E7B8}.Publish|Any CPU.ActiveCfg = Debug|Any CPU
+		{AD9ECE32-088A-49D8-8ACB-890E79F1E7B8}.Publish|Any CPU.Build.0 = Debug|Any CPU
+		{AD9ECE32-088A-49D8-8ACB-890E79F1E7B8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AD9ECE32-088A-49D8-8ACB-890E79F1E7B8}.Release|Any CPU.Build.0 = Release|Any CPU
 		{36DDC119-C030-407E-AC51-A877E9E0F660}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{36DDC119-C030-407E-AC51-A877E9E0F660}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{36DDC119-C030-407E-AC51-A877E9E0F660}.Publish|Any CPU.ActiveCfg = Debug|Any CPU
@@ -937,6 +945,7 @@ Global
 		{2918478E-BC86-4D53-9D01-9C318F80C14F} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C}
 		{E06818E3-00A5-41AC-97ED-9491070CDEA1} = {5D4C0700-BBB5-418F-A7B2-F392B9A18263}
 		{385A8FE5-87E2-4458-AE09-35E10BD2E67F} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C}
+		{AD9ECE32-088A-49D8-8ACB-890E79F1E7B8} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C}
 		{36DDC119-C030-407E-AC51-A877E9E0F660} = {1B4CBDE0-10C2-4E7D-9CD0-FE7586C96ED1}
 		{7AAD7388-307D-41FB-B80A-EF9E3A4E31F0} = {1B4CBDE0-10C2-4E7D-9CD0-FE7586C96ED1}
 		{8CF06B22-50F3-4F71-A002-622DB49DF0F5} = {1B4CBDE0-10C2-4E7D-9CD0-FE7586C96ED1}

dotnet/src/Connectors/Connectors.Memory.Weaviate/Connectors.Memory.Weaviate.csproj

@@ -26,4 +26,8 @@
     <ProjectReference Include="..\..\SemanticKernel.Core\SemanticKernel.Core.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <InternalsVisibleTo Include="SemanticKernel.Connectors.Weaviate.UnitTests" />
+  </ItemGroup>
+
 </Project>

dotnet/src/Connectors/Connectors.Memory.Weaviate/Converters/WeaviateDateTimeOffsetConverter.cs

@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Globalization;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.SemanticKernel.Connectors.Weaviate;
+
+/// <summary>
+/// Converts datetime type to RFC 3339 formatted string.
+/// </summary>
+internal sealed class WeaviateDateTimeOffsetConverter : JsonConverter<DateTimeOffset>
+{
+    private const string DateTimeFormat = "yyyy-MM-dd'T'HH:mm:ss.fffK";
+
+    public override DateTimeOffset Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        var dateString = reader.GetString();
+
+        if (string.IsNullOrWhiteSpace(dateString))
+        {
+            return default;
+        }
+
+        return DateTimeOffset.Parse(dateString, CultureInfo.InvariantCulture);
+    }
+
+    public override void Write(Utf8JsonWriter writer, DateTimeOffset value, JsonSerializerOptions options)
+    {
+        writer.WriteStringValue(value.ToString(DateTimeFormat, CultureInfo.InvariantCulture));
+    }
+}

dotnet/src/Connectors/Connectors.Memory.Weaviate/Converters/WeaviateNullableDateTimeOffsetConverter.cs

@@ -0,0 +1,45 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Globalization;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.SemanticKernel.Connectors.Weaviate;
+
+/// <summary>
+/// Converts datetime type to RFC 3339 formatted string.
+/// </summary>
+internal sealed class WeaviateNullableDateTimeOffsetConverter : JsonConverter<DateTimeOffset?>
+{
+    private const string DateTimeFormat = "yyyy-MM-dd'T'HH:mm:ss.fffK";
+
+    public override DateTimeOffset? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        if (reader.TokenType == JsonTokenType.Null)
+        {
+            return null;
+        }
+
+        var dateString = reader.GetString();
+
+        if (string.IsNullOrWhiteSpace(dateString))
+        {
+            return null;
+        }
+
+        return DateTimeOffset.Parse(dateString, CultureInfo.InvariantCulture);
+    }
+
+    public override void Write(Utf8JsonWriter writer, DateTimeOffset? value, JsonSerializerOptions options)
+    {
+        if (value.HasValue)
+        {
+            writer.WriteStringValue(value.Value.ToString(DateTimeFormat, CultureInfo.InvariantCulture));
+        }
+        else
+        {
+            writer.WriteNullValue();
+        }
+    }
+}

dotnet/src/Connectors/Connectors.Memory.Weaviate/Http/HttpRequest.cs

@@ -33,9 +33,20 @@ public static HttpRequestMessage CreatePostRequest(string url, object? payload =
         };
     }
 
-    public static HttpRequestMessage CreateDeleteRequest(string url)
+    public static HttpRequestMessage CreateDeleteRequest(string url, object? payload = null)
     {
-        return new(HttpMethod.Delete, url);
+        return new(HttpMethod.Delete, url)
+        {
+            Content = GetJsonContent(payload)
+        };
+    }
+
+    public static HttpRequestMessage CreatePutRequest(string url, object? payload = null)
+    {
+        return new(HttpMethod.Put, url)
+        {
+            Content = GetJsonContent(payload)
+        };
     }
 
     private static StringContent? GetJsonContent(object? payload)

dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateCreateCollectionSchemaRequest.cs

@@ -0,0 +1,36 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.SemanticKernel.Connectors.Weaviate;
+
+internal sealed class WeaviateCreateCollectionSchemaRequest
+{
+    private const string ApiRoute = "schema";
+
+    [JsonConstructor]
+    public WeaviateCreateCollectionSchemaRequest() { }
+
+    public WeaviateCreateCollectionSchemaRequest(WeaviateCollectionSchema collectionSchema)
+    {
+        this.CollectionName = collectionSchema.CollectionName;
+        this.VectorConfigurations = collectionSchema.VectorConfigurations;
+        this.Properties = collectionSchema.Properties;
+    }
+
+    [JsonPropertyName("class")]
+    public string? CollectionName { get; set; }
+
+    [JsonPropertyName("vectorConfig")]
+    public Dictionary<string, WeaviateCollectionSchemaVectorConfig>? VectorConfigurations { get; set; }
+
+    [JsonPropertyName("properties")]
+    public List<WeaviateCollectionSchemaProperty>? Properties { get; set; }
+
+    public HttpRequestMessage Build()
+    {
+        return HttpRequest.CreatePostRequest(ApiRoute, this);
+    }
+}

dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateDeleteCollectionSchemaRequest.cs

@@ -0,0 +1,19 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Net.Http;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.SemanticKernel.Connectors.Weaviate;
+
+internal sealed class WeaviateDeleteCollectionSchemaRequest(string collectionName)
+{
+    private const string ApiRoute = "schema";
+
+    [JsonIgnore]
+    public string CollectionName { get; set; } = collectionName;
+
+    public HttpRequestMessage Build()
+    {
+        return HttpRequest.CreateDeleteRequest($"{ApiRoute}/{this.CollectionName}");
+    }
+}

dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateDeleteObjectBatchRequest.cs

@@ -0,0 +1,27 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Net.Http;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.SemanticKernel.Connectors.Weaviate;
+
+internal sealed class WeaviateDeleteObjectBatchRequest
+{
+    private const string ApiRoute = "batch/objects";
+
+    [JsonConstructor]
+    public WeaviateDeleteObjectBatchRequest() { }
+
+    public WeaviateDeleteObjectBatchRequest(WeaviateQueryMatch match)
+    {
+        this.Match = match;
+    }
+
+    [JsonPropertyName("match")]
+    public WeaviateQueryMatch? Match { get; set; }
+
+    public HttpRequestMessage Build()
+    {
+        return HttpRequest.CreateDeleteRequest(ApiRoute, this);
+    }
+}

dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateDeleteObjectRequest.cs

@@ -0,0 +1,23 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Net.Http;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.SemanticKernel.Connectors.Weaviate;
+
+internal sealed class WeaviateDeleteObjectRequest(string collectionName, Guid id)
+{
+    private const string ApiRoute = "objects";
+
+    [JsonIgnore]
+    public string CollectionName { get; set; } = collectionName;
+
+    [JsonIgnore]
+    public Guid Id { get; set; } = id;
+
+    public HttpRequestMessage Build()
+    {
+        return HttpRequest.CreateDeleteRequest($"{ApiRoute}/{this.CollectionName}/{this.Id}");
+    }
+}

dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionObjectRequest.cs

@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Net.Http;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.SemanticKernel.Connectors.Weaviate;
+
+internal sealed class WeaviateGetCollectionObjectRequest(string collectionName, Guid id, bool includeVectors)
+{
+    private const string ApiRoute = "objects";
+    private const string IncludeQueryParameterName = "include";
+    private const string IncludeVectorQueryParameterValue = "vector";
+
+    [JsonIgnore]
+    public string CollectionName { get; set; } = collectionName;
+
+    [JsonIgnore]
+    public Guid Id { get; set; } = id;
+
+    [JsonIgnore]
+    public bool IncludeVectors { get; set; } = includeVectors;
+
+    public HttpRequestMessage Build()
+    {
+        var uri = $"{ApiRoute}/{this.CollectionName}/{this.Id}";
+
+        if (this.IncludeVectors)
+        {
+            uri += $"?{IncludeQueryParameterName}={IncludeVectorQueryParameterValue}";
+        }
+
+        return HttpRequest.CreateGetRequest(uri);
+    }
+}

dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionSchemaRequest.cs

@@ -0,0 +1,19 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Net.Http;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.SemanticKernel.Connectors.Weaviate;
+
+internal sealed class WeaviateGetCollectionSchemaRequest(string collectionName)
+{
+    private const string ApiRoute = "schema";
+
+    [JsonIgnore]
+    public string CollectionName { get; set; } = collectionName;
+
+    public HttpRequestMessage Build()
+    {
+        return HttpRequest.CreateGetRequest($"{ApiRoute}/{this.CollectionName}");
+    }
+}

dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionSchemaResponse.cs

@@ -0,0 +1,11 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Text.Json.Serialization;
+
+namespace Microsoft.SemanticKernel.Connectors.Weaviate;
+
+internal sealed class WeaviateGetCollectionSchemaResponse
+{
+    [JsonPropertyName("class")]
+    public string? CollectionName { get; set; }
+}

dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionsRequest.cs

@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Net.Http;
+
+namespace Microsoft.SemanticKernel.Connectors.Weaviate;
+
+internal sealed class WeaviateGetCollectionsRequest
+{
+    private const string ApiRoute = "schema";
+
+    public HttpRequestMessage Build()
+    {
+        return HttpRequest.CreateGetRequest(ApiRoute, this);
+    }
+}

dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionsResponse.cs

@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.SemanticKernel.Connectors.Weaviate;
+
+internal sealed class WeaviateGetCollectionsResponse
+{
+    [JsonPropertyName("classes")]
+    public List<WeaviateCollectionSchema>? Collections { get; set; }
+}

dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateUpsertCollectionObjectBatchRequest.cs

@@ -0,0 +1,32 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.SemanticKernel.Connectors.Weaviate;
+
+internal sealed class WeaviateUpsertCollectionObjectBatchRequest
+{
+    private const string ApiRoute = "batch/objects";
+
+    [JsonConstructor]
+    public WeaviateUpsertCollectionObjectBatchRequest() { }
+
+    public WeaviateUpsertCollectionObjectBatchRequest(List<JsonNode> collectionObjects)
+    {
+        this.CollectionObjects = collectionObjects;
+    }
+
+    [JsonPropertyName("fields")]
+    public List<string> Fields { get; set; } = [WeaviateConstants.ReservedKeyPropertyName];
+
+    [JsonPropertyName("objects")]
+    public List<JsonNode>? CollectionObjects { get; set; }
+
+    public HttpRequestMessage Build()
+    {
+        return HttpRequest.CreatePostRequest(ApiRoute, this);
+    }
+}