Merge branch 'main' into feature_agent_template

dotnet/src/IntegrationTests/Connectors/AzureOpenAI/AzureOpenAIChatCompletion_AutoFunctionChoiceBehaviorTests.cs

@@ -7,6 +7,7 @@
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Azure.Identity;
 using Microsoft.Extensions.Configuration;
 using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.ChatCompletion;
@@ -323,7 +324,6 @@ private Kernel InitializeKernel()
         var azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get<AzureOpenAIConfiguration>();
         Assert.NotNull(azureOpenAIConfiguration);
         Assert.NotNull(azureOpenAIConfiguration.ChatDeploymentName);
-        Assert.NotNull(azureOpenAIConfiguration.ApiKey);
         Assert.NotNull(azureOpenAIConfiguration.Endpoint);
 
         var kernelBuilder = base.CreateKernelBuilder();
@@ -332,7 +332,7 @@ private Kernel InitializeKernel()
             deploymentName: azureOpenAIConfiguration.ChatDeploymentName,
             modelId: azureOpenAIConfiguration.ChatModelId,
             endpoint: azureOpenAIConfiguration.Endpoint,
-            apiKey: azureOpenAIConfiguration.ApiKey);
+            credentials: new AzureCliCredential());
 
         return kernelBuilder.Build();
     }

dotnet/src/IntegrationTests/Connectors/AzureOpenAI/AzureOpenAIChatCompletion_NoneFunctionChoiceBehaviorTests.cs

@@ -6,14 +6,14 @@
 using System.Globalization;
 using System.Text;
 using System.Threading.Tasks;
+using Azure.Identity;
 using Microsoft.Extensions.Configuration;
 using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.Connectors.AzureOpenAI;
-using SemanticKernel.IntegrationTests.Connectors.AzureOpenAI;
 using SemanticKernel.IntegrationTests.TestSettings;
 using Xunit;
 
-namespace SemanticKernel.IntegrationTests.Connectors.OpenAI;
+namespace SemanticKernel.IntegrationTests.Connectors.AzureOpenAI;
 
 public sealed class AzureOpenAINoneFunctionChoiceBehaviorTests : BaseIntegrationTest
 {
@@ -165,7 +165,6 @@ private Kernel InitializeKernel()
         var azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get<AzureOpenAIConfiguration>();
         Assert.NotNull(azureOpenAIConfiguration);
         Assert.NotNull(azureOpenAIConfiguration.ChatDeploymentName);
-        Assert.NotNull(azureOpenAIConfiguration.ApiKey);
         Assert.NotNull(azureOpenAIConfiguration.Endpoint);
 
         var kernelBuilder = base.CreateKernelBuilder();
@@ -174,7 +173,7 @@ private Kernel InitializeKernel()
             deploymentName: azureOpenAIConfiguration.ChatDeploymentName,
             modelId: azureOpenAIConfiguration.ChatModelId,
             endpoint: azureOpenAIConfiguration.Endpoint,
-            apiKey: azureOpenAIConfiguration.ApiKey);
+            credentials: new AzureCliCredential());
 
         return kernelBuilder.Build();
     }

dotnet/src/IntegrationTests/Connectors/AzureOpenAI/AzureOpenAIChatCompletion_RequiredFunctionChoiceBehaviorTests.cs

@@ -6,6 +6,7 @@
 using System.Globalization;
 using System.Linq;
 using System.Threading.Tasks;
+using Azure.Identity;
 using Microsoft.Extensions.Configuration;
 using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.ChatCompletion;
@@ -409,7 +410,6 @@ private Kernel InitializeKernel()
         var azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get<AzureOpenAIConfiguration>();
         Assert.NotNull(azureOpenAIConfiguration);
         Assert.NotNull(azureOpenAIConfiguration.ChatDeploymentName);
-        Assert.NotNull(azureOpenAIConfiguration.ApiKey);
         Assert.NotNull(azureOpenAIConfiguration.Endpoint);
 
         var kernelBuilder = base.CreateKernelBuilder();
@@ -418,7 +418,7 @@ private Kernel InitializeKernel()
             deploymentName: azureOpenAIConfiguration.ChatDeploymentName,
             modelId: azureOpenAIConfiguration.ChatModelId,
             endpoint: azureOpenAIConfiguration.Endpoint,
-            apiKey: azureOpenAIConfiguration.ApiKey);
+            credentials: new AzureCliCredential());
 
         return kernelBuilder.Build();
     }

dotnet/src/IntegrationTests/Planners/PlanTests.cs

@@ -569,14 +569,14 @@ private Kernel InitializeKernel(bool useEmbeddings = false, bool useChatModel =
             c.AddAzureOpenAIChatCompletion(
                 deploymentName: azureOpenAIConfiguration.ChatDeploymentName!,
                 endpoint: azureOpenAIConfiguration.Endpoint,
-                apiKey: azureOpenAIConfiguration.ApiKey);
+                credentials: new AzureCliCredential());
         }
         else
         {
             c.AddAzureOpenAITextGeneration(
                 deploymentName: azureOpenAIConfiguration.DeploymentName,
                 endpoint: azureOpenAIConfiguration.Endpoint,
-                apiKey: azureOpenAIConfiguration.ApiKey);
+                credentials: new AzureCliCredential());
         }
 
         if (useEmbeddings)

dotnet/src/IntegrationTests/Planners/SequentialPlanner/SequentialPlanParserTests.cs

@@ -34,7 +34,7 @@ public void CanCallToPlanFromXml()
             .WithAzureOpenAITextGeneration(
                 deploymentName: azureOpenAIConfiguration.DeploymentName,
                 endpoint: azureOpenAIConfiguration.Endpoint,
-                apiKey: azureOpenAIConfiguration.ApiKey,
+                credentials: new AzureCliCredential(),
                 serviceId: azureOpenAIConfiguration.ServiceId)
             .Build();
         kernel.ImportPluginFromType<EmailPluginFake>("email");

dotnet/src/IntegrationTests/Planners/SequentialPlanner/SequentialPlannerTests.cs

@@ -121,14 +121,14 @@ private Kernel InitializeKernel(bool useEmbeddings = false, bool useChatModel =
             builder.Services.AddAzureOpenAIChatCompletion(
                 deploymentName: azureOpenAIConfiguration.ChatDeploymentName!,
                 endpoint: azureOpenAIConfiguration.Endpoint,
-                apiKey: azureOpenAIConfiguration.ApiKey);
+                credentials: new AzureCliCredential());
         }
         else
         {
             builder.Services.AddAzureOpenAITextGeneration(
                 deploymentName: azureOpenAIConfiguration.DeploymentName,
                 endpoint: azureOpenAIConfiguration.Endpoint,
-                apiKey: azureOpenAIConfiguration.ApiKey);
+                credentials: new AzureCliCredential());
         }
 
         if (useEmbeddings)

dotnet/src/IntegrationTests/Planners/StepwisePlanner/FunctionCallingStepwisePlannerTests.cs

@@ -75,7 +75,7 @@ private Kernel InitializeKernel(bool useEmbeddings = false)
             .WithAzureOpenAIChatCompletion(
                 deploymentName: azureOpenAIConfiguration.ChatDeploymentName!,
                 endpoint: azureOpenAIConfiguration.Endpoint,
-                apiKey: azureOpenAIConfiguration.ApiKey);
+                credentials: new AzureCliCredential());
         if (useEmbeddings)
         {
             builder.WithAzureOpenAITextEmbeddingGeneration(

dotnet/src/IntegrationTests/Planners/StepwisePlanner/StepwisePlannerTests.cs

@@ -150,14 +150,14 @@ private Kernel InitializeKernel(bool useEmbeddings = false, bool useChatModel =
             builder.Services.AddAzureOpenAIChatCompletion(
                 deploymentName: azureOpenAIConfiguration.ChatDeploymentName!,
                 endpoint: azureOpenAIConfiguration.Endpoint,
-                apiKey: azureOpenAIConfiguration.ApiKey);
+                credentials: new AzureCliCredential());
         }
         else
         {
             builder.Services.AddAzureOpenAITextGeneration(
                 deploymentName: azureOpenAIConfiguration.DeploymentName,
                 endpoint: azureOpenAIConfiguration.Endpoint,
-                apiKey: azureOpenAIConfiguration.ApiKey);
+                credentials: new AzureCliCredential());
         }
 
         if (useEmbeddings)

dotnet/src/IntegrationTests/README.md

@@ -3,16 +3,29 @@
 ## Requirements
 
 1. **Azure OpenAI**: go to the [Azure OpenAI Quickstart](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/quickstart)
-   and deploy an instance of Azure OpenAI, deploy a model like "gpt-35-turbo-instruct" find your Endpoint and API key.
-2. **OpenAI**: go to [OpenAI](https://platform.openai.com) to register and procure your API key.
-3. **HuggingFace API key**: see https://huggingface.co/docs/huggingface_hub/guides/inference for details.
-4. **Azure Bing Web Search API**: go to [Bing Web Search API](https://www.microsoft.com/en-us/bing/apis/bing-web-search-api)
+    1. Deploy the following models:
+        1. `dall-e-3` DALL-E 3 generates images  and is used in Text to Image tests.
+        1. `tts` TTS is a model that converts text to natural sounding speech and is used in Text to Audio tests.
+        1. `whisper` The Whisper models are trained for speech recognition and translation tasks and is used in Audio to Text tests.
+        1. `text-embedding-ada-002` Text Embedding Ada 002 is used in Text Embedding tests.
+        1. `gpt-35-turbo-instruct` GPT-3.5 Turbo Instruct is used in inference tests.
+        1. `gpt-4o` GPT-4o is used in chat completion tests.
+    1. Assign users who are running the integration tests the following roles: `Cognitive Services OpenAI Contributor` and `Cognitive Services OpenAI User`
+    1. Users must [Authenticate to Azure using Azure CLI](https://learn.microsoft.com/en-us/cli/azure/authenticate-azure-cli)
+1. **OpenAI**: go to [OpenAI](https://platform.openai.com) to register and procure your API key.
+1. **HuggingFace API key**: see https://huggingface.co/docs/huggingface_hub/guides/inference for details.
+1. **Azure Bing Web Search API**: go to [Bing Web Search API](https://www.microsoft.com/en-us/bing/apis/bing-web-search-api)
    and select `Try Now` to get started.
-5. **Postgres**: start a postgres with the [pgvector](https://github.com/pgvector/pgvector) extension installed. You can easily do it using the docker image [ankane/pgvector](https://hub.docker.com/r/ankane/pgvector).
-6. **Weaviate**: go to `IntegrationTests/Connectors/Weaviate` where `docker-compose.yml` is located and run `docker-compose up --build`. 
+1. **Postgres**: start a postgres with the [pgvector](https://github.com/pgvector/pgvector) extension installed. You can easily do it using the docker image [ankane/pgvector](https://hub.docker.com/r/ankane/pgvector).
+1. **Weaviate**: go to `IntegrationTests/Connectors/Weaviate` where `docker-compose.yml` is located and run `docker-compose up --build`. 
 
 ## Setup
 
+> [!IMPORTANT]  
+> To run integration tests that depend on Azure OpenAI, you must have the Azure OpenAI models deployed and have the necessary permissions to access them.
+> These test authenticate using [AzureCliCredential](https://learn.microsoft.com/en-us/dotnet/api/azure.identity.azureclicredential?view=azure-dotnet).
+> Users must [Authenticate to Azure using Azure CLI](https://learn.microsoft.com/en-us/cli/azure/authenticate-azure-cli).
+
 ### Option 1: Use Secret Manager
 
 Integration tests will require secrets and credentials, to access OpenAI, Azure OpenAI,

dotnet/src/SemanticKernel.Abstractions/Contents/KernelContent.cs

@@ -9,13 +9,14 @@ namespace Microsoft.SemanticKernel;
 /// <summary>
 /// Base class for all AI non-streaming results
 /// </summary>
-[JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")]
+[JsonPolymorphic(TypeDiscriminatorPropertyName = "$type", UnknownDerivedTypeHandling = JsonUnknownDerivedTypeHandling.FallBackToNearestAncestor)]
 [JsonDerivedType(typeof(TextContent), typeDiscriminator: nameof(TextContent))]
 [JsonDerivedType(typeof(ImageContent), typeDiscriminator: nameof(ImageContent))]
 [JsonDerivedType(typeof(FunctionCallContent), typeDiscriminator: nameof(FunctionCallContent))]
 [JsonDerivedType(typeof(FunctionResultContent), typeDiscriminator: nameof(FunctionResultContent))]
 [JsonDerivedType(typeof(BinaryContent), typeDiscriminator: nameof(BinaryContent))]
 [JsonDerivedType(typeof(AudioContent), typeDiscriminator: nameof(AudioContent))]
+[JsonDerivedType(typeof(ChatMessageContent), typeDiscriminator: nameof(ChatMessageContent))]
 #pragma warning disable SKEXP0110
 [JsonDerivedType(typeof(AnnotationContent), typeDiscriminator: nameof(AnnotationContent))]
 [JsonDerivedType(typeof(FileReferenceContent), typeDiscriminator: nameof(FileReferenceContent))]

dotnet/src/SemanticKernel.UnitTests/Contents/ChatMessageContentTests.cs

@@ -202,7 +202,7 @@ public void ItCanBeSerializeAndDeserialized()
             new FunctionCallContent("function-name", "plugin-name", "function-id", new KernelArguments { ["parameter"] = "argument" }),
             new FunctionResultContent(new FunctionCallContent("function-name", "plugin-name", "function-id"), "function-result"),
             new FileReferenceContent(fileId: "file-id-1") { ModelId = "model-7", Metadata = new Dictionary<string, object?>() { ["metadata-key-7"] = "metadata-value-7" } },
-            new AnnotationContent("quote-8") { ModelId = "model-8", FileId = "file-id-2", StartIndex = 2, EndIndex = 24, Metadata = new Dictionary<string, object?>() { ["metadata-key-8"] = "metadata-value-8" } }
+            new AnnotationContent("quote-8") { ModelId = "model-8", FileId = "file-id-2", StartIndex = 2, EndIndex = 24, Metadata = new Dictionary<string, object?>() { ["metadata-key-8"] = "metadata-value-8" } },
         ];
 
         // Act
@@ -320,4 +320,92 @@ public void ItCanBeSerializeAndDeserialized()
         Assert.Single(annotationContent.Metadata);
         Assert.Equal("metadata-value-8", annotationContent.Metadata["metadata-key-8"]?.ToString());
     }
+
+    [Fact]
+    public void ItCanBePolymorphicallySerializedAndDeserializedAsKernelContentType()
+    {
+        // Arrange
+        KernelContent sut = new ChatMessageContent(AuthorRole.User, "test-content", "test-model", metadata: new Dictionary<string, object?>()
+        {
+            ["test-metadata-key"] = "test-metadata-value"
+        })
+        {
+            MimeType = "test-mime-type"
+        };
+
+        // Act
+        var json = JsonSerializer.Serialize(sut);
+
+        var deserialized = JsonSerializer.Deserialize<KernelContent>(json)!;
+
+        // Assert
+        Assert.IsType<ChatMessageContent>(deserialized);
+        Assert.Equal("test-content", ((ChatMessageContent)deserialized).Content);
+        Assert.Equal("test-model", deserialized.ModelId);
+        Assert.Equal("test-mime-type", deserialized.MimeType);
+        Assert.NotNull(deserialized.Metadata);
+        Assert.Single(deserialized.Metadata);
+        Assert.Equal("test-metadata-value", deserialized.Metadata["test-metadata-key"]?.ToString());
+    }
+
+    [Fact]
+    public void UnknownDerivativeCanBePolymorphicallySerializedAndDeserializedAsChatMessageContentType()
+    {
+        // Arrange
+        KernelContent sut = new UnknownExternalChatMessageContent(AuthorRole.User, "test-content")
+        {
+            MimeType = "test-mime-type",
+        };
+
+        // Act
+        var json = JsonSerializer.Serialize(sut);
+
+        var deserialized = JsonSerializer.Deserialize<KernelContent>(json)!;
+
+        // Assert
+        Assert.IsType<ChatMessageContent>(deserialized);
+        Assert.Equal("test-content", ((ChatMessageContent)deserialized).Content);
+        Assert.Equal("test-mime-type", deserialized.MimeType);
+    }
+
+    [Fact]
+    public void ItCanBeSerializeAndDeserializedWithFunctionResultOfChatMessageType()
+    {
+        // Arrange
+        ChatMessageContentItemCollection items = [
+            new FunctionResultContent(new FunctionCallContent("function-name-1", "plugin-name-1", "function-id-1"), new ChatMessageContent(AuthorRole.User, "test-content-1")),
+            new FunctionResultContent(new FunctionCallContent("function-name-2", "plugin-name-2", "function-id-2"), new UnknownExternalChatMessageContent(AuthorRole.Assistant, "test-content-2")),
+        ];
+
+        // Act
+        var chatMessageJson = JsonSerializer.Serialize(new ChatMessageContent(AuthorRole.User, items: items, "message-model"));
+
+        var deserializedMessage = JsonSerializer.Deserialize<ChatMessageContent>(chatMessageJson)!;
+
+        // Assert
+        var functionResultContentWithResultOfChatMessageContentType = deserializedMessage.Items[0] as FunctionResultContent;
+        Assert.NotNull(functionResultContentWithResultOfChatMessageContentType);
+        Assert.Equal("function-name-1", functionResultContentWithResultOfChatMessageContentType.FunctionName);
+        Assert.Equal("function-id-1", functionResultContentWithResultOfChatMessageContentType.CallId);
+        Assert.Equal("plugin-name-1", functionResultContentWithResultOfChatMessageContentType.PluginName);
+        var chatMessageContent = Assert.IsType<JsonElement>(functionResultContentWithResultOfChatMessageContentType.Result);
+        Assert.Equal("user", chatMessageContent.GetProperty("Role").GetProperty("Label").GetString());
+        Assert.Equal("test-content-1", chatMessageContent.GetProperty("Items")[0].GetProperty("Text").GetString());
+
+        var functionResultContentWithResultOfUnknownChatMessageContentType = deserializedMessage.Items[1] as FunctionResultContent;
+        Assert.NotNull(functionResultContentWithResultOfUnknownChatMessageContentType);
+        Assert.Equal("function-name-2", functionResultContentWithResultOfUnknownChatMessageContentType.FunctionName);
+        Assert.Equal("function-id-2", functionResultContentWithResultOfUnknownChatMessageContentType.CallId);
+        Assert.Equal("plugin-name-2", functionResultContentWithResultOfUnknownChatMessageContentType.PluginName);
+        var unknownChatMessageContent = Assert.IsType<JsonElement>(functionResultContentWithResultOfUnknownChatMessageContentType.Result);
+        Assert.Equal("assistant", unknownChatMessageContent.GetProperty("Role").GetProperty("Label").GetString());
+        Assert.Equal("test-content-2", unknownChatMessageContent.GetProperty("Items")[0].GetProperty("Text").GetString());
+    }
+
+    private sealed class UnknownExternalChatMessageContent : ChatMessageContent
+    {
+        public UnknownExternalChatMessageContent(AuthorRole role, string? content) : base(role, content)
+        {
+        }
+    }
 }