diff --git a/dotnet/samples/KernelSyntaxExamples/Example85_AgentCharts.cs b/dotnet/samples/KernelSyntaxExamples/Example85_AgentCharts.cs new file mode 100644 index 000000000000..848c702ec3cb --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Example85_AgentCharts.cs @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics; +using System.IO; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Connectors.OpenAI; +using Microsoft.SemanticKernel.Experimental.Agents; +using Xunit; +using Xunit.Abstractions; + +namespace Examples; + +// ReSharper disable once InconsistentNaming +/// +/// Showcase usage of code_interpreter and retrieval tools. +/// +public sealed class Example85_AgentCharts : BaseTest +{ + /// + /// Specific model is required that supports agents and parallel function calling. + /// Currently this is limited to Open AI hosted services. + /// + private const string OpenAIFunctionEnabledModel = "gpt-4-1106-preview"; + + /// + /// Create a chart and retrieve by file_id. + /// + [Fact(Skip = "Launches external processes")] + public async Task CreateChartAsync() + { + this.WriteLine("======== Using CodeInterpreter tool ========"); + + if (TestConfiguration.OpenAI.ApiKey == null) + { + this.WriteLine("OpenAI apiKey not found. Skipping example."); + return; + } + + this.WriteLine(Environment.CurrentDirectory); + + var fileService = new OpenAIFileService(TestConfiguration.OpenAI.ApiKey); + + var agent = + await new AgentBuilder() + .WithOpenAIChatCompletion(OpenAIFunctionEnabledModel, TestConfiguration.OpenAI.ApiKey) + .WithCodeInterpreter() + .BuildAsync(); + + try + { + var thread = await agent.NewThreadAsync(); + + await InvokeAgentAsync( + thread, + "1-first", @" +Display this data using a bar-chart: + +Banding Brown Pink Yellow Sum +X00000 339 433 126 898 +X00300 48 421 222 691 +X12345 16 395 352 763 +Others 23 373 156 552 +Sum 426 1622 856 2904 +"); + await InvokeAgentAsync(thread, "2-colors", "Can you regenerate this same chart using the category names as the bar colors?"); + await InvokeAgentAsync(thread, "3-line", "Can you regenerate this as a line chart?"); + } + finally + { + await agent.DeleteAsync(); + } + + async Task InvokeAgentAsync(IAgentThread thread, string imageName, string question) + { + await foreach (var message in thread.InvokeAsync(agent, question)) + { + if (message.ContentType == ChatMessageType.Image) + { + var filename = $"{imageName}.jpg"; + var content = fileService.GetFileContent(message.Content); + await using var outputStream = File.OpenWrite(filename); + await using var inputStream = await content.GetStreamAsync(); + await inputStream.CopyToAsync(outputStream); + var path = Path.Combine(Environment.CurrentDirectory, filename); + this.WriteLine($"# {message.Role}: {path}"); + Process.Start( + new ProcessStartInfo + { + FileName = "cmd.exe", + Arguments = $"/C start {path}" + }); + } + else + { + this.WriteLine($"# {message.Role}: {message.Content}"); + } + } + + this.WriteLine(); + } + } + + public Example85_AgentCharts(ITestOutputHelper output) : base(output) { } +} diff --git a/dotnet/src/Experimental/Agents/IChatMessage.cs b/dotnet/src/Experimental/Agents/IChatMessage.cs index e722f73df3ec..1eb2e63da6de 100644 --- a/dotnet/src/Experimental/Agents/IChatMessage.cs +++ b/dotnet/src/Experimental/Agents/IChatMessage.cs @@ -5,6 +5,22 @@ namespace Microsoft.SemanticKernel.Experimental.Agents; +/// +/// $$$ +/// +public enum ChatMessageType +{ + /// + /// $$$ + /// + Text, + + /// + /// $$$ + /// + Image, +} + /// /// Represents a message that is part of an agent thread. /// @@ -20,6 +36,11 @@ public interface IChatMessage /// string? AgentId { get; } + /// + /// $$$ + /// + ChatMessageType ContentType { get; } + /// /// The chat message content. /// diff --git a/dotnet/src/Experimental/Agents/Internal/ChatMessage.cs b/dotnet/src/Experimental/Agents/Internal/ChatMessage.cs index fa3115554094..09e1d86ac8b1 100644 --- a/dotnet/src/Experimental/Agents/Internal/ChatMessage.cs +++ b/dotnet/src/Experimental/Agents/Internal/ChatMessage.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Linq; @@ -19,6 +20,9 @@ internal sealed class ChatMessage : IChatMessage /// public string? AgentId { get; } + /// + public ChatMessageType ContentType { get; } + /// public string Content { get; } @@ -36,13 +40,17 @@ internal sealed class ChatMessage : IChatMessage internal ChatMessage(ThreadMessageModel model) { var content = model.Content.First(); - var text = content.Text?.Value ?? string.Empty; - this.Annotations = content.Text!.Annotations.Select(a => new Annotation(a.Text, a.StartIndex, a.EndIndex, a.FileCitation?.FileId ?? a.FilePath!.FileId, a.FileCitation?.Quote)).ToArray(); + + this.Annotations = + content.Text == null ? + Array.Empty() : + content.Text.Annotations.Select(a => new Annotation(a.Text, a.StartIndex, a.EndIndex, a.FileCitation?.FileId ?? a.FilePath!.FileId, a.FileCitation?.Quote)).ToArray(); this.Id = model.Id; this.AgentId = string.IsNullOrWhiteSpace(model.AssistantId) ? null : model.AssistantId; this.Role = model.Role; - this.Content = text; + this.ContentType = content.Text == null ? ChatMessageType.Image : ChatMessageType.Text; + this.Content = content.Text?.Value ?? content.Image?.FileId ?? string.Empty; this.Properties = new ReadOnlyDictionary(model.Metadata); } diff --git a/dotnet/src/Experimental/Agents/Models/ThreadMessageModel.cs b/dotnet/src/Experimental/Agents/Models/ThreadMessageModel.cs index bb13c14f9b98..25156680370f 100644 --- a/dotnet/src/Experimental/Agents/Models/ThreadMessageModel.cs +++ b/dotnet/src/Experimental/Agents/Models/ThreadMessageModel.cs @@ -88,6 +88,12 @@ public sealed class ContentModel [JsonPropertyName("type")] public string Type { get; set; } = string.Empty; + /// + /// Text context. + /// + [JsonPropertyName("image_file")] + public ImageContentModel? Image { get; set; } + /// /// Text context. /// @@ -95,6 +101,18 @@ public sealed class ContentModel public TextContentModel? Text { get; set; } } + /// + /// Text content. + /// + public sealed class ImageContentModel + { + /// + /// The image file identifier. + /// + [JsonPropertyName("file_id")] + public string FileId { get; set; } = string.Empty; + } + /// /// Text content. ///