diff --git a/dotnet/samples/KernelSyntaxExamples/Example85_AgentCharts.cs b/dotnet/samples/KernelSyntaxExamples/Example85_AgentCharts.cs
new file mode 100644
index 000000000000..848c702ec3cb
--- /dev/null
+++ b/dotnet/samples/KernelSyntaxExamples/Example85_AgentCharts.cs
@@ -0,0 +1,105 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Threading.Tasks;
+using Microsoft.SemanticKernel.Connectors.OpenAI;
+using Microsoft.SemanticKernel.Experimental.Agents;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Examples;
+
+// ReSharper disable once InconsistentNaming
+///
+/// Showcase usage of code_interpreter and retrieval tools.
+///
+public sealed class Example85_AgentCharts : BaseTest
+{
+ ///
+ /// Specific model is required that supports agents and parallel function calling.
+ /// Currently this is limited to Open AI hosted services.
+ ///
+ private const string OpenAIFunctionEnabledModel = "gpt-4-1106-preview";
+
+ ///
+ /// Create a chart and retrieve by file_id.
+ ///
+ [Fact(Skip = "Launches external processes")]
+ public async Task CreateChartAsync()
+ {
+ this.WriteLine("======== Using CodeInterpreter tool ========");
+
+ if (TestConfiguration.OpenAI.ApiKey == null)
+ {
+ this.WriteLine("OpenAI apiKey not found. Skipping example.");
+ return;
+ }
+
+ this.WriteLine(Environment.CurrentDirectory);
+
+ var fileService = new OpenAIFileService(TestConfiguration.OpenAI.ApiKey);
+
+ var agent =
+ await new AgentBuilder()
+ .WithOpenAIChatCompletion(OpenAIFunctionEnabledModel, TestConfiguration.OpenAI.ApiKey)
+ .WithCodeInterpreter()
+ .BuildAsync();
+
+ try
+ {
+ var thread = await agent.NewThreadAsync();
+
+ await InvokeAgentAsync(
+ thread,
+ "1-first", @"
+Display this data using a bar-chart:
+
+Banding Brown Pink Yellow Sum
+X00000 339 433 126 898
+X00300 48 421 222 691
+X12345 16 395 352 763
+Others 23 373 156 552
+Sum 426 1622 856 2904
+");
+ await InvokeAgentAsync(thread, "2-colors", "Can you regenerate this same chart using the category names as the bar colors?");
+ await InvokeAgentAsync(thread, "3-line", "Can you regenerate this as a line chart?");
+ }
+ finally
+ {
+ await agent.DeleteAsync();
+ }
+
+ async Task InvokeAgentAsync(IAgentThread thread, string imageName, string question)
+ {
+ await foreach (var message in thread.InvokeAsync(agent, question))
+ {
+ if (message.ContentType == ChatMessageType.Image)
+ {
+ var filename = $"{imageName}.jpg";
+ var content = fileService.GetFileContent(message.Content);
+ await using var outputStream = File.OpenWrite(filename);
+ await using var inputStream = await content.GetStreamAsync();
+ await inputStream.CopyToAsync(outputStream);
+ var path = Path.Combine(Environment.CurrentDirectory, filename);
+ this.WriteLine($"# {message.Role}: {path}");
+ Process.Start(
+ new ProcessStartInfo
+ {
+ FileName = "cmd.exe",
+ Arguments = $"/C start {path}"
+ });
+ }
+ else
+ {
+ this.WriteLine($"# {message.Role}: {message.Content}");
+ }
+ }
+
+ this.WriteLine();
+ }
+ }
+
+ public Example85_AgentCharts(ITestOutputHelper output) : base(output) { }
+}
diff --git a/dotnet/src/Experimental/Agents/IChatMessage.cs b/dotnet/src/Experimental/Agents/IChatMessage.cs
index e722f73df3ec..1eb2e63da6de 100644
--- a/dotnet/src/Experimental/Agents/IChatMessage.cs
+++ b/dotnet/src/Experimental/Agents/IChatMessage.cs
@@ -5,6 +5,22 @@
namespace Microsoft.SemanticKernel.Experimental.Agents;
+///
+/// $$$
+///
+public enum ChatMessageType
+{
+ ///
+ /// $$$
+ ///
+ Text,
+
+ ///
+ /// $$$
+ ///
+ Image,
+}
+
///
/// Represents a message that is part of an agent thread.
///
@@ -20,6 +36,11 @@ public interface IChatMessage
///
string? AgentId { get; }
+ ///
+ /// $$$
+ ///
+ ChatMessageType ContentType { get; }
+
///
/// The chat message content.
///
diff --git a/dotnet/src/Experimental/Agents/Internal/ChatMessage.cs b/dotnet/src/Experimental/Agents/Internal/ChatMessage.cs
index fa3115554094..09e1d86ac8b1 100644
--- a/dotnet/src/Experimental/Agents/Internal/ChatMessage.cs
+++ b/dotnet/src/Experimental/Agents/Internal/ChatMessage.cs
@@ -1,5 +1,6 @@
// Copyright (c) Microsoft. All rights reserved.
+using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Linq;
@@ -19,6 +20,9 @@ internal sealed class ChatMessage : IChatMessage
///
public string? AgentId { get; }
+ ///
+ public ChatMessageType ContentType { get; }
+
///
public string Content { get; }
@@ -36,13 +40,17 @@ internal sealed class ChatMessage : IChatMessage
internal ChatMessage(ThreadMessageModel model)
{
var content = model.Content.First();
- var text = content.Text?.Value ?? string.Empty;
- this.Annotations = content.Text!.Annotations.Select(a => new Annotation(a.Text, a.StartIndex, a.EndIndex, a.FileCitation?.FileId ?? a.FilePath!.FileId, a.FileCitation?.Quote)).ToArray();
+
+ this.Annotations =
+ content.Text == null ?
+ Array.Empty() :
+ content.Text.Annotations.Select(a => new Annotation(a.Text, a.StartIndex, a.EndIndex, a.FileCitation?.FileId ?? a.FilePath!.FileId, a.FileCitation?.Quote)).ToArray();
this.Id = model.Id;
this.AgentId = string.IsNullOrWhiteSpace(model.AssistantId) ? null : model.AssistantId;
this.Role = model.Role;
- this.Content = text;
+ this.ContentType = content.Text == null ? ChatMessageType.Image : ChatMessageType.Text;
+ this.Content = content.Text?.Value ?? content.Image?.FileId ?? string.Empty;
this.Properties = new ReadOnlyDictionary(model.Metadata);
}
diff --git a/dotnet/src/Experimental/Agents/Models/ThreadMessageModel.cs b/dotnet/src/Experimental/Agents/Models/ThreadMessageModel.cs
index bb13c14f9b98..25156680370f 100644
--- a/dotnet/src/Experimental/Agents/Models/ThreadMessageModel.cs
+++ b/dotnet/src/Experimental/Agents/Models/ThreadMessageModel.cs
@@ -88,6 +88,12 @@ public sealed class ContentModel
[JsonPropertyName("type")]
public string Type { get; set; } = string.Empty;
+ ///
+ /// Text context.
+ ///
+ [JsonPropertyName("image_file")]
+ public ImageContentModel? Image { get; set; }
+
///
/// Text context.
///
@@ -95,6 +101,18 @@ public sealed class ContentModel
public TextContentModel? Text { get; set; }
}
+ ///
+ /// Text content.
+ ///
+ public sealed class ImageContentModel
+ {
+ ///
+ /// The image file identifier.
+ ///
+ [JsonPropertyName("file_id")]
+ public string FileId { get; set; } = string.Empty;
+ }
+
///
/// Text content.
///