forked from microsoft/semantic-kernel
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
.Net Add image support for Agent responses (microsoft#5113)
### Motivation and Context <!-- Thank you for your contribution to the semantic-kernel repo! Please help reviewers and future users, providing the following information: 1. Why is this change required? 2. What problem does it solve? 3. What scenario does it contribute to? 4. If it fixes an open issue, please link to the issue here. --> Update model and message to support image response. ### Description <!-- Describe your changes, the overall approach, the underlying design. These notes will help understanding how your code works. Thanks! --> With code-interpreter tool enabled, the need to support image based responses is critical. ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone 😄
- Loading branch information
Showing
4 changed files
with
155 additions
and
3 deletions.
There are no files selected for viewing
105 changes: 105 additions & 0 deletions
105
dotnet/samples/KernelSyntaxExamples/Example85_AgentCharts.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,105 @@ | ||
// Copyright (c) Microsoft. All rights reserved. | ||
|
||
using System; | ||
using System.Diagnostics; | ||
using System.IO; | ||
using System.Threading.Tasks; | ||
using Microsoft.SemanticKernel.Connectors.OpenAI; | ||
using Microsoft.SemanticKernel.Experimental.Agents; | ||
using Xunit; | ||
using Xunit.Abstractions; | ||
|
||
namespace Examples; | ||
|
||
// ReSharper disable once InconsistentNaming | ||
/// <summary> | ||
/// Showcase usage of code_interpreter and retrieval tools. | ||
/// </summary> | ||
public sealed class Example85_AgentCharts : BaseTest | ||
{ | ||
/// <summary> | ||
/// Specific model is required that supports agents and parallel function calling. | ||
/// Currently this is limited to Open AI hosted services. | ||
/// </summary> | ||
private const string OpenAIFunctionEnabledModel = "gpt-4-1106-preview"; | ||
|
||
/// <summary> | ||
/// Create a chart and retrieve by file_id. | ||
/// </summary> | ||
[Fact(Skip = "Launches external processes")] | ||
public async Task CreateChartAsync() | ||
{ | ||
this.WriteLine("======== Using CodeInterpreter tool ========"); | ||
|
||
if (TestConfiguration.OpenAI.ApiKey == null) | ||
{ | ||
this.WriteLine("OpenAI apiKey not found. Skipping example."); | ||
return; | ||
} | ||
|
||
this.WriteLine(Environment.CurrentDirectory); | ||
|
||
var fileService = new OpenAIFileService(TestConfiguration.OpenAI.ApiKey); | ||
|
||
var agent = | ||
await new AgentBuilder() | ||
.WithOpenAIChatCompletion(OpenAIFunctionEnabledModel, TestConfiguration.OpenAI.ApiKey) | ||
.WithCodeInterpreter() | ||
.BuildAsync(); | ||
|
||
try | ||
{ | ||
var thread = await agent.NewThreadAsync(); | ||
|
||
await InvokeAgentAsync( | ||
thread, | ||
"1-first", @" | ||
Display this data using a bar-chart: | ||
Banding Brown Pink Yellow Sum | ||
X00000 339 433 126 898 | ||
X00300 48 421 222 691 | ||
X12345 16 395 352 763 | ||
Others 23 373 156 552 | ||
Sum 426 1622 856 2904 | ||
"); | ||
await InvokeAgentAsync(thread, "2-colors", "Can you regenerate this same chart using the category names as the bar colors?"); | ||
await InvokeAgentAsync(thread, "3-line", "Can you regenerate this as a line chart?"); | ||
} | ||
finally | ||
{ | ||
await agent.DeleteAsync(); | ||
} | ||
|
||
async Task InvokeAgentAsync(IAgentThread thread, string imageName, string question) | ||
{ | ||
await foreach (var message in thread.InvokeAsync(agent, question)) | ||
{ | ||
if (message.ContentType == ChatMessageType.Image) | ||
{ | ||
var filename = $"{imageName}.jpg"; | ||
var content = fileService.GetFileContent(message.Content); | ||
await using var outputStream = File.OpenWrite(filename); | ||
await using var inputStream = await content.GetStreamAsync(); | ||
await inputStream.CopyToAsync(outputStream); | ||
var path = Path.Combine(Environment.CurrentDirectory, filename); | ||
this.WriteLine($"# {message.Role}: {path}"); | ||
Process.Start( | ||
new ProcessStartInfo | ||
{ | ||
FileName = "cmd.exe", | ||
Arguments = $"/C start {path}" | ||
}); | ||
} | ||
else | ||
{ | ||
this.WriteLine($"# {message.Role}: {message.Content}"); | ||
} | ||
} | ||
|
||
this.WriteLine(); | ||
} | ||
} | ||
|
||
public Example85_AgentCharts(ITestOutputHelper output) : base(output) { } | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters