Merge branch 'main' into feature-connectors-openai

microsoft · Jul 12, 2024 · 5dadd59 · 5dadd59
2 parents a10e9f2 + 6b59077
commit 5dadd59
Show file tree

Hide file tree

Showing 199 changed files with 10,475 additions and 3,861 deletions.
diff --git a/.github/ISSUE_TEMPLATE/feature_graduation.md b/.github/ISSUE_TEMPLATE/feature_graduation.md
@@ -16,14 +16,14 @@ about: Plan the graduation of an experimental feature
 
 Checklist to be completed when graduating an experimental feature
 
-- [ ] Notify PM's and EM's that feature is read for graduation
+- [ ] Notify PM's and EM's that feature is ready for graduation
 - [ ] Contact PM for list of sample use cases
 - [ ] Verify there are sample implementations for each of the use cases
 - [ ] Verify telemetry and logging are complete
 - [ ] Verify API docs are complete and arrange to have them published
 - [ ] Make appropriate updates to Learn docs
 - [ ] Make appropriate updates to Concept samples
-- [ ] Male appropriate updates to Blog posts
+- [ ] Make appropriate updates to Blog posts
 - [ ] Verify there are no serious open Issues
 - [ ] Update table in EXPERIMENTS.md
 - [ ] Remove SKEXP flag from the experimental code
diff --git a/.github/workflows/python-integration-tests.yml b/.github/workflows/python-integration-tests.yml
@@ -96,6 +96,8 @@ jobs:
           AZURE_KEY_VAULT_CLIENT_ID: ${{secrets.AZURE_KEY_VAULT_CLIENT_ID}}
           AZURE_KEY_VAULT_CLIENT_SECRET: ${{secrets.AZURE_KEY_VAULT_CLIENT_SECRET}}
           ACA_POOL_MANAGEMENT_ENDPOINT: ${{secrets.ACA_POOL_MANAGEMENT_ENDPOINT}}
+          MISTRALAI_API_KEY: ${{secrets.MISTRALAI_API_KEY}}
+          MISTRALAI_CHAT_MODEL_ID: ${{ vars.MISTRALAI_CHAT_MODEL_ID }}
         run: |
           if ${{ matrix.os == 'ubuntu-latest' }}; then
             docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest
@@ -163,6 +165,8 @@ jobs:
           AZURE_KEY_VAULT_CLIENT_ID: ${{secrets.AZURE_KEY_VAULT_CLIENT_ID}}
           AZURE_KEY_VAULT_CLIENT_SECRET: ${{secrets.AZURE_KEY_VAULT_CLIENT_SECRET}}
           ACA_POOL_MANAGEMENT_ENDPOINT: ${{secrets.ACA_POOL_MANAGEMENT_ENDPOINT}}
+          MISTRALAI_API_KEY: ${{secrets.MISTRALAI_API_KEY}}
+          MISTRALAI_CHAT_MODEL_ID: ${{ vars.MISTRALAI_CHAT_MODEL_ID }}
         run: |
           if ${{ matrix.os == 'ubuntu-latest' }}; then
             docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest

diff --git a/.github/workflows/python-samples-tests.yml b/.github/workflows/python-samples-tests.yml
diff --git a/.github/workflows/python-test-coverage.yml b/.github/workflows/python-test-coverage.yml
@@ -14,6 +14,7 @@ jobs:
   python-tests-coverage:
     name: Create Test Coverage Messages
     runs-on: ${{ matrix.os }}
+    continue-on-error: true
     permissions:
       pull-requests: write
       contents: read
@@ -33,11 +34,13 @@ jobs:
           wait-interval: 10
           allowed-conclusions: success
       - uses: actions/checkout@v4
+      - name: Setup filename variables
+        run: echo "FILE_ID=${{ github.run_id }}-${{ matrix.os }}-${{ matrix.python-version }}" >> $GITHUB_ENV
       - name: Download coverage
         continue-on-error: true
         uses: dawidd6/action-download-artifact@v3
         with:
-          name: python-coverage-${{ matrix.os }}-${{ matrix.python-version }}.txt
+          name: python-coverage-${{ env.FILE_ID }}.txt
           github_token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
           workflow: python-unit-tests.yml
           search_artifacts: true
@@ -46,7 +49,7 @@ jobs:
         continue-on-error: true
         uses: dawidd6/action-download-artifact@v3
         with:
-          name: pytest-${{ matrix.os }}-${{ matrix.python-version }}.xml
+          name: pytest-${{ env.FILE_ID }}.xml
           github_token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
           workflow: python-unit-tests.yml
           search_artifacts: true

diff --git a/.github/workflows/python-unit-tests.yml b/.github/workflows/python-unit-tests.yml
@@ -10,15 +10,26 @@ jobs:
   python-unit-tests:
     name: Python Unit Tests
     runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         python-version: ["3.10", "3.11", "3.12"]
         os: [ubuntu-latest, windows-latest, macos-latest]
+        experimental: [false]
+        include:
+          - python-version: "3.13.0-beta.3"
+            os: "ubuntu-latest"
+            experimental: true
     permissions:
       contents: write
+    defaults:
+      run:
+        working-directory: python
     steps:
       - uses: actions/checkout@v4
+      - name: Setup filename variables
+        run: echo "FILE_ID=${{ github.run_id }}-${{ matrix.os }}-${{ matrix.python-version }}" >> $GITHUB_ENV
       - name: Install poetry
         run: pipx install poetry
       - name: Set up Python ${{ matrix.python-version }}
@@ -27,20 +38,20 @@ jobs:
           python-version: ${{ matrix.python-version }}
           cache: "poetry"
       - name: Install dependencies
-        run: cd python && poetry install --with unit-tests
+        run: poetry install --with unit-tests
       - name: Test with pytest
-        run: cd python && poetry run pytest -q --junitxml=pytest-${{ matrix.os }}-${{ matrix.python-version }}.xml  --cov=semantic_kernel --cov-report=term-missing:skip-covered ./tests/unit | tee python-coverage-${{ matrix.os }}-${{ matrix.python-version }}.txt
+        run: poetry run pytest -q --junitxml=pytest.xml  --cov=semantic_kernel --cov-report=term-missing:skip-covered ./tests/unit | tee python-coverage.txt
       - name: Upload coverage
         uses: actions/upload-artifact@v4
         with:
-          name: python-coverage-${{ matrix.os }}-${{ matrix.python-version }}.txt
-          path: python/python-coverage-${{ matrix.os }}-${{ matrix.python-version }}.txt
+          name: python-coverage-${{ env.FILE_ID }}.txt
+          path: python/python-coverage.txt
           overwrite: true
-          retention-days: 1  
+          retention-days: 1
       - name: Upload pytest.xml
         uses: actions/upload-artifact@v4
         with:
-          name: pytest-${{ matrix.os }}-${{ matrix.python-version }}.xml
-          path: python/pytest-${{ matrix.os }}-${{ matrix.python-version }}.xml
+          name: pytest-${{ env.FILE_ID }}.xml
+          path: python/pytest.xml
           overwrite: true
           retention-days: 1
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -37,7 +37,7 @@ repos:
     - id: pyupgrade
       args: [--py310-plus]
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.5
+    rev: v0.5.1
     hooks:
       - id: ruff
         args: [ --fix, --exit-non-zero-on-fix ]

diff --git a/README.md b/README.md
@@ -19,7 +19,7 @@ is an SDK that integrates Large Language Models (LLMs) like
 [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service),
 and [Hugging Face](https://huggingface.co/)
 with conventional programming languages like C#, Python, and Java. Semantic Kernel achieves this
-by allowing you to define [plugins](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/plugins)
+by allowing you to define [plugins](https://learn.microsoft.com/en-us/semantic-kernel/concepts/plugins)
 that can be chained together
 in just a [few lines of code](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/chaining-functions?tabs=Csharp#using-the-runasync-method-to-simplify-your-code).
 

diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props
@@ -30,7 +30,7 @@
     <PackageVersion Include="Microsoft.Bcl.TimeProvider" Version="8.0.1" />
     <PackageVersion Include="Microsoft.Extensions.Logging.Debug" Version="8.0.0" />
     <PackageVersion Include="Microsoft.Identity.Client" Version="4.61.3" />
-    <PackageVersion Include="Microsoft.ML.OnnxRuntime" Version="1.18.0" />
+    <PackageVersion Include="Microsoft.ML.OnnxRuntime" Version="1.18.1" />
     <PackageVersion Include="FastBertTokenizer" Version="1.0.28" />
     <PackageVersion Include="System.Diagnostics.DiagnosticSource" Version="8.0.1" />
     <PackageVersion Include="System.Linq.Async" Version="6.0.1" />

diff --git a/dotnet/docs/EXPERIMENTS.md b/dotnet/docs/EXPERIMENTS.md
@@ -26,57 +26,57 @@ You can use the following diagnostic IDs to ignore warnings or errors for a part
 
 ## Experimental Features Tracking
 
-| SKEXP | Features | API docs | Learn docs | Samples | Issues | Implementations |
-|-------|----------|----------|------------|---------|--------|-----------------|
-| SKEXP0001 | Embedding services | | | | | |
-| SKEXP0001 | Image services | | | | | |
-| SKEXP0001 | Memory connectors | | | | | |
-| SKEXP0001 | Kernel filters | | | | | |
-| SKEXP0001 | Audio services | | | | | |
+| SKEXP | Features |
+|-------|----------|
+| SKEXP0001 | Embedding services |
+| SKEXP0001 | Image services |
+| SKEXP0001 | Memory connectors |
+| SKEXP0001 | Kernel filters |
+| SKEXP0001 | Audio services |
 | | | | | | | |
-| SKEXP0010 | Azure OpenAI with your data service | | | | | |
-| SKEXP0010 | OpenAI embedding service | | | | | |
-| SKEXP0010 | OpenAI image service | | | | | |
-| SKEXP0010 | OpenAI parameters | | | | | |
-| SKEXP0010 | OpenAI chat history extension | | | | | |
-| SKEXP0010 | OpenAI file service | | | | | |
+| SKEXP0010 | Azure OpenAI with your data service |
+| SKEXP0010 | OpenAI embedding service |
+| SKEXP0010 | OpenAI image service |
+| SKEXP0010 | OpenAI parameters |
+| SKEXP0010 | OpenAI chat history extension |
+| SKEXP0010 | OpenAI file service |
 | | | | | | | |
-| SKEXP0020 | Azure AI Search memory connector | | | | | |
-| SKEXP0020 | Chroma memory connector | | | | | |
-| SKEXP0020 | DuckDB memory connector | | | | | |
-| SKEXP0020 | Kusto memory connector | | | | | |
-| SKEXP0020 | Milvus memory connector | | | | | |
-| SKEXP0020 | Qdrant memory connector | | | | | |
-| SKEXP0020 | Redis memory connector | | | | | |
-| SKEXP0020 | Sqlite memory connector | | | | | |
-| SKEXP0020 | Weaviate memory connector | | | | | |
-| SKEXP0020 | MongoDB memory connector | | | | | |
-| SKEXP0020 | Pinecone memory connector | | | | | |
-| SKEXP0020 | Postgres memory connector | | | | | |
+| SKEXP0020 | Azure AI Search memory connector |
+| SKEXP0020 | Chroma memory connector |
+| SKEXP0020 | DuckDB memory connector |
+| SKEXP0020 | Kusto memory connector |
+| SKEXP0020 | Milvus memory connector |
+| SKEXP0020 | Qdrant memory connector |
+| SKEXP0020 | Redis memory connector |
+| SKEXP0020 | Sqlite memory connector |
+| SKEXP0020 | Weaviate memory connector |
+| SKEXP0020 | MongoDB memory connector |
+| SKEXP0020 | Pinecone memory connector |
+| SKEXP0020 | Postgres memory connector |
 | | | | | | | |
-| SKEXP0040 | GRPC functions | | | | | |
-| SKEXP0040 | Markdown functions | | | | | |
-| SKEXP0040 | OpenAPI functions | | | | | |
-| SKEXP0040 | OpenAPI function extensions | | | | | |
-| SKEXP0040 | Prompty Format support | | | | | |
+| SKEXP0040 | GRPC functions |
+| SKEXP0040 | Markdown functions |
+| SKEXP0040 | OpenAPI functions |
+| SKEXP0040 | OpenAPI function extensions |
+| SKEXP0040 | Prompty Format support |
 | | | | | | | |
-| SKEXP0050 | Core plugins | | | | | |
-| SKEXP0050 | Document plugins | | | | | |
-| SKEXP0050 | Memory plugins | | | | | |
-| SKEXP0050 | Microsoft 365 plugins | | | | | |
-| SKEXP0050 | Web plugins | | | | | |
-| SKEXP0050 | Text chunker plugin | | | | | |
+| SKEXP0050 | Core plugins |
+| SKEXP0050 | Document plugins |
+| SKEXP0050 | Memory plugins |
+| SKEXP0050 | Microsoft 365 plugins |
+| SKEXP0050 | Web plugins |
+| SKEXP0050 | Text chunker plugin |
 | | | | | | | |
-| SKEXP0060 | Handlebars planner | | | | | |
-| SKEXP0060 | OpenAI Stepwise planner | | | | | |
+| SKEXP0060 | Handlebars planner |
+| SKEXP0060 | OpenAI Stepwise planner |
 | | | | | | | |
-| SKEXP0070 | Ollama AI connector | | | | | |
-| SKEXP0070 | Gemini AI connector | | | | | |
-| SKEXP0070 | Mistral AI connector | | | | | |
-| SKEXP0070 | ONNX AI connector | | | | | |
-| SKEXP0070 | Hugging Face AI connector | | | | | |
+| SKEXP0070 | Ollama AI connector |
+| SKEXP0070 | Gemini AI connector |
+| SKEXP0070 | Mistral AI connector |
+| SKEXP0070 | ONNX AI connector |
+| SKEXP0070 | Hugging Face AI connector |
 | | | | | | | |
-| SKEXP0101 | Experiment with Assistants | | | | | |
-| SKEXP0101 | Experiment with Flow Orchestration | | | | | |
+| SKEXP0101 | Experiment with Assistants |
+| SKEXP0101 | Experiment with Flow Orchestration |
 | | | | | | | |
-| SKEXP0110 | Agent Framework | | | | | |
+| SKEXP0110 | Agent Framework |
diff --git a/dotnet/samples/Concepts/Agents/ComplexChat_NestedShopper.cs b/dotnet/samples/Concepts/Agents/ComplexChat_NestedShopper.cs
@@ -154,7 +154,7 @@ public async Task NestedChatWithAggregatorAgentAsync()
         Console.WriteLine(">>>> AGGREGATED CHAT");
         Console.WriteLine(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
 
-        await foreach (var content in chat.GetChatMessagesAsync(personalShopperAgent).Reverse())
+        await foreach (ChatMessageContent content in chat.GetChatMessagesAsync(personalShopperAgent).Reverse())
         {
             Console.WriteLine($">>>> {content.Role} - {content.AuthorName ?? "*"}: '{content.Content}'");
         }
@@ -165,7 +165,7 @@ async Task InvokeChatAsync(string input)
 
             Console.WriteLine($"# {AuthorRole.User}: '{input}'");
 
-            await foreach (var content in chat.InvokeAsync(personalShopperAgent))
+            await foreach (ChatMessageContent content in chat.InvokeAsync(personalShopperAgent))
             {
                 Console.WriteLine($"# {content.Role} - {content.AuthorName ?? "*"}: '{content.Content}'");
             }

diff --git a/dotnet/samples/Concepts/Agents/MixedChat_Agents.cs b/dotnet/samples/Concepts/Agents/MixedChat_Agents.cs
@@ -56,8 +56,8 @@ await OpenAIAssistantAgent.CreateAsync(
                 });
 
         // Create a chat for agent interaction.
-        var chat =
-            new AgentGroupChat(agentWriter, agentReviewer)
+        AgentGroupChat chat =
+            new(agentWriter, agentReviewer)
             {
                 ExecutionSettings =
                     new()
@@ -80,7 +80,7 @@ await OpenAIAssistantAgent.CreateAsync(
         chat.AddChatMessage(new ChatMessageContent(AuthorRole.User, input));
         Console.WriteLine($"# {AuthorRole.User}: '{input}'");
 
-        await foreach (var content in chat.InvokeAsync())
+        await foreach (ChatMessageContent content in chat.InvokeAsync())
         {
             Console.WriteLine($"# {content.Role} - {content.AuthorName ?? "*"}: '{content.Content}'");
         }

diff --git a/dotnet/samples/Concepts/Agents/OpenAIAssistant_ChartMaker.cs b/dotnet/samples/Concepts/Agents/OpenAIAssistant_ChartMaker.cs
@@ -37,7 +37,7 @@ await OpenAIAssistantAgent.CreateAsync(
                 });
 
         // Create a chat for agent interaction.
-        var chat = new AgentGroupChat();
+        AgentGroupChat chat = new();
 
         // Respond to user input
         try
@@ -68,14 +68,14 @@ async Task InvokeAgentAsync(string input)
 
             Console.WriteLine($"# {AuthorRole.User}: '{input}'");
 
-            await foreach (var message in chat.InvokeAsync(agent))
+            await foreach (ChatMessageContent message in chat.InvokeAsync(agent))
             {
                 if (!string.IsNullOrWhiteSpace(message.Content))
                 {
                     Console.WriteLine($"# {message.Role} - {message.AuthorName ?? "*"}: '{message.Content}'");
                 }
 
-                foreach (var fileReference in message.Items.OfType<FileReferenceContent>())
+                foreach (FileReferenceContent fileReference in message.Items.OfType<FileReferenceContent>())
                 {
                     Console.WriteLine($"# {message.Role} - {message.AuthorName ?? "*"}: @{fileReference.FileId}");
                 }

diff --git a/dotnet/samples/Concepts/Agents/OpenAIAssistant_CodeInterpreter.cs b/dotnet/samples/Concepts/Agents/OpenAIAssistant_CodeInterpreter.cs
@@ -28,7 +28,7 @@ await OpenAIAssistantAgent.CreateAsync(
                 });
 
         // Create a chat for agent interaction.
-        var chat = new AgentGroupChat();
+        AgentGroupChat chat = new();
 
         // Respond to user input
         try