Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve postgres memory #1794

Merged
merged 9 commits into from
Jul 3, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ public static async Task RunAsync()
dataSourceBuilder.UseVector();
using NpgsqlDataSource dataSource = dataSourceBuilder.Build();

PostgresMemoryStore memoryStore = new(dataSource, vectorSize: 1536, schema: "public", numberOfLists: 100);
PostgresMemoryStore memoryStore = new(dataSource, vectorSize: 1536, schema: "public");

IKernel kernel = Kernel.Builder
.WithLogger(ConsoleLogger.Log)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
using System.Net.Http;
using System.Text;
using System.Text.Json.Serialization;
using Microsoft.SemanticKernel.Connectors.Memory.Pinecone.Http;

namespace Microsoft.SemanticKernel.Connectors.Memory.Pinecone.Model;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// Copyright (c) Microsoft. All rights reserved.

using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
Expand All @@ -13,76 +14,95 @@ namespace Microsoft.SemanticKernel.Connectors.Memory.Postgres;
public interface IPostgresDbClient
{
/// <summary>
/// Check if a collection exists.
/// Check if a table exists.
/// </summary>
/// <param name="collectionName">The name assigned to a collection of entries.</param>
/// <param name="tableName">The name assigned to a table of entries.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns></returns>
Task<bool> DoesCollectionExistsAsync(string collectionName, CancellationToken cancellationToken = default);
Task<bool> DoesTableExistsAsync(string tableName, CancellationToken cancellationToken = default);

/// <summary>
/// Create a collection.
/// Create a table.
/// </summary>
/// <param name="collectionName">The name assigned to a collection of entries.</param>
/// <param name="tableName">The name assigned to a table of entries.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns></returns>
Task CreateCollectionAsync(string collectionName, CancellationToken cancellationToken = default);
Task CreateTableAsync(string tableName, CancellationToken cancellationToken = default);

/// <summary>
/// Get all collections.
/// Get all tables.
/// </summary>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns></returns>
IAsyncEnumerable<string> GetCollectionsAsync(CancellationToken cancellationToken = default);
/// <returns>A group of tables.</returns>
IAsyncEnumerable<string> GetTablesAsync(CancellationToken cancellationToken = default);

/// <summary>
/// Delete a collection.
/// Delete a table.
/// </summary>
/// <param name="collectionName">The name assigned to a collection of entries.</param>
/// <param name="tableName">The name assigned to a table of entries.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns></returns>
Task DeleteCollectionAsync(string collectionName, CancellationToken cancellationToken = default);
Task DeleteTableAsync(string tableName, CancellationToken cancellationToken = default);

/// <summary>
/// Upsert entry into a collection.
/// Upsert entry into a table.
/// </summary>
/// <param name="collectionName">The name assigned to a collection of entries.</param>
/// <param name="tableName">The name assigned to a table of entries.</param>
/// <param name="key">The key of the entry to upsert.</param>
/// <param name="metadata">The metadata of the entry.</param>
/// <param name="embedding">The embedding of the entry.</param>
/// <param name="timestamp">The timestamp of the entry</param>
/// <param name="timestamp">The timestamp of the entry. Its 'DateTimeKind' must be <see cref="DateTimeKind.Utc"/></param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns></returns>
Task UpsertAsync(string collectionName, string key, string? metadata, Vector? embedding, long? timestamp, CancellationToken cancellationToken = default);
Task UpsertAsync(string tableName, string key, string? metadata, Vector? embedding, DateTime? timestamp, CancellationToken cancellationToken = default);

/// <summary>
/// Gets the nearest matches to the <see cref="Vector"/>.
/// </summary>
/// <param name="collectionName">The name assigned to a collection of entries.</param>
/// <param name="embeddingFilter">The <see cref="Vector"/> to compare the collection's embeddings with.</param>
/// <param name="tableName">The name assigned to a table of entries.</param>
/// <param name="embedding">The <see cref="Vector"/> to compare the table's embeddings with.</param>
/// <param name="limit">The maximum number of similarity results to return.</param>
/// <param name="minRelevanceScore">The minimum relevance threshold for returned results.</param>
/// <param name="withEmbeddings">If true, the embeddings will be returned in the entries.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns></returns>
IAsyncEnumerable<(PostgresMemoryEntry, double)> GetNearestMatchesAsync(string collectionName, Vector embeddingFilter, int limit, double minRelevanceScore = 0, bool withEmbeddings = false, CancellationToken cancellationToken = default);
/// <returns>An asynchronous stream of <see cref="PostgresMemoryEntry"/> objects that the nearest matches to the <see cref="Vector"/>.</returns>
IAsyncEnumerable<(PostgresMemoryEntry, double)> GetNearestMatchesAsync(string tableName, Vector embedding, int limit, double minRelevanceScore = 0, bool withEmbeddings = false, CancellationToken cancellationToken = default);

/// <summary>
/// Read a entry by its key.
/// </summary>
/// <param name="collectionName">The name assigned to a collection of entries.</param>
/// <param name="tableName">The name assigned to a table of entries.</param>
/// <param name="key">The key of the entry to read.</param>
/// <param name="withEmbeddings">If true, the embeddings will be returned in the entries.</param>
/// <param name="withEmbeddings">If true, the embeddings will be returned in the entry.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns></returns>
Task<PostgresMemoryEntry?> ReadAsync(string collectionName, string key, bool withEmbeddings = false, CancellationToken cancellationToken = default);
Task<PostgresMemoryEntry?> ReadAsync(string tableName, string key, bool withEmbeddings = false, CancellationToken cancellationToken = default);

/// <summary>
/// Read multiple entries by their keys.
/// </summary>
/// <param name="tableName">The name assigned to a table of entries.</param>
/// <param name="keys">The keys of the entries to read.</param>
/// <param name="withEmbeddings">If true, the embeddings will be returned in the entries.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>An asynchronous stream of <see cref="PostgresMemoryEntry"/> objects that match the given keys.</returns>
IAsyncEnumerable<PostgresMemoryEntry> ReadBatchAsync(string tableName, IEnumerable<string> keys, bool withEmbeddings = false, CancellationToken cancellationToken = default);

/// <summary>
/// Delete a entry by its key.
/// </summary>
/// <param name="collectionName">The name assigned to a collection of entries.</param>
/// <param name="tableName">The name assigned to a table of entries.</param>
/// <param name="key">The key of the entry to delete.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns></returns>
Task DeleteAsync(string collectionName, string key, CancellationToken cancellationToken = default);
Task DeleteAsync(string tableName, string key, CancellationToken cancellationToken = default);

/// <summary>
/// Delete multiple entries by their key.
/// </summary>
/// <param name="tableName">The name assigned to a table of entries.</param>
/// <param name="keys">The keys of the entries to delete.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns></returns>
Task DeleteBatchAsync(string tableName, IEnumerable<string> keys, CancellationToken cancellationToken = default);
}
Loading
Loading