add hooks to debug OpenSSL memory

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Crypto.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
+using System.Collections.Concurrent;
 using System.Diagnostics;
 using System.Runtime.InteropServices;
 using System.Security.Cryptography.X509Certificates;
@@ -170,5 +171,106 @@ internal static byte[] GetDynamicBuffer<THandle>(NegativeSizeReadMethod<THandle>
 
             return bytes;
         }
+
+        [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_GetMemoryUse")]
+        internal static partial int GetMemoryUse(ref int memoryUse, ref int allocationCount);
+
+        public static int GetOpenSslAllocatedMemory()
+        {
+            int used = 0;
+            int count = 0;
+            GetMemoryUse(ref used, ref count);
+            return used;
+        }
+
+        public static int GetOpenSslAllocationCount()
+        {
+            int used = 0;
+            int count = 0;
+            GetMemoryUse(ref used, ref count);
+            return count;
+        }
+#if DEBUG
+        [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SetMemoryTracking")]
+        private static unsafe partial int SetMemoryTracking(delegate* unmanaged<MemoryOperation, UIntPtr, UIntPtr, int, char*, int, void> trackingCallback);
+
+        [StructLayout(LayoutKind.Sequential)]
+        private unsafe struct MemoryEntry
+        {
+            public int Size;
+            public int Line;
+            public char* File;
+        }
+
+        private enum MemoryOperation
+        {
+            Malloc = 1,
+            Realloc = 2,
+            Free = 3,
+        }
+
+        private static readonly unsafe nuint Offset = (nuint)sizeof(MemoryEntry);
+        // We only need to store the keys but we use ConcurrentDictionary to avoid locking
+        private static ConcurrentDictionary<UIntPtr, UIntPtr>? _allocations;
+
+        [UnmanagedCallersOnly]
+        private static unsafe void MemoryTrackinCallback(MemoryOperation operation, UIntPtr ptr, UIntPtr oldPtr, int size, char* file, int line)
+        {
+            ref MemoryEntry entry = ref *(MemoryEntry*)ptr;
+
+            Debug.Assert(entry.File != null);
+            Debug.Assert(ptr != UIntPtr.Zero);
+
+            switch (operation)
+            {
+                case MemoryOperation.Malloc:
+                    Debug.Assert(size == entry.Size);
+                    _allocations!.TryAdd(ptr, ptr);
+                    break;
+                case MemoryOperation.Realloc:
+                    if ((IntPtr)oldPtr != IntPtr.Zero)
+                    {
+                        _allocations!.TryRemove(oldPtr, out _);
+                    }
+                    _allocations!.TryAdd(ptr, ptr);
+                    break;
+                case MemoryOperation.Free:
+                    _allocations!.TryRemove(ptr, out _);
+                    break;
+            }
+        }
+
+        public static unsafe void EnableTracking()
+        {
+            _allocations ??= new ConcurrentDictionary<UIntPtr, UIntPtr>();
+            _allocations!.Clear();
+            SetMemoryTracking(&MemoryTrackinCallback);
+        }
+
+        public static unsafe void DisableTracking()
+        {
+            SetMemoryTracking(null);
+            _allocations!.Clear();
+        }
+
+        public static unsafe Tuple<UIntPtr, int, string>[] GetIncrementalAllocations()
+        {
+            if (_allocations == null || _allocations.IsEmpty)
+            {
+                return Array.Empty<Tuple<UIntPtr, int, string>>();
+            }
+
+            Tuple<UIntPtr, int, string>[] allocations = new Tuple<UIntPtr, int, string>[_allocations.Count];
+            int index = 0;
+            foreach ((UIntPtr ptr, UIntPtr value) in _allocations)
+            {
+                ref MemoryEntry entry = ref *(MemoryEntry*)ptr;
+                allocations[index] = new Tuple<UIntPtr, int, string>(ptr + Offset, entry.Size, $"{Marshal.PtrToStringAnsi((IntPtr)entry.File)}:{entry.Line}");
+                index++;
+            }
+
+            return allocations;
+        }
+#endif
     }
 }

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OpenSsl.cs

@@ -23,8 +23,12 @@ internal static partial class OpenSsl
     {
         private const string TlsCacheSizeCtxName = "System.Net.Security.TlsCacheSize";
         private const string TlsCacheSizeEnvironmentVariable = "DOTNET_SYSTEM_NET_SECURITY_TLSCACHESIZE";
+        private const string OpenSslDebugEnvironmentVariable = "DOTNET_SYSTEM_NET_SECURITY_OPENSSL_MEMORY_DEBUG";
         private const SslProtocols FakeAlpnSslProtocol = (SslProtocols)1;   // used to distinguish server sessions with ALPN
         private static readonly ConcurrentDictionary<SslProtocols, SafeSslContextHandle> s_clientSslContexts = new ConcurrentDictionary<SslProtocols, SafeSslContextHandle>();
+#pragma warning disable CA1823
+        private static readonly bool MemoryDebug = GetMemoryDebug();
+#pragma warning restore CA1823
 
         #region internal methods
         internal static SafeChannelBindingHandle? QueryChannelBinding(SafeSslHandle context, ChannelBindingKind bindingType)
@@ -63,6 +67,23 @@ private static int GetCacheSize()
             return cacheSize;
         }
 
+        private static bool GetMemoryDebug()
+        {
+            string? value = Environment.GetEnvironmentVariable(OpenSslDebugEnvironmentVariable);
+            if (int.TryParse(value, CultureInfo.InvariantCulture, out int enabled) && enabled == 1)
+            {
+                Interop.Crypto.GetOpenSslAllocationCount();
+                Interop.Crypto.GetOpenSslAllocatedMemory();
+#if DEBUG
+                Interop.Crypto.EnableTracking();
+                Interop.Crypto.GetIncrementalAllocations();
+                Interop.Crypto.DisableTracking();
+#endif
+            }
+
+            return enabled == 1;
+        }
+
         // This is helper function to adjust requested protocols based on CipherSuitePolicy and system capability.
         private static SslProtocols CalculateEffectiveProtocols(SslAuthenticationOptions sslAuthenticationOptions)
         {

src/libraries/System.Net.Http/src/System.Net.Http.csproj

@@ -432,11 +432,6 @@
              Link="Common\System\Net\Security\CertificateHelper.Unix.cs" />
   </ItemGroup>
 
-  <ItemGroup Condition="'$(TargetPlatformIdentifier)' != '' and '$(TargetPlatformIdentifier)' != 'windows' and '$(TargetPlatformIdentifier)' != 'browser' and '$(TargetPlatformIdentifier)' != 'osx' and '$(TargetPlatformIdentifier)' != 'ios' and '$(TargetPlatformIdentifier)' != 'tvos'">
-    <Compile Include="$(CommonPath)Interop\Unix\System.Security.Cryptography.Native\Interop.Initialization.cs"
-             Link="Common\Interop\Unix\System.Security.Cryptography.Native\Interop.Initialization.cs" />
-  </ItemGroup>
-
   <ItemGroup Condition="'$(TargetPlatformIdentifier)' == 'browser'">
     <Compile Include="$(CommonPath)\System\Net\HttpStatusDescription.cs"
              Link="Common\System\Net\HttpStatusDescription.cs" />

src/native/libs/System.Security.Cryptography.Native/apibridge_30.h

@@ -6,6 +6,17 @@
 #pragma once
 #include "pal_types.h"
 
+typedef void *(*CRYPTO_malloc_fn)(size_t num, const char *file, int line);
+typedef void *(*CRYPTO_realloc_fn)(void *addr, size_t num, const char *file, int line);
+typedef void (*CRYPTO_free_fn)(void *addr, const char *file, int line);
+
+#ifndef CRYPTO_RWLOCK
+typedef void CRYPTO_RWLOCK;
+#endif
+
+CRYPTO_RWLOCK *CRYPTO_THREAD_lock_new(void);
+int CRYPTO_atomic_add(int *val, int amount, int *ret, CRYPTO_RWLOCK *lock);
+
 typedef struct evp_mac_st EVP_MAC;
 typedef struct evp_mac_ctx_st EVP_MAC_CTX;
 
@@ -14,3 +25,5 @@ int local_EVP_PKEY_CTX_set_rsa_oaep_md(EVP_PKEY_CTX* ctx, const EVP_MD* md);
 int local_EVP_PKEY_CTX_set_rsa_padding(EVP_PKEY_CTX* ctx, int pad_mode);
 int local_EVP_PKEY_CTX_set_rsa_pss_saltlen(EVP_PKEY_CTX* ctx, int saltlen);
 int local_EVP_PKEY_CTX_set_signature_md(EVP_PKEY_CTX* ctx, const EVP_MD* md);
+
+int CRYPTO_set_mem_functions11(CRYPTO_malloc_fn malloc_fn, CRYPTO_realloc_fn realloc_fn, CRYPTO_free_fn free_fn);

src/native/libs/System.Security.Cryptography.Native/entrypoints.c

@@ -192,6 +192,8 @@ static const Entry s_cryptoNative[] =
     DllImportEntry(CryptoNative_GetECKeyParameters)
     DllImportEntry(CryptoNative_GetMaxMdSize)
     DllImportEntry(CryptoNative_GetMemoryBioSize)
+    DllImportEntry(CryptoNative_GetMemoryUse)
+    DllImportEntry(CryptoNative_SetMemoryTracking)
     DllImportEntry(CryptoNative_GetObjectDefinitionByName)
     DllImportEntry(CryptoNative_GetOcspRequestDerSize)
     DllImportEntry(CryptoNative_GetPkcs7Certificates)

src/native/libs/System.Security.Cryptography.Native/openssl.c

@@ -1478,6 +1478,118 @@ int32_t CryptoNative_OpenSslAvailable(void)
 #endif
 }
 
+static CRYPTO_RWLOCK* g_allocLock = NULL;
+static int g_allocatedMemory;
+static int g_allocationCount;
+
+static CRYPTO_allocation_cb g_memoryCallback;
+
+struct memoryEntry
+{
+    int size;
+    int line;
+    const char* file;
+} __attribute__((aligned(8)));
+
+static void* mallocFunction(size_t size, const char *file, int line)
+{
+    void* ptr = malloc(size + sizeof(struct memoryEntry));
+    if (ptr != NULL)
+    {
+        int newCount;
+        CRYPTO_atomic_add(&g_allocatedMemory, (int)size, &newCount, g_allocLock);
+        CRYPTO_atomic_add(&g_allocationCount, 1, &newCount, g_allocLock);
+        struct memoryEntry* entry = (struct memoryEntry*)ptr;
+        entry->size = (int)size;
+        entry->line = line;
+        entry->file = file;
+
+        if (g_memoryCallback != NULL)
+        {
+            g_memoryCallback(MallocOperation, ptr, NULL, entry->size, file, line);
+        }
+    }
+
+    return (void*)((char*)ptr + sizeof(struct memoryEntry));
+}
+
+static void* reallocFunction (void *ptr, size_t size, const char *file, int line)
+{
+    struct memoryEntry* entry;
+    int newCount;
+
+    if (ptr != NULL)
+    {
+        ptr = (void*)((char*)ptr - sizeof(struct memoryEntry));
+        entry = (struct memoryEntry*)ptr;
+        CRYPTO_atomic_add(&g_allocatedMemory, (int)(-entry->size), &newCount, g_allocLock);
+    }
+
+    void* newPtr = realloc(ptr, size + sizeof(struct memoryEntry));
+    if (newPtr != NULL)
+    {
+        CRYPTO_atomic_add(&g_allocatedMemory, (int)size, &newCount, g_allocLock);
+        CRYPTO_atomic_add(&g_allocationCount, 1, &newCount, g_allocLock);
+
+        entry = (struct memoryEntry*)newPtr;
+        entry->size = (int)size;
+        entry->line = line;
+        entry->file = file;
+
+        if (g_memoryCallback != NULL)
+        {
+#if defined(__GNUC__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Werror=use-after-free"
+#endif
+        // Now try just the _majorVer added
+            g_memoryCallback(ReallocOperation, newPtr, ptr, entry->size, file, line);
+
+#if defined(__GNUC__)
+#pragma GCC diagnostic pop
+#endif
+        }
+
+        return (void*)((char*)newPtr + sizeof(struct memoryEntry));
+    }
+
+    return NULL;
+}
+
+static void freeFunction(void *ptr, const char *file, int line)
+{
+    if (ptr != NULL)
+    {
+        int newCount;
+        struct memoryEntry* entry = (struct memoryEntry*)((char*)ptr - sizeof(struct memoryEntry));
+        CRYPTO_atomic_add(&g_allocatedMemory, (int)-entry->size, &newCount, g_allocLock);
+        if (g_memoryCallback != NULL)
+        {
+            g_memoryCallback(FreeOperation, entry, NULL, entry->size, file, line);
+        }
+
+        free(entry);
+    }
+}
+
+int32_t CryptoNative_GetMemoryUse(int* totalUsed, int* allocationCount)
+{
+    if (totalUsed == NULL || allocationCount == NULL)
+    {
+        return 0;
+    }
+    *totalUsed = g_allocatedMemory;
+    *allocationCount = g_allocationCount;
+
+    return 1;
+}
+
+PALEXPORT int32_t CryptoNative_SetMemoryTracking(CRYPTO_allocation_cb callback)
+{
+    g_memoryCallback = callback;
+    return 1;
+}
+
 static int32_t g_initStatus = 1;
 int g_x509_ocsp_index = -1;
 
@@ -1490,7 +1602,39 @@ static int32_t EnsureOpenSslInitializedCore(void)
     // Otherwise call the 1.1 one.
 #ifdef FEATURE_DISTRO_AGNOSTIC_SSL
     InitializeOpenSSLShim();
+#endif
+
+    const char* debug = getenv("DOTNET_SYSTEM_NET_SECURITY_OPENSSL_MEMORY_DEBUG");
+    if (debug != NULL && strcmp(debug, "1") == 0)
+    {
+        // This needs to be done before any allocation is done e.g. EnsureOpenSsl* is called.
+        // And it also needs to be after the pointers are loaded for DISTRO_AGNOSTIC_SSL
+#ifdef FEATURE_DISTRO_AGNOSTIC_SSL
+        if (API_EXISTS(CRYPTO_THREAD_lock_new))
+        {
+            // This should cover 1.1.1+
 
+            CRYPTO_set_mem_functions11(mallocFunction, reallocFunction, freeFunction);
+            g_allocLock = CRYPTO_THREAD_lock_new();
+
+            if (!API_EXISTS(SSL_state))
+            {
+                // CRYPTO_set_mem_functions exists in OpenSSL 1.0.1 as well but it has different prototype
+                // and that makes it difficult to use with managed callbacks.
+                // Since 1.0 is long time out of support we use it only on 1.1.1+
+                CRYPTO_set_mem_functions11(mallocFunction, reallocFunction, freeFunction);
+                g_allocLock = CRYPTO_THREAD_lock_new();
+            }
+        }
+#elif OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_1_1_0_RTM
+        // OpenSSL 1.0 has different prototypes and it is out of support so we enable this only
+        // on 1.1.1+
+        CRYPTO_set_mem_functions(mallocFunction, reallocFunction, freeFunction);
+        g_allocLock = CRYPTO_THREAD_lock_new();
+#endif
+    }
+
+#ifdef FEATURE_DISTRO_AGNOSTIC_SSL
     if (API_EXISTS(SSL_state))
     {
         ret = EnsureOpenSsl10Initialized();

src/native/libs/System.Security.Cryptography.Native/openssl.h

@@ -76,3 +76,16 @@ PALEXPORT int64_t CryptoNative_OpenSslVersionNumber(void);
 PALEXPORT void CryptoNative_RegisterLegacyAlgorithms(void);
 
 PALEXPORT int32_t CryptoNative_OpenSslAvailable(void);
+
+PALEXPORT int32_t CryptoNative_GetMemoryUse(int* totalUsed, int* allocationCount);
+
+typedef enum
+{
+    MallocOperation = 1,
+    ReallocOperation = 2,
+    FreeOperation = 3,
+} MemoryOperation;
+
+typedef void (*CRYPTO_allocation_cb)(MemoryOperation operation, void* ptr, void* oldPtr, int size, const char *file, int line);
+
+PALEXPORT int32_t CryptoNative_SetMemoryTracking(CRYPTO_allocation_cb callback);

src/native/libs/System.Security.Cryptography.Native/opensslshim.h

@@ -252,6 +252,9 @@ extern bool g_libSslUses32BitTime;
     REQUIRED_FUNCTION(CRYPTO_malloc) \
     LEGACY_FUNCTION(CRYPTO_num_locks) \
     LEGACY_FUNCTION(CRYPTO_set_locking_callback) \
+    LIGHTUP_FUNCTION(CRYPTO_THREAD_lock_new) \
+    LIGHTUP_FUNCTION(CRYPTO_atomic_add) \
+    RENAMED_FUNCTION(CRYPTO_set_mem_functions11, CRYPTO_set_mem_functions) \
     REQUIRED_FUNCTION(d2i_ASN1_BIT_STRING) \
     REQUIRED_FUNCTION(d2i_BASIC_CONSTRAINTS) \
     REQUIRED_FUNCTION(d2i_EXTENDED_KEY_USAGE) \
@@ -781,6 +784,9 @@ extern TYPEOF(OPENSSL_gmtime)* OPENSSL_gmtime_ptr;
 #define CRYPTO_malloc CRYPTO_malloc_ptr
 #define CRYPTO_num_locks CRYPTO_num_locks_ptr
 #define CRYPTO_set_locking_callback CRYPTO_set_locking_callback_ptr
+#define CRYPTO_THREAD_lock_new CRYPTO_THREAD_lock_new_ptr
+#define CRYPTO_atomic_add CRYPTO_atomic_add_ptr
+#define CRYPTO_set_mem_functions11 CRYPTO_set_mem_functions11_ptr
 #define d2i_ASN1_BIT_STRING d2i_ASN1_BIT_STRING_ptr
 #define d2i_BASIC_CONSTRAINTS d2i_BASIC_CONSTRAINTS_ptr
 #define d2i_EXTENDED_KEY_USAGE d2i_EXTENDED_KEY_USAGE_ptr