Adjust managed type system for new function pointer handling (#84819)

After #81006, the calling convention is no longer part of the type system identity of a function pointer type - it serves more like a modopt as far as the type system is concerned. The type system only cares whether the pointer is managed or not. Adjust the managed type system accordingly:

* If we're reading/representing a standalone method signature, read it as usual. Calling convention is available in flags/modopts.
* If we're reading/representing a function pointer type, collapse the calling convention information into the managed/unmanaged bit only.

src/coreclr/tools/Common/TypeSystem/Common/MethodDesc.cs

@@ -27,7 +27,8 @@ public enum EmbeddedSignatureDataKind
     {
         RequiredCustomModifier = 0,
         OptionalCustomModifier = 1,
-        ArrayShape = 2
+        ArrayShape = 2,
+        UnmanagedCallConv = 3,
     }
 
     public struct EmbeddedSignatureData

src/coreclr/tools/Common/TypeSystem/Common/TypeSystemContext.cs

@@ -242,6 +242,10 @@ protected override FunctionPointerType CreateValueFromKey(MethodSignature key)
 
         public FunctionPointerType GetFunctionPointerType(MethodSignature signature)
         {
+            // The type system only distinguishes between unmanaged and managed signatures.
+            // The caller should have normalized the signature by modifying flags and stripping modopts.
+            Debug.Assert((signature.Flags & MethodSignatureFlags.UnmanagedCallingConventionMask) is 0 or MethodSignatureFlags.UnmanagedCallingConvention);
+            Debug.Assert(!signature.HasEmbeddedSignatureData);
             return _functionPointerTypes.GetOrCreateValue(signature);
         }
 

src/coreclr/tools/Common/TypeSystem/Ecma/EcmaSignatureParser.cs

@@ -371,7 +371,19 @@ private MethodSignature ParseMethodSignatureImpl(bool skipEmbeddedSignatureData)
                 Debug.Assert((int)MethodSignatureFlags.CallingConventionVarargs == (int)SignatureCallingConvention.VarArgs);
                 Debug.Assert((int)MethodSignatureFlags.UnmanagedCallingConvention == (int)SignatureCallingConvention.Unmanaged);
 
-                flags = (MethodSignatureFlags)signatureCallConv;
+                // If skipEmbeddedSignatureData is true, we're building the signature for the purposes of building a type.
+                // We normalize unmanaged calling convention into a single value - "unmanaged".
+                if (skipEmbeddedSignatureData)
+                {
+                    flags = MethodSignatureFlags.UnmanagedCallingConvention;
+
+                    // But we still need to remember this signature is different, so add this to the EmbeddedSignatureData of the owner signature.
+                    _embeddedSignatureDataList?.Add(new EmbeddedSignatureData { index = string.Join(".", _indexStack) + "|" + ((int)signatureCallConv).ToString(), kind = EmbeddedSignatureDataKind.UnmanagedCallConv, type = null });
+                }
+                else
+                {
+                    flags = (MethodSignatureFlags)signatureCallConv;
+                }
             }
 
             if (!header.IsInstance)

src/coreclr/tools/Common/TypeSystem/MetadataEmitter/TypeSystemMetadataEmitter.cs

@@ -514,6 +514,27 @@ public void Push()
                 }
             }
 
+            public void UpdateSignatureCallingConventionAtCurrentIndexStack(ref SignatureCallingConvention callConv)
+            {
+                if (!Complete)
+                {
+                    if (_embeddedDataIndex < _embeddedData.Length)
+                    {
+                        if (_embeddedData[_embeddedDataIndex].kind == EmbeddedSignatureDataKind.UnmanagedCallConv)
+                        {
+                            string indexData = string.Join(".", _indexStack);
+
+                            var unmanagedCallConvPossibility = _embeddedData[_embeddedDataIndex].index.Split('|');
+                            if (unmanagedCallConvPossibility[0] == indexData)
+                            {
+                                callConv = (SignatureCallingConvention)int.Parse(unmanagedCallConvPossibility[1]);
+                                _embeddedDataIndex++;
+                            }
+                        }
+                    }
+                }
+            }
+
             public void EmitArrayShapeAtCurrentIndexStack(BlobBuilder signatureBuilder, int rank)
             {
                 var shapeEncoder = new ArrayShapeEncoder(signatureBuilder);
@@ -665,6 +686,9 @@ private void EncodeMethodSignature(BlobBuilder signatureBuilder, MethodSignature
                     break;
             }
 
+            if (sigCallingConvention != SignatureCallingConvention.Default)
+                signatureDataEmitter.UpdateSignatureCallingConventionAtCurrentIndexStack(ref sigCallingConvention);
+
             signatureEncoder.MethodSignature(sigCallingConvention, genericParameterCount, isInstanceMethod);
             signatureBuilder.WriteCompressedInteger(sig.Length);
             EncodeType(signatureBuilder, sig.ReturnType, signatureDataEmitter);

src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/CoreTestAssembly/CoreTestAssembly.csproj

@@ -4,6 +4,7 @@
     <AssemblyName>CoreTestAssembly</AssemblyName>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <IsCoreAssembly>true</IsCoreAssembly>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <SkipTestRun>true</SkipTestRun>
     <TargetFramework>netstandard2.0</TargetFramework>
     <!-- Don't add references to the netstandard platform since this is a core assembly -->

src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/CoreTestAssembly/Platform.cs

@@ -60,6 +60,11 @@ public struct RuntimeTypeHandle { }
     public struct RuntimeMethodHandle { }
     public struct RuntimeFieldHandle { }
 
+    public class Type
+    {
+        public static Type GetTypeFromHandle(RuntimeTypeHandle handle) => null;
+    }
+
     public class Attribute { }
     public class AttributeUsageAttribute : Attribute
     {
@@ -159,9 +164,14 @@ public sealed class IsByRefLikeAttribute : Attribute
     {
     }
 
+    public class CallConvCdecl { }
+    public class CallConvStdcall { }
+    public class CallConvSuppressGCTransition { }
+
     public static class RuntimeFeature
     {
         public const string ByRefFields = nameof(ByRefFields);
+        public const string UnmanagedSignatureCallingConvention = nameof(UnmanagedSignatureCallingConvention);
         public const string VirtualStaticsInInterfaces = nameof(VirtualStaticsInInterfaces);
     }
 

src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/CoreTestAssembly/VirtualFunctionOverride.cs

@@ -42,4 +42,22 @@ class ClassWithFinalizer
 
         }
     }
+
+    unsafe class FunctionPointerOverloadBase
+    {
+        // Do not reorder these, the test assumes this order
+        public virtual Type Method(delegate* unmanaged[Cdecl]<void> p) => typeof(delegate* unmanaged[Cdecl]<void>);
+        public virtual Type Method(delegate* unmanaged[Stdcall]<void> p) => typeof(delegate* unmanaged[Stdcall]<void>);
+        public virtual Type Method(delegate* unmanaged[Stdcall, SuppressGCTransition]<void> p) => typeof(delegate* unmanaged[Stdcall, SuppressGCTransition]<void>);
+        public virtual Type Method(delegate*<void> p) => typeof(delegate*<void>);
+    }
+
+    unsafe class FunctionPointerOverloadDerived : FunctionPointerOverloadBase
+    {
+        // Do not reorder these, the test assumes this order
+        public override Type Method(delegate* unmanaged[Cdecl]<void> p) => null;
+        public override Type Method(delegate* unmanaged[Stdcall]<void> p) => null;
+        public override Type Method(delegate* unmanaged[Stdcall, SuppressGCTransition]<void> p) => null;
+        public override Type Method(delegate*<void> p) => null;
+    }
 }

src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/VirtualFunctionOverrideTests.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using Internal.TypeSystem;
 
@@ -268,5 +269,25 @@ public void TestExactMethodImplGenericDeclResolutionOnInterfaces()
             Assert.Contains("!0,!1", md1.Name);
             Assert.Contains("!1,!0", md2.Name);
         }
+
+        [Fact]
+        public void TestFunctionPointerOverloads()
+        {
+            MetadataType baseClass = _testModule.GetType("VirtualFunctionOverride", "FunctionPointerOverloadBase");
+            MetadataType derivedClass = _testModule.GetType("VirtualFunctionOverride", "FunctionPointerOverloadDerived");
+
+            var resolvedMethods = new List<MethodDesc>();
+            foreach (MethodDesc baseMethod in baseClass.GetVirtualMethods())
+                resolvedMethods.Add(derivedClass.FindVirtualFunctionTargetMethodOnObjectType(baseMethod));
+
+            var expectedMethods = new List<MethodDesc>();
+            foreach (MethodDesc derivedMethod in derivedClass.GetVirtualMethods())
+                expectedMethods.Add(derivedMethod);
+
+            Assert.Equal(expectedMethods, resolvedMethods);
+
+            Assert.Equal(expectedMethods[0].Signature[0], expectedMethods[1].Signature[0]);
+            Assert.NotEqual(expectedMethods[0].Signature[0], expectedMethods[3].Signature[0]);
+        }
     }
 }