[release/9.0] [mono][mini] Interlocked.CompareExchange and Interlocked.Exchange intrinsics for small types and enums

src/libraries/System.Private.CoreLib/src/System/Threading/Interlocked.cs

@@ -84,7 +84,7 @@ public static short Exchange(ref short location1, short value) =>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static unsafe byte Exchange(ref byte location1, byte value)
         {
-#if !MONO && (TARGET_X86 || TARGET_AMD64 || TARGET_ARM64)
+#if (MONO && (TARGET_AMD64 || TARGET_ARM64 || TARGET_WASM)) || (!MONO && (TARGET_X86 || TARGET_AMD64 || TARGET_ARM64))
             return Exchange(ref location1, value); // Must expand intrinsic
 #else
             // this relies on GC keeping 4B alignment for refs and on subtracting to such alignment being in the same object
@@ -123,7 +123,7 @@ public static unsafe byte Exchange(ref byte location1, byte value)
         [CLSCompliant(false)]
         public static unsafe ushort Exchange(ref ushort location1, ushort value)
         {
-#if !MONO && (TARGET_X86 || TARGET_AMD64 || TARGET_ARM64)
+#if ((MONO && (TARGET_AMD64 || TARGET_ARM64 || TARGET_WASM)) || !MONO && (TARGET_X86 || TARGET_AMD64 || TARGET_ARM64))
             return Exchange(ref location1, value); // Must expand intrinsic
 #else
             // this relies on GC keeping 4B alignment for refs and on subtracting to such alignment being in the same object
@@ -322,7 +322,7 @@ public static short CompareExchange(ref short location1, short value, short comp
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static unsafe byte CompareExchange(ref byte location1, byte value, byte comparand)
         {
-#if !MONO && (TARGET_X86 || TARGET_AMD64 || TARGET_ARM64)
+#if (MONO && (TARGET_ARM64 || TARGET_AMD64 || TARGET_WASM)) || (!MONO && (TARGET_X86 || TARGET_AMD64 || TARGET_ARM64))
             return CompareExchange(ref location1, value, comparand); // Must expand intrinsic
 #else
             // this relies on GC keeping 4B alignment for refs and on subtracting to such alignment being in the same object
@@ -365,7 +365,7 @@ public static unsafe byte CompareExchange(ref byte location1, byte value, byte c
         [CLSCompliant(false)]
         public static unsafe ushort CompareExchange(ref ushort location1, ushort value, ushort comparand)
         {
-#if !MONO && (TARGET_X86 || TARGET_AMD64 || TARGET_ARM64)
+#if (MONO && (TARGET_ARM64 || TARGET_AMD64 || TARGET_WASM)) || (!MONO && (TARGET_X86 || TARGET_AMD64 || TARGET_ARM64))
             return CompareExchange(ref location1, value, comparand); // Must expand intrinsic
 #else
             // this relies on GC keeping 4B alignment for refs and on subtracting to such alignment being in the same object

src/mono/browser/runtime/jiterpreter-opcodes.ts

@@ -511,3 +511,73 @@ export const enum WasmSimdOpcode {
     i32x4_extadd_pairwise_i16x8_s = 0x7e,
     i32x4_extadd_pairwise_i16x8_u = 0x7f,
 }
+
+export const enum WasmAtomicOpcode {
+    memory_atomic_notify = 0x00,
+    memory_atomic_wait32 = 0x01,
+    memory_atomic_wait64 = 0x02,
+    atomic_fence = 0x03,
+    i32_atomic_load = 0x10,
+    i64_atomic_load = 0x11,
+    i32_atomic_load8_u = 0x12,
+    i32_atomic_load16_u = 0x13,
+    i64_atomic_load8_u = 0x14,
+    i64_atomic_load16_u = 0x15,
+    i64_atomic_load32_u = 0x16,
+    i32_atomic_store = 0x17,
+    i64_atomic_store = 0x18,
+    i32_atomic_store8 = 0x19,
+    i32_atomic_store16 = 0x1A,
+    i64_atomic_store8 = 0x1B,
+    i64_atomic_store16 = 0x1C,
+    i64_atomic_store32 = 0x1D,
+    i32_atomic_rmw_add = 0x1E,
+    i64_atomic_rmw_add = 0x1F,
+    i32_atomic_rmw8_add_u = 0x20,
+    i32_atomic_rmw16_add_u = 0x21,
+    i64_atomic_rmw8_add_u = 0x22,
+    i64_atomic_rmw16_add_u = 0x23,
+    i64_atomic_rmw32_add_u = 0x24,
+    i32_atomic_rmw_sub = 0x25,
+    i64_atomic_rmw_sub = 0x26,
+    i32_atomic_rmw8_sub_u = 0x27,
+    i32_atomic_rmw16_sub_u = 0x28,
+    i64_atomic_rmw8_sub_u = 0x29,
+    i64_atomic_rmw16_sub_u = 0x2A,
+    i64_atomic_rmw32_sub_u = 0x2B,
+    i32_atomic_rmw_and = 0x2C,
+    i64_atomic_rmw_and = 0x2D,
+    i32_atomic_rmw8_and_u = 0x2E,
+    i32_atomic_rmw16_and_u = 0x2F,
+    i64_atomic_rmw8_and_u = 0x30,
+    i64_atomic_rmw16_and_u = 0x31,
+    i64_atomic_rmw32_and_u = 0x32,
+    i32_atomic_rmw_or = 0x33,
+    i64_atomic_rmw_or = 0x34,
+    i32_atomic_rmw8_or_u = 0x35,
+    i32_atomic_rmw16_or_u = 0x36,
+    i64_atomic_rmw8_or_u = 0x37,
+    i64_atomic_rmw16_or_u = 0x38,
+    i64_atomic_rmw32_or_u = 0x39,
+    i32_atomic_rmw_xor = 0x3A,
+    i64_atomic_rmw_xor = 0x3B,
+    i32_atomic_rmw8_xor_u = 0x3C,
+    i32_atomic_rmw16_xor_u = 0x3D,
+    i64_atomic_rmw8_xor_u = 0x3E,
+    i64_atomic_rmw16_xor_u = 0x3F,
+    i64_atomic_rmw32_xor_u = 0x40,
+    i32_atomic_rmw_xchg = 0x41,
+    i64_atomic_rmw_xchg = 0x42,
+    i32_atomic_rmw8_xchg_u = 0x43,
+    i32_atomic_rmw16_xchg_u = 0x44,
+    i64_atomic_rmw8_xchg_u = 0x45,
+    i64_atomic_rmw16_xchg_u = 0x46,
+    i64_atomic_rmw32_xchg_u = 0x47,
+    i32_atomic_rmw_cmpxchg = 0x48,
+    i64_atomic_rmw_cmpxchg = 0x49,
+    i32_atomic_rmw8_cmpxchg_u = 0x4A,
+    i32_atomic_rmw16_cmpxchg_u = 0x4B,
+    i64_atomic_rmw8_cmpxchg_u = 0x4C,
+    i64_atomic_rmw16_cmpxchg_u = 0x4D,
+    i64_atomic_rmw32_cmpxchg_u = 0x4E,
+}

src/mono/browser/runtime/jiterpreter-support.ts

@@ -4,7 +4,7 @@
 import WasmEnableThreads from "consts:wasmEnableThreads";
 import { NativePointer, ManagedPointer, VoidPtr } from "./types/emscripten";
 import { Module, mono_assert, runtimeHelpers } from "./globals";
-import { WasmOpcode, WasmSimdOpcode, WasmValtype } from "./jiterpreter-opcodes";
+import { WasmOpcode, WasmSimdOpcode, WasmAtomicOpcode, WasmValtype } from "./jiterpreter-opcodes";
 import { MintOpcode } from "./mintops";
 import cwraps from "./cwraps";
 import { mono_log_error, mono_log_info } from "./logging";
@@ -105,6 +105,9 @@ export class WasmBuilder {
     nextConstantSlot = 0;
     backBranchTraceLevel = 0;
 
+    containsSimd!: boolean;
+    containsAtomics!: boolean;
+
     compressImportNames = false;
     lockImports = false;
 
@@ -153,6 +156,9 @@ export class WasmBuilder {
         this.callHandlerReturnAddresses.length = 0;
 
         this.allowNullCheckOptimization = this.options.eliminateNullChecks;
+
+        this.containsSimd = false;
+        this.containsAtomics = false;
     }
 
     _push () {
@@ -257,11 +263,18 @@ export class WasmBuilder {
 
     appendSimd (value: WasmSimdOpcode, allowLoad?: boolean) {
         this.current.appendU8(WasmOpcode.PREFIX_simd);
-        // Yes that's right. We're using LEB128 to encode 8-bit opcodes. Why? I don't know
         mono_assert(((value | 0) !== 0) || ((value === WasmSimdOpcode.v128_load) && (allowLoad === true)), "Expected non-v128_load simd opcode or allowLoad==true");
+        // Yes that's right. We're using LEB128 to encode 8-bit opcodes. Why? I don't know
         return this.current.appendULeb(value);
     }
 
+    appendAtomic (value: WasmAtomicOpcode, allowNotify?: boolean) {
+        this.current.appendU8(WasmOpcode.PREFIX_atomic);
+        mono_assert(((value | 0) !== 0) || ((value === WasmAtomicOpcode.memory_atomic_notify) && (allowNotify === true)), "Expected non-notify atomic opcode or allowNotify==true");
+        // Unlike SIMD, the spec appears to say that atomic opcodes are just two sequential bytes with explicit values.
+        return this.current.appendU8(value);
+    }
+
     appendU32 (value: number) {
         return this.current.appendU32(value);
     }
@@ -517,7 +530,7 @@ export class WasmBuilder {
             // memtype (limits = 0x03 n:u32 m:u32    => {min n, max m, shared})
             this.appendU8(0x02);
             this.appendU8(0x03);
-            // emcc seems to generate this min/max by default
+            // HACK: emcc seems to generate this min/max by default
             this.appendULeb(256);
             this.appendULeb(32768);
         } else {
@@ -1900,6 +1913,7 @@ export type JiterpreterOptions = {
     enableCallResume: boolean;
     enableWasmEh: boolean;
     enableSimd: boolean;
+    enableAtomics: boolean;
     zeroPageOptimization: boolean;
     cprop: boolean;
     // For locations where the jiterpreter heuristic says we will be unable to generate
@@ -1946,6 +1960,7 @@ const optionNames: { [jsName: string]: string } = {
     "enableCallResume": "jiterpreter-call-resume-enabled",
     "enableWasmEh": "jiterpreter-wasm-eh-enabled",
     "enableSimd": "jiterpreter-simd-enabled",
+    "enableAtomics": "jiterpreter-atomics-enabled",
     "zeroPageOptimization": "jiterpreter-zero-page-optimization",
     "cprop": "jiterpreter-constant-propagation",
     "enableStats": "jiterpreter-stats-enabled",

src/mono/browser/runtime/jiterpreter-tables.ts

@@ -2,7 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 import {
-    WasmOpcode, WasmSimdOpcode, JiterpSpecialOpcode
+    WasmOpcode, WasmSimdOpcode, WasmAtomicOpcode, JiterpSpecialOpcode
 } from "./jiterpreter-opcodes";
 import {
     MintOpcode, SimdIntrinsic2, SimdIntrinsic3, SimdIntrinsic4
@@ -325,6 +325,24 @@ export const mathIntrinsicTable: { [opcode: number]: [isUnary: boolean, isF32: b
     [MintOpcode.MINT_REM_R4]:   [false, true, "fmodf"],
 };
 
+export const xchgTable: { [opcode: number]: [wasmOpcode: WasmAtomicOpcode, resultFixupOpcode: WasmOpcode, alignmentPower: number] } = {
+    [MintOpcode.MINT_MONO_EXCHANGE_U1]: [WasmAtomicOpcode.i32_atomic_rmw8_xchg_u, WasmOpcode.unreachable, 0],
+    [MintOpcode.MINT_MONO_EXCHANGE_I1]: [WasmAtomicOpcode.i32_atomic_rmw8_xchg_u, WasmOpcode.i32_extend_8_s, 0],
+    [MintOpcode.MINT_MONO_EXCHANGE_U2]: [WasmAtomicOpcode.i32_atomic_rmw16_xchg_u, WasmOpcode.unreachable, 1],
+    [MintOpcode.MINT_MONO_EXCHANGE_I2]: [WasmAtomicOpcode.i32_atomic_rmw16_xchg_u, WasmOpcode.i32_extend_16_s, 1],
+    [MintOpcode.MINT_MONO_EXCHANGE_I4]: [WasmAtomicOpcode.i32_atomic_rmw_xchg, WasmOpcode.unreachable, 2],
+    [MintOpcode.MINT_MONO_EXCHANGE_I8]: [WasmAtomicOpcode.i64_atomic_rmw_xchg, WasmOpcode.unreachable, 3],
+};
+
+export const cmpxchgTable: { [opcode: number]: [wasmOpcode: WasmAtomicOpcode, resultFixupOpcode: WasmOpcode, alignmentPower: number] } = {
+    [MintOpcode.MINT_MONO_CMPXCHG_U1]: [WasmAtomicOpcode.i32_atomic_rmw8_cmpxchg_u, WasmOpcode.unreachable, 0],
+    [MintOpcode.MINT_MONO_CMPXCHG_I1]: [WasmAtomicOpcode.i32_atomic_rmw8_cmpxchg_u, WasmOpcode.i32_extend_8_s, 0],
+    [MintOpcode.MINT_MONO_CMPXCHG_U2]: [WasmAtomicOpcode.i32_atomic_rmw16_cmpxchg_u, WasmOpcode.unreachable, 1],
+    [MintOpcode.MINT_MONO_CMPXCHG_I2]: [WasmAtomicOpcode.i32_atomic_rmw16_cmpxchg_u, WasmOpcode.i32_extend_16_s, 1],
+    [MintOpcode.MINT_MONO_CMPXCHG_I4]: [WasmAtomicOpcode.i32_atomic_rmw_cmpxchg, WasmOpcode.unreachable, 2],
+    [MintOpcode.MINT_MONO_CMPXCHG_I8]: [WasmAtomicOpcode.i64_atomic_rmw_cmpxchg, WasmOpcode.unreachable, 3],
+};
+
 export const simdCreateSizes = {
     [MintOpcode.MINT_SIMD_V128_I1_CREATE]: 1,
     [MintOpcode.MINT_SIMD_V128_I2_CREATE]: 2,

src/mono/browser/runtime/jiterpreter-trace-generator.ts

@@ -54,6 +54,7 @@ import {
     bitmaskTable, createScalarTable,
     simdExtractTable, simdReplaceTable,
     simdLoadTable, simdStoreTable,
+    xchgTable, cmpxchgTable,
 } from "./jiterpreter-tables";
 import { mono_log_error, mono_log_info } from "./logging";
 import { mono_assert, runtimeHelpers } from "./globals";
@@ -244,7 +245,6 @@ export function generateWasmBody (
 ): number {
     const abort = <MintOpcodePtr><any>0;
     let isFirstInstruction = true, isConditionallyExecuted = false,
-        containsSimd = false,
         pruneOpcodes = false, hasEmittedUnreachable = false;
     let result = 0,
         prologueOpcodeCounter = 0,
@@ -1465,26 +1465,6 @@ export function generateWasmBody (
                 break;
             }
 
-            case MintOpcode.MINT_MONO_CMPXCHG_I4:
-                builder.local("pLocals");
-                append_ldloc(builder, getArgU16(ip, 2), WasmOpcode.i32_load); // dest
-                append_ldloc(builder, getArgU16(ip, 3), WasmOpcode.i32_load); // newVal
-                append_ldloc(builder, getArgU16(ip, 4), WasmOpcode.i32_load); // expected
-                builder.callImport("cmpxchg_i32");
-                append_stloc_tail(builder, getArgU16(ip, 1), WasmOpcode.i32_store);
-                break;
-            case MintOpcode.MINT_MONO_CMPXCHG_I8:
-                // because i64 values can't pass through JS cleanly (c.f getRawCwrap and
-                // EMSCRIPTEN_KEEPALIVE), we pass addresses of newVal, expected and the return value
-                // to the helper function.  The "dest" for the compare-exchange is already a
-                // pointer, so load it normally
-                append_ldloc(builder, getArgU16(ip, 2), WasmOpcode.i32_load); // dest
-                append_ldloca(builder, getArgU16(ip, 3), 0); // newVal
-                append_ldloca(builder, getArgU16(ip, 4), 0); // expected
-                append_ldloca(builder, getArgU16(ip, 1), 8); // oldVal
-                builder.callImport("cmpxchg_i64");
-                break;
-
             case MintOpcode.MINT_LOG2_I4:
             case MintOpcode.MINT_LOG2_I8: {
                 const isI64 = (opcode === MintOpcode.MINT_LOG2_I8);
@@ -1647,13 +1627,19 @@ export function generateWasmBody (
                     (opcode >= MintOpcode.MINT_SIMD_V128_LDC) &&
                     (opcode <= MintOpcode.MINT_SIMD_INTRINS_P_PPP)
                 ) {
+                    builder.containsSimd = true;
                     if (!emit_simd(builder, ip, opcode, opname, simdIntrinsArgCount, simdIntrinsIndex))
                         ip = abort;
-                    else {
-                        containsSimd = true;
+                    else
                         // We need to do dreg invalidation differently for simd, especially to handle ldc
                         skipDregInvalidation = true;
-                    }
+                } else if (
+                    (opcode >= MintOpcode.MINT_MONO_MEMORY_BARRIER) &&
+                    (opcode <= MintOpcode.MINT_MONO_CMPXCHG_I8)
+                ) {
+                    builder.containsAtomics = true;
+                    if (!emit_atomics(builder, ip, opcode))
+                        ip = abort;
                 } else if (opcodeValue === 0) {
                     // This means it was explicitly marked as no-value in the opcode value table
                     //  so we can just skip over it. This is done for things like nops.
@@ -1740,7 +1726,7 @@ export function generateWasmBody (
     // HACK: Traces containing simd will be *much* shorter than non-simd traces,
     //  which will cause both the heuristic and our length requirement outside
     //  to reject them. For now, just add a big constant to the length
-    if (containsSimd)
+    if (builder.containsSimd)
         result += 10240;
     return result;
 }
@@ -3963,3 +3949,53 @@ function emit_simd_4 (builder: WasmBuilder, ip: MintOpcodePtr, index: SimdIntrin
             return false;
     }
 }
+
+function emit_atomics (
+    builder: WasmBuilder, ip: MintOpcodePtr, opcode: number
+) {
+    if (!builder.options.enableAtomics)
+        return false;
+
+    // FIXME: memory barrier might be worthwhile to implement
+    // FIXME: We could probably unify most of the xchg/cmpxchg implementation into one implementation
+
+    const xchg = xchgTable[opcode];
+    if (xchg) {
+        const is64 = xchg[2] > 2;
+        // TODO: Generate alignment check to produce a better runtime error when address is not aligned?
+        builder.local("pLocals"); // stloc head
+        append_ldloc_cknull(builder, getArgU16(ip, 2), ip, true); // address
+        append_ldloc(builder, getArgU16(ip, 3), is64 ? WasmOpcode.i64_load : WasmOpcode.i32_load); // replacement
+        builder.appendAtomic(xchg[0], false);
+        builder.appendMemarg(0, xchg[2]);
+        // Fixup the result if necessary
+        if (xchg[1] !== WasmOpcode.unreachable)
+            builder.appendU8(xchg[1]);
+        // store old value
+        append_stloc_tail(builder, getArgU16(ip, 1), is64 ? WasmOpcode.i64_store : WasmOpcode.i32_store);
+        return true;
+    }
+
+    const cmpxchg = cmpxchgTable[opcode];
+    if (cmpxchg) {
+        const is64 = cmpxchg[2] > 2;
+        // TODO: Generate alignment check to produce a better runtime error when address is not aligned?
+        builder.local("pLocals"); // stloc head
+        append_ldloc_cknull(builder, getArgU16(ip, 2), ip, true); // address
+        // FIXME: Do these loads need to be sized? I think it's well-defined even if there are garbage bytes in the i32,
+        //  based on language from the spec that looks like this: 'expected wrapped from i32 to i8, 8-bit compare equal'
+        append_ldloc(builder, getArgU16(ip, 4), is64 ? WasmOpcode.i64_load : WasmOpcode.i32_load); // expected
+        append_ldloc(builder, getArgU16(ip, 3), is64 ? WasmOpcode.i64_load : WasmOpcode.i32_load); // replacement
+        builder.appendAtomic(cmpxchg[0], false);
+        builder.appendMemarg(0, cmpxchg[2]);
+        // Fixup the result if necessary
+        if (cmpxchg[1] !== WasmOpcode.unreachable)
+            builder.appendU8(cmpxchg[1]);
+        // store old value
+        append_stloc_tail(builder, getArgU16(ip, 1), is64 ? WasmOpcode.i64_store : WasmOpcode.i32_store);
+        return true;
+    }
+
+    return false;
+}
+

src/mono/browser/runtime/jiterpreter.ts

@@ -283,8 +283,6 @@ function getTraceImports () {
         importDef("array_rank", getRawCwrap("mono_jiterp_get_array_rank")),
         ["a_elesize", "array_rank", getRawCwrap("mono_jiterp_get_array_element_size")],
         importDef("stfld_o", getRawCwrap("mono_jiterp_set_object_field")),
-        importDef("cmpxchg_i32", getRawCwrap("mono_jiterp_cas_i32")),
-        importDef("cmpxchg_i64", getRawCwrap("mono_jiterp_cas_i64")),
         ["stelemr_tc", "stelemr", getRawCwrap("mono_jiterp_stelem_ref")],
         importDef("fma", getRawCwrap("fma")),
         importDef("fmaf", getRawCwrap("fmaf")),
@@ -629,25 +627,6 @@ function initialize_builder (builder: WasmBuilder) {
         },
         WasmValtype.void, true
     );
-    builder.defineType(
-        "cmpxchg_i32",
-        {
-            "dest": WasmValtype.i32,
-            "newVal": WasmValtype.i32,
-            "expected": WasmValtype.i32,
-        },
-        WasmValtype.i32, true
-    );
-    builder.defineType(
-        "cmpxchg_i64",
-        {
-            "dest": WasmValtype.i32,
-            "newVal": WasmValtype.i32,
-            "expected": WasmValtype.i32,
-            "oldVal": WasmValtype.i32,
-        },
-        WasmValtype.void, true
-    );
     builder.defineType(
         "stelemr",
         {
@@ -896,7 +875,12 @@ function generate_wasm (
     } catch (exc: any) {
         threw = true;
         rejected = false;
-        mono_log_error(`${methodFullName || traceName} code generation failed: ${exc} ${exc.stack}`);
+        let desc = builder.containsSimd
+            ? " (simd)"
+            : "";
+        if (builder.containsAtomics)
+            desc += " (atomics)";
+        mono_log_error(`${methodFullName || traceName}${desc} code generation failed: ${exc} ${exc.stack}`);
         recordFailure();
         return 0;
     } finally {