[AArch64] Assembly support for VMSA

Virtual Memory System Architecture (VMSA)

This is part of the 2022 A-Profile Architecture extensions and adds support for
the following:

 - Translation Hardening Extension (FEAT_THE)
 - 128-bit Page Table Descriptors (FEAT_D128)
 - 56-bit Virtual Address (FEAT_LVA3)
 - Support for 128-bit System Registers (FEAT_SYSREG128)
 - System Instructions that can take 128-bit inputs (FEAT_SYSINSTR128)
 - 128-bit Atomic Instructions (FEAT_LSE128)
 - Permission Indirection Extension (FEAT_S1PIE, FEAT_S2PIE)
 - Permission Overlay Extension (FEAT_S1POE, FEAT_S2POE)
 - Memory Attribute Index Enhancement (FEAT_AIE)

New instructions added:
 - FEAT_SYSREG128 adds MRRS and MSRR.
 - FEAT_SYSINSTR128 adds the SYSP instruction and TLBIP aliases.
 - FEAT_LSE128 adds LDCLRP, LDSET, and SWPP instructions.
 - FEAT_THE adds the set of RCW* instructions.

Specs for individual instructions can be found here:
https://developer.arm.com/documentation/ddi0602/2022-09/Base-Instructions/

Contributors:
  Keith Walker
  Lucas Prates
  Sam Elliott
  Son Tuan Vu
  Tomas Matheson

Differential Revision: https://reviews.llvm.org/D138920

clang/lib/Basic/Targets/AArch64.cpp

@@ -461,6 +461,9 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
   if (HasMOPS)
     Builder.defineMacro("__ARM_FEATURE_MOPS", "1");
 
+  if (HasD128)
+    Builder.defineMacro("__ARM_FEATURE_SYSREG128", "1");
+
   switch (ArchKind) {
   default:
     break;
@@ -596,6 +599,7 @@ bool AArch64TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
   HasMatmulFP32 = false;
   HasLSE = false;
   HasMOPS = false;
+  HasD128 = false;
   HasRCPC = false;
 
   ArchKind = llvm::AArch64::ArchKind::INVALID;
@@ -719,6 +723,16 @@ bool AArch64TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasFlagM = true;
     if (Feature == "+mops")
       HasMOPS = true;
+    if (Feature == "+d128")
+      HasD128 = true;
+  }
+
+  // Check features that are manually disabled by command line options.
+  // This needs to be checked after architecture-related features are handled,
+  // making sure they are properly disabled when required.
+  for (const auto &Feature : Features) {
+    if (Feature == "-d128")
+      HasD128 = false;
   }
 
   setDataLayout();

clang/lib/Basic/Targets/AArch64.h

@@ -53,6 +53,7 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo {
   bool HasLSE;
   bool HasFlagM;
   bool HasMOPS;
+  bool HasD128;
   bool HasRCPC;
 
   llvm::AArch64::ArchKind ArchKind;

clang/test/Driver/aarch64-d128.c

@@ -0,0 +1,10 @@
+// Test that target feature d128 is implemented and available correctly
+
+// FEAT_D128 is optional (off by default) for v9.4a and older, and can be enabled using +d128
+// RUN: %clang -### -target aarch64-none-none-eabi -march=armv9.4-a        %s 2>&1 | FileCheck %s --check-prefix=NOT_ENABLED
+// RUN: %clang -### -target aarch64-none-none-eabi -march=armv9.4-a+d128   %s 2>&1 | FileCheck %s --check-prefix=ENABLED
+// RUN: %clang -### -target aarch64-none-none-eabi -march=armv9.4-a+nod128 %s 2>&1 | FileCheck %s --check-prefix=DISABLED
+
+// ENABLED: "-target-feature" "+d128"
+// NOT_ENABLED-NOT: "-target-feature" "+d128"
+// DISABLED: "-target-feature" "-d128"

clang/test/Driver/aarch64-lse128.c

@@ -0,0 +1,10 @@
+// Test that target feature lse128 is implemented and available correctly
+
+// FEAT_LSE128 is optional (off by default) for v9.4a and older, and can be enabled using +lse128
+// RUN: %clang -### -target aarch64-none-none-eabi -march=armv9.4-a          %s 2>&1 | FileCheck %s --check-prefix=NOT_ENABLED
+// RUN: %clang -### -target aarch64-none-none-eabi -march=armv9.4-a+lse128   %s 2>&1 | FileCheck %s --check-prefix=ENABLED
+// RUN: %clang -### -target aarch64-none-none-eabi -march=armv9.4-a+nolse128 %s 2>&1 | FileCheck %s --check-prefix=DISABLED
+
+// ENABLED: "-target-feature" "+lse128"
+// NOT_ENABLED-NOT: "-target-feature" "+lse128"
+// DISABLED: "-target-feature" "-lse128"

clang/test/Driver/aarch64-the.c

@@ -0,0 +1,26 @@
+// Test that target feature the is implemented and available correctly
+
+// FEAT_THE is optional (off by default) for v8.9a/9.4a, and can be disabled using +nothe
+// RUN: %clang -### -target aarch64-none-none-eabi -march=armv8.9-a       %s 2>&1 | FileCheck %s --check-prefix=NOT_ENABLED
+// RUN: %clang -### -target aarch64-none-none-eabi -march=armv8.9-a+the   %s 2>&1 | FileCheck %s --check-prefix=ENABLED
+// RUN: %clang -### -target aarch64-none-none-eabi -march=armv8.9-a+nothe %s 2>&1 | FileCheck %s --check-prefix=DISABLED
+// RUN: %clang -### -target aarch64-none-none-eabi -march=armv9.4-a       %s 2>&1 | FileCheck %s --check-prefix=NOT_ENABLED
+// RUN: %clang -### -target aarch64-none-none-eabi -march=armv9.4-a+the   %s 2>&1 | FileCheck %s --check-prefix=ENABLED
+// RUN: %clang -### -target aarch64-none-none-eabi -march=armv9.4-a+nothe %s 2>&1 | FileCheck %s --check-prefix=DISABLED
+
+// FEAT_THE is optional (off by default) for v8.8a/9.3a and older, and can be enabled using +the
+// RUN: %clang -### -target aarch64-none-none-eabi -march=armv8.8-a       %s 2>&1 | FileCheck %s --check-prefix=NOT_ENABLED
+// RUN: %clang -### -target aarch64-none-none-eabi -march=armv8.8-a+the   %s 2>&1 | FileCheck %s --check-prefix=ENABLED
+// RUN: %clang -### -target aarch64-none-none-eabi -march=armv8.8-a+nothe %s 2>&1 | FileCheck %s --check-prefix=DISABLED
+// RUN: %clang -### -target aarch64-none-none-eabi -march=armv9.3-a       %s 2>&1 | FileCheck %s --check-prefix=NOT_ENABLED
+// RUN: %clang -### -target aarch64-none-none-eabi -march=armv9.3-a+the   %s 2>&1 | FileCheck %s --check-prefix=ENABLED
+// RUN: %clang -### -target aarch64-none-none-eabi -march=armv9.3-a+nothe %s 2>&1 | FileCheck %s --check-prefix=DISABLED
+
+// FEAT_THE is invalid before v8
+// RUN: %clang -### -target arm-none-none-eabi -march=armv7-a+the     %s 2>&1 | FileCheck %s --check-prefix=INVALID
+
+// INVALID: error: unsupported argument 'armv7-a+the' to option '-march='
+// ENABLED: "-target-feature" "+the"
+// NOT_ENABLED-NOT: "-target-feature" "+the"
+// DISABLED: "-target-feature" "-the"
+

llvm/include/llvm/Support/AArch64TargetParser.def

@@ -153,6 +153,9 @@ AARCH64_ARCH_EXT_NAME("mops",         AArch64::AEK_MOPS,        "+mops",
 AARCH64_ARCH_EXT_NAME("pmuv3",        AArch64::AEK_PERFMON,     "+perfmon",      "-perfmon")
 AARCH64_ARCH_EXT_NAME("cssc",         AArch64::AEK_CSSC,        "+cssc",         "-cssc")
 AARCH64_ARCH_EXT_NAME("rcpc3",        AArch64::AEK_RCPC3,       "+rcpc3",        "-rcpc3")
+AARCH64_ARCH_EXT_NAME("the",          AArch64::AEK_THE,         "+the",          "-the")
+AARCH64_ARCH_EXT_NAME("d128",         AArch64::AEK_D128,        "+d128",         "-d128")
+AARCH64_ARCH_EXT_NAME("lse128",       AArch64::AEK_LSE128,      "+lse128",       "-lse128")
 #undef AARCH64_ARCH_EXT_NAME
 
 #ifndef AARCH64_CPU_NAME

llvm/include/llvm/Support/AArch64TargetParser.h

@@ -78,6 +78,9 @@ enum ArchExtKind : uint64_t {
   AEK_SMEF16F16 =   1ULL << 47, // FEAT_SMEF16F16
   AEK_CSSC =        1ULL << 48, // FEAT_CSSC
   AEK_RCPC3 =       1ULL << 49, // FEAT_LRCPC3
+  AEK_THE =         1ULL << 50, // FEAT_THE
+  AEK_D128 =        1ULL << 51, // FEAT_D128
+  AEK_LSE128 =      1ULL << 52, // FEAT_LSE128
 };
 
 enum class ArchKind {

llvm/lib/Target/AArch64/AArch64.td

@@ -515,6 +515,20 @@ def FeatureRCPC3 : SubtargetFeature<"rcpc3", "HasRCPC3",
     "true", "Enable Armv8.9-A RCPC instructions for A64 and Advanced SIMD and floating-point instruction set (FEAT_LRCPC3)",
     [FeatureRCPC_IMMO]>;
 
+def FeatureTHE : SubtargetFeature<"the", "HasTHE",
+    "true", "Enable Armv8.9-A Translation Hardening Extension (FEAT_THE)">;
+
+def FeatureLSE128 : SubtargetFeature<"lse128", "HasLSE128",
+    "true", "Enable Armv9.4-A 128-bit Atomic Instructions (FEAT_LSE128)",
+    [FeatureLSE]>;
+
+// FEAT_D128, FEAT_LVA3, FEAT_SYSREG128, and FEAT_SYSINSTR128 are mutually implicit.
+// Therefore group them all under a single feature flag, d128:
+def FeatureD128 : SubtargetFeature<"d128", "HasD128",
+    "true", "Enable Armv9.4-A 128-bit Page Table Descriptors, System Registers "
+    "and Instructions (FEAT_D128, FEAT_LVA3, FEAT_SYSREG128, FEAT_SYSINSTR128)",
+    [FeatureLSE128]>;
+
 //===----------------------------------------------------------------------===//
 // Architectures.
 //

llvm/lib/Target/AArch64/AArch64InstrFormats.td

@@ -11842,6 +11842,247 @@ def TRCIT : RtSystemI<0b0, (outs), (ins GPR64:$Rt), "trcit", "\t$Rt"> {
   let Inst{7-5}   = 0b111;
 }
 
+// * RCWCAS family
+// * RCW<OP> family
+
+//--------------------------------------------------------------------
+// Read-Check-Write Compare And Swap family (RCWCAS[S|P|PS]?[A|L|AL]?)
+
+// Instruction encoding:
+//
+//          31 30|29  24|23|22|21|20 16|15|14 13|12 11 10|9 5|4 0
+// RCWCAS    0  0|011001| A| R| 1|   Rs| 0| 0  0| 0  1  0| Rn| Rt
+// RCWSCAS   0  1|011001| A| R| 1|   Rs| 0| 0  0| 0  1  0| Rn| Rt
+// RCWCASP   0  0|011001| A| R| 1|   Rs| 0| 0  0| 0  1  1| Rn| Rt
+// RCWSCASP  0  1|011001| A| R| 1|   Rs| 0| 0  0| 0  1  1| Rn| Rt
+
+// Instruction syntax:
+//
+// RCW[S]CAS{<order>}   <Xs>,           <Xt>,          [<Xn|SP>]
+// RCW[S]CASP{<order>}  <Xs>, <X(s+1)>, <Xt>, <X(t+1)> [<Xn|SP>]
+
+class BaseRCWCASEncoding<dag oops, dag iops, string asm>
+      : I<oops, iops, asm, "\t$Rs, $Rt, [$Rn]", "$out = $Rs", []>,
+        Sched<[]> {
+  bit Acq;
+  bit Rel;
+  bit SC;
+  bit Pair;
+  bits<5> Rs;
+  bits<5> Rn;
+  bits<5> Rt;
+  let Inst{31} = 0b0;
+  let Inst{30} = SC;
+  let Inst{29-24} = 0b011001;
+  let Inst{23} = Acq;
+  let Inst{22} = Rel;
+  let Inst{21} = 0b1;
+  let Inst{20-16} = Rs;
+  let Inst{15-13} = 0b000;
+  let Inst{12-11} = 0b01;
+  let Inst{10} = Pair;
+  let Inst{9-5} = Rn;
+  let Inst{4-0} = Rt;
+  let mayLoad = 1;
+  let mayStore = 1;
+  let hasSideEffects = 1;
+  let Defs = [NZCV];
+}
+
+multiclass BaseRCWCAS<dag oops, dag iops, string prefix> {
+  let Acq = 0b0, Rel = 0b0 in
+    def "" : BaseRCWCASEncoding<oops, iops, prefix # "">;
+  let Acq = 0b1, Rel = 0b0 in
+    def A  : BaseRCWCASEncoding<oops, iops, prefix # "a">;
+  let Acq = 0b0, Rel = 0b1 in
+    def L  : BaseRCWCASEncoding<oops, iops, prefix # "l">;
+  let Acq = 0b1, Rel = 0b1 in
+    def AL : BaseRCWCASEncoding<oops, iops, prefix # "al">;
+}
+
+multiclass ReadCheckWriteCompareAndSwap {
+  let SC = 0b0, Pair = 0b0, Predicates = [HasTHE] in
+    defm CAS  : BaseRCWCAS<(outs GPR64:$out),
+                           (ins GPR64:$Rs, GPR64:$Rt, GPR64sp:$Rn), "rcwcas" >;
+  let SC = 0b1, Pair = 0b0, Predicates = [HasTHE] in
+    defm SCAS : BaseRCWCAS<(outs GPR64:$out),
+                           (ins GPR64:$Rs, GPR64:$Rt, GPR64sp:$Rn), "rcwscas">;
+  let SC = 0b0, Pair = 0b1, Predicates = [HasTHE, HasD128] in
+    defm CASP : BaseRCWCAS<(outs XSeqPairClassOperand:$out),
+                           (ins XSeqPairClassOperand:$Rs,
+                                XSeqPairClassOperand:$Rt, GPR64sp:$Rn),
+                           "rcwcasp">;
+  let SC = 0b1, Pair = 0b1, Predicates = [HasTHE, HasD128] in
+    defm SCASP: BaseRCWCAS<(outs XSeqPairClassOperand:$out),
+                           (ins XSeqPairClassOperand:$Rs,
+                                XSeqPairClassOperand:$Rt, GPR64sp:$Rn),
+                           "rcwscasp">;
+}
+
+//------------------------------------------------------------------
+// Read-Check-Write <OP> family (RCW[CLR|SET|SWP][S|P|PS]?[A|L|AL]?)
+
+// Instruction encoding:
+//
+//          31 30|29  24|23|22|21|20 16|15|14 12|11 10|9 5|4 0
+// RCWCLR    0  0|111000| A| R| 1|   Rs| 1|  001| 0  0| Rn| Rt
+// RCWSCLR   0  1|111000| A| R| 1|   Rs| 1|  001| 0  0| Rn| Rt
+// RCWSET    0  0|111000| A| R| 1|   Rs| 1|  011| 0  0| Rn| Rt
+// RCWSSET   0  1|111000| A| R| 1|   Rs| 1|  011| 0  0| Rn| Rt
+// RCWSWP    0  0|111000| A| R| 1|   Rs| 1|  010| 0  0| Rn| Rt
+// RCWSSWP   0  1|111000| A| R| 1|   Rs| 1|  010| 0  0| Rn| Rt
+
+//          31 30|29  24|23|22|21|20 16|15|14 12|11 10|9 5|4 0
+// RCWCLRP   0  0|011001| A| R| 1|  Rt2| 1|  001| 0  0| Rn| Rt
+// RCWSCLRP  0  1|011001| A| R| 1|  Rt2| 1|  001| 0  0| Rn| Rt
+// RCWSETP   0  0|011001| A| R| 1|  Rt2| 1|  011| 0  0| Rn| Rt
+// RCWSSETP  0  1|011001| A| R| 1|  Rt2| 1|  011| 0  0| Rn| Rt
+// RCWSWPP   0  0|011001| A| R| 1|  Rt2| 1|  010| 0  0| Rn| Rt
+// RCWSSWPP  0  1|011001| A| R| 1|  Rt2| 1|  010| 0  0| Rn| Rt
+
+// Instruction syntax:
+//
+// RCW[S]<OP>{<order>}   <Xs>,  <Xt>, [<Xn|SP>]
+// RCW[S]<OP>P{<order>}  <Xt1>, <Xt2>, [<Xn|SP>]
+
+class BaseRCWOPEncoding<string asm>
+      : I<(outs GPR64:$Rt),(ins GPR64:$Rs, GPR64sp:$Rn), asm,
+          "\t$Rs, $Rt, [$Rn]", "", []>,
+        Sched<[]> {
+  bit Acq;
+  bit Rel;
+  bit SC;
+  bits<3> opc;
+  bits<5> Rs;
+  bits<5> Rn;
+  bits<5> Rt;
+  let Inst{31} = 0b0;
+  let Inst{30} = SC;
+  let Inst{29-24} = 0b111000;
+  let Inst{23} = Acq;
+  let Inst{22} = Rel;
+  let Inst{21} = 0b1;
+  let Inst{20-16} = Rs;
+  let Inst{15} = 0b1;
+  let Inst{14-12} = opc;
+  let Inst{11-10} = 0b00;
+  let Inst{9-5} = Rn;
+  let Inst{4-0} = Rt;
+  let mayLoad = 1;
+  let mayStore = 1;
+  let hasSideEffects = 1;
+  let Defs = [NZCV];
+  let Predicates = [HasTHE];
+}
+
+class BaseRCWOPPEncoding<string asm>
+      : I<(outs GPR64common:$Rt_wb, GPR64common:$Rt2_wb),
+          (ins GPR64common:$Rt, GPR64common:$Rt2, GPR64sp:$Rn), asm,
+          "\t$Rt, $Rt2, [$Rn]", "$Rt = $Rt_wb, $Rt2 = $Rt2_wb", []>,
+        Sched<[]> {
+  bit Acq;
+  bit Rel;
+  bit SC;
+  bits<3> opc;
+  bits<5> Rt2;
+  bits<5> Rn;
+  bits<5> Rt;
+  let Inst{31} = 0b0;
+  let Inst{30} = SC;
+  let Inst{29-24} = 0b011001;
+  let Inst{23} = Acq;
+  let Inst{22} = Rel;
+  let Inst{21} = 0b1;
+  let Inst{20-16} = Rt2;
+  let Inst{15} = 0b1;
+  let Inst{14-12} = opc;
+  let Inst{11-10} = 0b00;
+  let Inst{9-5} = Rn;
+  let Inst{4-0} = Rt;
+  let mayLoad = 1;
+  let mayStore = 1;
+  let hasSideEffects = 1;
+  let Defs = [NZCV];
+  let Predicates = [HasTHE, HasD128];
+}
+
+multiclass BaseRCWOP<string prefix> {
+  let Acq = 0b0, Rel = 0b0 in def "" : BaseRCWOPEncoding<prefix # "">;
+  let Acq = 0b1, Rel = 0b0 in def A  : BaseRCWOPEncoding<prefix # "a">;
+  let Acq = 0b0, Rel = 0b1 in def L  : BaseRCWOPEncoding<prefix # "l">;
+  let Acq = 0b1, Rel = 0b1 in def AL : BaseRCWOPEncoding<prefix # "al">;
+
+  let Acq = 0b0, Rel = 0b0 in def P   : BaseRCWOPPEncoding<prefix # "p">;
+  let Acq = 0b1, Rel = 0b0 in def PA  : BaseRCWOPPEncoding<prefix # "pa">;
+  let Acq = 0b0, Rel = 0b1 in def PL  : BaseRCWOPPEncoding<prefix # "pl">;
+  let Acq = 0b1, Rel = 0b1 in def PAL : BaseRCWOPPEncoding<prefix # "pal">;
+}
+
+multiclass ReadCheckWriteOperation<bits<3> opc, string op> {
+  let SC = 0b0, opc = opc in defm ""  : BaseRCWOP<"rcw" # ""  # op>;
+  let SC = 0b1, opc = opc in defm S   : BaseRCWOP<"rcw" # "s" # op >;
+}
+
+//---
+// 128-bit atomic instructions (FEAT_LSE128)
+//---
+
+let mayLoad = 1, mayStore = 1, hasSideEffects = 0 in
+class LSE128Base<bits<3> op0, bits<2> AR, bit o3, string asm>
+: I<(outs GPR64common:$Rt_wb, GPR64common:$Rt2_wb),
+    (ins GPR64common:$Rt, GPR64common:$Rt2, GPR64sp:$Rn),
+    asm, "\t$Rt, $Rt2, [$Rn]",
+    "$Rt = $Rt_wb, $Rt2 = $Rt2_wb", []>,
+  Sched<[]> {
+  bits<5> Rt;
+  bits<5> Rt2;
+  bits<5> Rn;
+  let Inst{31-24} = 0b00011001;
+  let Inst{23-22} = AR;
+  let Inst{21} = 0b1;
+  let Inst{20-16} = Rt2;
+  let Inst{15} = o3;
+  let Inst{14-12} = op0;
+  let Inst{11-10} = 0b00;
+  let Inst{9-5} = Rn;
+  let Inst{4-0} = Rt;
+}
+
+//---
+// 128-bit System Instructions (FEAT_SYSINSTR128)
+//---
+
+// Instruction encoding:
+//
+//          31          19|18 16|15 12|11 8|7 5|4 0
+// SYSP      1101010101001|  op1|   Cn|  Cm|op2| Rt
+
+// Instruction syntax:
+//
+// SYSP #<op1>, <Cn>, <Cm>, #<op2>{, <Xt>, <Xt+1>}
+
+class RtSystemI128<bit L, dag oops, dag iops, string asm, string operands, list<dag> pattern = []> :
+  RtSystemI<L, oops, iops, asm, operands, pattern> {
+  let Inst{22}    = 0b1; // override BaseSystemI
+}
+
+class BaseSYSPEncoding<bit L, string asm, string operands, dag outputs, dag inputs>
+  : RtSystemI128<L, outputs, inputs, asm, operands> {
+  bits<3> op1;
+  bits<4> Cn;
+  bits<4> Cm;
+  bits<3> op2;
+  let Inst{20-19} = 0b01;
+  let Inst{18-16} = op1;
+  let Inst{15-12} = Cn;
+  let Inst{11-8}  = Cm;
+  let Inst{7-5}   = op2;
+}
+class SystemPXtI<bit L, string asm> :
+  BaseSYSPEncoding<L, asm, "\t$op1, $Cn, $Cm, $op2, $Rt", (outs),
+  (ins imm0_7:$op1, sys_cr_op:$Cn, sys_cr_op:$Cm, imm0_7:$op2, XSeqPairClassOperand:$Rt)>;
+
+
 //----------------------------------------------------------------------------
 // Allow the size specifier tokens to be upper case, not just lower.
 def : TokenAlias<".4B", ".4b">;  // Add dot product