[LLVM][AArch64] Add register classes for Armv9.6 assembly (#111717)

Add new register classes/operands and their encoder/decoder behaviour
required for the new Armv9.6 instructions (see
https://developer.arm.com/documentation/109697/2024_09/Feature-descriptions/The-Armv9-6-architecture-extension).

This work is the basis ofthe 2024 Armv9.6 architecture update effort for
SME.

Co-authored-by: Caroline Concatto [email protected]
Co-authored-by: Marian Lukac [email protected]
Co-authored-by: Momchil Velikov [email protected]

Author: SpencerAbson

llvm/lib/Target/AArch64/AArch64RegisterInfo.td

@@ -923,10 +923,9 @@ class ZPRRegOp <string Suffix, AsmOperandClass C, ElementSizeEnum Size,
 //******************************************************************************
 
 // SVE predicate register classes.
-class PPRClass<int firstreg, int lastreg> : RegisterClass<
-                                  "AArch64",
+class PPRClass<int firstreg, int lastreg, int step = 1> : RegisterClass<"AArch64",
                                   [ nxv16i1, nxv8i1, nxv4i1, nxv2i1, nxv1i1 ], 16,
-                                  (sequence "P%u", firstreg, lastreg)> {
+                                  (sequence "P%u", firstreg, lastreg, step)> {
   let Size = 16;
 }
 
@@ -940,6 +939,8 @@ def PPR_p8to15 : PPRClass<8, 15> {
   let DecoderMethod = "DecodeSimpleRegisterClass<AArch64::PNRRegClassID, 8, 8>";
 }
 
+def PPRMul2 : PPRClass<0, 14, 2>;
+
 class PPRAsmOperand <string name, string RegClass, int Width>: AsmOperandClass {
   let Name = "SVE" # name # "Reg";
   let PredicateMethod = "isSVEPredicateVectorRegOfWidth<"
@@ -1098,10 +1099,11 @@ class PPRVectorListMul<int ElementWidth, int NumRegs> : PPRVectorList<ElementWid
   let DiagnosticType = "Invalid" # Name;
   let PredicateMethod =
       "isTypedVectorListMultiple<RegKind::SVEPredicateVector, " # NumRegs # ", 0, "
-                                                                # ElementWidth # ">";
+                                                                # ElementWidth #
+                                                                ", AArch64::PPRMul2RegClassID>";
 }
 
-let EncoderMethod = "EncodeRegAsMultipleOf<2>",
+let EncoderMethod = "EncodeRegMul_MinMax<2, 0, 14>",
     DecoderMethod = "DecodePPR2Mul2RegisterClass" in {
   def PP_b_mul_r : RegisterOperand<PPR2Mul2, "printTypedVectorList<0,'b'>"> {
     let ParserMatchClass = PPRVectorListMul<8, 2>;
@@ -1124,23 +1126,28 @@ let EncoderMethod = "EncodeRegAsMultipleOf<2>",
 //******************************************************************************
 
 // SVE vector register classes
-class ZPRClass<int lastreg> : RegisterClass<"AArch64",
+class ZPRClass<int firstreg, int lastreg, int step = 1> : RegisterClass<"AArch64",
                                             [nxv16i8, nxv8i16, nxv4i32, nxv2i64,
                                              nxv2f16, nxv4f16, nxv8f16,
                                              nxv2bf16, nxv4bf16, nxv8bf16,
                                              nxv2f32, nxv4f32,
                                              nxv2f64],
-                                            128, (sequence "Z%u", 0, lastreg)> {
+                                            128, (sequence "Z%u", firstreg, lastreg, step)> {
   let Size = 128;
 }
 
-def ZPR    : ZPRClass<31> {
+def ZPRMul2    : ZPRClass<0,  30, 2>;
+def ZPRMul4    : ZPRClass<0,  28, 4>;
+def ZPRMul2_Lo : ZPRClass<0,  14, 2>;
+def ZPRMul2_Hi : ZPRClass<16, 30, 2>;
+
+def ZPR    : ZPRClass<0, 31> {
   let DecoderMethod = "DecodeSimpleRegisterClass<AArch64::ZPRRegClassID, 0, 32>";
 }
-def ZPR_4b : ZPRClass<15> { // Restricted 4 bit SVE vector register class.
+def ZPR_4b : ZPRClass<0, 15> { // Restricted 4 bit SVE vector register class.
   let DecoderMethod = "DecodeSimpleRegisterClass<AArch64::ZPRRegClassID, 0, 16>";
 }
-def ZPR_3b : ZPRClass<7> {  // Restricted 3 bit SVE vector register class.
+def ZPR_3b : ZPRClass<0, 7> {  // Restricted 3 bit SVE vector register class.
   let DecoderMethod = "DecodeSimpleRegisterClass<AArch64::ZPRRegClassID, 0, 8>";
 }
 
@@ -1188,6 +1195,39 @@ def ZPR4b16 : ZPRRegOp<"h", ZPRAsmOp4b16, ElementSizeH, ZPR_4b>;
 def ZPR4b32 : ZPRRegOp<"s", ZPRAsmOp4b32, ElementSizeS, ZPR_4b>;
 def ZPR4b64 : ZPRRegOp<"d", ZPRAsmOp4b64, ElementSizeD, ZPR_4b>;
 
+class ZPRMul2_MinToMaxRegOp<string Suffix, AsmOperandClass C, int  Min, int Max, ElementSizeEnum  Width, RegisterClass RC>
+    : ZPRRegOp<Suffix, C, Width, RC> {
+  let EncoderMethod = "EncodeRegMul_MinMax<2," # Min # ", " # Max # ">";
+  let DecoderMethod = "DecodeZPRMul2_MinMax<" # Min # ", " # Max # ">";
+}
+
+def ZPRMul2AsmOp8_Lo  : ZPRAsmOperand<"VectorB_Lo",  8, "Mul2_Lo">;
+def ZPRMul2AsmOp8_Hi  : ZPRAsmOperand<"VectorB_Hi",  8, "Mul2_Hi">;
+def ZPRMul2AsmOp16_Lo : ZPRAsmOperand<"VectorH_Lo", 16, "Mul2_Lo">;
+def ZPRMul2AsmOp16_Hi : ZPRAsmOperand<"VectorH_Hi", 16, "Mul2_Hi">;
+def ZPRMul2AsmOp32_Lo : ZPRAsmOperand<"VectorS_Lo", 32, "Mul2_Lo">;
+def ZPRMul2AsmOp32_Hi : ZPRAsmOperand<"VectorS_Hi", 32, "Mul2_Hi">;
+def ZPRMul2AsmOp64_Lo : ZPRAsmOperand<"VectorD_Lo", 64, "Mul2_Lo">;
+def ZPRMul2AsmOp64_Hi : ZPRAsmOperand<"VectorD_Hi", 64, "Mul2_Hi">;
+
+def ZPR_K : RegisterClass<"AArch64", [untyped], 128,
+                          (add Z20, Z21, Z22, Z23, Z28, Z29, Z30, Z31)>;
+
+def ZK : RegisterOperand<ZPR_K, "printSVERegOp<>">{
+  let EncoderMethod = "EncodeZK";
+  let DecoderMethod = "DecodeZK";
+  let ParserMatchClass = ZPRAsmOperand<"Vector_20to23or28to31", 0, "_K">;
+}
+
+def ZPR8Mul2_Lo  : ZPRMul2_MinToMaxRegOp<"b", ZPRMul2AsmOp8_Lo,   0, 14, ElementSizeB, ZPRMul2_Lo>;
+def ZPR8Mul2_Hi  : ZPRMul2_MinToMaxRegOp<"b", ZPRMul2AsmOp8_Hi,  16, 30, ElementSizeB, ZPRMul2_Hi>;
+def ZPR16Mul2_Lo : ZPRMul2_MinToMaxRegOp<"h", ZPRMul2AsmOp16_Lo,  0, 14, ElementSizeH, ZPRMul2_Lo>;
+def ZPR16Mul2_Hi : ZPRMul2_MinToMaxRegOp<"h", ZPRMul2AsmOp16_Hi, 16, 30, ElementSizeH, ZPRMul2_Hi>;
+def ZPR32Mul2_Lo : ZPRMul2_MinToMaxRegOp<"s", ZPRMul2AsmOp32_Lo,  0, 14, ElementSizeS, ZPRMul2_Lo>;
+def ZPR32Mul2_Hi : ZPRMul2_MinToMaxRegOp<"s", ZPRMul2AsmOp32_Hi, 16, 30, ElementSizeS, ZPRMul2_Hi>;
+def ZPR64Mul2_Lo : ZPRMul2_MinToMaxRegOp<"d", ZPRMul2AsmOp64_Lo,  0, 14, ElementSizeD, ZPRMul2_Lo>;
+def ZPR64Mul2_Hi : ZPRMul2_MinToMaxRegOp<"d", ZPRMul2AsmOp64_Hi, 16, 30, ElementSizeD, ZPRMul2_Hi>;
+
 class FPRasZPR<int Width> : AsmOperandClass{
   let Name = "FPR" # Width # "asZPR";
   let PredicateMethod = "isFPRasZPR<AArch64::FPR" # Width # "RegClassID>";
@@ -1327,64 +1367,117 @@ def ZPR4Mul4 : RegisterClass<"AArch64", [untyped], 128, (add (decimate ZSeqQuads
   let Size = 512;
 }
 
-class ZPRVectorListMul<int ElementWidth, int NumRegs> : ZPRVectorList<ElementWidth, NumRegs> {
-  let Name = "SVEVectorListMul" # NumRegs # "x" # ElementWidth;
+class ZPRVectorListMul<int ElementWidth, int NumRegs, string RegClassSuffix = "">
+  : ZPRVectorList<ElementWidth, NumRegs> {
+  let Name = "SVEVectorList" # NumRegs # "x" # ElementWidth # RegClassSuffix;
   let DiagnosticType = "Invalid" # Name;
   let PredicateMethod =
-      "isTypedVectorListMultiple<RegKind::SVEDataVector, " # NumRegs # ", 0, "
-                                                           # ElementWidth # ">";
+      "isTypedVectorListMultiple<RegKind::SVEDataVector, "
+                                 # NumRegs # ", 0, "
+                                 # ElementWidth #  ", "
+                                 # "AArch64::ZPR" # RegClassSuffix # "RegClassID" # ">";
 }
 
-let EncoderMethod = "EncodeRegAsMultipleOf<2>",
-    DecoderMethod = "DecodeZPR2Mul2RegisterClass" in {
+let EncoderMethod = "EncodeRegMul_MinMax<2, 0, 30>",
+    DecoderMethod = "DecodeZPR2Mul2RegisterClass<0, 30>" in {
   def ZZ_mul_r : RegisterOperand<ZPR2Mul2, "printTypedVectorList<0,0>"> {
-    let ParserMatchClass = ZPRVectorListMul<0, 2>;
+    let ParserMatchClass = ZPRVectorListMul<0, 2, "Mul2">;
   }
 
   def ZZ_b_mul_r : RegisterOperand<ZPR2Mul2, "printTypedVectorList<0,'b'>"> {
-    let ParserMatchClass = ZPRVectorListMul<8, 2>;
+    let ParserMatchClass = ZPRVectorListMul<8, 2, "Mul2">;
   }
 
   def ZZ_h_mul_r : RegisterOperand<ZPR2Mul2, "printTypedVectorList<0,'h'>"> {
-    let ParserMatchClass = ZPRVectorListMul<16, 2>;
+    let ParserMatchClass = ZPRVectorListMul<16, 2, "Mul2">;
   }
 
   def ZZ_s_mul_r : RegisterOperand<ZPR2Mul2, "printTypedVectorList<0,'s'>"> {
-    let ParserMatchClass = ZPRVectorListMul<32, 2>;
+    let ParserMatchClass = ZPRVectorListMul<32, 2, "Mul2">;
   }
 
   def ZZ_d_mul_r : RegisterOperand<ZPR2Mul2, "printTypedVectorList<0,'d'>"> {
-    let ParserMatchClass = ZPRVectorListMul<64, 2>;
+    let ParserMatchClass = ZPRVectorListMul<64, 2, "Mul2">;
   }
 
   def ZZ_q_mul_r : RegisterOperand<ZPR2Mul2, "printTypedVectorList<0,'q'>"> {
-    let ParserMatchClass = ZPRVectorListMul<128, 2>;
+    let ParserMatchClass = ZPRVectorListMul<128, 2, "Mul2">;
   }
 } // end let EncoderMethod/DecoderMethod
 
-let EncoderMethod = "EncodeRegAsMultipleOf<4>",
+let EncoderMethod = "EncodeRegMul_MinMax<4, 0, 28>",
     DecoderMethod = "DecodeZPR4Mul4RegisterClass" in {
   def ZZZZ_b_mul_r : RegisterOperand<ZPR4Mul4, "printTypedVectorList<0,'b'>"> {
-    let ParserMatchClass = ZPRVectorListMul<8, 4>;
+    let ParserMatchClass = ZPRVectorListMul<8, 4, "Mul4">;
   }
 
   def ZZZZ_h_mul_r : RegisterOperand<ZPR4Mul4, "printTypedVectorList<0,'h'>"> {
-    let ParserMatchClass = ZPRVectorListMul<16, 4>;
+    let ParserMatchClass = ZPRVectorListMul<16, 4, "Mul4">;
   }
 
   def ZZZZ_s_mul_r : RegisterOperand<ZPR4Mul4, "printTypedVectorList<0,'s'>"> {
-    let ParserMatchClass = ZPRVectorListMul<32, 4>;
+    let ParserMatchClass = ZPRVectorListMul<32, 4, "Mul4">;
   }
 
   def ZZZZ_d_mul_r : RegisterOperand<ZPR4Mul4, "printTypedVectorList<0,'d'>"> {
-    let ParserMatchClass = ZPRVectorListMul<64, 4>;
+    let ParserMatchClass = ZPRVectorListMul<64, 4, "Mul4">;
   }
 
   def ZZZZ_q_mul_r : RegisterOperand<ZPR4Mul4, "printTypedVectorList<0,'q'>"> {
-    let ParserMatchClass = ZPRVectorListMul<128, 4>;
+    let ParserMatchClass = ZPRVectorListMul<128, 4, "Mul4">;
   }
 } // end let EncoderMethod/DecoderMethod
 
+// Pairs of consecutive ZPR, starting with an even register, split into
+// Lo=0-14 and Hi=16-30.
+def ZPR2Mul2_Lo : RegisterClass<"AArch64", [untyped], 128,
+                                (trunc (decimate ZSeqPairs, 2), 8)>  {
+  let Size = 256;
+}
+
+def ZPR2Mul2_Hi : RegisterClass<"AArch64", [untyped], 128,
+                                (trunc (rotr (decimate ZSeqPairs, 2), 8), 8)>  {
+  let Size = 256;
+}
+
+let EncoderMethod = "EncodeRegMul_MinMax<2, 0, 14>",
+    DecoderMethod = "DecodeZPR2Mul2RegisterClass<0, 16>" in {
+  def ZZ_b_mul_r_Lo : RegisterOperand<ZPR2Mul2_Lo, "printTypedVectorList<0,'b'>"> {
+    let ParserMatchClass = ZPRVectorListMul<8, 2, "Mul2_Lo">;
+  }
+
+  def ZZ_h_mul_r_Lo : RegisterOperand<ZPR2Mul2_Lo, "printTypedVectorList<0,'h'>"> {
+    let ParserMatchClass = ZPRVectorListMul<16, 2, "Mul2_Lo">;
+  }
+
+  def ZZ_s_mul_r_Lo : RegisterOperand<ZPR2Mul2_Lo, "printTypedVectorList<0,'s'>"> {
+    let ParserMatchClass = ZPRVectorListMul<32, 2, "Mul2_Lo">;
+  }
+
+  def ZZ_d_mul_r_Lo : RegisterOperand<ZPR2Mul2_Lo, "printTypedVectorList<0,'d'>"> {
+    let ParserMatchClass = ZPRVectorListMul<64, 2, "Mul2_Lo">;
+  }
+}
+
+let EncoderMethod = "EncodeRegMul_MinMax<2, 16, 30>",
+    DecoderMethod = "DecodeZPR2Mul2RegisterClass<16, 31>" in {
+  def ZZ_b_mul_r_Hi : RegisterOperand<ZPR2Mul2_Hi, "printTypedVectorList<0,'b'>"> {
+    let ParserMatchClass = ZPRVectorListMul<8, 2, "Mul2_Hi">;
+  }
+
+  def ZZ_h_mul_r_Hi : RegisterOperand<ZPR2Mul2_Hi, "printTypedVectorList<0,'h'>"> {
+    let ParserMatchClass = ZPRVectorListMul<16, 2, "Mul2_Hi">;
+  }
+
+  def ZZ_s_mul_r_Hi : RegisterOperand<ZPR2Mul2_Hi, "printTypedVectorList<0,'s'>"> {
+    let ParserMatchClass = ZPRVectorListMul<32, 2, "Mul2_Hi">;
+  }
+
+  def ZZ_d_mul_r_Hi : RegisterOperand<ZPR2Mul2_Hi, "printTypedVectorList<0,'d'>"> {
+    let ParserMatchClass = ZPRVectorListMul<64, 2, "Mul2_Hi">;
+  }
+ } // end let EncoderMethod/DecoderMethod
+
 // SME2 strided multi-vector operands
 
 // ZStridedPairs

llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp

@@ -1262,6 +1262,9 @@ class AArch64Operand : public MCParsedAsmOperand {
     case AArch64::ZPRRegClassID:
     case AArch64::ZPR_3bRegClassID:
     case AArch64::ZPR_4bRegClassID:
+    case AArch64::ZPRMul2_LoRegClassID:
+    case AArch64::ZPRMul2_HiRegClassID:
+    case AArch64::ZPR_KRegClassID:
       RK = RegKind::SVEDataVector;
       break;
     case AArch64::PPRRegClassID:
@@ -1442,13 +1445,13 @@ class AArch64Operand : public MCParsedAsmOperand {
   }
 
   template <RegKind VectorKind, unsigned NumRegs, unsigned NumElements,
-            unsigned ElementWidth>
+            unsigned ElementWidth, unsigned RegClass>
   DiagnosticPredicate isTypedVectorListMultiple() const {
     bool Res =
         isTypedVectorList<VectorKind, NumRegs, NumElements, ElementWidth>();
     if (!Res)
       return DiagnosticPredicateTy::NoMatch;
-    if (((VectorList.RegNum - AArch64::Z0) % NumRegs) != 0)
+    if (!AArch64MCRegisterClasses[RegClass].contains(VectorList.RegNum))
       return DiagnosticPredicateTy::NearMatch;
     return DiagnosticPredicateTy::Match;
   }
@@ -6092,6 +6095,33 @@ bool AArch64AsmParser::showMatchError(SMLoc Loc, unsigned ErrCode,
     return Error(Loc, "Invalid restricted vector register, expected z0.s..z15.s");
   case Match_InvalidZPR_4b64:
     return Error(Loc, "Invalid restricted vector register, expected z0.d..z15.d");
+  case Match_InvalidZPRMul2_Lo8:
+    return Error(Loc, "Invalid restricted vector register, expected even "
+                      "register in z0.b..z14.b");
+  case Match_InvalidZPRMul2_Hi8:
+    return Error(Loc, "Invalid restricted vector register, expected even "
+                      "register in z16.b..z30.b");
+  case Match_InvalidZPRMul2_Lo16:
+    return Error(Loc, "Invalid restricted vector register, expected even "
+                      "register in z0.h..z14.h");
+  case Match_InvalidZPRMul2_Hi16:
+    return Error(Loc, "Invalid restricted vector register, expected even "
+                      "register in z16.h..z30.h");
+  case Match_InvalidZPRMul2_Lo32:
+    return Error(Loc, "Invalid restricted vector register, expected even "
+                      "register in z0.s..z14.s");
+  case Match_InvalidZPRMul2_Hi32:
+    return Error(Loc, "Invalid restricted vector register, expected even "
+                      "register in z16.s..z30.s");
+  case Match_InvalidZPRMul2_Lo64:
+    return Error(Loc, "Invalid restricted vector register, expected even "
+                      "register in z0.d..z14.d");
+  case Match_InvalidZPRMul2_Hi64:
+    return Error(Loc, "Invalid restricted vector register, expected even "
+                      "register in z16.d..z30.d");
+  case Match_InvalidZPR_K0:
+    return Error(Loc, "invalid restricted vector register, expected register "
+                      "in z20..z23 or z28..z31");
   case Match_InvalidSVEPattern:
     return Error(Loc, "invalid predicate pattern");
   case Match_InvalidSVEPPRorPNRAnyReg:
@@ -6171,19 +6201,36 @@ bool AArch64AsmParser::showMatchError(SMLoc Loc, unsigned ErrCode,
     return Error(Loc, "operand must be a register in range [w12, w15]");
   case Match_InvalidMatrixIndexGPR32_8_11:
     return Error(Loc, "operand must be a register in range [w8, w11]");
-  case Match_InvalidSVEVectorListMul2x8:
-  case Match_InvalidSVEVectorListMul2x16:
-  case Match_InvalidSVEVectorListMul2x32:
-  case Match_InvalidSVEVectorListMul2x64:
-  case Match_InvalidSVEVectorListMul2x128:
+  case Match_InvalidSVEVectorList2x8Mul2:
+  case Match_InvalidSVEVectorList2x16Mul2:
+  case Match_InvalidSVEVectorList2x32Mul2:
+  case Match_InvalidSVEVectorList2x64Mul2:
+  case Match_InvalidSVEVectorList2x128Mul2:
     return Error(Loc, "Invalid vector list, expected list with 2 consecutive "
                       "SVE vectors, where the first vector is a multiple of 2 "
                       "and with matching element types");
-  case Match_InvalidSVEVectorListMul4x8:
-  case Match_InvalidSVEVectorListMul4x16:
-  case Match_InvalidSVEVectorListMul4x32:
-  case Match_InvalidSVEVectorListMul4x64:
-  case Match_InvalidSVEVectorListMul4x128:
+  case Match_InvalidSVEVectorList2x8Mul2_Lo:
+  case Match_InvalidSVEVectorList2x16Mul2_Lo:
+  case Match_InvalidSVEVectorList2x32Mul2_Lo:
+  case Match_InvalidSVEVectorList2x64Mul2_Lo:
+    return Error(Loc, "Invalid vector list, expected list with 2 consecutive "
+                      "SVE vectors in the range z0-z14, where the first vector "
+                      "is a multiple of 2 "
+                      "and with matching element types");
+  case Match_InvalidSVEVectorList2x8Mul2_Hi:
+  case Match_InvalidSVEVectorList2x16Mul2_Hi:
+  case Match_InvalidSVEVectorList2x32Mul2_Hi:
+  case Match_InvalidSVEVectorList2x64Mul2_Hi:
+    return Error(Loc,
+                 "Invalid vector list, expected list with 2 consecutive "
+                 "SVE vectors in the range z16-z30, where the first vector "
+                 "is a multiple of 2 "
+                 "and with matching element types");
+  case Match_InvalidSVEVectorList4x8Mul4:
+  case Match_InvalidSVEVectorList4x16Mul4:
+  case Match_InvalidSVEVectorList4x32Mul4:
+  case Match_InvalidSVEVectorList4x64Mul4:
+  case Match_InvalidSVEVectorList4x128Mul4:
     return Error(Loc, "Invalid vector list, expected list with 4 consecutive "
                       "SVE vectors, where the first vector is a multiple of 4 "
                       "and with matching element types");
@@ -6776,16 +6823,33 @@ bool AArch64AsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
   case Match_InvalidMatrixIndexGPR32_12_15:
   case Match_InvalidMatrixIndexGPR32_8_11:
   case Match_InvalidLookupTable:
-  case Match_InvalidSVEVectorListMul2x8:
-  case Match_InvalidSVEVectorListMul2x16:
-  case Match_InvalidSVEVectorListMul2x32:
-  case Match_InvalidSVEVectorListMul2x64:
-  case Match_InvalidSVEVectorListMul2x128:
-  case Match_InvalidSVEVectorListMul4x8:
-  case Match_InvalidSVEVectorListMul4x16:
-  case Match_InvalidSVEVectorListMul4x32:
-  case Match_InvalidSVEVectorListMul4x64:
-  case Match_InvalidSVEVectorListMul4x128:
+  case Match_InvalidZPRMul2_Lo8:
+  case Match_InvalidZPRMul2_Hi8:
+  case Match_InvalidZPRMul2_Lo16:
+  case Match_InvalidZPRMul2_Hi16:
+  case Match_InvalidZPRMul2_Lo32:
+  case Match_InvalidZPRMul2_Hi32:
+  case Match_InvalidZPRMul2_Lo64:
+  case Match_InvalidZPRMul2_Hi64:
+  case Match_InvalidZPR_K0:
+  case Match_InvalidSVEVectorList2x8Mul2:
+  case Match_InvalidSVEVectorList2x16Mul2:
+  case Match_InvalidSVEVectorList2x32Mul2:
+  case Match_InvalidSVEVectorList2x64Mul2:
+  case Match_InvalidSVEVectorList2x128Mul2:
+  case Match_InvalidSVEVectorList4x8Mul4:
+  case Match_InvalidSVEVectorList4x16Mul4:
+  case Match_InvalidSVEVectorList4x32Mul4:
+  case Match_InvalidSVEVectorList4x64Mul4:
+  case Match_InvalidSVEVectorList4x128Mul4:
+  case Match_InvalidSVEVectorList2x8Mul2_Lo:
+  case Match_InvalidSVEVectorList2x16Mul2_Lo:
+  case Match_InvalidSVEVectorList2x32Mul2_Lo:
+  case Match_InvalidSVEVectorList2x64Mul2_Lo:
+  case Match_InvalidSVEVectorList2x8Mul2_Hi:
+  case Match_InvalidSVEVectorList2x16Mul2_Hi:
+  case Match_InvalidSVEVectorList2x32Mul2_Hi:
+  case Match_InvalidSVEVectorList2x64Mul2_Hi:
   case Match_InvalidSVEVectorListStrided2x8:
   case Match_InvalidSVEVectorListStrided2x16:
   case Match_InvalidSVEVectorListStrided2x32: