[RISCV] Implement support for the Zicbom and Zicboz extensions

Implements the ratified RISC-V Base Cache Management Operation ISA
Extensions: Zicbom and Zicboz, as described in
https://github.com/riscv/riscv-CMOs/blob/master/specifications/cmobase-v1.0.pdf.

Zicbop is implemented in a separate patch due to it requiring a new ASM
operand type to be defined.

As discussed in the relevant issue in the upstream spec
https://github.com/riscv/riscv-CMOs/issues/47, the cbo.* instructions
use the format (rs1) or 0(rs1) for their operand, similar to the AMOs.

Differential Revision: https://reviews.llvm.org/D117432

clang/test/Preprocessor/riscv-target-features.c

@@ -42,6 +42,8 @@
 // CHECK-NOT: __riscv_zkr
 // CHECK-NOT: __riscv_zkt
 // CHECK-NOT: __riscv_zk
+// CHECK-NOT: __riscv_zicbom
+// CHECK-NOT: __riscv_zicboz
 
 // RUN: %clang -target riscv32-unknown-linux-gnu -march=rv32im -x c -E -dM %s \
 // RUN: -o - | FileCheck --check-prefix=CHECK-M-EXT %s
@@ -433,3 +435,15 @@
 // RUN: -march=rv64i_zbkb_zbkc_zbkx_zksed_zksh -x c -E -dM %s -o - \
 // RUN: | FileCheck --check-prefix=CHECK-COMBINE-INTO-ZKS %s
 // CHECK-COMBINE-INTO-ZKS: __riscv_zks 1
+
+// RUN: %clang -target riscv32 -march=rv32izicbom -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZICBOM-EXT %s
+// RUN: %clang -target riscv64 -march=rv64izicbom -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZICBOM-EXT %s
+// CHECK-ZICBOM-EXT: __riscv_zicbom 1000000{{$}}
+
+// RUN: %clang -target riscv32 -march=rv32izicboz -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZICBOZ-EXT %s
+// RUN: %clang -target riscv64 -march=rv64izicboz -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZICBOZ-EXT %s
+// CHECK-ZICBOZ-EXT: __riscv_zicboz 1000000{{$}}

llvm/lib/Support/RISCVISAInfo.cpp

@@ -95,6 +95,9 @@ static const RISCVSupportedExtension SupportedExtensions[] = {
     {"zve64x", RISCVExtensionVersion{1, 0}},
     {"zve64f", RISCVExtensionVersion{1, 0}},
     {"zve64d", RISCVExtensionVersion{1, 0}},
+
+    {"zicbom", RISCVExtensionVersion{1, 0}},
+    {"zicboz", RISCVExtensionVersion{1, 0}},
 };
 
 static const RISCVSupportedExtension SupportedExperimentalExtensions[] = {

llvm/lib/Target/RISCV/RISCV.td

@@ -400,6 +400,20 @@ def FeatureStdExtZvfh
                        "'Zvfh' (Vector Half-Precision Floating-Point)",
                        [FeatureStdExtZve32f]>;
 
+def FeatureStdExtZicbom
+    : SubtargetFeature<"zicbom", "HasStdExtZicbom", "true",
+                       "'Zicbom' (Cache-Block Management Instructions)">;
+def HasStdExtZicbom : Predicate<"Subtarget->hasStdExtZicbom()">,
+                                AssemblerPredicate<(all_of FeatureStdExtZicbom),
+                                "'Zicbom' (Cache-Block Management Instructions)">;
+
+def FeatureStdExtZicboz
+    : SubtargetFeature<"zicboz", "HasStdExtZicboz", "true",
+                       "'Zicboz' (Cache-Block Zero Instructions)">;
+def HasStdExtZicboz : Predicate<"Subtarget->hasStdExtZicboz()">,
+                                AssemblerPredicate<(all_of FeatureStdExtZicboz),
+                                "'Zicboz' (Cache-Block Zero Instructions)">;
+
 def Feature64Bit
     : SubtargetFeature<"64bit", "HasRV64", "true", "Implements RV64">;
 def IsRV64 : Predicate<"Subtarget->is64Bit()">,

llvm/lib/Target/RISCV/RISCVInstrInfo.td

@@ -1679,3 +1679,4 @@ include "RISCVInstrInfoZb.td"
 include "RISCVInstrInfoZk.td"
 include "RISCVInstrInfoV.td"
 include "RISCVInstrInfoZfh.td"
+include "RISCVInstrInfoZicbo.td"

llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td

@@ -0,0 +1,37 @@
+//===-- RISCVInstrInfoZicbo.td - RISC-V CMO instructions ---*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the RISC-V instructions from the standard Base Cache
+// Management Operation ISA Extensions document (Zicbop, Zicboz, and Zicbop).
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Instruction Class Templates
+//===----------------------------------------------------------------------===//
+let hasSideEffects = 0, mayLoad = 0, mayStore = 1 in
+class CBO_r<bits<12> optype, string opcodestr>
+    : RVInstI<0b010, OPC_MISC_MEM, (outs), (ins GPRMemZeroOffset:$rs1),
+              opcodestr, "$rs1"> {
+  let imm12 = optype;
+  let rd = 0b00000;
+}
+
+//===----------------------------------------------------------------------===//
+// Instructions
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasStdExtZicbom] in {
+def CBO_CLEAN : CBO_r<0b000000000001, "cbo.clean">, Sched<[]>;
+def CBO_FLUSH : CBO_r<0b000000000010, "cbo.flush">, Sched<[]>;
+def CBO_INVAL : CBO_r<0b000000000000, "cbo.inval">, Sched<[]>;
+} // Predicates = [HasStdExtZicbom]
+
+let Predicates = [HasStdExtZicboz] in {
+def CBO_ZERO : CBO_r<0b000000000100, "cbo.zero">, Sched<[]>;
+} // Predicates = [HasStdExtZicboz]

llvm/lib/Target/RISCV/RISCVSubtarget.h

@@ -86,6 +86,8 @@ private:
   bool HasStdExtZks = false;
   bool HasStdExtZkt = false;
   bool HasStdExtZk = false;
+  bool HasStdExtZicbom = false;
+  bool HasStdExtZicboz = false;
   bool HasRV64 = false;
   bool IsRV32E = false;
   bool EnableLinkerRelax = false;
@@ -178,6 +180,8 @@ public:
   bool hasStdExtZksed() const { return HasStdExtZksed; }
   bool hasStdExtZksh() const { return HasStdExtZksh; }
   bool hasStdExtZkr() const { return HasStdExtZkr; }
+  bool hasStdExtZicbom() const { return HasStdExtZicbom; }
+  bool hasStdExtZicboz() const { return HasStdExtZicboz; }
   bool is64Bit() const { return HasRV64; }
   bool isRV32E() const { return IsRV32E; }
   bool enableLinkerRelax() const { return EnableLinkerRelax; }

llvm/test/CodeGen/RISCV/attributes.ll

@@ -36,6 +36,8 @@
 ; RUN: llc -mtriple=riscv32 -mattr=+zkn,+zkr,+zkt %s -o - | FileCheck --check-prefix=RV32COMBINEINTOZK %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zbkb,+zbkc,+zbkx,+zkne,+zknd,+zknh %s -o - | FileCheck --check-prefix=RV32COMBINEINTOZKN %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zbkb,+zbkc,+zbkx,+zksed,+zksh %s -o - | FileCheck --check-prefix=RV32COMBINEINTOZKS %s
+; RUN: llc -mtriple=riscv32 -mattr=+zicbom %s -o - | FileCheck --check-prefix=RV32ZICBOM %s
+; RUN: llc -mtriple=riscv32 -mattr=+zicboz %s -o - | FileCheck --check-prefix=RV32ZICBOZ %s
 ; RUN: llc -mtriple=riscv64 -mattr=+m %s -o - | FileCheck --check-prefix=RV64M %s
 ; RUN: llc -mtriple=riscv64 -mattr=+a %s -o - | FileCheck --check-prefix=RV64A %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f %s -o - | FileCheck --check-prefix=RV64F %s
@@ -72,6 +74,8 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+zkn,+zkr,+zkt %s -o - | FileCheck --check-prefix=RV64COMBINEINTOZK %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zbkb,+zbkc,+zbkx,+zkne,+zknd,+zknh %s -o - | FileCheck --check-prefix=RV64COMBINEINTOZKN %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zbkb,+zbkc,+zbkx,+zksed,+zksh %s -o - | FileCheck --check-prefix=RV64COMBINEINTOZKS %s
+; RUN: llc -mtriple=riscv64 -mattr=+zicbom %s -o - | FileCheck --check-prefix=RV64ZICBOM %s
+; RUN: llc -mtriple=riscv64 -mattr=+zicboz %s -o - | FileCheck --check-prefix=RV64ZICBOZ %s
 
 ; RV32M: .attribute 5, "rv32i2p0_m2p0"
 ; RV32A: .attribute 5, "rv32i2p0_a2p0"
@@ -109,6 +113,8 @@
 ; RV32COMBINEINTOZK: .attribute 5, "rv32i2p0_zbkb1p0_zbkc1p0_zbkx1p0_zk1p0_zkn1p0_zknd1p0_zkne1p0_zknh1p0_zkr1p0_zkt1p0"
 ; RV32COMBINEINTOZKN: .attribute 5, "rv32i2p0_zbkb1p0_zbkc1p0_zbkx1p0_zkn1p0_zknd1p0_zkne1p0_zknh1p0"
 ; RV32COMBINEINTOZKS: .attribute 5, "rv32i2p0_zbkb1p0_zbkc1p0_zbkx1p0_zks1p0_zksed1p0_zksh1p0"
+; RV32ZICBOM: .attribute 5, "rv32i2p0_zicbom1p0"
+; RV32ZICBOZ: .attribute 5, "rv32i2p0_zicboz1p0"
 
 ; RV64M: .attribute 5, "rv64i2p0_m2p0"
 ; RV64A: .attribute 5, "rv64i2p0_a2p0"
@@ -146,6 +152,8 @@
 ; RV64COMBINEINTOZK: .attribute 5, "rv64i2p0_zbkb1p0_zbkc1p0_zbkx1p0_zk1p0_zkn1p0_zknd1p0_zkne1p0_zknh1p0_zkr1p0_zkt1p0"
 ; RV64COMBINEINTOZKN: .attribute 5, "rv64i2p0_zbkb1p0_zbkc1p0_zbkx1p0_zkn1p0_zknd1p0_zkne1p0_zknh1p0"
 ; RV64COMBINEINTOZKS: .attribute 5, "rv64i2p0_zbkb1p0_zbkc1p0_zbkx1p0_zks1p0_zksed1p0_zksh1p0"
+; RV64ZICBOM: .attribute 5, "rv64i2p0_zicbom1p0"
+; RV64ZICBOZ: .attribute 5, "rv64i2p0_zicboz1p0"
 
 define i32 @addi(i32 %a) {
   %1 = add i32 %a, 1

llvm/test/MC/RISCV/attribute-arch.s

@@ -87,6 +87,12 @@
 .attribute arch, "rv32ifdzve64d"
 # CHECK: attribute      5, "rv32i2p0_f2p0_d2p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvl32b1p0_zvl64b1p0"
 
+.attribute arch, "rv32izicbom"
+# CHECK: attribute      5, "rv32i2p0_zicbom1p0"
+
+.attribute arch, "rv32izicboz"
+# CHECK: attribute      5, "rv32i2p0_zicboz1p0"
+
 ## Experimental extensions require version string to be explicitly specified
 
 .attribute arch, "rv32izba1p0"

llvm/test/MC/RISCV/rv32zicbom-invalid.s

@@ -0,0 +1,24 @@
+# RUN: not llvm-mc -triple riscv32 -mattr=+zicbom < %s 2>&1 | FileCheck %s
+# RUN: not llvm-mc -triple riscv64 -mattr=+zicbom < %s 2>&1 | FileCheck %s
+
+# Must have a single register argument.
+cbo.clean # CHECK: :[[@LINE]]:1: error: too few operands for instruction
+cbo.flush # CHECK: :[[@LINE]]:1: error: too few operands for instruction
+cbo.inval # CHECK: :[[@LINE]]:1: error: too few operands for instruction
+
+cbo.clean 1 # CHECK: :[[@LINE]]:13: error: expected '(' after optional integer offset
+cbo.flush 2 # CHECK: :[[@LINE]]:13: error: expected '(' after optional integer offset
+cbo.inval 3 # CHECK: :[[@LINE]]:13: error: expected '(' after optional integer offset
+
+cbo.clean t0, t1 # CHECK: :[[@LINE]]:11: error: expected '(' or optional integer offset
+cbo.flush t0, t1 # CHECK: :[[@LINE]]:11: error: expected '(' or optional integer offset
+cbo.inval t0, t1 # CHECK: :[[@LINE]]:11: error: expected '(' or optional integer offset
+
+# Non-zero offsets are not supported.
+cbo.clean 1(t0) # CHECK: :[[@LINE]]:11: error: optional integer offset must be 0
+cbo.flush 2(t0) # CHECK: :[[@LINE]]:11: error: optional integer offset must be 0
+cbo.inval 3(t0) # CHECK: :[[@LINE]]:11: error: optional integer offset must be 0
+
+# Instructions from other zicbo* extensions aren't available without enabling
+# the appropriate -mattr flag.
+cbo.zero (t0) # CHECK: :[[@LINE]]:1: error: instruction requires the following: 'Zicboz' (Cache-Block Zero Instructions)

llvm/test/MC/RISCV/rv32zicbom-valid.s

@@ -0,0 +1,31 @@
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+zicbom -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
+# RUN: llvm-mc %s -triple=riscv64 -mattr=+zicbom -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
+# RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+zicbom < %s \
+# RUN:     | llvm-objdump --mattr=+zicbom -d -r - \
+# RUN:     | FileCheck --check-prefix=CHECK-ASM-AND-OBJ %s
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+zicbom < %s \
+# RUN:     | llvm-objdump --mattr=+zicbom -d -r - \
+# RUN:     | FileCheck --check-prefix=CHECK-ASM-AND-OBJ %s
+
+# CHECK-ASM-AND-OBJ: cbo.clean (t0)
+# CHECK-ASM: encoding: [0x0f,0xa0,0x12,0x00]
+cbo.clean (t0)
+# CHECK-ASM-AND-OBJ: cbo.clean (t0)
+# CHECK-ASM: encoding: [0x0f,0xa0,0x12,0x00]
+cbo.clean 0(t0)
+
+# CHECK-ASM-AND-OBJ: cbo.flush (t1)
+# CHECK-ASM: encoding: [0x0f,0x20,0x23,0x00]
+cbo.flush (t1)
+# CHECK-ASM-AND-OBJ: cbo.flush (t1)
+# CHECK-ASM: encoding: [0x0f,0x20,0x23,0x00]
+cbo.flush 0(t1)
+
+# CHECK-ASM-AND-OBJ: cbo.inval (t2)
+# CHECK-ASM: encoding: [0x0f,0xa0,0x03,0x00]
+cbo.inval (t2)
+# CHECK-ASM-AND-OBJ: cbo.inval (t2)
+# CHECK-ASM: encoding: [0x0f,0xa0,0x03,0x00]
+cbo.inval 0(t2)

llvm/test/MC/RISCV/rv32zicboz-invalid.s

@@ -0,0 +1,16 @@
+# RUN: not llvm-mc -triple riscv32 -mattr=+zicboz < %s 2>&1 | FileCheck %s
+# RUN: not llvm-mc -triple riscv64 -mattr=+zicboz < %s 2>&1 | FileCheck %s
+
+# Must have a single register argument.
+cbo.zero # CHECK: :[[@LINE]]:1: error: too few operands for instruction
+cbo.zero 1 # CHECK: :[[@LINE]]:12: error: expected '(' after optional integer offset
+cbo.zero t0, t1 # CHECK: :[[@LINE]]:10: error: expected '(' or optional integer offset
+
+# Non-zero offsets are not supported.
+cbo.zero 1(t0) # CHECK: :[[@LINE]]:10: error: optional integer offset must be 0
+
+# Instructions from other zicbo* extensions aren't available without enabling
+# the appropriate -mattr flag.
+cbo.clean (t0) # CHECK: :[[@LINE]]:1: error: instruction requires the following: 'Zicbom' (Cache-Block Management Instructions)
+cbo.flush (t1) # CHECK: :[[@LINE]]:1: error: instruction requires the following: 'Zicbom' (Cache-Block Management Instructions)
+cbo.inval (t2) # CHECK: :[[@LINE]]:1: error: instruction requires the following: 'Zicbom' (Cache-Block Management Instructions)

llvm/test/MC/RISCV/rv32zicboz-valid.s

@@ -0,0 +1,17 @@
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+zicboz -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
+# RUN: llvm-mc %s -triple=riscv64 -mattr=+zicboz -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
+# RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+zicboz < %s \
+# RUN:     | llvm-objdump --mattr=+zicboz -d -r - \
+# RUN:     | FileCheck --check-prefix=CHECK-ASM-AND-OBJ %s
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+zicboz < %s \
+# RUN:     | llvm-objdump --mattr=+zicboz -d -r - \
+# RUN:     | FileCheck --check-prefix=CHECK-ASM-AND-OBJ %s
+
+# CHECK-ASM-AND-OBJ: cbo.zero (t0)
+# CHECK-ASM: encoding: [0x0f,0xa0,0x42,0x00]
+cbo.zero (t0)
+# CHECK-ASM-AND-OBJ: cbo.zero (t0)
+# CHECK-ASM: encoding: [0x0f,0xa0,0x42,0x00]
+cbo.zero 0(t0)