forked from OSchip/llvm-project
629 lines
22 KiB
C++
629 lines
22 KiB
C++
//==- LoongArchExpandAtomicPseudoInsts.cpp - Expand atomic pseudo instrs. -===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// This file contains a pass that expands atomic pseudo instructions into
|
|
// target instructions. This pass should be run at the last possible moment,
|
|
// avoiding the possibility for other passes to break the requirements for
|
|
// forward progress in the LL/SC block.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "LoongArch.h"
|
|
#include "LoongArchInstrInfo.h"
|
|
#include "LoongArchTargetMachine.h"
|
|
|
|
#include "llvm/CodeGen/LivePhysRegs.h"
|
|
#include "llvm/CodeGen/MachineFunctionPass.h"
|
|
#include "llvm/CodeGen/MachineInstrBuilder.h"
|
|
|
|
using namespace llvm;
|
|
|
|
#define LoongArch_EXPAND_ATOMIC_PSEUDO_NAME \
|
|
"LoongArch atomic pseudo instruction expansion pass"
|
|
|
|
namespace {
|
|
|
|
class LoongArchExpandAtomicPseudo : public MachineFunctionPass {
|
|
public:
|
|
const LoongArchInstrInfo *TII;
|
|
static char ID;
|
|
|
|
LoongArchExpandAtomicPseudo() : MachineFunctionPass(ID) {
|
|
initializeLoongArchExpandAtomicPseudoPass(*PassRegistry::getPassRegistry());
|
|
}
|
|
|
|
bool runOnMachineFunction(MachineFunction &MF) override;
|
|
|
|
StringRef getPassName() const override {
|
|
return LoongArch_EXPAND_ATOMIC_PSEUDO_NAME;
|
|
}
|
|
|
|
private:
|
|
bool expandMBB(MachineBasicBlock &MBB);
|
|
bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
|
|
MachineBasicBlock::iterator &NextMBBI);
|
|
bool expandAtomicBinOp(MachineBasicBlock &MBB,
|
|
MachineBasicBlock::iterator MBBI, AtomicRMWInst::BinOp,
|
|
bool IsMasked, int Width,
|
|
MachineBasicBlock::iterator &NextMBBI);
|
|
bool expandAtomicMinMaxOp(MachineBasicBlock &MBB,
|
|
MachineBasicBlock::iterator MBBI,
|
|
AtomicRMWInst::BinOp, bool IsMasked, int Width,
|
|
MachineBasicBlock::iterator &NextMBBI);
|
|
bool expandAtomicCmpXchg(MachineBasicBlock &MBB,
|
|
MachineBasicBlock::iterator MBBI, bool IsMasked,
|
|
int Width, MachineBasicBlock::iterator &NextMBBI);
|
|
};
|
|
|
|
char LoongArchExpandAtomicPseudo::ID = 0;
|
|
|
|
bool LoongArchExpandAtomicPseudo::runOnMachineFunction(MachineFunction &MF) {
|
|
TII =
|
|
static_cast<const LoongArchInstrInfo *>(MF.getSubtarget().getInstrInfo());
|
|
bool Modified = false;
|
|
for (auto &MBB : MF)
|
|
Modified |= expandMBB(MBB);
|
|
return Modified;
|
|
}
|
|
|
|
bool LoongArchExpandAtomicPseudo::expandMBB(MachineBasicBlock &MBB) {
|
|
bool Modified = false;
|
|
|
|
MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
|
|
while (MBBI != E) {
|
|
MachineBasicBlock::iterator NMBBI = std::next(MBBI);
|
|
Modified |= expandMI(MBB, MBBI, NMBBI);
|
|
MBBI = NMBBI;
|
|
}
|
|
|
|
return Modified;
|
|
}
|
|
|
|
bool LoongArchExpandAtomicPseudo::expandMI(
|
|
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
|
|
MachineBasicBlock::iterator &NextMBBI) {
|
|
switch (MBBI->getOpcode()) {
|
|
case LoongArch::PseudoMaskedAtomicSwap32:
|
|
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32,
|
|
NextMBBI);
|
|
case LoongArch::PseudoAtomicSwap32:
|
|
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, false, 32,
|
|
NextMBBI);
|
|
case LoongArch::PseudoMaskedAtomicLoadAdd32:
|
|
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, true, 32, NextMBBI);
|
|
case LoongArch::PseudoMaskedAtomicLoadSub32:
|
|
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, true, 32, NextMBBI);
|
|
case LoongArch::PseudoAtomicLoadNand32:
|
|
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32,
|
|
NextMBBI);
|
|
case LoongArch::PseudoAtomicLoadNand64:
|
|
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64,
|
|
NextMBBI);
|
|
case LoongArch::PseudoMaskedAtomicLoadNand32:
|
|
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, true, 32,
|
|
NextMBBI);
|
|
case LoongArch::PseudoAtomicLoadAdd32:
|
|
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, false, 32,
|
|
NextMBBI);
|
|
case LoongArch::PseudoAtomicLoadSub32:
|
|
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, false, 32,
|
|
NextMBBI);
|
|
case LoongArch::PseudoAtomicLoadAnd32:
|
|
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::And, false, 32,
|
|
NextMBBI);
|
|
case LoongArch::PseudoAtomicLoadOr32:
|
|
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Or, false, 32, NextMBBI);
|
|
case LoongArch::PseudoAtomicLoadXor32:
|
|
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xor, false, 32,
|
|
NextMBBI);
|
|
case LoongArch::PseudoMaskedAtomicLoadUMax32:
|
|
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, true, 32,
|
|
NextMBBI);
|
|
case LoongArch::PseudoMaskedAtomicLoadUMin32:
|
|
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, true, 32,
|
|
NextMBBI);
|
|
case LoongArch::PseudoCmpXchg32:
|
|
return expandAtomicCmpXchg(MBB, MBBI, false, 32, NextMBBI);
|
|
case LoongArch::PseudoCmpXchg64:
|
|
return expandAtomicCmpXchg(MBB, MBBI, false, 64, NextMBBI);
|
|
case LoongArch::PseudoMaskedCmpXchg32:
|
|
return expandAtomicCmpXchg(MBB, MBBI, true, 32, NextMBBI);
|
|
case LoongArch::PseudoMaskedAtomicLoadMax32:
|
|
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, true, 32,
|
|
NextMBBI);
|
|
case LoongArch::PseudoMaskedAtomicLoadMin32:
|
|
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, true, 32,
|
|
NextMBBI);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
static void doAtomicBinOpExpansion(const LoongArchInstrInfo *TII,
|
|
MachineInstr &MI, DebugLoc DL,
|
|
MachineBasicBlock *ThisMBB,
|
|
MachineBasicBlock *LoopMBB,
|
|
MachineBasicBlock *DoneMBB,
|
|
AtomicRMWInst::BinOp BinOp, int Width) {
|
|
Register DestReg = MI.getOperand(0).getReg();
|
|
Register ScratchReg = MI.getOperand(1).getReg();
|
|
Register AddrReg = MI.getOperand(2).getReg();
|
|
Register IncrReg = MI.getOperand(3).getReg();
|
|
AtomicOrdering Ordering =
|
|
static_cast<AtomicOrdering>(MI.getOperand(4).getImm());
|
|
|
|
// .loop:
|
|
// if(Ordering != AtomicOrdering::Monotonic)
|
|
// dbar 0
|
|
// ll.[w|d] dest, (addr)
|
|
// binop scratch, dest, val
|
|
// sc.[w|d] scratch, scratch, (addr)
|
|
// beqz scratch, loop
|
|
if (Ordering != AtomicOrdering::Monotonic)
|
|
BuildMI(LoopMBB, DL, TII->get(LoongArch::DBAR)).addImm(0);
|
|
BuildMI(LoopMBB, DL,
|
|
TII->get(Width == 32 ? LoongArch::LL_W : LoongArch::LL_D), DestReg)
|
|
.addReg(AddrReg)
|
|
.addImm(0);
|
|
switch (BinOp) {
|
|
default:
|
|
llvm_unreachable("Unexpected AtomicRMW BinOp");
|
|
case AtomicRMWInst::Xchg:
|
|
BuildMI(LoopMBB, DL, TII->get(LoongArch::OR), ScratchReg)
|
|
.addReg(IncrReg)
|
|
.addReg(LoongArch::R0);
|
|
break;
|
|
case AtomicRMWInst::Nand:
|
|
BuildMI(LoopMBB, DL, TII->get(LoongArch::AND), ScratchReg)
|
|
.addReg(DestReg)
|
|
.addReg(IncrReg);
|
|
BuildMI(LoopMBB, DL, TII->get(LoongArch::NOR), ScratchReg)
|
|
.addReg(ScratchReg)
|
|
.addReg(LoongArch::R0);
|
|
break;
|
|
case AtomicRMWInst::Add:
|
|
BuildMI(LoopMBB, DL, TII->get(LoongArch::ADD_W), ScratchReg)
|
|
.addReg(DestReg)
|
|
.addReg(IncrReg);
|
|
break;
|
|
case AtomicRMWInst::Sub:
|
|
BuildMI(LoopMBB, DL, TII->get(LoongArch::SUB_W), ScratchReg)
|
|
.addReg(DestReg)
|
|
.addReg(IncrReg);
|
|
break;
|
|
case AtomicRMWInst::And:
|
|
BuildMI(LoopMBB, DL, TII->get(LoongArch::AND), ScratchReg)
|
|
.addReg(DestReg)
|
|
.addReg(IncrReg);
|
|
break;
|
|
case AtomicRMWInst::Or:
|
|
BuildMI(LoopMBB, DL, TII->get(LoongArch::OR), ScratchReg)
|
|
.addReg(DestReg)
|
|
.addReg(IncrReg);
|
|
break;
|
|
case AtomicRMWInst::Xor:
|
|
BuildMI(LoopMBB, DL, TII->get(LoongArch::XOR), ScratchReg)
|
|
.addReg(DestReg)
|
|
.addReg(IncrReg);
|
|
break;
|
|
}
|
|
BuildMI(LoopMBB, DL,
|
|
TII->get(Width == 32 ? LoongArch::SC_W : LoongArch::SC_D), ScratchReg)
|
|
.addReg(ScratchReg)
|
|
.addReg(AddrReg)
|
|
.addImm(0);
|
|
BuildMI(LoopMBB, DL, TII->get(LoongArch::BEQZ))
|
|
.addReg(ScratchReg)
|
|
.addMBB(LoopMBB);
|
|
}
|
|
|
|
static void insertMaskedMerge(const LoongArchInstrInfo *TII, DebugLoc DL,
|
|
MachineBasicBlock *MBB, Register DestReg,
|
|
Register OldValReg, Register NewValReg,
|
|
Register MaskReg, Register ScratchReg) {
|
|
assert(OldValReg != ScratchReg && "OldValReg and ScratchReg must be unique");
|
|
assert(OldValReg != MaskReg && "OldValReg and MaskReg must be unique");
|
|
assert(ScratchReg != MaskReg && "ScratchReg and MaskReg must be unique");
|
|
|
|
// res = oldval ^ ((oldval ^ newval) & masktargetdata);
|
|
BuildMI(MBB, DL, TII->get(LoongArch::XOR), ScratchReg)
|
|
.addReg(OldValReg)
|
|
.addReg(NewValReg);
|
|
BuildMI(MBB, DL, TII->get(LoongArch::AND), ScratchReg)
|
|
.addReg(ScratchReg)
|
|
.addReg(MaskReg);
|
|
BuildMI(MBB, DL, TII->get(LoongArch::XOR), DestReg)
|
|
.addReg(OldValReg)
|
|
.addReg(ScratchReg);
|
|
}
|
|
|
|
static void doMaskedAtomicBinOpExpansion(
|
|
const LoongArchInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
|
|
MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopMBB,
|
|
MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width) {
|
|
assert(Width == 32 && "Should never need to expand masked 64-bit operations");
|
|
Register DestReg = MI.getOperand(0).getReg();
|
|
Register ScratchReg = MI.getOperand(1).getReg();
|
|
Register AddrReg = MI.getOperand(2).getReg();
|
|
Register IncrReg = MI.getOperand(3).getReg();
|
|
Register MaskReg = MI.getOperand(4).getReg();
|
|
AtomicOrdering Ordering =
|
|
static_cast<AtomicOrdering>(MI.getOperand(5).getImm());
|
|
|
|
// .loop:
|
|
// if(Ordering != AtomicOrdering::Monotonic)
|
|
// dbar 0
|
|
// ll.w destreg, (alignedaddr)
|
|
// binop scratch, destreg, incr
|
|
// xor scratch, destreg, scratch
|
|
// and scratch, scratch, masktargetdata
|
|
// xor scratch, destreg, scratch
|
|
// sc.w scratch, scratch, (alignedaddr)
|
|
// beqz scratch, loop
|
|
if (Ordering != AtomicOrdering::Monotonic)
|
|
BuildMI(LoopMBB, DL, TII->get(LoongArch::DBAR)).addImm(0);
|
|
BuildMI(LoopMBB, DL, TII->get(LoongArch::LL_W), DestReg)
|
|
.addReg(AddrReg)
|
|
.addImm(0);
|
|
switch (BinOp) {
|
|
default:
|
|
llvm_unreachable("Unexpected AtomicRMW BinOp");
|
|
case AtomicRMWInst::Xchg:
|
|
BuildMI(LoopMBB, DL, TII->get(LoongArch::ADDI_W), ScratchReg)
|
|
.addReg(IncrReg)
|
|
.addImm(0);
|
|
break;
|
|
case AtomicRMWInst::Add:
|
|
BuildMI(LoopMBB, DL, TII->get(LoongArch::ADD_W), ScratchReg)
|
|
.addReg(DestReg)
|
|
.addReg(IncrReg);
|
|
break;
|
|
case AtomicRMWInst::Sub:
|
|
BuildMI(LoopMBB, DL, TII->get(LoongArch::SUB_W), ScratchReg)
|
|
.addReg(DestReg)
|
|
.addReg(IncrReg);
|
|
break;
|
|
case AtomicRMWInst::Nand:
|
|
BuildMI(LoopMBB, DL, TII->get(LoongArch::AND), ScratchReg)
|
|
.addReg(DestReg)
|
|
.addReg(IncrReg);
|
|
BuildMI(LoopMBB, DL, TII->get(LoongArch::NOR), ScratchReg)
|
|
.addReg(ScratchReg)
|
|
.addReg(LoongArch::R0);
|
|
// TODO: support other AtomicRMWInst.
|
|
}
|
|
|
|
insertMaskedMerge(TII, DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg,
|
|
ScratchReg);
|
|
|
|
BuildMI(LoopMBB, DL, TII->get(LoongArch::SC_W), ScratchReg)
|
|
.addReg(ScratchReg)
|
|
.addReg(AddrReg)
|
|
.addImm(0);
|
|
BuildMI(LoopMBB, DL, TII->get(LoongArch::BEQZ))
|
|
.addReg(ScratchReg)
|
|
.addMBB(LoopMBB);
|
|
}
|
|
|
|
bool LoongArchExpandAtomicPseudo::expandAtomicBinOp(
|
|
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
|
|
AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
|
|
MachineBasicBlock::iterator &NextMBBI) {
|
|
MachineInstr &MI = *MBBI;
|
|
DebugLoc DL = MI.getDebugLoc();
|
|
|
|
MachineFunction *MF = MBB.getParent();
|
|
auto LoopMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
|
|
// Insert new MBBs.
|
|
MF->insert(++MBB.getIterator(), LoopMBB);
|
|
MF->insert(++LoopMBB->getIterator(), DoneMBB);
|
|
|
|
// Set up successors and transfer remaining instructions to DoneMBB.
|
|
LoopMBB->addSuccessor(LoopMBB);
|
|
LoopMBB->addSuccessor(DoneMBB);
|
|
DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
|
|
DoneMBB->transferSuccessors(&MBB);
|
|
MBB.addSuccessor(LoopMBB);
|
|
|
|
if (IsMasked)
|
|
doMaskedAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp,
|
|
Width);
|
|
else
|
|
doAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp, Width);
|
|
|
|
NextMBBI = MBB.end();
|
|
MI.eraseFromParent();
|
|
|
|
LivePhysRegs LiveRegs;
|
|
computeAndAddLiveIns(LiveRegs, *LoopMBB);
|
|
computeAndAddLiveIns(LiveRegs, *DoneMBB);
|
|
|
|
return true;
|
|
}
|
|
|
|
static void insertSext(const LoongArchInstrInfo *TII, DebugLoc DL,
|
|
MachineBasicBlock *MBB, Register ValReg,
|
|
Register ShamtReg) {
|
|
BuildMI(MBB, DL, TII->get(LoongArch::SLL_W), ValReg)
|
|
.addReg(ValReg)
|
|
.addReg(ShamtReg);
|
|
BuildMI(MBB, DL, TII->get(LoongArch::SRA_W), ValReg)
|
|
.addReg(ValReg)
|
|
.addReg(ShamtReg);
|
|
}
|
|
|
|
bool LoongArchExpandAtomicPseudo::expandAtomicMinMaxOp(
|
|
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
|
|
AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
|
|
MachineBasicBlock::iterator &NextMBBI) {
|
|
assert(IsMasked == true &&
|
|
"Should only need to expand masked atomic max/min");
|
|
assert(Width == 32 && "Should never need to expand masked 64-bit operations");
|
|
|
|
MachineInstr &MI = *MBBI;
|
|
DebugLoc DL = MI.getDebugLoc();
|
|
MachineFunction *MF = MBB.getParent();
|
|
auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
auto TailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
|
|
// Insert new MBBs.
|
|
MF->insert(++MBB.getIterator(), LoopHeadMBB);
|
|
MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
|
|
MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
|
|
MF->insert(++LoopTailMBB->getIterator(), TailMBB);
|
|
MF->insert(++TailMBB->getIterator(), DoneMBB);
|
|
|
|
// Set up successors and transfer remaining instructions to DoneMBB.
|
|
LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
|
|
LoopHeadMBB->addSuccessor(LoopTailMBB);
|
|
LoopIfBodyMBB->addSuccessor(LoopTailMBB);
|
|
LoopTailMBB->addSuccessor(LoopHeadMBB);
|
|
LoopTailMBB->addSuccessor(TailMBB);
|
|
TailMBB->addSuccessor(DoneMBB);
|
|
DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
|
|
DoneMBB->transferSuccessors(&MBB);
|
|
MBB.addSuccessor(LoopHeadMBB);
|
|
|
|
Register DestReg = MI.getOperand(0).getReg();
|
|
Register Scratch1Reg = MI.getOperand(1).getReg();
|
|
Register Scratch2Reg = MI.getOperand(2).getReg();
|
|
Register AddrReg = MI.getOperand(3).getReg();
|
|
Register IncrReg = MI.getOperand(4).getReg();
|
|
Register MaskReg = MI.getOperand(5).getReg();
|
|
|
|
//
|
|
// .loophead:
|
|
// dbar 0
|
|
// ll.w destreg, (alignedaddr)
|
|
// and scratch2, destreg, mask
|
|
// move scratch1, destreg
|
|
BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::DBAR)).addImm(0);
|
|
BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::LL_W), DestReg)
|
|
.addReg(AddrReg)
|
|
.addImm(0);
|
|
BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::AND), Scratch2Reg)
|
|
.addReg(DestReg)
|
|
.addReg(MaskReg);
|
|
BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::OR), Scratch1Reg)
|
|
.addReg(DestReg)
|
|
.addReg(LoongArch::R0);
|
|
|
|
switch (BinOp) {
|
|
default:
|
|
llvm_unreachable("Unexpected AtomicRMW BinOp");
|
|
// bgeu scratch2, incr, .looptail
|
|
case AtomicRMWInst::UMax:
|
|
BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGEU))
|
|
.addReg(Scratch2Reg)
|
|
.addReg(IncrReg)
|
|
.addMBB(LoopTailMBB);
|
|
break;
|
|
// bgeu incr, scratch2, .looptail
|
|
case AtomicRMWInst::UMin:
|
|
BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGEU))
|
|
.addReg(IncrReg)
|
|
.addReg(Scratch2Reg)
|
|
.addMBB(LoopTailMBB);
|
|
break;
|
|
case AtomicRMWInst::Max:
|
|
insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
|
|
// bge scratch2, incr, .looptail
|
|
BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGE))
|
|
.addReg(Scratch2Reg)
|
|
.addReg(IncrReg)
|
|
.addMBB(LoopTailMBB);
|
|
break;
|
|
case AtomicRMWInst::Min:
|
|
insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
|
|
// bge incr, scratch2, .looptail
|
|
BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGE))
|
|
.addReg(IncrReg)
|
|
.addReg(Scratch2Reg)
|
|
.addMBB(LoopTailMBB);
|
|
break;
|
|
// TODO: support other AtomicRMWInst.
|
|
}
|
|
|
|
// .loopifbody:
|
|
// xor scratch1, destreg, incr
|
|
// and scratch1, scratch1, mask
|
|
// xor scratch1, destreg, scratch1
|
|
insertMaskedMerge(TII, DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg,
|
|
MaskReg, Scratch1Reg);
|
|
|
|
// .looptail:
|
|
// sc.w scratch1, scratch1, (addr)
|
|
// beqz scratch1, loop
|
|
// dbar 0x700
|
|
BuildMI(LoopTailMBB, DL, TII->get(LoongArch::SC_W), Scratch1Reg)
|
|
.addReg(Scratch1Reg)
|
|
.addReg(AddrReg)
|
|
.addImm(0);
|
|
BuildMI(LoopTailMBB, DL, TII->get(LoongArch::BEQZ))
|
|
.addReg(Scratch1Reg)
|
|
.addMBB(LoopHeadMBB);
|
|
|
|
// .tail:
|
|
// dbar 0x700
|
|
BuildMI(TailMBB, DL, TII->get(LoongArch::DBAR)).addImm(0x700);
|
|
|
|
NextMBBI = MBB.end();
|
|
MI.eraseFromParent();
|
|
|
|
LivePhysRegs LiveRegs;
|
|
computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
|
|
computeAndAddLiveIns(LiveRegs, *LoopIfBodyMBB);
|
|
computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
|
|
computeAndAddLiveIns(LiveRegs, *TailMBB);
|
|
computeAndAddLiveIns(LiveRegs, *DoneMBB);
|
|
|
|
return true;
|
|
}
|
|
|
|
bool LoongArchExpandAtomicPseudo::expandAtomicCmpXchg(
|
|
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsMasked,
|
|
int Width, MachineBasicBlock::iterator &NextMBBI) {
|
|
MachineInstr &MI = *MBBI;
|
|
DebugLoc DL = MI.getDebugLoc();
|
|
MachineFunction *MF = MBB.getParent();
|
|
auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
auto TailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
|
|
// Insert new MBBs.
|
|
MF->insert(++MBB.getIterator(), LoopHeadMBB);
|
|
MF->insert(++LoopHeadMBB->getIterator(), LoopTailMBB);
|
|
MF->insert(++LoopTailMBB->getIterator(), TailMBB);
|
|
MF->insert(++TailMBB->getIterator(), DoneMBB);
|
|
|
|
// Set up successors and transfer remaining instructions to DoneMBB.
|
|
LoopHeadMBB->addSuccessor(LoopTailMBB);
|
|
LoopHeadMBB->addSuccessor(TailMBB);
|
|
LoopTailMBB->addSuccessor(DoneMBB);
|
|
LoopTailMBB->addSuccessor(LoopHeadMBB);
|
|
TailMBB->addSuccessor(DoneMBB);
|
|
DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
|
|
DoneMBB->transferSuccessors(&MBB);
|
|
MBB.addSuccessor(LoopHeadMBB);
|
|
|
|
Register DestReg = MI.getOperand(0).getReg();
|
|
Register ScratchReg = MI.getOperand(1).getReg();
|
|
Register AddrReg = MI.getOperand(2).getReg();
|
|
Register CmpValReg = MI.getOperand(3).getReg();
|
|
Register NewValReg = MI.getOperand(4).getReg();
|
|
|
|
if (!IsMasked) {
|
|
// .loophead:
|
|
// ll.[w|d] dest, (addr)
|
|
// bne dest, cmpval, tail
|
|
BuildMI(LoopHeadMBB, DL,
|
|
TII->get(Width == 32 ? LoongArch::LL_W : LoongArch::LL_D), DestReg)
|
|
.addReg(AddrReg)
|
|
.addImm(0);
|
|
BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BNE))
|
|
.addReg(DestReg)
|
|
.addReg(CmpValReg)
|
|
.addMBB(TailMBB);
|
|
// .looptail:
|
|
// dbar 0
|
|
// move scratch, newval
|
|
// sc.[w|d] scratch, scratch, (addr)
|
|
// beqz scratch, loophead
|
|
// b done
|
|
BuildMI(LoopTailMBB, DL, TII->get(LoongArch::DBAR)).addImm(0);
|
|
BuildMI(LoopTailMBB, DL, TII->get(LoongArch::OR), ScratchReg)
|
|
.addReg(NewValReg)
|
|
.addReg(LoongArch::R0);
|
|
BuildMI(LoopTailMBB, DL,
|
|
TII->get(Width == 32 ? LoongArch::SC_W : LoongArch::SC_D),
|
|
ScratchReg)
|
|
.addReg(ScratchReg)
|
|
.addReg(AddrReg)
|
|
.addImm(0);
|
|
BuildMI(LoopTailMBB, DL, TII->get(LoongArch::BEQZ))
|
|
.addReg(ScratchReg)
|
|
.addMBB(LoopHeadMBB);
|
|
BuildMI(LoopTailMBB, DL, TII->get(LoongArch::B)).addMBB(DoneMBB);
|
|
} else {
|
|
// .loophead:
|
|
// ll.[w|d] dest, (addr)
|
|
// and scratch, dest, mask
|
|
// bne scratch, cmpval, tail
|
|
Register MaskReg = MI.getOperand(5).getReg();
|
|
BuildMI(LoopHeadMBB, DL,
|
|
TII->get(Width == 32 ? LoongArch::LL_W : LoongArch::LL_D), DestReg)
|
|
.addReg(AddrReg)
|
|
.addImm(0);
|
|
BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::AND), ScratchReg)
|
|
.addReg(DestReg)
|
|
.addReg(MaskReg);
|
|
BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BNE))
|
|
.addReg(ScratchReg)
|
|
.addReg(CmpValReg)
|
|
.addMBB(TailMBB);
|
|
|
|
// .looptail:
|
|
// dbar 0
|
|
// andn scratch, dest, mask
|
|
// or scratch, scratch, newval
|
|
// sc.[w|d] scratch, scratch, (addr)
|
|
// beqz scratch, loophead
|
|
// b done
|
|
BuildMI(LoopTailMBB, DL, TII->get(LoongArch::DBAR)).addImm(0);
|
|
BuildMI(LoopTailMBB, DL, TII->get(LoongArch::ANDN), ScratchReg)
|
|
.addReg(DestReg)
|
|
.addReg(MaskReg);
|
|
BuildMI(LoopTailMBB, DL, TII->get(LoongArch::OR), ScratchReg)
|
|
.addReg(ScratchReg)
|
|
.addReg(NewValReg);
|
|
BuildMI(LoopTailMBB, DL,
|
|
TII->get(Width == 32 ? LoongArch::SC_W : LoongArch::SC_D),
|
|
ScratchReg)
|
|
.addReg(ScratchReg)
|
|
.addReg(AddrReg)
|
|
.addImm(0);
|
|
BuildMI(LoopTailMBB, DL, TII->get(LoongArch::BEQZ))
|
|
.addReg(ScratchReg)
|
|
.addMBB(LoopHeadMBB);
|
|
BuildMI(LoopTailMBB, DL, TII->get(LoongArch::B)).addMBB(DoneMBB);
|
|
}
|
|
|
|
// .tail:
|
|
// dbar 0x700
|
|
BuildMI(TailMBB, DL, TII->get(LoongArch::DBAR)).addImm(0x700);
|
|
|
|
NextMBBI = MBB.end();
|
|
MI.eraseFromParent();
|
|
|
|
LivePhysRegs LiveRegs;
|
|
computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
|
|
computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
|
|
computeAndAddLiveIns(LiveRegs, *TailMBB);
|
|
computeAndAddLiveIns(LiveRegs, *DoneMBB);
|
|
|
|
return true;
|
|
}
|
|
|
|
} // end namespace
|
|
|
|
INITIALIZE_PASS(LoongArchExpandAtomicPseudo, "loongarch-expand-atomic-pseudo",
|
|
LoongArch_EXPAND_ATOMIC_PSEUDO_NAME, false, false)
|
|
|
|
namespace llvm {
|
|
|
|
FunctionPass *createLoongArchExpandAtomicPseudoPass() {
|
|
return new LoongArchExpandAtomicPseudo();
|
|
}
|
|
|
|
} // end namespace llvm
|