Skip to content

Commit

Permalink
[AArch64] Use neon instructions for i64/i128 ISD::PARITY calculation
Browse files Browse the repository at this point in the history 
As noticed on D129765 and reported on Issue llvm#56531 - aarch64 targets can use the neon ctpop + add-reduce instructions to speed up scalar ctpop instructions, but we fail to do this for parity calculations.

I'm not sure where the cutoff should be for specific CPUs, but i64 (+ i128 special case) shows a definite reduction in instruction count. i32 is about the same (but scalar <-> neon transfers are probably more costly?), and sub-i32 promotion looks to be a definite regression compared to parity expansion optimized for those widths.

Differential Revision: https://reviews.llvm.org/D130246
  • Loading branch information
@RKSimon
RKSimon committed Jul 22, 2022
1 parent 8f0ba6c commit 939cf9b
Show file tree
Hide file tree
Showing 3 changed files with 37 additions and 25 deletions.
24 changes: 21 additions & 3 deletions llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -521,6 +521,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::CTPOP, MVT::i64, Custom);
setOperationAction(ISD::CTPOP, MVT::i128, Custom);

setOperationAction(ISD::PARITY, MVT::i64, Custom);
setOperationAction(ISD::PARITY, MVT::i128, Custom);

setOperationAction(ISD::ABS, MVT::i32, Custom);
setOperationAction(ISD::ABS, MVT::i64, Custom);

Expand Down Expand Up @@ -5463,7 +5466,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
case ISD::SRA_PARTS:
return LowerShiftParts(Op, DAG);
case ISD::CTPOP:
return LowerCTPOP(Op, DAG);
case ISD::PARITY:
return LowerCTPOP_PARITY(Op, DAG);
case ISD::FCOPYSIGN:
return LowerFCOPYSIGN(Op, DAG);
case ISD::OR:
Expand Down Expand Up @@ -7783,14 +7787,17 @@ SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op,
return BitCast(VT, BSP, DAG);
}

SDValue AArch64TargetLowering::LowerCTPOP(SDValue Op, SelectionDAG &DAG) const {
SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op,
SelectionDAG &DAG) const {
if (DAG.getMachineFunction().getFunction().hasFnAttribute(
Attribute::NoImplicitFloat))
return SDValue();

if (!Subtarget->hasNEON())
return SDValue();

bool IsParity = Op.getOpcode() == ISD::PARITY;

// While there is no integer popcount instruction, it can
// be more efficiently lowered to the following sequence that uses
// AdvSIMD registers/instructions as long as the copies to/from
Expand All @@ -7813,6 +7820,10 @@ SDValue AArch64TargetLowering::LowerCTPOP(SDValue Op, SelectionDAG &DAG) const {
ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
DAG.getConstant(Intrinsic::aarch64_neon_uaddlv, DL, MVT::i32), CtPop);

if (IsParity)
UaddLV = DAG.getNode(ISD::AND, DL, MVT::i32, UaddLV,
DAG.getConstant(1, DL, MVT::i32));

if (VT == MVT::i64)
UaddLV = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, UaddLV);
return UaddLV;
Expand All @@ -7824,9 +7835,15 @@ SDValue AArch64TargetLowering::LowerCTPOP(SDValue Op, SelectionDAG &DAG) const {
ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
DAG.getConstant(Intrinsic::aarch64_neon_uaddlv, DL, MVT::i32), CtPop);

if (IsParity)
UaddLV = DAG.getNode(ISD::AND, DL, MVT::i32, UaddLV,
DAG.getConstant(1, DL, MVT::i32));

return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i128, UaddLV);
}

assert(!IsParity && "ISD::PARITY of vector types not supported");

if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT))
return LowerToPredicatedOp(Op, DAG, AArch64ISD::CTPOP_MERGE_PASSTHRU);

Expand Down Expand Up @@ -20058,7 +20075,8 @@ void AArch64TargetLowering::ReplaceNodeResults(
return;

case ISD::CTPOP:
if (SDValue Result = LowerCTPOP(SDValue(N, 0), DAG))
case ISD::PARITY:
if (SDValue Result = LowerCTPOP_PARITY(SDValue(N, 0), DAG))
Results.push_back(Result);
return;
case AArch64ISD::SADDV:
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Target/AArch64/AArch64ISelLowering.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1002,7 +1002,7 @@ class AArch64TargetLowering : public TargetLowering {
SDValue LowerVectorSRA_SRL_SHL(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerCTPOP(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerCTPOP_PARITY(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerCTTZ(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerBitreverse(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerMinMax(SDValue Op, SelectionDAG &DAG) const;
Expand Down
36 changes: 15 additions & 21 deletions llvm/test/CodeGen/AArch64/parity.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,13 +77,11 @@ define i32 @parity_32(i32 %x) {
define i64 @parity_64(i64 %x) {
; CHECK-LABEL: parity_64:
; CHECK: // %bb.0:
; CHECK-NEXT: eor x8, x0, x0, lsr #32
; CHECK-NEXT: eor x8, x8, x8, lsr #16
; CHECK-NEXT: eor x8, x8, x8, lsr #8
; CHECK-NEXT: eor x8, x8, x8, lsr #4
; CHECK-NEXT: eor x8, x8, x8, lsr #2
; CHECK-NEXT: eor w8, w8, w8, lsr #1
; CHECK-NEXT: and x0, x8, #0x1
; CHECK-NEXT: fmov d0, x0
; CHECK-NEXT: cnt v0.8b, v0.8b
; CHECK-NEXT: uaddlv h0, v0.8b
; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: and w0, w8, #0x1
; CHECK-NEXT: ret
%1 = tail call i64 @llvm.ctpop.i64(i64 %x)
%2 = and i64 %1, 1
Expand All @@ -93,15 +91,13 @@ define i64 @parity_64(i64 %x) {
define i128 @parity_128(i128 %x) {
; CHECK-LABEL: parity_128:
; CHECK: // %bb.0:
; CHECK-NEXT: eor x8, x0, x1
; CHECK-NEXT: fmov d0, x0
; CHECK-NEXT: mov v0.d[1], x1
; CHECK-NEXT: mov x1, xzr
; CHECK-NEXT: eor x8, x8, x8, lsr #32
; CHECK-NEXT: eor x8, x8, x8, lsr #16
; CHECK-NEXT: eor x8, x8, x8, lsr #8
; CHECK-NEXT: eor x8, x8, x8, lsr #4
; CHECK-NEXT: eor x8, x8, x8, lsr #2
; CHECK-NEXT: eor w8, w8, w8, lsr #1
; CHECK-NEXT: and x0, x8, #0x1
; CHECK-NEXT: cnt v0.16b, v0.16b
; CHECK-NEXT: uaddlv h0, v0.16b
; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: and w0, w8, #0x1
; CHECK-NEXT: ret
%1 = tail call i128 @llvm.ctpop.i128(i128 %x)
%2 = and i128 %1, 1
Expand All @@ -111,12 +107,10 @@ define i128 @parity_128(i128 %x) {
define i32 @parity_64_trunc(i64 %x) {
; CHECK-LABEL: parity_64_trunc:
; CHECK: // %bb.0:
; CHECK-NEXT: eor x8, x0, x0, lsr #32
; CHECK-NEXT: eor x8, x8, x8, lsr #16
; CHECK-NEXT: eor x8, x8, x8, lsr #8
; CHECK-NEXT: eor x8, x8, x8, lsr #4
; CHECK-NEXT: eor x8, x8, x8, lsr #2
; CHECK-NEXT: eor w8, w8, w8, lsr #1
; CHECK-NEXT: fmov d0, x0
; CHECK-NEXT: cnt v0.8b, v0.8b
; CHECK-NEXT: uaddlv h0, v0.8b
; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: and w0, w8, #0x1
; CHECK-NEXT: ret
%1 = tail call i64 @llvm.ctpop.i64(i64 %x)
Expand Down

0 comments on commit 939cf9b

Please sign in to comment.