Skip to content

Commit

Permalink
[SPIR-V] Fix validity of atomic instructions (#87051)
Browse files Browse the repository at this point in the history 
This PR fixes validity of atomic instructions and improves type
inference. More tests are able now to be accepted by `spirv-val`.
  • Loading branch information
@VyacheslavLevytskyy
VyacheslavLevytskyy authored Apr 2, 2024
1 parent 77e5c0a commit 6cce67a
Show file tree
Hide file tree
Showing 9 changed files with 252 additions and 37 deletions.
7 changes: 7 additions & 0 deletions llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,13 @@ static SPIRVType *getArgSPIRVType(const Function &F, unsigned ArgIdx,
cast<ConstantInt>(II->getOperand(2))->getZExtValue(), ST));
}

// Replace PointerType with TypedPointerType to be able to map SPIR-V types to
// LLVM types in a consistent manner
if (isUntypedPointerTy(OriginalArgType)) {
OriginalArgType =
TypedPointerType::get(Type::getInt8Ty(F.getContext()),
getPointerAddressSpace(OriginalArgType));
}
return GR->getOrCreateSPIRVType(OriginalArgType, MIRBuilder, ArgAccessQual);
}

Expand Down
89 changes: 79 additions & 10 deletions llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@ class SPIRVEmitIntrinsics
Type *deduceElementType(Value *I);
Type *deduceElementTypeHelper(Value *I);
Type *deduceElementTypeHelper(Value *I, std::unordered_set<Value *> &Visited);
Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
std::unordered_set<Value *> &Visited);
Type *deduceElementTypeByUsersDeep(Value *Op,
std::unordered_set<Value *> &Visited);

// deduce nested types of composites
Type *deduceNestedTypeHelper(User *U);
Expand Down Expand Up @@ -176,6 +180,44 @@ static inline void reportFatalOnTokenType(const Instruction *I) {
false);
}

// Set element pointer type to the given value of ValueTy and tries to
// specify this type further (recursively) by Operand value, if needed.
Type *SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(
Type *ValueTy, Value *Operand, std::unordered_set<Value *> &Visited) {
Type *Ty = ValueTy;
if (Operand) {
if (auto *PtrTy = dyn_cast<PointerType>(Ty)) {
if (Type *NestedTy = deduceElementTypeHelper(Operand, Visited))
Ty = TypedPointerType::get(NestedTy, PtrTy->getAddressSpace());
} else {
Ty = deduceNestedTypeHelper(dyn_cast<User>(Operand), Ty, Visited);
}
}
return Ty;
}

// Traverse User instructions to deduce an element pointer type of the operand.
Type *SPIRVEmitIntrinsics::deduceElementTypeByUsersDeep(
Value *Op, std::unordered_set<Value *> &Visited) {
if (!Op || !isPointerTy(Op->getType()))
return nullptr;

if (auto PType = dyn_cast<TypedPointerType>(Op->getType()))
return PType->getElementType();

// maybe we already know operand's element type
if (Type *KnownTy = GR->findDeducedElementType(Op))
return KnownTy;

for (User *OpU : Op->users()) {
if (Instruction *Inst = dyn_cast<Instruction>(OpU)) {
if (Type *Ty = deduceElementTypeHelper(Inst, Visited))
return Ty;
}
}
return nullptr;
}

// Deduce and return a successfully deduced Type of the Instruction,
// or nullptr otherwise.
Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(Value *I) {
Expand Down Expand Up @@ -206,21 +248,27 @@ Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
} else if (auto *Ref = dyn_cast<GetElementPtrInst>(I)) {
Ty = Ref->getResultElementType();
} else if (auto *Ref = dyn_cast<GlobalValue>(I)) {
Ty = Ref->getValueType();
if (Value *Op = Ref->getNumOperands() > 0 ? Ref->getOperand(0) : nullptr) {
if (auto *PtrTy = dyn_cast<PointerType>(Ty)) {
if (Type *NestedTy = deduceElementTypeHelper(Op, Visited))
Ty = TypedPointerType::get(NestedTy, PtrTy->getAddressSpace());
} else {
Ty = deduceNestedTypeHelper(dyn_cast<User>(Op), Ty, Visited);
}
}
Ty = deduceElementTypeByValueDeep(
Ref->getValueType(),
Ref->getNumOperands() > 0 ? Ref->getOperand(0) : nullptr, Visited);
} else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(I)) {
Ty = deduceElementTypeHelper(Ref->getPointerOperand(), Visited);
} else if (auto *Ref = dyn_cast<BitCastInst>(I)) {
if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy();
isPointerTy(Src) && isPointerTy(Dest))
Ty = deduceElementTypeHelper(Ref->getOperand(0), Visited);
} else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(I)) {
Value *Op = Ref->getNewValOperand();
Ty = deduceElementTypeByValueDeep(Op->getType(), Op, Visited);
} else if (auto *Ref = dyn_cast<AtomicRMWInst>(I)) {
Value *Op = Ref->getValOperand();
Ty = deduceElementTypeByValueDeep(Op->getType(), Op, Visited);
} else if (auto *Ref = dyn_cast<PHINode>(I)) {
for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) {
Ty = deduceElementTypeByUsersDeep(Ref->getIncomingValue(i), Visited);
if (Ty)
break;
}
}

// remember the found relationship
Expand Down Expand Up @@ -293,6 +341,22 @@ Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(
return NewTy;
}
}
} else if (auto *VecTy = dyn_cast<VectorType>(OrigTy)) {
if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(0) : nullptr) {
Type *OpTy = VecTy->getElementType();
Type *Ty = OpTy;
if (auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
if (Type *NestedTy = deduceElementTypeHelper(Op, Visited))
Ty = TypedPointerType::get(NestedTy, PtrTy->getAddressSpace());
} else {
Ty = deduceNestedTypeHelper(dyn_cast<User>(Op), OpTy, Visited);
}
if (Ty != OpTy) {
Type *NewTy = VectorType::get(Ty, VecTy->getElementCount());
GR->addDeducedCompositeType(U, NewTy);
return NewTy;
}
}
}

return OrigTy;
Expand Down Expand Up @@ -578,7 +642,8 @@ void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I,

// Handle calls to builtins (non-intrinsics):
CallInst *CI = dyn_cast<CallInst>(I);
if (!CI || CI->isIndirectCall() || CI->getCalledFunction()->isIntrinsic())
if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
!CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
return;

// collect information about formal parameter types
Expand Down Expand Up @@ -929,6 +994,10 @@ Type *SPIRVEmitIntrinsics::deduceFunParamElementType(
// maybe we already know operand's element type
if (Type *KnownTy = GR->findDeducedElementType(OpArg))
return KnownTy;
// try to deduce from the operand itself
Visited.clear();
if (Type *Ty = deduceElementTypeHelper(OpArg, Visited))
return Ty;
// search in actual parameter's users
for (User *OpU : OpArg->users()) {
Instruction *Inst = dyn_cast<Instruction>(OpU);
Expand Down
42 changes: 42 additions & 0 deletions llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,17 @@ void validateForwardCalls(const SPIRVSubtarget &STI,
}
}

// Validation of an access chain.
void validateAccessChain(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI,
SPIRVGlobalRegistry &GR, MachineInstr &I) {
SPIRVType *BaseTypeInst = GR.getSPIRVTypeForVReg(I.getOperand(0).getReg());
if (BaseTypeInst && BaseTypeInst->getOpcode() == SPIRV::OpTypePointer) {
SPIRVType *BaseElemType =
GR.getSPIRVTypeForVReg(BaseTypeInst->getOperand(2).getReg());
validatePtrTypes(STI, MRI, GR, I, 2, BaseElemType);
}
}

// TODO: the logic of inserting additional bitcast's is to be moved
// to pre-IRTranslation passes eventually
void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const {
Expand All @@ -213,16 +224,47 @@ void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const {
MBBI != MBBE;) {
MachineInstr &MI = *MBBI++;
switch (MI.getOpcode()) {
case SPIRV::OpAtomicLoad:
case SPIRV::OpAtomicExchange:
case SPIRV::OpAtomicCompareExchange:
case SPIRV::OpAtomicCompareExchangeWeak:
case SPIRV::OpAtomicIIncrement:
case SPIRV::OpAtomicIDecrement:
case SPIRV::OpAtomicIAdd:
case SPIRV::OpAtomicISub:
case SPIRV::OpAtomicSMin:
case SPIRV::OpAtomicUMin:
case SPIRV::OpAtomicSMax:
case SPIRV::OpAtomicUMax:
case SPIRV::OpAtomicAnd:
case SPIRV::OpAtomicOr:
case SPIRV::OpAtomicXor:
// for the above listed instructions
// OpAtomicXXX <ResType>, ptr %Op, ...
// implies that %Op is a pointer to <ResType>
case SPIRV::OpLoad:
// OpLoad <ResType>, ptr %Op implies that %Op is a pointer to <ResType>
validatePtrTypes(STI, MRI, GR, MI, 2,
GR.getSPIRVTypeForVReg(MI.getOperand(0).getReg()));
break;
case SPIRV::OpAtomicStore:
// OpAtomicStore ptr %Op, <Scope>, <Mem>, <Obj>
// implies that %Op points to the <Obj>'s type
validatePtrTypes(STI, MRI, GR, MI, 0,
GR.getSPIRVTypeForVReg(MI.getOperand(3).getReg()));
break;
case SPIRV::OpStore:
// OpStore ptr %Op, <Obj> implies that %Op points to the <Obj>'s type
validatePtrTypes(STI, MRI, GR, MI, 0,
GR.getSPIRVTypeForVReg(MI.getOperand(1).getReg()));
break;
case SPIRV::OpPtrCastToGeneric:
validateAccessChain(STI, MRI, GR, MI);
break;
case SPIRV::OpInBoundsPtrAccessChain:
if (MI.getNumOperands() == 4)
validateAccessChain(STI, MRI, GR, MI);
break;

case SPIRV::OpFunctionCall:
// ensure there is no mismatch between actual and expected arg types:
Expand Down
1 change: 1 addition & 0 deletions llvm/test/CodeGen/SPIRV/ExecutionMode.ll
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}

; CHECK-DAG: %[[#VOID:]] = OpTypeVoid

Expand Down
28 changes: 19 additions & 9 deletions llvm/test/CodeGen/SPIRV/instructions/atomic.ll
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}

; CHECK-DAG: OpName [[ADD:%.*]] "test_add"
; CHECK-DAG: OpName [[SUB:%.*]] "test_sub"
Expand All @@ -20,7 +21,8 @@
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter
; CHECK-NEXT: OpLabel
; CHECK-NEXT: [[R:%.*]] = OpAtomicIAdd [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicIAdd [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
define i32 @test_add(i32* %ptr, i32 %val) {
Expand All @@ -32,7 +34,8 @@ define i32 @test_add(i32* %ptr, i32 %val) {
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter
; CHECK-NEXT: OpLabel
; CHECK-NEXT: [[R:%.*]] = OpAtomicISub [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicISub [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
define i32 @test_sub(i32* %ptr, i32 %val) {
Expand All @@ -44,7 +47,8 @@ define i32 @test_sub(i32* %ptr, i32 %val) {
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter
; CHECK-NEXT: OpLabel
; CHECK-NEXT: [[R:%.*]] = OpAtomicSMin [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicSMin [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
define i32 @test_min(i32* %ptr, i32 %val) {
Expand All @@ -56,7 +60,8 @@ define i32 @test_min(i32* %ptr, i32 %val) {
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter
; CHECK-NEXT: OpLabel
; CHECK-NEXT: [[R:%.*]] = OpAtomicSMax [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicSMax [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
define i32 @test_max(i32* %ptr, i32 %val) {
Expand All @@ -68,7 +73,8 @@ define i32 @test_max(i32* %ptr, i32 %val) {
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter
; CHECK-NEXT: OpLabel
; CHECK-NEXT: [[R:%.*]] = OpAtomicUMin [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicUMin [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
define i32 @test_umin(i32* %ptr, i32 %val) {
Expand All @@ -80,7 +86,8 @@ define i32 @test_umin(i32* %ptr, i32 %val) {
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter
; CHECK-NEXT: OpLabel
; CHECK-NEXT: [[R:%.*]] = OpAtomicUMax [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicUMax [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
define i32 @test_umax(i32* %ptr, i32 %val) {
Expand All @@ -92,7 +99,8 @@ define i32 @test_umax(i32* %ptr, i32 %val) {
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter
; CHECK-NEXT: OpLabel
; CHECK-NEXT: [[R:%.*]] = OpAtomicAnd [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicAnd [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
define i32 @test_and(i32* %ptr, i32 %val) {
Expand All @@ -104,7 +112,8 @@ define i32 @test_and(i32* %ptr, i32 %val) {
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter
; CHECK-NEXT: OpLabel
; CHECK-NEXT: [[R:%.*]] = OpAtomicOr [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicOr [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
define i32 @test_or(i32* %ptr, i32 %val) {
Expand All @@ -116,7 +125,8 @@ define i32 @test_or(i32* %ptr, i32 %val) {
; CHECK-NEXT: [[A:%.*]] = OpFunctionParameter
; CHECK-NEXT: [[B:%.*]] = OpFunctionParameter
; CHECK-NEXT: OpLabel
; CHECK-NEXT: [[R:%.*]] = OpAtomicXor [[I32Ty]] [[A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: [[BC_A:%.*]] = OpBitcast %[[#]] [[A]]
; CHECK-NEXT: [[R:%.*]] = OpAtomicXor [[I32Ty]] [[BC_A]] [[SCOPE]] [[RELAXED]] [[B]]
; CHECK-NEXT: OpReturnValue [[R]]
; CHECK-NEXT: OpFunctionEnd
define i32 @test_xor(i32* %ptr, i32 %val) {
Expand Down
Loading

0 comments on commit 6cce67a

Please sign in to comment.