[clang & libcxx] constexpr pointer tagging (not for merging, just for…

… review)

clang/include/clang/AST/APValue.h

@@ -198,6 +198,8 @@ class APValue {
       /// The QualType, if this is a DynamicAllocLValue.
       void *DynamicAllocType;
     };
+  public:
+    uint64_t Metadata{0};
   };
 
   /// A FieldDecl or CXXRecordDecl, along with a flag indicating whether we
@@ -527,6 +529,8 @@ class APValue {
   }
 
   const LValueBase getLValueBase() const;
+  uint64_t getLValueMetadata() const;
+  uint64_t & getLValueMetadata();
   CharUnits &getLValueOffset();
   const CharUnits &getLValueOffset() const {
     return const_cast<APValue*>(this)->getLValueOffset();

clang/include/clang/Basic/Builtins.td

@@ -4867,3 +4867,40 @@ def ArithmeticFence : LangBuiltin<"ALL_LANGUAGES"> {
   let Attributes = [CustomTypeChecking, Constexpr];
   let Prototype = "void(...)";
 }
+
+// support for pointer tagging
+// (ptr & mask) | (val & ~mask)
+def TagPointerMaskOr : Builtin {
+  let Spellings = ["__builtin_tag_pointer_mask_or"];
+  let Attributes = [Constexpr, NoThrow];
+  let Prototype = "void*(void*, size_t, size_t)";
+}
+
+// (ptr & mask) -> void *
+def TagPointerMask : Builtin {
+  let Spellings = ["__builtin_tag_pointer_mask"];
+  let Attributes = [Constexpr, NoThrow];
+  let Prototype = "void*(void*, size_t)";
+}
+
+// (ptr & mask) -> uintptr_t
+def TagPointerMaskAsInt : Builtin {
+  let Spellings = ["__builtin_tag_pointer_mask_as_int"];
+  let Attributes = [Constexpr, NoThrow];
+  let Prototype = "size_t(void*, size_t)";
+}
+
+// (ptr << shift) | (value & ~mask) -> void *
+// mask = (1 << shift) - 1
+def TagPointerShiftOr : Builtin {
+  let Spellings = ["__builtin_tag_pointer_shift_or"];
+  let Attributes = [Constexpr, NoThrow];
+  let Prototype = "void*(void*, size_t, size_t)";
+}
+
+// (ptr >> unshift) -> void *
+def TagPointerUnshift : Builtin {
+  let Spellings = ["__builtin_tag_pointer_unshift"];
+  let Attributes = [Constexpr, NoThrow];
+  let Prototype = "void*(void*, size_t)";
+}

clang/include/clang/Basic/DiagnosticASTKinds.td

@@ -218,6 +218,12 @@ def note_constexpr_access_null : Note<
 def note_constexpr_access_past_end : Note<
   "%sub{access_kind}0 dereferenced one-past-the-end pointer "
   "is not allowed in a constant expression">;
+def note_constexpr_dereferencing_tagged_pointer: Note<
+  "dereferencing tagged pointer">;
+def note_constexpr_tagging_with_shift_zero: Note<
+  "you must shift pointer at least by one bit to store a tag">;
+def note_constexpr_tagging_with_empty_mask: Note<
+  "you must provide non-zero mask for pointer tagging">;
 def note_constexpr_access_unsized_array : Note<
   "%sub{access_kind}0 element of array without known bound "
   "is not allowed in a constant expression">;

clang/lib/AST/APValue.cpp

@@ -975,6 +975,16 @@ const APValue::LValueBase APValue::getLValueBase() const {
   return ((const LV *)(const void *)&Data)->Base;
 }
 
+uint64_t APValue::getLValueMetadata() const {
+  assert(isLValue() && "Invalid accessor");
+  return ((const LV *)(const void *)&Data)->Base.Metadata;
+}
+
+uint64_t & APValue::getLValueMetadata() {
+  assert(isLValue() && "Invalid accessor");
+  return ((LV *)(void *)&Data)->Base.Metadata;
+}
+
 bool APValue::isLValueOnePastTheEnd() const {
   assert(isLValue() && "Invalid accessor");
   return ((const LV *)(const void *)&Data)->IsOnePastTheEnd;

clang/lib/AST/ExprConstant.cpp

@@ -4498,6 +4498,11 @@ handleLValueToRValueConversion(EvalInfo &Info, const Expr *Conv, QualType Type,
                                bool WantObjectRepresentation = false) {
   if (LVal.Designator.Invalid)
     return false;
+
+  if (LVal.Base.Metadata != 0) {
+    Info.FFDiag(Conv, diag::note_constexpr_dereferencing_tagged_pointer);
+    return false;
+  }
 
   // Check for special cases where there is no existing APValue to look at.
   const Expr *Base = LVal.Base.dyn_cast<const Expr*>();
@@ -9735,6 +9740,87 @@ bool PointerExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
     return Success(E);
 
   switch (BuiltinOp) {
+  // emulation of pointer tagging without actually touching pointer value
+  // as there is no such thing as address here, so tag is stored as a metadata in lvalue base
+  case Builtin::BI__builtin_tag_pointer_mask_or: { 
+      APSInt Value, Mask;
+      if (!evaluatePointer(E->getArg(0), Result))
+        return Error(E);
+
+      if (!EvaluateInteger(E->getArg(1), Value, Info))
+        return Error(E);
+
+      if (!EvaluateInteger(E->getArg(2), Mask, Info))
+        return Error(E);
+
+      if (Mask.getLimitedValue() == 0) {
+        CCEDiag(E->getArg(2), diag::note_constexpr_tagging_with_empty_mask);
+        return false;
+      }
+
+      Result.Base.Metadata = (Result.Base.Metadata & ~Mask.getLimitedValue()) | (Value.getLimitedValue() & Mask.getLimitedValue());
+      return true;
+    }
+
+  // alternative approach to tagging which shifts pointer
+  // here we are only shifting metadata
+  case Builtin::BI__builtin_tag_pointer_shift_or: {
+    APSInt Value, Shift;
+    if (!evaluatePointer(E->getArg(0), Result))
+      return Error(E);
+
+    if (!EvaluateInteger(E->getArg(1), Value, Info))
+      return Error(E);
+
+    if (!EvaluateInteger(E->getArg(2), Shift, Info))
+      return Error(E);
+
+    if (Shift.getLimitedValue() == 0) {
+      CCEDiag(E->getArg(2), diag::note_constexpr_tagging_with_shift_zero);
+      return false;
+    }
+
+    const uint64_t Mask = (1ull << static_cast<uint64_t>(Shift.getLimitedValue())) - 1ull;
+    Result.Base.Metadata = (Result.Base.Metadata << static_cast<uint64_t>(Shift.getLimitedValue())) | (Value.getLimitedValue() & Mask);
+    return true;
+  }
+
+  // recover pointer by masking metadata
+  // exprconstant allows dereferencing only metadata == 0 pointer
+  case Builtin::BI__builtin_tag_pointer_mask: {
+    APSInt Mask;
+    if (!evaluatePointer(E->getArg(0), Result))
+        return Error(E);
+
+    if (!EvaluateInteger(E->getArg(1), Mask, Info))
+      return Error(E);
+
+    if (Mask.getLimitedValue() == 0) {
+      CCEDiag(E->getArg(2), diag::note_constexpr_tagging_with_empty_mask);
+      return false;
+    }
+
+    Result.Base.Metadata = (Result.Base.Metadata & Mask.getLimitedValue());
+    return true;
+  }
+
+  // shifting back pointer (also can convert tagged pointer back to normal pointer)
+  case Builtin::BI__builtin_tag_pointer_unshift: {
+    APSInt Shift;
+    if (!evaluatePointer(E->getArg(0), Result))
+        return Error(E);
+
+    if (!EvaluateInteger(E->getArg(1), Shift, Info))
+      return Error(E);
+
+    if (Shift.getLimitedValue() == 0) {
+      CCEDiag(E->getArg(2), diag::note_constexpr_tagging_with_shift_zero);
+      return false;
+    }
+
+    Result.Base.Metadata = (Result.Base.Metadata >> static_cast<uint64_t>(Shift.getLimitedValue()));
+    return true;
+  }
   case Builtin::BIaddressof:
   case Builtin::BI__addressof:
   case Builtin::BI__builtin_addressof:
@@ -12662,6 +12748,25 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
   default:
     return false;
 
+  case Builtin::BI__builtin_tag_pointer_mask_as_int: {
+    LValue Pointer;
+    APSInt Mask;
+
+    if (!EvaluatePointer(E->getArg(0), Pointer, Info))
+      return Error(E);
+
+    if (!EvaluateInteger(E->getArg(1), Mask, Info))
+      return Error(E);
+
+    if (Mask.getLimitedValue() == 0) {
+      CCEDiag(E->getArg(2), diag::note_constexpr_tagging_with_empty_mask);
+      return false;
+    }
+
+    const uint64_t Result = Pointer.Base.Metadata & (static_cast<uint64_t>(Mask.getLimitedValue()));
+    return Success(Result, E);
+  }
+
   case Builtin::BI__builtin_dynamic_object_size:
   case Builtin::BI__builtin_object_size: {
     // The type was checked when we built the expression.
@@ -14219,6 +14324,13 @@ EvaluateComparisonBinaryOperator(EvalInfo &Info, const BinaryOperator *E,
       return Success(CmpResult::Less, E);
     if (CompareLHS > CompareRHS)
       return Success(CmpResult::Greater, E);
+
+    // this makes tagged pointer not equal to original pointer
+    if (LHSValue.Base.Metadata < RHSValue.Base.Metadata)
+      return Success(CmpResult::Less, E);
+    if (LHSValue.Base.Metadata > RHSValue.Base.Metadata)
+      return Success(CmpResult::Greater, E);
+
     return Success(CmpResult::Equal, E);
   }
 

clang/lib/CodeGen/CGBuiltin.cpp

@@ -5320,6 +5320,19 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
 
     return RValue::get(Carry);
   }
+
+  // support for pointer tagging
+  case Builtin::BI__builtin_tag_pointer_mask_or:
+    return EmitBuiltinTagPointerMaskOr(E);
+  case Builtin::BI__builtin_tag_pointer_mask:
+    return EmitBuiltinTagPointerMask(E);
+  case Builtin::BI__builtin_tag_pointer_mask_as_int:
+    return EmitBuiltinTagPointerMaskAsInt(E);
+  case Builtin::BI__builtin_tag_pointer_shift_or:
+    return EmitBuiltinTagPointerShiftOr(E);
+  case Builtin::BI__builtin_tag_pointer_unshift:
+    return EmitBuiltinTagPointerUnshift(E);
+
   case Builtin::BIaddressof:
   case Builtin::BI__addressof:
   case Builtin::BI__builtin_addressof:
@@ -21245,6 +21258,95 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
   }
 }
 
+/// Generate (x & ~mask) | (value & mask).
+RValue CodeGenFunction::EmitBuiltinTagPointerMaskOr(const CallExpr *E) {
+  llvm::Value * Ptr = EmitScalarExpr(E->getArg(0));
+  llvm::Value * Value = EmitScalarExpr(E->getArg(1));
+  llvm::Value * Mask = EmitScalarExpr(E->getArg(2));
+
+  llvm::IntegerType * IntType = IntegerType::get(getLLVMContext(), CGM.getDataLayout().getIndexTypeSizeInBits(Ptr->getType()));
+
+  // TODO: avoid using bitcast and go path of ptr.tag (mirror to ptr.mask)
+  // to keep pointer's provenance, but this turns out a bit harder to do as it touches 
+  // a lot of places in llvm
+  llvm::Value * PointerInt = Builder.CreateBitOrPointerCast(Ptr, IntType, "pointer_int");
+  llvm::Value * InvertedMask = Builder.CreateNot(Mask, "inverted_mask");
+
+  llvm::Value * MaskedPtr = Builder.CreateAnd(PointerInt, InvertedMask, "masked_ptr");
+  llvm::Value * MaskedValue = Builder.CreateAnd(Value, Mask, "masked_value");
+
+  llvm::Value * ResultInt = Builder.CreateOr(MaskedPtr, MaskedValue, "result_int");
+  llvm::Value * Result = Builder.CreateBitOrPointerCast(ResultInt, Ptr->getType(), "result_ptr");
+
+  return RValue::get(Result);
+}
+
+/// Generate (x << shift) | (value & ((1 << shift) - 1)).
+RValue CodeGenFunction::EmitBuiltinTagPointerShiftOr(const CallExpr *E) {
+  llvm::Value * Ptr = EmitScalarExpr(E->getArg(0));
+  llvm::Value * Value = EmitScalarExpr(E->getArg(1));
+  llvm::Value * Shift = EmitScalarExpr(E->getArg(2));
+
+  llvm::IntegerType * IntType = IntegerType::get(getLLVMContext(), CGM.getDataLayout().getIndexTypeSizeInBits(Ptr->getType()));
+
+  // TODO: again, for now a bitcast, later ptr.shift_tag
+  llvm::Value * PointerInt = Builder.CreateBitOrPointerCast(Ptr, IntType, "pointer_int");
+  llvm::Value * ShiftedPointerInt = Builder.CreateShl(PointerInt, Shift);
+
+  auto *One = llvm::ConstantInt::get(IntType, 1);
+
+  llvm::Value * Mask = Builder.CreateSub(Builder.CreateShl(One, Shift), One, "mask");
+  llvm::Value * MaskedValue = Builder.CreateAdd(Value, Mask, "masked_value");
+  llvm::Value * PointerWithTag = Builder.CreateOr(ShiftedPointerInt, MaskedValue, "pointer_with_tag_int");
+
+  llvm::Value * Result = Builder.CreateBitOrPointerCast(PointerWithTag, Ptr->getType(), "result_ptr");
+  return RValue::get(Result);
+}
+
+/// Generate (x >> shift)
+RValue CodeGenFunction::EmitBuiltinTagPointerUnshift(const CallExpr *E) {
+  llvm::Value * Ptr = EmitScalarExpr(E->getArg(0));
+  llvm::Value * Shift = EmitScalarExpr(E->getArg(1));
+
+  llvm::IntegerType * IntType = IntegerType::get(getLLVMContext(), CGM.getDataLayout().getIndexTypeSizeInBits(Ptr->getType()));
+
+  // for now I'm going path of bitcast
+  llvm::Value * PointerInt = Builder.CreateBitOrPointerCast(Ptr, IntType, "pointer_int");
+  llvm::Value * UnShiftedPointerInt = Builder.CreateAShr(PointerInt, Shift, "unshifted_pointer_int");
+
+  llvm::Value * Result = Builder.CreateBitOrPointerCast(UnShiftedPointerInt, Ptr->getType(), "result_ptr");
+  return RValue::get(Result);
+}
+
+/// Generate (x & mask).
+RValue CodeGenFunction::EmitBuiltinTagPointerMask(const CallExpr *E) {
+  llvm::Value * Ptr = EmitScalarExpr(E->getArg(0));
+  llvm::Value * Mask = EmitScalarExpr(E->getArg(1));
+
+  llvm::Value *Result = Builder.CreateIntrinsic(
+        Intrinsic::ptrmask, {Ptr->getType(), Mask->getType()},
+        {Ptr, Mask}, nullptr, "result");
+
+  return RValue::get(Result);
+}
+
+/// Generate (x & mask) (but return it as number).
+RValue CodeGenFunction::EmitBuiltinTagPointerMaskAsInt(const CallExpr *E) {
+  llvm::Value * Ptr = EmitScalarExpr(E->getArg(0));
+  llvm::Value * Mask = EmitScalarExpr(E->getArg(1));
+
+  llvm::IntegerType * IntType = IntegerType::get(getLLVMContext(), CGM.getDataLayout().getIndexTypeSizeInBits(Ptr->getType()));
+
+  llvm::Value *Result = Builder.CreateIntrinsic(
+        Intrinsic::ptrmask, {Ptr->getType(), Mask->getType()},
+        {Ptr, Mask}, nullptr, "result");
+
+  llvm::Value * IntResult = Builder.CreateBitOrPointerCast(Result, IntType, "int_result");
+
+  return RValue::get(IntResult);
+}
+
+
 namespace {
 struct BuiltinAlignArgs {
   llvm::Value *Src = nullptr;

clang/lib/CodeGen/CodeGenFunction.h

@@ -4556,6 +4556,13 @@ class CodeGenFunction : public CodeGenTypeCache {
 
   RValue emitRotate(const CallExpr *E, bool IsRotateRight);
 
+  /// Emit IR for pointer tagging
+  RValue EmitBuiltinTagPointerMaskOr(const CallExpr *E);
+  RValue EmitBuiltinTagPointerMask(const CallExpr *E);
+  RValue EmitBuiltinTagPointerMaskAsInt(const CallExpr *E);
+  RValue EmitBuiltinTagPointerShiftOr(const CallExpr *E);
+  RValue EmitBuiltinTagPointerUnshift(const CallExpr *E);
+
   /// Emit IR for __builtin_os_log_format.
   RValue emitBuiltinOSLogFormat(const CallExpr &E);
 

libcxx/include/CMakeLists.txt

@@ -553,6 +553,7 @@ set(files
   __memory/raw_storage_iterator.h
   __memory/shared_ptr.h
   __memory/swap_allocator.h
+  __memory/tagged_ptr.h
   __memory/temp_value.h
   __memory/temporary_buffer.h
   __memory/uninitialized_algorithms.h