diff --git a/ydb/library/yql/udfs/common/roaring/roaring.cpp b/ydb/library/yql/udfs/common/roaring/roaring.cpp new file mode 100644 index 000000000000..fdb582ef41da --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/roaring.cpp @@ -0,0 +1,436 @@ +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +using namespace NKikimr; +using namespace NUdf; + +namespace { + + using namespace roaring::api; + + inline roaring_bitmap_t* DeserializePortable(TStringRef binaryString) { + auto bitmap = roaring_bitmap_portable_deserialize_safe(binaryString.Data(), binaryString.Size()); + Y_ENSURE(bitmap); + return bitmap; + } + + struct TRoaringWrapper: public TBoxedValue { + TRoaringWrapper(TStringRef binaryString) + : Roaring(DeserializePortable(binaryString)) + { + } + + ~TRoaringWrapper() { + roaring_bitmap_free(Roaring); + } + + roaring_bitmap_t* Roaring; + }; + + inline roaring_bitmap_t* GetBitmapFromArg(TUnboxedValuePod arg) { + return static_cast(arg.AsBoxed().Get())->Roaring; + } + + class TRoaringOrWithBinary: public TBoxedValue { + public: + TRoaringOrWithBinary() { + } + + static TStringRef Name() { + return TStringRef::Of("OrWithBinary"); + } + + private: + TUnboxedValue Run(const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const override { + Y_UNUSED(valueBuilder); + auto binaryString = args[1].AsStringRef(); + auto bitmap = DeserializePortable(binaryString); + + roaring_bitmap_or_inplace(GetBitmapFromArg(args[0]), bitmap); + roaring_bitmap_free(bitmap); + + return args[0]; + } + }; + + class TRoaringAndWithBinary: public TBoxedValue { + public: + TRoaringAndWithBinary() { + } + + static TStringRef Name() { + return TStringRef::Of("AndWithBinary"); + } + + private: + TUnboxedValue Run(const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const override { + Y_UNUSED(valueBuilder); + auto binaryString = args[1].AsStringRef(); + auto bitmap = DeserializePortable(binaryString); + + roaring_bitmap_and_inplace(GetBitmapFromArg(args[0]), bitmap); + roaring_bitmap_free(bitmap); + + return args[0]; + } + }; + + class TRoaringAnd: public TBoxedValue { + public: + TRoaringAnd() { + } + + static TStringRef Name() { + return TStringRef::Of("And"); + } + + private: + TUnboxedValue Run(const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const override { + Y_UNUSED(valueBuilder); + roaring_bitmap_and_inplace(GetBitmapFromArg(args[0]), GetBitmapFromArg(args[1])); + return args[0]; + } + }; + + class TRoaringOr: public TBoxedValue { + public: + TRoaringOr() { + } + + static TStringRef Name() { + return TStringRef::Of("Or"); + } + + private: + TUnboxedValue Run(const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const override { + Y_UNUSED(valueBuilder); + roaring_bitmap_or_inplace(GetBitmapFromArg(args[0]), GetBitmapFromArg(args[1])); + return args[0]; + } + }; + + class TRoaringUint32List: public TBoxedValue { + public: + static TStringRef Name() { + return TStringRef::Of("Uint32List"); + } + + private: + class TIterator: public TManagedBoxedValue { + public: + TIterator(roaring_bitmap_t* Roaring) { + Iter_ = roaring_iterator_create(Roaring); + } + // Any iterator. + bool Skip() override { + if (!Iter_->has_value) { + return false; + } + roaring_uint32_iterator_advance(Iter_); + return true; + }; + + // List iterator. + bool Next(TUnboxedValue& value) override { + if (!Iter_->has_value) { + return false; + } + value = TUnboxedValuePod(Iter_->current_value); + roaring_uint32_iterator_advance(Iter_); + return true; + }; + + private: + roaring_uint32_iterator_t* Iter_; + }; + + class TList: public TBoxedValue { + public: + TList(roaring_bitmap_t* Roaring) + : Roaring_(Roaring) + { + Length_ = roaring_bitmap_get_cardinality(Roaring_); + } + + bool HasFastListLength() const override { + return true; + }; + + ui64 GetListLength() const override { + return Length_; + }; + + ui64 GetEstimatedListLength() const override { + return GetListLength(); + }; + + TUnboxedValue GetListIterator() const override { + return TUnboxedValuePod(new TIterator(Roaring_)); + }; + + private: + roaring_bitmap_t* Roaring_; + ui64 Length_; + }; + + TUnboxedValue Run(const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const override { + Y_UNUSED(valueBuilder); + auto bitmap = GetBitmapFromArg(args[0]); + + return TUnboxedValuePod(new TList(bitmap)); + } + }; + + class TRoaringDeserialize: public TBoxedValue { + public: + TRoaringDeserialize() { + } + + static TStringRef Name() { + return TStringRef::Of("Deserialize"); + } + + private: + TUnboxedValue Run(const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const override { + Y_UNUSED(valueBuilder); + return TUnboxedValuePod(new TRoaringWrapper(args[0].AsStringRef())); + } + }; + + class TRoaringSerialize: public TBoxedValue { + public: + TRoaringSerialize() { + } + + static TStringRef Name() { + return TStringRef::Of("Serialize"); + } + + private: + TUnboxedValue Run(const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const override { + auto bitmap = GetBitmapFromArg(args[0]); + roaring_bitmap_run_optimize(bitmap); + + auto sizeInBytes = roaring_bitmap_portable_size_in_bytes(bitmap); + auto buf = (char*)UdfAllocateWithSize(sizeInBytes); + roaring_bitmap_portable_serialize(bitmap, buf); + auto string = valueBuilder->NewString(TStringRef(buf, sizeInBytes)); + UdfFreeWithSize((void*)buf, sizeInBytes); + + return string; + } + }; + + class TRoaringCardinality: public TBoxedValue { + public: + TRoaringCardinality() { + } + + static TStringRef Name() { + return TStringRef::Of("Cardinality"); + } + + private: + TUnboxedValue Run(const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const override { + Y_UNUSED(valueBuilder); + auto bitmap = GetBitmapFromArg(args[0]); + auto cardinality = (ui32)roaring_bitmap_get_cardinality(bitmap); + return TUnboxedValuePod(cardinality); + } + }; + + class TRoaringModule: public IUdfModule { + public: + TStringRef Name() const { + return TStringRef::Of("Roaring"); + } + + void GetAllFunctions(IFunctionsSink& sink) const final { + sink.Add(TRoaringSerialize::Name()); + sink.Add(TRoaringDeserialize::Name()); + + sink.Add(TRoaringCardinality::Name()); + + sink.Add(TRoaringUint32List::Name())->SetTypeAwareness(); + + sink.Add(TRoaringOrWithBinary::Name()); + sink.Add(TRoaringOr::Name()); + + sink.Add(TRoaringAndWithBinary::Name()); + sink.Add(TRoaringAnd::Name()); + } + + void CleanupOnTerminate() const final { + } + + void BuildFunctionTypeInfo(const TStringRef& name, NUdf::TType* userType, + const TStringRef& typeConfig, ui32 flags, + IFunctionTypeInfoBuilder& builder) const final { + try { + Y_UNUSED(typeConfig); + Y_UNUSED(userType); + + auto typesOnly = (flags & TFlags::TypesOnly); + + if (TRoaringDeserialize::Name() == name) { + builder.Returns>>().Args()->Add>(); + + if (!typesOnly) { + builder.Implementation(new TRoaringDeserialize()); + } + } else if (TRoaringSerialize::Name() == name) { + builder.Returns(builder.SimpleType()) + .Args() + ->Add>>(); + + if (!typesOnly) { + builder.Implementation(new TRoaringSerialize()); + } + } else if (TRoaringCardinality::Name() == name) { + builder.Returns(builder.SimpleType()) + .Args() + ->Add>>(); + + if (!typesOnly) { + builder.Implementation(new TRoaringCardinality()); + } + } else if (TRoaringUint32List::Name() == name) { + auto ui32ListType = + builder.List()->Item(builder.SimpleType()).Build(); + builder.Returns(builder.Optional()->Item(ui32ListType).Build()) + .Args() + ->Add>>(); + + if (!typesOnly) { + builder.Implementation(new TRoaringUint32List()); + } + } else if (TRoaringOrWithBinary::Name() == name) { + builder.Returns>>() + .Args() + ->Add>>() + .Add>(); + + if (!typesOnly) { + builder.Implementation(new TRoaringOrWithBinary()); + } + } else if (TRoaringOr::Name() == name) { + builder.Returns>>() + .Args() + ->Add>>() + .Add>>(); + + if (!typesOnly) { + builder.Implementation(new TRoaringOr()); + } + } else if (TRoaringAndWithBinary::Name() == name) { + builder.Returns>>() + .Args() + ->Add>>() + .Add>(); + + if (!typesOnly) { + builder.Implementation(new TRoaringAndWithBinary()); + } + } else if (TRoaringAnd::Name() == name) { + builder.Returns>>() + .Args() + ->Add>>() + .Add>>(); + + if (!typesOnly) { + builder.Implementation(new TRoaringAnd()); + } + } else { + TStringBuilder sb; + sb << "Unknown function: " << name.Data(); + builder.SetError(sb); + } + } catch (const std::exception& e) { + builder.SetError(CurrentExceptionMessage()); + } + } + + private: + inline static const char RoaringResourceName[] = "roaring_bitmap"; + + static void* RoaringMallocUdf(size_t size) { + auto allocationSize = size + 2 * sizeof(void*); + auto allocatedMemPointer = UdfAllocateWithSize(allocationSize); + + auto roaringMemPointer = ((char*)allocatedMemPointer) + 2 * sizeof(void*); + + ((void**)roaringMemPointer)[-1] = allocatedMemPointer; + ((void**)roaringMemPointer)[-2] = ((char*)allocatedMemPointer) + allocationSize; + return roaringMemPointer; + } + + static void* RoaringReallocUdf(void* oldPointer, size_t newSize) { + if (oldPointer == nullptr) { + return RoaringMallocUdf(newSize); + } + + if (oldPointer != nullptr && newSize == 0) { + RoaringFreeUdf(oldPointer); + return nullptr; + } + + auto reallocatedPointer = RoaringMallocUdf(newSize); + auto oldAllocatedMemPointer = (char*)((void**)oldPointer)[-1]; + auto oldSizePointer = (char*)((void**)oldPointer)[-2]; + memcpy(reallocatedPointer, oldPointer, oldSizePointer - oldAllocatedMemPointer); + RoaringFreeUdf(oldPointer); + + return reallocatedPointer; + } + + static void* RoaringCallocUdf(size_t elements, size_t elementSize) { + auto newMem = RoaringMallocUdf(elements * elementSize); + memset(newMem, 0, elements * elementSize); + return newMem; + } + + static void RoaringFreeUdf(void* pointer) { + if (pointer == nullptr) { + return; + } + auto allocatedMemPointer = (char*)((void**)pointer)[-1]; + auto sizePointer = (char*)((void**)pointer)[-2]; + UdfFreeWithSize(allocatedMemPointer, sizePointer - allocatedMemPointer); + } + + static void* RoaringAlignedMallocUdf(size_t alignment, size_t size) { + auto allocationSize = size + (alignment - 1) + 2 * sizeof(void*); + auto allocatedMemPointer = UdfAllocateWithSize(allocationSize); + + auto roaringMemPointer = ((char*)allocatedMemPointer) + 2 * sizeof(void*); + if ((size_t)roaringMemPointer & (alignment - 1)) { + roaringMemPointer += alignment - ((size_t)roaringMemPointer & (alignment - 1)); + } + + ((void**)roaringMemPointer)[-1] = allocatedMemPointer; + ((void**)roaringMemPointer)[-2] = ((char*)allocatedMemPointer) + allocationSize; + return roaringMemPointer; + } + }; + +} // namespace + +REGISTER_MODULES(TRoaringModule) diff --git a/ydb/library/yql/udfs/common/roaring/test/canondata/result.json b/ydb/library/yql/udfs/common/roaring/test/canondata/result.json new file mode 100644 index 000000000000..8c592fb352ad --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/test/canondata/result.json @@ -0,0 +1,22 @@ +{ + "test.test[cardinality]": [ + { + "uri": "file://test.test_cardinality_/results.txt" + } + ], + "test.test[intersect]": [ + { + "uri": "file://test.test_intersect_/results.txt" + } + ], + "test.test[serialize_deserialize]": [ + { + "uri": "file://test.test_serialize_deserialize_/results.txt" + } + ], + "test.test[union]": [ + { + "uri": "file://test.test_union_/results.txt" + } + ] +} diff --git a/ydb/library/yql/udfs/common/roaring/test/canondata/test.test_cardinality_/results.txt b/ydb/library/yql/udfs/common/roaring/test/canondata/test.test_cardinality_/results.txt new file mode 100644 index 000000000000..315e3b881a91 --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/test/canondata/test.test_cardinality_/results.txt @@ -0,0 +1,69 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "OrCardinality"; + [ + "OptionalType"; + [ + "DataType"; + "Uint32" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "3" + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "AndCardinality"; + [ + "OptionalType"; + [ + "ListType"; + [ + "DataType"; + "Uint32" + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + [ + "1" + ] + ] + ] + ] + } + ] + } +] \ No newline at end of file diff --git a/ydb/library/yql/udfs/common/roaring/test/canondata/test.test_intersect_/results.txt b/ydb/library/yql/udfs/common/roaring/test/canondata/test.test_intersect_/results.txt new file mode 100644 index 000000000000..326216669869 --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/test/canondata/test.test_intersect_/results.txt @@ -0,0 +1,106 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "AndList"; + [ + "OptionalType"; + [ + "ListType"; + [ + "DataType"; + "Uint32" + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + [ + "1" + ] + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "AndWithBinaryList"; + [ + "OptionalType"; + [ + "ListType"; + [ + "DataType"; + "Uint32" + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + [ + "1" + ] + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "AndWithBinaryListEmpty"; + [ + "OptionalType"; + [ + "ListType"; + [ + "DataType"; + "Uint32" + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + # + ] + ] + } + ] + } +] \ No newline at end of file diff --git a/ydb/library/yql/udfs/common/roaring/test/canondata/test.test_serialize_deserialize_/results.txt b/ydb/library/yql/udfs/common/roaring/test/canondata/test.test_serialize_deserialize_/results.txt new file mode 100644 index 000000000000..05f944c62233 --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/test/canondata/test.test_serialize_deserialize_/results.txt @@ -0,0 +1,176 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "DeserializedList"; + [ + "OptionalType"; + [ + "ListType"; + [ + "DataType"; + "Uint32" + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + [ + "10"; + "567" + ] + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "Serialized"; + [ + "OptionalType"; + [ + "DataType"; + "String" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + ":0\0\0\1\0\0\0\0\0\1\0\x10\0\0\0\n\0007\2" + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "LimitedList"; + [ + "OptionalType"; + [ + "ListType"; + [ + "DataType"; + "Uint32" + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + [ + "10" + ] + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "OffsetedList"; + [ + "OptionalType"; + [ + "ListType"; + [ + "DataType"; + "Uint32" + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + [ + "567" + ] + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "EmptyList"; + [ + "OptionalType"; + [ + "ListType"; + [ + "DataType"; + "Uint32" + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + [] + ] + ] + ] + } + ] + } +] \ No newline at end of file diff --git a/ydb/library/yql/udfs/common/roaring/test/canondata/test.test_union_/results.txt b/ydb/library/yql/udfs/common/roaring/test/canondata/test.test_union_/results.txt new file mode 100644 index 000000000000..208f1ff2c42e --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/test/canondata/test.test_union_/results.txt @@ -0,0 +1,78 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "OrList"; + [ + "OptionalType"; + [ + "ListType"; + [ + "DataType"; + "Uint32" + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + [ + "1"; + "2"; + "3" + ] + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "OrWithBinaryList"; + [ + "OptionalType"; + [ + "ListType"; + [ + "DataType"; + "Uint32" + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + [ + "1"; + "2"; + "3" + ] + ] + ] + ] + } + ] + } +] \ No newline at end of file diff --git a/ydb/library/yql/udfs/common/roaring/test/cases/cardinality.in b/ydb/library/yql/udfs/common/roaring/test/cases/cardinality.in new file mode 100644 index 000000000000..12431657f3b8 --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/test/cases/cardinality.in @@ -0,0 +1 @@ +{"left"=":0\x00\x00\x01\x00\x00\x00\x00\x00\x01\x00\x10\x00\x00\x00\x01\x00\x02\x00";"right"=":0\x00\x00\x01\x00\x00\x00\x00\x00\x01\x00\x10\x00\x00\x00\x01\x00\x03\x00";}; diff --git a/ydb/library/yql/udfs/common/roaring/test/cases/cardinality.in.attr b/ydb/library/yql/udfs/common/roaring/test/cases/cardinality.in.attr new file mode 100644 index 000000000000..6bfd01839a56 --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/test/cases/cardinality.in.attr @@ -0,0 +1 @@ +{schema=[{name=left;type=string};{name=right;type=string}]} diff --git a/ydb/library/yql/udfs/common/roaring/test/cases/cardinality.sql b/ydb/library/yql/udfs/common/roaring/test/cases/cardinality.sql new file mode 100644 index 000000000000..729fc50bd383 --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/test/cases/cardinality.sql @@ -0,0 +1,2 @@ +SELECT Roaring::Cardinality(Roaring::OrWithBinary(Roaring::Deserialize(left), right)) AS OrCardinality FROM Input; +SELECT Roaring::Uint32List(Roaring::AndWithBinary(Roaring::Deserialize(right), left)) AS AndCardinality FROM Input; diff --git a/ydb/library/yql/udfs/common/roaring/test/cases/intersect.in b/ydb/library/yql/udfs/common/roaring/test/cases/intersect.in new file mode 100644 index 000000000000..12431657f3b8 --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/test/cases/intersect.in @@ -0,0 +1 @@ +{"left"=":0\x00\x00\x01\x00\x00\x00\x00\x00\x01\x00\x10\x00\x00\x00\x01\x00\x02\x00";"right"=":0\x00\x00\x01\x00\x00\x00\x00\x00\x01\x00\x10\x00\x00\x00\x01\x00\x03\x00";}; diff --git a/ydb/library/yql/udfs/common/roaring/test/cases/intersect.in.attr b/ydb/library/yql/udfs/common/roaring/test/cases/intersect.in.attr new file mode 100644 index 000000000000..6bfd01839a56 --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/test/cases/intersect.in.attr @@ -0,0 +1 @@ +{schema=[{name=left;type=string};{name=right;type=string}]} diff --git a/ydb/library/yql/udfs/common/roaring/test/cases/intersect.sql b/ydb/library/yql/udfs/common/roaring/test/cases/intersect.sql new file mode 100644 index 000000000000..60c968946ead --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/test/cases/intersect.sql @@ -0,0 +1,3 @@ +SELECT Roaring::Uint32List(Roaring::And(Roaring::Deserialize(left), Roaring::Deserialize(right))) AS AndList FROM Input; +SELECT Roaring::Uint32List(Roaring::AndWithBinary(Roaring::Deserialize(right), left)) AS AndWithBinaryList FROM Input; +SELECT Roaring::Uint32List(Roaring::AndWithBinary(Roaring::Deserialize(right), NULL)) AS AndWithBinaryListEmpty FROM Input; diff --git a/ydb/library/yql/udfs/common/roaring/test/cases/serialize_deserialize.in b/ydb/library/yql/udfs/common/roaring/test/cases/serialize_deserialize.in new file mode 100644 index 000000000000..0023cbae96fe --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/test/cases/serialize_deserialize.in @@ -0,0 +1 @@ +{"binaryString"=":0\x00\x00\x01\x00\x00\x00\x00\x00\x01\x00\x10\x00\x00\x00\n\x007\x02";}; diff --git a/ydb/library/yql/udfs/common/roaring/test/cases/serialize_deserialize.in.attr b/ydb/library/yql/udfs/common/roaring/test/cases/serialize_deserialize.in.attr new file mode 100644 index 000000000000..158676c2c5b3 --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/test/cases/serialize_deserialize.in.attr @@ -0,0 +1 @@ +{schema=[{name=binaryString;type=string}]} \ No newline at end of file diff --git a/ydb/library/yql/udfs/common/roaring/test/cases/serialize_deserialize.sql b/ydb/library/yql/udfs/common/roaring/test/cases/serialize_deserialize.sql new file mode 100644 index 000000000000..9735432181c9 --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/test/cases/serialize_deserialize.sql @@ -0,0 +1,15 @@ +/* syntax version 1 */ +SELECT Roaring::Uint32List(Roaring::Deserialize(binaryString)) AS DeserializedList +FROM Input; + +SELECT Roaring::Serialize(Roaring::Deserialize(binaryString)) AS Serialized +FROM Input; + +SELECT ListTake(Roaring::Uint32List(Roaring::Deserialize(binaryString)), 1) AS LimitedList +FROM Input; + +SELECT ListTake(ListSkip(Roaring::Uint32List(Roaring::Deserialize(binaryString)), 1), 1) AS OffsetedList +FROM Input; + +SELECT ListTake(ListSkip(Roaring::Uint32List(Roaring::Deserialize(binaryString)), 10), 1) AS EmptyList +FROM Input; diff --git a/ydb/library/yql/udfs/common/roaring/test/cases/union.in b/ydb/library/yql/udfs/common/roaring/test/cases/union.in new file mode 100644 index 000000000000..12431657f3b8 --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/test/cases/union.in @@ -0,0 +1 @@ +{"left"=":0\x00\x00\x01\x00\x00\x00\x00\x00\x01\x00\x10\x00\x00\x00\x01\x00\x02\x00";"right"=":0\x00\x00\x01\x00\x00\x00\x00\x00\x01\x00\x10\x00\x00\x00\x01\x00\x03\x00";}; diff --git a/ydb/library/yql/udfs/common/roaring/test/cases/union.in.attr b/ydb/library/yql/udfs/common/roaring/test/cases/union.in.attr new file mode 100644 index 000000000000..6bfd01839a56 --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/test/cases/union.in.attr @@ -0,0 +1 @@ +{schema=[{name=left;type=string};{name=right;type=string}]} diff --git a/ydb/library/yql/udfs/common/roaring/test/cases/union.sql b/ydb/library/yql/udfs/common/roaring/test/cases/union.sql new file mode 100644 index 000000000000..54d44054b03e --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/test/cases/union.sql @@ -0,0 +1,3 @@ +SELECT Roaring::Uint32List(Roaring::Or(Roaring::Deserialize(left), Roaring::Deserialize(right))) AS OrList FROM Input; +SELECT Roaring::Uint32List(Roaring::OrWithBinary(Roaring::Deserialize(right), left)) AS OrWithBinaryList FROM Input; + diff --git a/ydb/library/yql/udfs/common/roaring/test/ya.make b/ydb/library/yql/udfs/common/roaring/test/ya.make new file mode 100644 index 000000000000..157190e5b3ad --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/test/ya.make @@ -0,0 +1,12 @@ +YQL_UDF_YDB_TEST() + +DEPENDS(ydb/library/yql/udfs/common/roaring) + +TIMEOUT(300) +SIZE(MEDIUM) + +IF (SANITIZER_TYPE == "memory") + TAG(ya:not_autocheck) # YQL-15385 +ENDIF() + +END() diff --git a/ydb/library/yql/udfs/common/roaring/ya.make b/ydb/library/yql/udfs/common/roaring/ya.make new file mode 100644 index 000000000000..841ba775520d --- /dev/null +++ b/ydb/library/yql/udfs/common/roaring/ya.make @@ -0,0 +1,22 @@ +YQL_UDF_YDB(roaring) + +YQL_ABI_VERSION( + 2 + 35 + 0 +) + +SRCS( + roaring.cpp +) + +PEERDIR( + contrib/libs/croaring +) + + +END() + +RECURSE_FOR_TESTS( + test +) \ No newline at end of file diff --git a/ydb/library/yql/udfs/common/ya.make b/ydb/library/yql/udfs/common/ya.make index 808b51a0735e..39ca89f8c5a9 100644 --- a/ydb/library/yql/udfs/common/ya.make +++ b/ydb/library/yql/udfs/common/ya.make @@ -15,6 +15,7 @@ RECURSE( pire protobuf re2 + roaring set stat streaming