diff --git a/folly/Memory.h b/folly/Memory.h index b96342e3025..835541ea5dd 100644 --- a/folly/Memory.h +++ b/folly/Memory.h @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -412,8 +413,9 @@ class SysAllocator { } return static_cast(p); } - void deallocate(T* p, size_t /* count */) { - std::free(p); + void deallocate(T* p, size_t count) { + using lifted = typename detail::lift_void_to_char::type; + sizedFree(p, count * sizeof(lifted)); } friend bool operator==(Self const&, Self const&) noexcept { diff --git a/folly/memory/Arena-inl.h b/folly/memory/Arena-inl.h index ad507b66061..c1cf991817c 100644 --- a/folly/memory/Arena-inl.h +++ b/folly/memory/Arena-inl.h @@ -22,27 +22,8 @@ namespace folly { -template -std::pair::Block*, size_t> -Arena::Block::allocate(Alloc& alloc, size_t size, bool allowSlack) { - size_t allocSize = sizeof(Block) + size; - if (allowSlack) { - allocSize = ArenaAllocatorTraits::goodSize(alloc, allocSize); - } - - void* mem = std::allocator_traits::allocate(alloc, allocSize); - return std::make_pair(new (mem) Block(), allocSize - sizeof(Block)); -} - -template -void Arena::Block::deallocate(Alloc& alloc) { - this->~Block(); - std::allocator_traits::deallocate(alloc, this, 1); -} - template void* Arena::allocateSlow(size_t size) { - std::pair p; char* start; size_t allocSize; @@ -55,31 +36,41 @@ void* Arena::allocateSlow(size_t size) { } if (size > minBlockSize()) { - // Allocate a large block for this chunk only, put it at the back of the - // list so it doesn't get used for small allocations; don't change ptr_ - // and end_, let them point into a normal block (or none, if they're - // null) - p = Block::allocate(alloc(), size, false); - start = p.first->start(); - blocks_.push_back(*p.first); + // Allocate a large block for this chunk only; don't change ptr_ and end_, + // let them point into a normal block (or none, if they're null) + allocSize = sizeof(LargeBlock) + size; + void* mem = AllocTraits::allocate(alloc(), allocSize); + auto blk = new (mem) LargeBlock(allocSize); + start = blk->start(); + largeBlocks_.push_front(*blk); } else { // Allocate a normal sized block and carve out size bytes from it - p = Block::allocate(alloc(), minBlockSize(), true); - start = p.first->start(); - blocks_.push_front(*p.first); + // Will allocate more than size bytes if convenient + // (via ArenaAllocatorTraits::goodSize()) as we'll try to pack small + // allocations in this block. + allocSize = blockGoodAllocSize(); + void* mem = AllocTraits::allocate(alloc(), allocSize); + auto blk = new (mem) Block(); + start = blk->start(); + blocks_.push_front(*blk); ptr_ = start + size; - end_ = start + p.second; + end_ = start + allocSize - sizeof(Block); + assert(ptr_ <= end_); } - assert(p.second >= size); - totalAllocatedSize_ += p.second + sizeof(Block); + totalAllocatedSize_ += allocSize; return start; } template void Arena::merge(Arena&& other) { + FOLLY_SAFE_CHECK( + blockGoodAllocSize() == other.blockGoodAllocSize(), + "cannot merge arenas of different minBlockSize"); blocks_.splice_after(blocks_.before_begin(), other.blocks_); other.blocks_.clear(); + largeBlocks_.splice_after(largeBlocks_.before_begin(), other.largeBlocks_); + other.largeBlocks_.clear(); other.ptr_ = other.end_ = nullptr; totalAllocatedSize_ += other.totalAllocatedSize_; other.totalAllocatedSize_ = 0; @@ -87,7 +78,15 @@ void Arena::merge(Arena&& other) { template Arena::~Arena() { - blocks_.clear_and_dispose([this](Block* b) { b->deallocate(this->alloc()); }); + blocks_.clear_and_dispose([this](Block* b) { + b->~Block(); + AllocTraits::deallocate(alloc(), b, blockGoodAllocSize()); + }); + largeBlocks_.clear_and_dispose([this](LargeBlock* b) { + auto size = b->allocSize; + b->~LargeBlock(); + AllocTraits::deallocate(alloc(), b, size); + }); } } // namespace folly diff --git a/folly/memory/Arena.h b/folly/memory/Arena.h index 1eb830a391f..24d0a38af38 100644 --- a/folly/memory/Arena.h +++ b/folly/memory/Arena.h @@ -126,30 +126,37 @@ class Arena { Arena& operator=(Arena&&) = delete; private: - struct Block; - typedef boost::intrusive::slist_member_hook> - BlockLink; + using AllocTraits = std::allocator_traits; + using BlockLink = boost::intrusive::slist_member_hook<>; struct alignas(max_align_v) Block { BlockLink link; - // Allocate a block with at least size bytes of storage. - // If allowSlack is true, allocate more than size bytes if convenient - // (via ArenaAllocatorTraits::goodSize()) as we'll try to pack small - // allocations in this block. - static std::pair - allocate(Alloc& alloc, size_t size, bool allowSlack); - void deallocate(Alloc& alloc); - char* start() { return reinterpret_cast(this + 1); } - private: Block() = default; ~Block() = default; }; + constexpr size_t blockGoodAllocSize() { + return ArenaAllocatorTraits::goodSize( + alloc(), sizeof(Block) + minBlockSize()); + } + + struct alignas(max_align_v) LargeBlock { + BlockLink link; + const size_t allocSize; + + char* start() { + return reinterpret_cast(this + 1); + } + + LargeBlock(size_t s) : allocSize(s) {} + ~LargeBlock() = default; + }; + public: static constexpr size_t kDefaultMinBlockSize = 4096 - sizeof(Block); static constexpr size_t kNoSizeLimit = 0; @@ -183,6 +190,13 @@ class Arena { boost::intrusive::cache_last> BlockList; + typedef boost::intrusive::slist< + LargeBlock, + boost::intrusive::member_hook, + boost::intrusive::constant_time_size, + boost::intrusive::cache_last> + LargeBlockList; + void* allocateSlow(size_t size); // Empty member optimization: package Alloc with a non-empty member @@ -206,6 +220,7 @@ class Arena { AllocAndSize allocAndSize_; BlockList blocks_; + LargeBlockList largeBlocks_; char* ptr_; char* end_; size_t totalAllocatedSize_; diff --git a/folly/memory/ThreadCachedArena.cpp b/folly/memory/ThreadCachedArena.cpp index ecf14dd6462..0d9c571fe95 100644 --- a/folly/memory/ThreadCachedArena.cpp +++ b/folly/memory/ThreadCachedArena.cpp @@ -21,7 +21,9 @@ namespace folly { ThreadCachedArena::ThreadCachedArena(size_t minBlockSize, size_t maxAlign) - : minBlockSize_(minBlockSize), maxAlign_(maxAlign) {} + : minBlockSize_(minBlockSize), + maxAlign_(maxAlign), + zombies_(folly::in_place, minBlockSize) {} SysArena* ThreadCachedArena::allocateThreadLocalArena() { auto arena = new SysArena(minBlockSize_, SysArena::kNoSizeLimit, maxAlign_);