/* * Copyright (C) 2015 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef TNT_UTILS_ALLOCATOR_H #define TNT_UTILS_ALLOCATOR_H #include #include #include #include #include #include #include #include #include #include namespace utils { namespace pointermath { template static inline P* add(P* a, T b) noexcept { return (P*)(uintptr_t(a) + uintptr_t(b)); } template static inline P* align(P* p, size_t alignment) noexcept { // alignment must be a power-of-two assert(alignment && !(alignment & alignment-1)); return (P*)((uintptr_t(p) + alignment - 1) & ~(alignment - 1)); } template static inline P* align(P* p, size_t alignment, size_t offset) noexcept { P* const r = align(add(p, offset), alignment); assert(pointermath::add(r, -offset) >= p); return r; } } /* ------------------------------------------------------------------------------------------------ * LinearAllocator * * + Allocates blocks linearly * + Cannot free individual blocks * + Can free top of memory back up to a specified point * + Doesn't call destructors * ------------------------------------------------------------------------------------------------ */ class LinearAllocator { public: // use memory area provided LinearAllocator(void* begin, void* end) noexcept; template explicit LinearAllocator(const AREA& area) : LinearAllocator(area.begin(), area.end()) { } // Allocators can't be copied LinearAllocator(const LinearAllocator& rhs) = delete; LinearAllocator& operator=(const LinearAllocator& rhs) = delete; // Allocators can be moved LinearAllocator(LinearAllocator&& rhs) noexcept; LinearAllocator& operator=(LinearAllocator&& rhs) noexcept; ~LinearAllocator() noexcept = default; // our allocator concept void* alloc(size_t size, size_t alignment = alignof(std::max_align_t), size_t extra = 0) UTILS_RESTRICT { // branch-less allocation void* const p = pointermath::align(current(), alignment, extra); void* const c = pointermath::add(p, size); bool success = c <= end(); set_current(success ? c : current()); return success ? p : nullptr; } // API specific to this allocator void *getCurrent() UTILS_RESTRICT noexcept { return current(); } // free memory back to the specified point void rewind(void* p) UTILS_RESTRICT noexcept { assert(p>=mBegin && p explicit HeapAllocator(const AREA&) { } // our allocator concept void* alloc(size_t size, size_t alignment = alignof(std::max_align_t), size_t extra = 0) { // this allocator doesn't support 'extra' assert(extra == 0); return aligned_alloc(size, alignment); } void free(void* p) noexcept { aligned_free(p); } void free(void* p, size_t) noexcept { free(p); } // Allocators can't be copied HeapAllocator(const HeapAllocator& rhs) = delete; HeapAllocator& operator=(const HeapAllocator& rhs) = delete; // Allocators can be moved HeapAllocator(HeapAllocator&& rhs) noexcept = default; HeapAllocator& operator=(HeapAllocator&& rhs) noexcept = default; ~HeapAllocator() noexcept = default; void swap(HeapAllocator& rhs) noexcept { } }; // ------------------------------------------------------------------------------------------------ class FreeList { public: FreeList() noexcept = default; FreeList(void* begin, void* end, size_t elementSize, size_t alignment, size_t extra) noexcept; FreeList(const FreeList& rhs) = delete; FreeList& operator=(const FreeList& rhs) = delete; FreeList(FreeList&& rhs) noexcept = default; FreeList& operator=(FreeList&& rhs) noexcept = default; void* pop() noexcept { Node* const head = mHead; mHead = head ? head->next : nullptr; // this could indicate a use after free assert(!mHead || mHead >= mBegin && mHead < mEnd); return head; } void push(void* p) noexcept { assert(p); assert(p >= mBegin && p < mEnd); // TODO: assert this is one of our pointer (i.e.: it's address match one of ours) Node* const head = static_cast(p); head->next = mHead; mHead = head; } void *getFirst() noexcept { return mHead; } private: struct Node { Node* next; }; static Node* init(void* begin, void* end, size_t elementSize, size_t alignment, size_t extra) noexcept; Node* mHead = nullptr; #ifndef NDEBUG // These are needed only for debugging... void* mBegin = nullptr; void* mEnd = nullptr; #endif }; class AtomicFreeList { public: AtomicFreeList() noexcept = default; AtomicFreeList(void* begin, void* end, size_t elementSize, size_t alignment, size_t extra) noexcept; AtomicFreeList(const FreeList& rhs) = delete; AtomicFreeList& operator=(const FreeList& rhs) = delete; void* pop() noexcept { Node* const storage = mStorage; HeadPtr currentHead = mHead.load(); while (currentHead.offset >= 0) { // The value of "next" we load here might already contain application data if another // thread raced ahead of us. But in that case, the computed "newHead" will be discarded // since compare_exchange_weak fails. Then this thread will loop with the updated // value of currentHead, and try again. Node* const next = storage[currentHead.offset].next.load(std::memory_order_relaxed); const HeadPtr newHead{ next ? int32_t(next - storage) : -1, currentHead.tag + 1 }; // In the rare case that the other thread that raced ahead of us already returned the // same mHead we just loaded, but it now has a different "next" value, the tag field will not // match, and compare_exchange_weak will fail and prevent that particular race condition. if (mHead.compare_exchange_weak(currentHead, newHead)) { // This assert needs to occur after we have validated that there was no race condition // Otherwise, next might already contain application data, if another thread // raced ahead of us after we loaded mHead, but before we loaded mHead->next. assert(!next || next >= storage); break; } } void* p = (currentHead.offset >= 0) ? (storage + currentHead.offset) : nullptr; assert(!p || p >= storage); return p; } void push(void* p) noexcept { Node* const storage = mStorage; assert(p && p >= storage); Node* const node = static_cast(p); HeadPtr currentHead = mHead.load(); HeadPtr newHead = { int32_t(node - storage), currentHead.tag + 1 }; do { newHead.tag = currentHead.tag + 1; Node* const n = (currentHead.offset >= 0) ? (storage + currentHead.offset) : nullptr; node->next.store(n, std::memory_order_relaxed); } while(!mHead.compare_exchange_weak(currentHead, newHead)); } void* getFirst() noexcept { return mStorage + mHead.load(std::memory_order_relaxed).offset; } private: struct Node { // This should be a regular (non-atomic) pointer, but this causes TSAN to complain // about a data-race that exists but is benin. We always use this atomic<> in // relaxed mode. // The data race TSAN complains about is when a pop() is interrupted by a // pop() + push() just after mHead->next is read -- it appears as though it is written // without synchronization (by the push), however in that case, the pop's CAS will fail // and things will auto-correct. // // Pop() | // | | // read head->next | // | pop() // | | // | read head->next // | CAS, tag++ // | | // | push() // | | // [TSAN: data-race here] write head->next // | CAS, tag++ // CAS fails // | // read head->next // | // CAS, tag++ // std::atomic next; }; // This struct is using a 32-bit offset into the arena rather than // a direct pointer, because together with the 32-bit tag, it needs to // fit into 8 bytes. If it was any larger, it would not be possible to // access it atomically. struct alignas(8) HeadPtr { int32_t offset; uint32_t tag; }; std::atomic mHead{}; Node* mStorage = nullptr; }; // ------------------------------------------------------------------------------------------------ template < size_t ELEMENT_SIZE, size_t ALIGNMENT = alignof(std::max_align_t), size_t OFFSET = 0, typename FREELIST = FreeList> class PoolAllocator { static_assert(ELEMENT_SIZE >= sizeof(void*), "ELEMENT_SIZE must accommodate at least a pointer"); public: // our allocator concept void* alloc(size_t size = ELEMENT_SIZE, size_t alignment = ALIGNMENT, size_t offset = OFFSET) noexcept { assert(size <= ELEMENT_SIZE); assert(alignment <= ALIGNMENT); assert(offset == OFFSET); return mFreeList.pop(); } void free(void* p, size_t = ELEMENT_SIZE) noexcept { mFreeList.push(p); } size_t getSize() const noexcept { return ELEMENT_SIZE; } PoolAllocator(void* begin, void* end) noexcept : mFreeList(begin, end, ELEMENT_SIZE, ALIGNMENT, OFFSET) { } template explicit PoolAllocator(const AREA& area) noexcept : PoolAllocator(area.begin(), area.end()) { } // Allocators can't be copied PoolAllocator(const PoolAllocator& rhs) = delete; PoolAllocator& operator=(const PoolAllocator& rhs) = delete; PoolAllocator() noexcept = default; ~PoolAllocator() noexcept = default; // API specific to this allocator void *getCurrent() noexcept { return mFreeList.getFirst(); } private: FREELIST mFreeList; }; #define UTILS_MAX(a,b) ((a) > (b) ? (a) : (b)) template using ObjectPoolAllocator = PoolAllocator; template using ThreadSafeObjectPoolAllocator = PoolAllocator; // ------------------------------------------------------------------------------------------------ // Areas // ------------------------------------------------------------------------------------------------ class HeapArea { public: HeapArea() noexcept = default; explicit HeapArea(size_t size) { if (size) { // TODO: policy committing memory mBegin = malloc(size); mEnd = pointermath::add(mBegin, size); } } ~HeapArea() noexcept { // TODO: policy for returning memory to system free(mBegin); } HeapArea(const HeapArea& rhs) = delete; HeapArea& operator=(const HeapArea& rhs) = delete; HeapArea(HeapArea&& rhs) noexcept = delete; HeapArea& operator=(HeapArea&& rhs) noexcept = delete; void* data() const noexcept { return mBegin; } void* begin() const noexcept { return mBegin; } void* end() const noexcept { return mEnd; } size_t getSize() const noexcept { return uintptr_t(mEnd) - uintptr_t(mBegin); } private: void* mBegin = nullptr; void* mEnd = nullptr; }; // ------------------------------------------------------------------------------------------------ // Policies // ------------------------------------------------------------------------------------------------ namespace LockingPolicy { struct NoLock { void lock() noexcept { } void unlock() noexcept { } }; #if defined(__SANITIZE_THREAD__) // Unfortunately TSAN doesn't support homegrown synchronization primitives using SpinLock = utils::Mutex; #elif defined(__ARM_ARCH_7A__) // We've had problems with "wfe" on some ARM-V7 devices, causing spurious SIGILL using SpinLock = utils::Mutex; #else using SpinLock = utils::SpinLock; #endif using Mutex = utils::Mutex; } // namespace LockingPolicy namespace TrackingPolicy { // default no-op tracker struct Untracked { Untracked() noexcept = default; Untracked(const char* name, void* base, size_t size) noexcept { } void onAlloc(void* p, size_t size, size_t alignment, size_t extra) noexcept { } void onFree(void* p, size_t = 0) noexcept { } void onReset() noexcept { } void onRewind(void* addr) noexcept { } }; // This just track the max memory usage and logs it in the destructor struct HighWatermark { HighWatermark() noexcept = default; HighWatermark(const char* name, void* base, size_t size) noexcept : mName(name), mBase(base), mSize(uint32_t(size)) { } ~HighWatermark() noexcept; void onAlloc(void* p, size_t size, size_t alignment, size_t extra) noexcept; void onFree(void* p, size_t size) noexcept; void onReset() noexcept; void onRewind(void const* addr) noexcept; protected: const char* mName = nullptr; void* mBase = nullptr; uint32_t mSize = 0; uint32_t mCurrent = 0; uint32_t mHighWaterMark = 0; }; // This just fills buffers with known values to help catch uninitialized access and use after free. struct Debug { Debug() noexcept = default; Debug(const char* name, void* base, size_t size) noexcept : mName(name), mBase(base), mSize(uint32_t(size)) { } void onAlloc(void* p, size_t size, size_t alignment, size_t extra) noexcept; void onFree(void* p, size_t size) noexcept; void onReset() noexcept; void onRewind(void* addr) noexcept; protected: const char* mName = nullptr; void* mBase = nullptr; uint32_t mSize = 0; }; struct DebugAndHighWatermark : protected HighWatermark, protected Debug { DebugAndHighWatermark() noexcept = default; DebugAndHighWatermark(const char* name, void* base, size_t size) noexcept : HighWatermark(name, base, size), Debug(name, base, size) { } void onAlloc(void* p, size_t size, size_t alignment, size_t extra) noexcept { HighWatermark::onAlloc(p, size, alignment, extra); Debug::onAlloc(p, size, alignment, extra); } void onFree(void* p, size_t size) noexcept { HighWatermark::onFree(p, size); Debug::onFree(p, size); } void onReset() noexcept { HighWatermark::onReset(); Debug::onReset(); } void onRewind(void* addr) noexcept { HighWatermark::onRewind(addr); Debug::onRewind(addr); } }; } // namespace TrackingPolicy // ------------------------------------------------------------------------------------------------ // Arenas // ------------------------------------------------------------------------------------------------ template class Arena { public: Arena() = default; // construct an arena with a name and forward argument to its allocator template Arena(const char* name, size_t size, ARGS&& ... args) : mArea(size), mAllocator(mArea, std::forward(args) ... ), mListener(name, mArea.data(), size), mArenaName(name) { } // allocate memory from arena with given size and alignment // (acceptable size/alignment may depend on the allocator provided) void* alloc(size_t size, size_t alignment = alignof(std::max_align_t), size_t extra = 0) noexcept { std::lock_guard guard(mLock); void* p = mAllocator.alloc(size, alignment, extra); mListener.onAlloc(p, size, alignment, extra); return p; } // Allocate an array of trivially destructible objects // for safety, we disable the object-based alloc method if the object type is not // trivially destructible, since free() won't call the destructor and this is allocating // an array. template ::value>::type> T* alloc(size_t count, size_t alignment = alignof(T), size_t extra = 0) noexcept { return (T*)alloc(count * sizeof(T), alignment, extra); } // return memory pointed by p to the arena // (actual behaviour may depend on allocator provided) void free(void* p) noexcept { if (p) { std::lock_guard guard(mLock); mListener.onFree(p); mAllocator.free(p); } } // some allocators require the size of the allocation for free void free(void* p, size_t size) noexcept { if (p) { std::lock_guard guard(mLock); mListener.onFree(p, size); mAllocator.free(p, size); } } // some allocators don't have a free() call, but a single reset() or rewind() instead void reset() noexcept { std::lock_guard guard(mLock); mListener.onReset(); mAllocator.reset(); } void* getCurrent() noexcept { return mAllocator.getCurrent(); } void rewind(void *addr) noexcept { std::lock_guard guard(mLock); mListener.onRewind(addr); mAllocator.rewind(addr); } // Allocate and construct an object template T* make(ARGS&& ... args) noexcept { void* const p = this->alloc(sizeof(T), ALIGN); return p ? new(p) T(std::forward(args)...) : nullptr; } // destroys an object created with make() above, and frees associated memory template void destroy(T* p) noexcept { if (p) { p->~T(); this->free((void*)p, sizeof(T)); } } char const* getName() const noexcept { return mArenaName; } AllocatorPolicy& getAllocator() noexcept { return mAllocator; } AllocatorPolicy const& getAllocator() const noexcept { return mAllocator; } TrackingPolicy& getListener() noexcept { return mListener; } TrackingPolicy const& getListener() const noexcept { return mListener; } HeapArea& getArea() noexcept { return mArea; } HeapArea const& getArea() const noexcept { return mArea; } void setListener(TrackingPolicy listener) noexcept { std::swap(mListener, listener); } template void emplaceListener(ARGS&& ... args) noexcept { mListener.~TrackingPolicy(); new (&mListener) TrackingPolicy(std::forward(args)...); } // An arena can't be copied Arena(Arena const& rhs) noexcept = delete; Arena& operator=(Arena const& rhs) noexcept = delete; private: HeapArea mArea; // We might want to make that a template parameter too eventually. AllocatorPolicy mAllocator; LockingPolicy mLock; TrackingPolicy mListener; char const* mArenaName = nullptr; }; // ------------------------------------------------------------------------------------------------ template using HeapArena = Arena; // ------------------------------------------------------------------------------------------------ // This doesn't implement our allocator concept, because it's too risky to use this as an allocator // in particular, doing ArenaScope. template class ArenaScope { struct Finalizer { void (*finalizer)(void* p) = nullptr; Finalizer* next = nullptr; }; template static void destruct(void* p) noexcept { static_cast(p)->~T(); } public: explicit ArenaScope(ARENA& allocator) : mArena(allocator), mRewind(allocator.getCurrent()) { } ArenaScope& operator=(const ArenaScope& rhs) = delete; ArenaScope(ArenaScope&& rhs) noexcept = delete; ArenaScope& operator=(ArenaScope&& rhs) noexcept = delete; ~ArenaScope() { // run the finalizer chain Finalizer* head = mFinalizerHead; while (head) { void* p = pointermath::add(head, sizeof(Finalizer)); head->finalizer(p); head = head->next; } // ArenaScope works only with Arena that implements rewind() mArena.rewind(mRewind); } template T* make(ARGS&& ... args) noexcept { T* o = nullptr; if (std::is_trivially_destructible::value) { o = mArena.template make(std::forward(args)...); } else { void* const p = (Finalizer*)mArena.alloc(sizeof(T), ALIGN, sizeof(Finalizer)); if (p != nullptr) { Finalizer* const f = static_cast(p) - 1; // constructor must be called before adding the dtor to the list // so that the ctor can allocate objects in a nested scope and have the // finalizers called in reverse order. o = new(p) T(std::forward(args)...); f->finalizer = &destruct; f->next = mFinalizerHead; mFinalizerHead = f; } } return o; } void* allocate(size_t size, size_t alignment = 1) noexcept { return mArena.template alloc(size, alignment, 0); } template T* allocate(size_t size, size_t alignment = alignof(T), size_t extra = 0) noexcept { return mArena.template alloc(size, alignment, extra); } // use with caution ARENA& getAllocator() noexcept { return mArena; } private: ARENA& mArena; void* mRewind = nullptr; Finalizer* mFinalizerHead = nullptr; }; template class STLAllocator { public: using value_type = TYPE; using pointer = TYPE*; using const_pointer = const TYPE*; using reference = TYPE&; using const_reference = const TYPE&; using size_type = std::size_t; using difference_type = std::ptrdiff_t; using propagate_on_container_move_assignment = std::true_type; using is_always_equal = std::true_type; template struct rebind { using other = STLAllocator; }; public: // we don't make this explicit, so that we can initialize a vector using a STLAllocator // from an Arena, avoiding to have to repeat the vector type. STLAllocator(ARENA& arena) : mArena(arena) { } // NOLINT(google-explicit-constructor) template explicit STLAllocator(STLAllocator const& rhs) : mArena(rhs.mArena) { } TYPE* allocate(std::size_t n) { return static_cast(mArena.alloc(n * sizeof(TYPE), alignof(TYPE))); } void deallocate(TYPE* p, std::size_t n) { mArena.free(p, n * sizeof(TYPE)); } // these should be out-of-class friends, but this doesn't seem to work with some compilers // which complain about multiple definition each time a STLAllocator<> is instantiated. template bool operator==(const STLAllocator& rhs) const noexcept { return std::addressof(mArena) == std::addressof(rhs.mArena); } template bool operator!=(const STLAllocator& rhs) const noexcept { return !operator==(rhs); } private: template friend class STLAllocator; ARENA& mArena; }; } // namespace utils #endif // TNT_UTILS_ALLOCATOR_H