Google Git
Sign in
android / platform / external / cronet / 181bb755f589c53bd3e47be5f2a1883b9c33a562 / . / base / allocator / partition_allocator / src / partition_alloc / partition_alloc_unittest.cc
blob: f4c5fb3ac780b6e802e5158c6bf7ebd759470dc5 [file] [log] [blame]
// Copyright 2013 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "partition_alloc/partition_alloc_for_testing.h"
#include <algorithm>
#include <bit>
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <limits>
#include <memory>
#include <random>
#include <set>
#include <tuple>
#include <vector>
#include "base/system/sys_info.h"
#include "base/test/gtest_util.h"
#include "build/build_config.h"
#include "partition_alloc/address_space_randomization.h"
#include "partition_alloc/chromecast_buildflags.h"
#include "partition_alloc/dangling_raw_ptr_checks.h"
#include "partition_alloc/freeslot_bitmap.h"
#include "partition_alloc/lightweight_quarantine.h"
#include "partition_alloc/memory_reclaimer.h"
#include "partition_alloc/page_allocator_constants.h"
#include "partition_alloc/partition_address_space.h"
#include "partition_alloc/partition_alloc_base/bits.h"
#include "partition_alloc/partition_alloc_base/compiler_specific.h"
#include "partition_alloc/partition_alloc_base/cpu.h"
#include "partition_alloc/partition_alloc_base/debug/debugging_buildflags.h"
#include "partition_alloc/partition_alloc_base/logging.h"
#include "partition_alloc/partition_alloc_base/numerics/checked_math.h"
#include "partition_alloc/partition_alloc_base/rand_util.h"
#include "partition_alloc/partition_alloc_base/thread_annotations.h"
#include "partition_alloc/partition_alloc_base/threading/platform_thread_for_testing.h"
#include "partition_alloc/partition_alloc_buildflags.h"
#include "partition_alloc/partition_alloc_config.h"
#include "partition_alloc/partition_alloc_constants.h"
#include "partition_alloc/partition_alloc_forward.h"
#include "partition_alloc/partition_bucket.h"
#include "partition_alloc/partition_cookie.h"
#include "partition_alloc/partition_freelist_entry.h"
#include "partition_alloc/partition_page.h"
#include "partition_alloc/partition_ref_count.h"
#include "partition_alloc/partition_root.h"
#include "partition_alloc/partition_stats.h"
#include "partition_alloc/reservation_offset_table.h"
#include "partition_alloc/tagging.h"
#include "partition_alloc/thread_isolation/thread_isolation.h"
#include "testing/gtest/include/gtest/gtest.h"
#if defined(__ARM_FEATURE_MEMORY_TAGGING)
#include <arm_acle.h>
#endif
#if BUILDFLAG(IS_POSIX)
#if BUILDFLAG(IS_LINUX)
// We need PKEY_DISABLE_WRITE in this file; glibc defines it in sys/mman.h but
// it's actually Linux-specific and other Linux libcs define it in linux/mman.h.
// We have to include both to be sure we get the definition.
#include <linux/mman.h>
#endif // BUILDFLAG(IS_LINUX)
#include <sys/mman.h>
#include <sys/resource.h>
#include <sys/time.h>
#endif // BUILDFLAG(IS_POSIX)
#if BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC) && BUILDFLAG(IS_MAC)
#include <OpenCL/opencl.h>
#endif
#if BUILDFLAG(IS_MAC)
#include "partition_alloc/partition_alloc_base/mac/mac_util.h"
#endif
#if BUILDFLAG(ENABLE_PKEYS)
#include <sys/syscall.h>
#endif
// In the MTE world, the upper bits of a pointer can be decorated with a tag,
// thus allowing many versions of the same pointer to exist. These macros take
// that into account when comparing.
#define PA_EXPECT_PTR_EQ(ptr1, ptr2) \
{ EXPECT_EQ(UntagPtr(ptr1), UntagPtr(ptr2)); }
#define PA_EXPECT_PTR_NE(ptr1, ptr2) \
{ EXPECT_NE(UntagPtr(ptr1), UntagPtr(ptr2)); }
#if !defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
namespace {
bool IsLargeMemoryDevice() {
// Treat any device with 4GiB or more of physical memory as a "large memory
// device". We check for slightly less than GiB so that devices with a small
// amount of memory not accessible to the OS still count as "large".
//
// Set to 4GiB, since we have 2GiB Android devices where tests flakily fail
// (e.g. Nexus 5X, crbug.com/1191195).
return base::SysInfo::AmountOfPhysicalMemory() >= 4000ULL * 1024 * 1024;
}
bool SetAddressSpaceLimit() {
#if !defined(ARCH_CPU_64_BITS) || !BUILDFLAG(IS_POSIX)
// 32 bits => address space is limited already.
return true;
#elif BUILDFLAG(IS_POSIX) && !BUILDFLAG(IS_APPLE)
// macOS will accept, but not enforce, |RLIMIT_AS| changes. See
// https://crbug.com/435269 and rdar://17576114.
//
// Note: This number must be not less than 6 GB, because with
// sanitizer_coverage_flags=edge, it reserves > 5 GB of address space. See
// https://crbug.com/674665.
const size_t kAddressSpaceLimit = static_cast<size_t>(6144) * 1024 * 1024;
struct rlimit limit;
if (getrlimit(RLIMIT_DATA, &limit) != 0) {
return false;
}
if (limit.rlim_cur == RLIM_INFINITY || limit.rlim_cur > kAddressSpaceLimit) {
limit.rlim_cur = kAddressSpaceLimit;
if (setrlimit(RLIMIT_DATA, &limit) != 0) {
return false;
}
}
return true;
#else
return false;
#endif
}
bool ClearAddressSpaceLimit() {
#if !defined(ARCH_CPU_64_BITS) || !BUILDFLAG(IS_POSIX)
return true;
#elif BUILDFLAG(IS_POSIX)
struct rlimit limit;
if (getrlimit(RLIMIT_DATA, &limit) != 0) {
return false;
}
limit.rlim_cur = limit.rlim_max;
if (setrlimit(RLIMIT_DATA, &limit) != 0) {
return false;
}
return true;
#else
return false;
#endif
}
const size_t kTestSizes[] = {
1,
17,
100,
partition_alloc::internal::SystemPageSize(),
partition_alloc::internal::SystemPageSize() + 1,
partition_alloc::PartitionRoot::GetDirectMapSlotSize(100),
1 << 20,
1 << 21,
};
constexpr size_t kTestSizesCount = std::size(kTestSizes);
template <
partition_alloc::AllocFlags alloc_flags,
partition_alloc::FreeFlags free_flags = partition_alloc::FreeFlags::kNone>
void AllocateRandomly(partition_alloc::PartitionRoot* root, size_t count) {
std::vector<void*> allocations(count, nullptr);
for (size_t i = 0; i < count; ++i) {
const size_t size =
kTestSizes[partition_alloc::internal::base::RandGenerator(
kTestSizesCount)];
allocations[i] = root->Alloc<alloc_flags>(size);
EXPECT_NE(nullptr, allocations[i]) << " size: " << size << " i: " << i;
}
for (size_t i = 0; i < count; ++i) {
if (allocations[i]) {
root->Free(allocations[i]);
}
}
}
void HandleOOM(size_t unused_size) {
PA_LOG(FATAL) << "Out of memory";
}
int g_dangling_raw_ptr_detected_count = 0;
int g_dangling_raw_ptr_released_count = 0;
class CountDanglingRawPtr {
public:
CountDanglingRawPtr() {
g_dangling_raw_ptr_detected_count = 0;
g_dangling_raw_ptr_released_count = 0;
old_detected_fn_ = partition_alloc::GetDanglingRawPtrDetectedFn();
old_released_fn_ = partition_alloc::GetDanglingRawPtrReleasedFn();
partition_alloc::SetDanglingRawPtrDetectedFn(
CountDanglingRawPtr::DanglingRawPtrDetected);
partition_alloc::SetDanglingRawPtrReleasedFn(
CountDanglingRawPtr::DanglingRawPtrReleased);
}
~CountDanglingRawPtr() {
partition_alloc::SetDanglingRawPtrDetectedFn(old_detected_fn_);
partition_alloc::SetDanglingRawPtrReleasedFn(old_released_fn_);
}
private:
static void DanglingRawPtrDetected(uintptr_t) {
g_dangling_raw_ptr_detected_count++;
}
static void DanglingRawPtrReleased(uintptr_t) {
g_dangling_raw_ptr_released_count++;
}
partition_alloc::DanglingRawPtrDetectedFn* old_detected_fn_;
partition_alloc::DanglingRawPtrReleasedFn* old_released_fn_;
};
} // namespace
// Note: This test exercises interfaces inside the `partition_alloc`
// namespace, but inspects objects inside `partition_alloc::internal`.
// For ease of reading, the tests are placed into the latter namespace.
namespace partition_alloc::internal {
using BucketDistribution = PartitionRoot::BucketDistribution;
using SlotSpan = SlotSpanMetadata;
const size_t kTestAllocSize = 16;
#if !BUILDFLAG(PA_DCHECK_IS_ON)
const size_t kPointerOffset = kPartitionRefCountOffsetAdjustment;
const size_t kExtraAllocSizeWithoutRefCount = 0ull;
#else
const size_t kPointerOffset = kPartitionRefCountOffsetAdjustment;
const size_t kExtraAllocSizeWithoutRefCount = kCookieSize;
#endif
const char* type_name = nullptr;
void SetDistributionForPartitionRoot(PartitionRoot* root,
BucketDistribution distribution) {
switch (distribution) {
case BucketDistribution::kNeutral:
root->ResetBucketDistributionForTesting();
break;
case BucketDistribution::kDenser:
root->SwitchToDenserBucketDistribution();
break;
}
}
struct PartitionAllocTestParam {
BucketDistribution bucket_distribution;
bool use_pkey_pool;
size_t ref_count_size;
};
const std::vector<PartitionAllocTestParam> GetPartitionAllocTestParams() {
std::vector<size_t> ref_count_sizes = {0, 8, 16};
// sizeof(PartitionRefCount) == 8 under some configurations, so we can't force
// the size down to 4.
#if !PA_CONFIG(REF_COUNT_STORE_REQUESTED_SIZE) && \
!PA_CONFIG(REF_COUNT_CHECK_COOKIE) && \
!BUILDFLAG(ENABLE_DANGLING_RAW_PTR_CHECKS)
ref_count_sizes.push_back(4);
#endif
// Using MTE or Mac13 workaroud increases extras size without increasing
// sizeof(PartitionRefCount), so we don't have to exclude it here, as long as
// ExtraAllocSize() accounts for it.
std::vector<PartitionAllocTestParam> params;
for (size_t ref_count_size : ref_count_sizes) {
params.emplace_back(PartitionAllocTestParam{BucketDistribution::kNeutral,
false, ref_count_size});
params.emplace_back(PartitionAllocTestParam{BucketDistribution::kDenser,
false, ref_count_size});
#if BUILDFLAG(ENABLE_PKEYS)
if (CPUHasPkeySupport()) {
params.emplace_back(PartitionAllocTestParam{BucketDistribution::kNeutral,
true, ref_count_size});
params.emplace_back(PartitionAllocTestParam{BucketDistribution::kDenser,
true, ref_count_size});
}
#endif
}
return params;
}
class PartitionAllocTest
: public testing::TestWithParam<PartitionAllocTestParam> {
protected:
class ScopedPageAllocation {
public:
ScopedPageAllocation(PartitionAllocator& allocator,
base::CheckedNumeric<size_t> npages)
: allocator_(allocator),
npages_(npages),
ptr_(static_cast<char*>(allocator_.root()->Alloc(
(npages * SystemPageSize() - ExtraAllocSize(allocator_))
.ValueOrDie(),
type_name))) {}
~ScopedPageAllocation() { allocator_.root()->Free(ptr_); }
void TouchAllPages() {
memset(ptr_, 'A',
((npages_ * SystemPageSize()) - ExtraAllocSize(allocator_))
.ValueOrDie());
}
void* PageAtIndex(size_t index) {
return ptr_ - kPointerOffset + (SystemPageSize() * index);
}
private:
PartitionAllocator& allocator_;
const base::CheckedNumeric<size_t> npages_;
char* ptr_;
};
PartitionAllocTest() = default;
~PartitionAllocTest() override = default;
struct PartitionTestOptions {
bool use_memory_reclaimer = false;
bool uncap_empty_slot_span_memory = false;
bool set_bucket_distribution = false;
};
void InitializeTestRoot(PartitionRoot* root,
PartitionOptions opts,
PartitionTestOptions test_opts) {
root->Init(opts);
if (test_opts.use_memory_reclaimer) {
MemoryReclaimer::Instance()->RegisterPartition(root);
}
if (test_opts.uncap_empty_slot_span_memory) {
root->UncapEmptySlotSpanMemoryForTesting();
}
if (test_opts.set_bucket_distribution) {
SetDistributionForPartitionRoot(root, GetBucketDistribution());
}
}
std::unique_ptr<PartitionRoot> CreateCustomTestRoot(
PartitionOptions opts,
PartitionTestOptions test_opts) {
auto root = std::make_unique<PartitionRoot>();
InitializeTestRoot(root.get(), opts, test_opts);
return root;
}
PartitionOptions GetCommonPartitionOptions() {
PartitionOptions opts;
opts.ref_count_size = GetParam().ref_count_size;
opts.zapping_by_free_flags = PartitionOptions::kEnabled;
return opts;
}
void InitializeMainTestAllocators() {
#if BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT)
PartitionOptions::EnableToggle enable_backup_ref_ptr =
PartitionOptions::kEnabled;
#endif
#if BUILDFLAG(ENABLE_PKEYS)
int pkey = PkeyAlloc(UseThreadIsolatedPool() ? 0 : PKEY_DISABLE_WRITE);
if (pkey != -1) {
pkey_ = pkey;
}
PartitionOptions pkey_opts = GetCommonPartitionOptions();
pkey_opts.aligned_alloc = PartitionOptions::kAllowed;
pkey_opts.thread_isolation = ThreadIsolationOption(pkey_);
// We always want to have a pkey allocator initialized to make sure that the
// other pools still work. As part of the initializition, we tag some memory
// with the new pkey, effectively making it read-only. So there's some
// potential for breakage that this should catch.
InitializeTestRoot(pkey_allocator.root(), pkey_opts,
PartitionTestOptions{.use_memory_reclaimer = true});
ThreadIsolationOption thread_isolation_opt;
if (UseThreadIsolatedPool() && pkey_ != kInvalidPkey) {
thread_isolation_opt = ThreadIsolationOption(pkey_);
#if BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT)
// BRP and thread isolated mode use different pools, so they can't be
// enabled at the same time.
enable_backup_ref_ptr = PartitionOptions::kDisabled;
#endif
}
#endif // BUILDFLAG(ENABLE_PKEYS)
PartitionOptions opts = GetCommonPartitionOptions();
#if !BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT) || \
BUILDFLAG(PUT_REF_COUNT_IN_PREVIOUS_SLOT)
// AlignedAlloc() can't be called when BRP is in the
// "before allocation" mode, because this mode adds extras before
// the allocation. Extras after the allocation are ok.
opts.aligned_alloc = PartitionOptions::kAllowed;
#endif
#if BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT)
opts.backup_ref_ptr = enable_backup_ref_ptr;
#endif
#if BUILDFLAG(ENABLE_PKEYS)
opts.thread_isolation = thread_isolation_opt;
#endif
#if PA_CONFIG(HAS_MEMORY_TAGGING)
opts.memory_tagging = {
.enabled =
partition_alloc::internal::base::CPU::GetInstanceNoAllocation()
.has_mte()
? PartitionOptions::kEnabled
: PartitionOptions::kDisabled,
};
#endif // PA_CONFIG(HAS_MEMORY_TAGGING)
InitializeTestRoot(
allocator.root(), opts,
PartitionTestOptions{.use_memory_reclaimer = true,
.uncap_empty_slot_span_memory = true,
.set_bucket_distribution = true});
PartitionOptions aligned_opts = GetCommonPartitionOptions();
aligned_opts.aligned_alloc = PartitionOptions::kAllowed;
InitializeTestRoot(
aligned_allocator.root(), aligned_opts,
PartitionTestOptions{.use_memory_reclaimer = true,
.uncap_empty_slot_span_memory = true,
.set_bucket_distribution = true});
}
size_t RealAllocSize() const {
return partition_alloc::internal::base::bits::AlignUp(
kTestAllocSize + ExtraAllocSize(allocator), kAlignment);
}
void SetUp() override {
PartitionRoot::SetStraightenLargerSlotSpanFreeListsMode(
StraightenLargerSlotSpanFreeListsMode::kOnlyWhenUnprovisioning);
PartitionRoot::SetSortSmallerSlotSpanFreeListsEnabled(true);
PartitionRoot::SetSortActiveSlotSpansEnabled(true);
PartitionAllocGlobalInit(HandleOOM);
InitializeMainTestAllocators();
test_bucket_index_ = SizeToIndex(RealAllocSize());
}
size_t SizeToIndex(size_t size) {
const auto distribution_to_use = GetBucketDistribution();
return PartitionRoot::SizeToBucketIndex(size, distribution_to_use);
}
size_t SizeToBucketSize(size_t size) {
const auto index = SizeToIndex(size);
return allocator.root()->buckets[index].slot_size;
}
void TearDown() override {
allocator.root()->PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans |
PurgeFlags::kDiscardUnusedSystemPages);
PartitionAllocGlobalUninitForTesting();
#if BUILDFLAG(ENABLE_PKEYS)
if (pkey_ != kInvalidPkey) {
PkeyFree(pkey_);
}
#endif
}
static size_t ExtraAllocSize(const PartitionAllocator& allocator) {
size_t ref_count_size = 0;
// Duplicate the logic from PartitionRoot::Init().
if (allocator.root()->brp_enabled()) {
ref_count_size = GetParam().ref_count_size;
if (!ref_count_size) {
ref_count_size = kPartitionRefCountSizeAdjustment;
}
ref_count_size = AlignUpRefCountSizeForMac(ref_count_size);
#if PA_CONFIG(INCREASE_REF_COUNT_SIZE_FOR_MTE)
if (allocator.root()->IsMemoryTaggingEnabled()) {
ref_count_size = partition_alloc::internal::base::bits::AlignUp(
ref_count_size, kMemTagGranuleSize);
}
#endif // PA_CONFIG(INCREASE_REF_COUNT_SIZE_FOR_MTE)
}
return kExtraAllocSizeWithoutRefCount + ref_count_size;
}
size_t GetNumPagesPerSlotSpan(size_t size) {
size_t real_size = size + ExtraAllocSize(allocator);
size_t bucket_index = SizeToIndex(real_size);
PartitionRoot::Bucket* bucket = &allocator.root()->buckets[bucket_index];
// TODO(tasak): make get_pages_per_slot_span() available at
// partition_alloc_unittest.cc. Is it allowable to make the code from
// partition_bucet.cc to partition_bucket.h?
return (bucket->num_system_pages_per_slot_span +
(NumSystemPagesPerPartitionPage() - 1)) /
NumSystemPagesPerPartitionPage();
}
SlotSpan* GetFullSlotSpan(size_t size) {
size_t real_size = size + ExtraAllocSize(allocator);
size_t bucket_index = SizeToIndex(real_size);
PartitionRoot::Bucket* bucket = &allocator.root()->buckets[bucket_index];
size_t num_slots =
(bucket->num_system_pages_per_slot_span * SystemPageSize()) /
bucket->slot_size;
uintptr_t first = 0;
uintptr_t last = 0;
size_t i;
for (i = 0; i < num_slots; ++i) {
void* ptr = allocator.root()->Alloc(size, type_name);
EXPECT_TRUE(ptr);
if (!i) {
first = allocator.root()->ObjectToSlotStart(ptr);
} else if (i == num_slots - 1) {
last = allocator.root()->ObjectToSlotStart(ptr);
}
}
EXPECT_EQ(SlotSpan::FromSlotStart(first), SlotSpan::FromSlotStart(last));
if (bucket->num_system_pages_per_slot_span ==
NumSystemPagesPerPartitionPage()) {
EXPECT_EQ(first & PartitionPageBaseMask(),
last & PartitionPageBaseMask());
}
EXPECT_EQ(num_slots, bucket->active_slot_spans_head->num_allocated_slots);
EXPECT_EQ(nullptr, bucket->active_slot_spans_head->get_freelist_head());
EXPECT_TRUE(bucket->is_valid());
EXPECT_TRUE(bucket->active_slot_spans_head !=
SlotSpan::get_sentinel_slot_span());
EXPECT_TRUE(bucket->active_slot_spans_head->is_full());
return bucket->active_slot_spans_head;
}
void CycleFreeCache(size_t size) {
for (size_t i = 0; i < kMaxFreeableSpans; ++i) {
void* ptr = allocator.root()->Alloc(size, type_name);
auto* slot_span =
SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
auto* bucket = slot_span->bucket;
EXPECT_EQ(1u, bucket->active_slot_spans_head->num_allocated_slots);
allocator.root()->Free(ptr);
EXPECT_EQ(0u, bucket->active_slot_spans_head->num_allocated_slots);
EXPECT_TRUE(bucket->active_slot_spans_head->in_empty_cache() ||
bucket->active_slot_spans_head ==
SlotSpanMetadata::get_sentinel_slot_span());
}
}
enum ReturnNullTestMode {
kPartitionAlloc,
kPartitionRealloc,
};
void DoReturnNullTest(size_t alloc_size, ReturnNullTestMode mode) {
// TODO(crbug.com/678782): Where necessary and possible, disable the
// platform's OOM-killing behavior. OOM-killing makes this test flaky on
// low-memory devices.
if (!IsLargeMemoryDevice()) {
PA_LOG(WARNING)
<< "Skipping test on this device because of crbug.com/678782";
PA_LOG(FATAL) << "Passed DoReturnNullTest";
}
ASSERT_TRUE(SetAddressSpaceLimit());
// Work out the number of allocations for 6 GB of memory.
const int num_allocations = (6 * 1024 * 1024) / (alloc_size / 1024);
void** ptrs = static_cast<void**>(
allocator.root()->Alloc(num_allocations * sizeof(void*), type_name));
int i;
for (i = 0; i < num_allocations; ++i) {
switch (mode) {
case kPartitionAlloc: {
ptrs[i] = allocator.root()->Alloc<AllocFlags::kReturnNull>(alloc_size,
type_name);
break;
}
case kPartitionRealloc: {
ptrs[i] =
allocator.root()->Alloc<AllocFlags::kReturnNull>(1, type_name);
ptrs[i] = allocator.root()->Realloc<AllocFlags::kReturnNull>(
ptrs[i], alloc_size, type_name);
break;
}
}
if (!i) {
EXPECT_TRUE(ptrs[0]);
}
if (!ptrs[i]) {
ptrs[i] = allocator.root()->Alloc<AllocFlags::kReturnNull>(alloc_size,
type_name);
EXPECT_FALSE(ptrs[i]);
break;
}
}
// We shouldn't succeed in allocating all 6 GB of memory. If we do, then
// we're not actually testing anything here.
EXPECT_LT(i, num_allocations);
// Free, reallocate and free again each block we allocated. We do this to
// check that freeing memory also works correctly after a failed allocation.
for (--i; i >= 0; --i) {
allocator.root()->Free(ptrs[i]);
ptrs[i] = allocator.root()->Alloc<AllocFlags::kReturnNull>(alloc_size,
type_name);
EXPECT_TRUE(ptrs[i]);
allocator.root()->Free(ptrs[i]);
}
allocator.root()->Free(ptrs);
EXPECT_TRUE(ClearAddressSpaceLimit());
PA_LOG(FATAL) << "Passed DoReturnNullTest";
}
void RunRefCountReallocSubtest(size_t orig_size, size_t new_size);
PA_NOINLINE PA_MALLOC_FN void* Alloc(size_t size) {
return allocator.root()->Alloc(size);
}
PA_NOINLINE void Free(void* ptr) { allocator.root()->Free(ptr); }
BucketDistribution GetBucketDistribution() const {
return GetParam().bucket_distribution;
}
bool UseThreadIsolatedPool() const { return GetParam().use_pkey_pool; }
bool UseBRPPool() const { return allocator.root()->brp_enabled(); }
partition_alloc::PartitionAllocatorForTesting allocator;
partition_alloc::PartitionAllocatorForTesting aligned_allocator;
#if BUILDFLAG(ENABLE_PKEYS)
partition_alloc::PartitionAllocatorForTesting pkey_allocator;
#endif
size_t test_bucket_index_;
#if BUILDFLAG(ENABLE_PKEYS)
int pkey_ = kInvalidPkey;
#endif
};
// Death tests misbehave on Android, http://crbug.com/643760.
#if defined(GTEST_HAS_DEATH_TEST) && !BUILDFLAG(IS_ANDROID)
#define PA_HAS_DEATH_TESTS
class PartitionAllocDeathTest : public PartitionAllocTest {};
INSTANTIATE_TEST_SUITE_P(AlternateTestParams,
PartitionAllocDeathTest,
testing::ValuesIn(GetPartitionAllocTestParams()));
#endif
namespace {
void FreeFullSlotSpan(PartitionRoot* root, SlotSpan* slot_span) {
EXPECT_TRUE(slot_span->is_full());
size_t size = slot_span->bucket->slot_size;
size_t num_slots =
(slot_span->bucket->num_system_pages_per_slot_span * SystemPageSize()) /
size;
EXPECT_EQ(num_slots, slot_span->num_allocated_slots);
uintptr_t address = SlotSpan::ToSlotSpanStart(slot_span);
size_t i;
for (i = 0; i < num_slots; ++i) {
root->Free(root->SlotStartToObject(address));
address += size;
}
EXPECT_TRUE(slot_span->is_empty());
}
#if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
bool CheckPageInCore(void* ptr, bool in_core) {
unsigned char ret = 0;
EXPECT_EQ(0, mincore(ptr, SystemPageSize(), &ret));
return in_core == (ret & 1);
}
#define CHECK_PAGE_IN_CORE(ptr, in_core) \
EXPECT_TRUE(CheckPageInCore(ptr, in_core))
#else
#define CHECK_PAGE_IN_CORE(ptr, in_core) (void)(0)
#endif // BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
class MockPartitionStatsDumper : public PartitionStatsDumper {
public:
MockPartitionStatsDumper() = default;
void PartitionDumpTotals(const char* partition_name,
const PartitionMemoryStats* stats) override {
EXPECT_GE(stats->total_mmapped_bytes, stats->total_resident_bytes);
EXPECT_EQ(total_resident_bytes, stats->total_resident_bytes);
EXPECT_EQ(total_active_bytes, stats->total_active_bytes);
EXPECT_EQ(total_decommittable_bytes, stats->total_decommittable_bytes);
EXPECT_EQ(total_discardable_bytes, stats->total_discardable_bytes);
}
void PartitionsDumpBucketStats(
[[maybe_unused]] const char* partition_name,
const PartitionBucketMemoryStats* stats) override {
EXPECT_TRUE(stats->is_valid);
EXPECT_EQ(0u, stats->bucket_slot_size & sizeof(void*));
bucket_stats.push_back(*stats);
total_resident_bytes += stats->resident_bytes;
total_active_bytes += stats->active_bytes;
total_decommittable_bytes += stats->decommittable_bytes;
total_discardable_bytes += stats->discardable_bytes;
}
bool IsMemoryAllocationRecorded() {
return total_resident_bytes != 0 && total_active_bytes != 0;
}
const PartitionBucketMemoryStats* GetBucketStats(size_t bucket_size) {
for (auto& stat : bucket_stats) {
if (stat.bucket_slot_size == bucket_size) {
return &stat;
}
}
return nullptr;
}
private:
size_t total_resident_bytes = 0;
size_t total_active_bytes = 0;
size_t total_decommittable_bytes = 0;
size_t total_discardable_bytes = 0;
std::vector<PartitionBucketMemoryStats> bucket_stats;
};
} // namespace
INSTANTIATE_TEST_SUITE_P(AlternateTestParams,
PartitionAllocTest,
testing::ValuesIn(GetPartitionAllocTestParams()));
// Check that the most basic of allocate / free pairs work.
TEST_P(PartitionAllocTest, Basic) {
PartitionRoot::Bucket* bucket =
&allocator.root()->buckets[test_bucket_index_];
auto* seed_slot_span = SlotSpan::get_sentinel_slot_span();
EXPECT_FALSE(bucket->empty_slot_spans_head);
EXPECT_FALSE(bucket->decommitted_slot_spans_head);
EXPECT_EQ(seed_slot_span, bucket->active_slot_spans_head);
EXPECT_EQ(nullptr, bucket->active_slot_spans_head->next_slot_span);
void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
EXPECT_TRUE(ptr);
EXPECT_EQ(kPointerOffset, UntagPtr(ptr) & PartitionPageOffsetMask());
// Check that the offset appears to include a guard page.
EXPECT_EQ(PartitionPageSize() +
partition_alloc::internal::ReservedFreeSlotBitmapSize() +
kPointerOffset,
UntagPtr(ptr) & kSuperPageOffsetMask);
allocator.root()->Free(ptr);
// Expect that the last active slot span gets noticed as empty but doesn't get
// decommitted.
EXPECT_TRUE(bucket->empty_slot_spans_head);
EXPECT_FALSE(bucket->decommitted_slot_spans_head);
}
// Test multiple allocations, and freelist handling.
TEST_P(PartitionAllocTest, MultiAlloc) {
void* ptr1 = allocator.root()->Alloc(kTestAllocSize, type_name);
void* ptr2 = allocator.root()->Alloc(kTestAllocSize, type_name);
EXPECT_TRUE(ptr1);
EXPECT_TRUE(ptr2);
ptrdiff_t diff = UntagPtr(ptr2) - UntagPtr(ptr1);
EXPECT_EQ(static_cast<ptrdiff_t>(RealAllocSize()), diff);
// Check that we re-use the just-freed slot.
allocator.root()->Free(ptr2);
ptr2 = allocator.root()->Alloc(kTestAllocSize, type_name);
EXPECT_TRUE(ptr2);
diff = UntagPtr(ptr2) - UntagPtr(ptr1);
EXPECT_EQ(static_cast<ptrdiff_t>(RealAllocSize()), diff);
allocator.root()->Free(ptr1);
ptr1 = allocator.root()->Alloc(kTestAllocSize, type_name);
EXPECT_TRUE(ptr1);
diff = UntagPtr(ptr2) - UntagPtr(ptr1);
EXPECT_EQ(static_cast<ptrdiff_t>(RealAllocSize()), diff);
void* ptr3 = allocator.root()->Alloc(kTestAllocSize, type_name);
EXPECT_TRUE(ptr3);
diff = UntagPtr(ptr3) - UntagPtr(ptr1);
EXPECT_EQ(static_cast<ptrdiff_t>(RealAllocSize() * 2), diff);
allocator.root()->Free(ptr1);
allocator.root()->Free(ptr2);
allocator.root()->Free(ptr3);
}
// Test a bucket with multiple slot spans.
TEST_P(PartitionAllocTest, MultiSlotSpans) {
PartitionRoot::Bucket* bucket =
&allocator.root()->buckets[test_bucket_index_];
auto* slot_span = GetFullSlotSpan(kTestAllocSize);
FreeFullSlotSpan(allocator.root(), slot_span);
EXPECT_TRUE(bucket->empty_slot_spans_head);
EXPECT_EQ(SlotSpan::get_sentinel_slot_span(), bucket->active_slot_spans_head);
EXPECT_EQ(nullptr, slot_span->next_slot_span);
EXPECT_EQ(0u, slot_span->num_allocated_slots);
slot_span = GetFullSlotSpan(kTestAllocSize);
auto* slot_span2 = GetFullSlotSpan(kTestAllocSize);
EXPECT_EQ(slot_span2, bucket->active_slot_spans_head);
EXPECT_EQ(nullptr, slot_span2->next_slot_span);
EXPECT_EQ(SlotSpan::ToSlotSpanStart(slot_span) & kSuperPageBaseMask,
SlotSpan::ToSlotSpanStart(slot_span2) & kSuperPageBaseMask);
// Fully free the non-current slot span. This will leave us with no current
// active slot span because one is empty and the other is full.
FreeFullSlotSpan(allocator.root(), slot_span);
EXPECT_EQ(0u, slot_span->num_allocated_slots);
EXPECT_TRUE(bucket->empty_slot_spans_head);
EXPECT_EQ(SlotSpanMetadata::get_sentinel_slot_span(),
bucket->active_slot_spans_head);
// Allocate a new slot span, it should pull from the freelist.
slot_span = GetFullSlotSpan(kTestAllocSize);
EXPECT_FALSE(bucket->empty_slot_spans_head);
EXPECT_EQ(slot_span, bucket->active_slot_spans_head);
FreeFullSlotSpan(allocator.root(), slot_span);
FreeFullSlotSpan(allocator.root(), slot_span2);
EXPECT_EQ(0u, slot_span->num_allocated_slots);
EXPECT_EQ(0u, slot_span2->num_allocated_slots);
EXPECT_EQ(0u, slot_span2->num_unprovisioned_slots);
EXPECT_TRUE(slot_span2->in_empty_cache());
}
// Test some finer aspects of internal slot span transitions.
TEST_P(PartitionAllocTest, SlotSpanTransitions) {
PartitionRoot::Bucket* bucket =
&allocator.root()->buckets[test_bucket_index_];
auto* slot_span1 = GetFullSlotSpan(kTestAllocSize);
EXPECT_EQ(slot_span1, bucket->active_slot_spans_head);
EXPECT_EQ(nullptr, slot_span1->next_slot_span);
auto* slot_span2 = GetFullSlotSpan(kTestAllocSize);
EXPECT_EQ(slot_span2, bucket->active_slot_spans_head);
EXPECT_EQ(nullptr, slot_span2->next_slot_span);
// Bounce slot_span1 back into the non-full list then fill it up again.
void* ptr = allocator.root()->SlotStartToObject(
SlotSpan::ToSlotSpanStart(slot_span1));
allocator.root()->Free(ptr);
EXPECT_EQ(slot_span1, bucket->active_slot_spans_head);
std::ignore = allocator.root()->Alloc(kTestAllocSize, type_name);
EXPECT_EQ(slot_span1, bucket->active_slot_spans_head);
EXPECT_EQ(slot_span2, bucket->active_slot_spans_head->next_slot_span);
// Allocating another slot span at this point should cause us to scan over
// slot_span1 (which is both full and NOT our current slot span), and evict it
// from the freelist. Older code had a O(n^2) condition due to failure to do
// this.
auto* slot_span3 = GetFullSlotSpan(kTestAllocSize);
EXPECT_EQ(slot_span3, bucket->active_slot_spans_head);
EXPECT_EQ(nullptr, slot_span3->next_slot_span);
// Work out a pointer into slot_span2 and free it.
ptr = allocator.root()->SlotStartToObject(
SlotSpan::ToSlotSpanStart(slot_span2));
allocator.root()->Free(ptr);
// Trying to allocate at this time should cause us to cycle around to
// slot_span2 and find the recently freed slot.
void* ptr2 = allocator.root()->Alloc(kTestAllocSize, type_name);
PA_EXPECT_PTR_EQ(ptr, ptr2);
EXPECT_EQ(slot_span2, bucket->active_slot_spans_head);
EXPECT_EQ(slot_span3, slot_span2->next_slot_span);
// Work out a pointer into slot_span1 and free it. This should pull the slot
// span back into the list of available slot spans.
ptr = allocator.root()->SlotStartToObject(
SlotSpan::ToSlotSpanStart(slot_span1));
allocator.root()->Free(ptr);
// This allocation should be satisfied by slot_span1.
ptr2 = allocator.root()->Alloc(kTestAllocSize, type_name);
PA_EXPECT_PTR_EQ(ptr, ptr2);
EXPECT_EQ(slot_span1, bucket->active_slot_spans_head);
EXPECT_EQ(slot_span2, slot_span1->next_slot_span);
FreeFullSlotSpan(allocator.root(), slot_span3);
FreeFullSlotSpan(allocator.root(), slot_span2);
FreeFullSlotSpan(allocator.root(), slot_span1);
// Allocating whilst in this state exposed a bug, so keep the test.
ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
allocator.root()->Free(ptr);
}
// Test that ExtraAllocSize() is exactly what PA takes away from the slot for
// extras.
TEST_P(PartitionAllocTest, ExtraAllocSize) {
// There is a bucket with a slot size exactly that (asserted below).
size_t slot_size = 64;
size_t bucket_index =
allocator.root()->SizeToBucketIndex(slot_size, GetBucketDistribution());
PartitionRoot::Bucket* bucket = &allocator.root()->buckets[bucket_index];
ASSERT_EQ(bucket->slot_size, slot_size);
// The first allocation is expected to span exactly the capcity of the slot.
// The second one should overflow into a higher-size slot, and not fill its
// capacity.
size_t requested_size1 = slot_size - ExtraAllocSize(allocator);
size_t requested_size2 = requested_size1 + 1;
void* ptr1 = allocator.root()->Alloc(requested_size1);
void* ptr2 = allocator.root()->Alloc(requested_size2);
size_t capacity1 = allocator.root()->AllocationCapacityFromSlotStart(
allocator.root()->ObjectToSlotStart(ptr1));
size_t capacity2 = allocator.root()->AllocationCapacityFromSlotStart(
allocator.root()->ObjectToSlotStart(ptr2));
EXPECT_EQ(capacity1, requested_size1);
EXPECT_LT(capacity1, capacity2);
EXPECT_LT(requested_size2, capacity2);
allocator.root()->Free(ptr1);
allocator.root()->Free(ptr2);
}
TEST_P(PartitionAllocTest, PreferSlotSpansWithProvisionedEntries) {
size_t size = SystemPageSize() - ExtraAllocSize(allocator);
size_t real_size = size + ExtraAllocSize(allocator);
size_t bucket_index =
allocator.root()->SizeToBucketIndex(real_size, GetBucketDistribution());
PartitionRoot::Bucket* bucket = &allocator.root()->buckets[bucket_index];
ASSERT_EQ(bucket->slot_size, real_size);
size_t slots_per_span = bucket->num_system_pages_per_slot_span;
// Make 10 full slot spans.
constexpr int kSpans = 10;
std::vector<std::vector<void*>> allocated_memory_spans(kSpans);
for (int span_index = 0; span_index < kSpans; span_index++) {
for (size_t i = 0; i < slots_per_span; i++) {
allocated_memory_spans[span_index].push_back(
allocator.root()->Alloc(size));
}
}
// Reverse ordering, since a newly non-full span is placed at the head of the
// active list.
for (int span_index = kSpans - 1; span_index >= 0; span_index--) {
allocator.root()->Free(allocated_memory_spans[span_index].back());
allocated_memory_spans[span_index].pop_back();
}
// Since slot spans are large enough and we freed memory from the end, the
// slot spans become partially provisioned after PurgeMemory().
allocator.root()->PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans |
PurgeFlags::kDiscardUnusedSystemPages);
std::vector<SlotSpanMetadata*> active_slot_spans;
for (auto* span = bucket->active_slot_spans_head; span;
span = span->next_slot_span) {
active_slot_spans.push_back(span);
ASSERT_EQ(span->num_unprovisioned_slots, 1u);
// But no freelist entries.
ASSERT_FALSE(span->get_freelist_head());
}
// Free one entry in the middle span, creating a freelist entry.
constexpr size_t kSpanIndex = 5;
allocator.root()->Free(allocated_memory_spans[kSpanIndex].back());
allocated_memory_spans[kSpanIndex].pop_back();
ASSERT_TRUE(active_slot_spans[kSpanIndex]->get_freelist_head());
ASSERT_FALSE(bucket->active_slot_spans_head->get_freelist_head());
// It must come from the middle slot span even though the first one has
// unprovisioned space.
void* new_ptr = allocator.root()->Alloc(size);
// Comes from the middle slot span, since it has a freelist entry.
auto* new_active_slot_span = active_slot_spans[kSpanIndex];
ASSERT_FALSE(new_active_slot_span->get_freelist_head());
// The middle slot span was moved to the front.
active_slot_spans.erase(active_slot_spans.begin() + kSpanIndex);
active_slot_spans.insert(active_slot_spans.begin(), new_active_slot_span);
// Check slot span ordering.
int index = 0;
for (auto* span = bucket->active_slot_spans_head; span;
span = span->next_slot_span) {
EXPECT_EQ(span, active_slot_spans[index]);
index++;
}
EXPECT_EQ(index, kSpans);
allocator.root()->Free(new_ptr);
for (int span_index = 0; span_index < kSpans; span_index++) {
for (void* ptr : allocated_memory_spans[span_index]) {
allocator.root()->Free(ptr);
}
}
}
// Test some corner cases relating to slot span transitions in the internal
// free slot span list metadata bucket.
TEST_P(PartitionAllocTest, FreeSlotSpanListSlotSpanTransitions) {
PartitionRoot::Bucket* bucket =
&allocator.root()->buckets[test_bucket_index_];
size_t num_to_fill_free_list_slot_span =
PartitionPageSize() / (sizeof(SlotSpan) + ExtraAllocSize(allocator));
// The +1 is because we need to account for the fact that the current slot
// span never gets thrown on the freelist.
++num_to_fill_free_list_slot_span;
auto slot_spans =
std::make_unique<SlotSpan*[]>(num_to_fill_free_list_slot_span);
size_t i;
for (i = 0; i < num_to_fill_free_list_slot_span; ++i) {
slot_spans[i] = GetFullSlotSpan(kTestAllocSize);
}
EXPECT_EQ(slot_spans[num_to_fill_free_list_slot_span - 1],
bucket->active_slot_spans_head);
for (i = 0; i < num_to_fill_free_list_slot_span; ++i) {
FreeFullSlotSpan(allocator.root(), slot_spans[i]);
}
EXPECT_EQ(SlotSpan::get_sentinel_slot_span(), bucket->active_slot_spans_head);
EXPECT_TRUE(bucket->empty_slot_spans_head);
// Allocate / free in a different bucket size so we get control of a
// different free slot span list. We need two slot spans because one will be
// the last active slot span and not get freed.
auto* slot_span1 = GetFullSlotSpan(kTestAllocSize * 2);
auto* slot_span2 = GetFullSlotSpan(kTestAllocSize * 2);
FreeFullSlotSpan(allocator.root(), slot_span1);
FreeFullSlotSpan(allocator.root(), slot_span2);
for (i = 0; i < num_to_fill_free_list_slot_span; ++i) {
slot_spans[i] = GetFullSlotSpan(kTestAllocSize);
}
EXPECT_EQ(slot_spans[num_to_fill_free_list_slot_span - 1],
bucket->active_slot_spans_head);
for (i = 0; i < num_to_fill_free_list_slot_span; ++i) {
FreeFullSlotSpan(allocator.root(), slot_spans[i]);
}
EXPECT_EQ(SlotSpan::get_sentinel_slot_span(), bucket->active_slot_spans_head);
EXPECT_TRUE(bucket->empty_slot_spans_head);
}
// Test a large series of allocations that cross more than one underlying
// super page.
TEST_P(PartitionAllocTest, MultiPageAllocs) {
size_t num_pages_per_slot_span = GetNumPagesPerSlotSpan(kTestAllocSize);
// 1 super page has 2 guard partition pages and a tag bitmap.
size_t num_slot_spans_needed =
(NumPartitionPagesPerSuperPage() - 2 -
partition_alloc::internal::NumPartitionPagesPerFreeSlotBitmap()) /
num_pages_per_slot_span;
// We need one more slot span in order to cross super page boundary.
++num_slot_spans_needed;
EXPECT_GT(num_slot_spans_needed, 1u);
auto slot_spans = std::make_unique<SlotSpan*[]>(num_slot_spans_needed);
uintptr_t first_super_page_base = 0;
size_t i;
for (i = 0; i < num_slot_spans_needed; ++i) {
slot_spans[i] = GetFullSlotSpan(kTestAllocSize);
uintptr_t slot_span_start = SlotSpan::ToSlotSpanStart(slot_spans[i]);
if (!i) {
first_super_page_base = slot_span_start & kSuperPageBaseMask;
}
if (i == num_slot_spans_needed - 1) {
uintptr_t second_super_page_base = slot_span_start & kSuperPageBaseMask;
uintptr_t second_super_page_offset =
slot_span_start & kSuperPageOffsetMask;
EXPECT_FALSE(second_super_page_base == first_super_page_base);
// Check that we allocated a guard page and the reserved tag bitmap for
// the second page.
EXPECT_EQ(PartitionPageSize() +
partition_alloc::internal::ReservedFreeSlotBitmapSize(),
second_super_page_offset);
}
}
for (i = 0; i < num_slot_spans_needed; ++i) {
FreeFullSlotSpan(allocator.root(), slot_spans[i]);
}
}
// Test the generic allocation functions that can handle arbitrary sizes and
// reallocing etc.
TEST_P(PartitionAllocTest, Alloc) {
void* ptr = allocator.root()->Alloc(1, type_name);
EXPECT_TRUE(ptr);
allocator.root()->Free(ptr);
ptr = allocator.root()->Alloc(kMaxBucketed + 1, type_name);
EXPECT_TRUE(ptr);
allocator.root()->Free(ptr);
// To make both alloc(x + 1) and alloc(x + kSmallestBucket) to allocate from
// the same bucket, partition_alloc::internal::base::bits::AlignUp(1 + x +
// ExtraAllocSize(allocator), kAlignment)
// == partition_alloc::internal::base::bits::AlignUp(kSmallestBucket + x +
// ExtraAllocSize(allocator), kAlignment), because slot_size is multiples of
// kAlignment. So (x + ExtraAllocSize(allocator)) must be multiples of
// kAlignment. x =
// partition_alloc::internal::base::bits::AlignUp(ExtraAllocSize(allocator),
// kAlignment) - ExtraAllocSize(allocator);
size_t base_size = partition_alloc::internal::base::bits::AlignUp(
ExtraAllocSize(allocator), kAlignment) -
ExtraAllocSize(allocator);
ptr = allocator.root()->Alloc(base_size + 1, type_name);
EXPECT_TRUE(ptr);
void* orig_ptr = ptr;
char* char_ptr = static_cast<char*>(ptr);
*char_ptr = 'A';
// Change the size of the realloc, remaining inside the same bucket.
void* new_ptr = allocator.root()->Realloc(ptr, base_size + 2, type_name);
PA_EXPECT_PTR_EQ(ptr, new_ptr);
new_ptr = allocator.root()->Realloc(ptr, base_size + 1, type_name);
PA_EXPECT_PTR_EQ(ptr, new_ptr);
new_ptr =
allocator.root()->Realloc(ptr, base_size + kSmallestBucket, type_name);
PA_EXPECT_PTR_EQ(ptr, new_ptr);
// Change the size of the realloc, switching buckets.
new_ptr = allocator.root()->Realloc(ptr, base_size + kSmallestBucket + 1,
type_name);
PA_EXPECT_PTR_NE(new_ptr, ptr);
// Check that the realloc copied correctly.
char* new_char_ptr = static_cast<char*>(new_ptr);
EXPECT_EQ(*new_char_ptr, 'A');
#if BUILDFLAG(PA_EXPENSIVE_DCHECKS_ARE_ON)
// Subtle: this checks for an old bug where we copied too much from the
// source of the realloc. The condition can be detected by a trashing of
// the uninitialized value in the space of the upsized allocation.
EXPECT_EQ(kUninitializedByte,
static_cast<unsigned char>(*(new_char_ptr + kSmallestBucket)));
#endif
*new_char_ptr = 'B';
// The realloc moved. To check that the old allocation was freed, we can
// do an alloc of the old allocation size and check that the old allocation
// address is at the head of the freelist and reused.
void* reused_ptr = allocator.root()->Alloc(base_size + 1, type_name);
PA_EXPECT_PTR_EQ(reused_ptr, orig_ptr);
allocator.root()->Free(reused_ptr);
// Downsize the realloc.
ptr = new_ptr;
new_ptr = allocator.root()->Realloc(ptr, base_size + 1, type_name);
PA_EXPECT_PTR_EQ(new_ptr, orig_ptr);
new_char_ptr = static_cast<char*>(new_ptr);
EXPECT_EQ(*new_char_ptr, 'B');
*new_char_ptr = 'C';
// Upsize the realloc to outside the partition.
ptr = new_ptr;
new_ptr = allocator.root()->Realloc(ptr, kMaxBucketed + 1, type_name);
PA_EXPECT_PTR_NE(new_ptr, ptr);
new_char_ptr = static_cast<char*>(new_ptr);
EXPECT_EQ(*new_char_ptr, 'C');
*new_char_ptr = 'D';
// Upsize and downsize the realloc, remaining outside the partition.
ptr = new_ptr;
new_ptr = allocator.root()->Realloc(ptr, kMaxBucketed * 10, type_name);
new_char_ptr = static_cast<char*>(new_ptr);
EXPECT_EQ(*new_char_ptr, 'D');
*new_char_ptr = 'E';
ptr = new_ptr;
new_ptr = allocator.root()->Realloc(ptr, kMaxBucketed * 2, type_name);
new_char_ptr = static_cast<char*>(new_ptr);
EXPECT_EQ(*new_char_ptr, 'E');
*new_char_ptr = 'F';
// Downsize the realloc to inside the partition.
ptr = new_ptr;
new_ptr = allocator.root()->Realloc(ptr, base_size + 1, type_name);
PA_EXPECT_PTR_NE(new_ptr, ptr);
PA_EXPECT_PTR_EQ(new_ptr, orig_ptr);
new_char_ptr = static_cast<char*>(new_ptr);
EXPECT_EQ(*new_char_ptr, 'F');
allocator.root()->Free(new_ptr);
}
// Test the generic allocation functions can handle some specific sizes of
// interest.
TEST_P(PartitionAllocTest, AllocSizes) {
{
void* ptr = allocator.root()->Alloc(0, type_name);
EXPECT_TRUE(ptr);
allocator.root()->Free(ptr);
}
{
// PartitionPageSize() is interesting because it results in just one
// allocation per page, which tripped up some corner cases.
const size_t size = PartitionPageSize() - ExtraAllocSize(allocator);
void* ptr = allocator.root()->Alloc(size, type_name);
EXPECT_TRUE(ptr);
void* ptr2 = allocator.root()->Alloc(size, type_name);
EXPECT_TRUE(ptr2);
allocator.root()->Free(ptr);
// Should be freeable at this point.
auto* slot_span =
SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
EXPECT_TRUE(slot_span->in_empty_cache());
allocator.root()->Free(ptr2);
}
{
// Single-slot slot span size.
const size_t size =
PartitionPageSize() * kMaxPartitionPagesPerRegularSlotSpan + 1;
void* ptr = allocator.root()->Alloc(size, type_name);
EXPECT_TRUE(ptr);
memset(ptr, 'A', size);
void* ptr2 = allocator.root()->Alloc(size, type_name);
EXPECT_TRUE(ptr2);
void* ptr3 = allocator.root()->Alloc(size, type_name);
EXPECT_TRUE(ptr3);
void* ptr4 = allocator.root()->Alloc(size, type_name);
EXPECT_TRUE(ptr4);
auto* slot_span = SlotSpanMetadata::FromSlotStart(
allocator.root()->ObjectToSlotStart(ptr));
auto* slot_span2 =
SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr3));
EXPECT_NE(slot_span, slot_span2);
allocator.root()->Free(ptr);
allocator.root()->Free(ptr3);
allocator.root()->Free(ptr2);
// Should be freeable at this point.
EXPECT_TRUE(slot_span->in_empty_cache());
EXPECT_EQ(0u, slot_span->num_allocated_slots);
EXPECT_EQ(0u, slot_span->num_unprovisioned_slots);
void* new_ptr_1 = allocator.root()->Alloc(size, type_name);
PA_EXPECT_PTR_EQ(ptr2, new_ptr_1);
void* new_ptr_2 = allocator.root()->Alloc(size, type_name);
PA_EXPECT_PTR_EQ(ptr3, new_ptr_2);
allocator.root()->Free(new_ptr_1);
allocator.root()->Free(new_ptr_2);
allocator.root()->Free(ptr4);
#if BUILDFLAG(PA_EXPENSIVE_DCHECKS_ARE_ON)
// |SlotSpanMetadata::Free| must poison the slot's contents with
// |kFreedByte|.
EXPECT_EQ(kFreedByte,
*(static_cast<unsigned char*>(new_ptr_1) + (size - 1)));
#endif
}
// Can we allocate a massive (128MB) size?
// Add +1, to test for cookie writing alignment issues.
// Test this only if the device has enough memory or it might fail due
// to OOM.
if (IsLargeMemoryDevice()) {
void* ptr = allocator.root()->Alloc(128 * 1024 * 1024 + 1, type_name);
allocator.root()->Free(ptr);
}
{
// Check a more reasonable, but still direct mapped, size.
// Chop a system page and a byte off to test for rounding errors.
size_t size = 20 * 1024 * 1024;
ASSERT_GT(size, kMaxBucketed);
size -= SystemPageSize();
size -= 1;
void* ptr = allocator.root()->Alloc(size, type_name);
char* char_ptr = static_cast<char*>(ptr);
*(char_ptr + (size - 1)) = 'A';
allocator.root()->Free(ptr);
// Can we free null?
allocator.root()->Free(nullptr);
// Do we correctly get a null for a failed allocation?
EXPECT_EQ(nullptr, allocator.root()->Alloc<AllocFlags::kReturnNull>(
3u * 1024 * 1024 * 1024, type_name));
}
}
// Test that we can fetch the real allocated size after an allocation.
TEST_P(PartitionAllocTest, AllocGetSizeAndStart) {
void* ptr;
size_t requested_size, actual_capacity, predicted_capacity;
// Allocate something small.
requested_size = 511 - ExtraAllocSize(allocator);
predicted_capacity =
allocator.root()->AllocationCapacityFromRequestedSize(requested_size);
ptr = allocator.root()->Alloc(requested_size, type_name);
EXPECT_TRUE(ptr);
uintptr_t slot_start = allocator.root()->ObjectToSlotStart(ptr);
actual_capacity =
allocator.root()->AllocationCapacityFromSlotStart(slot_start);
EXPECT_EQ(predicted_capacity, actual_capacity);
EXPECT_LT(requested_size, actual_capacity);
#if BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT)
if (UseBRPPool()) {
uintptr_t address = UntagPtr(ptr);
for (size_t offset = 0; offset < requested_size; ++offset) {
EXPECT_EQ(PartitionAllocGetSlotStartInBRPPool(address + offset),
slot_start);
}
}
#endif // BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT)
allocator.root()->Free(ptr);
// Allocate a size that should be a perfect match for a bucket, because it
// is an exact power of 2.
requested_size = (256 * 1024) - ExtraAllocSize(allocator);
predicted_capacity =
allocator.root()->AllocationCapacityFromRequestedSize(requested_size);
ptr = allocator.root()->Alloc(requested_size, type_name);
EXPECT_TRUE(ptr);
slot_start = allocator.root()->ObjectToSlotStart(ptr);
actual_capacity =
allocator.root()->AllocationCapacityFromSlotStart(slot_start);
EXPECT_EQ(predicted_capacity, actual_capacity);
EXPECT_EQ(requested_size, actual_capacity);
#if BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT)
if (UseBRPPool()) {
uintptr_t address = UntagPtr(ptr);
for (size_t offset = 0; offset < requested_size; offset += 877) {
EXPECT_EQ(PartitionAllocGetSlotStartInBRPPool(address + offset),
slot_start);
}
}
#endif // BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT)
allocator.root()->Free(ptr);
// Allocate a size that is a system page smaller than a bucket.
// AllocationCapacityFromSlotStart() should return a larger size than we asked
// for now.
size_t num = 64;
while (num * SystemPageSize() >= 1024 * 1024) {
num /= 2;
}
requested_size =
num * SystemPageSize() - SystemPageSize() - ExtraAllocSize(allocator);
predicted_capacity =
allocator.root()->AllocationCapacityFromRequestedSize(requested_size);
ptr = allocator.root()->Alloc(requested_size, type_name);
EXPECT_TRUE(ptr);
slot_start = allocator.root()->ObjectToSlotStart(ptr);
actual_capacity =
allocator.root()->AllocationCapacityFromSlotStart(slot_start);
EXPECT_EQ(predicted_capacity, actual_capacity);
EXPECT_EQ(requested_size + SystemPageSize(), actual_capacity);
#if BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT)
if (UseBRPPool()) {
uintptr_t address = UntagPtr(ptr);
for (size_t offset = 0; offset < requested_size; offset += 4999) {
EXPECT_EQ(PartitionAllocGetSlotStartInBRPPool(address + offset),
slot_start);
}
}
#endif // BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT)
allocator.root()->Free(ptr);
// Allocate the maximum allowed bucketed size.
requested_size = kMaxBucketed - ExtraAllocSize(allocator);
predicted_capacity =
allocator.root()->AllocationCapacityFromRequestedSize(requested_size);
ptr = allocator.root()->Alloc(requested_size, type_name);
EXPECT_TRUE(ptr);
slot_start = allocator.root()->ObjectToSlotStart(ptr);
actual_capacity =
allocator.root()->AllocationCapacityFromSlotStart(slot_start);
EXPECT_EQ(predicted_capacity, actual_capacity);
EXPECT_EQ(requested_size, actual_capacity);
#if BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT)
if (UseBRPPool()) {
uintptr_t address = UntagPtr(ptr);
for (size_t offset = 0; offset < requested_size; offset += 4999) {
EXPECT_EQ(PartitionAllocGetSlotStartInBRPPool(address + offset),
slot_start);
}
}
#endif // BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT)
// Check that we can write at the end of the reported size too.
char* char_ptr = static_cast<char*>(ptr);
*(char_ptr + (actual_capacity - 1)) = 'A';
allocator.root()->Free(ptr);
// Allocate something very large, and uneven.
if (IsLargeMemoryDevice()) {
requested_size = 128 * 1024 * 1024 - 33;
predicted_capacity =
allocator.root()->AllocationCapacityFromRequestedSize(requested_size);
ptr = allocator.root()->Alloc(requested_size, type_name);
EXPECT_TRUE(ptr);
slot_start = allocator.root()->ObjectToSlotStart(ptr);
actual_capacity =
allocator.root()->AllocationCapacityFromSlotStart(slot_start);
EXPECT_EQ(predicted_capacity, actual_capacity);
EXPECT_LT(requested_size, actual_capacity);
#if BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT)
if (UseBRPPool()) {
uintptr_t address = UntagPtr(ptr);
for (size_t offset = 0; offset < requested_size; offset += 16111) {
EXPECT_EQ(PartitionAllocGetSlotStartInBRPPool(address + offset),
slot_start);
}
}
#endif // BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT)
allocator.root()->Free(ptr);
}
// Too large allocation.
requested_size = MaxDirectMapped() + 1;
predicted_capacity =
allocator.root()->AllocationCapacityFromRequestedSize(requested_size);
EXPECT_EQ(requested_size, predicted_capacity);
}
#if PA_CONFIG(HAS_MEMORY_TAGGING)
TEST_P(PartitionAllocTest, MTEProtectsFreedPtr) {
// This test checks that Arm's memory tagging extension (MTE) is correctly
// protecting freed pointers.
base::CPU cpu;
if (!cpu.has_mte()) {
// This test won't pass without MTE support.
GTEST_SKIP();
}
// Create an arbitrarily-sized small allocation.
size_t alloc_size = 64 - ExtraAllocSize(allocator);
uint64_t* ptr1 =
static_cast<uint64_t*>(allocator.root()->Alloc(alloc_size, type_name));
EXPECT_TRUE(ptr1);
// Invalidate the pointer by freeing it.
allocator.root()->Free(ptr1);
// When we immediately reallocate a pointer, we should see the same allocation
// slot but with a different tag (PA_EXPECT_PTR_EQ ignores the MTE tag).
uint64_t* ptr2 =
static_cast<uint64_t*>(allocator.root()->Alloc(alloc_size, type_name));
PA_EXPECT_PTR_EQ(ptr1, ptr2);
// The different tag bits mean that ptr1 is not the same as ptr2.
EXPECT_NE(ptr1, ptr2);
// When we free again, we expect a new tag for that area that's different from
// ptr1 and ptr2.
allocator.root()->Free(ptr2);
uint64_t* ptr3 =
static_cast<uint64_t*>(allocator.root()->Alloc(alloc_size, type_name));
PA_EXPECT_PTR_EQ(ptr2, ptr3);
EXPECT_NE(ptr1, ptr3);
EXPECT_NE(ptr2, ptr3);
// We don't check anything about ptr3, but we do clean it up to avoid DCHECKs.
allocator.root()->Free(ptr3);
}
#endif // PA_CONFIG(HAS_MEMORY_TAGGING)
#if BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT)
TEST_P(PartitionAllocTest, IsPtrWithinSameAlloc) {
if (!UseBRPPool()) {
return;
}
const size_t kMinReasonableTestSize =
partition_alloc::internal::base::bits::AlignUp(
ExtraAllocSize(allocator) + 1, kAlignment);
ASSERT_GT(kMinReasonableTestSize, ExtraAllocSize(allocator));
const size_t kSizes[] = {kMinReasonableTestSize,
256,
SystemPageSize(),
PartitionPageSize(),
MaxRegularSlotSpanSize(),
MaxRegularSlotSpanSize() + 1,
MaxRegularSlotSpanSize() + SystemPageSize(),
MaxRegularSlotSpanSize() + PartitionPageSize(),
kMaxBucketed,
kMaxBucketed + 1,
kMaxBucketed + SystemPageSize(),
kMaxBucketed + PartitionPageSize(),
kSuperPageSize};
#if BUILDFLAG(HAS_64_BIT_POINTERS)
constexpr size_t kFarFarAwayDelta = 512 * kGiB;
#else
constexpr size_t kFarFarAwayDelta = kGiB;
#endif
for (size_t size : kSizes) {
size_t requested_size = size - ExtraAllocSize(allocator);
// For regular slot-span allocations, confirm the size fills the entire
// slot. Otherwise the test would be ineffective, as Partition Alloc has no
// ability to check against the actual allocated size.
// Single-slot slot-spans and direct map don't have that problem.
if (size <= MaxRegularSlotSpanSize()) {
ASSERT_EQ(requested_size,
allocator.root()->AllocationCapacityFromRequestedSize(
requested_size));
}
constexpr size_t kNumRepeats = 3;
void* ptrs[kNumRepeats];
for (void*& ptr : ptrs) {
ptr = allocator.root()->Alloc(requested_size, type_name);
// Double check.
if (size <= MaxRegularSlotSpanSize()) {
uintptr_t slot_start = allocator.root()->ObjectToSlotStart(ptr);
EXPECT_EQ(
requested_size,
allocator.root()->AllocationCapacityFromSlotStart(slot_start));
}
uintptr_t address = UntagPtr(ptr);
EXPECT_EQ(IsPtrWithinSameAlloc(address, address - kFarFarAwayDelta, 0u),
PtrPosWithinAlloc::kFarOOB);
EXPECT_EQ(IsPtrWithinSameAlloc(address, address - kSuperPageSize, 0u),
PtrPosWithinAlloc::kFarOOB);
EXPECT_EQ(IsPtrWithinSameAlloc(address, address - 1, 0u),
PtrPosWithinAlloc::kFarOOB);
EXPECT_EQ(IsPtrWithinSameAlloc(address, address, 0u),
PtrPosWithinAlloc::kInBounds);
EXPECT_EQ(IsPtrWithinSameAlloc(address, address + requested_size / 2, 0u),
PtrPosWithinAlloc::kInBounds);
#if BUILDFLAG(BACKUP_REF_PTR_POISON_OOB_PTR)
EXPECT_EQ(IsPtrWithinSameAlloc(address, address + requested_size - 1, 1u),
PtrPosWithinAlloc::kInBounds);
EXPECT_EQ(IsPtrWithinSameAlloc(address, address + requested_size, 1u),
PtrPosWithinAlloc::kAllocEnd);
EXPECT_EQ(IsPtrWithinSameAlloc(address, address + requested_size - 4, 4u),
PtrPosWithinAlloc::kInBounds);
for (size_t subtrahend = 0; subtrahend < 4; subtrahend++) {
EXPECT_EQ(IsPtrWithinSameAlloc(
address, address + requested_size - subtrahend, 4u),
PtrPosWithinAlloc::kAllocEnd);
}
#else // BUILDFLAG(BACKUP_REF_PTR_POISON_OOB_PTR)
EXPECT_EQ(IsPtrWithinSameAlloc(address, address + requested_size, 0u),
PtrPosWithinAlloc::kInBounds);
#endif
EXPECT_EQ(IsPtrWithinSameAlloc(address, address + requested_size + 1, 0u),
PtrPosWithinAlloc::kFarOOB);
EXPECT_EQ(IsPtrWithinSameAlloc(
address, address + requested_size + kSuperPageSize, 0u),
PtrPosWithinAlloc::kFarOOB);
EXPECT_EQ(IsPtrWithinSameAlloc(
address, address + requested_size + kFarFarAwayDelta, 0u),
PtrPosWithinAlloc::kFarOOB);
EXPECT_EQ(
IsPtrWithinSameAlloc(address + requested_size,
address + requested_size + kFarFarAwayDelta, 0u),
PtrPosWithinAlloc::kFarOOB);
EXPECT_EQ(
IsPtrWithinSameAlloc(address + requested_size,
address + requested_size + kSuperPageSize, 0u),
PtrPosWithinAlloc::kFarOOB);
EXPECT_EQ(IsPtrWithinSameAlloc(address + requested_size,
address + requested_size + 1, 0u),
PtrPosWithinAlloc::kFarOOB);
#if BUILDFLAG(BACKUP_REF_PTR_POISON_OOB_PTR)
EXPECT_EQ(IsPtrWithinSameAlloc(address + requested_size - 1,
address + requested_size - 1, 1u),
PtrPosWithinAlloc::kInBounds);
EXPECT_EQ(IsPtrWithinSameAlloc(address + requested_size - 1,
address + requested_size, 1u),
PtrPosWithinAlloc::kAllocEnd);
EXPECT_EQ(IsPtrWithinSameAlloc(address + requested_size,
address + requested_size, 1u),
PtrPosWithinAlloc::kAllocEnd);
EXPECT_EQ(IsPtrWithinSameAlloc(address + requested_size - 4,
address + requested_size - 4, 4u),
PtrPosWithinAlloc::kInBounds);
for (size_t addend = 1; addend < 4; addend++) {
EXPECT_EQ(
IsPtrWithinSameAlloc(address + requested_size - 4,
address + requested_size - 4 + addend, 4u),
PtrPosWithinAlloc::kAllocEnd);
}
#else // BUILDFLAG(BACKUP_REF_PTR_POISON_OOB_PTR)
EXPECT_EQ(IsPtrWithinSameAlloc(address + requested_size,
address + requested_size, 0u),
PtrPosWithinAlloc::kInBounds);
#endif
EXPECT_EQ(IsPtrWithinSameAlloc(
address + requested_size,
address + requested_size - (requested_size / 2), 0u),
PtrPosWithinAlloc::kInBounds);
EXPECT_EQ(IsPtrWithinSameAlloc(address + requested_size, address, 0u),
PtrPosWithinAlloc::kInBounds);
EXPECT_EQ(IsPtrWithinSameAlloc(address + requested_size, address - 1, 0u),
PtrPosWithinAlloc::kFarOOB);
EXPECT_EQ(IsPtrWithinSameAlloc(address + requested_size,
address - kSuperPageSize, 0u),
PtrPosWithinAlloc::kFarOOB);
EXPECT_EQ(IsPtrWithinSameAlloc(address + requested_size,
address - kFarFarAwayDelta, 0u),
PtrPosWithinAlloc::kFarOOB);
}
for (void* ptr : ptrs) {
allocator.root()->Free(ptr);
}
}
}
TEST_P(PartitionAllocTest, GetSlotStartMultiplePages) {
if (!UseBRPPool()) {
return;
}
auto* root = allocator.root();
// Find the smallest bucket with multiple PartitionPages. When searching for
// a bucket here, we need to check two conditions:
// (1) The bucket is used in our current bucket distribution.
// (2) The bucket is large enough that our requested size (see below) will be
// non-zero.
size_t real_size = 0;
for (const auto& bucket : root->buckets) {
if ((root->buckets + SizeToIndex(bucket.slot_size))->slot_size !=
bucket.slot_size) {
continue;
}
if (bucket.slot_size <= ExtraAllocSize(allocator)) {
continue;
}
if (bucket.num_system_pages_per_slot_span >
NumSystemPagesPerPartitionPage()) {
real_size = bucket.slot_size;
break;
}
}
// Make sure that we've managed to find an appropriate bucket.
ASSERT_GT(real_size, 0u);
const size_t requested_size = real_size - ExtraAllocSize(allocator);
// Double check we don't end up with 0 or negative size.
EXPECT_GT(requested_size, 0u);
EXPECT_LE(requested_size, real_size);
const auto* bucket = allocator.root()->buckets + SizeToIndex(real_size);
EXPECT_EQ(bucket->slot_size, real_size);
// Make sure the test is testing multiple partition pages case.
EXPECT_GT(bucket->num_system_pages_per_slot_span,
PartitionPageSize() / SystemPageSize());
size_t num_slots =
(bucket->num_system_pages_per_slot_span * SystemPageSize()) / real_size;
std::vector<void*> ptrs;
for (size_t i = 0; i < num_slots; ++i) {
ptrs.push_back(allocator.root()->Alloc(requested_size, type_name));
}
for (void* ptr : ptrs) {
uintptr_t address = UntagPtr(ptr);
uintptr_t slot_start = allocator.root()->ObjectToSlotStart(ptr);
EXPECT_EQ(allocator.root()->AllocationCapacityFromSlotStart(slot_start),
requested_size);
for (size_t offset = 0; offset < requested_size; offset += 13) {
EXPECT_EQ(PartitionAllocGetSlotStartInBRPPool(address + offset),
slot_start);
}
allocator.root()->Free(ptr);
}
}
#endif // BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT)
// Test the realloc() contract.
TEST_P(PartitionAllocTest, Realloc) {
// realloc(0, size) should be equivalent to malloc().
void* ptr = allocator.root()->Realloc(nullptr, kTestAllocSize, type_name);
memset(ptr, 'A', kTestAllocSize);
auto* slot_span =
SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
// realloc(ptr, 0) should be equivalent to free().
void* ptr2 = allocator.root()->Realloc(ptr, 0, type_name);
EXPECT_EQ(nullptr, ptr2);
EXPECT_EQ(allocator.root()->ObjectToSlotStart(ptr),
UntagPtr(slot_span->get_freelist_head()));
// Test that growing an allocation with realloc() copies everything from the
// old allocation.
size_t size = SystemPageSize() - ExtraAllocSize(allocator);
// Confirm size fills the entire slot.
ASSERT_EQ(size, allocator.root()->AllocationCapacityFromRequestedSize(size));
ptr = allocator.root()->Alloc(size, type_name);
memset(ptr, 'A', size);
ptr2 = allocator.root()->Realloc(ptr, size + 1, type_name);
PA_EXPECT_PTR_NE(ptr, ptr2);
char* char_ptr2 = static_cast<char*>(ptr2);
EXPECT_EQ('A', char_ptr2[0]);
EXPECT_EQ('A', char_ptr2[size - 1]);
#if BUILDFLAG(PA_EXPENSIVE_DCHECKS_ARE_ON)
EXPECT_EQ(kUninitializedByte, static_cast<unsigned char>(char_ptr2[size]));
#endif
// Test that shrinking an allocation with realloc() also copies everything
// from the old allocation. Use |size - 1| to test what happens to the extra
// space before the cookie.
ptr = allocator.root()->Realloc(ptr2, size - 1, type_name);
PA_EXPECT_PTR_NE(ptr2, ptr);
char* char_ptr = static_cast<char*>(ptr);
EXPECT_EQ('A', char_ptr[0]);
EXPECT_EQ('A', char_ptr[size - 2]);
#if BUILDFLAG(PA_EXPENSIVE_DCHECKS_ARE_ON)
EXPECT_EQ(kUninitializedByte, static_cast<unsigned char>(char_ptr[size - 1]));
#endif
allocator.root()->Free(ptr);
// Single-slot slot spans...
// Test that growing an allocation with realloc() copies everything from the
// old allocation.
size = MaxRegularSlotSpanSize() + 1;
ASSERT_LE(2 * size, kMaxBucketed); // should be in single-slot span range
// Confirm size doesn't fill the entire slot.
ASSERT_LT(size, allocator.root()->AllocationCapacityFromRequestedSize(size));
ptr = allocator.root()->Alloc(size, type_name);
memset(ptr, 'A', size);
ptr2 = allocator.root()->Realloc(ptr, size * 2, type_name);
PA_EXPECT_PTR_NE(ptr, ptr2);
char_ptr2 = static_cast<char*>(ptr2);
EXPECT_EQ('A', char_ptr2[0]);
EXPECT_EQ('A', char_ptr2[size - 1]);
#if BUILDFLAG(PA_EXPENSIVE_DCHECKS_ARE_ON)
EXPECT_EQ(kUninitializedByte, static_cast<unsigned char>(char_ptr2[size]));
#endif
allocator.root()->Free(ptr2);
// Test that shrinking an allocation with realloc() also copies everything
// from the old allocation.
size = 2 * (MaxRegularSlotSpanSize() + 1);
ASSERT_GT(size / 2, MaxRegularSlotSpanSize()); // in single-slot span range
ptr = allocator.root()->Alloc(size, type_name);
memset(ptr, 'A', size);
ptr2 = allocator.root()->Realloc(ptr2, size / 2, type_name);
PA_EXPECT_PTR_NE(ptr, ptr2);
char_ptr2 = static_cast<char*>(ptr2);
EXPECT_EQ('A', char_ptr2[0]);
EXPECT_EQ('A', char_ptr2[size / 2 - 1]);
#if BUILDFLAG(PA_DCHECK_IS_ON)
// For single-slot slot spans, the cookie is always placed immediately after
// the allocation.
EXPECT_EQ(kCookieValue[0], static_cast<unsigned char>(char_ptr2[size / 2]));
#endif
allocator.root()->Free(ptr2);
// Test that shrinking a direct mapped allocation happens in-place.
// Pick a large size so that Realloc doesn't think it's worthwhile to
// downsize even if one less super page is used (due to high granularity on
// 64-bit systems).
size = 10 * kSuperPageSize + SystemPageSize() - 42;
ASSERT_GT(size - 32 * SystemPageSize(), kMaxBucketed);
ptr = allocator.root()->Alloc(size, type_name);
uintptr_t slot_start = allocator.root()->ObjectToSlotStart(ptr);
size_t actual_capacity =
allocator.root()->AllocationCapacityFromSlotStart(slot_start);
ptr2 = allocator.root()->Realloc(ptr, size - SystemPageSize(), type_name);
uintptr_t slot_start2 = allocator.root()->ObjectToSlotStart(ptr2);
EXPECT_EQ(slot_start, slot_start2);
EXPECT_EQ(actual_capacity - SystemPageSize(),
allocator.root()->AllocationCapacityFromSlotStart(slot_start2));
void* ptr3 =
allocator.root()->Realloc(ptr2, size - 32 * SystemPageSize(), type_name);
uintptr_t slot_start3 = allocator.root()->ObjectToSlotStart(ptr3);
EXPECT_EQ(slot_start2, slot_start3);
EXPECT_EQ(actual_capacity - 32 * SystemPageSize(),
allocator.root()->AllocationCapacityFromSlotStart(slot_start3));
// Test that a previously in-place shrunk direct mapped allocation can be
// expanded up again up to its original size.
ptr = allocator.root()->Realloc(ptr3, size, type_name);
slot_start = allocator.root()->ObjectToSlotStart(ptr);
EXPECT_EQ(slot_start3, slot_start);
EXPECT_EQ(actual_capacity,
allocator.root()->AllocationCapacityFromSlotStart(slot_start));
// Test that the allocation can be expanded in place up to its capacity.
ptr2 = allocator.root()->Realloc(ptr, actual_capacity, type_name);
slot_start2 = allocator.root()->ObjectToSlotStart(ptr2);
EXPECT_EQ(slot_start, slot_start2);
EXPECT_EQ(actual_capacity,
allocator.root()->AllocationCapacityFromSlotStart(slot_start2));
// Test that a direct mapped allocation is performed not in-place when the
// new size is small enough.
ptr3 = allocator.root()->Realloc(ptr2, SystemPageSize(), type_name);
slot_start3 = allocator.root()->ObjectToSlotStart(ptr3);
EXPECT_NE(slot_start, slot_start3);
allocator.root()->Free(ptr3);
}
TEST_P(PartitionAllocTest, ReallocDirectMapAligned) {
size_t alignments[] = {
PartitionPageSize(),
2 * PartitionPageSize(),
kMaxSupportedAlignment / 2,
kMaxSupportedAlignment,
};
for (size_t alignment : alignments) {
// Test that shrinking a direct mapped allocation happens in-place.
// Pick a large size so that Realloc doesn't think it's worthwhile to
// downsize even if one less super page is used (due to high granularity on
// 64-bit systems), even if the alignment padding is taken out.
size_t size = 10 * kSuperPageSize + SystemPageSize() - 42;
ASSERT_GT(size, kMaxBucketed);
void* ptr =
allocator.root()->AllocInternalForTesting(size, alignment, type_name);
uintptr_t slot_start = allocator.root()->ObjectToSlotStart(ptr);
size_t actual_capacity =
allocator.root()->AllocationCapacityFromSlotStart(slot_start);
void* ptr2 =
allocator.root()->Realloc(ptr, size - SystemPageSize(), type_name);
uintptr_t slot_start2 = allocator.root()->ObjectToSlotStart(ptr2);
EXPECT_EQ(slot_start, slot_start2);
EXPECT_EQ(actual_capacity - SystemPageSize(),
allocator.root()->AllocationCapacityFromSlotStart(slot_start2));
void* ptr3 = allocator.root()->Realloc(ptr2, size - 32 * SystemPageSize(),
type_name);
uintptr_t slot_start3 = allocator.root()->ObjectToSlotStart(ptr3);
EXPECT_EQ(slot_start2, slot_start3);
EXPECT_EQ(actual_capacity - 32 * SystemPageSize(),
allocator.root()->AllocationCapacityFromSlotStart(slot_start3));
// Test that a previously in-place shrunk direct mapped allocation can be
// expanded up again up to its original size.
ptr = allocator.root()->Realloc(ptr3, size, type_name);
slot_start = allocator.root()->ObjectToSlotStart(ptr);
EXPECT_EQ(slot_start3, slot_start);
EXPECT_EQ(actual_capacity,
allocator.root()->AllocationCapacityFromSlotStart(slot_start));
// Test that the allocation can be expanded in place up to its capacity.
ptr2 = allocator.root()->Realloc(ptr, actual_capacity, type_name);
slot_start2 = allocator.root()->ObjectToSlotStart(ptr2);
EXPECT_EQ(slot_start, slot_start2);
EXPECT_EQ(actual_capacity,
allocator.root()->AllocationCapacityFromSlotStart(slot_start2));
// Test that a direct mapped allocation is performed not in-place when the
// new size is small enough.
ptr3 = allocator.root()->Realloc(ptr2, SystemPageSize(), type_name);
slot_start3 = allocator.root()->ObjectToSlotStart(ptr3);
EXPECT_NE(slot_start2, slot_start3);
allocator.root()->Free(ptr3);
}
}
TEST_P(PartitionAllocTest, ReallocDirectMapAlignedRelocate) {
// Pick size such that the alignment will put it cross the super page
// boundary.
size_t size = 2 * kSuperPageSize - kMaxSupportedAlignment + SystemPageSize();
ASSERT_GT(size, kMaxBucketed);
void* ptr = allocator.root()->AllocInternalForTesting(
size, kMaxSupportedAlignment, type_name);
// Reallocating with the same size will actually relocate, because without a
// need for alignment we can downsize the reservation significantly.
void* ptr2 = allocator.root()->Realloc(ptr, size, type_name);
PA_EXPECT_PTR_NE(ptr, ptr2);
allocator.root()->Free(ptr2);
// Again pick size such that the alignment will put it cross the super page
// boundary, but this time make it so large that Realloc doesn't fing it worth
// shrinking.
size = 10 * kSuperPageSize - kMaxSupportedAlignment + SystemPageSize();
ASSERT_GT(size, kMaxBucketed);
ptr = allocator.root()->AllocInternalForTesting(size, kMaxSupportedAlignment,
type_name);
ptr2 = allocator.root()->Realloc(ptr, size, type_name);
EXPECT_EQ(ptr, ptr2);
allocator.root()->Free(ptr2);
}
// Tests the handing out of freelists for partial slot spans.
TEST_P(PartitionAllocTest, PartialPageFreelists) {
size_t big_size = SystemPageSize() - ExtraAllocSize(allocator);
size_t bucket_index = SizeToIndex(big_size + ExtraAllocSize(allocator));
PartitionRoot::Bucket* bucket = &allocator.root()->buckets[bucket_index];
EXPECT_EQ(nullptr, bucket->empty_slot_spans_head);
void* ptr = allocator.root()->Alloc(big_size, type_name);
EXPECT_TRUE(ptr);
auto* slot_span =
SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
size_t total_slots =
(slot_span->bucket->num_system_pages_per_slot_span * SystemPageSize()) /
(big_size + ExtraAllocSize(allocator));
EXPECT_EQ(4u, total_slots);
// The freelist should have one entry, because we were able to exactly fit
// one object slot and one freelist pointer (the null that the head points
// to) into a system page.
EXPECT_FALSE(slot_span->get_freelist_head());
EXPECT_EQ(1u, slot_span->num_allocated_slots);
EXPECT_EQ(3u, slot_span->num_unprovisioned_slots);
void* ptr2 = allocator.root()->Alloc(big_size, type_name);
EXPECT_TRUE(ptr2);
EXPECT_FALSE(slot_span->get_freelist_head());
EXPECT_EQ(2u, slot_span->num_allocated_slots);
EXPECT_EQ(2u, slot_span->num_unprovisioned_slots);
void* ptr3 = allocator.root()->Alloc(big_size, type_name);
EXPECT_TRUE(ptr3);
EXPECT_FALSE(slot_span->get_freelist_head());
EXPECT_EQ(3u, slot_span->num_allocated_slots);
EXPECT_EQ(1u, slot_span->num_unprovisioned_slots);
void* ptr4 = allocator.root()->Alloc(big_size, type_name);
EXPECT_TRUE(ptr4);
EXPECT_FALSE(slot_span->get_freelist_head());
EXPECT_EQ(4u, slot_span->num_allocated_slots);
EXPECT_EQ(0u, slot_span->num_unprovisioned_slots);
void* ptr5 = allocator.root()->Alloc(big_size, type_name);
EXPECT_TRUE(ptr5);
auto* slot_span2 =
SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr5));
EXPECT_EQ(1u, slot_span2->num_allocated_slots);
// Churn things a little whilst there's a partial slot span freelist.
allocator.root()->Free(ptr);
ptr = allocator.root()->Alloc(big_size, type_name);
void* ptr6 = allocator.root()->Alloc(big_size, type_name);
allocator.root()->Free(ptr);
allocator.root()->Free(ptr2);
allocator.root()->Free(ptr3);
allocator.root()->Free(ptr4);
allocator.root()->Free(ptr5);
allocator.root()->Free(ptr6);
EXPECT_TRUE(slot_span->in_empty_cache());
EXPECT_TRUE(slot_span2->in_empty_cache());
EXPECT_TRUE(slot_span2->get_freelist_head());
EXPECT_EQ(0u, slot_span2->num_allocated_slots);
// Size that's just above half a page.
size_t non_dividing_size =
SystemPageSize() / 2 + 1 - ExtraAllocSize(allocator);
bucket_index = SizeToIndex(non_dividing_size + ExtraAllocSize(allocator));
bucket = &allocator.root()->buckets[bucket_index];
EXPECT_EQ(nullptr, bucket->empty_slot_spans_head);
ptr = allocator.root()->Alloc(non_dividing_size, type_name);
EXPECT_TRUE(ptr);
slot_span = SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
total_slots =
(slot_span->bucket->num_system_pages_per_slot_span * SystemPageSize()) /
bucket->slot_size;
EXPECT_FALSE(slot_span->get_freelist_head());
EXPECT_EQ(1u, slot_span->num_allocated_slots);
EXPECT_EQ(total_slots - 1, slot_span->num_unprovisioned_slots);
ptr2 = allocator.root()->Alloc(non_dividing_size, type_name);
EXPECT_TRUE(ptr2);
EXPECT_TRUE(slot_span->get_freelist_head());
EXPECT_EQ(2u, slot_span->num_allocated_slots);
// 2 slots got provisioned: the first one fills the rest of the first (already
// provision page) and exceeds it by just a tad, thus leading to provisioning
// a new page, and the second one fully fits within that new page.
EXPECT_EQ(total_slots - 3, slot_span->num_unprovisioned_slots);
ptr3 = allocator.root()->Alloc(non_dividing_size, type_name);
EXPECT_TRUE(ptr3);
EXPECT_FALSE(slot_span->get_freelist_head());
EXPECT_EQ(3u, slot_span->num_allocated_slots);
EXPECT_EQ(total_slots - 3, slot_span->num_unprovisioned_slots);
allocator.root()->Free(ptr);
allocator.root()->Free(ptr2);
allocator.root()->Free(ptr3);
EXPECT_TRUE(slot_span->in_empty_cache());
EXPECT_TRUE(slot_span2->get_freelist_head());
EXPECT_EQ(0u, slot_span2->num_allocated_slots);
// And test a couple of sizes that do not cross SystemPageSize() with a
// single allocation.
size_t medium_size = (SystemPageSize() / 2) - ExtraAllocSize(allocator);
bucket_index = SizeToIndex(medium_size + ExtraAllocSize(allocator));
bucket = &allocator.root()->buckets[bucket_index];
EXPECT_EQ(nullptr, bucket->empty_slot_spans_head);
ptr = allocator.root()->Alloc(medium_size, type_name);
EXPECT_TRUE(ptr);
slot_span = SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
EXPECT_EQ(1u, slot_span->num_allocated_slots);
total_slots =
(slot_span->bucket->num_system_pages_per_slot_span * SystemPageSize()) /
(medium_size + ExtraAllocSize(allocator));
size_t first_slot_span_slots =
SystemPageSize() / (medium_size + ExtraAllocSize(allocator));
EXPECT_EQ(2u, first_slot_span_slots);
EXPECT_EQ(total_slots - first_slot_span_slots,
slot_span->num_unprovisioned_slots);
allocator.root()->Free(ptr);
size_t small_size = (SystemPageSize() / 4) - ExtraAllocSize(allocator);
bucket_index = SizeToIndex(small_size + ExtraAllocSize(allocator));
bucket = &allocator.root()->buckets[bucket_index];
EXPECT_EQ(nullptr, bucket->empty_slot_spans_head);
ptr = allocator.root()->Alloc(small_size, type_name);
EXPECT_TRUE(ptr);
slot_span = SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
EXPECT_EQ(1u, slot_span->num_allocated_slots);
total_slots =
(slot_span->bucket->num_system_pages_per_slot_span * SystemPageSize()) /
(small_size + ExtraAllocSize(allocator));
first_slot_span_slots =
SystemPageSize() / (small_size + ExtraAllocSize(allocator));
EXPECT_EQ(total_slots - first_slot_span_slots,
slot_span->num_unprovisioned_slots);
allocator.root()->Free(ptr);
EXPECT_TRUE(slot_span->get_freelist_head());
EXPECT_EQ(0u, slot_span->num_allocated_slots);
ASSERT_LT(ExtraAllocSize(allocator), 64u);
size_t very_small_size = (ExtraAllocSize(allocator) <= 32)
? (32 - ExtraAllocSize(allocator))
: (64 - ExtraAllocSize(allocator));
size_t very_small_adjusted_size =
allocator.root()->AdjustSize0IfNeeded(very_small_size);
bucket_index =
SizeToIndex(very_small_adjusted_size + ExtraAllocSize(allocator));
bucket = &allocator.root()->buckets[bucket_index];
EXPECT_EQ(nullptr, bucket->empty_slot_spans_head);
ptr = allocator.root()->Alloc(very_small_size, type_name);
EXPECT_TRUE(ptr);
slot_span = SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
EXPECT_EQ(1u, slot_span->num_allocated_slots);
size_t very_small_actual_size = allocator.root()->GetUsableSize(ptr);
total_slots =
(slot_span->bucket->num_system_pages_per_slot_span * SystemPageSize()) /
(very_small_actual_size + ExtraAllocSize(allocator));
first_slot_span_slots =
SystemPageSize() / (very_small_actual_size + ExtraAllocSize(allocator));
EXPECT_EQ(total_slots - first_slot_span_slots,
slot_span->num_unprovisioned_slots);
allocator.root()->Free(ptr);
EXPECT_TRUE(slot_span->get_freelist_head());
EXPECT_EQ(0u, slot_span->num_allocated_slots);
// And try an allocation size (against the generic allocator) that is
// larger than a system page.
size_t page_and_a_half_size =
(SystemPageSize() + (SystemPageSize() / 2)) - ExtraAllocSize(allocator);
ptr = allocator.root()->Alloc(page_and_a_half_size, type_name);
EXPECT_TRUE(ptr);
slot_span = SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
EXPECT_EQ(1u, slot_span->num_allocated_slots);
// Only the first slot was provisioned, and that's the one that was just
// allocated so the free list is empty.
EXPECT_TRUE(!slot_span->get_freelist_head());
total_slots =
(slot_span->bucket->num_system_pages_per_slot_span * SystemPageSize()) /
(page_and_a_half_size + ExtraAllocSize(allocator));
EXPECT_EQ(total_slots - 1, slot_span->num_unprovisioned_slots);
ptr2 = allocator.root()->Alloc(page_and_a_half_size, type_name);
EXPECT_TRUE(ptr);
slot_span = SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
EXPECT_EQ(2u, slot_span->num_allocated_slots);
// As above, only one slot was provisioned.
EXPECT_TRUE(!slot_span->get_freelist_head());
EXPECT_EQ(total_slots - 2, slot_span->num_unprovisioned_slots);
allocator.root()->Free(ptr);
allocator.root()->Free(ptr2);
// And then make sure than exactly the page size only faults one page.
size_t page_size = SystemPageSize() - ExtraAllocSize(allocator);
ptr = allocator.root()->Alloc(page_size, type_name);
EXPECT_TRUE(ptr);
slot_span = SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
EXPECT_EQ(1u, slot_span->num_allocated_slots);
EXPECT_TRUE(slot_span->get_freelist_head());
total_slots =
(slot_span->bucket->num_system_pages_per_slot_span * SystemPageSize()) /
(page_size + ExtraAllocSize(allocator));
EXPECT_EQ(total_slots - 2, slot_span->num_unprovisioned_slots);
allocator.root()->Free(ptr);
}
// Test some of the fragmentation-resistant properties of the allocator.
TEST_P(PartitionAllocTest, SlotSpanRefilling) {
PartitionRoot::Bucket* bucket =
&allocator.root()->buckets[test_bucket_index_];
// Grab two full slot spans and a non-full slot span.
auto* slot_span1 = GetFullSlotSpan(kTestAllocSize);
auto* slot_span2 = GetFullSlotSpan(kTestAllocSize);
void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
EXPECT_TRUE(ptr);
EXPECT_NE(slot_span1, bucket->active_slot_spans_head);
EXPECT_NE(slot_span2, bucket->active_slot_spans_head);
auto* slot_span =
SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
EXPECT_EQ(1u, slot_span->num_allocated_slots);
// Work out a pointer into slot_span2 and free it; and then slot_span1 and
// free it.
void* ptr2 = allocator.root()->SlotStartToObject(
SlotSpan::ToSlotSpanStart(slot_span1));
allocator.root()->Free(ptr2);
ptr2 = allocator.root()->SlotStartToObject(
SlotSpan::ToSlotSpanStart(slot_span2));
allocator.root()->Free(ptr2);
// If we perform two allocations from the same bucket now, we expect to
// refill both the nearly full slot spans.
std::ignore = allocator.root()->Alloc(kTestAllocSize, type_name);
std::ignore = allocator.root()->Alloc(kTestAllocSize, type_name);
EXPECT_EQ(1u, slot_span->num_allocated_slots);
FreeFullSlotSpan(allocator.root(), slot_span2);
FreeFullSlotSpan(allocator.root(), slot_span1);
allocator.root()->Free(ptr);
}
// Basic tests to ensure that allocations work for partial page buckets.
TEST_P(PartitionAllocTest, PartialPages) {
// Find a size that is backed by a partial partition page.
size_t size = sizeof(void*);
size_t bucket_index;
PartitionRoot::Bucket* bucket = nullptr;
constexpr size_t kMaxSize = 4000u;
while (size < kMaxSize) {
bucket_index = SizeToIndex(size + ExtraAllocSize(allocator));
bucket = &allocator.root()->buckets[bucket_index];
if (bucket->num_system_pages_per_slot_span %
NumSystemPagesPerPartitionPage()) {
break;
}
size += sizeof(void*);
}
EXPECT_LT(size, kMaxSize);
auto* slot_span1 = GetFullSlotSpan(size);
auto* slot_span2 = GetFullSlotSpan(size);
FreeFullSlotSpan(allocator.root(), slot_span2);
FreeFullSlotSpan(allocator.root(), slot_span1);
}
// Test correct handling if our mapping collides with another.
TEST_P(PartitionAllocTest, MappingCollision) {
size_t num_pages_per_slot_span = GetNumPagesPerSlotSpan(kTestAllocSize);
// The -2 is because the first and last partition pages in a super page are
// guard pages. We also discount the partition pages used for the tag bitmap.
size_t num_slot_span_needed =
(NumPartitionPagesPerSuperPage() - 2 -
partition_alloc::internal::NumPartitionPagesPerFreeSlotBitmap()) /
num_pages_per_slot_span;
size_t num_partition_pages_needed =
num_slot_span_needed * num_pages_per_slot_span;
auto first_super_page_pages =
std::make_unique<SlotSpan*[]>(num_partition_pages_needed);
auto second_super_page_pages =
std::make_unique<SlotSpan*[]>(num_partition_pages_needed);
size_t i;
for (i = 0; i < num_partition_pages_needed; ++i) {
first_super_page_pages[i] = GetFullSlotSpan(kTestAllocSize);
}
uintptr_t slot_span_start =
SlotSpan::ToSlotSpanStart(first_super_page_pages[0]);
EXPECT_EQ(PartitionPageSize() +
partition_alloc::internal::ReservedFreeSlotBitmapSize(),
slot_span_start & kSuperPageOffsetMask);
uintptr_t super_page =
slot_span_start - PartitionPageSize() -
partition_alloc::internal::ReservedFreeSlotBitmapSize();
// Map a single system page either side of the mapping for our allocations,
// with the goal of tripping up alignment of the next mapping.
uintptr_t map1 =
AllocPages(super_page - PageAllocationGranularity(),
PageAllocationGranularity(), PageAllocationGranularity(),
PageAccessibilityConfiguration(
PageAccessibilityConfiguration::kInaccessible),
PageTag::kPartitionAlloc);
EXPECT_TRUE(map1);
uintptr_t map2 =
AllocPages(super_page + kSuperPageSize, PageAllocationGranularity(),
PageAllocationGranularity(),
PageAccessibilityConfiguration(
PageAccessibilityConfiguration::kInaccessible),
PageTag::kPartitionAlloc);
EXPECT_TRUE(map2);
for (i = 0; i < num_partition_pages_needed; ++i) {
second_super_page_pages[i] = GetFullSlotSpan(kTestAllocSize);
}
FreePages(map1, PageAllocationGranularity());
FreePages(map2, PageAllocationGranularity());
super_page = SlotSpan::ToSlotSpanStart(second_super_page_pages[0]);
EXPECT_EQ(PartitionPageSize() +
partition_alloc::internal::ReservedFreeSlotBitmapSize(),
super_page & kSuperPageOffsetMask);
super_page -= PartitionPageSize() +
partition_alloc::internal::ReservedFreeSlotBitmapSize();
// Map a single system page either side of the mapping for our allocations,
// with the goal of tripping up alignment of the next mapping.
map1 = AllocPages(super_page - PageAllocationGranularity(),
PageAllocationGranularity(), PageAllocationGranularity(),
PageAccessibilityConfiguration(
PageAccessibilityConfiguration::kReadWriteTagged),
PageTag::kPartitionAlloc);
EXPECT_TRUE(map1);
map2 = AllocPages(super_page + kSuperPageSize, PageAllocationGranularity(),
PageAllocationGranularity(),
PageAccessibilityConfiguration(
PageAccessibilityConfiguration::kReadWriteTagged),
PageTag::kPartitionAlloc);
EXPECT_TRUE(map2);
EXPECT_TRUE(TrySetSystemPagesAccess(
map1, PageAllocationGranularity(),
PageAccessibilityConfiguration(
PageAccessibilityConfiguration::kInaccessible)));
EXPECT_TRUE(TrySetSystemPagesAccess(
map2, PageAllocationGranularity(),
PageAccessibilityConfiguration(
PageAccessibilityConfiguration::kInaccessible)));
auto* slot_span_in_third_super_page = GetFullSlotSpan(kTestAllocSize);
FreePages(map1, PageAllocationGranularity());
FreePages(map2, PageAllocationGranularity());
EXPECT_EQ(0u, SlotSpan::ToSlotSpanStart(slot_span_in_third_super_page) &
PartitionPageOffsetMask());
// And make sure we really did get a page in a new superpage.
EXPECT_NE(
SlotSpan::ToSlotSpanStart(first_super_page_pages[0]) & kSuperPageBaseMask,
SlotSpan::ToSlotSpanStart(slot_span_in_third_super_page) &
kSuperPageBaseMask);
EXPECT_NE(SlotSpan::ToSlotSpanStart(second_super_page_pages[0]) &
kSuperPageBaseMask,
SlotSpan::ToSlotSpanStart(slot_span_in_third_super_page) &
kSuperPageBaseMask);
FreeFullSlotSpan(allocator.root(), slot_span_in_third_super_page);
for (i = 0; i < num_partition_pages_needed; ++i) {
FreeFullSlotSpan(allocator.root(), first_super_page_pages[i]);
FreeFullSlotSpan(allocator.root(), second_super_page_pages[i]);
}
}
// Tests that slot spans in the free slot span cache do get freed as
// appropriate.
TEST_P(PartitionAllocTest, FreeCache) {
EXPECT_EQ(0U, allocator.root()->get_total_size_of_committed_pages());
size_t big_size = 1000 - ExtraAllocSize(allocator);
size_t bucket_index = SizeToIndex(big_size + ExtraAllocSize(allocator));
PartitionBucket* bucket = &allocator.root()->buckets[bucket_index];
void* ptr = allocator.root()->Alloc(big_size, type_name);
EXPECT_TRUE(ptr);
auto* slot_span =
SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
EXPECT_EQ(nullptr, bucket->empty_slot_spans_head);
EXPECT_EQ(1u, slot_span->num_allocated_slots);
// Lazy commit commits only needed pages.
size_t expected_committed_size =
kUseLazyCommit ? SystemPageSize() : PartitionPageSize();
EXPECT_EQ(expected_committed_size,
allocator.root()->get_total_size_of_committed_pages());
allocator.root()->Free(ptr);
EXPECT_EQ(0u, slot_span->num_allocated_slots);
EXPECT_TRUE(slot_span->in_empty_cache());
EXPECT_TRUE(slot_span->get_freelist_head());
CycleFreeCache(kTestAllocSize);
// Flushing the cache should have really freed the unused slot spans.
EXPECT_FALSE(slot_span->get_freelist_head());
EXPECT_FALSE(slot_span->in_empty_cache());
EXPECT_EQ(0u, slot_span->num_allocated_slots);
size_t num_system_pages_per_slot_span = allocator.root()
->buckets[test_bucket_index_]
.num_system_pages_per_slot_span;
size_t expected_size =
kUseLazyCommit ? SystemPageSize()
: num_system_pages_per_slot_span * SystemPageSize();
EXPECT_EQ(expected_size,
allocator.root()->get_total_size_of_committed_pages());
// Check that an allocation works ok whilst in this state (a free'd slot span
// as the active slot spans head).
ptr = allocator.root()->Alloc(big_size, type_name);
EXPECT_FALSE(bucket->empty_slot_spans_head);
allocator.root()->Free(ptr);
// Also check that a slot span that is bouncing immediately between empty and
// used does not get freed.
for (size_t i = 0; i < kMaxFreeableSpans * 2; ++i) {
ptr = allocator.root()->Alloc(big_size, type_name);
EXPECT_TRUE(slot_span->get_freelist_head());
allocator.root()->Free(ptr);
EXPECT_TRUE(slot_span->get_freelist_head());
}
EXPECT_EQ(expected_committed_size,
allocator.root()->get_total_size_of_committed_pages());
}
// Tests for a bug we had with losing references to free slot spans.
TEST_P(PartitionAllocTest, LostFreeSlotSpansBug) {
size_t size = PartitionPageSize() - ExtraAllocSize(allocator);
void* ptr = allocator.root()->Alloc(size, type_name);
EXPECT_TRUE(ptr);
void* ptr2 = allocator.root()->Alloc(size, type_name);
EXPECT_TRUE(ptr2);
SlotSpanMetadata* slot_span =
SlotSpanMetadata::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
SlotSpanMetadata* slot_span2 = SlotSpanMetadata::FromSlotStart(
allocator.root()->ObjectToSlotStart(ptr2));
PartitionBucket* bucket = slot_span->bucket;
EXPECT_EQ(nullptr, bucket->empty_slot_spans_head);
EXPECT_EQ(1u, slot_span->num_allocated_slots);
EXPECT_EQ(1u, slot_span2->num_allocated_slots);
EXPECT_TRUE(slot_span->is_full());
EXPECT_TRUE(slot_span2->is_full());
// The first span was kicked out from the active list, but the second one
// wasn't.
EXPECT_TRUE(slot_span->marked_full);
EXPECT_FALSE(slot_span2->marked_full);
allocator.root()->Free(ptr);
allocator.root()->Free(ptr2);
EXPECT_TRUE(bucket->empty_slot_spans_head);
EXPECT_TRUE(bucket->empty_slot_spans_head->next_slot_span);
EXPECT_EQ(0u, slot_span->num_allocated_slots);
EXPECT_EQ(0u, slot_span2->num_allocated_slots);
EXPECT_FALSE(slot_span->is_full());
EXPECT_FALSE(slot_span->is_full());
EXPECT_FALSE(slot_span->marked_full);
EXPECT_FALSE(slot_span2->marked_full);
EXPECT_TRUE(slot_span->get_freelist_head());
EXPECT_TRUE(slot_span2->get_freelist_head());
CycleFreeCache(kTestAllocSize);
EXPECT_FALSE(slot_span->get_freelist_head());
EXPECT_FALSE(slot_span2->get_freelist_head());
EXPECT_TRUE(bucket->empty_slot_spans_head);
EXPECT_TRUE(bucket->empty_slot_spans_head->next_slot_span);
EXPECT_EQ(SlotSpanMetadata::get_sentinel_slot_span(),
bucket->active_slot_spans_head);
// At this moment, we have two decommitted slot spans, on the empty list.
ptr = allocator.root()->Alloc(size, type_name);
EXPECT_TRUE(ptr);
allocator.root()->Free(ptr);
EXPECT_EQ(SlotSpanMetadata::get_sentinel_slot_span(),
bucket->active_slot_spans_head);
EXPECT_TRUE(bucket->empty_slot_spans_head);
EXPECT_TRUE(bucket->decommitted_slot_spans_head);
CycleFreeCache(kTestAllocSize);
// We're now set up to trigger a historical bug by scanning over the active
// slot spans list. The current code gets into a different state, but we'll
// keep the test as being an interesting corner case.
ptr = allocator.root()->Alloc(size, type_name);
EXPECT_TRUE(ptr);
allocator.root()->Free(ptr);
EXPECT_TRUE(bucket->is_valid());
EXPECT_TRUE(bucket->empty_slot_spans_head);
EXPECT_TRUE(bucket->decommitted_slot_spans_head);
}
#if defined(PA_HAS_DEATH_TESTS)
// Unit tests that check if an allocation fails in "return null" mode,
// repeating it doesn't crash, and still returns null. The tests need to
// stress memory subsystem limits to do so, hence they try to allocate
// 6 GB of memory, each with a different per-allocation block sizes.
//
// On 64-bit systems we need to restrict the address space to force allocation
// failure, so these tests run only on POSIX systems that provide setrlimit(),
// and use it to limit address space to 6GB.
//
// Disable these tests on Android because, due to the allocation-heavy behavior,
// they tend to get OOM-killed rather than pass.
//
// Disable these test on Windows, since they run slower, so tend to timout and
// cause flake.
#if !BUILDFLAG(IS_WIN) && \
(!defined(ARCH_CPU_64_BITS) || \
(BUILDFLAG(IS_POSIX) && \
!(BUILDFLAG(IS_APPLE) || BUILDFLAG(IS_ANDROID)))) || \
BUILDFLAG(IS_FUCHSIA)
#define MAYBE_RepeatedAllocReturnNullDirect RepeatedAllocReturnNullDirect
#define MAYBE_RepeatedReallocReturnNullDirect RepeatedReallocReturnNullDirect
#else
#define MAYBE_RepeatedAllocReturnNullDirect \
DISABLED_RepeatedAllocReturnNullDirect
#define MAYBE_RepeatedReallocReturnNullDirect \
DISABLED_RepeatedReallocReturnNullDirect
#endif
// The following four tests wrap a called function in an expect death statement
// to perform their test, because they are non-hermetic. Specifically they are
// going to attempt to exhaust the allocatable memory, which leaves the
// allocator in a bad global state.
// Performing them as death tests causes them to be forked into their own
// process, so they won't pollute other tests.
//
// These tests are *very* slow when BUILDFLAG(PA_DCHECK_IS_ON), because they
// memset() many GiB of data (see crbug.com/1168168).
// TODO(lizeb): make these tests faster.
TEST_P(PartitionAllocDeathTest, MAYBE_RepeatedAllocReturnNullDirect) {
// A direct-mapped allocation size.
size_t direct_map_size = 32 * 1024 * 1024;
ASSERT_GT(direct_map_size, kMaxBucketed);
EXPECT_DEATH(DoReturnNullTest(direct_map_size, kPartitionAlloc),
"Passed DoReturnNullTest");
}
// Repeating above test with Realloc
TEST_P(PartitionAllocDeathTest, MAYBE_RepeatedReallocReturnNullDirect) {
size_t direct_map_size = 32 * 1024 * 1024;
ASSERT_GT(direct_map_size, kMaxBucketed);
EXPECT_DEATH(DoReturnNullTest(direct_map_size, kPartitionRealloc),
"Passed DoReturnNullTest");
}
// TODO(crbug.com/1348221) re-enable the tests below, once the allocator
// actually returns nullptr for non direct-mapped allocations.
// When doing so, they will need to be made MAYBE_ like those above.
//
// Tests "return null" with a 512 kB block size.
TEST_P(PartitionAllocDeathTest, DISABLED_RepeatedAllocReturnNull) {
// A single-slot but non-direct-mapped allocation size.
size_t single_slot_size = 512 * 1024;
ASSERT_GT(single_slot_size, MaxRegularSlotSpanSize());
ASSERT_LE(single_slot_size, kMaxBucketed);
EXPECT_DEATH(DoReturnNullTest(single_slot_size, kPartitionAlloc),
"Passed DoReturnNullTest");
}
// Repeating above test with Realloc.
TEST_P(PartitionAllocDeathTest, DISABLED_RepeatedReallocReturnNull) {
size_t single_slot_size = 512 * 1024;
ASSERT_GT(single_slot_size, MaxRegularSlotSpanSize());
ASSERT_LE(single_slot_size, kMaxBucketed);
EXPECT_DEATH(DoReturnNullTest(single_slot_size, kPartitionRealloc),
"Passed DoReturnNullTest");
}
#if PA_CONFIG(HAS_MEMORY_TAGGING)
// Check that Arm's memory tagging extension (MTE) is correctly protecting
// freed pointers. Writes to a free pointer should result in a crash.
TEST_P(PartitionAllocDeathTest, MTEProtectsFreedPtr) {
base::CPU cpu;
if (!cpu.has_mte()) {
// This test won't pass on systems without MTE.
GTEST_SKIP();
}
constexpr uint64_t kCookie = 0x1234567890ABCDEF;
constexpr uint64_t kQuarantined = 0xEFEFEFEFEFEFEFEF;
// Make an arbitrary-sized small allocation.
size_t alloc_size = 64 - ExtraAllocSize(allocator);
uint64_t* ptr =
static_cast<uint64_t*>(allocator.root()->Alloc(alloc_size, type_name));
EXPECT_TRUE(ptr);
// Check that the allocation's writable.
*ptr = kCookie;
// Invalidate ptr by freeing it.
allocator.root()->Free(ptr);
// Writing to ptr after free() should crash
EXPECT_EXIT(
{
// Should be in synchronous MTE mode for running this test.
*ptr = kQuarantined;
},
testing::KilledBySignal(SIGSEGV), "");
}
#endif // PA_CONFIG(HAS_MEMORY_TAGGING)
// Make sure that malloc(-1) dies.
// In the past, we had an integer overflow that would alias malloc(-1) to
// malloc(0), which is not good.
TEST_P(PartitionAllocDeathTest, LargeAllocs) {
// Largest alloc.
EXPECT_DEATH(allocator.root()->Alloc(static_cast<size_t>(-1), type_name), "");
// And the smallest allocation we expect to die.
// TODO(bartekn): Separate into its own test, as it wouldn't run (same below).
EXPECT_DEATH(allocator.root()->Alloc(MaxDirectMapped() + 1, type_name), "");
}
// These tests don't work deterministically when BRP is enabled on certain
// architectures. On Free(), BRP's ref-count gets overwritten by an encoded
// freelist pointer. On little-endian 64-bit architectures, this happens to be
// always an even number, which will triggers BRP's own CHECK (sic!). On other
// architectures, it's likely to be an odd number >1, which will fool BRP into
// thinking the memory isn't freed and still referenced, thus making it
// quarantine it and return early, before PA_CHECK(slot_start != freelist_head)
// is reached.
// TODO(bartekn): Enable in the BUILDFLAG(PUT_REF_COUNT_IN_PREVIOUS_SLOT) case.
#if !BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT) || \
(BUILDFLAG(HAS_64_BIT_POINTERS) && defined(ARCH_CPU_LITTLE_ENDIAN))
// Check that our immediate double-free detection works.
TEST_P(PartitionAllocDeathTest, ImmediateDoubleFree) {
void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
EXPECT_TRUE(ptr);
allocator.root()->Free(ptr);
EXPECT_DEATH(allocator.root()->Free(ptr), "");
}
// As above, but when this isn't the only slot in the span.
TEST_P(PartitionAllocDeathTest, ImmediateDoubleFree2ndSlot) {
void* ptr0 = allocator.root()->Alloc(kTestAllocSize, type_name);
EXPECT_TRUE(ptr0);
void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
EXPECT_TRUE(ptr);
allocator.root()->Free(ptr);
EXPECT_DEATH(allocator.root()->Free(ptr), "");
allocator.root()->Free(ptr0);
}
// Check that our double-free detection based on |num_allocated_slots| not going
// below 0 works.
//
// Unlike in ImmediateDoubleFree test, we can't have a 2ndSlot version, as this
// protection wouldn't work when there is another slot present in the span. It
// will prevent |num_allocated_slots| from going below 0.
TEST_P(PartitionAllocDeathTest, NumAllocatedSlotsDoubleFree) {
void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
EXPECT_TRUE(ptr);
void* ptr2 = allocator.root()->Alloc(kTestAllocSize, type_name);
EXPECT_TRUE(ptr2);
allocator.root()->Free(ptr);
allocator.root()->Free(ptr2);
// This is not an immediate double-free so our immediate detection won't
// fire. However, it does take |num_allocated_slots| to -1, which is illegal
// and should be trapped.
EXPECT_DEATH(allocator.root()->Free(ptr), "");
}
#endif // !BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT) || \
// (BUILDFLAG(HAS_64_BIT_POINTERS) && defined(ARCH_CPU_LITTLE_ENDIAN))
// Check that guard pages are present where expected.
TEST_P(PartitionAllocDeathTest, DirectMapGuardPages) {
const size_t kSizes[] = {
kMaxBucketed + ExtraAllocSize(allocator) + 1,
kMaxBucketed + SystemPageSize(), kMaxBucketed + PartitionPageSize(),
partition_alloc::internal::base::bits::AlignUp(
kMaxBucketed + kSuperPageSize, kSuperPageSize) -
PartitionRoot::GetDirectMapMetadataAndGuardPagesSize()};
for (size_t size : kSizes) {
ASSERT_GT(size, kMaxBucketed);
size -= ExtraAllocSize(allocator);
EXPECT_GT(size, kMaxBucketed)
<< "allocation not large enough for direct allocation";
void* ptr = allocator.root()->Alloc(size, type_name);
EXPECT_TRUE(ptr);
char* char_ptr = static_cast<char*>(ptr) - kPointerOffset;
EXPECT_DEATH(*(char_ptr - 1) = 'A', "");
EXPECT_DEATH(*(char_ptr + partition_alloc::internal::base::bits::AlignUp(
size, SystemPageSize())) = 'A',
"");
allocator.root()->Free(ptr);
}
}
// These tests rely on precise layout. They handle cookie, not ref-count.
#if !BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT) && \
PA_CONFIG(HAS_FREELIST_SHADOW_ENTRY)
TEST_P(PartitionAllocDeathTest, UseAfterFreeDetection) {
base::CPU cpu;
void* data = allocator.root()->Alloc(100);
allocator.root()->Free(data);
// use after free, not crashing here, but the next allocation should crash,
// since we corrupted the freelist.
memset(data, 0x42, 100);
EXPECT_DEATH(allocator.root()->Alloc(100), "");
}
TEST_P(PartitionAllocDeathTest, FreelistCorruption) {
base::CPU cpu;
const size_t alloc_size = 2 * sizeof(void*);
void** fake_freelist_entry =
static_cast<void**>(allocator.root()->Alloc(alloc_size));
fake_freelist_entry[0] = nullptr;
fake_freelist_entry[1] = nullptr;
void** uaf_data = static_cast<void**>(allocator.root()->Alloc(alloc_size));
allocator.root()->Free(uaf_data);
// Try to confuse the allocator. This is still easy to circumvent willingly,
// "just" need to set uaf_data[1] to ~uaf_data[0].
void* previous_uaf_data = uaf_data[0];
uaf_data[0] = fake_freelist_entry;
EXPECT_DEATH(allocator.root()->Alloc(alloc_size), "");
// Restore the freelist entry value, otherwise freelist corruption is detected
// in TearDown(), crashing this process.
uaf_data[0] = previous_uaf_data;
allocator.root()->Free(fake_freelist_entry);
}
// With BUILDFLAG(PA_DCHECK_IS_ON), cookie already handles off-by-one detection.
#if !BUILDFLAG(PA_DCHECK_IS_ON)
TEST_P(PartitionAllocDeathTest, OffByOneDetection) {
base::CPU cpu;
const size_t alloc_size = 2 * sizeof(void*);
char* array = static_cast<char*>(allocator.root()->Alloc(alloc_size));
if (cpu.has_mte()) {
EXPECT_DEATH(array[alloc_size] = 'A', "");
} else {
char previous_value = array[alloc_size];
// volatile is required to prevent the compiler from getting too clever and
// eliding the out-of-bounds write. The root cause is that the PA_MALLOC_FN
// annotation tells the compiler (among other things) that the returned
// value cannot alias anything.
*const_cast<volatile char*>(&array[alloc_size]) = 'A';
// Crash at the next allocation. This assumes that we are touching a new,
// non-randomized slot span, where the next slot to be handed over to the
// application directly follows the current one.
EXPECT_DEATH(allocator.root()->Alloc(alloc_size), "");
// Restore integrity, otherwise the process will crash in TearDown().
array[alloc_size] = previous_value;
}
}
TEST_P(PartitionAllocDeathTest, OffByOneDetectionWithRealisticData) {
base::CPU cpu;
const size_t alloc_size = 2 * sizeof(void*);
void** array = static_cast<void**>(allocator.root()->Alloc(alloc_size));
char valid;
if (cpu.has_mte()) {
EXPECT_DEATH(array[2] = &valid, "");
} else {
void* previous_value = array[2];
// As above, needs volatile to convince the compiler to perform the write.
*const_cast<void* volatile*>(&array[2]) = &valid;
// Crash at the next allocation. This assumes that we are touching a new,
// non-randomized slot span, where the next slot to be handed over to the
// application directly follows the current one.
EXPECT_DEATH(allocator.root()->Alloc(alloc_size), "");
array[2] = previous_value;
}
}
#endif // !BUILDFLAG(PA_DCHECK_IS_ON)
#endif // !BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT) &&
// PA_CONFIG(HAS_FREELIST_SHADOW_ENTRY)
#endif // !defined(PA_HAS_DEATH_TESTS)
// Tests that |PartitionDumpStats| and |PartitionDumpStats| run without
// crashing and return non-zero values when memory is allocated.
TEST_P(PartitionAllocTest, DumpMemoryStats) {
{
void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
MockPartitionStatsDumper mock_stats_dumper;
allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
&mock_stats_dumper);
EXPECT_TRUE(mock_stats_dumper.IsMemoryAllocationRecorded());
allocator.root()->Free(ptr);
}
// This series of tests checks the active -> empty -> decommitted states.
{
{
void* ptr =
allocator.root()->Alloc(2048 - ExtraAllocSize(allocator), type_name);
MockPartitionStatsDumper dumper;
allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
&dumper);
EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
const PartitionBucketMemoryStats* stats = dumper.GetBucketStats(2048);
EXPECT_TRUE(stats);
EXPECT_TRUE(stats->is_valid);
EXPECT_EQ(2048u, stats->bucket_slot_size);
EXPECT_EQ(2048u, stats->active_bytes);
EXPECT_EQ(1u, stats->active_count);
EXPECT_EQ(SystemPageSize(), stats->resident_bytes);
EXPECT_EQ(0u, stats->decommittable_bytes);
EXPECT_EQ(0u, stats->discardable_bytes);
EXPECT_EQ(0u, stats->num_full_slot_spans);
EXPECT_EQ(1u, stats->num_active_slot_spans);
EXPECT_EQ(0u, stats->num_empty_slot_spans);
EXPECT_EQ(0u, stats->num_decommitted_slot_spans);
allocator.root()->Free(ptr);
}
{
MockPartitionStatsDumper dumper;
allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
&dumper);
EXPECT_FALSE(dumper.IsMemoryAllocationRecorded());
const PartitionBucketMemoryStats* stats = dumper.GetBucketStats(2048);
EXPECT_TRUE(stats);
EXPECT_TRUE(stats->is_valid);
EXPECT_EQ(2048u, stats->bucket_slot_size);
EXPECT_EQ(0u, stats->active_bytes);
EXPECT_EQ(0u, stats->active_count);
EXPECT_EQ(SystemPageSize(), stats->resident_bytes);
EXPECT_EQ(SystemPageSize(), stats->decommittable_bytes);
EXPECT_EQ(0u, stats->discardable_bytes);
EXPECT_EQ(0u, stats->num_full_slot_spans);
EXPECT_EQ(0u, stats->num_active_slot_spans);
EXPECT_EQ(1u, stats->num_empty_slot_spans);
EXPECT_EQ(0u, stats->num_decommitted_slot_spans);
}
// TODO(crbug.com/722911): Commenting this out causes this test to fail when
// run singly (--gtest_filter=PartitionAllocTest.DumpMemoryStats), but not
// when run with the others (--gtest_filter=PartitionAllocTest.*).
CycleFreeCache(kTestAllocSize);
{
MockPartitionStatsDumper dumper;
allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
&dumper);
EXPECT_FALSE(dumper.IsMemoryAllocationRecorded());
const PartitionBucketMemoryStats* stats = dumper.GetBucketStats(2048);
EXPECT_TRUE(stats);
EXPECT_TRUE(stats->is_valid);
EXPECT_EQ(2048u, stats->bucket_slot_size);
EXPECT_EQ(0u, stats->active_bytes);
EXPECT_EQ(0u, stats->active_count);
EXPECT_EQ(0u, stats->resident_bytes);
EXPECT_EQ(0u, stats->decommittable_bytes);
EXPECT_EQ(0u, stats->discardable_bytes);
EXPECT_EQ(0u, stats->num_full_slot_spans);
EXPECT_EQ(0u, stats->num_active_slot_spans);
EXPECT_EQ(0u, stats->num_empty_slot_spans);
EXPECT_EQ(1u, stats->num_decommitted_slot_spans);
}
}
// This test checks for correct empty slot span list accounting.
{
size_t size = PartitionPageSize() - ExtraAllocSize(allocator);
void* ptr1 = allocator.root()->Alloc(size, type_name);
void* ptr2 = allocator.root()->Alloc(size, type_name);
allocator.root()->Free(ptr1);
allocator.root()->Free(ptr2);
CycleFreeCache(kTestAllocSize);
ptr1 = allocator.root()->Alloc(size, type_name);
{
MockPartitionStatsDumper dumper;
allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
&dumper);
EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
const PartitionBucketMemoryStats* stats =
dumper.GetBucketStats(PartitionPageSize());
EXPECT_TRUE(stats);
EXPECT_TRUE(stats->is_valid);
EXPECT_EQ(PartitionPageSize(), stats->bucket_slot_size);
EXPECT_EQ(PartitionPageSize(), stats->active_bytes);
EXPECT_EQ(1u, stats->active_count);
EXPECT_EQ(PartitionPageSize(), stats->resident_bytes);
EXPECT_EQ(0u, stats->decommittable_bytes);
EXPECT_EQ(0u, stats->discardable_bytes);
EXPECT_EQ(1u, stats->num_full_slot_spans);
EXPECT_EQ(0u, stats->num_active_slot_spans);
EXPECT_EQ(0u, stats->num_empty_slot_spans);
EXPECT_EQ(1u, stats->num_decommitted_slot_spans);
}
allocator.root()->Free(ptr1);
}
// This test checks for correct direct mapped accounting.
{
size_t size_smaller = kMaxBucketed + 1;
size_t size_bigger = (kMaxBucketed * 2) + 1;
size_t real_size_smaller =
(size_smaller + SystemPageOffsetMask()) & SystemPageBaseMask();
size_t real_size_bigger =
(size_bigger + SystemPageOffsetMask()) & SystemPageBaseMask();
void* ptr = allocator.root()->Alloc(size_smaller, type_name);
void* ptr2 = allocator.root()->Alloc(size_bigger, type_name);
{
MockPartitionStatsDumper dumper;
allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
&dumper);
EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
const PartitionBucketMemoryStats* stats =
dumper.GetBucketStats(real_size_smaller);
EXPECT_TRUE(stats);
EXPECT_TRUE(stats->is_valid);
EXPECT_TRUE(stats->is_direct_map);
EXPECT_EQ(real_size_smaller, stats->bucket_slot_size);
EXPECT_EQ(real_size_smaller, stats->active_bytes);
EXPECT_EQ(1u, stats->active_count);
EXPECT_EQ(real_size_smaller, stats->resident_bytes);
EXPECT_EQ(0u, stats->decommittable_bytes);
EXPECT_EQ(0u, stats->discardable_bytes);
EXPECT_EQ(1u, stats->num_full_slot_spans);
EXPECT_EQ(0u, stats->num_active_slot_spans);
EXPECT_EQ(0u, stats->num_empty_slot_spans);
EXPECT_EQ(0u, stats->num_decommitted_slot_spans);
stats = dumper.GetBucketStats(real_size_bigger);
EXPECT_TRUE(stats);
EXPECT_TRUE(stats->is_valid);
EXPECT_TRUE(stats->is_direct_map);
EXPECT_EQ(real_size_bigger, stats->bucket_slot_size);
EXPECT_EQ(real_size_bigger, stats->active_bytes);
EXPECT_EQ(1u, stats->active_count);
EXPECT_EQ(real_size_bigger, stats->resident_bytes);
EXPECT_EQ(0u, stats->decommittable_bytes);
EXPECT_EQ(0u, stats->discardable_bytes);
EXPECT_EQ(1u, stats->num_full_slot_spans);
EXPECT_EQ(0u, stats->num_active_slot_spans);
EXPECT_EQ(0u, stats->num_empty_slot_spans);
EXPECT_EQ(0u, stats->num_decommitted_slot_spans);
}
allocator.root()->Free(ptr2);
allocator.root()->Free(ptr);
// Whilst we're here, allocate again and free with different ordering to
// give a workout to our linked list code.
ptr = allocator.root()->Alloc(size_smaller, type_name);
ptr2 = allocator.root()->Alloc(size_bigger, type_name);
allocator.root()->Free(ptr);
allocator.root()->Free(ptr2);
}
// This test checks large-but-not-quite-direct allocations.
{
const size_t requested_size = 16 * SystemPageSize();
void* ptr = allocator.root()->Alloc(requested_size + 1, type_name);
{
MockPartitionStatsDumper dumper;
allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
&dumper);
EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
size_t slot_size = SizeToBucketSize(requested_size + 1);
const PartitionBucketMemoryStats* stats =
dumper.GetBucketStats(slot_size);
ASSERT_TRUE(stats);
EXPECT_TRUE(stats->is_valid);
EXPECT_FALSE(stats->is_direct_map);
EXPECT_EQ(slot_size, stats->bucket_slot_size);
EXPECT_EQ(requested_size + 1 + ExtraAllocSize(allocator),
stats->active_bytes);
EXPECT_EQ(1u, stats->active_count);
EXPECT_EQ(slot_size, stats->resident_bytes);
EXPECT_EQ(0u, stats->decommittable_bytes);
EXPECT_EQ((slot_size - (requested_size + 1)) / SystemPageSize() *
SystemPageSize(),
stats->discardable_bytes);
EXPECT_EQ(1u, stats->num_full_slot_spans);
EXPECT_EQ(0u, stats->num_active_slot_spans);
EXPECT_EQ(0u, stats->num_empty_slot_spans);
EXPECT_EQ(0u, stats->num_decommitted_slot_spans);
}
allocator.root()->Free(ptr);
{
MockPartitionStatsDumper dumper;
allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
&dumper);
EXPECT_FALSE(dumper.IsMemoryAllocationRecorded());
size_t slot_size = SizeToBucketSize(requested_size + 1);
const PartitionBucketMemoryStats* stats =
dumper.GetBucketStats(slot_size);
EXPECT_TRUE(stats);
EXPECT_TRUE(stats->is_valid);
EXPECT_FALSE(stats->is_direct_map);
EXPECT_EQ(slot_size, stats->bucket_slot_size);
EXPECT_EQ(0u, stats->active_bytes);
EXPECT_EQ(0u, stats->active_count);
EXPECT_EQ(slot_size, stats->resident_bytes);
EXPECT_EQ(slot_size, stats->decommittable_bytes);
EXPECT_EQ(0u, stats->num_full_slot_spans);
EXPECT_EQ(0u, stats->num_active_slot_spans);
EXPECT_EQ(1u, stats->num_empty_slot_spans);
EXPECT_EQ(0u, stats->num_decommitted_slot_spans);
}
void* ptr2 = allocator.root()->Alloc(requested_size + SystemPageSize() + 1,
type_name);
EXPECT_EQ(ptr, ptr2);
{
MockPartitionStatsDumper dumper;
allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
&dumper);
EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
size_t slot_size =
SizeToBucketSize(requested_size + SystemPageSize() + 1);
const PartitionBucketMemoryStats* stats =
dumper.GetBucketStats(slot_size);
EXPECT_TRUE(stats);
EXPECT_TRUE(stats->is_valid);
EXPECT_FALSE(stats->is_direct_map);
EXPECT_EQ(slot_size, stats->bucket_slot_size);
EXPECT_EQ(
requested_size + SystemPageSize() + 1 + ExtraAllocSize(allocator),
stats->active_bytes);
EXPECT_EQ(1u, stats->active_count);
EXPECT_EQ(slot_size, stats->resident_bytes);
EXPECT_EQ(0u, stats->decommittable_bytes);
EXPECT_EQ((slot_size - (requested_size + SystemPageSize() + 1)) /
SystemPageSize() * SystemPageSize(),
stats->discardable_bytes);
EXPECT_EQ(1u, stats->num_full_slot_spans);
EXPECT_EQ(0u, stats->num_active_slot_spans);
EXPECT_EQ(0u, stats->num_empty_slot_spans);
EXPECT_EQ(0u, stats->num_decommitted_slot_spans);
}
allocator.root()->Free(ptr2);
}
}
// Tests the API to purge freeable memory.
TEST_P(PartitionAllocTest, Purge) {
char* ptr = static_cast<char*>(
allocator.root()->Alloc(2048 - ExtraAllocSize(allocator), type_name));
allocator.root()->Free(ptr);
{
MockPartitionStatsDumper dumper;
allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
&dumper);
EXPECT_FALSE(dumper.IsMemoryAllocationRecorded());
const PartitionBucketMemoryStats* stats = dumper.GetBucketStats(2048);
EXPECT_TRUE(stats);
EXPECT_TRUE(stats->is_valid);
EXPECT_EQ(SystemPageSize(), stats->decommittable_bytes);
EXPECT_EQ(SystemPageSize(), stats->resident_bytes);
}
allocator.root()->PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans);
{
MockPartitionStatsDumper dumper;
allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
&dumper);
EXPECT_FALSE(dumper.IsMemoryAllocationRecorded());
const PartitionBucketMemoryStats* stats = dumper.GetBucketStats(2048);
EXPECT_TRUE(stats);
EXPECT_TRUE(stats->is_valid);
EXPECT_EQ(0u, stats->decommittable_bytes);
EXPECT_EQ(0u, stats->resident_bytes);
}
// Calling purge again here is a good way of testing we didn't mess up the
// state of the free cache ring.
allocator.root()->PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans);
// A single-slot but non-direct-mapped allocation size.
size_t single_slot_size = 512 * 1024;
ASSERT_GT(single_slot_size, MaxRegularSlotSpanSize());
ASSERT_LE(single_slot_size, kMaxBucketed);
char* big_ptr =
static_cast<char*>(allocator.root()->Alloc(single_slot_size, type_name));
allocator.root()->Free(big_ptr);
allocator.root()->PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans);
CHECK_PAGE_IN_CORE(ptr - kPointerOffset, false);
CHECK_PAGE_IN_CORE(big_ptr - kPointerOffset, false);
}
// Tests that we prefer to allocate into a non-empty partition page over an
// empty one. This is an important aspect of minimizing memory usage for some
// allocation sizes, particularly larger ones.
TEST_P(PartitionAllocTest, PreferActiveOverEmpty) {
size_t size = (SystemPageSize() * 2) - ExtraAllocSize(allocator);
// Allocate 3 full slot spans worth of 8192-byte allocations.
// Each slot span for this size is 16384 bytes, or 1 partition page and 2
// slots.
void* ptr1 = allocator.root()->Alloc(size, type_name);
void* ptr2 = allocator.root()->Alloc(size, type_name);
void* ptr3 = allocator.root()->Alloc(size, type_name);
void* ptr4 = allocator.root()->Alloc(size, type_name);
void* ptr5 = allocator.root()->Alloc(size, type_name);
void* ptr6 = allocator.root()->Alloc(size, type_name);
SlotSpanMetadata* slot_span1 = SlotSpanMetadata::FromSlotStart(
allocator.root()->ObjectToSlotStart(ptr1));
SlotSpanMetadata* slot_span2 = SlotSpanMetadata::FromSlotStart(
allocator.root()->ObjectToSlotStart(ptr3));
SlotSpanMetadata* slot_span3 = SlotSpanMetadata::FromSlotStart(
allocator.root()->ObjectToSlotStart(ptr6));
EXPECT_NE(slot_span1, slot_span2);
EXPECT_NE(slot_span2, slot_span3);
PartitionBucket* bucket = slot_span1->bucket;
EXPECT_EQ(slot_span3, bucket->active_slot_spans_head);
// Free up the 2nd slot in each slot span.
// This leaves the active list containing 3 slot spans, each with 1 used and 1
// free slot. The active slot span will be the one containing ptr1.
allocator.root()->Free(ptr6);
allocator.root()->Free(ptr4);
allocator.root()->Free(ptr2);
EXPECT_EQ(slot_span1, bucket->active_slot_spans_head);
// Empty the middle slot span in the active list.
allocator.root()->Free(ptr3);
EXPECT_EQ(slot_span1, bucket->active_slot_spans_head);
// Empty the first slot span in the active list -- also the current slot span.
allocator.root()->Free(ptr1);
// A good choice here is to re-fill the third slot span since the first two
// are empty. We used to fail that.
void* ptr7 = allocator.root()->Alloc(size, type_name);
PA_EXPECT_PTR_EQ(ptr6, ptr7);
EXPECT_EQ(slot_span3, bucket->active_slot_spans_head);
allocator.root()->Free(ptr5);
allocator.root()->Free(ptr7);
}
// Tests the API to purge discardable memory.
TEST_P(PartitionAllocTest, PurgeDiscardableSecondPage) {
// Free the second of two 4096 byte allocations and then purge.
void* ptr1 = allocator.root()->Alloc(
SystemPageSize() - ExtraAllocSize(allocator), type_name);
char* ptr2 = static_cast<char*>(allocator.root()->Alloc(
SystemPageSize() - ExtraAllocSize(allocator), type_name));
allocator.root()->Free(ptr2);
SlotSpanMetadata* slot_span = SlotSpanMetadata::FromSlotStart(
allocator.root()->ObjectToSlotStart(ptr1));
EXPECT_EQ(2u, slot_span->num_unprovisioned_slots);
{
MockPartitionStatsDumper dumper;
allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
&dumper);
EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
const PartitionBucketMemoryStats* stats =
dumper.GetBucketStats(SystemPageSize());
EXPECT_TRUE(stats);
EXPECT_TRUE(stats->is_valid);
EXPECT_EQ(0u, stats->decommittable_bytes);
EXPECT_EQ(SystemPageSize(), stats->discardable_bytes);
EXPECT_EQ(SystemPageSize(), stats->active_bytes);
EXPECT_EQ(2 * SystemPageSize(), stats->resident_bytes);
}
CHECK_PAGE_IN_CORE(ptr2 - kPointerOffset, true);
allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
CHECK_PAGE_IN_CORE(ptr2 - kPointerOffset, false);
EXPECT_EQ(3u, slot_span->num_unprovisioned_slots);
allocator.root()->Free(ptr1);
}
TEST_P(PartitionAllocTest, PurgeDiscardableFirstPage) {
// Free the first of two 4096 byte allocations and then purge.
char* ptr1 = static_cast<char*>(allocator.root()->Alloc(
SystemPageSize() - ExtraAllocSize(allocator), type_name));
void* ptr2 = allocator.root()->Alloc(
SystemPageSize() - ExtraAllocSize(allocator), type_name);
allocator.root()->Free(ptr1);
{
MockPartitionStatsDumper dumper;
allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
&dumper);
EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
const PartitionBucketMemoryStats* stats =
dumper.GetBucketStats(SystemPageSize());
EXPECT_TRUE(stats);
EXPECT_TRUE(stats->is_valid);
EXPECT_EQ(0u, stats->decommittable_bytes);
#if BUILDFLAG(IS_WIN)
EXPECT_EQ(0u, stats->discardable_bytes);
#else
EXPECT_EQ(SystemPageSize(), stats->discardable_bytes);
#endif
EXPECT_EQ(SystemPageSize(), stats->active_bytes);
EXPECT_EQ(2 * SystemPageSize(), stats->resident_bytes);
}
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, false);
allocator.root()->Free(ptr2);
}
TEST_P(PartitionAllocTest, PurgeDiscardableNonPageSizedAlloc) {
const size_t requested_size = 2.5 * SystemPageSize();
char* ptr1 = static_cast<char*>(allocator.root()->Alloc(
requested_size - ExtraAllocSize(allocator), type_name));
void* ptr2 = allocator.root()->Alloc(
requested_size - ExtraAllocSize(allocator), type_name);
void* ptr3 = allocator.root()->Alloc(
requested_size - ExtraAllocSize(allocator), type_name);
void* ptr4 = allocator.root()->Alloc(
requested_size - ExtraAllocSize(allocator), type_name);
memset(ptr1, 'A', requested_size - ExtraAllocSize(allocator));
memset(ptr2, 'A', requested_size - ExtraAllocSize(allocator));
allocator.root()->Free(ptr1);
allocator.root()->Free(ptr2);
{
MockPartitionStatsDumper dumper;
allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
&dumper);
EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
const PartitionBucketMemoryStats* stats =
dumper.GetBucketStats(requested_size);
EXPECT_TRUE(stats);
EXPECT_TRUE(stats->is_valid);
EXPECT_EQ(0u, stats->decommittable_bytes);
#if BUILDFLAG(IS_WIN)
EXPECT_EQ(3 * SystemPageSize(), stats->discardable_bytes);
#else
EXPECT_EQ(4 * SystemPageSize(), stats->discardable_bytes);
#endif
EXPECT_EQ(requested_size * 2, stats->active_bytes);
EXPECT_EQ(10 * SystemPageSize(), stats->resident_bytes);
}
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 2), true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 3), true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 4), true);
allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
// Except for Windows, the first page is discardable because the freelist
// pointer on this page is nullptr. Note that CHECK_PAGE_IN_CORE only executes
// checks for Linux and ChromeOS, not for Windows.
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, false);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), false);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 2), true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 3), false);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 4), false);
allocator.root()->Free(ptr3);
allocator.root()->Free(ptr4);
}
TEST_P(PartitionAllocTest, PurgeDiscardableNonPageSizedAllocOnSlotBoundary) {
const size_t requested_size = 2.5 * SystemPageSize();
char* ptr1 = static_cast<char*>(allocator.root()->Alloc(
requested_size - ExtraAllocSize(allocator), type_name));
void* ptr2 = allocator.root()->Alloc(
requested_size - ExtraAllocSize(allocator), type_name);
void* ptr3 = allocator.root()->Alloc(
requested_size - ExtraAllocSize(allocator), type_name);
void* ptr4 = allocator.root()->Alloc(
requested_size - ExtraAllocSize(allocator), type_name);
memset(ptr1, 'A', requested_size - ExtraAllocSize(allocator));
memset(ptr2, 'A', requested_size - ExtraAllocSize(allocator));
allocator.root()->Free(ptr2);
allocator.root()->Free(ptr1);
{
MockPartitionStatsDumper dumper;
allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
&dumper);
EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
const PartitionBucketMemoryStats* stats =
dumper.GetBucketStats(requested_size);
EXPECT_TRUE(stats);
EXPECT_TRUE(stats->is_valid);
EXPECT_EQ(0u, stats->decommittable_bytes);
#if BUILDFLAG(IS_WIN)
EXPECT_EQ(3 * SystemPageSize(), stats->discardable_bytes);
#else
EXPECT_EQ(4 * SystemPageSize(), stats->discardable_bytes);
#endif
EXPECT_EQ(requested_size * 2, stats->active_bytes);
EXPECT_EQ(10 * SystemPageSize(), stats->resident_bytes);
}
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 2), true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 3), true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 4), true);
allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), false);
// Except for Windows, the third page is discardable because the freelist
// pointer on this page is nullptr. Note that CHECK_PAGE_IN_CORE only executes
// checks for Linux and ChromeOS, not for Windows.
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 2), false);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 3), false);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 4), false);
allocator.root()->Free(ptr3);
allocator.root()->Free(ptr4);
}
TEST_P(PartitionAllocTest, PurgeDiscardableManyPages) {
// On systems with large pages, use less pages because:
// 1) There must be a bucket for kFirstAllocPages * SystemPageSize(), and
// 2) On low-end systems, using too many large pages can OOM during the test
const bool kHasLargePages = SystemPageSize() > 4096;
const size_t kFirstAllocPages = kHasLargePages ? 32 : 64;
const size_t kSecondAllocPages = kHasLargePages ? 31 : 61;
// Detect case (1) from above.
PA_DCHECK(kFirstAllocPages * SystemPageSize() < (1UL << kMaxBucketedOrder));
const size_t kDeltaPages = kFirstAllocPages - kSecondAllocPages;
{
ScopedPageAllocation p(allocator, kFirstAllocPages);
p.TouchAllPages();
}
ScopedPageAllocation p(allocator, kSecondAllocPages);
MockPartitionStatsDumper dumper;
allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
&dumper);
EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
const PartitionBucketMemoryStats* stats =
dumper.GetBucketStats(kFirstAllocPages * SystemPageSize());
EXPECT_TRUE(stats);
EXPECT_TRUE(stats->is_valid);
EXPECT_EQ(0u, stats->decommittable_bytes);
EXPECT_EQ(kDeltaPages * SystemPageSize(), stats->discardable_bytes);
EXPECT_EQ(kSecondAllocPages * SystemPageSize(), stats->active_bytes);
EXPECT_EQ(kFirstAllocPages * SystemPageSize(), stats->resident_bytes);
for (size_t i = 0; i < kFirstAllocPages; i++) {
CHECK_PAGE_IN_CORE(p.PageAtIndex(i), true);
}
allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
for (size_t i = 0; i < kSecondAllocPages; i++) {
CHECK_PAGE_IN_CORE(p.PageAtIndex(i), true);
}
for (size_t i = kSecondAllocPages; i < kFirstAllocPages; i++) {
CHECK_PAGE_IN_CORE(p.PageAtIndex(i), false);
}
}
TEST_P(PartitionAllocTest, PurgeDiscardableWithFreeListStraightening) {
// This sub-test tests truncation of the provisioned slots in a trickier
// case where the freelist is rewritten.
allocator.root()->PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans);
char* ptr1 = static_cast<char*>(allocator.root()->Alloc(
SystemPageSize() - ExtraAllocSize(allocator), type_name));
void* ptr2 = allocator.root()->Alloc(
SystemPageSize() - ExtraAllocSize(allocator), type_name);
void* ptr3 = allocator.root()->Alloc(
SystemPageSize() - ExtraAllocSize(allocator), type_name);
void* ptr4 = allocator.root()->Alloc(
SystemPageSize() - ExtraAllocSize(allocator), type_name);
ptr1[0] = 'A';
ptr1[SystemPageSize()] = 'A';
ptr1[SystemPageSize() * 2] = 'A';
ptr1[SystemPageSize() * 3] = 'A';
SlotSpanMetadata* slot_span = SlotSpanMetadata::FromSlotStart(
allocator.root()->ObjectToSlotStart(ptr1));
allocator.root()->Free(ptr2);
allocator.root()->Free(ptr4);
allocator.root()->Free(ptr1);
EXPECT_EQ(0u, slot_span->num_unprovisioned_slots);
{
MockPartitionStatsDumper dumper;
allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
&dumper);
EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
const PartitionBucketMemoryStats* stats =
dumper.GetBucketStats(SystemPageSize());
EXPECT_TRUE(stats);
EXPECT_TRUE(stats->is_valid);
EXPECT_EQ(0u, stats->decommittable_bytes);
#if BUILDFLAG(IS_WIN)
EXPECT_EQ(SystemPageSize(), stats->discardable_bytes);
#else
EXPECT_EQ(2 * SystemPageSize(), stats->discardable_bytes);
#endif
EXPECT_EQ(SystemPageSize(), stats->active_bytes);
EXPECT_EQ(4 * SystemPageSize(), stats->resident_bytes);
}
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 2), true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 3), true);
allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
EXPECT_EQ(1u, slot_span->num_unprovisioned_slots);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), false);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 2), true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 3), false);
// Let's check we didn't brick the freelist.
void* ptr1b = allocator.root()->Alloc(
SystemPageSize() - ExtraAllocSize(allocator), type_name);
PA_EXPECT_PTR_EQ(ptr1, ptr1b);
void* ptr2b = allocator.root()->Alloc(
SystemPageSize() - ExtraAllocSize(allocator), type_name);
PA_EXPECT_PTR_EQ(ptr2, ptr2b);
EXPECT_FALSE(slot_span->get_freelist_head()); // ptr4 was unprovisioned
void* ptr4b = allocator.root()->Alloc(
SystemPageSize() - ExtraAllocSize(allocator), type_name);
PA_EXPECT_PTR_EQ(ptr4, ptr4b);
EXPECT_FALSE(slot_span->get_freelist_head());
// Free objects such that they're in this order on the list:
// head -> ptr2 -> ptr3 -> ptr1
// However, ptr4 is still unfreed preventing any unprovisioning.
allocator.root()->Free(ptr1);
allocator.root()->Free(ptr3);
allocator.root()->Free(ptr2);
allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
// The test by default runs in
// StraightenLargerSlotSpanFreeListsMode::kOnlyWhenUnprovisioning mode, so the
// freelist wasn't modified, and the allocations will happen in LIFO order.
ptr2b = allocator.root()->Alloc(SystemPageSize() - ExtraAllocSize(allocator),
type_name);
PA_EXPECT_PTR_EQ(ptr2, ptr2b);
void* ptr3b = allocator.root()->Alloc(
SystemPageSize() - ExtraAllocSize(allocator), type_name);
PA_EXPECT_PTR_EQ(ptr3, ptr3b);
ptr1b = allocator.root()->Alloc(SystemPageSize() - ExtraAllocSize(allocator),
type_name);
PA_EXPECT_PTR_EQ(ptr1, ptr1b);
EXPECT_FALSE(slot_span->get_freelist_head());
// Free objects such that they're in this order on the list:
// head -> ptr2 -> ptr3 -> ptr1
// However, ptr4 is still unfreed preventing any unprovisioning.
allocator.root()->Free(ptr1);
allocator.root()->Free(ptr3);
allocator.root()->Free(ptr2);
PartitionRoot::SetStraightenLargerSlotSpanFreeListsMode(
StraightenLargerSlotSpanFreeListsMode::kAlways);
allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
// In StraightenLargerSlotSpanFreeListsMode::kAlways mode, the freelist is
// ordered from left to right.
ptr1b = allocator.root()->Alloc(SystemPageSize() - ExtraAllocSize(allocator),
type_name);
PA_EXPECT_PTR_EQ(ptr1, ptr1b);
ptr2b = allocator.root()->Alloc(SystemPageSize() - ExtraAllocSize(allocator),
type_name);
PA_EXPECT_PTR_EQ(ptr2, ptr2b);
ptr3b = allocator.root()->Alloc(SystemPageSize() - ExtraAllocSize(allocator),
type_name);
PA_EXPECT_PTR_EQ(ptr3, ptr3b);
EXPECT_FALSE(slot_span->get_freelist_head());
// Free objects such that they're in this order on the list:
// head -> ptr2 -> ptr4 -> ptr1
// ptr3 is still unfreed preventing unprovisioning of ptr1 and ptr2, but not
// ptr4.
allocator.root()->Free(ptr1);
allocator.root()->Free(ptr4);
allocator.root()->Free(ptr2);
PartitionRoot::SetStraightenLargerSlotSpanFreeListsMode(
StraightenLargerSlotSpanFreeListsMode::kNever);
allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
// In StraightenLargerSlotSpanFreeListsMode::kNever mode, unprovisioned
// entries willbe removed form the freelist but the list won't be reordered.
ptr2b = allocator.root()->Alloc(SystemPageSize() - ExtraAllocSize(allocator),
type_name);
PA_EXPECT_PTR_EQ(ptr2, ptr2b);
ptr1b = allocator.root()->Alloc(SystemPageSize() - ExtraAllocSize(allocator),
type_name);
PA_EXPECT_PTR_EQ(ptr1, ptr1b);
EXPECT_FALSE(slot_span->get_freelist_head());
ptr4b = allocator.root()->Alloc(SystemPageSize() - ExtraAllocSize(allocator),
type_name);
PA_EXPECT_PTR_EQ(ptr4, ptr4b);
EXPECT_FALSE(slot_span->get_freelist_head());
// Clean up.
allocator.root()->Free(ptr1);
allocator.root()->Free(ptr2);
allocator.root()->Free(ptr3);
allocator.root()->Free(ptr4);
}
TEST_P(PartitionAllocTest, PurgeDiscardableDoubleTruncateFreeList) {
// This sub-test is similar, but tests a double-truncation.
allocator.root()->PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans);
char* ptr1 = static_cast<char*>(allocator.root()->Alloc(
SystemPageSize() - ExtraAllocSize(allocator), type_name));
void* ptr2 = allocator.root()->Alloc(
SystemPageSize() - ExtraAllocSize(allocator), type_name);
void* ptr3 = allocator.root()->Alloc(
SystemPageSize() - ExtraAllocSize(allocator), type_name);
void* ptr4 = allocator.root()->Alloc(
SystemPageSize() - ExtraAllocSize(allocator), type_name);
ptr1[0] = 'A';
ptr1[SystemPageSize()] = 'A';
ptr1[SystemPageSize() * 2] = 'A';
ptr1[SystemPageSize() * 3] = 'A';
SlotSpanMetadata* slot_span = SlotSpanMetadata::FromSlotStart(
allocator.root()->ObjectToSlotStart(ptr1));
allocator.root()->Free(ptr4);
allocator.root()->Free(ptr3);
EXPECT_EQ(0u, slot_span->num_unprovisioned_slots);
{
MockPartitionStatsDumper dumper;
allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
&dumper);
EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
const PartitionBucketMemoryStats* stats =
dumper.GetBucketStats(SystemPageSize());
EXPECT_TRUE(stats);
EXPECT_TRUE(stats->is_valid);
EXPECT_EQ(0u, stats->decommittable_bytes);
EXPECT_EQ(2 * SystemPageSize(), stats->discardable_bytes);
EXPECT_EQ(2 * SystemPageSize(), stats->active_bytes);
EXPECT_EQ(4 * SystemPageSize(), stats->resident_bytes);
}
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 2), true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 3), true);
allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
EXPECT_EQ(2u, slot_span->num_unprovisioned_slots);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 2), false);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 3), false);
EXPECT_FALSE(slot_span->get_freelist_head());
allocator.root()->Free(ptr1);
allocator.root()->Free(ptr2);
}
TEST_P(PartitionAllocTest, PurgeDiscardableSmallSlotsWithTruncate) {
size_t requested_size = 0.5 * SystemPageSize();
char* ptr1 = static_cast<char*>(allocator.root()->Alloc(
requested_size - ExtraAllocSize(allocator), type_name));
void* ptr2 = allocator.root()->Alloc(
requested_size - ExtraAllocSize(allocator), type_name);
void* ptr3 = allocator.root()->Alloc(
requested_size - ExtraAllocSize(allocator), type_name);
void* ptr4 = allocator.root()->Alloc(
requested_size - ExtraAllocSize(allocator), type_name);
allocator.root()->Free(ptr3);
allocator.root()->Free(ptr4);
SlotSpanMetadata* slot_span = SlotSpanMetadata::FromSlotStart(
allocator.root()->ObjectToSlotStart(ptr1));
EXPECT_EQ(4u, slot_span->num_unprovisioned_slots);
{
MockPartitionStatsDumper dumper;
allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
&dumper);
EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
const PartitionBucketMemoryStats* stats =
dumper.GetBucketStats(requested_size);
EXPECT_TRUE(stats);
EXPECT_TRUE(stats->is_valid);
EXPECT_EQ(0u, stats->decommittable_bytes);
EXPECT_EQ(SystemPageSize(), stats->discardable_bytes);
EXPECT_EQ(requested_size * 2, stats->active_bytes);
EXPECT_EQ(2 * SystemPageSize(), stats->resident_bytes);
}
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), true);
allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), false);
EXPECT_EQ(6u, slot_span->num_unprovisioned_slots);
allocator.root()->Free(ptr1);
allocator.root()->Free(ptr2);
}
TEST_P(PartitionAllocTest, ActiveListMaintenance) {
size_t size = SystemPageSize() - ExtraAllocSize(allocator);
size_t real_size = size + ExtraAllocSize(allocator);
size_t bucket_index =
allocator.root()->SizeToBucketIndex(real_size, GetBucketDistribution());
PartitionRoot::Bucket* bucket = &allocator.root()->buckets[bucket_index];
ASSERT_EQ(bucket->slot_size, real_size);
size_t slots_per_span = bucket->num_system_pages_per_slot_span;
// Make 10 full slot spans.
constexpr int kSpans = 10;
std::vector<std::vector<void*>> allocated_memory_spans(kSpans);
for (int span_index = 0; span_index < kSpans; span_index++) {
for (size_t i = 0; i < slots_per_span; i++) {
allocated_memory_spans[span_index].push_back(
allocator.root()->Alloc(size));
}
}
// Free one entry in the middle span, creating a partial slot span.
constexpr size_t kSpanIndex = 5;
allocator.root()->Free(allocated_memory_spans[kSpanIndex].back());
allocated_memory_spans[kSpanIndex].pop_back();
// Empty the last slot span.
for (void* ptr : allocated_memory_spans[kSpans - 1]) {
allocator.root()->Free(ptr);
}
allocated_memory_spans.pop_back();
// The active list now is:
// Partial -> Empty -> Full -> Full -> ... -> Full
bucket->MaintainActiveList();
// Only one entry in the active list.
ASSERT_NE(bucket->active_slot_spans_head,
SlotSpanMetadata::get_sentinel_slot_span());
EXPECT_FALSE(bucket->active_slot_spans_head->next_slot_span);
// The empty list has 1 entry.
ASSERT_NE(bucket->empty_slot_spans_head,
SlotSpanMetadata::get_sentinel_slot_span());
EXPECT_FALSE(bucket->empty_slot_spans_head->next_slot_span);
// The rest are full slot spans.
EXPECT_EQ(8u, bucket->num_full_slot_spans);
// Free all memory.
for (const auto& span : allocated_memory_spans) {
for (void* ptr : span) {
allocator.root()->Free(ptr);
}
}
}
TEST_P(PartitionAllocTest, ReallocMovesCookie) {
// Resize so as to be sure to hit a "resize in place" case, and ensure that
// use of the entire result is compatible with the debug mode's cookie, even
// when the bucket size is large enough to span more than one partition page
// and we can track the "raw" size. See https://crbug.com/709271
static const size_t kSize = MaxRegularSlotSpanSize();
void* ptr = allocator.root()->Alloc(kSize + 1, type_name);
EXPECT_TRUE(ptr);
memset(ptr, 0xbd, kSize + 1);
ptr = allocator.root()->Realloc(ptr, kSize + 2, type_name);
EXPECT_TRUE(ptr);
memset(ptr, 0xbd, kSize + 2);
allocator.root()->Free(ptr);
}
TEST_P(PartitionAllocTest, SmallReallocDoesNotMoveTrailingCookie) {
// For crbug.com/781473
static constexpr size_t kSize = 264;
void* ptr = allocator.root()->Alloc(kSize, type_name);
EXPECT_TRUE(ptr);
ptr = allocator.root()->Realloc(ptr, kSize + 16, type_name);
EXPECT_TRUE(ptr);
allocator.root()->Free(ptr);
}
TEST_P(PartitionAllocTest, ZeroFill) {
static constexpr size_t kAllZerosSentinel =
std::numeric_limits<size_t>::max();
for (size_t size : kTestSizes) {
char* p = static_cast<char*>(
allocator.root()->Alloc<AllocFlags::kZeroFill>(size));
size_t non_zero_position = kAllZerosSentinel;
for (size_t i = 0; i < size; ++i) {
if (0 != p[i]) {
non_zero_position = i;
break;
}
}
EXPECT_EQ(kAllZerosSentinel, non_zero_position)
<< "test allocation size: " << size;
allocator.root()->Free(p);
}
for (int i = 0; i < 10; ++i) {
SCOPED_TRACE(i);
AllocateRandomly<AllocFlags::kZeroFill>(allocator.root(), 250);
}
}
TEST_P(PartitionAllocTest, SchedulerLoopQuarantine) {
SchedulerLoopQuarantineBranch& branch =
allocator.root()->GetSchedulerLoopQuarantineBranchForTesting();
constexpr size_t kCapacityInBytes = std::numeric_limits<size_t>::max();
size_t original_capacity_in_bytes = branch.GetRoot().GetCapacityInBytes();
branch.GetRoot().SetCapacityInBytesForTesting(kCapacityInBytes);
for (size_t size : kTestSizes) {
SCOPED_TRACE(size);
void* object = allocator.root()->Alloc(size);
allocator.root()->Free<FreeFlags::kSchedulerLoopQuarantine>(object);
ASSERT_TRUE(branch.IsQuarantinedForTesting(object));
}
for (int i = 0; i < 10; ++i) {
SCOPED_TRACE(i);
AllocateRandomly<AllocFlags::kNone, FreeFlags::kSchedulerLoopQuarantine>(
allocator.root(), 250);
}
branch.Purge();
branch.GetRoot().SetCapacityInBytesForTesting(original_capacity_in_bytes);
}
TEST_P(PartitionAllocTest, ZapOnFree) {
void* ptr = allocator.root()->Alloc(1, type_name);
EXPECT_TRUE(ptr);
memset(ptr, 'A', 1);
allocator.root()->Free<FreeFlags::kZap>(ptr);
EXPECT_NE('A', *static_cast<unsigned char*>(ptr));
constexpr size_t size = 1024;
ptr = allocator.root()->Alloc(size, type_name);
EXPECT_TRUE(ptr);
memset(ptr, 'A', size);
allocator.root()->Free<FreeFlags::kZap>(ptr);
EXPECT_NE('A', *static_cast<unsigned char*>(ptr));
EXPECT_EQ(kFreedByte,
*(static_cast<unsigned char*>(ptr) + 2 * sizeof(void*)));
EXPECT_EQ(kFreedByte, *(static_cast<unsigned char*>(ptr) + size - 1));
}
TEST_P(PartitionAllocTest, Bug_897585) {
// Need sizes big enough to be direct mapped and a delta small enough to
// allow re-use of the slot span when cookied. These numbers fall out of the
// test case in the indicated bug.
size_t kInitialSize = 983050;
size_t kDesiredSize = 983100;
ASSERT_GT(kInitialSize, kMaxBucketed);
ASSERT_GT(kDesiredSize, kMaxBucketed);
void* ptr = allocator.root()->Alloc<AllocFlags::kReturnNull>(kInitialSize);
ASSERT_NE(nullptr, ptr);
ptr = allocator.root()->Realloc<AllocFlags::kReturnNull>(ptr, kDesiredSize,
nullptr);
ASSERT_NE(nullptr, ptr);
memset(ptr, 0xbd, kDesiredSize);
allocator.root()->Free(ptr);
}
TEST_P(PartitionAllocTest, OverrideHooks) {
constexpr size_t kOverriddenSize = 1234;
constexpr const char* kOverriddenType = "Overridden type";
constexpr unsigned char kOverriddenChar = 'A';
// Marked static so that we can use them in non-capturing lambdas below.
// (Non-capturing lambdas convert directly to function pointers.)
static volatile bool free_called = false;
static void* overridden_allocation = nullptr;
overridden_allocation = malloc(kOverriddenSize);
memset(overridden_allocation, kOverriddenChar, kOverriddenSize);
PartitionAllocHooks::SetOverrideHooks(
[](void** out, AllocFlags flags, size_t size,
const char* type_name) -> bool {
if (size == kOverriddenSize && type_name == kOverriddenType) {
*out = overridden_allocation;
return true;
}
return false;
},
[](void* address) -> bool {
if (address == overridden_allocation) {
free_called = true;
return true;
}
return false;
},
[](size_t* out, void* address) -> bool {
if (address == overridden_allocation) {
*out = kOverriddenSize;
return true;
}
return false;
});
void* ptr = allocator.root()->Alloc<AllocFlags::kReturnNull>(kOverriddenSize,
kOverriddenType);
ASSERT_EQ(ptr, overridden_allocation);
allocator.root()->Free(ptr);
EXPECT_TRUE(free_called);
// overridden_allocation has not actually been freed so we can now immediately
// realloc it.
free_called = false;
ptr = allocator.root()->Realloc<AllocFlags::kReturnNull>(ptr, 1, nullptr);
ASSERT_NE(ptr, nullptr);
EXPECT_NE(ptr, overridden_allocation);
EXPECT_TRUE(free_called);
EXPECT_EQ(*(char*)ptr, kOverriddenChar);
allocator.root()->Free(ptr);
PartitionAllocHooks::SetOverrideHooks(nullptr, nullptr, nullptr);
free(overridden_allocation);
}
TEST_P(PartitionAllocTest, Alignment) {
std::vector<void*> allocated_ptrs;
for (size_t size = 1; size <= PartitionPageSize(); size <<= 1) {
if (size <= ExtraAllocSize(allocator)) {
continue;
}
size_t requested_size = size - ExtraAllocSize(allocator);
// All allocations which are not direct-mapped occupy contiguous slots of a
// span, starting on a page boundary. This means that allocations are first
// rounded up to the nearest bucket size, then have an address of the form:
// (partition-page-aligned address) + i * bucket_size.
//
// All powers of two are bucket sizes, meaning that all power of two
// allocations smaller than a page will be aligned on the allocation size.
size_t expected_alignment = size;
for (int index = 0; index < 3; index++) {
void* ptr = allocator.root()->Alloc(requested_size);
allocated_ptrs.push_back(ptr);
EXPECT_EQ(0u,
allocator.root()->ObjectToSlotStart(ptr) % expected_alignment)
<< (index + 1) << "-th allocation of size=" << size;
}
}
for (void* ptr : allocated_ptrs) {
allocator.root()->Free(ptr);
}
}
TEST_P(PartitionAllocTest, FundamentalAlignment) {
// See the test above for details. Essentially, checking the bucket size is
// sufficient to ensure that alignment will always be respected, as long as
// the fundamental alignment is <= 16 bytes.
size_t fundamental_alignment = kAlignment;
for (size_t size = 0; size < SystemPageSize(); size++) {
// Allocate several pointers, as the first one in use in a size class will
// be aligned on a page boundary.
void* ptr = allocator.root()->Alloc(size);
void* ptr2 = allocator.root()->Alloc(size);
void* ptr3 = allocator.root()->Alloc(size);
EXPECT_EQ(UntagPtr(ptr) % fundamental_alignment, 0u);
EXPECT_EQ(UntagPtr(ptr2) % fundamental_alignment, 0u);
EXPECT_EQ(UntagPtr(ptr3) % fundamental_alignment, 0u);
uintptr_t slot_start = allocator.root()->ObjectToSlotStart(ptr);
#if BUILDFLAG(PUT_REF_COUNT_IN_PREVIOUS_SLOT)
// The capacity(C) is slot size - ExtraAllocSize(allocator).
// Since slot size is multiples of kAlignment,
// C % kAlignment == (slot_size - ExtraAllocSize(allocator)) % kAlignment.
// C % kAlignment == (-ExtraAllocSize(allocator)) % kAlignment.
// Since kCookieSize is a multiple of kAlignment,
// C % kAlignment == (-kInSlotRefCountBufferSize) % kAlignment
// == (kAlignment - kInSlotRefCountBufferSize) % kAlignment.
EXPECT_EQ(allocator.root()->AllocationCapacityFromSlotStart(slot_start) %
fundamental_alignment,
UseBRPPool()
? (-ExtraAllocSize(allocator) % fundamental_alignment)
: 0);
#else
EXPECT_EQ(allocator.root()->AllocationCapacityFromSlotStart(slot_start) %
fundamental_alignment,
-ExtraAllocSize(allocator) % fundamental_alignment);
#endif
allocator.root()->Free(ptr);
allocator.root()->Free(ptr2);
allocator.root()->Free(ptr3);
}
}
void VerifyAlignment(PartitionRoot* root, size_t size, size_t alignment) {
std::vector<void*> allocated_ptrs;
for (int index = 0; index < 3; index++) {
void* ptr = root->AlignedAlloc(alignment, size);
ASSERT_TRUE(ptr);
allocated_ptrs.push_back(ptr);
EXPECT_EQ(0ull, UntagPtr(ptr) % alignment)
<< (index + 1) << "-th allocation of size=" << size
<< ", alignment=" << alignment;
}
for (void* ptr : allocated_ptrs) {
root->Free(ptr);
}
}
TEST_P(PartitionAllocTest, AlignedAllocations) {
size_t alloc_sizes[] = {1,
10,
100,
1000,
10000,
60000,
70000,
130000,
500000,
900000,
kMaxBucketed + 1,
2 * kMaxBucketed,
kSuperPageSize - 2 * PartitionPageSize(),
4 * kMaxBucketed};
for (size_t alloc_size : alloc_sizes) {
for (size_t alignment = 1; alignment <= kMaxSupportedAlignment;
alignment <<= 1) {
VerifyAlignment(aligned_allocator.root(), alloc_size, alignment);
// Verify alignment on the regular allocator only when BRP is off, or when
// it's on in the "previous slot" mode. See the comment in SetUp().
#if !BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT) || \
BUILDFLAG(PUT_REF_COUNT_IN_PREVIOUS_SLOT)
VerifyAlignment(allocator.root(), alloc_size, alignment);
#endif
}
}
}
// Test that the optimized `GetSlotNumber` implementation produces valid
// results.
TEST_P(PartitionAllocTest, OptimizedGetSlotNumber) {
for (size_t i = 0; i < kNumBuckets; ++i) {
auto& bucket = allocator.root()->buckets[i];
if (SizeToIndex(bucket.slot_size) != i) {
continue;
}
for (size_t slot = 0, offset = 0; slot < bucket.get_slots_per_span();
++slot, offset += bucket.slot_size) {
EXPECT_EQ(slot, bucket.GetSlotNumber(offset));
EXPECT_EQ(slot, bucket.GetSlotNumber(offset + bucket.slot_size / 2));
EXPECT_EQ(slot, bucket.GetSlotNumber(offset + bucket.slot_size - 1));
}
}
}
TEST_P(PartitionAllocTest, GetUsableSizeNull) {
EXPECT_EQ(0ULL, PartitionRoot::GetUsableSize(nullptr));
}
TEST_P(PartitionAllocTest, GetUsableSize) {
#if PA_CONFIG(ENABLE_MAC11_MALLOC_SIZE_HACK)
allocator.root()->EnableMac11MallocSizeHackForTesting(
GetParam().ref_count_size);
#endif
size_t delta = 31;
for (size_t size = 1; size <= kMinDirectMappedDownsize; size += delta) {
void* ptr = allocator.root()->Alloc(size);
EXPECT_TRUE(ptr);
size_t usable_size = PartitionRoot::GetUsableSize(ptr);
size_t usable_size_with_hack =
PartitionRoot::GetUsableSizeWithMac11MallocSizeHack(ptr);
#if PA_CONFIG(ENABLE_MAC11_MALLOC_SIZE_HACK)
if (size != internal::kMac11MallocSizeHackRequestedSize)
#endif
EXPECT_EQ(usable_size_with_hack, usable_size);
EXPECT_LE(size, usable_size);
memset(ptr, 0xDE, usable_size);
// Should not crash when free the ptr.
allocator.root()->Free(ptr);
}
}
#if PA_CONFIG(ENABLE_MAC11_MALLOC_SIZE_HACK)
TEST_P(PartitionAllocTest, GetUsableSizeWithMac11MallocSizeHack) {
if (internal::base::mac::MacOSMajorVersion() != 11) {
GTEST_SKIP() << "Skipping because the test is for Mac11.";
}
allocator.root()->EnableMac11MallocSizeHackForTesting(
GetParam().ref_count_size);
size_t size = internal::kMac11MallocSizeHackRequestedSize;
void* ptr = allocator.root()->Alloc(size);
size_t usable_size = PartitionRoot::GetUsableSize(ptr);
size_t usable_size_with_hack =
PartitionRoot::GetUsableSizeWithMac11MallocSizeHack(ptr);
EXPECT_EQ(usable_size,
allocator.root()->settings.mac11_malloc_size_hack_usable_size_);
EXPECT_EQ(usable_size_with_hack, size);
allocator.root()->Free(ptr);
}
#endif // PA_CONFIG(ENABLE_MAC11_MALLOC_SIZE_HACK)
TEST_P(PartitionAllocTest, Bookkeeping) {
auto& root = *allocator.root();
EXPECT_EQ(0U, root.total_size_of_committed_pages);
EXPECT_EQ(0U, root.max_size_of_committed_pages);
EXPECT_EQ(0U, root.get_total_size_of_allocated_bytes());
EXPECT_EQ(0U, root.get_max_size_of_allocated_bytes());
EXPECT_EQ(0U, root.total_size_of_super_pages);
size_t small_size = 1000;
// A full slot span of size 1 partition page is committed.
void* ptr = root.Alloc(small_size - ExtraAllocSize(allocator), type_name);
// Lazy commit commits only needed pages.
size_t expected_committed_size =
kUseLazyCommit ? SystemPageSize() : PartitionPageSize();
size_t expected_super_pages_size = kSuperPageSize;
size_t expected_max_committed_size = expected_committed_size;
size_t bucket_index = SizeToIndex(small_size - ExtraAllocSize(allocator));
PartitionBucket* bucket = &root.buckets[bucket_index];
size_t expected_total_allocated_size = bucket->slot_size;
size_t expected_max_allocated_size = expected_total_allocated_size;
EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
EXPECT_EQ(expected_total_allocated_size,
root.get_total_size_of_allocated_bytes());
EXPECT_EQ(expected_max_allocated_size,
root.get_max_size_of_allocated_bytes());
EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
// Freeing memory doesn't result in decommitting pages right away.
root.Free(ptr);
expected_total_allocated_size = 0U;
EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
EXPECT_EQ(expected_total_allocated_size,
root.get_total_size_of_allocated_bytes());
EXPECT_EQ(expected_max_allocated_size,
root.get_max_size_of_allocated_bytes());
EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
// Allocating the same size lands it in the same slot span.
ptr = root.Alloc(small_size - ExtraAllocSize(allocator), type_name);
EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
EXPECT_EQ(expected_max_allocated_size,
root.get_max_size_of_allocated_bytes());
EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
// Freeing memory doesn't result in decommitting pages right away.
root.Free(ptr);
EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
EXPECT_EQ(expected_max_allocated_size,
root.get_max_size_of_allocated_bytes());
EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
// Allocating another size commits another slot span.
ptr = root.Alloc(2 * small_size - ExtraAllocSize(allocator), type_name);
expected_committed_size +=
kUseLazyCommit ? SystemPageSize() : PartitionPageSize();
expected_max_committed_size =
std::max(expected_max_committed_size, expected_committed_size);
expected_max_allocated_size =
std::max(expected_max_allocated_size, static_cast<size_t>(2048));
EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
EXPECT_EQ(expected_max_allocated_size,
root.get_max_size_of_allocated_bytes());
EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
// Freeing memory doesn't result in decommitting pages right away.
root.Free(ptr);
EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
EXPECT_EQ(expected_max_allocated_size,
root.get_max_size_of_allocated_bytes());
EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
// Single-slot slot spans...
//
// When the system page size is larger than 4KiB, we don't necessarily have
// enough space in the superpage to store two of the largest bucketed
// allocations, particularly when we reserve extra space for e.g. bitmaps.
// To avoid this, we use something just below kMaxBucketed.
size_t big_size = kMaxBucketed * 4 / 5 - SystemPageSize();
ASSERT_GT(big_size, MaxRegularSlotSpanSize());
ASSERT_LE(big_size, kMaxBucketed);
bucket_index = SizeToIndex(big_size - ExtraAllocSize(allocator));
bucket = &root.buckets[bucket_index];
// Assert the allocation doesn't fill the entire span nor entire partition
// page, to make the test more interesting.
ASSERT_LT(big_size, bucket->get_bytes_per_span());
ASSERT_NE(big_size % PartitionPageSize(), 0U);
ptr = root.Alloc(big_size - ExtraAllocSize(allocator), type_name);
expected_committed_size += bucket->get_bytes_per_span();
expected_max_committed_size =
std::max(expected_max_committed_size, expected_committed_size);
expected_total_allocated_size += bucket->get_bytes_per_span();
expected_max_allocated_size =
std::max(expected_max_allocated_size, expected_total_allocated_size);
EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
EXPECT_EQ(expected_total_allocated_size,
root.get_total_size_of_allocated_bytes());
EXPECT_EQ(expected_max_allocated_size,
root.get_max_size_of_allocated_bytes());
EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
// Allocating 2nd time doesn't overflow the super page...
void* ptr2 = root.Alloc(big_size - ExtraAllocSize(allocator), type_name);
expected_committed_size += bucket->get_bytes_per_span();
expected_max_committed_size =
std::max(expected_max_committed_size, expected_committed_size);
expected_total_allocated_size += bucket->get_bytes_per_span();
expected_max_allocated_size =
std::max(expected_max_allocated_size, expected_total_allocated_size);
EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
EXPECT_EQ(expected_total_allocated_size,
root.get_total_size_of_allocated_bytes());
EXPECT_EQ(expected_max_allocated_size,
root.get_max_size_of_allocated_bytes());
EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
// ... but 3rd time does.
void* ptr3 = root.Alloc(big_size - ExtraAllocSize(allocator), type_name);
expected_committed_size += bucket->get_bytes_per_span();
expected_max_committed_size =
std::max(expected_max_committed_size, expected_committed_size);
expected_total_allocated_size += bucket->get_bytes_per_span();
expected_max_allocated_size =
std::max(expected_max_allocated_size, expected_total_allocated_size);
expected_super_pages_size += kSuperPageSize;
EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
EXPECT_EQ(expected_total_allocated_size,
root.get_total_size_of_allocated_bytes());
EXPECT_EQ(expected_max_allocated_size,
root.get_max_size_of_allocated_bytes());
EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
// Freeing memory doesn't result in decommitting pages right away.
root.Free(ptr);
root.Free(ptr2);
root.Free(ptr3);
expected_total_allocated_size -= 3 * bucket->get_bytes_per_span();
expected_max_allocated_size =
std::max(expected_max_allocated_size, expected_total_allocated_size);
EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
EXPECT_EQ(expected_total_allocated_size,
root.get_total_size_of_allocated_bytes());
EXPECT_EQ(expected_max_allocated_size,
root.get_max_size_of_allocated_bytes());
EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
// Now everything should be decommitted. The reserved space for super pages
// stays the same and will never go away (by design).
root.PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans);
expected_committed_size = 0;
EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
EXPECT_EQ(expected_total_allocated_size,
root.get_total_size_of_allocated_bytes());
EXPECT_EQ(expected_max_allocated_size,
root.get_max_size_of_allocated_bytes());
EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
// None of the above should affect the direct map space.
EXPECT_EQ(0U, root.total_size_of_direct_mapped_pages);
size_t huge_sizes[] = {
kMaxBucketed + SystemPageSize(),
kMaxBucketed + SystemPageSize() + 123,
kSuperPageSize - PageAllocationGranularity(),
kSuperPageSize - SystemPageSize() - PartitionPageSize(),
kSuperPageSize - PartitionPageSize(),
kSuperPageSize - SystemPageSize(),
kSuperPageSize,
kSuperPageSize + SystemPageSize(),
kSuperPageSize + PartitionPageSize(),
kSuperPageSize + SystemPageSize() + PartitionPageSize(),
kSuperPageSize + PageAllocationGranularity(),
kSuperPageSize + DirectMapAllocationGranularity(),
};
size_t alignments[] = {
PartitionPageSize(),
2 * PartitionPageSize(),
kMaxSupportedAlignment / 2,
kMaxSupportedAlignment,
};
for (size_t huge_size : huge_sizes) {
ASSERT_GT(huge_size, kMaxBucketed);
for (size_t alignment : alignments) {
// For direct map, we commit only as many pages as needed.
size_t aligned_size = partition_alloc::internal::base::bits::AlignUp(
huge_size, SystemPageSize());
ptr = root.AllocInternalForTesting(huge_size - ExtraAllocSize(allocator),
alignment, type_name);
expected_committed_size += aligned_size;
expected_max_committed_size =
std::max(expected_max_committed_size, expected_committed_size);
expected_total_allocated_size += aligned_size;
expected_max_allocated_size =
std::max(expected_max_allocated_size, expected_total_allocated_size);
// The total reserved map includes metadata and guard pages at the ends.
// It also includes alignment. However, these would double count the first
// partition page, so it needs to be subtracted.
size_t surrounding_pages_size =
PartitionRoot::GetDirectMapMetadataAndGuardPagesSize() + alignment -
PartitionPageSize();
size_t expected_direct_map_size =
partition_alloc::internal::base::bits::AlignUp(
aligned_size + surrounding_pages_size,
DirectMapAllocationGranularity());
EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
EXPECT_EQ(expected_total_allocated_size,
root.get_total_size_of_allocated_bytes());
EXPECT_EQ(expected_max_allocated_size,
root.get_max_size_of_allocated_bytes());
EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
EXPECT_EQ(expected_direct_map_size,
root.total_size_of_direct_mapped_pages);
// Freeing memory in the diret map decommits pages right away. The address
// space is released for re-use too.
root.Free(ptr);
expected_committed_size -= aligned_size;
expected_direct_map_size = 0;
expected_max_committed_size =
std::max(expected_max_committed_size, expected_committed_size);
expected_total_allocated_size -= aligned_size;
expected_max_allocated_size =
std::max(expected_max_allocated_size, expected_total_allocated_size);
EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
EXPECT_EQ(expected_total_allocated_size,
root.get_total_size_of_allocated_bytes());
EXPECT_EQ(expected_max_allocated_size,
root.get_max_size_of_allocated_bytes());
EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
EXPECT_EQ(expected_direct_map_size,
root.total_size_of_direct_mapped_pages);
}
}
}
#if BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT)
TEST_P(PartitionAllocTest, RefCountBasic) {
if (!UseBRPPool()) {
return;
}
constexpr uint64_t kCookie = 0x1234567890ABCDEF;
constexpr uint64_t kQuarantined = 0xEFEFEFEFEFEFEFEF;
size_t alloc_size = 64 - ExtraAllocSize(allocator);
uint64_t* ptr1 =
static_cast<uint64_t*>(allocator.root()->Alloc(alloc_size, type_name));
EXPECT_TRUE(ptr1);
*ptr1 = kCookie;
auto* ref_count =
PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr1));
EXPECT_TRUE(ref_count->IsAliveWithNoKnownRefs());
ref_count->Acquire();
EXPECT_FALSE(ref_count->Release());
EXPECT_TRUE(ref_count->IsAliveWithNoKnownRefs());
EXPECT_EQ(*ptr1, kCookie);
ref_count->AcquireFromUnprotectedPtr();
EXPECT_FALSE(ref_count->IsAliveWithNoKnownRefs());
allocator.root()->Free(ptr1);
// The allocation shouldn't be reclaimed, and its contents should be zapped.
// Retag ptr1 to get its correct MTE tag.
ptr1 = TagPtr(ptr1);
EXPECT_NE(*ptr1, kCookie);
EXPECT_EQ(*ptr1, kQuarantined);
// The allocator should not reuse the original slot since its reference count
// doesn't equal zero.
uint64_t* ptr2 =
static_cast<uint64_t*>(allocator.root()->Alloc(alloc_size, type_name));
EXPECT_NE(ptr1, ptr2);
allocator.root()->Free(ptr2);
// When the last reference is released, the slot should become reusable.
// Retag ref_count because PartitionAlloc retags ptr to enforce quarantine.
ref_count = TagPtr(ref_count);
EXPECT_TRUE(ref_count->ReleaseFromUnprotectedPtr());
PartitionAllocFreeForRefCounting(allocator.root()->ObjectToSlotStart(ptr1));
uint64_t* ptr3 =
static_cast<uint64_t*>(allocator.root()->Alloc(alloc_size, type_name));
EXPECT_EQ(ptr1, ptr3);
allocator.root()->Free(ptr3);
}
void PartitionAllocTest::RunRefCountReallocSubtest(size_t orig_size,
size_t new_size) {
void* ptr1 = allocator.root()->Alloc(orig_size, type_name);
EXPECT_TRUE(ptr1);
auto* ref_count1 =
PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr1));
EXPECT_TRUE(ref_count1->IsAliveWithNoKnownRefs());
ref_count1->AcquireFromUnprotectedPtr();
EXPECT_FALSE(ref_count1->IsAliveWithNoKnownRefs());
void* ptr2 = allocator.root()->Realloc(ptr1, new_size, type_name);
EXPECT_TRUE(ptr2);
// PartitionAlloc may retag memory areas on realloc (even if they
// do not move), so recover the true tag here.
ref_count1 = TagPtr(ref_count1);
// Re-query ref-count. It may have moved if Realloc changed the slot.
auto* ref_count2 =
PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr2));
if (UntagPtr(ptr1) == UntagPtr(ptr2)) {
// If the slot didn't change, ref-count should stay the same.
EXPECT_EQ(ref_count1, ref_count2);
EXPECT_FALSE(ref_count2->IsAliveWithNoKnownRefs());
EXPECT_FALSE(ref_count2->ReleaseFromUnprotectedPtr());
} else {
// If the allocation was moved to another slot, the old ref-count stayed
// in the same location in memory, is no longer alive, but still has a
// reference. The new ref-count is alive, but has no references.
EXPECT_NE(ref_count1, ref_count2);
EXPECT_FALSE(ref_count1->IsAlive());
EXPECT_FALSE(ref_count1->IsAliveWithNoKnownRefs());
EXPECT_TRUE(ref_count2->IsAliveWithNoKnownRefs());
EXPECT_TRUE(ref_count1->ReleaseFromUnprotectedPtr());
PartitionAllocFreeForRefCounting(allocator.root()->ObjectToSlotStart(ptr1));
}
allocator.root()->Free(ptr2);
}
TEST_P(PartitionAllocTest, RefCountRealloc) {
if (!UseBRPPool()) {
return;
}
size_t alloc_sizes[] = {500, 5000, 50000, 400000};
for (size_t alloc_size : alloc_sizes) {
alloc_size -= ExtraAllocSize(allocator);
RunRefCountReallocSubtest(alloc_size, alloc_size - 9);
RunRefCountReallocSubtest(alloc_size, alloc_size + 9);
RunRefCountReallocSubtest(alloc_size, alloc_size * 2);
RunRefCountReallocSubtest(alloc_size, alloc_size / 2);
}
}
int g_unretained_dangling_raw_ptr_detected_count = 0;
class UnretainedDanglingRawPtrTest : public PartitionAllocTest {
public:
void SetUp() override {
PartitionAllocTest::SetUp();
g_unretained_dangling_raw_ptr_detected_count = 0;
old_detected_fn_ = partition_alloc::GetUnretainedDanglingRawPtrDetectedFn();
partition_alloc::SetUnretainedDanglingRawPtrDetectedFn(
&UnretainedDanglingRawPtrTest::DanglingRawPtrDetected);
old_unretained_dangling_ptr_enabled_ =
partition_alloc::SetUnretainedDanglingRawPtrCheckEnabled(true);
}
void TearDown() override {
partition_alloc::SetUnretainedDanglingRawPtrDetectedFn(old_detected_fn_);
partition_alloc::SetUnretainedDanglingRawPtrCheckEnabled(
old_unretained_dangling_ptr_enabled_);
PartitionAllocTest::TearDown();
}
private:
static void DanglingRawPtrDetected(uintptr_t) {
g_unretained_dangling_raw_ptr_detected_count++;
}
partition_alloc::DanglingRawPtrDetectedFn* old_detected_fn_;
bool old_unretained_dangling_ptr_enabled_;
};
INSTANTIATE_TEST_SUITE_P(AlternateTestParams,
UnretainedDanglingRawPtrTest,
testing::ValuesIn(GetPartitionAllocTestParams()));
TEST_P(UnretainedDanglingRawPtrTest, UnretainedDanglingPtrNoReport) {
if (!UseBRPPool()) {
return;
}
void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
EXPECT_TRUE(ptr);
auto* ref_count =
PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
ref_count->Acquire();
EXPECT_TRUE(ref_count->IsAlive());
// Allocation is still live, so calling ReportIfDangling() should not result
// in any detections.
ref_count->ReportIfDangling();
EXPECT_EQ(g_unretained_dangling_raw_ptr_detected_count, 0);
EXPECT_FALSE(ref_count->Release());
allocator.root()->Free(ptr);
}
TEST_P(UnretainedDanglingRawPtrTest, UnretainedDanglingPtrShouldReport) {
if (!UseBRPPool()) {
return;
}
void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
EXPECT_TRUE(ptr);
auto* ref_count =
PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
ref_count->AcquireFromUnprotectedPtr();
EXPECT_TRUE(ref_count->IsAlive());
allocator.root()->Free(ptr);
// At this point, memory shouldn't be alive...
EXPECT_FALSE(ref_count->IsAlive());
// ...and we should report the ptr as dangling.
ref_count->ReportIfDangling();
EXPECT_EQ(g_unretained_dangling_raw_ptr_detected_count, 1);
EXPECT_TRUE(ref_count->ReleaseFromUnprotectedPtr());
PartitionAllocFreeForRefCounting(allocator.root()->ObjectToSlotStart(ptr));
}
#if !BUILDFLAG(HAS_64_BIT_POINTERS)
TEST_P(PartitionAllocTest, BackupRefPtrGuardRegion) {
if (!UseBRPPool()) {
return;
}
size_t alignment = internal::PageAllocationGranularity();
uintptr_t requested_address;
memset(&requested_address, internal::kQuarantinedByte,
sizeof(requested_address));
requested_address = RoundDownToPageAllocationGranularity(requested_address);
uintptr_t allocated_address =
AllocPages(requested_address, alignment, alignment,
PageAccessibilityConfiguration(
PageAccessibilityConfiguration::kReadWrite),
PageTag::kPartitionAlloc);
EXPECT_NE(allocated_address, requested_address);
if (allocated_address) {
FreePages(allocated_address, alignment);
}
}
#endif // !BUILDFLAG(HAS_64_BIT_POINTERS)
#endif // BUILDFLAG(ENABLE_BACKUP_REF_PTR_SUPPORT)
#if BUILDFLAG(ENABLE_DANGLING_RAW_PTR_CHECKS)
// Allocate memory, and reference it from 3 raw_ptr. Among them 2 will be
// dangling.
TEST_P(PartitionAllocTest, DanglingPtr) {
if (!UseBRPPool()) {
return;
}
CountDanglingRawPtr dangling_checks;
// Allocate memory, and reference it from 3 raw_ptr.
uint64_t* ptr = static_cast<uint64_t*>(
allocator.root()->Alloc(64 - ExtraAllocSize(allocator), type_name));
auto* ref_count =
PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
ref_count->Acquire();
ref_count->Acquire();
ref_count->Acquire();
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// The first raw_ptr stops referencing it, before the memory has been
// released.
EXPECT_FALSE(ref_count->Release());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
#if BUILDFLAG(ENABLE_DANGLING_RAW_PTR_PERF_EXPERIMENT)
// Free it. This creates two dangling pointer.
allocator.root()->Free(ptr);
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// The dangling raw_ptr stop referencing it.
EXPECT_FALSE(ref_count->Release());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// The dangling raw_ptr stop referencing it again.
EXPECT_TRUE(ref_count->Release());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
#else
// Free it. This creates two dangling pointer.
allocator.root()->Free(ptr);
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 1);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// The dangling raw_ptr stop referencing it.
EXPECT_FALSE(ref_count->Release());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 1);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 1);
// The dangling raw_ptr stop referencing it again.
EXPECT_TRUE(ref_count->Release());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 1);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 2);
#endif
PartitionAllocFreeForRefCounting(allocator.root()->ObjectToSlotStart(ptr));
}
// Allocate memory, and reference it from 3
// raw_ptr<T, DisableDanglingPtrDetection>. Among them 2 will be dangling. This
// doesn't trigger any dangling raw_ptr checks.
TEST_P(PartitionAllocTest, DanglingDanglingPtr) {
if (!UseBRPPool()) {
return;
}
CountDanglingRawPtr dangling_checks;
// Allocate memory, and reference it from 3 raw_ptr.
uint64_t* ptr = static_cast<uint64_t*>(
allocator.root()->Alloc(64 - ExtraAllocSize(allocator), type_name));
auto* ref_count =
PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
ref_count->AcquireFromUnprotectedPtr();
ref_count->AcquireFromUnprotectedPtr();
ref_count->AcquireFromUnprotectedPtr();
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// The first raw_ptr<T, DisableDanglingPtrDetection> stops referencing it,
// before the memory has been released.
EXPECT_FALSE(ref_count->ReleaseFromUnprotectedPtr());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// Free it. This creates two dangling raw_ptr<T, DisableDanglingPtrDetection>.
allocator.root()->Free(ptr);
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// The dangling raw_ptr<T, DisableDanglingPtrDetection> stop referencing it.
EXPECT_FALSE(ref_count->ReleaseFromUnprotectedPtr());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// The dangling raw_ptr<T, DisableDanglingPtrDetection> stop referencing it
// again.
EXPECT_TRUE(ref_count->ReleaseFromUnprotectedPtr());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
PartitionAllocFreeForRefCounting(allocator.root()->ObjectToSlotStart(ptr));
}
// When 'free' is called, it remain one raw_ptr<> and one
// raw_ptr<T, DisableDanglingPtrDetection>. The raw_ptr<> is released first.
TEST_P(PartitionAllocTest, DanglingMixedReleaseRawPtrFirst) {
if (!UseBRPPool()) {
return;
}
CountDanglingRawPtr dangling_checks;
uint64_t* ptr = static_cast<uint64_t*>(
allocator.root()->Alloc(64 - ExtraAllocSize(allocator), type_name));
auto* ref_count =
PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
// Acquire a raw_ptr<T, DisableDanglingPtrDetection> and a raw_ptr<>.
ref_count->AcquireFromUnprotectedPtr();
ref_count->Acquire();
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
#if BUILDFLAG(ENABLE_DANGLING_RAW_PTR_PERF_EXPERIMENT)
// Free it.
allocator.root()->Free(ptr);
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// The raw_ptr<> stops referencing it.
EXPECT_FALSE(ref_count->Release());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// The raw_ptr<T, DisableDanglingPtrDetection> stop referencing it.
EXPECT_TRUE(ref_count->ReleaseFromUnprotectedPtr());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
#else
// Free it.
allocator.root()->Free(ptr);
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 1);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// The raw_ptr<> stops referencing it.
EXPECT_FALSE(ref_count->Release());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 1);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 1);
// The raw_ptr<T, DisableDanglingPtrDetection> stop referencing it.
EXPECT_TRUE(ref_count->ReleaseFromUnprotectedPtr());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 1);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 1);
#endif
PartitionAllocFreeForRefCounting(allocator.root()->ObjectToSlotStart(ptr));
}
// When 'free' is called, it remain one raw_ptr<> and one
// raw_ptr<T, DisableDanglingPtrDetection>.
// The raw_ptr<T, DisableDanglingPtrDetection> is released first. This
// triggers the dangling raw_ptr<> checks.
TEST_P(PartitionAllocTest, DanglingMixedReleaseDanglingPtrFirst) {
if (!UseBRPPool()) {
return;
}
CountDanglingRawPtr dangling_checks;
void* ptr =
allocator.root()->Alloc(64 - ExtraAllocSize(allocator), type_name);
auto* ref_count =
PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
// Acquire a raw_ptr<T, DisableDanglingPtrDetection> and a raw_ptr<>.
ref_count->AcquireFromUnprotectedPtr();
ref_count->Acquire();
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
#if BUILDFLAG(ENABLE_DANGLING_RAW_PTR_PERF_EXPERIMENT)
// Free it.
allocator.root()->Free(ptr);
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// The raw_ptr<> stops referencing it.
EXPECT_FALSE(ref_count->ReleaseFromUnprotectedPtr());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// The raw_ptr<T, DisableDanglingPtrDetection> stop referencing it.
EXPECT_TRUE(ref_count->Release());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
#else
// Free it.
allocator.root()->Free(ptr);
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 1);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// The raw_ptr<> stops referencing it.
EXPECT_FALSE(ref_count->ReleaseFromUnprotectedPtr());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 1);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// The raw_ptr<T, DisableDanglingPtrDetection> stop referencing it.
EXPECT_TRUE(ref_count->Release());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 1);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 1);
#endif
PartitionAllocFreeForRefCounting(allocator.root()->ObjectToSlotStart(ptr));
}
// When 'free' is called, it remains one
// raw_ptr<T, DisableDanglingPtrDetection>, then it is used to acquire one
// dangling raw_ptr<>. Release the raw_ptr<> first.
TEST_P(PartitionAllocTest, DanglingPtrUsedToAcquireNewRawPtr) {
if (!UseBRPPool()) {
return;
}
CountDanglingRawPtr dangling_checks;
void* ptr =
allocator.root()->Alloc(64 - ExtraAllocSize(allocator), type_name);
auto* ref_count =
PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
// Acquire a raw_ptr<T, DisableDanglingPtrDetection>.
ref_count->AcquireFromUnprotectedPtr();
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// Free it once.
allocator.root()->Free(ptr);
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// A raw_ptr<> starts referencing it.
ref_count->Acquire();
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// The raw_ptr<> stops referencing it.
EXPECT_FALSE(ref_count->Release());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// The raw_ptr<T, DisableDanglingPtrDetection> stops referencing it.
EXPECT_TRUE(ref_count->ReleaseFromUnprotectedPtr());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
PartitionAllocFreeForRefCounting(allocator.root()->ObjectToSlotStart(ptr));
}
// Same as 'DanglingPtrUsedToAcquireNewRawPtr', but release the
// raw_ptr<T, DisableDanglingPtrDetection> before the raw_ptr<>.
TEST_P(PartitionAllocTest, DanglingPtrUsedToAcquireNewRawPtrVariant) {
if (!UseBRPPool()) {
return;
}
CountDanglingRawPtr dangling_checks;
void* ptr =
allocator.root()->Alloc(64 - ExtraAllocSize(allocator), type_name);
auto* ref_count =
PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
// Acquire a raw_ptr<T, DisableDanglingPtrDetection>.
ref_count->AcquireFromUnprotectedPtr();
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// Free it.
allocator.root()->Free(ptr);
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// A raw_ptr<> starts referencing it.
ref_count->Acquire();
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// The raw_ptr<> stops referencing it.
EXPECT_FALSE(ref_count->ReleaseFromUnprotectedPtr());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// The raw_ptr<T, DisableDanglingPtrDetection> stops referencing it.
EXPECT_TRUE(ref_count->Release());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
PartitionAllocFreeForRefCounting(allocator.root()->ObjectToSlotStart(ptr));
}
// Acquire a raw_ptr<T>, and release it before freeing memory. In the
// background, there is one raw_ptr<T, DisableDanglingPtrDetection>. This
// doesn't trigger any dangling raw_ptr<T> checks.
TEST_P(PartitionAllocTest, RawPtrReleasedBeforeFree) {
if (!UseBRPPool()) {
return;
}
CountDanglingRawPtr dangling_checks;
void* ptr =
allocator.root()->Alloc(64 - ExtraAllocSize(allocator), type_name);
auto* ref_count =
PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
// Acquire a raw_ptr<T, DisableDanglingPtrDetection> and a raw_ptr<>.
ref_count->Acquire();
ref_count->AcquireFromUnprotectedPtr();
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// Release the raw_ptr<>.
EXPECT_FALSE(ref_count->Release());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// Free it.
allocator.root()->Free(ptr);
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
// The raw_ptr<T, DisableDanglingPtrDetection> stop referencing it.
EXPECT_TRUE(ref_count->ReleaseFromUnprotectedPtr());
EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
PartitionAllocFreeForRefCounting(allocator.root()->ObjectToSlotStart(ptr));
}
#if defined(PA_HAS_DEATH_TESTS)
// DCHECK message are stripped in official build. It causes death tests with
// matchers to fail.
#if !defined(OFFICIAL_BUILD) || !defined(NDEBUG)
// Acquire() once, Release() twice => CRASH
TEST_P(PartitionAllocDeathTest, ReleaseUnderflowRawPtr) {
if (!UseBRPPool()) {
return;
}
void* ptr =
allocator.root()->Alloc(64 - ExtraAllocSize(allocator), type_name);
auto* ref_count =
PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
ref_count->Acquire();
EXPECT_FALSE(ref_count->Release());
EXPECT_DCHECK_DEATH(ref_count->Release());
allocator.root()->Free(ptr);
}
// AcquireFromUnprotectedPtr() once, ReleaseFromUnprotectedPtr() twice => CRASH
TEST_P(PartitionAllocDeathTest, ReleaseUnderflowDanglingPtr) {
if (!UseBRPPool()) {
return;
}
void* ptr =
allocator.root()->Alloc(64 - ExtraAllocSize(allocator), type_name);
auto* ref_count =
PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
ref_count->AcquireFromUnprotectedPtr();
EXPECT_FALSE(ref_count->ReleaseFromUnprotectedPtr());
EXPECT_DCHECK_DEATH(ref_count->ReleaseFromUnprotectedPtr());
allocator.root()->Free(ptr);
}
#endif //! defined(OFFICIAL_BUILD) || !defined(NDEBUG)
#endif // defined(PA_HAS_DEATH_TESTS)
#endif // BUILDFLAG(ENABLE_DANGLING_RAW_PTR_CHECKS)
TEST_P(PartitionAllocTest, ReservationOffset) {
// For normal buckets, offset should be kOffsetTagNormalBuckets.
void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
EXPECT_TRUE(ptr);
uintptr_t address = UntagPtr(ptr);
EXPECT_EQ(kOffsetTagNormalBuckets, *ReservationOffsetPointer(address));
allocator.root()->Free(ptr);
// For direct-map,
size_t large_size = kSuperPageSize * 5 + PartitionPageSize() * .5f;
ASSERT_GT(large_size, kMaxBucketed);
ptr = allocator.root()->Alloc(large_size, type_name);
EXPECT_TRUE(ptr);
address = UntagPtr(ptr);
EXPECT_EQ(0U, *ReservationOffsetPointer(address));
EXPECT_EQ(1U, *ReservationOffsetPointer(address + kSuperPageSize));
EXPECT_EQ(2U, *ReservationOffsetPointer(address + kSuperPageSize * 2));
EXPECT_EQ(3U, *ReservationOffsetPointer(address + kSuperPageSize * 3));
EXPECT_EQ(4U, *ReservationOffsetPointer(address + kSuperPageSize * 4));
EXPECT_EQ(5U, *ReservationOffsetPointer(address + kSuperPageSize * 5));
// In-place realloc doesn't affect the offsets.
void* new_ptr = allocator.root()->Realloc(ptr, large_size * .8, type_name);
EXPECT_EQ(new_ptr, ptr);
EXPECT_EQ(0U, *ReservationOffsetPointer(address));
EXPECT_EQ(1U, *ReservationOffsetPointer(address + kSuperPageSize));
EXPECT_EQ(2U, *ReservationOffsetPointer(address + kSuperPageSize * 2));
EXPECT_EQ(3U, *ReservationOffsetPointer(address + kSuperPageSize * 3));
EXPECT_EQ(4U, *ReservationOffsetPointer(address + kSuperPageSize * 4));
EXPECT_EQ(5U, *ReservationOffsetPointer(address + kSuperPageSize * 5));
allocator.root()->Free(ptr);
// After free, the offsets must be kOffsetTagNotAllocated.
EXPECT_EQ(kOffsetTagNotAllocated, *ReservationOffsetPointer(address));
EXPECT_EQ(kOffsetTagNotAllocated,
*ReservationOffsetPointer(address + kSuperPageSize));
EXPECT_EQ(kOffsetTagNotAllocated,
*ReservationOffsetPointer(address + kSuperPageSize * 2));
EXPECT_EQ(kOffsetTagNotAllocated,
*ReservationOffsetPointer(address + kSuperPageSize * 3));
EXPECT_EQ(kOffsetTagNotAllocated,
*ReservationOffsetPointer(address + kSuperPageSize * 4));
EXPECT_EQ(kOffsetTagNotAllocated,
*ReservationOffsetPointer(address + kSuperPageSize * 5));
}
TEST_P(PartitionAllocTest, GetReservationStart) {
size_t large_size = kSuperPageSize * 3 + PartitionPageSize() * .5f;
ASSERT_GT(large_size, kMaxBucketed);
void* ptr = allocator.root()->Alloc(large_size, type_name);
EXPECT_TRUE(ptr);
uintptr_t slot_start = allocator.root()->ObjectToSlotStart(ptr);
uintptr_t reservation_start = slot_start - PartitionPageSize();
EXPECT_EQ(0U, reservation_start & DirectMapAllocationGranularityOffsetMask());
uintptr_t address = UntagPtr(ptr);
for (uintptr_t a = address; a < address + large_size; ++a) {
uintptr_t address2 = GetDirectMapReservationStart(a) + PartitionPageSize();
EXPECT_EQ(slot_start, address2);
}
EXPECT_EQ(reservation_start, GetDirectMapReservationStart(slot_start));
allocator.root()->Free(ptr);
}
TEST_P(PartitionAllocTest, CheckReservationType) {
void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
EXPECT_TRUE(ptr);
uintptr_t address = UntagPtr(ptr);
uintptr_t address_to_check = address;
EXPECT_FALSE(IsReservationStart(address_to_check));
EXPECT_TRUE(IsManagedByNormalBuckets(address_to_check));
EXPECT_FALSE(IsManagedByDirectMap(address_to_check));
EXPECT_TRUE(IsManagedByNormalBucketsOrDirectMap(address_to_check));
address_to_check = address + kTestAllocSize - 1;
EXPECT_FALSE(IsReservationStart(address_to_check));
EXPECT_TRUE(IsManagedByNormalBuckets(address_to_check));
EXPECT_FALSE(IsManagedByDirectMap(address_to_check));
EXPECT_TRUE(IsManagedByNormalBucketsOrDirectMap(address_to_check));
address_to_check =
partition_alloc::internal::base::bits::AlignDown(address, kSuperPageSize);
EXPECT_TRUE(IsReservationStart(address_to_check));
EXPECT_TRUE(IsManagedByNormalBuckets(address_to_check));
EXPECT_FALSE(IsManagedByDirectMap(address_to_check));
EXPECT_TRUE(IsManagedByNormalBucketsOrDirectMap(address_to_check));
allocator.root()->Free(ptr);
// Freeing keeps a normal-bucket super page in memory.
address_to_check =
partition_alloc::internal::base::bits::AlignDown(address, kSuperPageSize);
EXPECT_TRUE(IsReservationStart(address_to_check));
EXPECT_TRUE(IsManagedByNormalBuckets(address_to_check));
EXPECT_FALSE(IsManagedByDirectMap(address_to_check));
EXPECT_TRUE(IsManagedByNormalBucketsOrDirectMap(address_to_check));
size_t large_size = 2 * kSuperPageSize;
ASSERT_GT(large_size, kMaxBucketed);
ptr = allocator.root()->Alloc(large_size, type_name);
EXPECT_TRUE(ptr);
address = UntagPtr(ptr);
address_to_check = address;
EXPECT_FALSE(IsReservationStart(address_to_check));
EXPECT_FALSE(IsManagedByNormalBuckets(address_to_check));
EXPECT_TRUE(IsManagedByDirectMap(address_to_check));
EXPECT_TRUE(IsManagedByNormalBucketsOrDirectMap(address_to_check));
address_to_check =
partition_alloc::internal::base::bits::AlignUp(address, kSuperPageSize);
EXPECT_FALSE(IsReservationStart(address_to_check));
EXPECT_FALSE(IsManagedByNormalBuckets(address_to_check));
EXPECT_TRUE(IsManagedByDirectMap(address_to_check));
EXPECT_TRUE(IsManagedByNormalBucketsOrDirectMap(address_to_check));
address_to_check = address + large_size - 1;
EXPECT_FALSE(IsReservationStart(address_to_check));
EXPECT_FALSE(IsManagedByNormalBuckets(address_to_check));
EXPECT_TRUE(IsManagedByDirectMap(address_to_check));
EXPECT_TRUE(IsManagedByNormalBucketsOrDirectMap(address_to_check));
address_to_check =
partition_alloc::internal::base::bits::AlignDown(address, kSuperPageSize);
EXPECT_TRUE(IsReservationStart(address_to_check));
EXPECT_FALSE(IsManagedByNormalBuckets(address_to_check));
EXPECT_TRUE(IsManagedByDirectMap(address_to_check));
EXPECT_TRUE(IsManagedByNormalBucketsOrDirectMap(address_to_check));
allocator.root()->Free(ptr);
// Freeing releases direct-map super pages.
address_to_check =
partition_alloc::internal::base::bits::AlignDown(address, kSuperPageSize);
// DCHECKs don't work with EXPECT_DEATH on official builds.
#if BUILDFLAG(PA_DCHECK_IS_ON) && (!defined(OFFICIAL_BUILD) || !defined(NDEBUG))
// Expect to DCHECK on unallocated region.
EXPECT_DEATH_IF_SUPPORTED(IsReservationStart(address_to_check), "");
#endif // BUILDFLAG(PA_DCHECK_IS_ON) && (!defined(OFFICIAL_BUILD) ||
// !defined(NDEBUG))
EXPECT_FALSE(IsManagedByNormalBuckets(address_to_check));
EXPECT_FALSE(IsManagedByDirectMap(address_to_check));
EXPECT_FALSE(IsManagedByNormalBucketsOrDirectMap(address_to_check));
}
// Test for crash http://crbug.com/1169003.
TEST_P(PartitionAllocTest, CrossPartitionRootRealloc) {
// Size is large enough to satisfy it from a single-slot slot span
size_t test_size = MaxRegularSlotSpanSize() - ExtraAllocSize(allocator);
void* ptr = allocator.root()->Alloc<AllocFlags::kReturnNull>(test_size);
EXPECT_TRUE(ptr);
// Create new root and call PurgeMemory to simulate ConfigurePartitions().
allocator.root()->PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans |
PurgeFlags::kDiscardUnusedSystemPages);
std::unique_ptr<PartitionRoot> new_root = CreateCustomTestRoot(
GetCommonPartitionOptions(),
PartitionTestOptions{.set_bucket_distribution = true});
// Realloc from |allocator.root()| into |new_root|.
void* ptr2 = new_root->Realloc<AllocFlags::kReturnNull>(ptr, test_size + 1024,
nullptr);
EXPECT_TRUE(ptr2);
PA_EXPECT_PTR_NE(ptr, ptr2);
}
TEST_P(PartitionAllocTest, FastPathOrReturnNull) {
size_t allocation_size = 64;
// The very first allocation is never a fast path one, since it needs a new
// super page and a new partition page.
EXPECT_FALSE(allocator.root()->Alloc<AllocFlags::kFastPathOrReturnNull>(
allocation_size, ""));
void* ptr = allocator.root()->Alloc(allocation_size);
ASSERT_TRUE(ptr);
// Next one is, since the partition page has been activated.
void* ptr2 = allocator.root()->Alloc<AllocFlags::kFastPathOrReturnNull>(
allocation_size, "");
EXPECT_TRUE(ptr2);
// First allocation of a different bucket is slow.
EXPECT_FALSE(allocator.root()->Alloc<AllocFlags::kFastPathOrReturnNull>(
2 * allocation_size, ""));
size_t allocated_size = 2 * allocation_size;
std::vector<void*> ptrs;
while (void* new_ptr =
allocator.root()->Alloc<AllocFlags::kFastPathOrReturnNull>(
allocation_size, "")) {
ptrs.push_back(new_ptr);
allocated_size += allocation_size;
}
EXPECT_LE(allocated_size,
PartitionPageSize() * kMaxPartitionPagesPerRegularSlotSpan);
for (void* ptr_to_free : ptrs) {
allocator.root()->Free<FreeFlags::kNoHooks>(ptr_to_free);
}
allocator.root()->Free<FreeFlags::kNoHooks>(ptr);
allocator.root()->Free<FreeFlags::kNoHooks>(ptr2);
}
#if defined(PA_HAS_DEATH_TESTS)
// DCHECK message are stripped in official build. It causes death tests with
// matchers to fail.
#if !defined(OFFICIAL_BUILD) || !defined(NDEBUG)
TEST_P(PartitionAllocDeathTest, CheckTriggered) {
EXPECT_DCHECK_DEATH_WITH(PA_CHECK(5 == 7), "Check failed.*5 == 7");
EXPECT_DEATH(PA_CHECK(5 == 7), "Check failed.*5 == 7");
}
#endif // !defined(OFFICIAL_BUILD) && !defined(NDEBUG)
#endif // defined(PA_HAS_DEATH_TESTS)
// Not on chromecast, since gtest considers extra output from itself as a test
// failure:
// https://ci.chromium.org/ui/p/chromium/builders/ci/Cast%20Audio%20Linux/98492/overview
#if BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC) && defined(PA_HAS_DEATH_TESTS) && \
!BUILDFLAG(PA_IS_CASTOS)
namespace {
PA_NOINLINE void FreeForTest(void* data) {
free(data);
}
class ThreadDelegateForPreforkHandler
: public base::PlatformThreadForTesting::Delegate {
public:
ThreadDelegateForPreforkHandler(std::atomic<bool>& please_stop,
std::atomic<int>& started_threads,
const int alloc_size)
: please_stop_(please_stop),
started_threads_(started_threads),
alloc_size_(alloc_size) {}
void ThreadMain() override {
started_threads_++;
while (!please_stop_.load(std::memory_order_relaxed)) {
void* ptr = malloc(alloc_size_);
// A simple malloc() / free() pair can be discarded by the compiler (and
// is), making the test fail. It is sufficient to make |FreeForTest()| a
// PA_NOINLINE function for the call to not be eliminated, but it is
// required.
FreeForTest(ptr);
}
}
private:
std::atomic<bool>& please_stop_;
std::atomic<int>& started_threads_;
const int alloc_size_;
};
} // namespace
// Disabled because executing it causes Gtest to show a warning in the output,
// which confuses the runner on some platforms, making the test report an
// "UNKNOWN" status even though it succeeded.
TEST_P(PartitionAllocTest, DISABLED_PreforkHandler) {
std::atomic<bool> please_stop;
std::atomic<int> started_threads{0};
// Continuously allocates / frees memory, bypassing the thread cache. This
// makes it likely that this thread will own the lock, and that the
// EXPECT_EXIT() part will deadlock.
constexpr size_t kAllocSize = ThreadCache::kLargeSizeThreshold + 1;
ThreadDelegateForPreforkHandler delegate(please_stop, started_threads,
kAllocSize);
constexpr int kThreads = 4;
base::PlatformThreadHandle thread_handles[kThreads];
for (auto& thread_handle : thread_handles) {
base::PlatformThreadForTesting::Create(0, &delegate, &thread_handle);
}
// Make sure all threads are actually already running.
while (started_threads != kThreads) {
}
EXPECT_EXIT(
{
void* ptr = malloc(kAllocSize);
FreeForTest(ptr);
exit(1);
},
::testing::ExitedWithCode(1), "");
please_stop.store(true);
for (auto& thread_handle : thread_handles) {
base::PlatformThreadForTesting::Join(thread_handle);
}
}
#endif // BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC) &&
// PA_CONFIG(HAS_DEATH_TESTS) && !BUILDFLAG(PA_IS_CASTOS)
// Checks the bucket index logic.
TEST_P(PartitionAllocTest, GetIndex) {
BucketIndexLookup lookup{};
for (size_t size = 0; size < kMaxBucketed; size++) {
size_t index = BucketIndexLookup::GetIndex(size);
ASSERT_GE(lookup.bucket_sizes()[index], size);
}
// Make sure that power-of-two have exactly matching buckets.
for (size_t size = (1 << (kMinBucketedOrder - 1)); size < kMaxBucketed;
size <<= 1) {
size_t index = BucketIndexLookup::GetIndex(size);
ASSERT_EQ(lookup.bucket_sizes()[index], size);
}
}
// Used to check alignment. If the compiler understands the annotations, the
// zeroing in the constructor uses aligned SIMD instructions.
TEST_P(PartitionAllocTest, MallocFunctionAnnotations) {
struct TestStruct {
uint64_t a = 0;
uint64_t b = 0;
};
void* buffer = Alloc(sizeof(TestStruct));
// Should use "mov*a*ps" on x86_64.
auto* x = new (buffer) TestStruct();
EXPECT_EQ(x->a, 0u);
Free(buffer);
}
// Test that the ConfigurablePool works properly.
TEST_P(PartitionAllocTest, ConfigurablePool) {
EXPECT_FALSE(IsConfigurablePoolAvailable());
// The rest is only applicable to 64-bit mode
#if defined(ARCH_CPU_64_BITS)
// Repeat the test for every possible Pool size
const size_t max_pool_size = PartitionAddressSpace::ConfigurablePoolMaxSize();
const size_t min_pool_size = PartitionAddressSpace::ConfigurablePoolMinSize();
for (size_t pool_size = max_pool_size; pool_size >= min_pool_size;
pool_size /= 2) {
PA_DCHECK(std::has_single_bit(pool_size));
EXPECT_FALSE(IsConfigurablePoolAvailable());
uintptr_t pool_base =
AllocPages(pool_size, pool_size,
PageAccessibilityConfiguration(
PageAccessibilityConfiguration::kInaccessible),
PageTag::kPartitionAlloc);
EXPECT_NE(0u, pool_base);
PartitionAddressSpace::InitConfigurablePool(pool_base, pool_size);
EXPECT_TRUE(IsConfigurablePoolAvailable());
PartitionOptions opts = GetCommonPartitionOptions();
opts.use_configurable_pool = PartitionOptions::kAllowed;
std::unique_ptr<PartitionRoot> root = CreateCustomTestRoot(
opts, PartitionTestOptions{.uncap_empty_slot_span_memory = true,
.set_bucket_distribution = true});
const size_t count = 250;
std::vector<void*> allocations(count, nullptr);
for (size_t i = 0; i < count; ++i) {
const size_t size = kTestSizes[base::RandGenerator(kTestSizesCount)];
allocations[i] = root->Alloc(size);
EXPECT_NE(nullptr, allocations[i]);
// We don't Untag allocations here because MTE is disabled for
// configurable pools used by V8.
// https://bugs.chromium.org/p/v8/issues/detail?id=13117
uintptr_t allocation_base = reinterpret_cast<uintptr_t>(allocations[i]);
EXPECT_EQ(allocation_base, UntagPtr(allocations[i]));
EXPECT_TRUE(allocation_base >= pool_base &&
allocation_base < pool_base + pool_size);
}
PartitionAddressSpace::UninitConfigurablePoolForTesting();
FreePages(pool_base, pool_size);
}
#endif // defined(ARCH_CPU_64_BITS)
}
TEST_P(PartitionAllocTest, EmptySlotSpanSizeIsCapped) {
// Use another root, since the ones from the test harness disable the empty
// slot span size cap.
std::unique_ptr<PartitionRoot> root = CreateCustomTestRoot(
GetCommonPartitionOptions(),
PartitionTestOptions{.set_bucket_distribution = true});
// Allocate some memory, don't free it to keep committed memory.
std::vector<void*> allocated_memory;
const size_t size = SystemPageSize();
const size_t count = 400;
for (size_t i = 0; i < count; i++) {
void* ptr = root->Alloc(size);
allocated_memory.push_back(ptr);
}
ASSERT_GE(root->total_size_of_committed_pages.load(std::memory_order_relaxed),
size * count);
// To create empty slot spans, allocate from single-slot slot spans, 128kiB at
// a time.
std::vector<void*> single_slot_allocated_memory;
constexpr size_t single_slot_count = kDefaultEmptySlotSpanRingSize - 1;
const size_t single_slot_size = MaxRegularSlotSpanSize() + 1;
// Make sure that even with allocation size rounding up, a single allocation
// is still below the threshold.
ASSERT_LT(MaxRegularSlotSpanSize() * 2,
((count * size) >> root->max_empty_slot_spans_dirty_bytes_shift));
for (size_t i = 0; i < single_slot_count; i++) {
void* ptr = root->Alloc(single_slot_size);
single_slot_allocated_memory.push_back(ptr);
}
// Free everything at once, creating as many empty slot spans as there are
// allocations (since they are from single-slot slot spans).
for (void* ptr : single_slot_allocated_memory) {
root->Free(ptr);
}
// Still have some committed empty slot spans.
// PA_TS_UNCHECKED_READ() is not an issue here, since everything is
// single-threaded.
EXPECT_GT(PA_TS_UNCHECKED_READ(root->empty_slot_spans_dirty_bytes), 0u);
// But not all, as the cap triggered.
EXPECT_LT(PA_TS_UNCHECKED_READ(root->empty_slot_spans_dirty_bytes),
single_slot_count * single_slot_size);
// Nothing left after explicit purge.
root->PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans);
EXPECT_EQ(PA_TS_UNCHECKED_READ(root->empty_slot_spans_dirty_bytes), 0u);
for (void* ptr : allocated_memory) {
root->Free(ptr);
}
}
TEST_P(PartitionAllocTest, IncreaseEmptySlotSpanRingSize) {
std::unique_ptr<PartitionRoot> root = CreateCustomTestRoot(
GetCommonPartitionOptions(),
PartitionTestOptions{.uncap_empty_slot_span_memory = true,
.set_bucket_distribution = true});
std::vector<void*> single_slot_allocated_memory;
constexpr size_t single_slot_count = kDefaultEmptySlotSpanRingSize + 10;
const size_t single_slot_size = MaxRegularSlotSpanSize() + 1;
const size_t bucket_size =
root->buckets[SizeToIndex(single_slot_size)].slot_size;
for (size_t i = 0; i < single_slot_count; i++) {
void* ptr = root->Alloc(single_slot_size);
single_slot_allocated_memory.push_back(ptr);
}
// Free everything at once, creating as many empty slot spans as there are
// allocations (since they are from single-slot slot spans).
for (void* ptr : single_slot_allocated_memory) {
root->Free(ptr);
}
single_slot_allocated_memory.clear();
// Some of the free()-s above overflowed the slot span ring.
EXPECT_EQ(PA_TS_UNCHECKED_READ(root->empty_slot_spans_dirty_bytes),
kDefaultEmptySlotSpanRingSize * bucket_size);
// Now can cache more slot spans.
root->EnableLargeEmptySlotSpanRing();
constexpr size_t single_slot_large_count = kDefaultEmptySlotSpanRingSize + 10;
for (size_t i = 0; i < single_slot_large_count; i++) {
void* ptr = root->Alloc(single_slot_size);
single_slot_allocated_memory.push_back(ptr);
}
for (void* ptr : single_slot_allocated_memory) {
root->Free(ptr);
}
single_slot_allocated_memory.clear();
// No overflow this time.
EXPECT_EQ(PA_TS_UNCHECKED_READ(root->empty_slot_spans_dirty_bytes),
single_slot_large_count * bucket_size);
constexpr size_t single_slot_too_many_count = kMaxFreeableSpans + 10;
for (size_t i = 0; i < single_slot_too_many_count; i++) {
void* ptr = root->Alloc(single_slot_size);
single_slot_allocated_memory.push_back(ptr);
}
for (void* ptr : single_slot_allocated_memory) {
root->Free(ptr);
}
single_slot_allocated_memory.clear();
// Overflow still works.
EXPECT_EQ(PA_TS_UNCHECKED_READ(root->empty_slot_spans_dirty_bytes),
kMaxFreeableSpans * bucket_size);
}
#if BUILDFLAG(PA_IS_CAST_ANDROID) && BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC)
extern "C" {
void* __real_malloc(size_t);
} // extern "C"
TEST_P(PartitionAllocTest, HandleMixedAllocations) {
void* ptr = __real_malloc(12);
// Should not crash, no test assertion.
free(ptr);
}
#endif
TEST_P(PartitionAllocTest, SortFreelist) {
const size_t count = 100;
const size_t allocation_size = 1;
void* first_ptr = allocator.root()->Alloc(allocation_size);
std::vector<void*> allocations;
for (size_t i = 0; i < count; ++i) {
allocations.push_back(allocator.root()->Alloc(allocation_size));
}
// Shuffle and free memory out of order.
std::random_device rd;
std::mt19937 generator(rd());
std::shuffle(allocations.begin(), allocations.end(), generator);
// Keep one allocation alive (first_ptr), so that the SlotSpan is not fully
// empty.
for (void* ptr : allocations) {
allocator.root()->Free(ptr);
}
allocations.clear();
allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
size_t bucket_index =
SizeToIndex(allocation_size + ExtraAllocSize(allocator));
auto& bucket = allocator.root()->buckets[bucket_index];
EXPECT_TRUE(bucket.active_slot_spans_head->freelist_is_sorted());
// Can sort again.
allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
EXPECT_TRUE(bucket.active_slot_spans_head->freelist_is_sorted());
for (size_t i = 0; i < count; ++i) {
allocations.push_back(allocator.root()->Alloc(allocation_size));
// Allocating keeps the freelist sorted.
EXPECT_TRUE(bucket.active_slot_spans_head->freelist_is_sorted());
}
// Check that it is sorted.
for (size_t i = 1; i < allocations.size(); i++) {
EXPECT_LT(UntagPtr(allocations[i - 1]), UntagPtr(allocations[i]));
}
for (void* ptr : allocations) {
allocator.root()->Free(ptr);
// Free()-ing memory destroys order. Not looking at the head of the active
// list, as it is not necessarily the one from which |ptr| came from.
auto* slot_span =
SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
EXPECT_FALSE(slot_span->freelist_is_sorted());
}
allocator.root()->Free(first_ptr);
}
#if BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC) && BUILDFLAG(IS_LINUX) && \
defined(ARCH_CPU_64_BITS)
TEST_P(PartitionAllocTest, CrashOnUnknownPointer) {
int not_a_heap_object = 42;
EXPECT_DEATH(allocator.root()->Free(&not_a_heap_object), "");
}
#endif // BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC) &&
// BUILDFLAG(IS_LINUX) && defined(ARCH_CPU_64_BITS)
#if BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC) && BUILDFLAG(IS_MAC)
// Adapted from crashpad tests.
class ScopedOpenCLNoOpKernel {
public:
ScopedOpenCLNoOpKernel()
: context_(nullptr),
program_(nullptr),
kernel_(nullptr),
success_(false) {}
ScopedOpenCLNoOpKernel(const ScopedOpenCLNoOpKernel&) = delete;
ScopedOpenCLNoOpKernel& operator=(const ScopedOpenCLNoOpKernel&) = delete;
~ScopedOpenCLNoOpKernel() {
if (kernel_) {
cl_int rv = clReleaseKernel(kernel_);
EXPECT_EQ(rv, CL_SUCCESS) << "clReleaseKernel";
}
if (program_) {
cl_int rv = clReleaseProgram(program_);
EXPECT_EQ(rv, CL_SUCCESS) << "clReleaseProgram";
}
if (context_) {
cl_int rv = clReleaseContext(context_);
EXPECT_EQ(rv, CL_SUCCESS) << "clReleaseContext";
}
}
void SetUp() {
cl_platform_id platform_id;
cl_int rv = clGetPlatformIDs(1, &platform_id, nullptr);
ASSERT_EQ(rv, CL_SUCCESS) << "clGetPlatformIDs";
cl_device_id device_id;
rv =
clGetDeviceIDs(platform_id, CL_DEVICE_TYPE_CPU, 1, &device_id, nullptr);
#if defined(ARCH_CPU_ARM64)
// CL_DEVICE_TYPE_CPU doesn’t seem to work at all on arm64, meaning that
// these weird OpenCL modules probably don’t show up there at all. Keep this
// test even on arm64 in case this ever does start working.
if (rv == CL_INVALID_VALUE) {
return;
}
#endif // ARCH_CPU_ARM64
ASSERT_EQ(rv, CL_SUCCESS) << "clGetDeviceIDs";
context_ = clCreateContext(nullptr, 1, &device_id, nullptr, nullptr, &rv);
ASSERT_EQ(rv, CL_SUCCESS) << "clCreateContext";
const char* sources[] = {
"__kernel void NoOp(void) {barrier(CLK_LOCAL_MEM_FENCE);}",
};
const size_t source_lengths[] = {
strlen(sources[0]),
};
static_assert(std::size(sources) == std::size(source_lengths),
"arrays must be parallel");
program_ = clCreateProgramWithSource(context_, std::size(sources), sources,
source_lengths, &rv);
ASSERT_EQ(rv, CL_SUCCESS) << "clCreateProgramWithSource";
rv = clBuildProgram(program_, 1, &device_id, "-cl-opt-disable", nullptr,
nullptr);
ASSERT_EQ(rv, CL_SUCCESS) << "clBuildProgram";
kernel_ = clCreateKernel(program_, "NoOp", &rv);
ASSERT_EQ(rv, CL_SUCCESS) << "clCreateKernel";
success_ = true;
}
bool success() const { return success_; }
private:
cl_context context_;
cl_program program_;
cl_kernel kernel_;
bool success_;
};
// On macOS 10.11, allocations are made with PartitionAlloc, but the pointer
// is incorrectly passed by CoreFoundation to the previous default zone,
// causing crashes. This is intended to detect these issues regressing in future
// versions of macOS.
TEST_P(PartitionAllocTest, OpenCL) {
ScopedOpenCLNoOpKernel kernel;
kernel.SetUp();
#if !defined(ARCH_CPU_ARM64)
ASSERT_TRUE(kernel.success());
#endif
}
#endif // BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC) &&
// BUILDFLAG(IS_MAC)
TEST_P(PartitionAllocTest, SmallSlotSpanWaste) {
for (PartitionRoot::Bucket& bucket : allocator.root()->buckets) {
const size_t slot_size = bucket.slot_size;
if (slot_size == kInvalidBucketSize) {
continue;
}
size_t small_system_page_count =
partition_alloc::internal::ComputeSystemPagesPerSlotSpan(
bucket.slot_size, true);
size_t small_waste =
(small_system_page_count * SystemPageSize()) % slot_size;
EXPECT_LT(small_waste, .05 * SystemPageSize());
if (slot_size <= MaxRegularSlotSpanSize()) {
EXPECT_LE(small_system_page_count, MaxSystemPagesPerRegularSlotSpan());
}
}
}
TEST_P(PartitionAllocTest, SortActiveSlotSpans) {
auto run_test = [](size_t count) {
PartitionBucket bucket;
bucket.Init(16);
bucket.active_slot_spans_head = nullptr;
std::vector<SlotSpanMetadata> slot_spans;
slot_spans.reserve(count);
// Add slot spans with random freelist length.
for (size_t i = 0; i < count; i++) {
slot_spans.emplace_back(&bucket);
auto& slot_span = slot_spans.back();
slot_span.num_unprovisioned_slots =
partition_alloc::internal::base::RandGenerator(
bucket.get_slots_per_span() / 2);
slot_span.num_allocated_slots =
partition_alloc::internal::base::RandGenerator(
bucket.get_slots_per_span() - slot_span.num_unprovisioned_slots);
slot_span.next_slot_span = bucket.active_slot_spans_head;
bucket.active_slot_spans_head = &slot_span;
}
bucket.SortActiveSlotSpans();
std::set<SlotSpanMetadata*> seen_slot_spans;
std::vector<SlotSpanMetadata*> sorted_slot_spans;
for (auto* slot_span = bucket.active_slot_spans_head; slot_span;
slot_span = slot_span->next_slot_span) {
sorted_slot_spans.push_back(slot_span);
seen_slot_spans.insert(slot_span);
}
// None repeated, none missing.
EXPECT_EQ(seen_slot_spans.size(), sorted_slot_spans.size());
EXPECT_EQ(seen_slot_spans.size(), slot_spans.size());
// The first slot spans are sorted.
size_t sorted_spans_count =
std::min(PartitionBucket::kMaxSlotSpansToSort, count);
EXPECT_TRUE(std::is_sorted(sorted_slot_spans.begin(),
sorted_slot_spans.begin() + sorted_spans_count,
partition_alloc::internal::CompareSlotSpans));
// Slot spans with no freelist entries are at the end of the sorted run.
auto has_empty_freelist = [](SlotSpanMetadata* a) {
return a->GetFreelistLength() == 0;
};
auto it = std::find_if(sorted_slot_spans.begin(),
sorted_slot_spans.begin() + sorted_spans_count,
has_empty_freelist);
if (it != sorted_slot_spans.end()) {
EXPECT_TRUE(std::all_of(it,
sorted_slot_spans.begin() + sorted_spans_count,
has_empty_freelist));
}
};
// Everything is sorted.
run_test(PartitionBucket::kMaxSlotSpansToSort / 2);
// Only the first slot spans are sorted.
run_test(PartitionBucket::kMaxSlotSpansToSort * 2);
// Corner cases.
run_test(0);
run_test(1);
}
#if BUILDFLAG(USE_FREESLOT_BITMAP)
TEST_P(PartitionAllocTest, FreeSlotBitmapMarkedAsUsedAfterAlloc) {
void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
uintptr_t slot_start = allocator.root()->ObjectToSlotStart(ptr);
EXPECT_TRUE(FreeSlotBitmapSlotIsUsed(slot_start));
allocator.root()->Free(ptr);
}
TEST_P(PartitionAllocTest, FreeSlotBitmapMarkedAsFreeAfterFree) {
void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
uintptr_t slot_start = allocator.root()->ObjectToSlotStart(ptr);
EXPECT_TRUE(FreeSlotBitmapSlotIsUsed(slot_start));
allocator.root()->Free(ptr);
EXPECT_FALSE(FreeSlotBitmapSlotIsUsed(slot_start));
}
TEST_P(PartitionAllocTest, FreeSlotBitmapResetAfterDecommit) {
void* ptr1 = allocator.root()->Alloc(
SystemPageSize() - ExtraAllocSize(allocator), type_name);
uintptr_t slot_start = allocator.root()->ObjectToSlotStart(ptr1);
allocator.root()->Free(ptr1);
EXPECT_FALSE(FreeSlotBitmapSlotIsUsed(slot_start));
// Decommit the slot span. Bitmap will be rewritten in Decommit().
allocator.root()->PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans);
EXPECT_TRUE(FreeSlotBitmapSlotIsUsed(slot_start));
}
TEST_P(PartitionAllocTest, FreeSlotBitmapResetAfterPurge) {
void* ptr1 = allocator.root()->Alloc(
SystemPageSize() - ExtraAllocSize(allocator), type_name);
char* ptr2 = static_cast<char*>(allocator.root()->Alloc(
SystemPageSize() - ExtraAllocSize(allocator), type_name));
uintptr_t slot_start = allocator.root()->ObjectToSlotStart(ptr2);
allocator.root()->Free(ptr2);
CHECK_PAGE_IN_CORE(ptr2 - kPointerOffset, true);
EXPECT_FALSE(FreeSlotBitmapSlotIsUsed(slot_start));
// Bitmap will be rewritten in PartitionPurgeSlotSpan().
allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
CHECK_PAGE_IN_CORE(ptr2 - kPointerOffset, false);
EXPECT_TRUE(FreeSlotBitmapSlotIsUsed(slot_start));
allocator.root()->Free(ptr1);
}
#endif // BUILDFLAG(USE_FREESLOT_BITMAP)
} // namespace partition_alloc::internal
#endif // !defined(MEMORY_TOOL_REPLACES_ALLOCATOR)