From 165879e6725a1546f33b1b2437669a248ffdbcea Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Thu, 21 Jul 2022 06:42:07 -0400 Subject: [PATCH 3/3] Backport of 16k page support on aarch64 --- .../address_space_randomization.h | 15 ++++++ .../page_allocator_constants.h | 50 +++++++++++++++++- .../partition_address_space.cc | 6 +++ .../partition_alloc_constants.h | 5 +- .../partition_allocator/partition_root.cc | 2 +- .../address_space_randomization.h | 15 ++++++ .../partition_allocator/page_allocator.cc | 8 +++ .../page_allocator_constants.h | 52 ++++++++++++++++++- .../partition_allocator/partition_alloc.cc | 2 +- .../partition_alloc_constants.h | 5 +- 10 files changed, 152 insertions(+), 8 deletions(-) diff --git a/chromium/base/allocator/partition_allocator/address_space_randomization.h b/chromium/base/allocator/partition_allocator/address_space_randomization.h index e77003eab25..31ac05b86f5 100644 --- a/chromium/base/allocator/partition_allocator/address_space_randomization.h +++ b/chromium/base/allocator/partition_allocator/address_space_randomization.h @@ -119,6 +119,21 @@ AslrMask(uintptr_t bits) { return AslrAddress(0x20000000ULL); } + #elif defined(OS_LINUX) + + // Linux on arm64 can use 39, 42, 48, or 52-bit user space, depending on + // page size and number of levels of translation pages used. We use + // 39-bit as base as all setups should support this, lowered to 38-bit + // as ASLROffset() could cause a carry. + PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE uintptr_t + ASLRMask() { + return AslrMask(38); + } + PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE uintptr_t + ASLROffset() { + return AslrAddress(0x1000000000ULL); + } + #else // ARM64 on Linux has 39-bit user space. Use 38 bits since ASLROffset() diff --git a/chromium/base/allocator/partition_allocator/page_allocator_constants.h b/chromium/base/allocator/partition_allocator/page_allocator_constants.h index f6f19e41611..200903d3342 100644 --- a/chromium/base/allocator/partition_allocator/page_allocator_constants.h +++ b/chromium/base/allocator/partition_allocator/page_allocator_constants.h @@ -24,6 +24,31 @@ // elimination. #define PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR __attribute__((const)) +#elif defined(OS_LINUX) && defined(ARCH_CPU_ARM64) +// This should work for all POSIX (if needed), but currently all other +// supported OS/architecture combinations use either hard-coded values +// (such as x86) or have means to determine these values without needing +// atomics (such as macOS on arm64). + +// Page allocator constants are run-time constant +#define PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR __attribute__((const)) + +#include +#include + +namespace base::internal { + +// Holds the current page size and shift, where size = 1 << shift +// Use PageAllocationGranularity(), PageAllocationGranularityShift() +// to initialize and retrieve these values safely. +struct PageCharacteristics { + std::atomic size; + std::atomic shift; +}; +extern PageCharacteristics page_characteristics; + +} // namespace partition_alloc::internal + #else // When defined, page size constants are fixed at compile time. When not @@ -38,6 +63,10 @@ namespace base { +// Forward declaration, implementation below +PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE size_t +PageAllocationGranularity(); + PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE size_t PageAllocationGranularityShift() { #if defined(OS_WIN) || defined(ARCH_CPU_PPC64) @@ -50,6 +79,15 @@ PageAllocationGranularityShift() { return 14; // 16kB #elif defined(OS_APPLE) && defined(ARCH_CPU_64_BITS) return vm_page_shift; +#elif defined(OS_LINUX) && defined(ARCH_CPU_ARM64) + // arm64 supports 4kb (shift = 12), 16kb (shift = 14), and 64kb (shift = 16) + // page sizes. Retrieve from or initialize cache. + int shift = internal::page_characteristics.shift.load(std::memory_order_relaxed); + if (UNLIKELY(shift == 0)) { + shift = __builtin_ctz((int)PageAllocationGranularity()); + internal::page_characteristics.shift.store(shift, std::memory_order_relaxed); + } + return shift; #else return 12; // 4kB #endif @@ -61,6 +99,15 @@ PageAllocationGranularity() { // This is literally equivalent to |1 << PageAllocationGranularityShift()| // below, but was separated out for OS_APPLE to avoid << on a non-constexpr. return vm_page_size; +#elif defined(OS_LINUX) && defined(ARCH_CPU_ARM64) + // arm64 supports 4kb, 16kb, and 64kb page sizes. Retrieve from or + // initialize cache. + int size = internal::page_characteristics.size.load(std::memory_order_relaxed); + if (UNLIKELY(size == 0)) { + size = getpagesize(); + internal::page_characteristics.size.store(size, std::memory_order_relaxed); + } + return size; #else return 1ULL << PageAllocationGranularityShift(); #endif @@ -90,7 +137,8 @@ SystemPageShift() { PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE size_t SystemPageSize() { -#if defined(OS_APPLE) && defined(ARCH_CPU_64_BITS) +#if (defined(OS_APPLE) && defined(ARCH_CPU_64_BITS)) || \ + (defined(OS_LINUX) && defined(ARCH_CPU_ARM64)) // This is literally equivalent to |1 << SystemPageShift()| below, but was // separated out for 64-bit OS_APPLE to avoid << on a non-constexpr. return PageAllocationGranularity(); diff --git a/chromium/base/allocator/partition_allocator/partition_address_space.cc b/chromium/base/allocator/partition_allocator/partition_address_space.cc index ee6294d8f55..3c9e4f47d4c 100644 --- a/chromium/base/allocator/partition_allocator/partition_address_space.cc +++ b/chromium/base/allocator/partition_allocator/partition_address_space.cc @@ -97,6 +97,12 @@ void PartitionAddressSpace::UninitForTesting() { internal::AddressPoolManager::GetInstance()->ResetForTesting(); } +#if defined(OS_LINUX) && defined(ARCH_CPU_ARM64) + +PageCharacteristics page_characteristics; + +#endif // defined(OS_LINUX) && defined(ARCH_CPU_ARM64) + #endif // defined(PA_HAS_64_BITS_POINTERS) } // namespace internal diff --git a/chromium/base/allocator/partition_allocator/partition_alloc_constants.h b/chromium/base/allocator/partition_allocator/partition_alloc_constants.h index 1eeac4993a5..61bfc4f4b95 100644 --- a/chromium/base/allocator/partition_allocator/partition_alloc_constants.h +++ b/chromium/base/allocator/partition_allocator/partition_alloc_constants.h @@ -47,10 +47,11 @@ PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE size_t PartitionPageShift() { return 18; // 256 KiB } -#elif defined(OS_APPLE) && defined(ARCH_CPU_64_BITS) +#elif (defined(OS_APPLE) && defined(ARCH_CPU_64_BITS)) || \ + (defined(OS_LINUX) && defined(ARCH_CPU_ARM64)) PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE size_t PartitionPageShift() { - return vm_page_shift + 2; + return PageAllocationGranularityShift() + 2; } #else PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE size_t diff --git a/chromium/base/allocator/partition_allocator/partition_root.cc b/chromium/base/allocator/partition_allocator/partition_root.cc index 37a190f371d..a268c74f62d 100644 --- a/chromium/base/allocator/partition_allocator/partition_root.cc +++ b/chromium/base/allocator/partition_allocator/partition_root.cc @@ -182,7 +182,7 @@ static size_t PartitionPurgeSlotSpan( constexpr size_t kMaxSlotCount = (PartitionPageSize() * kMaxPartitionPagesPerRegularSlotSpan) / SystemPageSize(); -#elif defined(OS_APPLE) +#elif defined(OS_APPLE) || (defined(OS_LINUX) && defined(ARCH_CPU_ARM64)) // It's better for slot_usage to be stack-allocated and fixed-size, which // demands that its size be constexpr. On OS_APPLE, PartitionPageSize() is // always SystemPageSize() << 2, so regardless of what the run time page size diff --git a/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/address_space_randomization.h b/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/address_space_randomization.h index 28c8271fd68..3957e0cdf76 100644 --- a/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/address_space_randomization.h +++ b/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/address_space_randomization.h @@ -120,6 +120,21 @@ AslrMask(uintptr_t bits) { return AslrAddress(0x20000000ULL); } + #elif defined(OS_LINUX) + + // Linux on arm64 can use 39, 42, 48, or 52-bit user space, depending on + // page size and number of levels of translation pages used. We use + // 39-bit as base as all setups should support this, lowered to 38-bit + // as ASLROffset() could cause a carry. + PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE uintptr_t + ASLRMask() { + return AslrMask(38); + } + PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE uintptr_t + ASLROffset() { + return AslrAddress(0x1000000000ULL); + } + #else // ARM64 on Linux has 39-bit user space. Use 38 bits since kASLROffset diff --git a/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/page_allocator.cc b/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/page_allocator.cc index e1cf290a66f..c5386a7c564 100644 --- a/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/page_allocator.cc +++ b/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/page_allocator.cc @@ -255,5 +255,13 @@ uint32_t GetAllocPageErrorCode() { return s_allocPageErrorCode; } +#if defined(OS_LINUX) && defined(ARCH_CPU_ARM64) + +namespace internal { +PageCharacteristics page_characteristics; +} + +#endif // defined(OS_LINUX) && defined(ARCH_CPU_ARM64) + } // namespace base } // namespace pdfium diff --git a/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/page_allocator_constants.h b/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/page_allocator_constants.h index fdc65ac47b7..f826308839d 100644 --- a/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/page_allocator_constants.h +++ b/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/page_allocator_constants.h @@ -24,6 +24,31 @@ // elimination. #define PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR __attribute__((const)) +#elif defined(OS_LINUX) && defined(ARCH_CPU_ARM64) +// This should work for all POSIX (if needed), but currently all other +// supported OS/architecture combinations use either hard-coded values +// (such as x86) or have means to determine these values without needing +// atomics (such as macOS on arm64). + +// Page allocator constants are run-time constant +#define PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR __attribute__((const)) + +#include +#include + +namespace pdfium::base::internal { + +// Holds the current page size and shift, where size = 1 << shift +// Use PageAllocationGranularity(), PageAllocationGranularityShift() +// to initialize and retrieve these values safely. +struct PageCharacteristics { + std::atomic size; + std::atomic shift; +}; +extern PageCharacteristics page_characteristics; + +} // namespace base::internal + #else // When defined, page size constants are fixed at compile time. When not @@ -37,11 +62,18 @@ #endif namespace pdfium { + +namespace base { +// Forward declaration, implementation below +PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE size_t +PageAllocationGranularity(); +} + namespace { #if !defined(OS_APPLE) -constexpr ALWAYS_INLINE int PageAllocationGranularityShift() { +PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE int PageAllocationGranularityShift() { #if defined(OS_WIN) || defined(ARCH_CPU_PPC64) // Modern ppc64 systems support 4kB (shift = 12) and 64kB (shift = 16) page // sizes. Since 64kB is the de facto standard on the platform and binaries @@ -50,6 +82,15 @@ constexpr ALWAYS_INLINE int PageAllocationGranularityShift() { return 16; // 64kB #elif defined(_MIPS_ARCH_LOONGSON) return 14; // 16kB +#elif defined(OS_LINUX) && defined(ARCH_CPU_ARM64) + // arm64 supports 4kb (shift = 12), 16kb (shift = 14), and 64kb (shift = 16) + // page sizes. Retrieve from or initialize cache. + int shift = base::internal::page_characteristics.shift.load(std::memory_order_relaxed); + if (UNLIKELY(shift == 0)) { + shift = __builtin_ctz((int)base::PageAllocationGranularity()); + base::internal::page_characteristics.shift.store(shift, std::memory_order_relaxed); + } + return shift; #else return 12; // 4kB #endif @@ -65,6 +106,15 @@ PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE size_t PageAllocationGranularity() { #if defined(OS_APPLE) return vm_page_size; +#elif defined(OS_LINUX) && defined(ARCH_CPU_ARM64) + // arm64 supports 4kb, 16kb, and 64kb page sizes. Retrieve from or + // initialize cache. + int size = internal::page_characteristics.size.load(std::memory_order_relaxed); + if (UNLIKELY(size == 0)) { + size = getpagesize(); + internal::page_characteristics.size.store(size, std::memory_order_relaxed); + } + return size; #else return 1ULL << PageAllocationGranularityShift(); #endif diff --git a/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/partition_alloc.cc b/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/partition_alloc.cc index 2e5e87fa7e6..89b9f6217a6 100644 --- a/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/partition_alloc.cc +++ b/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/partition_alloc.cc @@ -486,7 +486,7 @@ static size_t PartitionPurgePage(internal::PartitionPage* page, bool discard) { #if defined(PAGE_ALLOCATOR_CONSTANTS_ARE_CONSTEXPR) constexpr size_t kMaxSlotCount = (PartitionPageSize() * kMaxPartitionPagesPerSlotSpan) / SystemPageSize(); -#elif defined(OS_APPLE) +#elif defined(OS_APPLE) || (defined(OS_LINUX) && defined(ARCH_CPU_ARM64)) // It's better for slot_usage to be stack-allocated and fixed-size, which // demands that its size be constexpr. On OS_APPLE, PartitionPageSize() is // always SystemPageSize() << 2, so regardless of what the run time page size diff --git a/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/partition_alloc_constants.h b/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/partition_alloc_constants.h index e3bcf5a9933..1dd3d429f69 100644 --- a/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/partition_alloc_constants.h +++ b/chromium/third_party/pdfium/third_party/base/allocator/partition_allocator/partition_alloc_constants.h @@ -49,10 +49,11 @@ PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE int PartitionPageShift() { return 18; // 256 KiB } -#elif defined(OS_APPLE) +#elif defined(OS_APPLE) || \ + (defined(OS_LINUX) && defined(ARCH_CPU_ARM64)) PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE int PartitionPageShift() { - return vm_page_shift + 2; + return PageAllocationGranularityShift() + 2; } #else PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE int -- 2.37.0