From c2eaee274c78938d9fd5576cda7ee9f994c55323 Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Mon, 27 Jan 2025 11:53:26 +0800 Subject: [PATCH 01/11] [WIP] support secondary allocation --- src/snmalloc/mem/secondary.h | 16 +++++++++++++++ src/snmalloc/mem/secondary/default.h | 29 +++++++++++++++++++++++++++ src/snmalloc/mem/secondary/gwp_asan.h | 1 + 3 files changed, 46 insertions(+) create mode 100644 src/snmalloc/mem/secondary.h create mode 100644 src/snmalloc/mem/secondary/default.h create mode 100644 src/snmalloc/mem/secondary/gwp_asan.h diff --git a/src/snmalloc/mem/secondary.h b/src/snmalloc/mem/secondary.h new file mode 100644 index 000000000..e3d5e95c5 --- /dev/null +++ b/src/snmalloc/mem/secondary.h @@ -0,0 +1,16 @@ +#pragma once +#ifdef SNMALLOC_ENABLE_GPW_ASAN_INTEGRATION +# include "snmalloc/mem/secondary/gwp_asan.h" + +namespace snmalloc +{ + using SecondaryAllocator = GwpAsanSecondaryAllocator; +} // namespace snmalloc +#else +# include "snmalloc/mem/secondary/default.h" + +namespace snmalloc +{ + using SecondaryAllocator = DefaultSecondaryAllocator; +} // namespace snmalloc +#endif diff --git a/src/snmalloc/mem/secondary/default.h b/src/snmalloc/mem/secondary/default.h new file mode 100644 index 000000000..16861bfbb --- /dev/null +++ b/src/snmalloc/mem/secondary/default.h @@ -0,0 +1,29 @@ +#pragma once + +#include "snmalloc/ds/ds.h" + +#include + +namespace snmalloc +{ + class DefaultSecondaryAllocator + { + SNMALLOC_FAST_PATH + static void* allocate([[maybe_unused]] size_t size) + { + return nullptr; + } + + SNMALLOC_FAST_PATH + static void deallocate(void* pointer) + { + // If pointer is not null, then dealloc has been call on something + // it shouldn't be called on. + // TODO: Should this be tested even in the !CHECK_CLIENT case? + snmalloc_check_client( + mitigations(sanity_checks), + pointer == nullptr, + "Not allocated by snmalloc."); + } + }; +} // namespace snmalloc diff --git a/src/snmalloc/mem/secondary/gwp_asan.h b/src/snmalloc/mem/secondary/gwp_asan.h new file mode 100644 index 000000000..6f70f09be --- /dev/null +++ b/src/snmalloc/mem/secondary/gwp_asan.h @@ -0,0 +1 @@ +#pragma once From 573c22115d10fd4f0eda54bd169d9fe74308fd51 Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Mon, 27 Jan 2025 05:35:51 +0000 Subject: [PATCH 02/11] almost work --- CMakeLists.txt | 20 +++++++ src/snmalloc/mem/corealloc.h | 5 ++ src/snmalloc/mem/localalloc.h | 15 +++--- src/snmalloc/mem/secondary.h | 2 +- src/snmalloc/mem/secondary/default.h | 7 +++ src/snmalloc/mem/secondary/gwp_asan.h | 76 +++++++++++++++++++++++++++ 6 files changed, 117 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 834bc827f..833e7540b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,12 @@ cmake_dependent_option(SNMALLOC_STATIC_LIBRARY "Build static libraries" ON "NOT cmake_dependent_option(SNMALLOC_CHECK_LOADS "Perform bounds checks on the source argument to memcpy with heap objects" OFF "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) cmake_dependent_option(SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE "Compile for current machine architecture" Off "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) cmake_dependent_option(SNMALLOC_PAGEID "Set an id to memory regions" OFF "NOT SNMALLOC_PAGEID" OFF) + +# GwpAsan secondary allocator +option(SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION "Enable GwpAsan as a secondary allocator" OFF) +set(SNMALLOC_GWP_ASAN_INCLUDE_PATH "" CACHE PATH "GwpAsan header directory") +set(SNMALLOC_GWP_ASAN_LIBRARY_PATH "" CACHE PATH "GwpAsan library directory") + if (NOT SNMALLOC_HEADER_ONLY_LIBRARY) # Pick a sensible default for the thread cleanup mechanism if (${CMAKE_SYSTEM_NAME} STREQUAL FreeBSD) @@ -241,6 +247,20 @@ if(SNMALLOC_USE_SELF_VENDORED_STL) target_compile_definitions(snmalloc INTERFACE SNMALLOC_USE_SELF_VENDORED_STL) endif() +if (SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION) + if (NOT EXISTS ${SNMALLOC_GWP_ASAN_INCLUDE_PATH}) + message(FATAL_ERROR "GwpAsan cannot be enabled without setting SNMALLOC_GWP_ASAN_INCLUDE_PATH") + endif() + if (NOT EXISTS ${SNMALLOC_GWP_ASAN_LIBRARY_PATH}) + message(FATAL_ERROR "GwpAsan cannot be enabled without setting SNMALLOC_GWP_ASAN_LIBRARY_PATH") + endif() + message(STATUS "GwpAsan is enabled: ${SNMALLOC_GWP_ASAN_LIBRARY_PATH}/libclang_rt.gwp_asan-${CMAKE_SYSTEM_PROCESSOR}.a") + target_include_directories(snmalloc INTERFACE ${SNMALLOC_GWP_ASAN_INCLUDE_PATH}) + target_link_directories(snmalloc INTERFACE ${SNMALLOC_GWP_ASAN_LIBRARY_PATH}) + target_compile_definitions(snmalloc INTERFACE -DSNMALLOC_ENABLE_GWP_ASAN_INTEGRATION) + target_link_libraries(snmalloc INTERFACE clang_rt.gwp_asan-${CMAKE_SYSTEM_PROCESSOR}) +endif() + # https://learn.microsoft.com/en-us/cpp/build/reference/zc-cplusplus if(MSVC) target_compile_options(snmalloc INTERFACE "/Zc:__cplusplus") diff --git a/src/snmalloc/mem/corealloc.h b/src/snmalloc/mem/corealloc.h index f82ccc1c2..869725ec7 100644 --- a/src/snmalloc/mem/corealloc.h +++ b/src/snmalloc/mem/corealloc.h @@ -5,6 +5,7 @@ #include "metadata.h" #include "pool.h" #include "remotecache.h" +#include "secondary.h" #include "sizeclasstable.h" #include "snmalloc/stl/new.h" #include "ticker.h" @@ -885,6 +886,10 @@ namespace snmalloc { size_t rsize = sizeclass_to_size(sizeclass); + void* result = SecondaryAllocator::allocate(rsize); + if (result != nullptr) + return capptr::Alloc::unsafe_from(result); + // No existing free list get a new slab. size_t slab_size = sizeclass_to_slab_size(sizeclass); diff --git a/src/snmalloc/mem/localalloc.h b/src/snmalloc/mem/localalloc.h index d30d4149d..236be946a 100644 --- a/src/snmalloc/mem/localalloc.h +++ b/src/snmalloc/mem/localalloc.h @@ -1,6 +1,7 @@ #pragma once #include "snmalloc/aal/address.h" +#include "snmalloc/mem/secondary.h" #if defined(_MSC_VER) # define ALLOCATOR __declspec(allocator) __declspec(restrict) #elif __has_attribute(malloc) @@ -186,6 +187,10 @@ namespace snmalloc return small_alloc(1); } + void* result = SecondaryAllocator::allocate(size); + if (result != nullptr) + return capptr::Alloc::unsafe_from(result); + return check_init([&](CoreAlloc* core_alloc) { if (size > bits::one_at_bit(bits::BITS - 1)) { @@ -697,13 +702,7 @@ namespace snmalloc return; } - // If p_tame is not null, then dealloc has been call on something - // it shouldn't be called on. - // TODO: Should this be tested even in the !CHECK_CLIENT case? - snmalloc_check_client( - mitigations(sanity_checks), - p_tame == nullptr, - "Not allocated by snmalloc."); + SecondaryAllocator::deallocate(p_tame.unsafe_ptr()); # ifdef SNMALLOC_TRACING message<1024>("nullptr deallocation"); @@ -719,6 +718,8 @@ namespace snmalloc #else if constexpr (mitigations(sanity_checks)) { + if (SecondaryAllocator::has_secondary_ownership(p)) + return; size = size == 0 ? 1 : size; auto sc = size_to_sizeclass_full(size); auto pm_sc = diff --git a/src/snmalloc/mem/secondary.h b/src/snmalloc/mem/secondary.h index e3d5e95c5..cde4c1ec8 100644 --- a/src/snmalloc/mem/secondary.h +++ b/src/snmalloc/mem/secondary.h @@ -1,5 +1,5 @@ #pragma once -#ifdef SNMALLOC_ENABLE_GPW_ASAN_INTEGRATION +#ifdef SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION # include "snmalloc/mem/secondary/gwp_asan.h" namespace snmalloc diff --git a/src/snmalloc/mem/secondary/default.h b/src/snmalloc/mem/secondary/default.h index 16861bfbb..5a03e5c52 100644 --- a/src/snmalloc/mem/secondary/default.h +++ b/src/snmalloc/mem/secondary/default.h @@ -8,6 +8,7 @@ namespace snmalloc { class DefaultSecondaryAllocator { + public: SNMALLOC_FAST_PATH static void* allocate([[maybe_unused]] size_t size) { @@ -25,5 +26,11 @@ namespace snmalloc pointer == nullptr, "Not allocated by snmalloc."); } + + SNMALLOC_FAST_PATH + static bool has_secondary_ownership([[maybe_unused]] void* pointer) + { + return false; + } }; } // namespace snmalloc diff --git a/src/snmalloc/mem/secondary/gwp_asan.h b/src/snmalloc/mem/secondary/gwp_asan.h index 6f70f09be..ac03a24a3 100644 --- a/src/snmalloc/mem/secondary/gwp_asan.h +++ b/src/snmalloc/mem/secondary/gwp_asan.h @@ -1 +1,77 @@ #pragma once + +#include "gwp_asan/guarded_pool_allocator.h" +#include "snmalloc/ds_core/defines.h" +#include "snmalloc/mem/sizeclasstable.h" +#if defined(SNMALLOC_BACKTRACE_HEADER) +# include SNMALLOC_BACKTRACE_HEADER +#endif + +namespace snmalloc +{ + class GwpAsanSecondaryAllocator + { + static size_t max_allocation_size; + + static void + initialize_gwp_asan(gwp_asan::GuardedPoolAllocator* allocator) noexcept + { + // for now, we use default options + gwp_asan::options::Options opt; + opt.setDefaults(); +#ifdef SNMALLOC_BACKTRACE_HEADER + opt.Backtrace = [](uintptr_t* buf, size_t length) { + return static_cast( + ::backtrace(reinterpret_cast(buf), static_cast(length))); + }; +#endif + allocator->init(opt); + max_allocation_size = + allocator->getAllocatorState()->maximumAllocationSize(); + } + + static gwp_asan::GuardedPoolAllocator& get() + { + static Singleton + singleton; + return singleton.get(); + } + + public: + SNMALLOC_FAST_PATH static void* allocate(size_t size) + { + auto& inner = get(); + if (SNMALLOC_UNLIKELY(inner.shouldSample())) + { + if (size > max_allocation_size) + return nullptr; + auto alignment = natural_alignment(size); + return get().allocate(size, alignment); + } + return nullptr; + } + + SNMALLOC_FAST_PATH + static void deallocate(void* pointer) + { + if (SNMALLOC_LIKELY(pointer == nullptr)) + return; + + auto& inner = get(); + snmalloc_check_client( + mitigations(sanity_checks), + inner.pointerIsMine(pointer), + "Not allocated by snmalloc or secondary allocator"); + + inner.deallocate(pointer); + } + + SNMALLOC_FAST_PATH + static bool has_secondary_ownership([[maybe_unused]] void* pointer) + { + return get().pointerIsMine(pointer); + } + }; + + inline size_t GwpAsanSecondaryAllocator::max_allocation_size; +} // namespace snmalloc From 9a98bc1e5c1236287f3d9eb0e14a55ef03c6f94c Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Mon, 27 Jan 2025 06:29:58 +0000 Subject: [PATCH 03/11] clean up --- src/snmalloc/backend/globalconfig.h | 3 +++ src/snmalloc/mem/secondary/default.h | 4 +++ src/snmalloc/mem/secondary/gwp_asan.h | 26 ++++++++----------- src/test/func/fixed_region/fixed_region.cc | 5 ++++ .../perf/external_pointer/externalpointer.cc | 2 ++ 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/snmalloc/backend/globalconfig.h b/src/snmalloc/backend/globalconfig.h index 97ca9ec94..f95f134f7 100644 --- a/src/snmalloc/backend/globalconfig.h +++ b/src/snmalloc/backend/globalconfig.h @@ -3,6 +3,7 @@ #include "../backend_helpers/backend_helpers.h" #include "backend.h" #include "meta_protected_range.h" +#include "snmalloc/mem/secondary.h" #include "standard_range.h" namespace snmalloc @@ -107,6 +108,8 @@ namespace snmalloc if (initialised) return; + SecondaryAllocator::initialize(); + LocalEntropy entropy; entropy.init(); // Initialise key for remote deallocation lists diff --git a/src/snmalloc/mem/secondary/default.h b/src/snmalloc/mem/secondary/default.h index 5a03e5c52..a0cbcd5b6 100644 --- a/src/snmalloc/mem/secondary/default.h +++ b/src/snmalloc/mem/secondary/default.h @@ -1,6 +1,7 @@ #pragma once #include "snmalloc/ds/ds.h" +#include "snmalloc/ds_core/defines.h" #include @@ -9,6 +10,9 @@ namespace snmalloc class DefaultSecondaryAllocator { public: + SNMALLOC_FAST_PATH + static void initialize() {} + SNMALLOC_FAST_PATH static void* allocate([[maybe_unused]] size_t size) { diff --git a/src/snmalloc/mem/secondary/gwp_asan.h b/src/snmalloc/mem/secondary/gwp_asan.h index ac03a24a3..a0bce0c0f 100644 --- a/src/snmalloc/mem/secondary/gwp_asan.h +++ b/src/snmalloc/mem/secondary/gwp_asan.h @@ -11,10 +11,16 @@ namespace snmalloc { class GwpAsanSecondaryAllocator { - static size_t max_allocation_size; + static inline gwp_asan::GuardedPoolAllocator singleton; + static inline size_t max_allocation_size; - static void - initialize_gwp_asan(gwp_asan::GuardedPoolAllocator* allocator) noexcept + static gwp_asan::GuardedPoolAllocator& get() + { + return singleton; + } + + public: + static void initialize() noexcept { // for now, we use default options gwp_asan::options::Options opt; @@ -25,19 +31,11 @@ namespace snmalloc ::backtrace(reinterpret_cast(buf), static_cast(length))); }; #endif - allocator->init(opt); + singleton.init(opt); max_allocation_size = - allocator->getAllocatorState()->maximumAllocationSize(); + singleton.getAllocatorState()->maximumAllocationSize(); } - static gwp_asan::GuardedPoolAllocator& get() - { - static Singleton - singleton; - return singleton.get(); - } - - public: SNMALLOC_FAST_PATH static void* allocate(size_t size) { auto& inner = get(); @@ -72,6 +70,4 @@ namespace snmalloc return get().pointerIsMine(pointer); } }; - - inline size_t GwpAsanSecondaryAllocator::max_allocation_size; } // namespace snmalloc diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index 2c00c7b8c..e8f6c97b0 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -1,3 +1,4 @@ +#include "snmalloc/mem/secondary.h" #include "test/setup.h" #include @@ -38,6 +39,10 @@ int main() while (true) { auto r1 = a.alloc(object_size); + + if (SecondaryAllocator::has_secondary_ownership(r1)) + continue; + count += object_size; i++; diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index 984909123..1b21e1b4f 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -76,6 +76,8 @@ namespace test size_t rand = (size_t)r.next(); size_t oid = rand & (((size_t)1 << count_log) - 1); size_t* external_ptr = objects[oid]; + if (SecondaryAllocator::has_secondary_ownership(external_ptr)) + continue; size_t size = *external_ptr; size_t offset = (size >> 4) * (rand & 15); void* interior_ptr = pointer_offset(external_ptr, offset); From 6d0b9856e3770e584e2b4560c9516198dc03ccee Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Mon, 27 Jan 2025 07:08:18 +0000 Subject: [PATCH 04/11] override alloc_size --- src/snmalloc/mem/localalloc.h | 3 +++ src/snmalloc/mem/secondary/default.h | 11 ++++++++++- src/snmalloc/mem/secondary/gwp_asan.h | 25 ++++++++++++------------- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/snmalloc/mem/localalloc.h b/src/snmalloc/mem/localalloc.h index 236be946a..664f2ba2e 100644 --- a/src/snmalloc/mem/localalloc.h +++ b/src/snmalloc/mem/localalloc.h @@ -768,6 +768,9 @@ namespace snmalloc #ifdef SNMALLOC_PASS_THROUGH return external_alloc::malloc_usable_size(const_cast(p_raw)); #else + + if (SecondaryAllocator::has_secondary_ownership(p_raw)) + return SecondaryAllocator::alloc_size(p_raw); // TODO What's the domestication policy here? At the moment we just // probe the pagemap with the raw address, without checks. There could // be implicit domestication through the `Config::Pagemap` or diff --git a/src/snmalloc/mem/secondary/default.h b/src/snmalloc/mem/secondary/default.h index a0cbcd5b6..d4a272a4d 100644 --- a/src/snmalloc/mem/secondary/default.h +++ b/src/snmalloc/mem/secondary/default.h @@ -32,9 +32,18 @@ namespace snmalloc } SNMALLOC_FAST_PATH - static bool has_secondary_ownership([[maybe_unused]] void* pointer) + static bool has_secondary_ownership([[maybe_unused]] const void* pointer) { return false; } + + SNMALLOC_FAST_PATH + static size_t alloc_size([[maybe_unused]] const void* pointer) + { + SNMALLOC_ASSERT( + false && + "secondary alloc_size should never be invoked with default setup"); + return 0; + } }; } // namespace snmalloc diff --git a/src/snmalloc/mem/secondary/gwp_asan.h b/src/snmalloc/mem/secondary/gwp_asan.h index a0bce0c0f..0d50e91d0 100644 --- a/src/snmalloc/mem/secondary/gwp_asan.h +++ b/src/snmalloc/mem/secondary/gwp_asan.h @@ -14,11 +14,6 @@ namespace snmalloc static inline gwp_asan::GuardedPoolAllocator singleton; static inline size_t max_allocation_size; - static gwp_asan::GuardedPoolAllocator& get() - { - return singleton; - } - public: static void initialize() noexcept { @@ -38,13 +33,12 @@ namespace snmalloc SNMALLOC_FAST_PATH static void* allocate(size_t size) { - auto& inner = get(); - if (SNMALLOC_UNLIKELY(inner.shouldSample())) + if (SNMALLOC_UNLIKELY(singleton.shouldSample())) { if (size > max_allocation_size) return nullptr; auto alignment = natural_alignment(size); - return get().allocate(size, alignment); + return singleton.allocate(size, alignment); } return nullptr; } @@ -55,19 +49,24 @@ namespace snmalloc if (SNMALLOC_LIKELY(pointer == nullptr)) return; - auto& inner = get(); snmalloc_check_client( mitigations(sanity_checks), - inner.pointerIsMine(pointer), + singleton.pointerIsMine(pointer), "Not allocated by snmalloc or secondary allocator"); - inner.deallocate(pointer); + singleton.deallocate(pointer); + } + + SNMALLOC_FAST_PATH + static bool has_secondary_ownership([[maybe_unused]] const void* pointer) + { + return singleton.pointerIsMine(pointer); } SNMALLOC_FAST_PATH - static bool has_secondary_ownership([[maybe_unused]] void* pointer) + static size_t alloc_size(const void* pointer) { - return get().pointerIsMine(pointer); + return singleton.getSize(pointer); } }; } // namespace snmalloc From f6709dbe8f831dcf410a3d7e76b6ba113479d7f4 Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Mon, 27 Jan 2025 07:23:22 +0000 Subject: [PATCH 05/11] fix --- src/snmalloc/mem/secondary/default.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snmalloc/mem/secondary/default.h b/src/snmalloc/mem/secondary/default.h index d4a272a4d..56323f885 100644 --- a/src/snmalloc/mem/secondary/default.h +++ b/src/snmalloc/mem/secondary/default.h @@ -1,7 +1,7 @@ #pragma once -#include "snmalloc/ds/ds.h" #include "snmalloc/ds_core/defines.h" +#include "snmalloc/ds_core/mitigations.h" #include From d7acd326d87b1ac6dd57c05c7f05e075d4e208ce Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Mon, 27 Jan 2025 07:29:03 +0000 Subject: [PATCH 06/11] avoid repeated leaking from gwp_asan --- src/test/func/fixed_region/fixed_region.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index e8f6c97b0..31b303f82 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -40,12 +40,15 @@ int main() { auto r1 = a.alloc(object_size); - if (SecondaryAllocator::has_secondary_ownership(r1)) - continue; - count += object_size; i++; + if (SecondaryAllocator::has_secondary_ownership(r1)) + { + a.dealloc(r1); + continue; + } + if (i == 1024) { i = 0; From 121102c650a2ddee44f684bad39112c479cc4212 Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Mon, 27 Jan 2025 08:55:23 +0000 Subject: [PATCH 07/11] add related workflows --- .github/workflows/main.yml | 47 +++++++++++++++++++++++- src/test/func/client_meta/client_meta.cc | 3 +- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 55e634f53..59c01a4ec 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -542,7 +542,52 @@ jobs: run: | cd ${{github.workspace}}/build ctest --parallel - + + gwp-asan: + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, ubuntu-24.04-arm] + profile: [RelWithDebInfo, Debug] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Install Ninja + run: | + sudo apt-get install -y ninja-build + - name: Install Compiler-RT + shell: bash + run: | + cd .. + git clone https://github.com/llvm/llvm-project --depth=1 -b llvmorg-19.1.7 + mkdir compiler-rt + cmake -G Ninja \ + -S llvm-project/runtimes \ + -B llvm-project/build \ + -DCMAKE_BUILD_TYPE=${{ matrix.profile }}\ + -DLLVM_ENABLE_RUNTIMES=compiler-rt \ + -DCMAKE_CXX_COMPILER=clang++-18 \ + -DCMAKE_C_COMPILER=clang-18 \ + -DCMAKE_INSTALL_PREFIX=$(realpath compiler-rt) + cmake --build llvm-project/build --parallel + cmake --build llvm-project/build --target=install + - name: Configure SnMalloc + run: > + cmake -GNinja + -B${{github.workspace}}/build + -DCMAKE_BUILD_TYPE=${{ matrix.profile }} + -DCMAKE_CXX_COMPILER=clang++-18 + -DCMAKE_C_COMPILER=clang-18 + -DSNMALLOC_ENABLE_GWP_ASAN_INTEGRATION=On + -DSNMALLOC_GWP_ASAN_INCLUDE_PATH=${{github.workspace}}/../llvm-project/compiler-rt/lib + -DSNMALLOC_GWP_ASAN_LIBRARY_PATH=${{github.workspace}}/../compiler-rt/lib/linux + - name: Build + run: cmake --build ${{github.workspace}}/build --parallel + - name: Test + run: | + cd ${{github.workspace}}/build + ctest --parallel + all-checks: # Currently FreeBSD and NetBSD CI are not working, so we do not require them to pass. # Add fuzzing back when the memove issue is fixed. diff --git a/src/test/func/client_meta/client_meta.cc b/src/test/func/client_meta/client_meta.cc index dd576d304..d1d99a341 100644 --- a/src/test/func/client_meta/client_meta.cc +++ b/src/test/func/client_meta/client_meta.cc @@ -23,7 +23,8 @@ namespace snmalloc int main() { -#ifdef SNMALLOC_PASS_THROUGH +#if defined(SNMALLOC_PASS_THROUGH) || \ + defined(SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION) // This test does not make sense in pass-through return 0; #else From dbaae4074ba49703f6764b89749b911f72cc2d86 Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Wed, 29 Jan 2025 12:58:17 +0000 Subject: [PATCH 08/11] address code reviews --- .github/workflows/main.yml | 2 +- src/snmalloc/ds_core/ptrwrap.h | 2 + src/snmalloc/mem/corealloc.h | 12 ++- src/snmalloc/mem/localalloc.h | 74 ++++++++++++------- src/snmalloc/mem/secondary.h | 4 + src/snmalloc/mem/secondary/default.h | 16 ++-- src/snmalloc/mem/secondary/gwp_asan.h | 20 ++--- src/test/func/fixed_region/fixed_region.cc | 11 +-- .../perf/external_pointer/externalpointer.cc | 4 +- 9 files changed, 88 insertions(+), 57 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 59c01a4ec..2df4e8303 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -586,7 +586,7 @@ jobs: - name: Test run: | cd ${{github.workspace}}/build - ctest --parallel + ctest --parallel --output-on-failure all-checks: # Currently FreeBSD and NetBSD CI are not working, so we do not require them to pass. diff --git a/src/snmalloc/ds_core/ptrwrap.h b/src/snmalloc/ds_core/ptrwrap.h index 1d6e41f34..853e5fb4c 100644 --- a/src/snmalloc/ds_core/ptrwrap.h +++ b/src/snmalloc/ds_core/ptrwrap.h @@ -4,6 +4,8 @@ #include "defines.h" #include "snmalloc/stl/atomic.h" +#include + namespace snmalloc { /* diff --git a/src/snmalloc/mem/corealloc.h b/src/snmalloc/mem/corealloc.h index 869725ec7..55fd5efe0 100644 --- a/src/snmalloc/mem/corealloc.h +++ b/src/snmalloc/mem/corealloc.h @@ -818,6 +818,14 @@ namespace snmalloc SNMALLOC_SLOW_PATH capptr::Alloc small_alloc(smallsizeclass_t sizeclass, freelist::Iter<>& fast_free_list) { + void* result = SecondaryAllocator::allocate( + [sizeclass]() -> stl::Pair { + auto size = sizeclass_to_size(sizeclass); + return {size, natural_alignment(size)}; + }); + + if (result != nullptr) + return capptr::Alloc::unsafe_from(result); // Look to see if we can grab a free list. auto& sl = alloc_classes[sizeclass].available; if (SNMALLOC_LIKELY(alloc_classes[sizeclass].length > 0)) @@ -886,10 +894,6 @@ namespace snmalloc { size_t rsize = sizeclass_to_size(sizeclass); - void* result = SecondaryAllocator::allocate(rsize); - if (result != nullptr) - return capptr::Alloc::unsafe_from(result); - // No existing free list get a new slab. size_t slab_size = sizeclass_to_slab_size(sizeclass); diff --git a/src/snmalloc/mem/localalloc.h b/src/snmalloc/mem/localalloc.h index 664f2ba2e..c3488b9fd 100644 --- a/src/snmalloc/mem/localalloc.h +++ b/src/snmalloc/mem/localalloc.h @@ -1,6 +1,7 @@ #pragma once #include "snmalloc/aal/address.h" +#include "snmalloc/mem/remoteallocator.h" #include "snmalloc/mem/secondary.h" #if defined(_MSC_VER) # define ALLOCATOR __declspec(allocator) __declspec(restrict) @@ -187,10 +188,6 @@ namespace snmalloc return small_alloc(1); } - void* result = SecondaryAllocator::allocate(size); - if (result != nullptr) - return capptr::Alloc::unsafe_from(result); - return check_init([&](CoreAlloc* core_alloc) { if (size > bits::one_at_bit(bits::BITS - 1)) { @@ -199,6 +196,15 @@ namespace snmalloc errno = ENOMEM; return capptr::Alloc{nullptr}; } + + // Check if secondary allocator wants to offer the memory + void* result = + SecondaryAllocator::allocate([size]() -> stl::Pair { + return {size, natural_alignment(size)}; + }); + if (result != nullptr) + return capptr::Alloc::unsafe_from(result); + // Grab slab of correct size // Set remote as large allocator remote. auto [chunk, meta] = Config::Backend::alloc_chunk( @@ -611,17 +617,13 @@ namespace snmalloc #endif } - SNMALLOC_FAST_PATH void dealloc(void* p_raw) - { -#ifdef SNMALLOC_PASS_THROUGH - external_alloc::free(p_raw); -#else - // Care is needed so that dealloc(nullptr) works before init - // The backend allocator must ensure that a minimal page map exists - // before init, that maps null to a remote_deallocator that will never - // be in thread local state. + // The domestic pointer with its origin allocator + using DomesticInfo = stl::Pair, const PagemapEntry&>; -# ifdef __CHERI_PURE_CAPABILITY__ + // Check whether the raw pointer is owned by snmalloc + SNMALLOC_FAST_PATH_INLINE DomesticInfo get_domestic_info(const void* p_raw) + { +#ifdef __CHERI_PURE_CAPABILITY__ /* * On CHERI platforms, snap the provided pointer to its base, ignoring * any client-provided offset, which may have taken the pointer out of @@ -637,10 +639,29 @@ namespace snmalloc * start of the allocation and so the offset is zero. */ p_raw = __builtin_cheri_offset_set(p_raw, 0); -# endif +#endif + capptr::AllocWild p_wild = + capptr_from_client(const_cast(p_raw)); + auto p_tame = + capptr_domesticate(core_alloc->backend_state_ptr(), p_wild); + const PagemapEntry& entry = + Config::Backend::get_metaentry(address_cast(p_tame)); + return {p_tame, entry}; + } - capptr::AllocWild p_wild = capptr_from_client(p_raw); + // Check if a pointer is domestic to SnMalloc + SNMALLOC_FAST_PATH bool check_domestication(const void* p_raw) + { + auto [_, entry] = get_domestic_info(p_raw); + RemoteAllocator* remote = entry.get_remote(); + return remote != nullptr; + } + SNMALLOC_FAST_PATH void dealloc(void* p_raw) + { +#ifdef SNMALLOC_PASS_THROUGH + external_alloc::free(p_raw); +#else /* * p_tame may be nullptr, even if p_raw/p_wild are not, in the case * where domestication fails. We exclusively use p_tame below so that @@ -653,11 +674,7 @@ namespace snmalloc * well-formedness) of this pointer. The remainder of the logic will * deal with the object's extent. */ - capptr::Alloc p_tame = - capptr_domesticate(core_alloc->backend_state_ptr(), p_wild); - - const PagemapEntry& entry = - Config::Backend::get_metaentry(address_cast(p_tame)); + auto [p_tame, entry] = get_domestic_info(p_raw); if (SNMALLOC_LIKELY(local_cache.remote_allocator == entry.get_remote())) { @@ -702,12 +719,15 @@ namespace snmalloc return; } - SecondaryAllocator::deallocate(p_tame.unsafe_ptr()); - + if (SNMALLOC_LIKELY(p_tame == nullptr)) + { # ifdef SNMALLOC_TRACING - message<1024>("nullptr deallocation"); + message<1024>("nullptr deallocation"); # endif - return; + return; + } + + SecondaryAllocator::deallocate(p_tame.unsafe_ptr()); #endif } @@ -718,7 +738,7 @@ namespace snmalloc #else if constexpr (mitigations(sanity_checks)) { - if (SecondaryAllocator::has_secondary_ownership(p)) + if (!check_domestication(p)) return; size = size == 0 ? 1 : size; auto sc = size_to_sizeclass_full(size); @@ -769,7 +789,7 @@ namespace snmalloc return external_alloc::malloc_usable_size(const_cast(p_raw)); #else - if (SecondaryAllocator::has_secondary_ownership(p_raw)) + if (!SecondaryAllocator::pass_through && !check_domestication(p_raw)) return SecondaryAllocator::alloc_size(p_raw); // TODO What's the domestication policy here? At the moment we just // probe the pagemap with the raw address, without checks. There could diff --git a/src/snmalloc/mem/secondary.h b/src/snmalloc/mem/secondary.h index cde4c1ec8..9f3dab307 100644 --- a/src/snmalloc/mem/secondary.h +++ b/src/snmalloc/mem/secondary.h @@ -1,4 +1,8 @@ #pragma once + +#include "snmalloc/ds_core/defines.h" +#include "snmalloc/ds_core/ptrwrap.h" + #ifdef SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION # include "snmalloc/mem/secondary/gwp_asan.h" diff --git a/src/snmalloc/mem/secondary/default.h b/src/snmalloc/mem/secondary/default.h index 56323f885..ae5d53bfb 100644 --- a/src/snmalloc/mem/secondary/default.h +++ b/src/snmalloc/mem/secondary/default.h @@ -10,11 +10,15 @@ namespace snmalloc class DefaultSecondaryAllocator { public: + // This flag is used to turn off checks on fast paths if the secondary + // allocator does not own the memory at all. + static constexpr inline bool pass_through = true; + SNMALLOC_FAST_PATH static void initialize() {} - SNMALLOC_FAST_PATH - static void* allocate([[maybe_unused]] size_t size) + template + SNMALLOC_FAST_PATH static void* allocate(SizeAlign&&) { return nullptr; } @@ -32,13 +36,7 @@ namespace snmalloc } SNMALLOC_FAST_PATH - static bool has_secondary_ownership([[maybe_unused]] const void* pointer) - { - return false; - } - - SNMALLOC_FAST_PATH - static size_t alloc_size([[maybe_unused]] const void* pointer) + static size_t alloc_size(const void*) { SNMALLOC_ASSERT( false && diff --git a/src/snmalloc/mem/secondary/gwp_asan.h b/src/snmalloc/mem/secondary/gwp_asan.h index 0d50e91d0..a46678321 100644 --- a/src/snmalloc/mem/secondary/gwp_asan.h +++ b/src/snmalloc/mem/secondary/gwp_asan.h @@ -15,6 +15,8 @@ namespace snmalloc static inline size_t max_allocation_size; public: + static constexpr inline bool pass_through = false; + static void initialize() noexcept { // for now, we use default options @@ -31,10 +33,17 @@ namespace snmalloc singleton.getAllocatorState()->maximumAllocationSize(); } - SNMALLOC_FAST_PATH static void* allocate(size_t size) + // Use thunk to avoid extra computation when allocation decision can be made + // before size and alignment are computed. + template + SNMALLOC_FAST_PATH static void* allocate(SizeAlign&& getter) { + // TODO: this `shouldSample` is only triggered on snmalloc's slowpath, + // which may reduce the chance of error detection. We may reconsider + // the logic to improve the precision in future commits. if (SNMALLOC_UNLIKELY(singleton.shouldSample())) { + auto [size, align] = getter(); if (size > max_allocation_size) return nullptr; auto alignment = natural_alignment(size); @@ -46,9 +55,6 @@ namespace snmalloc SNMALLOC_FAST_PATH static void deallocate(void* pointer) { - if (SNMALLOC_LIKELY(pointer == nullptr)) - return; - snmalloc_check_client( mitigations(sanity_checks), singleton.pointerIsMine(pointer), @@ -57,12 +63,6 @@ namespace snmalloc singleton.deallocate(pointer); } - SNMALLOC_FAST_PATH - static bool has_secondary_ownership([[maybe_unused]] const void* pointer) - { - return singleton.pointerIsMine(pointer); - } - SNMALLOC_FAST_PATH static size_t alloc_size(const void* pointer) { diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index 31b303f82..ff9dc6abf 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -43,7 +43,12 @@ int main() count += object_size; i++; - if (SecondaryAllocator::has_secondary_ownership(r1)) + // Run until we exhaust the fixed region. + // This should return null. + if (r1 == nullptr) + break; + + if (!a.check_domestication(r1)) { a.dealloc(r1); continue; @@ -54,10 +59,6 @@ int main() i = 0; std::cout << "."; } - // Run until we exhaust the fixed region. - // This should return null. - if (r1 == nullptr) - break; if (oe_base > r1) { diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index 1b21e1b4f..1704b2671 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -76,14 +76,16 @@ namespace test size_t rand = (size_t)r.next(); size_t oid = rand & (((size_t)1 << count_log) - 1); size_t* external_ptr = objects[oid]; - if (SecondaryAllocator::has_secondary_ownership(external_ptr)) + if (!alloc.check_domestication(external_ptr)) continue; size_t size = *external_ptr; size_t offset = (size >> 4) * (rand & 15); void* interior_ptr = pointer_offset(external_ptr, offset); void* calced_external = alloc.external_pointer(interior_ptr); if (calced_external != external_ptr) + { abort(); + } } } From 7bc335c6a2bb3e4854b506eb0188e3d1bb5ec3df Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Wed, 29 Jan 2025 14:00:14 +0000 Subject: [PATCH 09/11] minor fix --- src/snmalloc/mem/secondary/gwp_asan.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/snmalloc/mem/secondary/gwp_asan.h b/src/snmalloc/mem/secondary/gwp_asan.h index a46678321..178a3eeee 100644 --- a/src/snmalloc/mem/secondary/gwp_asan.h +++ b/src/snmalloc/mem/secondary/gwp_asan.h @@ -46,8 +46,7 @@ namespace snmalloc auto [size, align] = getter(); if (size > max_allocation_size) return nullptr; - auto alignment = natural_alignment(size); - return singleton.allocate(size, alignment); + return singleton.allocate(size, align); } return nullptr; } From 3492df6f376d5be9a0a20d36e3786b783260fba9 Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Wed, 29 Jan 2025 14:02:13 +0000 Subject: [PATCH 10/11] add special check for nullptr --- src/snmalloc/mem/localalloc.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/snmalloc/mem/localalloc.h b/src/snmalloc/mem/localalloc.h index c3488b9fd..f97e576ff 100644 --- a/src/snmalloc/mem/localalloc.h +++ b/src/snmalloc/mem/localalloc.h @@ -789,7 +789,9 @@ namespace snmalloc return external_alloc::malloc_usable_size(const_cast(p_raw)); #else - if (!SecondaryAllocator::pass_through && !check_domestication(p_raw)) + if ( + !SecondaryAllocator::pass_through && !check_domestication(p_raw) && + p_raw != nullptr) return SecondaryAllocator::alloc_size(p_raw); // TODO What's the domestication policy here? At the moment we just // probe the pagemap with the raw address, without checks. There could From 0081383b69a994aa00c03f6154e1480f0104dd7d Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Tue, 4 Feb 2025 10:48:30 +0000 Subject: [PATCH 11/11] rename according to CRs --- src/snmalloc/mem/localalloc.h | 6 +++--- src/test/func/fixed_region/fixed_region.cc | 2 +- src/test/perf/external_pointer/externalpointer.cc | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/snmalloc/mem/localalloc.h b/src/snmalloc/mem/localalloc.h index f97e576ff..09256a063 100644 --- a/src/snmalloc/mem/localalloc.h +++ b/src/snmalloc/mem/localalloc.h @@ -650,7 +650,7 @@ namespace snmalloc } // Check if a pointer is domestic to SnMalloc - SNMALLOC_FAST_PATH bool check_domestication(const void* p_raw) + SNMALLOC_FAST_PATH bool is_snmalloc_owned(const void* p_raw) { auto [_, entry] = get_domestic_info(p_raw); RemoteAllocator* remote = entry.get_remote(); @@ -738,7 +738,7 @@ namespace snmalloc #else if constexpr (mitigations(sanity_checks)) { - if (!check_domestication(p)) + if (!is_snmalloc_owned(p)) return; size = size == 0 ? 1 : size; auto sc = size_to_sizeclass_full(size); @@ -790,7 +790,7 @@ namespace snmalloc #else if ( - !SecondaryAllocator::pass_through && !check_domestication(p_raw) && + !SecondaryAllocator::pass_through && !is_snmalloc_owned(p_raw) && p_raw != nullptr) return SecondaryAllocator::alloc_size(p_raw); // TODO What's the domestication policy here? At the moment we just diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index ff9dc6abf..f5e513b0c 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -48,7 +48,7 @@ int main() if (r1 == nullptr) break; - if (!a.check_domestication(r1)) + if (!a.is_snmalloc_owned(r1)) { a.dealloc(r1); continue; diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index 1704b2671..b87b8800a 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -76,7 +76,7 @@ namespace test size_t rand = (size_t)r.next(); size_t oid = rand & (((size_t)1 << count_log) - 1); size_t* external_ptr = objects[oid]; - if (!alloc.check_domestication(external_ptr)) + if (!alloc.is_snmalloc_owned(external_ptr)) continue; size_t size = *external_ptr; size_t offset = (size >> 4) * (rand & 15);