diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 55e634f53..2df4e8303 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -542,7 +542,52 @@ jobs: run: | cd ${{github.workspace}}/build ctest --parallel - + + gwp-asan: + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, ubuntu-24.04-arm] + profile: [RelWithDebInfo, Debug] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Install Ninja + run: | + sudo apt-get install -y ninja-build + - name: Install Compiler-RT + shell: bash + run: | + cd .. + git clone https://github.com/llvm/llvm-project --depth=1 -b llvmorg-19.1.7 + mkdir compiler-rt + cmake -G Ninja \ + -S llvm-project/runtimes \ + -B llvm-project/build \ + -DCMAKE_BUILD_TYPE=${{ matrix.profile }}\ + -DLLVM_ENABLE_RUNTIMES=compiler-rt \ + -DCMAKE_CXX_COMPILER=clang++-18 \ + -DCMAKE_C_COMPILER=clang-18 \ + -DCMAKE_INSTALL_PREFIX=$(realpath compiler-rt) + cmake --build llvm-project/build --parallel + cmake --build llvm-project/build --target=install + - name: Configure SnMalloc + run: > + cmake -GNinja + -B${{github.workspace}}/build + -DCMAKE_BUILD_TYPE=${{ matrix.profile }} + -DCMAKE_CXX_COMPILER=clang++-18 + -DCMAKE_C_COMPILER=clang-18 + -DSNMALLOC_ENABLE_GWP_ASAN_INTEGRATION=On + -DSNMALLOC_GWP_ASAN_INCLUDE_PATH=${{github.workspace}}/../llvm-project/compiler-rt/lib + -DSNMALLOC_GWP_ASAN_LIBRARY_PATH=${{github.workspace}}/../compiler-rt/lib/linux + - name: Build + run: cmake --build ${{github.workspace}}/build --parallel + - name: Test + run: | + cd ${{github.workspace}}/build + ctest --parallel --output-on-failure + all-checks: # Currently FreeBSD and NetBSD CI are not working, so we do not require them to pass. # Add fuzzing back when the memove issue is fixed. diff --git a/CMakeLists.txt b/CMakeLists.txt index 834bc827f..833e7540b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,12 @@ cmake_dependent_option(SNMALLOC_STATIC_LIBRARY "Build static libraries" ON "NOT cmake_dependent_option(SNMALLOC_CHECK_LOADS "Perform bounds checks on the source argument to memcpy with heap objects" OFF "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) cmake_dependent_option(SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE "Compile for current machine architecture" Off "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) cmake_dependent_option(SNMALLOC_PAGEID "Set an id to memory regions" OFF "NOT SNMALLOC_PAGEID" OFF) + +# GwpAsan secondary allocator +option(SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION "Enable GwpAsan as a secondary allocator" OFF) +set(SNMALLOC_GWP_ASAN_INCLUDE_PATH "" CACHE PATH "GwpAsan header directory") +set(SNMALLOC_GWP_ASAN_LIBRARY_PATH "" CACHE PATH "GwpAsan library directory") + if (NOT SNMALLOC_HEADER_ONLY_LIBRARY) # Pick a sensible default for the thread cleanup mechanism if (${CMAKE_SYSTEM_NAME} STREQUAL FreeBSD) @@ -241,6 +247,20 @@ if(SNMALLOC_USE_SELF_VENDORED_STL) target_compile_definitions(snmalloc INTERFACE SNMALLOC_USE_SELF_VENDORED_STL) endif() +if (SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION) + if (NOT EXISTS ${SNMALLOC_GWP_ASAN_INCLUDE_PATH}) + message(FATAL_ERROR "GwpAsan cannot be enabled without setting SNMALLOC_GWP_ASAN_INCLUDE_PATH") + endif() + if (NOT EXISTS ${SNMALLOC_GWP_ASAN_LIBRARY_PATH}) + message(FATAL_ERROR "GwpAsan cannot be enabled without setting SNMALLOC_GWP_ASAN_LIBRARY_PATH") + endif() + message(STATUS "GwpAsan is enabled: ${SNMALLOC_GWP_ASAN_LIBRARY_PATH}/libclang_rt.gwp_asan-${CMAKE_SYSTEM_PROCESSOR}.a") + target_include_directories(snmalloc INTERFACE ${SNMALLOC_GWP_ASAN_INCLUDE_PATH}) + target_link_directories(snmalloc INTERFACE ${SNMALLOC_GWP_ASAN_LIBRARY_PATH}) + target_compile_definitions(snmalloc INTERFACE -DSNMALLOC_ENABLE_GWP_ASAN_INTEGRATION) + target_link_libraries(snmalloc INTERFACE clang_rt.gwp_asan-${CMAKE_SYSTEM_PROCESSOR}) +endif() + # https://learn.microsoft.com/en-us/cpp/build/reference/zc-cplusplus if(MSVC) target_compile_options(snmalloc INTERFACE "/Zc:__cplusplus") diff --git a/src/snmalloc/backend/globalconfig.h b/src/snmalloc/backend/globalconfig.h index 97ca9ec94..f95f134f7 100644 --- a/src/snmalloc/backend/globalconfig.h +++ b/src/snmalloc/backend/globalconfig.h @@ -3,6 +3,7 @@ #include "../backend_helpers/backend_helpers.h" #include "backend.h" #include "meta_protected_range.h" +#include "snmalloc/mem/secondary.h" #include "standard_range.h" namespace snmalloc @@ -107,6 +108,8 @@ namespace snmalloc if (initialised) return; + SecondaryAllocator::initialize(); + LocalEntropy entropy; entropy.init(); // Initialise key for remote deallocation lists diff --git a/src/snmalloc/ds_core/ptrwrap.h b/src/snmalloc/ds_core/ptrwrap.h index 1d6e41f34..853e5fb4c 100644 --- a/src/snmalloc/ds_core/ptrwrap.h +++ b/src/snmalloc/ds_core/ptrwrap.h @@ -4,6 +4,8 @@ #include "defines.h" #include "snmalloc/stl/atomic.h" +#include + namespace snmalloc { /* diff --git a/src/snmalloc/mem/corealloc.h b/src/snmalloc/mem/corealloc.h index f82ccc1c2..55fd5efe0 100644 --- a/src/snmalloc/mem/corealloc.h +++ b/src/snmalloc/mem/corealloc.h @@ -5,6 +5,7 @@ #include "metadata.h" #include "pool.h" #include "remotecache.h" +#include "secondary.h" #include "sizeclasstable.h" #include "snmalloc/stl/new.h" #include "ticker.h" @@ -817,6 +818,14 @@ namespace snmalloc SNMALLOC_SLOW_PATH capptr::Alloc small_alloc(smallsizeclass_t sizeclass, freelist::Iter<>& fast_free_list) { + void* result = SecondaryAllocator::allocate( + [sizeclass]() -> stl::Pair { + auto size = sizeclass_to_size(sizeclass); + return {size, natural_alignment(size)}; + }); + + if (result != nullptr) + return capptr::Alloc::unsafe_from(result); // Look to see if we can grab a free list. auto& sl = alloc_classes[sizeclass].available; if (SNMALLOC_LIKELY(alloc_classes[sizeclass].length > 0)) diff --git a/src/snmalloc/mem/localalloc.h b/src/snmalloc/mem/localalloc.h index d30d4149d..09256a063 100644 --- a/src/snmalloc/mem/localalloc.h +++ b/src/snmalloc/mem/localalloc.h @@ -1,6 +1,8 @@ #pragma once #include "snmalloc/aal/address.h" +#include "snmalloc/mem/remoteallocator.h" +#include "snmalloc/mem/secondary.h" #if defined(_MSC_VER) # define ALLOCATOR __declspec(allocator) __declspec(restrict) #elif __has_attribute(malloc) @@ -194,6 +196,15 @@ namespace snmalloc errno = ENOMEM; return capptr::Alloc{nullptr}; } + + // Check if secondary allocator wants to offer the memory + void* result = + SecondaryAllocator::allocate([size]() -> stl::Pair { + return {size, natural_alignment(size)}; + }); + if (result != nullptr) + return capptr::Alloc::unsafe_from(result); + // Grab slab of correct size // Set remote as large allocator remote. auto [chunk, meta] = Config::Backend::alloc_chunk( @@ -606,17 +617,13 @@ namespace snmalloc #endif } - SNMALLOC_FAST_PATH void dealloc(void* p_raw) - { -#ifdef SNMALLOC_PASS_THROUGH - external_alloc::free(p_raw); -#else - // Care is needed so that dealloc(nullptr) works before init - // The backend allocator must ensure that a minimal page map exists - // before init, that maps null to a remote_deallocator that will never - // be in thread local state. + // The domestic pointer with its origin allocator + using DomesticInfo = stl::Pair, const PagemapEntry&>; -# ifdef __CHERI_PURE_CAPABILITY__ + // Check whether the raw pointer is owned by snmalloc + SNMALLOC_FAST_PATH_INLINE DomesticInfo get_domestic_info(const void* p_raw) + { +#ifdef __CHERI_PURE_CAPABILITY__ /* * On CHERI platforms, snap the provided pointer to its base, ignoring * any client-provided offset, which may have taken the pointer out of @@ -632,10 +639,29 @@ namespace snmalloc * start of the allocation and so the offset is zero. */ p_raw = __builtin_cheri_offset_set(p_raw, 0); -# endif +#endif + capptr::AllocWild p_wild = + capptr_from_client(const_cast(p_raw)); + auto p_tame = + capptr_domesticate(core_alloc->backend_state_ptr(), p_wild); + const PagemapEntry& entry = + Config::Backend::get_metaentry(address_cast(p_tame)); + return {p_tame, entry}; + } - capptr::AllocWild p_wild = capptr_from_client(p_raw); + // Check if a pointer is domestic to SnMalloc + SNMALLOC_FAST_PATH bool is_snmalloc_owned(const void* p_raw) + { + auto [_, entry] = get_domestic_info(p_raw); + RemoteAllocator* remote = entry.get_remote(); + return remote != nullptr; + } + SNMALLOC_FAST_PATH void dealloc(void* p_raw) + { +#ifdef SNMALLOC_PASS_THROUGH + external_alloc::free(p_raw); +#else /* * p_tame may be nullptr, even if p_raw/p_wild are not, in the case * where domestication fails. We exclusively use p_tame below so that @@ -648,11 +674,7 @@ namespace snmalloc * well-formedness) of this pointer. The remainder of the logic will * deal with the object's extent. */ - capptr::Alloc p_tame = - capptr_domesticate(core_alloc->backend_state_ptr(), p_wild); - - const PagemapEntry& entry = - Config::Backend::get_metaentry(address_cast(p_tame)); + auto [p_tame, entry] = get_domestic_info(p_raw); if (SNMALLOC_LIKELY(local_cache.remote_allocator == entry.get_remote())) { @@ -697,18 +719,15 @@ namespace snmalloc return; } - // If p_tame is not null, then dealloc has been call on something - // it shouldn't be called on. - // TODO: Should this be tested even in the !CHECK_CLIENT case? - snmalloc_check_client( - mitigations(sanity_checks), - p_tame == nullptr, - "Not allocated by snmalloc."); - + if (SNMALLOC_LIKELY(p_tame == nullptr)) + { # ifdef SNMALLOC_TRACING - message<1024>("nullptr deallocation"); + message<1024>("nullptr deallocation"); # endif - return; + return; + } + + SecondaryAllocator::deallocate(p_tame.unsafe_ptr()); #endif } @@ -719,6 +738,8 @@ namespace snmalloc #else if constexpr (mitigations(sanity_checks)) { + if (!is_snmalloc_owned(p)) + return; size = size == 0 ? 1 : size; auto sc = size_to_sizeclass_full(size); auto pm_sc = @@ -767,6 +788,11 @@ namespace snmalloc #ifdef SNMALLOC_PASS_THROUGH return external_alloc::malloc_usable_size(const_cast(p_raw)); #else + + if ( + !SecondaryAllocator::pass_through && !is_snmalloc_owned(p_raw) && + p_raw != nullptr) + return SecondaryAllocator::alloc_size(p_raw); // TODO What's the domestication policy here? At the moment we just // probe the pagemap with the raw address, without checks. There could // be implicit domestication through the `Config::Pagemap` or diff --git a/src/snmalloc/mem/secondary.h b/src/snmalloc/mem/secondary.h new file mode 100644 index 000000000..9f3dab307 --- /dev/null +++ b/src/snmalloc/mem/secondary.h @@ -0,0 +1,20 @@ +#pragma once + +#include "snmalloc/ds_core/defines.h" +#include "snmalloc/ds_core/ptrwrap.h" + +#ifdef SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION +# include "snmalloc/mem/secondary/gwp_asan.h" + +namespace snmalloc +{ + using SecondaryAllocator = GwpAsanSecondaryAllocator; +} // namespace snmalloc +#else +# include "snmalloc/mem/secondary/default.h" + +namespace snmalloc +{ + using SecondaryAllocator = DefaultSecondaryAllocator; +} // namespace snmalloc +#endif diff --git a/src/snmalloc/mem/secondary/default.h b/src/snmalloc/mem/secondary/default.h new file mode 100644 index 000000000..ae5d53bfb --- /dev/null +++ b/src/snmalloc/mem/secondary/default.h @@ -0,0 +1,47 @@ +#pragma once + +#include "snmalloc/ds_core/defines.h" +#include "snmalloc/ds_core/mitigations.h" + +#include + +namespace snmalloc +{ + class DefaultSecondaryAllocator + { + public: + // This flag is used to turn off checks on fast paths if the secondary + // allocator does not own the memory at all. + static constexpr inline bool pass_through = true; + + SNMALLOC_FAST_PATH + static void initialize() {} + + template + SNMALLOC_FAST_PATH static void* allocate(SizeAlign&&) + { + return nullptr; + } + + SNMALLOC_FAST_PATH + static void deallocate(void* pointer) + { + // If pointer is not null, then dealloc has been call on something + // it shouldn't be called on. + // TODO: Should this be tested even in the !CHECK_CLIENT case? + snmalloc_check_client( + mitigations(sanity_checks), + pointer == nullptr, + "Not allocated by snmalloc."); + } + + SNMALLOC_FAST_PATH + static size_t alloc_size(const void*) + { + SNMALLOC_ASSERT( + false && + "secondary alloc_size should never be invoked with default setup"); + return 0; + } + }; +} // namespace snmalloc diff --git a/src/snmalloc/mem/secondary/gwp_asan.h b/src/snmalloc/mem/secondary/gwp_asan.h new file mode 100644 index 000000000..178a3eeee --- /dev/null +++ b/src/snmalloc/mem/secondary/gwp_asan.h @@ -0,0 +1,71 @@ +#pragma once + +#include "gwp_asan/guarded_pool_allocator.h" +#include "snmalloc/ds_core/defines.h" +#include "snmalloc/mem/sizeclasstable.h" +#if defined(SNMALLOC_BACKTRACE_HEADER) +# include SNMALLOC_BACKTRACE_HEADER +#endif + +namespace snmalloc +{ + class GwpAsanSecondaryAllocator + { + static inline gwp_asan::GuardedPoolAllocator singleton; + static inline size_t max_allocation_size; + + public: + static constexpr inline bool pass_through = false; + + static void initialize() noexcept + { + // for now, we use default options + gwp_asan::options::Options opt; + opt.setDefaults(); +#ifdef SNMALLOC_BACKTRACE_HEADER + opt.Backtrace = [](uintptr_t* buf, size_t length) { + return static_cast( + ::backtrace(reinterpret_cast(buf), static_cast(length))); + }; +#endif + singleton.init(opt); + max_allocation_size = + singleton.getAllocatorState()->maximumAllocationSize(); + } + + // Use thunk to avoid extra computation when allocation decision can be made + // before size and alignment are computed. + template + SNMALLOC_FAST_PATH static void* allocate(SizeAlign&& getter) + { + // TODO: this `shouldSample` is only triggered on snmalloc's slowpath, + // which may reduce the chance of error detection. We may reconsider + // the logic to improve the precision in future commits. + if (SNMALLOC_UNLIKELY(singleton.shouldSample())) + { + auto [size, align] = getter(); + if (size > max_allocation_size) + return nullptr; + return singleton.allocate(size, align); + } + return nullptr; + } + + SNMALLOC_FAST_PATH + static void deallocate(void* pointer) + { + snmalloc_check_client( + mitigations(sanity_checks), + singleton.pointerIsMine(pointer), + "Not allocated by snmalloc or secondary allocator"); + + singleton.deallocate(pointer); + } + + SNMALLOC_FAST_PATH + static size_t alloc_size(const void* pointer) + { + return singleton.getSize(pointer); + } + }; +} // namespace snmalloc diff --git a/src/test/func/client_meta/client_meta.cc b/src/test/func/client_meta/client_meta.cc index dd576d304..d1d99a341 100644 --- a/src/test/func/client_meta/client_meta.cc +++ b/src/test/func/client_meta/client_meta.cc @@ -23,7 +23,8 @@ namespace snmalloc int main() { -#ifdef SNMALLOC_PASS_THROUGH +#if defined(SNMALLOC_PASS_THROUGH) || \ + defined(SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION) // This test does not make sense in pass-through return 0; #else diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index 2c00c7b8c..f5e513b0c 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -1,3 +1,4 @@ +#include "snmalloc/mem/secondary.h" #include "test/setup.h" #include @@ -38,18 +39,26 @@ int main() while (true) { auto r1 = a.alloc(object_size); + count += object_size; i++; + // Run until we exhaust the fixed region. + // This should return null. + if (r1 == nullptr) + break; + + if (!a.is_snmalloc_owned(r1)) + { + a.dealloc(r1); + continue; + } + if (i == 1024) { i = 0; std::cout << "."; } - // Run until we exhaust the fixed region. - // This should return null. - if (r1 == nullptr) - break; if (oe_base > r1) { diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index 984909123..b87b8800a 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -76,12 +76,16 @@ namespace test size_t rand = (size_t)r.next(); size_t oid = rand & (((size_t)1 << count_log) - 1); size_t* external_ptr = objects[oid]; + if (!alloc.is_snmalloc_owned(external_ptr)) + continue; size_t size = *external_ptr; size_t offset = (size >> 4) * (rand & 15); void* interior_ptr = pointer_offset(external_ptr, offset); void* calced_external = alloc.external_pointer(interior_ptr); if (calced_external != external_ptr) + { abort(); + } } }