From c7682c487e928ce7bc6740c39fb75f49bf083e0b Mon Sep 17 00:00:00 2001 From: Jake Hillion Date: Thu, 15 Aug 2024 15:48:08 +0100 Subject: [PATCH] nix: add building oid to the flake OI's build is challenging and has often been a problem for the Open Source community. It requires an extremely specific set of dependencies that are very hard to achieve on most systems. There are frequent breakages, like when updating to CentOS Stream 9, or when trying to update the CI's clang from clang-12 to clang-15 - OI requires the clang libraries to be version 15 but can't be compiled with it on the CI! This changes provides a mostly working build environment with `nix`. This environment is pinned to a specific nixpkgs revision using `flake.lock`, and only updates when we explicitly tell it to. Summary of changes: - Update CMakeLists.txt required version to 3.24. This allows specifying `FIND_PACKAGE_ARGS` in `FetchContent`, meaning we can use system packages. This is available on most up to date distros (3.30.2 is current). - Extends `flake.nix` to be able to build OI. Adds instructions for building and developing OI using `nix`. - Partially runs the tests in GitHub Actions. A huge amount must be excluded because of incompatibilites between the clangStdenv compiler and drgn. We have similar, though fewer, issues when building with the clang-12/libstdcxx mix on the Ubuntu 22.04 CircleCI, though this is at least reproducible. - Updates CircleCI to build CMake from source as we don't have a newer image available. Also add some newly found dependencies (not sure how it was working without them before). Test plan: This change requires less testing than previous build related changes because it deprecates most of the build types. - The internal BUCK build is unaffected. No special testing. - The semi-internal CMake build is gone. Use Nix. - The Nix build for clang-15 and some tests are continuously tested in GitHub actions. - Tested the set of Nix commands in the README. All work except the one that points to GitHub as this must be merged first. - The existing CircleCI runs on Ubuntu 20.04 are maintained. - Unable to test the new `test-report.yml` as it must be merged due to the permissions it needs. Will follow up with testing after this is merged. See: https://github.com/dorny/test-reporter?tab=readme-ov-file#recommended-setup-for-public-repositories The list of exclusions for GitHub Actions/nix testing is currently very long, I think 29% of the tests. This should be stable and reproducible though, and likely needs deep changes to OI to fix. That's why fixes are excluded from this PR. It's all to do with the forked drgn not being able to parse clang's newer DWARF output, and can't be fixed by rolling back as we required a relatively new libcxx. --- .circleci/config.yml | 18 +- .github/workflows/ci.yml | 54 ++++ .github/workflows/object-introspection.yml | 15 -- .github/workflows/test-report.yml | 20 ++ .github/workflows/tests_failing_under_nix.txt | 235 ++++++++++++++++++ CMakeLists.txt | 234 +++++++++-------- README.md | 58 +++++ cmake/Finduring.cmake | 27 ++ flake.nix | 96 ++++++- oi/CMakeLists.txt | 3 - oi/type_graph/CMakeLists.txt | 3 - tools/config_gen.py | 6 +- 12 files changed, 640 insertions(+), 129 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/object-introspection.yml create mode 100644 .github/workflows/test-report.yml create mode 100644 .github/workflows/tests_failing_under_nix.txt create mode 100644 cmake/Finduring.cmake diff --git a/.circleci/config.yml b/.circleci/config.yml index 589245ab..dba98d7a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -83,20 +83,26 @@ jobs: libgtest-dev \ libjemalloc-dev \ libmsgpack-dev \ + libomp-12-dev \ + liburing-dev \ libzstd-dev \ llvm-15-dev \ ninja-build \ pkg-config \ python3-setuptools pip3 install toml + + # Ubuntu 22.04 CMake is too old and we don't have a newer image yet + git clone --depth 1 --branch v3.30.2 https://github.com/Kitware/CMake.git /tmp/cmake + (cd /tmp/cmake && cmake -B build/ -G Ninja && cmake --build build/) environment: DEBIAN_FRONTEND: noninteractive - checkout - run: name: Build command: | - cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >> - cmake --build build/ + /tmp/cmake/build/bin/cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >> + ninja -C build/ # Testing rubbish: cp test/ci.oid.toml build/testing.oid.toml - persist_to_workspace: @@ -131,10 +137,12 @@ jobs: sudo apt-get install -y \ clang-15 \ libboost-all-dev \ - libgflags-dev \ - llvm-15-dev \ libfmt-dev \ - libjemalloc-dev + libgflags-dev \ + libgoogle-glog-dev \ + libjemalloc-dev \ + libomp-12-dev \ + llvm-15-dev environment: DEBIAN_FRONTEND: noninteractive - run: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..8a12ac8e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,54 @@ +name: CI +on: + pull_request: +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.7 + - uses: cachix/install-nix-action@v27 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + - name: nix fmt + run: |- + nix --experimental-features 'nix-command flakes' fmt + git diff --exit-code + + build-test: + runs-on: 16-core-ubuntu + strategy: + matrix: + llvm_version: [15] + steps: + - uses: actions/checkout@v4.1.7 + - uses: cachix/install-nix-action@v27 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + - name: build (LLVM ${{ matrix.llvm_version }}) + # Run the build manually in `nix develop` to keep non-outputs around + run: | + nix develop .#oid-llvm${{ matrix.llvm_version }} --command cmake -B build -G Ninja -DWITH_FLAKY_TESTS=Off -DFORCE_BOOST_STATIC=Off + nix develop .#oid-llvm${{ matrix.llvm_version }} --command ninja -C build + - name: test (LLVM ${{ matrix.llvm_version }}) + env: + # disable drgn multithreading as tests are already run in parallel + OMP_NUM_THREADS: 1 + run: | + echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope + nix develop .#oid-llvm${{ matrix.llvm_version }} --command ./tools/config_gen.py -c clang++ build/testing.oid.toml + nix develop .#oid-llvm${{ matrix.llvm_version }} --command ctest \ + --test-dir build/test/ \ + --test-action Test \ + --parallel \ + --no-compress-output \ + --schedule-random \ + --timeout 60 \ + --repeat until-pass:2 \ + --exclude-from-file ../../.github/workflows/tests_failing_under_nix.txt \ + --output-junit results.xml + - name: upload results + uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: test-results-${{ matrix.llvm_version }} + path: build/test/results.xml diff --git a/.github/workflows/object-introspection.yml b/.github/workflows/object-introspection.yml deleted file mode 100644 index 70b41aa5..00000000 --- a/.github/workflows/object-introspection.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: facebookexperimental/object-introspection -on: - pull_request: -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4.1.0 - - uses: cachix/install-nix-action@v25 - with: - github_access_token: ${{ secrets.GITHUB_TOKEN }} - - name: nix fmt - run: |- - nix --experimental-features 'nix-command flakes' fmt - git diff --exit-code diff --git a/.github/workflows/test-report.yml b/.github/workflows/test-report.yml new file mode 100644 index 00000000..91524b10 --- /dev/null +++ b/.github/workflows/test-report.yml @@ -0,0 +1,20 @@ +name: 'Test Report' +on: + workflow_run: + workflows: ['CI'] + types: + - completed +permissions: + contents: read + actions: read + checks: write +jobs: + report: + runs-on: ubuntu-latest + steps: + - uses: dorny/test-reporter@v1 + with: + artifact: test-results-15 + name: CTest Tests + path: results.xml + reporter: jest-junit diff --git a/.github/workflows/tests_failing_under_nix.txt b/.github/workflows/tests_failing_under_nix.txt new file mode 100644 index 00000000..ed514f47 --- /dev/null +++ b/.github/workflows/tests_failing_under_nix.txt @@ -0,0 +1,235 @@ +AddChildrenTest.InheritancePolymorphic +ClangTypeParserTest.MemberAlignment +ClangTypeParserTest.SimpleStruct +DrgnParserTest.ClassTemplateInt +DrgnParserTest.Container +DrgnParserTest.TemplateEnumValue +DrgnParserTest.TemplateEnumValueGaps +DrgnParserTest.TemplateEnumValueNegative +OidIntegration.alignment_wrapper_member_alignment +OidIntegration.alignment_wrapper_member_lower +OidIntegration.alignment_wrapper_member_override +OidIntegration.alignment_wrapper_struct +OidIntegration.alignment_wrapper_two_members +OidIntegration.alignment_wrapper_union_member +OidIntegration.anonymous_anon_struct +OidIntegration.anonymous_anon_typedef +OidIntegration.anonymous_anon_union +OidIntegration.anonymous_nested_anon_struct +OidIntegration.anonymous_regular_struct +OidIntegration.arrays_member_int0 +OidIntegration.arrays_member_int10 +OidIntegration.arrays_multidim +OidIntegration.arrays_multidim_legacy +OidIntegration.bitfields_enum +OidIntegration.bitfields_mixed +OidIntegration.bitfields_single +OidIntegration.bitfields_straddle_bytes +OidIntegration.bitfields_within_bytes +OidIntegration.bitfields_zero_bits +OidIntegration.cycles_raw_ptr +OidIntegration.cycles_raw_ptr_wrapped +OidIntegration.cycles_shared_ptr +OidIntegration.cycles_unique_ptr +OidIntegration.enums_params_scoped_enum_val +OidIntegration.enums_params_scoped_enum_val_cast +OidIntegration.enums_params_scoped_enum_val_gaps +OidIntegration.enums_params_scoped_enum_val_negative +OidIntegration.enums_params_unscoped_enum_val_cast +OidIntegration.fbstring_empty +OidIntegration.fbstring_heap_allocated +OidIntegration.fbstring_inline +OidIntegration.fbstring_string_pooled_unique +OidIntegration.folly_f14_fast_map_a +OidIntegration.folly_f14_fast_set_a +OidIntegration.folly_f14_node_map_a +OidIntegration.folly_f14_node_set_a +OidIntegration.folly_f14_value_map_a +OidIntegration.folly_f14_value_set_a +OidIntegration.folly_f14_vector_map_a +OidIntegration.folly_f14_vector_set_a +OidIntegration.folly_small_vector_int_always_heap +OidIntegration.folly_small_vector_int_default_empty +OidIntegration.folly_small_vector_int_default_inlined +OidIntegration.folly_small_vector_int_default_overflow +OidIntegration.folly_small_vector_vector_3_empty +OidIntegration.folly_small_vector_vector_3_inlined +OidIntegration.folly_small_vector_vector_3_overflow +OidIntegration.folly_sorted_vector_map_int_int_empty +OidIntegration.folly_sorted_vector_map_int_int_reserve +OidIntegration.folly_sorted_vector_map_int_int_some +OidIntegration.ignored_member +OidIntegration.ignored_roottype +OidIntegration.ignored_subtype +OidIntegration.inheritance_access_private +OidIntegration.inheritance_access_protected +OidIntegration.inheritance_access_public +OidIntegration.inheritance_access_public_as_base +OidIntegration.inheritance_multiple_a +OidIntegration.inheritance_polymorphic_a_as_a +OidIntegration.inheritance_polymorphic_b_as_a +OidIntegration.inheritance_polymorphic_b_as_b +OidIntegration.inheritance_polymorphic_c_as_a +OidIntegration.inheritance_polymorphic_c_as_b +OidIntegration.inheritance_polymorphic_c_as_c +OidIntegration.inheritance_polymorphic_diamond_child_as_child +OidIntegration.inheritance_polymorphic_diamond_child_as_middle1 +OidIntegration.inheritance_polymorphic_diamond_child_as_middle1_root +OidIntegration.inheritance_polymorphic_diamond_child_as_middle2 +OidIntegration.inheritance_polymorphic_diamond_child_as_middle2_root +OidIntegration.inheritance_polymorphic_diamond_middle1_as_middle1 +OidIntegration.inheritance_polymorphic_diamond_middle1_as_root +OidIntegration.inheritance_polymorphic_diamond_middle2_as_middle2 +OidIntegration.inheritance_polymorphic_diamond_middle2_as_root +OidIntegration.inheritance_polymorphic_diamond_root_as_root +OidIntegration.inheritance_polymorphic_non_dynamic_base_a_as_a +OidIntegration.inheritance_polymorphic_non_dynamic_base_a_no_polymorphic +OidIntegration.inheritance_polymorphic_non_dynamic_base_b_as_a +OidIntegration.inheritance_polymorphic_non_dynamic_base_b_as_b +OidIntegration.inheritance_polymorphic_non_dynamic_base_b_no_polymorphic +OidIntegration.inheritance_polymorphic_non_dynamic_base_c_as_a +OidIntegration.inheritance_polymorphic_non_dynamic_base_c_as_b +OidIntegration.inheritance_polymorphic_non_dynamic_base_c_as_c +OidIntegration.inheritance_polymorphic_non_dynamic_base_c_no_polymorphic +OidIntegration.multi_arg_tb_all_fail_crashes +OidIntegration.multi_arg_tb_fail_first_arg +OidIntegration.namespaces_queue +OidIntegration.namespaces_stack +OidIntegration.packed_a +OidIntegration.padding_bool_padding +OidIntegration.padding_nested_padding +OidIntegration.pointers_feature_config +OidIntegration.pointers_feature_flag_disabled +OidIntegration.pointers_incomplete_containing_struct +OidIntegration.pointers_incomplete_containing_struct_no_follow +OidIntegration.pointers_incomplete_shared_ptr +OidIntegration.pointers_incomplete_shared_ptr_null +OidIntegration.pointers_incomplete_unique_ptr +OidIntegration.pointers_incomplete_unique_ptr_null +OidIntegration.pointers_struct_primitive_ptrs +OidIntegration.pointers_struct_primitive_ptrs_no_follow +OidIntegration.pointers_struct_primitive_ptrs_null +OidIntegration.pointers_struct_vector_ptr +OidIntegration.pointers_struct_vector_ptr_no_follow +OidIntegration.pointers_struct_vector_ptr_null +OidIntegration.pointers_vector_of_pointers +OidIntegration.primitives_long_double +OidIntegration.simple_class +OidIntegration.simple_struct +OidIntegration.simple_union +OidIntegration.sorted_vector_set_no_ints +OidIntegration.sorted_vector_set_some_ints +OidIntegration.std_array_uint64_length_0 +OidIntegration.std_array_uint64_length_1 +OidIntegration.std_array_uint64_length_8 +OidIntegration.std_array_vector_length_1 +OidIntegration.std_array_vector_length_2 +OidIntegration.std_conditional_a +OidIntegration.std_deque_del_allocator_a +OidIntegration.std_deque_deque_int_empty +OidIntegration.std_deque_deque_int_some +OidIntegration.std_deque_int_empty +OidIntegration.std_deque_int_some +OidIntegration.std_list_del_allocator_a +OidIntegration.std_list_int_empty +OidIntegration.std_list_int_some +OidIntegration.std_list_list_int_empty +OidIntegration.std_list_list_int_some +OidIntegration.std_list_struct_some +OidIntegration.std_map_custom_comparator_a +OidIntegration.std_multimap_custom_comparator_a +OidIntegration.std_multiset_custom_comparator_a +OidIntegration.std_optional_uint64_empty +OidIntegration.std_optional_uint64_present +OidIntegration.std_optional_vector_empty +OidIntegration.std_optional_vector_present +OidIntegration.std_pair_uint64_uint32 +OidIntegration.std_pair_uint64_uint64 +OidIntegration.std_pair_vector_vector +OidIntegration.std_priority_queue_adapter_deque_empty +OidIntegration.std_priority_queue_adapter_deque_some +OidIntegration.std_priority_queue_int_empty +OidIntegration.std_priority_queue_int_some +OidIntegration.std_queue_adapter_vector_empty +OidIntegration.std_queue_adapter_vector_some +OidIntegration.std_queue_int_empty +OidIntegration.std_queue_int_some +OidIntegration.std_queue_queue_int_empty +OidIntegration.std_queue_queue_int_some +OidIntegration.std_reference_wrapper_int +OidIntegration.std_reference_wrapper_vector +OidIntegration.std_set_custom_comparator_a +OidIntegration.std_smart_ptr_shared_ptr_const_uint64_empty +OidIntegration.std_smart_ptr_shared_ptr_const_vector_empty +OidIntegration.std_smart_ptr_shared_ptr_uint64_empty +OidIntegration.std_smart_ptr_shared_ptr_uint64_present +OidIntegration.std_smart_ptr_shared_ptr_vector_empty +OidIntegration.std_smart_ptr_shared_ptr_vector_present +OidIntegration.std_smart_ptr_shared_ptr_void_empty +OidIntegration.std_smart_ptr_shared_ptr_void_present +OidIntegration.std_smart_ptr_unique_ptr_const_uint64_empty +OidIntegration.std_smart_ptr_unique_ptr_const_vector_empty +OidIntegration.std_smart_ptr_unique_ptr_uint64_empty +OidIntegration.std_smart_ptr_unique_ptr_uint64_present +OidIntegration.std_smart_ptr_unique_ptr_vector_empty +OidIntegration.std_smart_ptr_unique_ptr_vector_present +OidIntegration.std_smart_ptr_unique_ptr_void_empty +OidIntegration.std_smart_ptr_unique_ptr_void_present +OidIntegration.std_smart_ptr_weak_ptr_int64_empty +OidIntegration.std_smart_ptr_weak_ptr_int64_expired +OidIntegration.std_smart_ptr_weak_ptr_int64_expired_chase +OidIntegration.std_smart_ptr_weak_ptr_int64_present +OidIntegration.std_smart_ptr_weak_ptr_int64_present_chase +OidIntegration.std_smart_ptr_weak_ptr_int64_void_empty +OidIntegration.std_stack_adapter_vector_empty +OidIntegration.std_stack_adapter_vector_some +OidIntegration.std_stack_int_empty +OidIntegration.std_stack_int_some +OidIntegration.std_stack_stack_int_empty +OidIntegration.std_stack_stack_int_some +OidIntegration.std_string_empty +OidIntegration.std_string_heap_allocated +OidIntegration.std_string_sso +OidIntegration.std_tuple_uint64_uint64 +OidIntegration.std_unordered_map_custom_operator_a +OidIntegration.std_unordered_multimap_custom_operator_a +OidIntegration.std_unordered_multiset_custom_operator_a +OidIntegration.std_unordered_set_custom_operator_a +OidIntegration.std_variant_256_params_256 +OidIntegration.std_variant_256_params_empty +OidIntegration.std_variant_char_int64_1 +OidIntegration.std_variant_char_int64_2 +OidIntegration.std_variant_empty +OidIntegration.std_variant_optional +OidIntegration.std_variant_vector_int_1 +OidIntegration.std_variant_vector_int_2 +OidIntegration.std_vector_del_allocator_a +OidIntegration.std_vector_int_empty +OidIntegration.std_vector_int_some +OidIntegration.std_vector_reserve +OidIntegration.std_vector_struct_some +OidIntegration.std_vector_vector_int_empty +OidIntegration.std_vector_vector_int_some +OidIntegration.templates_int +OidIntegration.templates_two +OidIntegration.templates_value +OidIntegration.templates_vector +OidIntegration.typedefed_parent_multilevel_typedef_parent +OidIntegration.typedefed_parent_simple_typedef_parent +OidIntegration.typedefs_anonymous +OidIntegration.typedefs_container +OidIntegration.unions_alignment +OidIntegration.unions_int +OidIntegration.unions_tagged_int +OidIntegration.unions_tagged_unordered_map +OidIntegration.unions_tagged_vector +OidIntegration.unions_unordered_map +OidIntegration.unions_vector +OilIntegration.folly_f14_fast_map_a +OilIntegration.folly_f14_fast_set_a +OilIntegration.folly_f14_node_map_a +OilIntegration.folly_f14_node_set_a +OilIntegration.folly_f14_value_map_a +OilIntegration.folly_f14_value_set_a +OilIntegration.folly_f14_vector_map_a +OilIntegration.folly_f14_vector_set_a diff --git a/CMakeLists.txt b/CMakeLists.txt index 8674d464..426b68c4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ # object-introspection -cmake_minimum_required(VERSION 3.20) +cmake_minimum_required(VERSION 3.24) project(object-introspection) # Lets find_program() locate SETUID binaries @@ -33,75 +33,108 @@ find_package(gflags REQUIRED) ### tomlplusplus (for configuration files) FetchContent_Declare( tomlplusplus - GIT_REPOSITORY https://github.com/marzer/tomlplusplus.git - GIT_TAG 4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0 - GIT_PROGRESS TRUE + GIT_REPOSITORY https://github.com/marzer/tomlplusplus.git + GIT_TAG 4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0 + GIT_PROGRESS TRUE + FIND_PACKAGE_ARGS ) FetchContent_MakeAvailable(tomlplusplus) ### glog -FetchContent_Declare( - glog - GIT_REPOSITORY https://github.com/google/glog.git - GIT_TAG 96a2f23dca4cc7180821ca5f32e526314395d26a - GIT_PROGRESS TRUE -) -FetchContent_MakeAvailable(glog) - -# These glog executable targets still generate warnings - disable warnings for -# them explicitly -target_compile_options(demangle_unittest PRIVATE "-w") -target_compile_options(logging_unittest PRIVATE "-w") -target_compile_options(stl_logging_unittest PRIVATE "-w") -target_compile_options(symbolize_unittest PRIVATE "-w") -target_compile_options(utilities_unittest PRIVATE "-w") +find_package(glog) +if (NOT glog_FOUND) + FetchContent_Declare( + glog + GIT_REPOSITORY https://github.com/google/glog.git + GIT_TAG 96a2f23dca4cc7180821ca5f32e526314395d26a + GIT_PROGRESS TRUE + FIND_PACKAGE_ARGS + ) + FetchContent_MakeAvailable(glog) + + # These glog executable targets still generate warnings - disable warnings for + # them explicitly + target_compile_options(demangle_unittest PRIVATE "-w") + target_compile_options(logging_unittest PRIVATE "-w") + target_compile_options(stl_logging_unittest PRIVATE "-w") + target_compile_options(symbolize_unittest PRIVATE "-w") + target_compile_options(utilities_unittest PRIVATE "-w") +endif() -### googletest +### GTest # Do this in the main file so it can be fetched before setting project warnings. # After this is fixed with FetchContent, move to test/CMakeLists.txt. FetchContent_Declare( - googletest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG 1ed6a8c67a0bd675149ece27bbec0ef1759854cf - GIT_PROGRESS TRUE + GTest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG 1ed6a8c67a0bd675149ece27bbec0ef1759854cf + GIT_PROGRESS TRUE + FIND_PACKAGE_ARGS ) -FetchContent_MakeAvailable(googletest) +FetchContent_MakeAvailable(GTest) + +### liburing (for RocksDB) +find_package(uring REQUIRED) ### rocksdb -FetchContent_Declare( - rocksdb - GIT_REPOSITORY https://github.com/facebook/rocksdb.git - GIT_TAG f32521662acf3352397d438b732144c7813bbbec # v8.5.3 - GIT_PROGRESS TRUE -) -FetchContent_Populate(rocksdb) -add_custom_target(librocksdb ALL - WORKING_DIRECTORY ${rocksdb_SOURCE_DIR} - COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off - COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb - BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a - COMMENT "Building RocksDB" - USES_TERMINAL -) -include_directories(SYSTEM "${rocksdb_SOURCE_DIR}/include") +find_package(RocksDB 8.11 CONFIG) +if (NOT RocksDB_FOUND) + FetchContent_Declare( + rocksdb + GIT_REPOSITORY https://github.com/facebook/rocksdb.git + GIT_TAG f32521662acf3352397d438b732144c7813bbbec # v8.5.3 + GIT_PROGRESS TRUE + ) + FetchContent_Populate(rocksdb) + + add_custom_target(librocksdb_build ALL + WORKING_DIRECTORY ${rocksdb_SOURCE_DIR} + COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off + COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb + BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a + COMMENT "Building RocksDB" + USES_TERMINAL + ) + + ### zstd (for rocksdb) + find_package(zstd REQUIRED) + + add_library(librocksdb INTERFACE) + add_dependencies(librocksdb librocksdb_build) + target_include_directories(librocksdb INTERFACE SYSTEM "${rocksdb_SOURCE_DIR}/include") + target_link_libraries(librocksdb INTERFACE ${rocksdb_BINARY_DIR}/librocksdb.a zstd::zstd) + + add_library(RocksDB::rocksdb ALIAS librocksdb) +endif() ### folly ### use folly as a header only library. some features won't be supported. -FetchContent_Declare( - folly - GIT_REPOSITORY https://github.com/facebook/folly.git - GIT_TAG c5aa5c46291a27f69acc920894d43605ceb43eba - GIT_PROGRESS TRUE - PATCH_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h -) -FetchContent_Populate(folly) +find_package(folly CONFIG) +if (NOT folly_FOUND) + FetchContent_Declare( + folly + GIT_REPOSITORY https://github.com/facebook/folly.git + GIT_TAG c5aa5c46291a27f69acc920894d43605ceb43eba + GIT_PROGRESS TRUE + PATCH_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h + ) + FetchContent_Populate(folly) + + add_library(folly_headers INTERFACE) + target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR}) + target_link_libraries(folly_headers INTERFACE Boost::headers) +else() + add_library(folly_headers INTERFACE) + target_include_directories(folly_headers SYSTEM INTERFACE ${FOLLY_INCLUDE_DIR}) +endif() ### range-v3 FetchContent_Declare( range-v3 - GIT_REPOSITORY https://github.com/ericniebler/range-v3.git - GIT_TAG a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0 - GIT_PROGRESS TRUE + GIT_REPOSITORY https://github.com/ericniebler/range-v3.git + GIT_TAG a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0 + GIT_PROGRESS TRUE + FIND_PACKAGE_ARGS ) FetchContent_MakeAvailable(range-v3) @@ -124,6 +157,18 @@ endif() ### (Re)download submodules find_package(Git QUIET) +if (DEFINED drgn_SOURCE_DIR) + # drgn's autotools build requires source modification. in case we have a + # readonly source (read: nix) we must copy this. do this always to avoid + # polluting the source. + file(COPY "${drgn_SOURCE_DIR}/" DESTINATION "${FETCHCONTENT_BASE_DIR}/drgn-src" NO_SOURCE_PERMISSIONS) + SET(drgn_SOURCE_DIR "${FETCHCONTENT_BASE_DIR}/drgn-src") + SET(drgn_BINARY_DIR "${CMAKE_BINARY_DIR}/_deps/drgn-build") +else() + SET(drgn_SOURCE_DIR "${PROJECT_SOURCE_DIR}/extern/drgn") + SET(drgn_BINARY_DIR "${drgn_SOURCE_DIR}/build") +endif() + # TODO: No idea if this huge block is required, just picked from an example. There may be a short-hand. if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git") # Update submodules as needed @@ -145,20 +190,17 @@ if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git") if(NOT GIT_SUBMOD_RESULT EQUAL "0") message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules") endif() + + if(NOT EXISTS "${drgn_SOURCE_DIR}") + message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.") + endif() endif() endif() -if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/drgn") - message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.") -endif() ### Select Python version find_program(PYTHON NAMES python3.9 python3) -add_library(folly_headers INTERFACE) -target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR}) -target_link_libraries(folly_headers INTERFACE Boost::headers) - ### bison & flex (for oid_parser) find_package(BISON 3.5 REQUIRED) find_package(FLEX) @@ -194,8 +236,11 @@ find_package(msgpack REQUIRED CONFIG) get_target_property(MSGPACK_INCLUDE_DIRS msgpackc INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${MSGPACK_INCLUDE_DIRS}) -### zstd (for rocksdb) -find_package(zstd REQUIRED) +### drgn/elfutils dependencies +find_package(BZip2 REQUIRED) +find_package(OpenMP REQUIRED) +find_package(LibLZMA REQUIRED) +find_package(ZLIB REQUIRED) ### drgn # The setup.py script in drgn is really meant to build drgn (python @@ -214,46 +259,38 @@ find_package(zstd REQUIRED) # make # # Since setup.py has a single cmd to do this, just use it for now. -# -# Another extemely annoying point. drgn pretty much has to be compiled with gcc only -# clang-12 does NOT work. clang fails with the following error :- -# configure: error: gcc with GNU99 support required set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no") if (ASAN) list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes") endif() -add_custom_target(libdrgn ALL - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn - COMMAND unset BISON_PKGDATADIR && CC=gcc CFLAGS="${DRGN_CFLAGS}" CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp build - BYPRODUCTS ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/.libs/libdrgnimpl.a - ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdw/libdw.a - ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libelf/libelf.a - ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdwelf/libdwelf.a + +add_custom_target(drgn_build ALL + WORKING_DIRECTORY ${drgn_SOURCE_DIR} + COMMAND unset BISON_PKGDATADIR && CFLAGS="${DRGN_CFLAGS}" + CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp "${drgn_BINARY_DIR}" + BYPRODUCTS ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a + ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a + ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a + ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a COMMENT "Building drgn" USES_TERMINAL ) -set(DRGN_PATH "${PROJECT_SOURCE_DIR}/extern/drgn/build") - -# Ideally drgn stuff should be together at the end. But looks like rpath needs -# to be set before add_executable() unfortunately. Maybe split libdrgn stuff -# into a separate file later. -set(CMAKE_SKIP_BUILD_RPATH FALSE) -set(CMAKE_INSTALL_RPATH - "${DRGN_PATH}/.libs" - "${DRGN_PATH}/velfutils/libdw" - "${DRGN_PATH}/velfutils/libelf" - "${DRGN_PATH}/velfutils/libdwelf" -) -set(CMAKE_BUILD_RPATH - "${DRGN_PATH}/.libs" - "${DRGN_PATH}/velfutils/libdw" - "${DRGN_PATH}/velfutils/libelf" - "${DRGN_PATH}/velfutils/libdwelf" -) -set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) -include_directories(SYSTEM "${DRGN_PATH}") +add_library(drgn INTERFACE) +add_dependencies(drgn drgn_build) +target_link_libraries(drgn INTERFACE + ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a + ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a + ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a + ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a + + BZip2::BZip2 + LibLZMA::LibLZMA + OpenMP::OpenMP_CXX + ZLIB::ZLIB +) +target_include_directories(drgn SYSTEM INTERFACE "${drgn_SOURCE_DIR}" "${drgn_BINARY_DIR}") if (STATIC_LINK) # glog links against the `gflags` target, which is an alias for `gflags_shared` @@ -289,7 +326,6 @@ add_library(oicore oi/PaddingHunter.cpp oi/Serialize.cpp ) -add_dependencies(oicore libdrgn) target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS}) target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS}) target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) @@ -323,7 +359,6 @@ else() endif() target_link_libraries(oicore - "-L${DRGN_PATH}/.libs" drgn dw pthread @@ -334,11 +369,9 @@ add_library(treebuilder oi/TreeBuilder.cpp oi/exporters/TypeCheckingWalker.cpp ) -add_dependencies(treebuilder librocksdb) target_link_libraries(treebuilder - ${rocksdb_BINARY_DIR}/librocksdb.a + RocksDB::rocksdb oicore # overkill but it does need a lot of stuff - zstd::zstd ) @@ -375,10 +408,8 @@ target_link_libraries(oip oicore) ### Object Introspection RocksDB Printer (OIRP) add_executable(oirp tools/OIRP.cpp) -add_dependencies(oirp librocksdb) target_link_libraries(oirp - ${rocksdb_BINARY_DIR}/librocksdb.a - zstd::zstd + RocksDB::rocksdb msgpackc ) @@ -417,3 +448,6 @@ endif() if (DEFINED ENV{CMAKE_HOOK}) include($ENV{CMAKE_HOOK}) endif() + +install(TARGETS oid DESTINATION ${CMAKE_INSTALL_BINDIR}) + diff --git a/README.md b/README.md index 9f95bf89..abce6f81 100644 --- a/README.md +++ b/README.md @@ -14,3 +14,61 @@ See the [CONTRIBUTING](CONTRIBUTING.md) file for how to help out. ## License Object Introspection is licensed under the [Apache 2.0 License](LICENSE). + +## Getting started with Nix + +Nix is the easiest way to get started with `oid` as it is non-trivial to build otherwise. Explicit Nix support for Object Introspection as a Library will come down the line, but Nix can currently provide you a reproducible development environment in which to build it. + +These examples expect you to have `nix` installed and available with no other dependencies required. Find the installation guide at https://nixos.org/download.html. + +We also required flake support. To enable flakes globally run: + + $ mkdir -p ~/.config/nix + $ echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf + +Or suffix every `nix` command with `nix --extra-experimental-features 'nix-command flakes'`. + +### Run upstream OID without modifying the source + + $ nix run github:facebookexperimental/object-introspection -- --help + +This will download the latest source into your Nix store along with all of its dependencies, running help afterwards. + +### Build OID locally + + $ git clone https://github.com/facebookexperimental/object-introspection + $ nix build + $ ./result/bin/oid --help + +This will build OID from your local sources. Please note that this will NOT pick up changes to `extern/drgn` or `extern/drgn/libdrgn/velfutils`. + +### Get a development environment + + $ nix develop + $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off + $ ninja -C build + $ build/oid --help + +This command provides a development shell with all the required dependencies. This is the most flexible option and will pick up source changes as CMake normally would. + +Sometimes this developer environment can be polluted by things installed on your normal system. If this is an issue, use: + + $ nix develop -i + +This removes the environment from your host system and makes the build pure. + +### Run the tests + + $ nix develop + $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off + $ ninja -C build + $ ./tools/config_gen.py -c clang++ build/testing.oid.toml + $ ctest -j --test-dir build/test + +Running tests under `nix` is new to the project and may take some time to mature. The CI is the source of truth for now. + +### Format source + + $ nix fmt + +This formats the Nix, C++, and Python code in the repository. diff --git a/cmake/Finduring.cmake b/cmake/Finduring.cmake new file mode 100644 index 00000000..3b007fee --- /dev/null +++ b/cmake/Finduring.cmake @@ -0,0 +1,27 @@ +# - Find liburing +# +# uring_INCLUDE_DIR - Where to find liburing.h +# uring_LIBRARIES - List of libraries when using uring. +# uring_FOUND - True if uring found. + +find_path(uring_INCLUDE_DIR + NAMES liburing.h) +find_library(uring_LIBRARIES + NAMES liburing.a liburing.so) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(uring + DEFAULT_MSG uring_LIBRARIES uring_INCLUDE_DIR) + +mark_as_advanced( + uring_INCLUDE_DIR + uring_LIBRARIES) + +if(uring_FOUND AND NOT TARGET uring::uring) + add_library(uring::uring UNKNOWN IMPORTED) + set_target_properties(uring::uring PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${uring_INCLUDE_DIR}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${uring_LIBRARIES}") +endif() + diff --git a/flake.nix b/flake.nix index 7a3382e1..f9256214 100644 --- a/flake.nix +++ b/flake.nix @@ -1,5 +1,5 @@ { - description = "A flake for building Object Introspection."; + description = "Object level memory profiler for C++"; inputs = { nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; @@ -18,7 +18,99 @@ treefmt-nix, ... }@inputs: - flake-utils.lib.eachDefaultSystem ( + flake-utils.lib.eachSystem [ flake-utils.lib.system.x86_64-linux ] ( + system: + let + pkgs = import nixpkgs { inherit system; }; + + drgnSrc = pkgs.fetchFromGitHub { + owner = "JakeHillion"; + repo = "drgn"; + rev = "b1f8c3e8526611b6720800250ba858a713dd9e4f"; + hash = "sha256-5WhMHgx/RKtqjxGx4AyiqVKMot5xulr+6c8i2E9IxiA="; + fetchSubmodules = true; + }; + + mkOidPackage = + llvmPackages: + with pkgs; + pkgs.llvmPackages.stdenv.mkDerivation rec { + name = "oid"; + + src = self; + + nativeBuildInputs = [ + autoconf + automake + bison + cmake + flex + gettext + git + hexdump + libtool + ninja + pkgconf + python312 + python312Packages.setuptools + python312Packages.toml + glibcLocales + ]; + + buildInputs = [ + llvmPackages.libclang + llvmPackages.llvm + + boost + bzip2 + curl + double-conversion + elfutils + flex + folly + folly.fmt + gflags + glog + gtest + icu + jemalloc + libarchive + libmicrohttpd + liburing + libxml2 + lzma + msgpack + range-v3 + rocksdb_8_11 + sqlite + tomlplusplus + zstd + + llvmPackages.openmp # should match the stdenv clang version, see: https://github.com/NixOS/nixpkgs/issues/79818 + ]; + + cmakeFlags = [ + "-Ddrgn_SOURCE_DIR=${drgnSrc}" + "-DFORCE_BOOST_STATIC=Off" + ]; + + outputs = [ "out" ]; + }; + in + { + packages = rec { + default = oid-llvm15; + + oid-llvm15 = mkOidPackage pkgs.llvmPackages_15; + }; + + apps.default = { + type = "app"; + program = "${self.packages.${system}.default}/bin/oid"; + }; + } + ) + // flake-utils.lib.eachDefaultSystem ( system: let pkgs = nixpkgs.legacyPackages.${system}; diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt index bb73871a..7d805e50 100644 --- a/oi/CMakeLists.txt +++ b/oi/CMakeLists.txt @@ -6,11 +6,8 @@ target_link_libraries(toml PUBLIC tomlplusplus::tomlplusplus) add_library(drgn_utils DrgnUtils.cpp) target_link_libraries(drgn_utils glog::glog - - "-L${DRGN_PATH}/.libs" drgn ) -add_dependencies(drgn_utils libdrgn) add_library(symbol_service Descs.cpp diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt index 6c4da13e..c878e085 100644 --- a/oi/type_graph/CMakeLists.txt +++ b/oi/type_graph/CMakeLists.txt @@ -20,12 +20,9 @@ add_library(type_graph TypeIdentifier.cpp Types.cpp ) -add_dependencies(type_graph libdrgn) target_link_libraries(type_graph container_info symbol_service - - "-L${DRGN_PATH}/.libs" drgn ) target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS}) diff --git a/tools/config_gen.py b/tools/config_gen.py index c20ca40c..5ac56ff7 100755 --- a/tools/config_gen.py +++ b/tools/config_gen.py @@ -16,6 +16,7 @@ import argparse import getpass +import os import pathlib import subprocess import typing @@ -170,7 +171,10 @@ def pull_base_toml() -> typing.Dict: # Now, we need to replace any placeholders that might be present in the base toml file with the real verisons. user = getpass.getuser() - pwd = str(repo_path.resolve()) + if "IN_NIX_SHELL" in os.environ and "src" in os.environ: + pwd = os.environ['src'] + else: + pwd = str(repo_path.resolve()) container_list = base.get("types", {}).get("containers") if container_list: