Skip to content

Commit

Permalink
Dynamic dispatch for block codecs (#579)
Browse files Browse the repository at this point in the history
Use dynamic dispatch for block codecs. The old class template
`block_freq_index` is replaced by the `BlockInvertedIndex` class. This class
contains a shared pointer to a `BlockCodec` object, which is a superclass of
all block codec implementations.

Changelog-changed: `BlockInvertedIndex` replaces `block_freq_index`
Changelog-performance: Expect a slight performance drop of all block codecs
elshize authored Jul 20, 2024
1 parent 36add4d commit bb2b3df
Showing 72 changed files with 2,129 additions and 1,797 deletions.
49 changes: 16 additions & 33 deletions benchmarks/index_perftest.cpp
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#include "mappable/mapper.hpp"
#include "memory_source.hpp"
#include "mio/mmap.hpp"
#include "spdlog/spdlog.h"

@@ -10,8 +11,7 @@ using pisa::do_not_optimize_away;
using pisa::get_time_usecs;

template <typename IndexType, bool with_freqs>
void perftest(IndexType const& index, std::string const& type)
{
void perftest(IndexType const& index, std::string const& type) {
std::string freqs_log = with_freqs ? "+freq()" : "";
{
size_t min_length = 4096;
@@ -22,7 +22,8 @@ void perftest(IndexType const& index, std::string const& type)
"Scanning {} posting lists with length between {} and {}",
max_number_of_lists,
min_length,
max_length);
max_length
);

std::vector<size_t> long_lists;
for (size_t i = 0; i < index.size() and long_lists.size() <= max_number_of_lists; ++i) {
@@ -53,7 +54,8 @@ void perftest(IndexType const& index, std::string const& type)
postings,
freqs_log,
uint64_t(elapsed / 1000000),
next_ns);
next_ns
);
spdlog::info("{}\tnext{}\t{:.1f}", type, (with_freqs ? "_freq" : ""), next_ns);
}

@@ -97,26 +99,15 @@ void perftest(IndexType const& index, std::string const& type)
calls,
freqs_log,
skip,
next_geq_ns);
next_geq_ns
);
spdlog::info(
"{}\tnext_geq{}\t{}\t{:.1f}", type, (with_freqs ? "_freq" : ""), skip, next_geq_ns);
"{}\tnext_geq{}\t{}\t{:.1f}", type, (with_freqs ? "_freq" : ""), skip, next_geq_ns
);
}
}

template <typename IndexType>
void perftest(const char* index_filename, std::string const& type)
{
spdlog::info("Loading index from {}", index_filename);
IndexType index;
mio::mmap_source m(index_filename);
pisa::mapper::map(index, m, pisa::mapper::map_flags::warmup);

perftest<IndexType, false>(index, type);
perftest<IndexType, true>(index, type);
}

int main(int argc, const char** argv)
{
int main(int argc, const char** argv) {
using namespace pisa;

if (argc != 3) {
@@ -127,17 +118,9 @@ int main(int argc, const char** argv)
std::string type = argv[1];
const char* index_filename = argv[2];

if (false) {
#define LOOP_BODY(R, DATA, T) \
} \
else if (type == BOOST_PP_STRINGIZE(T)) \
{ \
perftest<BOOST_PP_CAT(T, _index)>(index_filename, type); \
/**/

BOOST_PP_SEQ_FOR_EACH(LOOP_BODY, _, PISA_INDEX_TYPES);
#undef LOOP_BODY
} else {
spdlog::error("Unknown type {}", type);
}
run_for_index(type, MemorySource::mapped_file(std::string_view(index_filename)), [&](auto index) {
using Index = std::decay_t<decltype(index)>;
perftest<Index, false>(index, type);
perftest<Index, true>(index, type);
});
}
2 changes: 1 addition & 1 deletion external/fmt
Submodule fmt updated 80 files
+8 −0 .github/dependabot.yml
+3 −2 .github/pull_request_template.md
+30 −0 .github/workflows/cifuzz.yml
+9 −1 .github/workflows/doc.yml
+26 −0 .github/workflows/lint.yml
+30 −13 .github/workflows/linux.yml
+17 −2 .github/workflows/macos.yml
+65 −0 .github/workflows/scorecard.yml
+12 −4 .github/workflows/windows.yml
+13 −26 .gitignore
+113 −47 CMakeLists.txt
+5,533 −0 ChangeLog.md
+0 −5,255 ChangeLog.rst
+1 −1 LICENSE
+490 −0 README.md
+0 −531 README.rst
+6 −6 doc/_static/bootstrap.min.js
+251 −189 doc/api.rst
+11 −7 doc/build.py
+145 −12 doc/syntax.rst
+38 −0 doc/usage.rst
+7 −6 include/fmt/args.h
+584 −413 include/fmt/chrono.h
+62 −70 include/fmt/color.h
+35 −111 include/fmt/compile.h
+797 −1,151 include/fmt/core.h
+161 −206 include/fmt/format-inl.h
+1,260 −942 include/fmt/format.h
+67 −90 include/fmt/os.h
+77 −69 include/fmt/ostream.h
+210 −175 include/fmt/printf.h
+172 −156 include/fmt/ranges.h
+418 −52 include/fmt/std.h
+68 −38 include/fmt/xchar.h
+39 −30 src/fmt.cc
+1 −5 src/format.cc
+103 −62 src/os.cc
+1 −1 support/AndroidManifest.xml
+3 −3 support/Vagrantfile
+0 −1 support/bazel/.bazelrc
+1 −1 support/bazel/.bazelversion
+2 −2 support/bazel/BUILD.bazel
+5 −4 support/bazel/README.md
+1 −1 support/build.gradle
+0 −54 support/cmake/cxx14.cmake
+45 −19 support/manage.py
+0 −159 support/rst2md.py
+21 −4 test/CMakeLists.txt
+1 −1 test/add-subdirectory-test/CMakeLists.txt
+1 −1 test/args-test.cc
+532 −176 test/chrono-test.cc
+2 −2 test/compile-error-test/CMakeLists.txt
+1 −0 test/compile-fp-test.cc
+28 −43 test/compile-test.cc
+119 −239 test/core-test.cc
+2 −0 test/enforce-checks-test.cc
+1 −1 test/find-package-test/CMakeLists.txt
+28 −99 test/format-impl-test.cc
+728 −744 test/format-test.cc
+1 −1 test/fuzzing/CMakeLists.txt
+3 −1 test/gtest-extra-test.cc
+2 −2 test/gtest-extra.cc
+2 −7 test/gtest-extra.h
+1 −7 test/gtest/CMakeLists.txt
+2 −2 test/gtest/gmock-gtest-all.cc
+2 −2 test/mock-allocator.h
+36 −88 test/module-test.cc
+15 −56 test/os-test.cc
+34 −48 test/ostream-test.cc
+1 −7 test/posix-mock-test.cc
+4 −6 test/posix-mock.h
+15 −39 test/printf-test.cc
+198 −62 test/ranges-test.cc
+103 −31 test/scan-test.cc
+560 −135 test/scan.h
+1 −1 test/static-export-test/CMakeLists.txt
+247 −20 test/std-test.cc
+6 −2 test/util.cc
+15 −17 test/util.h
+187 −79 test/xchar-test.cc
2 changes: 1 addition & 1 deletion external/spdlog
Submodule spdlog updated 160 files
Loading

0 comments on commit bb2b3df

Please sign in to comment.