From 8c8fb467ee2ebf09c088a488a84b58371c220c75 Mon Sep 17 00:00:00 2001 From: Michal Siedlaczek Date: Tue, 9 Jan 2024 21:19:11 -0500 Subject: [PATCH] Add metric evaluation to benchmark Docker Benchmark Docker image now can also evaluate metrics on Robust04. --- test/docker/benchmark/Dockerfile | 7 ++++ test/docker/benchmark/bench.sh | 14 +++++++ test/docker/benchmark/build.sh | 32 +++++++++++++++ test/docker/benchmark/evaluate.sh | 22 +++++++++++ test/docker/benchmark/expected-eval.txt | 3 ++ test/docker/benchmark/run.sh | 52 +++---------------------- test/docker/benchmark/setup.sh | 16 ++++++++ 7 files changed, 99 insertions(+), 47 deletions(-) create mode 100755 test/docker/benchmark/bench.sh create mode 100755 test/docker/benchmark/build.sh create mode 100755 test/docker/benchmark/evaluate.sh create mode 100644 test/docker/benchmark/expected-eval.txt mode change 100644 => 100755 test/docker/benchmark/run.sh create mode 100755 test/docker/benchmark/setup.sh diff --git a/test/docker/benchmark/Dockerfile b/test/docker/benchmark/Dockerfile index 027b01ca..1b8d6ed0 100644 --- a/test/docker/benchmark/Dockerfile +++ b/test/docker/benchmark/Dockerfile @@ -5,6 +5,8 @@ ARG USE_SANITIZERS=OFF ENV DEBIAN_FRONTEND=noninteractive ENV TZ=America/New_York +ENV COLLECTION_PATH=/opt/disk45 +ENV WORKDIR=/opt/workdir COPY . /pisa RUN ./pisa/test/docker/install-cmake.sh @@ -22,5 +24,10 @@ RUN cmake \ && cmake --build . --config Debug -- -j 4 COPY ./test/docker/benchmark/run.sh /pisa/build +COPY ./test/docker/benchmark/build.sh /pisa/build +COPY ./test/docker/benchmark/evaluate.sh /pisa/build +COPY ./test/docker/benchmark/bench.sh /pisa/build +COPY ./test/docker/benchmark/setup.sh /pisa/build +COPY ./test/docker/benchmark/expected-eval.txt /pisa/build CMD ["bash", "/pisa/test/docker/benchmark/run.sh"] diff --git a/test/docker/benchmark/bench.sh b/test/docker/benchmark/bench.sh new file mode 100755 index 00000000..afaa9dbc --- /dev/null +++ b/test/docker/benchmark/bench.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -e + +./bin/queries \ + -e block_simdbp \ + -a block_max_wand \ + -i "$WORKDIR/inv.block_simdbp" \ + -w "$WORKDIR/inv.bm25.bmw" \ + -F lowercase -F porter2 \ + --terms "$WORKDIR/fwd.termlex" \ + -k 1000 \ + --scorer bm25 \ + -q "$WORKDIR/topics.robust2004.title" diff --git a/test/docker/benchmark/build.sh b/test/docker/benchmark/build.sh new file mode 100755 index 00000000..707ee774 --- /dev/null +++ b/test/docker/benchmark/build.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -e + +gzip -dc $(find "$COLLECTION_PATH" -type f -name '*.*z' \ + \( -path '*/disk4/fr94/[0-9]*/*' -o -path '*/disk4/ft/ft*' \ + -o -path '*/disk5/fbis/fb*' -o -path '*/disk5/latimes/la*' \)) \ + | ./bin/parse_collection -f trectext -b 10000 -F lowercase -F porter2 --html -o "$WORKDIR/fwd" + +./bin/invert \ + -i "$WORKDIR/fwd" \ + -o "$WORKDIR/inv" \ + --batch-size 400000 + +./bin/reorder-docids \ + --bp \ + -c "$WORKDIR/inv" \ + -o "$WORKDIR/inv.bp" \ + --documents "$WORKDIR/fwd.doclex" \ + --reordered-documents "$WORKDIR/fwd.bp.doclex" + +./bin/create_wand_data \ + -c "$WORKDIR/inv.bp" \ + -b 64 \ + -o "$WORKDIR/inv.bm25.bmw" \ + -s bm25 + +./bin/compress_inverted_index \ + -e block_simdbp \ + -c "$WORKDIR/inv.bp" \ + -o "$WORKDIR/inv.block_simdbp" \ + --check diff --git a/test/docker/benchmark/evaluate.sh b/test/docker/benchmark/evaluate.sh new file mode 100755 index 00000000..bb39576e --- /dev/null +++ b/test/docker/benchmark/evaluate.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -e + +./bin/evaluate_queries \ + -e block_simdbp \ + -a block_max_wand \ + -i "$WORKDIR/inv.block_simdbp" \ + -w "$WORKDIR/inv.bm25.bmw" \ + -F lowercase -F porter2 \ + --terms "$WORKDIR/fwd.termlex" \ + --documents "$WORKDIR/fwd.bp.doclex" \ + -k 1000 \ + --scorer bm25 \ + -q "$WORKDIR/topics.robust2004.title" \ + > "$WORKDIR/results.txt" + +trec_eval -m map -m P.30 -m ndcg_cut.20 "$WORKDIR/qrels.robust2004.txt" "$WORKDIR/results.txt" > 'eval.txt' + +cat 'eval.txt' + +diff 'eval.txt' expected-eval.txt diff --git a/test/docker/benchmark/expected-eval.txt b/test/docker/benchmark/expected-eval.txt new file mode 100644 index 00000000..99dafcc6 --- /dev/null +++ b/test/docker/benchmark/expected-eval.txt @@ -0,0 +1,3 @@ +map all 0.2543 +P_30 all 0.3139 +ndcg_cut_20 all 0.4250 diff --git a/test/docker/benchmark/run.sh b/test/docker/benchmark/run.sh old mode 100644 new mode 100755 index 65f2ce29..98047243 --- a/test/docker/benchmark/run.sh +++ b/test/docker/benchmark/run.sh @@ -1,50 +1,8 @@ -#!/bin/bash +#!/usr/bin/env bash set -e -collection_path='/opt/disk45' -workdir='/opt/workdir' - -gzip -dc $(find "$collection_path" -type f -name '*.*z' \ - \( -path '*/disk4/fr94/[0-9]*/*' -o -path '*/disk4/ft/ft*' \ - -o -path '*/disk5/fbis/fb*' -o -path '*/disk5/latimes/la*' \)) \ - | ./bin/parse_collection -f trectext -b 10000 -F lowercase -F porter2 --html -o "$workdir/fwd" - -./bin/invert \ - -i "$workdir/fwd" \ - -o "$workdir/inv" \ - --batch-size 400000 - -./bin/reorder-docids \ - --bp \ - -c "$workdir/inv" \ - -o "$workdir/inv.bp" \ - --documents "$workdir/fwd.doclex" \ - --reordered-documents "$workdir/fwd.bp.doclex" - -./bin/create_wand_data \ - -c "$workdir/inv.bp" \ - -b 64 \ - -o "$workdir/inv.bm25.bmw" \ - -s bm25 - -./bin/compress_inverted_index \ - -e block_simdbp \ - -c "$workdir/inv.bp" \ - -o "$workdir/inv.block_simdbp" \ - --check - -wget http://trec.nist.gov/data/robust/04.testset.gz -gunzip 04.testset.gz -./bin/extract_topics -f trec -i 04.testset -o "$workdir/topics.robust2004" - -./bin/queries \ - -e block_simdbp \ - -a block_max_wand \ - -i "$workdir/inv.block_simdbp" \ - -w "$workdir/inv.bm25.bmw" \ - -F lowercase -F porter2 \ - --terms "$workdir/fwd.termlex" \ - -k 1000 \ - --scorer bm25 \ - -q "$workdir/topics.robust2004.title" +./setup.sh +./build.sh +./evaluate.sh +./bench.sh diff --git a/test/docker/benchmark/setup.sh b/test/docker/benchmark/setup.sh new file mode 100755 index 00000000..5d1e47e2 --- /dev/null +++ b/test/docker/benchmark/setup.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -e + +wget http://trec.nist.gov/data/robust/04.testset.gz +gunzip 04.testset.gz +./bin/extract_topics -f trec -i 04.testset -o "$WORKDIR/topics.robust2004" + +wget http://trec.nist.gov/data/robust/qrels.robust2004.txt +cp qrels.robust2004.txt "$WORKDIR/" + +wget https://github.com/usnistgov/trec_eval/archive/refs/tags/v9.0.8.tar.gz +tar -xzvf v9.0.8.tar.gz +cd trec_eval-9.0.8 +make install +cd ..