Skip to content

Commit

Permalink
Merge azure-script into master (#113)
Browse files Browse the repository at this point in the history 
* Update Azure script

* and make sure it's uncommented...

* Enable timings for terrier and pisa

* Capture logs and scp back to local.

* Update robust04 runbook

* Remove P@5 from OldDog config

* robust04, core17, and core18 runbooks.

* Update terrier version in runbook

* Update core17 runbook

* Update terrier runbook

* Whoops...

* Update olddog version

* Add anserini-bm25prf to runbook

* Update anserini-bm25prf

* Update anserini-bm25prf

* Update robust05 path for irc-centre2019

* Add ielab and galago to runbook

* Write an additional file with the trec_eval results

* Update runbook for official releases

* Add NDCG@20 by default and fix anserini-bm25prf output directory

* Don't create .trec_eval files for non-trec files

* Merge master into azure-script (#111)

* Add files via upload (#104)

* Evaluate on all topics instead of those with documents retrieved (#108)

* Added TREC7 and TREC8 qrels (primarily to avoid confusion with the WT qrels in `qrels/qrels.401-450.txt`

* Missing TREC7 topics files

* Change default to setting also used in the "official" evaluation

* Add -c flag for trec_eval

* Revert "Change default to setting also used in the "official" evaluation"

This reverts commit 85b13df.

* TREC Terabyte 04-06 topic/qrel files merged (#109)
  • Loading branch information
Ryan Clancy authored Jun 28, 2019
1 parent e51fe7a commit 7f6efa3
Show file tree
Hide file tree
Showing 7 changed files with 405 additions and 124 deletions.
32 changes: 21 additions & 11 deletions azure/azure.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ ssh jig@${IP_ADDRESS} << EOF
git clone https://github.com/osirrc2019/jig.git; cd jig
mkdir logs
virtualenv -p /usr/bin/python3 venv
source venv/bin/activate
Expand All @@ -120,30 +121,39 @@ TOPIC_PATH=$(cat ${RUN_FILE} | jq -r ".topic.path")
QRELS_PATH=$(cat ${RUN_FILE} | jq -r ".qrels.path")

# The number of images
NUM_IMAGES=$(cat ${RUN_FILE} | jq -r '.images | length')
NUM_IMAGES=$(cat ${RUN_FILE} | jq -r ".images | length")

for i in $(seq 0 $((${NUM_IMAGES} - 1))); do

NAME=$(cat ${RUN_FILE} | jq -r ".images[$i].name")

# Get the search command and substitute in variables
PREPARE=$(cat ${RUN_FILE} | jq -r ".images[$i].command.prepare")
PREPARE=${PREPARE/"[COLLECTION_NAME]"/${COLLECTION_NAME}}
PREPARE=${PREPARE/"[COLLECTION_PATH]"/${COLLECTION_PATH}}
PREPARE=${PREPARE/"[COLLECTION_FORMAT]"/${COLLECTION_FORMAT}}

# Get the search command and substitute in variables
SEARCH=$(cat ${RUN_FILE} | jq -r ".images[$i].command.search")
SEARCH=${SEARCH/"[COLLECTION_NAME]"/${COLLECTION_NAME}}
SEARCH=${SEARCH/"[TOPIC_PATH]"/${TOPIC_PATH}}
SEARCH=${SEARCH/"[QRELS_PATH]"/${QRELS_PATH}}
SEARCH=${SEARCH/"[OUTPUT]"/${OUTPUT}}

echo ${PREPARE}
ssh jig@${IP_ADDRESS} "cd jig && source venv/bin/activate && eval ${PREPARE}"
ssh jig@${IP_ADDRESS} "cd jig && source venv/bin/activate && ${PREPARE} > logs/${NAME}-${COLLECTION_NAME}-prepare.log"

NUM_SEARCHES=$(cat ${RUN_FILE} | jq -r ".images[$i].command.search | length")

for j in $(seq 0 $((${NUM_SEARCHES} - 1))); do

# Get the search command and substitute in variables
SEARCH=$(cat ${RUN_FILE} | jq -r ".images[$i].command.search[$j]")
SEARCH=${SEARCH/"[COLLECTION_NAME]"/${COLLECTION_NAME}}
SEARCH=${SEARCH/"[TOPIC_PATH]"/${TOPIC_PATH}}
SEARCH=${SEARCH/"[QRELS_PATH]"/${QRELS_PATH}}
SEARCH=${SEARCH/"[OUTPUT]"/${OUTPUT}}

echo ${SEARCH}
ssh jig@${IP_ADDRESS} "cd jig && source venv/bin/activate && ${SEARCH} > logs/${NAME}-${COLLECTION_NAME}-search-${j}.log"

echo ${SEARCH}
ssh jig@${IP_ADDRESS} "cd jig && source venv/bin/activate && eval ${SEARCH}"
done

scp -r jig@${IP_ADDRESS}:${OUTPUT} .
scp -r jig@${IP_ADDRESS}:/home/jig/jig/logs .

done

Expand Down
79 changes: 0 additions & 79 deletions azure/osirrc2019.json
View file

This file was deleted.

93 changes: 93 additions & 0 deletions azure/osirrc2019/core17.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
{
"collection": {
"name": "core17",
"path": "/home/jig/collections/NYTcorpus",
"format": "trectext"
},
"topic": {
"path": "topics/topics.core17.txt"
},
"qrels": {
"path": "qrels/qrels.core17.txt"
},
"output": "/tmp/output",
"images": [
{
"name": "anserini",
"command": {
"prepare": "python run.py prepare --repo osirrc2019/anserini --tag v0.1.1 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]",
"search": [
"python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"bm25.run\" search_args=\"-bm25\"",
"python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"bm25.rm3.run\" search_args=\"-bm25 -rm3\"",
"python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"bm25.ax.run\" search_args=\"-bm25 -axiom -rerankCutoff 20 -axiom.deterministic\"",
"python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"ql.run\" search_args=\"-ql\"",
"python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"ql.rm3.run\" search_args=\"-ql -rm3\"",
"python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"ql.ax.run\" search_args=\"-ql -axiom -rerankCutoff 20 -axiom.deterministic\""
]
}
},
{
"name": "atire",
"command": {
"prepare": "python run.py prepare --repo osirrc2019/atire --tag v0.1.1 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]",
"search": [
"python run.py search --repo osirrc2019/atire --tag v0.1.1 --output [OUTPUT]/atire --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --top_k 100 --timings"
]
}
},
{
"name": "ielab",
"command": {
"prepare": "python run.py prepare --repo osirrc2019/ielab --tag v0.0.1 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]",
"search": [
"python run.py search --repo osirrc2019/ielab --tag v0.0.1 --output [OUTPUT]/ielab --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME]"
]
}
},
{
"name": "irc-centre2019-wcrobust04",
"command": {
"prepare": "python run.py prepare --repo osirrc2019/irc-centre2019 --tag v0.1.3 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT] robust04=/home/jig/collections/disk45=trectext --opts run=\"wcrobust04\"",
"search": [
"python run.py search --repo osirrc2019/irc-centre2019 --tag v0.1.3 --output [OUTPUT]/irc-centre2019 --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME]"
]
}
},
{
"name": "irc-centre2019-wcrobust0405",
"command": {
"prepare": "python run.py prepare --repo osirrc2019/irc-centre2019 --tag v0.1.3 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT] robust04=/home/jig/collections/disk45=trectext robust05=/home/jig/collections/AQUAINT=trectext --opts run=\"wcrobust0405\"",
"search": [
"python run.py search --repo osirrc2019/irc-centre2019 --tag v0.1.3 --output [OUTPUT]/irc-centre2019 --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME]"
]
}
},
{
"name": "jass",
"command": {
"prepare": "python run.py prepare --repo osirrc2019/jass --tag v0.1.1 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]",
"search": [
"python run.py search --repo osirrc2019/jass --tag v0.1.1 --output [OUTPUT]/jass --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --top_k 100 --timings"
]
}
},
{
"name": "jassv2",
"command": {
"prepare": "python run.py prepare --repo osirrc2019/jassv2 --tag v0.1.1 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]",
"search": [
"python run.py search --repo osirrc2019/jassv2 --tag v0.1.1 --output [OUTPUT]/jassv2 --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --top_k 100 --timings"
]
}
},
{
"name": "pisa",
"command": {
"prepare": "python run.py prepare --repo osirrc2019/pisa --tag v0.1.3 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]",
"search": [
"python run.py search --repo osirrc2019/pisa --tag v0.1.3 --output [OUTPUT]/pisa --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME]"
]
}
}
]
}
Loading

0 comments on commit 7f6efa3

Please sign in to comment.