index

#!/usr/bin/env python3

import argparse
import json
import os
import subprocess
import pymonetdb

subprocess.run('monetdbd create /work/mydbfarm'.split())                        
subprocess.run('monetdbd start /work/mydbfarm'.split())                         

parser = argparse.ArgumentParser()
parser.add_argument("--json", type=json.loads, required=True, help="the args")

# Parse the args
args, unknown = parser.parse_known_args()

# Iterate over collections
for collection in args.json["collections"]:
    
    # Select processor depending on choice of collection
    # TODO: a more generic solution would be preferable
    if (collection["name"] == "robust04"):
        cname = "TrecCollection"
        gname = "JsoupGenerator"
        dbname = "robust04"
    elif (collection["name"] == "core18"):
        cname = "WashingtonPostCollection"
        gname = "WapoGenerator"
        dbname = "core18"
    else:
        print( "Unknown collection name: {}".format(collection["name"]))
        continue

    print("CREATE DATABASE {}".format(dbname))

    # TODO: do we really want collection specific databases?
    subprocess.run('monetdb create {}'.format(dbname).split())
    subprocess.run('monetdb release {}'.format(dbname).split())

    # Updated manually for WAPO
    # Command need to be adapted based on a collection chosen to index
    subprocess.run("""
        java -cp anserini.jar io.anserini.index.IndexCollection
            -collection {2} -generator {3}
            -threads 1 -index {0}
            -input {1}
            -storePositions -storeDocvectors -storeRawDocs
    """.format(collection["name"], collection["path"], cname, gname).split())

    subprocess.run("""
        /work/olddog/target/appassembler/bin/nl.ru.convert.Convert
        -index /work/{0}
        -docs docs-{0}
        -dict dict-{0}
        -terms terms-{0}
    """.format(dbname).split())

    subprocess.run('mv dict-{0} /tmp/dict-{0}'.format(dbname).split())
    subprocess.run('mv terms-{0} /tmp/terms-{0}'.format(dbname).split())
    subprocess.run('mv docs-{0} /tmp/docs-{0}'.format(dbname).split())

    subprocess.run('rm -rf /work/{0}'.split())

    print("CONNECT TO {}".format(dbname))
    connection = pymonetdb.connect(username='monetdb',
                                   password='monetdb',
                                   hostname='localhost',
                                   database=dbname)
    cursor = connection.cursor()

    print("CREATE TABLES")
    cursor.execute("CREATE TABLE docs (collection_id STRING, id INT, len INT)")
    cursor.execute("CREATE TABLE dict (termid INT, term STRING, df INT)")
    cursor.execute("CREATE TABLE terms (termid INT, docid INT, count INT)")

    print("LOAD DATA")
    cursor.execute("COPY INTO docs FROM '/tmp/docs-{0}' using delimiters '|', '\n', ''".format(dbname))
    cursor.execute("COPY INTO dict FROM '/tmp/dict-{0}' using delimiters '|', '\n', ''".format(dbname))
    cursor.execute("COPY INTO terms FROM '/tmp/terms-{0}' using delimiters '|', '\n', ''".format(dbname))

    print("DATA LOADED")
    connection.commit()
    connection.close()