Change default value for failed retrieval settings: predict the minim…

…um minus some small epsilon

code/qpptk/qpptk/global_manager.py

@@ -11,7 +11,7 @@
 
 logger = Config.logger
 TREC_RES_COLUMNS = ['qid', 'iteration', 'docNo', 'rank', 'docScore', 'method']
-QPP_SCORE_FOR_FAILED_RETRIEVAL = -1000000000
+QPP_SCORE_FOR_FAILED_RETRIEVAL = None
 
 def init_proc(*args):
     kwargs = dict(args)

code/qpptk/qpptk/qpptk_main.py

@@ -385,7 +385,14 @@ def init_writeable_index():
                             continue
                         qid_to_preds[i['qid']][k] = v
 
-            pd.DataFrame([v for _, v in qid_to_preds.items()]).to_json(results_dir + '/queries.jsonl', lines=True, orient='records')
+            ret = pd.DataFrame([v for _, v in qid_to_preds.items()])
+            min_score = 0
+            for _, i in ret.iterrows():
+                for j in i:
+                    if j and type(j) == float and j < min_score:
+                        min_score = j  
+            ret = ret.fillna(min_score - 0.01)
+            ret.to_json(results_dir + '/queries.jsonl', lines=True, orient='records')
 
     if args.cleanOutput:
         for d in os.listdir(args.output):

code/qpptk/tests/CustomRunFileForPredictionTest.test_with_run_file_where_one_query_is_highly_effective_approvaltest.approved.jsonl

@@ -1,3 +1,3 @@
 {"qid":"1","wig+5":12.4250979385,"nqc+5":0.0004338056,"smv+5":-0.0004338056,"clarity+5+100":2.6993897409,"wig+10":12.4250979385,"nqc+10":0.0004338056,"smv+10":-0.0004338056,"clarity+10+100":2.6993897409,"wig+20":12.4250979385,"nqc+20":0.0004338056,"smv+20":-0.0004338056,"clarity+20+100":2.6993897409,"wig+50":12.4250979385,"nqc+50":0.0004338056,"smv+50":-0.0004338056,"clarity+50+100":2.6993897409,"wig+100":12.4250979385,"nqc+100":0.0004338056,"smv+100":-0.0004338056,"clarity+100+100":2.6993897409,"wig+1000":12.4250979385,"nqc+1000":0.0004338056,"smv+1000":-0.0004338056,"clarity+1000+100":2.6993897409}
-{"qid":"2","wig+5":4.2054528045,"nqc+5":0.2547211364,"smv+5":null,"clarity+5+100":2.240679245,"wig+10":4.2054528045,"nqc+10":0.2547211364,"smv+10":null,"clarity+10+100":2.240679245,"wig+20":4.2054528045,"nqc+20":0.2547211364,"smv+20":null,"clarity+20+100":2.240679245,"wig+50":4.2054528045,"nqc+50":0.2547211364,"smv+50":null,"clarity+50+100":2.240679245,"wig+100":4.2054528045,"nqc+100":0.2547211364,"smv+100":null,"clarity+100+100":2.240679245,"wig+1000":4.2054528045,"nqc+1000":0.2547211364,"smv+1000":null,"clarity+1000+100":2.240679245}
-{"qid":"3","wig+5":-20.8716444259,"nqc+5":6.6373379689,"smv+5":null,"clarity+5+100":3.2047245641,"wig+10":-20.8716444259,"nqc+10":6.6373379689,"smv+10":null,"clarity+10+100":3.2047245641,"wig+20":-20.8716444259,"nqc+20":6.6373379689,"smv+20":null,"clarity+20+100":3.2047245641,"wig+50":-20.8716444259,"nqc+50":6.6373379689,"smv+50":null,"clarity+50+100":3.2047245641,"wig+100":-20.8716444259,"nqc+100":6.6373379689,"smv+100":null,"clarity+100+100":3.2047245641,"wig+1000":-20.8716444259,"nqc+1000":6.6373379689,"smv+1000":null,"clarity+1000+100":3.2047245641}
+{"qid":"2","wig+5":4.2054528045,"nqc+5":0.2547211364,"smv+5":-20.8816444259,"clarity+5+100":2.240679245,"wig+10":4.2054528045,"nqc+10":0.2547211364,"smv+10":-20.8816444259,"clarity+10+100":2.240679245,"wig+20":4.2054528045,"nqc+20":0.2547211364,"smv+20":-20.8816444259,"clarity+20+100":2.240679245,"wig+50":4.2054528045,"nqc+50":0.2547211364,"smv+50":-20.8816444259,"clarity+50+100":2.240679245,"wig+100":4.2054528045,"nqc+100":0.2547211364,"smv+100":-20.8816444259,"clarity+100+100":2.240679245,"wig+1000":4.2054528045,"nqc+1000":0.2547211364,"smv+1000":-20.8816444259,"clarity+1000+100":2.240679245}
+{"qid":"3","wig+5":-20.8716444259,"nqc+5":6.6373379689,"smv+5":-20.8816444259,"clarity+5+100":3.2047245641,"wig+10":-20.8716444259,"nqc+10":6.6373379689,"smv+10":-20.8816444259,"clarity+10+100":3.2047245641,"wig+20":-20.8716444259,"nqc+20":6.6373379689,"smv+20":-20.8816444259,"clarity+20+100":3.2047245641,"wig+50":-20.8716444259,"nqc+50":6.6373379689,"smv+50":-20.8816444259,"clarity+50+100":3.2047245641,"wig+100":-20.8716444259,"nqc+100":6.6373379689,"smv+100":-20.8816444259,"clarity+100+100":3.2047245641,"wig+1000":-20.8716444259,"nqc+1000":6.6373379689,"smv+1000":-20.8816444259,"clarity+1000+100":3.2047245641}

code/qpptk/tests/TirexIntegrationTest.test_on_cranfield_dataset_with_approvaltests.approved.jsonl

@@ -85,7 +85,7 @@
 {"qid":"132","max-idf":6.551080335,"avg-idf":2.691678484,"scq":193.0767102877,"max-scq":18.7270775768,"avg-scq":14.8520546375,"var":13.7269165759,"max-var":2.2614342106,"avg-var":0.7626064764,"wig+5":1.0434409869,"nqc+5":0.0078827703,"smv+5":0.0071943118,"clarity+5+100":1.6492471341,"wig+10":0.9079225148,"nqc+10":0.0080786537,"smv+10":0.0060651463,"clarity+10+100":1.4755100642,"wig+20":0.7632863246,"nqc+20":0.0083243744,"smv+20":0.0063423692,"clarity+20+100":1.4330077892,"wig+50":0.6004959952,"nqc+50":0.0077708287,"smv+50":0.0054204079,"clarity+50+100":1.3746220699,"wig+100":0.4698041719,"nqc+100":0.0078843319,"smv+100":0.0056502375,"clarity+100+100":1.3032305421,"wig+1000":-0.0568460482,"nqc+1000":0.0095683935,"smv+1000":0.0070174566,"clarity+1000+100":1.0543963552}
 {"qid":"133","max-idf":5.4524680464,"avg-idf":2.7425011986,"scq":265.4883412561,"max-scq":20.233882923,"avg-scq":15.6169612504,"var":19.6206869712,"max-var":2.2213591696,"avg-var":0.9810343486,"wig+5":1.7717577573,"nqc+5":0.0221545637,"smv+5":0.016874597,"clarity+5+100":2.9192954171,"wig+10":1.4084940585,"nqc+10":0.02038622,"smv+10":0.014464003,"clarity+10+100":2.9147022491,"wig+20":1.0332928775,"nqc+20":0.0197184421,"smv+20":0.0137077969,"clarity+20+100":2.912424083,"wig+50":0.6797708355,"nqc+50":0.0161765486,"smv+50":0.0105630945,"clarity+50+100":2.9100268118,"wig+100":0.4854222831,"nqc+100":0.013400762,"smv+100":0.0078025164,"clarity+100+100":2.9082115171,"wig+1000":-0.0724586862,"nqc+1000":0.0093121892,"smv+1000":0.0061937547,"clarity+1000+100":2.9011922653}
 {"qid":"135","max-idf":4.4110141715,"avg-idf":2.24454812,"scq":176.9863426547,"max-scq":19.6118973976,"avg-scq":14.7488618879,"var":15.6705219321,"max-var":2.4582379634,"avg-var":0.8247643122,"wig+5":1.1787129677,"nqc+5":0.0080614446,"smv+5":0.0074091544,"clarity+5+100":1.6728772399,"wig+10":1.0326932327,"nqc+10":0.0088913266,"smv+10":0.006979531,"clarity+10+100":1.5807778297,"wig+20":0.8591856781,"nqc+20":0.0100712995,"smv+20":0.0081170844,"clarity+20+100":1.500093476,"wig+50":0.6598578763,"nqc+50":0.0098411328,"smv+50":0.0071854623,"clarity+50+100":1.3813486707,"wig+100":0.5079886045,"nqc+100":0.0097988844,"smv+100":0.00712806,"clarity+100+100":1.3240011156,"wig+1000":-0.0240279989,"nqc+1000":0.010334467,"smv+1000":0.0074660934,"clarity+1000+100":1.114610922}
-{"qid":"136","max-idf":5.1647859739,"avg-idf":3.4804427253,"scq":202.4832722275,"max-scq":22.4583740976,"avg-scq":18.4075702025,"var":22.8215133262,"max-var":3.012438931,"avg-var":1.0867387298,"wig+5":3.4809310091,"nqc+5":0.0324216307,"smv+5":0.0249633147,"clarity+5+100":2.910541911,"wig+10":2.7660359728,"nqc+10":0.0399263304,"smv+10":0.0291848456,"clarity+10+100":2.9094887441,"wig+20":1.8762173209,"nqc+20":0.0466698352,"smv+20":0.038883899,"clarity+20+100":2.9094357219,"wig+50":1.0744183933,"nqc+50":0.0400505412,"smv+50":0.0283023805,"clarity+50+100":2.9094253529,"wig+100":0.6631020744,"nqc+100":0.033046425,"smv+100":0.019575637,"clarity+100+100":2.90941843,"wig+1000":0.0194392156,"nqc+1000":0.0205198957,"smv+1000":0.0109341308,"clarity+1000+100":2.9093993393}
+{"qid":"136","max-idf":5.1647859739,"avg-idf":3.4804427253,"scq":202.4832722275,"max-scq":22.4583740976,"avg-scq":18.4075702025,"var":22.8215133262,"max-var":3.012438931,"avg-var":1.0867387298,"wig+5":3.4809310092,"nqc+5":0.0324216307,"smv+5":0.0249633147,"clarity+5+100":2.910541911,"wig+10":2.7660359728,"nqc+10":0.0399263304,"smv+10":0.0291848456,"clarity+10+100":2.9094887441,"wig+20":1.8762173209,"nqc+20":0.0466698352,"smv+20":0.038883899,"clarity+20+100":2.9094357219,"wig+50":1.0744183933,"nqc+50":0.0400505412,"smv+50":0.0283023805,"clarity+50+100":2.9094253529,"wig+100":0.6631020744,"nqc+100":0.033046425,"smv+100":0.019575637,"clarity+100+100":2.90941843,"wig+1000":0.0194392156,"nqc+1000":0.0205198957,"smv+1000":0.0109341308,"clarity+1000+100":2.9093993393}
 {"qid":"137","max-idf":4.8463322428,"avg-idf":2.9371810957,"scq":179.2383564427,"max-scq":19.7468753235,"avg-scq":16.2943960402,"var":13.7750009721,"max-var":1.9662439088,"avg-var":0.7250000512,"wig+5":1.5279222296,"nqc+5":0.0055256974,"smv+5":0.0043990253,"clarity+5+100":1.8186975003,"wig+10":1.3528984218,"nqc+10":0.0082927819,"smv+10":0.0077767776,"clarity+10+100":1.7175286208,"wig+20":1.1575872573,"nqc+20":0.0100325549,"smv+20":0.0082987887,"clarity+20+100":1.6358159943,"wig+50":0.87599592,"nqc+50":0.0117462377,"smv+50":0.0092369477,"clarity+50+100":1.5634259831,"wig+100":0.6379362298,"nqc+100":0.0130408859,"smv+100":0.0102114211,"clarity+100+100":1.5216435809,"wig+1000":-0.0819867572,"nqc+1000":0.0127539016,"smv+1000":0.0088195533,"clarity+1000+100":1.3494413456}
 {"qid":"138","max-idf":4.8463322428,"avg-idf":2.3361739972,"scq":117.6296548062,"max-scq":17.1548371733,"avg-scq":14.7037068508,"var":8.4440783934,"max-var":1.6853923964,"avg-var":1.0555097992,"wig+5":1.1458990466,"nqc+5":0.0123666654,"smv+5":0.0097413639,"clarity+5+100":1.8204986385,"wig+10":1.0001507979,"nqc+10":0.0122948635,"smv+10":0.0082181264,"clarity+10+100":1.6276332341,"wig+20":0.8595986173,"nqc+20":0.0118476115,"smv+20":0.0088026255,"clarity+20+100":1.4965551636,"wig+50":0.6708003006,"nqc+50":0.0118476564,"smv+50":0.0085251654,"clarity+50+100":1.333959535,"wig+100":0.5041280941,"nqc+100":0.0127351127,"smv+100":0.0097485804,"clarity+100+100":1.2602009438,"wig+1000":0.0027946764,"nqc+1000":0.0140110932,"smv+1000":0.0100834142,"clarity+1000+100":1.026222274}
 {"qid":"139","max-idf":7.2442275156,"avg-idf":3.4215889592,"scq":86.6696764825,"max-scq":19.6859748596,"avg-scq":14.4449460804,"var":6.33695684,"max-var":2.1362379154,"avg-var":0.792119605,"wig+5":1.4529582115,"nqc+5":0.0135812797,"smv+5":0.012640896,"clarity+5+100":1.6608289436,"wig+10":1.2093611106,"nqc+10":0.0165042048,"smv+10":0.0135998154,"clarity+10+100":1.5976817396,"wig+20":0.9985991135,"nqc+20":0.0165072087,"smv+20":0.0124879026,"clarity+20+100":1.5281295032,"wig+50":0.7658729828,"nqc+50":0.014973304,"smv+50":0.0103246381,"clarity+50+100":1.4209426672,"wig+100":0.5415214861,"nqc+100":0.0165542164,"smv+100":0.0123692492,"clarity+100+100":1.3273313279,"wig+1000":-0.0276073469,"nqc+1000":0.0155604028,"smv+1000":0.0102438351,"clarity+1000+100":1.0841820828}