forked from Yushi-Hu/IC-DST
-
Notifications
You must be signed in to change notification settings - Fork 0
/
evaluate_metrics.py
executable file
·70 lines (58 loc) · 1.88 KB
/
evaluate_metrics.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
def compute_acc(gold, pred, n_slot=30):
if type(gold) == dict:
gold = [f"{k}-{v}" for k, v in gold.items()]
if type(pred) == dict:
pred = [f"{k}-{v}" for k, v in pred.items()]
miss_gold = 0
miss_slot = []
for g in gold:
if g not in pred:
miss_gold += 1
miss_slot.append(g.rsplit("-", 1)[0])
wrong_pred = 0
for p in pred:
if p not in gold and p.rsplit("-", 1)[0] not in miss_slot:
wrong_pred += 1
ACC_TOTAL = n_slot
ACC = n_slot - miss_gold - wrong_pred
ACC = ACC / float(ACC_TOTAL)
return ACC
def compute_prf(gold, pred):
if type(gold) == dict:
gold = [f"{k}-{v}" for k, v in gold.items()]
if type(pred) == dict:
pred = [f"{k}-{v}" for k, v in pred.items()]
TP, FP, FN = 0, 0, 0
if len(gold) != 0:
count = 1
for g in gold:
if g in pred:
TP += 1
else:
FN += 1
for p in pred:
if p not in gold:
FP += 1
precision = TP / float(TP+FP) if (TP+FP) != 0 else 0
recall = TP / float(TP+FN) if (TP+FN) != 0 else 0
F1 = 2 * precision * recall / \
float(precision + recall) if (precision+recall) != 0 else 0
else:
if len(pred) == 0:
precision, recall, F1, count = 1, 1, 1, 1
else:
precision, recall, F1, count = 0, 0, 0, 1
return F1, recall, precision, count
def evaluate(preds: dict, golds: dict):
gold_slots = list(golds.keys())
for k in gold_slots:
if '|' in golds[k]:
gold_values = golds[k].split('|')
if k in preds and preds[k] in gold_values:
golds[k] = preds[k]
jga, acc, f1 = 0, 0, 0
if preds == golds:
jga = 1
acc = compute_acc(golds, preds)
f1 = compute_prf(golds, preds)[0]
return jga, acc, f1