LogicOCR/get_score.py at main · MiliLab/LogicOCR · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
import sys
import json
from argparse import ArgumentParser
from tqdm import tqdm
import os


def _get_args():
    parser = ArgumentParser()
    parser.add_argument("--gen_json", type=str, required=True)
    parser.add_argument("--real_json", type=str, required=True)
    args = parser.parse_args()
    return args


if __name__ == "__main__":
    args = _get_args()

    total_score = 0
    total_ques = 0

    with open(args.gen_json, 'r') as f:
        gen_res = json.load(f)
    assert len(gen_res) <= 1100  # QvQ API may reject some questions

    gen_score_by_rtype = {
        "1 reasoning type": 0,
        "2 reasoning type": 0,
        "3 reasoning type": 0,
        ">3 reasoning type": 0,
    }
    gen_ques_by_rtype = {
        "1 reasoning type": 0,
        "2 reasoning type": 0,
        "3 reasoning type": 0,
        ">3 reasoning type": 0,
    }
    for item in gen_res:
        if "score" not in item.keys():
            continue
        total_ques += 1

        if len(item["type"].keys()) <= 1:
            gen_ques_by_rtype["1 reasoning type"] += 1
        elif len(item["type"].keys()) == 2:
            gen_ques_by_rtype["2 reasoning type"] += 1
        elif len(item["type"].keys()) == 3:
            gen_ques_by_rtype["3 reasoning type"] += 1
        elif len(item["type"].keys()) > 3:
             gen_ques_by_rtype[">3 reasoning type"] += 1

        if item["score"]:
            total_score += 1
            if len(item["type"].keys()) <= 1:
                gen_score_by_rtype["1 reasoning type"] += 1
            elif len(item["type"].keys()) == 2:
                gen_score_by_rtype["2 reasoning type"] += 1
            elif len(item["type"].keys()) == 3:
                gen_score_by_rtype["3 reasoning type"] += 1
            elif len(item["type"].keys()) > 3:
                gen_score_by_rtype[">3 reasoning type"] += 1
    assert sum(gen_ques_by_rtype.values()) <= len(gen_res)

    print(f"LogicOCR_Gen accuracy: {round(sum(gen_score_by_rtype.values())/sum(gen_ques_by_rtype.values())*100, 3)}")
    for key, value in gen_score_by_rtype.items():
        gen_score_by_rtype[key] = round(value / gen_ques_by_rtype[key] * 100, 3)
    print(gen_score_by_rtype)
    print('-'*30)


    with open(args.real_json, 'r') as f:
        real_res = json.load(f)
    real_score_by_rtype = {
        "numerical": 0,
        "temporal": 0,
        "decision": 0,
        "conditional": 0,
    }
    real_ques_by_rtype = {
        "numerical": 0,
        "temporal": 0,
        "decision": 0,
        "conditional": 0,
    }
    for item in real_res:
        if "evaluation" not in item.keys():
            continue
        total_ques += 1

        if item["type"] in ["data comparison analysis", "data statistical analysis", "mathematical reasoning"]:
            real_ques_by_rtype["numerical"] += 1
        elif item["type"] == "temporal reasoning":
            real_ques_by_rtype["temporal"] += 1
        elif item["type"] == "decision reasoning":
            real_ques_by_rtype["decision"] += 1
        elif item["type"] == "conditional reasoning":
            real_ques_by_rtype["conditional"] += 1
        else:
            raise ValueError

        if item["evaluation"]["correct"]:
            total_score += 1
            if item["type"] in ["data comparison analysis", "data statistical analysis", "mathematical reasoning"]:
                real_score_by_rtype["numerical"] += 1
            elif item["type"] == "temporal reasoning":
                real_score_by_rtype["temporal"] += 1
            elif item["type"] == "decision reasoning":
                real_score_by_rtype["decision"] += 1
            elif item["type"] == "conditional reasoning":
                real_score_by_rtype["conditional"] += 1
            else:
                raise ValueError
    # assert sum(real_ques_by_rtype.values()) == len(real_res)

    print(f"LogicOCR_Real accuracy: {round(sum(real_score_by_rtype.values())/sum(real_ques_by_rtype.values())*100, 3)}")
    for key, value in real_score_by_rtype.items():
        real_score_by_rtype[key] = round(value / real_ques_by_rtype[key] * 100, 3)
    print(real_score_by_rtype)
    print('-'*30)

    print(f"{total_ques} questions.\naccuracy: {round(total_score/total_ques*100, 3)}")