JTransBench/4_result.py at main · webbigdata-jp/JTransBench · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
import os
import json

def get_score_from_spbleu(file_path):
    try:
        with open(file_path, 'r') as f:
            data = json.load(f)
            return data.get('score', '')
    except Exception as e:
        return ''

def get_score_from_chrf(file_path):
    try:
        with open(file_path, 'r') as f:
            data = json.load(f)
            return data.get('score', '')
    except Exception as e:
        return ''

def get_score_from_comet(file_path):
    try:
        with open(file_path, 'r') as f:
            lines = f.readlines()
            if lines:
                last_line = lines[-1]
                if 'score:' in last_line:
                    return float(last_line.split('score:')[1].strip())
        return ''
    except Exception as e:
        return ''

# Initialize result storage
results = []

# Define the directory to scan
work_dir = 'work'

# Find all relevant files in the work directory
for file_name in os.listdir(work_dir):
    if file_name.endswith('.hyp.spBLEU'):
        base_name = file_name.replace('.hyp.spBLEU', '')
        spBLEU_score = get_score_from_spbleu(os.path.join(work_dir, file_name))
        chrf_score = get_score_from_chrf(os.path.join(work_dir, base_name + '.hyp.chrf'))
        comet_score = get_score_from_comet(os.path.join(work_dir, base_name + '.hyp.comet'))
        xlcomet_score = get_score_from_comet(os.path.join(work_dir, base_name + '.hyp.xlcomet'))
        xxlcomet_score = get_score_from_comet(os.path.join(work_dir, base_name + '.hyp.xxlcomet'))
        parts = base_name.split('_')
        bench_name = parts[0]

        import re
        pattern = r'_(\w{4})$'
        match = re.search(pattern, base_name)

        direction = ""
        if match:
            direction = match.group(1)

        results.append({
            'filename': bench_name,
            'direction': direction,
            'spBLEU': spBLEU_score,
            'chrF2++': chrf_score,
            'comet': comet_score,
            'xlcomet': xlcomet_score,
            'xxlcomet': xxlcomet_score
        })

# Print the header
print("filename direction spBLEU chrF2++ comet xlcomet xxlcomet")

# Print each result
for result in results:
    print(f"{result['filename']} {result['direction']} {result['spBLEU']} {result['chrF2++']} {result['comet']} {result['xlcomet']} {result['xxlcomet']}")