lmms-eval/test_parse.py at main · mm-evaluation/lmms-eval · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
from lmms_eval.filters.extraction import MultiChoiceRegexFilter


def parse_multi_choice_answer(answer):
    # Example responses and documents
    model_responses = [["The answer is (B)", "I believe it is (A)", "(C) seems correct"], ["Answer is: B!", "Answer: B", "Answer: B"]]  # Model response set 1  # Model response set 2

    documents = [{"choices": ["A. Apple", "B. Banana", "C. Cherry"]}, {"choices": ["A. Alpha", "B. Beta", "C. Gamma"]}]  # Multiple choice options for question 1  # Multiple choice options for question 2

    # Instantiate the filter
    multi_choice_filter = MultiChoiceRegexFilter(regex_pattern=r"\(([A-D])\)", group_select=0, ignore_case=False, ignore_punctuation=True)

    filtered_responses = multi_choice_filter.apply(model_responses, documents)

    # Print the filtered answers
    for i, filtered in enumerate(filtered_responses):
        print(f"Question {i+1} filtered responses: {filtered}")


parse_multi_choice_answer("a")