diff --git a/pattern_finder.py b/pattern_finder.py
new file mode 100644
index 00000000..5d4ef31d
--- /dev/null
+++ b/pattern_finder.py
@@ -0,0 +1,101 @@
+import argparse
+import csv
+import json
+import re
+from collections import namedtuple
+from pathlib import Path
+from typing import Tuple
+
+
+def parse_arguments():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-m", "--model_name", type=str, default=None)
+ parser.add_argument('-t', '--task_name', type=str, default=None)
+ return parser.parse_args()
+
+
+def save_results(folder_name: str, task_name: str, results: dict) -> None:
+
+ folder = Path(__file__).parent / folder_name
+ if not folder.exists():
+ folder.mkdir()
+
+ fieldnames = ['answer', 'certainty', 'score', 'count', 'prompt', 'prompt_number']
+ with open(folder / f'{task_name}.csv', 'w') as f:
+ writer = csv.DictWriter(f, fieldnames=fieldnames)
+ writer.writeheader()
+ for k, v in sorted(results.items(), key=lambda x: -x[1]):
+ writer.writerow({'answer': k.answer,
+ 'certainty': k.certainty,
+ 'score': k.score,
+ 'prompt': k.prompt,
+ 'prompt_number': k.prompt_number,
+ 'count': v})
+
+
+def load_json(path: Path) -> dict:
+ with open(path) as f:
+ data = json.load(f)
+ return data
+
+
+def replace_with_metadata(metadata: dict,
+ prompt: str,
+ prediction_string: str) -> Tuple[str, str]:
+ for k, v in metadata.items():
+ if k not in {'answer', 'distractor', 'query'}:
+ continue
+ if isinstance(v, str) or isinstance(v, int):
+ prediction_string = re.sub(fr'\b({v})\b'.lower(), fr'{k}', prediction_string)
+ prompt = re.sub(fr'\b({v})\b'.lower(), fr'{k}', prompt)
+ if isinstance(v, list):
+ for i in v:
+ prediction_string = re.sub(fr'\b({i})\b'.lower(), fr'{k}', prediction_string)
+ prompt = re.sub(fr'\b({i})\b'.lower(), fr'{k}', prompt)
+ prompt = prompt.replace('a:', '').strip()
+ return prompt, prediction_string
+
+
+def process_predictions(data: dict, predictions: dict) -> Tuple[dict, dict]:
+
+ Answer = namedtuple('Answer',
+ ['answer', 'certainty', 'score', 'prompt', 'prompt_number'])
+ positive, negative = {}, {}
+
+ for prompt_number, (example, prediction) in enumerate(zip(data['examples'].values(),
+ predictions.values())):
+ prediction_string = prediction['prediction'].lower()
+ prompt = prediction['input'].lower()
+ prediction_certainty = prediction['certainty']
+ prediction_score = prediction['score']
+ metadata = example['metadata']
+ prompt, prediction_string = replace_with_metadata(metadata, prompt, prediction_string)
+ ans = prediction_string.split('\n', 1)[1]
+ ans = ans.replace('a: ', '').strip()
+ prompt_number = prompt_number // data['settings']['num_examples_per_template'] + 1
+ answer = Answer(ans, prediction_certainty, prediction_score, prompt, prompt_number)
+ if answer.score == 0:
+ negative[answer] = negative.get(answer, 0) + 1
+ else:
+ positive[answer] = positive.get(answer, 0) + 1
+
+ return positive, negative
+
+
+def main():
+ args = parse_arguments()
+ model, task = args.model_name, args.task_name
+
+ predictions_path = Path(__file__).parent / 'predictions' / task / f'{model}.json'
+ data_path = Path(__file__).parent / 'data' / f'{task}.json'
+
+ predictions_json = load_json(predictions_path)
+ data_json = load_json(data_path)
+
+ positive, negative = process_predictions(data_json, predictions_json)
+ save_results('positive_patterns', task, positive)
+ save_results('negative_patterns', task, negative)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/statistics_viz.ipynb b/statistics_viz.ipynb
new file mode 100644
index 00000000..3340f844
--- /dev/null
+++ b/statistics_viz.ipynb
@@ -0,0 +1,241 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "initial_id",
+ "metadata": {
+ "collapsed": true,
+ "ExecuteTime": {
+ "end_time": "2024-01-06T20:24:19.571506757Z",
+ "start_time": "2024-01-06T20:24:19.490337322Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "import pandas as pd"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "outputs": [
+ {
+ "data": {
+ "text/plain": " task_name template_1 template_2 template_3\n0 smaller_number 55.3 58.6 48.4\n1 less_letters 46.3 52.5 24.1\n2 homophones 40.0 36.2 59.5\n3 first_alphabetically 57.8 56.3 56.5\n4 rhyming_word 0.6 47.8 47.6\n5 bigger_number 96.2 92.0 28.0",
+ "text/html": "\n\n
\n \n \n | \n task_name | \n template_1 | \n template_2 | \n template_3 | \n
\n \n \n \n | 0 | \n smaller_number | \n 55.3 | \n 58.6 | \n 48.4 | \n
\n \n | 1 | \n less_letters | \n 46.3 | \n 52.5 | \n 24.1 | \n
\n \n | 2 | \n homophones | \n 40.0 | \n 36.2 | \n 59.5 | \n
\n \n | 3 | \n first_alphabetically | \n 57.8 | \n 56.3 | \n 56.5 | \n
\n \n | 4 | \n rhyming_word | \n 0.6 | \n 47.8 | \n 47.6 | \n
\n \n | 5 | \n bigger_number | \n 96.2 | \n 92.0 | \n 28.0 | \n
\n \n
\n
\n\n
\n \n \n | \n task_name | \n template_1 | \n template_2 | \n template_3 | \n prompts_per_templ | \n
\n \n \n \n | 0 | \n smaller_number | \n 0.553 | \n 0.586 | \n 0.484 | \n 1000.0 | \n
\n \n | 1 | \n less_letters | \n 0.463 | \n 0.525 | \n 0.241 | \n 1000.0 | \n
\n \n | 2 | \n homophones | \n 0.400 | \n 0.362 | \n 0.595 | \n 800.0 | \n
\n \n | 3 | \n first_alphabetically | \n 0.578 | \n 0.563 | \n 0.565 | \n 1000.0 | \n
\n \n | 4 | \n rhyming_word | \n 0.006 | \n 0.478 | \n 0.476 | \n 1000.0 | \n
\n \n | 5 | \n bigger_number | \n 0.962 | \n 0.920 | \n 0.280 | \n 1000.0 | \n
\n \n
\n
\n\n
\n \n \n | \n task_name | \n template_1 | \n template_2 | \n template_3 | \n prompts_per_templ | \n template_1_std | \n template_2_std | \n template_3_std | \n
\n \n \n \n | 0 | \n smaller_number | \n 0.553 | \n 0.586 | \n 0.484 | \n 1000.0 | \n 0.015722 | \n 0.015576 | \n 0.015803 | \n
\n \n | 1 | \n less_letters | \n 0.463 | \n 0.525 | \n 0.241 | \n 1000.0 | \n 0.015768 | \n 0.015792 | \n 0.013525 | \n
\n \n | 2 | \n homophones | \n 0.400 | \n 0.362 | \n 0.595 | \n 800.0 | \n 0.017321 | \n 0.016991 | \n 0.017356 | \n
\n \n | 3 | \n first_alphabetically | \n 0.578 | \n 0.563 | \n 0.565 | \n 1000.0 | \n 0.015618 | \n 0.015685 | \n 0.015677 | \n
\n \n | 4 | \n rhyming_word | \n 0.006 | \n 0.478 | \n 0.476 | \n 1000.0 | \n 0.002442 | \n 0.015796 | \n 0.015793 | \n
\n \n | 5 | \n bigger_number | \n 0.962 | \n 0.920 | \n 0.280 | \n 1000.0 | \n 0.006046 | \n 0.008579 | \n 0.014199 | \n
\n \n
\n