diff --git a/.github/workflows/build_leaderboard.yml b/.github/workflows/build_leaderboard.yml deleted file mode 100644 index e642108..0000000 --- a/.github/workflows/build_leaderboard.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Build leaderboard - -on: - push: - branches: [ "main" ] - paths: - - "results/submitted/**" - - "leaderboards/build_leaderboard.py" - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - name: Install - run: | - python -m pip install --upgrade pip - python -m pip install pandas tabulate - - name: Build leaderboard - run: | - python leaderboards/build_leaderboard.py - - name: Show diff - run: | - git status --porcelain - git diff -- leaderboards/leaderboard.md || true diff --git a/.github/workflows/validate_results.yml b/.github/workflows/validate_results.yml deleted file mode 100644 index 87e0ea9..0000000 --- a/.github/workflows/validate_results.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Validate results submissions - -on: - pull_request: - paths: - - "results/submitted/**" - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - name: Install validator deps - run: | - python -m pip install --upgrade pip - python -m pip install pandas - - name: Validate metrics schema - run: | - python - << 'PY' - from pathlib import Path - import pandas as pd - import json - - required_cols = {"dataset", "metric", "score", "split"} - - root = Path("results/submitted") - for metrics in root.rglob("metrics.csv"): - df = pd.read_csv(metrics) - missing = required_cols - set(df.columns) - if missing: - raise SystemExit(f"{metrics}: missing columns {sorted(missing)}") - - # Basic types - try: - df["score"].astype(float) - except Exception as e: - raise SystemExit(f"{metrics}: score column not numeric: {e}") - - meta = metrics.with_name("run-metadata.json") - if not meta.exists(): - raise SystemExit(f"{metrics}: missing run-metadata.json") - - with meta.open("r", encoding="utf-8") as f: - j = json.load(f) - for k in ["algorithm", "archive_version", "evaluation_regime", "software", "hardware", "command", "git_commit"]: - if k not in j: - raise SystemExit(f"{meta}: missing key '{k}'") - - print("All result submissions validated.") - PY diff --git a/docs/datasets.md b/docs/datasets.md index 279c453..197cfd8 100644 --- a/docs/datasets.md +++ b/docs/datasets.md @@ -3,10 +3,10 @@ ## Downloading and loading You can retrieve a dataset directly from zenodo with the following aeon code. It will by default -store the data in your home directory, or you can specify the +store the data in your home directory, or you can specify the path as an argument ```python -from aeon.classification import load_classification +from aeon.datasets import load_classification X,y = load_classification("BasicMotions") X,y = load_classification("BasicMotions", extract_path="C:\\Temp\\") @@ -14,27 +14,25 @@ X,y = load_classification("BasicMotions", extract_path="C:\\Temp\\") ## Packaging convention -For each dataset, data is store one zip file on Zenodo containing: +If you download the data using aeon, it is stored in the `extract_path` in a +directory with the name of the problems and files - `_TRAIN.ts` - `_TEST.ts` - -If the original dataset has unequal length or missing values, we also store a file - +and possibly - `_TRAIN_eq.ts` or - `_TEST_nmv.ts` -If you download the data using aeon, it is stored in the `extract_path` in a directory +The extra files are for cases where time series in the original dataset has unequal +length or missing values. Versions with lengths equalised or missing imputed are +stored in files with `_eq` or `_nmv` suffix. -- `_TRAIN.ts` -- `_TEST.ts` -and possibly -- `_TRAIN_eq.ts` -or -- `_TEST_nmv.ts` -When you first call ``load_classification`` again, it looks first in the `extract_path` -or your. + +- When you first call ``load_classification`` again, it looks first in the `extract_path` +or in your home directory to see if the file exists. If it does, it does not + download it again. You can load the combined train/test splits or the train/test + separately with the argument `split`: ```python X,y = load_classification("BasicMotions") # Load combined train/test @@ -42,8 +40,9 @@ trainX,trainy = load_classification("BasicMotions", split="train") testX,testy = load_classification("BasicMotions", split="test") ``` Equal length datasets are stored in 3D numpy arrays of shape ``(n_cases, n_channels, n_timepoints)``. -Note this is different to some other packages which assume a single time series is shape -``(n_timepoints, n_channels)`` so if you are not using aeon you may need to reshape it. +Note this is different to some other packages such as tensorflow which assume a single +time series is shape ``(n_timepoints, n_channels)`` so if you are not using ``aeon`` +you may need to reshape it. To use ``sklearn`` classifiers directly on multivariate equal length datasets, one option is to flatten the data so that the 3D array `(n_cases, n_channels, n_timepoints)` becomes a 2D array @@ -54,18 +53,30 @@ flatTrainX = X.reshape(X.shape[0], X.shape[1] * X.shape[2]) flatTestX = X.reshape(X.shape[0], X.shape[1] * X.shape[2]) ``` -Unequal length datasets are stored in -a list of 2D numpy arrays. You can control whether to load the equal length version with the -parameter ``equal_length``. +Unequal length datasets are stored in a list of 2D numpy arrays. You can control +whether to load the equal length version with the parameter ``load_equal_length``. ```python -X,y = load_classification("JapaneseVowels", load_equal = False) # Unequal length example +X,y = load_classification("JapaneseVowels", load_equal_length = False) # Unequal length example ``` - +Imputed missing value versions can be loaded with the argument ``load_no_missing``. You can download whole archives from zenodo or in code ```python -from aeon.classification import load_classification +from aeon.datasets import download_archive + +download_archive(archive="UEA", extract_path="C:\\Temp\\") + + ``` +Currently should be one of "EEG","UCR","UEA","Imbalanced","TSR", "Unequal". See +``aeon`` documentation for more details. There are lists of datasets in aeon and a dictionary of all zenodo keys. +```python + +from aeon.datasets.tsc_datasets import multiverse_core, multiverse2026, eeg2026 +print(len(multiverse_core)) # 66 +print(len(multiverse2026)) # 133 +print(len(eeg2026)) # 28 +``` diff --git a/docs/leaderboard.md b/docs/leaderboard.md index 07c5af2..f609a29 100644 --- a/docs/leaderboard.md +++ b/docs/leaderboard.md @@ -1,10 +1,73 @@ # Leaderboards -The leaderboards can be interactively generated on the WEBSITE. +The leaderboards can be interactively generated on the WEBSITE. These are some +illustrative static leaderboards. -## Multiverse-mini +## Multiverse-core ## EEG archive +The EEG archive is a collection of EEG classification problems, described in [1]. On +release, it contains 30 datasets. Two of these are univariate and two are not +available on zenodo. The resulting list is contained in the multiverse +eeg = [ + "Alzheimers", + "Blink", + "ButtonPress", + "EpilepticSeizures", + "EyesOpenShut", + "FaceDetection", + "FeedbackButton", + "FeetHands", + "FingerMovements", + "HandMovementDirection", + "ImaginedFeetHands", + "ImaginedOpenCloseFist", + "InnerSpeech", + "LongIntervalTask", + "LowCost", + "MatchingPennies", + "MindReading", + "MotorImagery", + "OpenCloseFist", + "PhotoStimulation", + "PronouncedSpeech", + "SelfRegulationSCP1", + "SelfRegulationSCP2", + "ShortIntervalTask", + "SitStand", + "Sleep", + "SongFamiliarity", + "VisualSpeech", +] + + + +We currently have results for the train/test splits with the following classifiers. + +all_classifiers = [ + "Arsenal", + "CNN", + "CSP-SVM", + "DrCIF", + "HC2", + "IT", + "MRHydra", + "R-KNN", + "R-MDM", + "STC", + "SVM", + "TDE", +] + +See the paper and aeon-neuro for details of these classifier. The overall accuracy +picture is + + + +## UEA archive + +People will still use the UEA archive, so it is worth maintaining a list for sanity +checks. The archive contains 30 datasets, but \ No newline at end of file diff --git a/img/all_eval_accuracy_critical_difference.pdf b/img/all_eval_accuracy_critical_difference.pdf new file mode 100644 index 0000000..d9a781d Binary files /dev/null and b/img/all_eval_accuracy_critical_difference.pdf differ diff --git a/img/all_eval_logloss_critical_difference.pdf b/img/all_eval_logloss_critical_difference.pdf new file mode 100644 index 0000000..cab8c34 Binary files /dev/null and b/img/all_eval_logloss_critical_difference.pdf differ diff --git a/multiverse/experiments/run_eeg_bakeoff.py b/multiverse/experiments/run_eeg_bakeoff.py new file mode 100644 index 0000000..308fc00 --- /dev/null +++ b/multiverse/experiments/run_eeg_bakeoff.py @@ -0,0 +1,34 @@ +"""Code to reproduce the results in [1]. + +Note this is for guidance only, showing you how to access the classifiers and +datasets. Running it like this on a single machine will take a +very long time. + + +This creates a single result file for each combination of classifier and datasets +which stores all predictions and probability estimates. There is an example file in + +Of course, you dont have to run it like this +[1] +""" +from aeon.datasets import load_classification +from aeon.datasets.tsc_datasets import eeg +from aeon.classification.hybrid import HIVECOTEV2 +from aeon_neuro import classifiers +from tsml_eval import + +classifiers = [] +datasets = eeg + + +def simple_experiment(): + + + + +if __name__ == "__main__": + for d in datasets: + for cls in classifiers: + single_experiment(d, cls) + + diff --git a/multiverse/experiments/run_single_dataset.py b/multiverse/experiments/run_single_dataset.py index 181a426..8940f28 100644 --- a/multiverse/experiments/run_single_dataset.py +++ b/multiverse/experiments/run_single_dataset.py @@ -18,4 +18,4 @@ def single_experiment(): if __name__ == "__main__": - main() + single_experiment() diff --git a/pyproject.toml b/pyproject.toml index 6955e16..62fa239 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,11 +44,13 @@ classifiers = [ requires-python = ">=3.10" license = {file = "LICENSE"} dependencies = [ - "numpy>=1.23", - "pandas>=2.0", - "requests>=2.31", - "tqdm>=4.66", - "aeon>=1.2", + "numpy>=1.23", + "pandas>=2.0", + "requests>=2.31", + "tqdm>=4.66", + "aeon>=1.4", + "tsml_eval>=0.0", + "aeon_neuro>=0.1", ] [tool.setuptools.packages.find] diff --git a/results/eeg/accuracy_mean.csv b/results/eeg/accuracy_mean.csv new file mode 100644 index 0000000..907e583 --- /dev/null +++ b/results/eeg/accuracy_mean.csv @@ -0,0 +1,27 @@ +Estimators:,Arsenal,CNN,CSP-SVM,DrCIF,HC2,IT,MRHydra,R-KNN,R-MDM,STC,SVM,TDE +Alzheimers,0.5116279069767442,0.4186046511627907,0.3953488372093023,0.4883720930232558,0.37209302325581395,0.37209302325581395,0.5116279069767442,0.3023255813953488,0.3953488372093023,0.3953488372093023,0.4186046511627907,0.37209302325581395 +Blink,0.9733333333333334,0.7844444444444445,0.8733333333333333,0.9777777777777777,1.0,0.7311111111111112,0.9777777777777777,0.5466666666666666,0.3288888888888889,0.9977777777777778,0.8044444444444444,0.9977777777777778 +ButtonPress,1.0,0.7083333333333334,0.675,1.0,1.0,0.9833333333333333,1.0,0.975,0.95,0.9083333333333333,0.7416666666666667,1.0 +EyesOpenShut,0.5238095238095238,0.5,0.3333333333333333,0.5238095238095238,0.4523809523809524,0.4523809523809524,0.42857142857142855,0.5238095238095238,0.5714285714285714,0.47619047619047616,0.5,0.42857142857142855 +FaceDetection,0.5820090805902384,0.6572077185017026,0.5366061293984109,0.5964812712826334,0.6625993189557321,0.6716799091940976,0.5913734392735528,0.49545970488081725,0.5153234960272418,0.6339387060158911,0.6923950056753689,0.5854143019296254 +FeetHands,0.6634920634920635,0.5042328042328043,0.555026455026455,0.674074074074074,0.7026455026455026,0.6386243386243386,0.647089947089947,0.5047619047619047,0.49682539682539684,0.6687830687830688,0.6507936507936508,0.6232804232804233 +FingerMovements,0.57,0.49,0.51,0.48,0.54,0.56,0.58,0.56,0.5,0.56,0.49,0.5 +HandMovementDirection,0.3918918918918919,0.5945945945945946,0.6081081081081081,0.5675675675675675,0.4189189189189189,0.4189189189189189,0.32432432432432434,0.3108108108108108,0.40540540540540543,0.527027027027027,0.5405405405405406,0.36486486486486486 +ImaginedFeetHands,0.6770025839793282,0.6708010335917313,0.5452196382428941,0.6863049095607235,0.6987080103359173,0.6330749354005168,0.6356589147286822,0.5147286821705427,0.49870801033591733,0.6945736434108527,0.6811369509043927,0.6537467700258398 +ImaginedOpenCloseFist,0.7052910052910053,0.5005291005291005,0.5814814814814815,0.7386243386243386,0.7777777777777778,0.735978835978836,0.7021164021164021,0.5126984126984127,0.5068783068783069,0.762962962962963,0.7322751322751323,0.68994708994709 +InnerSpeech,0.26,0.235,0.26,0.25,0.27,0.23,0.255,0.26,0.26,0.205,0.26,0.27 +LongIntervalTask,0.6794642857142857,0.6410714285714286,0.6785714285714286,0.6794642857142857,0.7116071428571429,0.6946428571428571,0.6928571428571428,0.6633928571428571,0.6008928571428571,0.6598214285714286,0.6785714285714286,0.6589285714285714 +LowCost,0.6033333333333334,0.65,0.5033333333333333,0.72,0.675,0.64,0.585,0.5666666666666667,0.5033333333333333,0.6583333333333333,0.5483333333333333,0.6483333333333333 +MatchingPennies,0.5914285714285714,0.5028571428571429,0.5885714285714285,0.5914285714285714,0.6128571428571429,0.52,0.5942857142857143,0.5571428571428572,0.57,0.5985714285714285,0.5528571428571428,0.5542857142857143 +MindReading,0.5022970903522205,0.23124042879019907,0.35834609494640124,0.44869831546707506,0.49770290964777947,0.19142419601837674,0.4119448698315467,0.5482388973966309,0.4104134762633997,0.44716692189892804,0.2450229709035222,0.35222052067381315 +MotorImagery,0.5,0.5,0.47,0.43,0.55,0.54,0.51,0.43,0.53,0.51,0.55,0.52 +OpenCloseFist,0.737037037037037,0.5005291005291005,0.6105820105820106,0.737037037037037,0.7777777777777778,0.7518518518518519,0.707936507936508,0.5201058201058201,0.5132275132275133,0.7571428571428571,0.725925925925926,0.6693121693121693 +PhotoStimulation,0.3888888888888889,0.4166666666666667,0.3888888888888889,0.3888888888888889,0.4444444444444444,0.2222222222222222,0.3888888888888889,0.3055555555555556,0.3333333333333333,0.4444444444444444,0.4444444444444444,0.4166666666666667 +PronouncedSpeech,0.28,0.24,0.255,0.315,0.32,0.235,0.27,0.245,0.25,0.325,0.265,0.3 +SelfRegulationSCP1,0.9180887372013652,0.8430034129692833,0.757679180887372,0.89419795221843,0.8771331058020477,0.8430034129692833,0.9488054607508533,0.5221843003412969,0.5733788395904437,0.8430034129692833,0.8191126279863481,0.8327645051194539 +SelfRegulationSCP2,0.5444444444444444,0.48333333333333334,0.5,0.5555555555555556,0.55,0.5,0.5333333333333333,0.5333333333333333,0.49444444444444446,0.48333333333333334,0.5055555555555555,0.5388888888888889 +ShortIntervalTask,0.7276785714285714,0.6455357142857143,0.6428571428571429,0.7133928571428572,0.6910714285714286,0.6919642857142857,0.7428571428571429,0.5678571428571428,0.5875,0.6785714285714286,0.6446428571428572,0.5794642857142858 +SitStand,0.423125,0.3234375,0.26125,0.4078125,0.4115625,0.336875,0.4196875,0.2840625,0.3403125,0.3553125,0.359375,0.348125 +Sleep,0.5742738589211618,0.4887966804979253,0.5452282157676348,0.6008298755186722,0.6074688796680497,0.5576763485477179,0.5742738589211618,0.366804979253112,0.3029045643153527,0.5991701244813278,0.5477178423236515,0.5717842323651452 +SongFamiliarity,0.5333333333333333,0.5186666666666667,0.5026666666666667,0.5253333333333333,0.504,0.4826666666666667,0.5133333333333333,0.52,0.5146666666666667,0.52,0.5093333333333333,0.48 +VisualSpeech,0.215,0.26,0.26,0.235,0.29,0.26,0.235,0.29,0.24,0.295,0.245,0.295 diff --git a/results/eeg/balacc_mean.csv b/results/eeg/balacc_mean.csv new file mode 100644 index 0000000..e94ba6b --- /dev/null +++ b/results/eeg/balacc_mean.csv @@ -0,0 +1,27 @@ +Estimators:,Arsenal,CNN,CSP-SVM,DrCIF,HC2,IT,MRHydra,R-KNN,R-MDM,STC,SVM,TDE +Alzheimers,0.4915824915824916,0.3333333333333333,0.3148148148148148,0.443001443001443,0.3357383357383357,0.3227513227513228,0.4915824915824916,0.2631072631072631,0.38720538720538716,0.39490139490139486,0.41221741221741226,0.32395382395382394 +Blink,0.975,0.799,0.885,0.98,1.0,0.758,0.9784999999999999,0.5605,0.33399999999999996,0.998,0.818,0.998 +ButtonPress,1.0,0.7151506617854126,0.675443537031822,1.0,1.0,0.9850746268656716,1.0,0.9776119402985075,0.9552238805970149,0.9100253449732469,0.7469726837510561,1.0 +EyesOpenShut,0.5238095238095237,0.5,0.3333333333333333,0.5238095238095238,0.45238095238095233,0.45238095238095233,0.4285714285714286,0.5238095238095238,0.5714285714285714,0.47619047619047616,0.5,0.4285714285714286 +FaceDetection,0.5820090805902384,0.6572077185017027,0.5366061293984109,0.5964812712826334,0.6625993189557322,0.6716799091940976,0.5913734392735528,0.49545970488081725,0.5153234960272418,0.633938706015891,0.6923950056753689,0.5854143019296254 +FeetHands,0.6639606880927611,0.5,0.5551155089639973,0.6747136773050559,0.7028660826172699,0.639498813498014,0.6473110247815974,0.5052460297818158,0.4985228918172238,0.669099770314717,0.6518795333726781,0.6236476173091545 +FingerMovements,0.5724289715886355,0.4899959983993597,0.5132052821128451,0.4817927170868347,0.5418167266906763,0.5598239295718287,0.5814325730292116,0.5614245698279312,0.498999599839936,0.5618247298919568,0.4935974389755902,0.5006002400960384 +HandMovementDirection,0.3869047619047619,0.6392857142857142,0.6476190476190476,0.555952380952381,0.3869047619047619,0.4488095238095238,0.29285714285714287,0.3297619047619047,0.4071428571428572,0.5059523809523809,0.5214285714285714,0.3369047619047619 +ImaginedFeetHands,0.6769808643927286,0.6708476843265787,0.545174113514576,0.6862992171408548,0.6987103335697864,0.6330582785645303,0.6356526746263045,0.5147432418573248,0.49861600160674147,0.6945786363209039,0.6811296546360475,0.6537472117052826 +ImaginedOpenCloseFist,0.7053438653384456,0.5,0.5815554789121009,0.7386554000788332,0.7778391174257355,0.7359578242018132,0.7021177482352099,0.5129100673666105,0.5067500985415846,0.7630130881140933,0.7323106657827785,0.6900016124986563 +InnerSpeech,0.26253590634289725,0.25,0.25,0.2503340124920672,0.2770195230301613,0.23686495874945723,0.26214344166471826,0.26242526470489996,0.2625025050936905,0.20692491399178328,0.25,0.2770195230301613 +LongIntervalTask,0.6300438596491228,0.49064327485380116,0.5,0.5598684210526316,0.5857456140350877,0.6507309941520467,0.6282163742690059,0.6065058479532164,0.5341374269005847,0.5527046783625731,0.5,0.5513157894736842 +LowCost,0.6033333333333333,0.6499999999999999,0.5033333333333333,0.72,0.675,0.64,0.585,0.5666666666666667,0.5033333333333334,0.6583333333333333,0.5483333333333333,0.6483333333333333 +MatchingPennies,0.5913335945663531,0.5,0.5885253396029257,0.5917580982236155,0.6131628787878788,0.5199026907001045,0.594092868338558,0.5578467868338558,0.56992881400209,0.5990726227795193,0.5523772204806687,0.555316091954023 +MindReading,0.4997641864692898,0.2,0.3339690949227373,0.42411479028697574,0.47986910790806386,0.2,0.40063191793273595,0.5589193091806258,0.43443137254901953,0.4440023893000909,0.2144,0.33068058693676144 +MotorImagery,0.5,0.5,0.47,0.43,0.55,0.54,0.51,0.43,0.53,0.51,0.55,0.52 +OpenCloseFist,0.7370675368187194,0.5,0.610567017235819,0.7370104274913104,0.7777864872612607,0.7518498942917548,0.7079720141899881,0.5201651915290071,0.513136265094779,0.7571162701114416,0.7259166606944494,0.669304520371233 +PhotoStimulation,0.3444444444444444,0.3333333333333333,0.3525252525252525,0.35555555555555557,0.3666666666666667,0.18888888888888888,0.3686868686868687,0.2636363636363636,0.2888888888888889,0.396969696969697,0.398989898989899,0.3444444444444445 +PronouncedSpeech,0.27857974293396287,0.24754901960784315,0.25,0.31834874417994624,0.3228736310577743,0.24540133779264214,0.2730097055544626,0.24636861433536625,0.25077054233064466,0.32093907797232607,0.25725867269984914,0.30151157452947736 +SelfRegulationSCP1,0.9182974559686888,0.8432811480756686,0.7584102134004287,0.8944413381791072,0.8771316745876432,0.8434442270058709,0.948932997856677,0.5226213773180506,0.5739446463516913,0.8429083962352064,0.8195881092162892,0.8331469574131023 +SelfRegulationSCP2,0.5444444444444444,0.4833333333333333,0.5,0.5555555555555556,0.55,0.5,0.5333333333333333,0.5333333333333333,0.49444444444444446,0.48333333333333334,0.5055555555555555,0.5388888888888889 +ShortIntervalTask,0.6720833333333334,0.5170833333333333,0.5,0.6081944444444445,0.5941666666666667,0.6576388888888889,0.6822222222222223,0.5138888888888888,0.571388888888889,0.5955555555555555,0.5025,0.4945833333333333 +SitStand,0.42312500000000003,0.3234375,0.26125,0.40781249999999997,0.4115625,0.336875,0.41968749999999994,0.2840625,0.3403125,0.35531250000000003,0.359375,0.348125 +Sleep,0.4256548484995929,0.3902322911083124,0.2819016088846892,0.42830174665043097,0.4508399692188262,0.46596436623323606,0.4256548484995929,0.2621813569865084,0.36585357399923485,0.4604899691322705,0.28274774380092876,0.3930115150898617 +SongFamiliarity,0.5361713036481068,0.5,0.48686880914910735,0.520882438812496,0.49852238497746193,0.5012853470437018,0.515694763902043,0.5180340243112178,0.515783776855208,0.5159404396527782,0.5015701884938296,0.4728937755022111 +VisualSpeech,0.20965213008280234,0.25,0.25,0.23511423800289347,0.28660983624218916,0.25,0.24109595761381478,0.2875284729276326,0.24282934327577185,0.2967412926709145,0.25,0.2914175285498815