From ea07efcbc08a1e74039cb5e7fdad9e34fafea07b Mon Sep 17 00:00:00 2001 From: Tony Bagnall Date: Mon, 9 Mar 2026 17:03:45 +0000 Subject: [PATCH 1/3] wrong yml --- .github/workflows/build_leaderboard.yml | 28 ------------- .github/workflows/validate_results.yml | 53 ------------------------- 2 files changed, 81 deletions(-) delete mode 100644 .github/workflows/build_leaderboard.yml delete mode 100644 .github/workflows/validate_results.yml diff --git a/.github/workflows/build_leaderboard.yml b/.github/workflows/build_leaderboard.yml deleted file mode 100644 index e642108..0000000 --- a/.github/workflows/build_leaderboard.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Build leaderboard - -on: - push: - branches: [ "main" ] - paths: - - "results/submitted/**" - - "leaderboards/build_leaderboard.py" - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - name: Install - run: | - python -m pip install --upgrade pip - python -m pip install pandas tabulate - - name: Build leaderboard - run: | - python leaderboards/build_leaderboard.py - - name: Show diff - run: | - git status --porcelain - git diff -- leaderboards/leaderboard.md || true diff --git a/.github/workflows/validate_results.yml b/.github/workflows/validate_results.yml deleted file mode 100644 index 87e0ea9..0000000 --- a/.github/workflows/validate_results.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Validate results submissions - -on: - pull_request: - paths: - - "results/submitted/**" - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - name: Install validator deps - run: | - python -m pip install --upgrade pip - python -m pip install pandas - - name: Validate metrics schema - run: | - python - << 'PY' - from pathlib import Path - import pandas as pd - import json - - required_cols = {"dataset", "metric", "score", "split"} - - root = Path("results/submitted") - for metrics in root.rglob("metrics.csv"): - df = pd.read_csv(metrics) - missing = required_cols - set(df.columns) - if missing: - raise SystemExit(f"{metrics}: missing columns {sorted(missing)}") - - # Basic types - try: - df["score"].astype(float) - except Exception as e: - raise SystemExit(f"{metrics}: score column not numeric: {e}") - - meta = metrics.with_name("run-metadata.json") - if not meta.exists(): - raise SystemExit(f"{metrics}: missing run-metadata.json") - - with meta.open("r", encoding="utf-8") as f: - j = json.load(f) - for k in ["algorithm", "archive_version", "evaluation_regime", "software", "hardware", "command", "git_commit"]: - if k not in j: - raise SystemExit(f"{meta}: missing key '{k}'") - - print("All result submissions validated.") - PY From b5a3b22da6c4759cca58c11450fe2b644a469b8b Mon Sep 17 00:00:00 2001 From: Tony Bagnall Date: Mon, 23 Mar 2026 20:07:17 +0000 Subject: [PATCH 2/3] datasets --- docs/datasets.md | 57 ++++++++++------- docs/leaderboard.md | 67 +++++++++++++++++++- multiverse/experiments/run_eeg_bakeoff.py | 34 ++++++++++ multiverse/experiments/run_single_dataset.py | 2 +- pyproject.toml | 12 ++-- 5 files changed, 141 insertions(+), 31 deletions(-) create mode 100644 multiverse/experiments/run_eeg_bakeoff.py diff --git a/docs/datasets.md b/docs/datasets.md index 279c453..197cfd8 100644 --- a/docs/datasets.md +++ b/docs/datasets.md @@ -3,10 +3,10 @@ ## Downloading and loading You can retrieve a dataset directly from zenodo with the following aeon code. It will by default -store the data in your home directory, or you can specify the +store the data in your home directory, or you can specify the path as an argument ```python -from aeon.classification import load_classification +from aeon.datasets import load_classification X,y = load_classification("BasicMotions") X,y = load_classification("BasicMotions", extract_path="C:\\Temp\\") @@ -14,27 +14,25 @@ X,y = load_classification("BasicMotions", extract_path="C:\\Temp\\") ## Packaging convention -For each dataset, data is store one zip file on Zenodo containing: +If you download the data using aeon, it is stored in the `extract_path` in a +directory with the name of the problems and files - `_TRAIN.ts` - `_TEST.ts` - -If the original dataset has unequal length or missing values, we also store a file - +and possibly - `_TRAIN_eq.ts` or - `_TEST_nmv.ts` -If you download the data using aeon, it is stored in the `extract_path` in a directory +The extra files are for cases where time series in the original dataset has unequal +length or missing values. Versions with lengths equalised or missing imputed are +stored in files with `_eq` or `_nmv` suffix. -- `_TRAIN.ts` -- `_TEST.ts` -and possibly -- `_TRAIN_eq.ts` -or -- `_TEST_nmv.ts` -When you first call ``load_classification`` again, it looks first in the `extract_path` -or your. + +- When you first call ``load_classification`` again, it looks first in the `extract_path` +or in your home directory to see if the file exists. If it does, it does not + download it again. You can load the combined train/test splits or the train/test + separately with the argument `split`: ```python X,y = load_classification("BasicMotions") # Load combined train/test @@ -42,8 +40,9 @@ trainX,trainy = load_classification("BasicMotions", split="train") testX,testy = load_classification("BasicMotions", split="test") ``` Equal length datasets are stored in 3D numpy arrays of shape ``(n_cases, n_channels, n_timepoints)``. -Note this is different to some other packages which assume a single time series is shape -``(n_timepoints, n_channels)`` so if you are not using aeon you may need to reshape it. +Note this is different to some other packages such as tensorflow which assume a single +time series is shape ``(n_timepoints, n_channels)`` so if you are not using ``aeon`` +you may need to reshape it. To use ``sklearn`` classifiers directly on multivariate equal length datasets, one option is to flatten the data so that the 3D array `(n_cases, n_channels, n_timepoints)` becomes a 2D array @@ -54,18 +53,30 @@ flatTrainX = X.reshape(X.shape[0], X.shape[1] * X.shape[2]) flatTestX = X.reshape(X.shape[0], X.shape[1] * X.shape[2]) ``` -Unequal length datasets are stored in -a list of 2D numpy arrays. You can control whether to load the equal length version with the -parameter ``equal_length``. +Unequal length datasets are stored in a list of 2D numpy arrays. You can control +whether to load the equal length version with the parameter ``load_equal_length``. ```python -X,y = load_classification("JapaneseVowels", load_equal = False) # Unequal length example +X,y = load_classification("JapaneseVowels", load_equal_length = False) # Unequal length example ``` - +Imputed missing value versions can be loaded with the argument ``load_no_missing``. You can download whole archives from zenodo or in code ```python -from aeon.classification import load_classification +from aeon.datasets import download_archive + +download_archive(archive="UEA", extract_path="C:\\Temp\\") + + ``` +Currently should be one of "EEG","UCR","UEA","Imbalanced","TSR", "Unequal". See +``aeon`` documentation for more details. There are lists of datasets in aeon and a dictionary of all zenodo keys. +```python + +from aeon.datasets.tsc_datasets import multiverse_core, multiverse2026, eeg2026 +print(len(multiverse_core)) # 66 +print(len(multiverse2026)) # 133 +print(len(eeg2026)) # 28 +``` diff --git a/docs/leaderboard.md b/docs/leaderboard.md index 07c5af2..f609a29 100644 --- a/docs/leaderboard.md +++ b/docs/leaderboard.md @@ -1,10 +1,73 @@ # Leaderboards -The leaderboards can be interactively generated on the WEBSITE. +The leaderboards can be interactively generated on the WEBSITE. These are some +illustrative static leaderboards. -## Multiverse-mini +## Multiverse-core ## EEG archive +The EEG archive is a collection of EEG classification problems, described in [1]. On +release, it contains 30 datasets. Two of these are univariate and two are not +available on zenodo. The resulting list is contained in the multiverse +eeg = [ + "Alzheimers", + "Blink", + "ButtonPress", + "EpilepticSeizures", + "EyesOpenShut", + "FaceDetection", + "FeedbackButton", + "FeetHands", + "FingerMovements", + "HandMovementDirection", + "ImaginedFeetHands", + "ImaginedOpenCloseFist", + "InnerSpeech", + "LongIntervalTask", + "LowCost", + "MatchingPennies", + "MindReading", + "MotorImagery", + "OpenCloseFist", + "PhotoStimulation", + "PronouncedSpeech", + "SelfRegulationSCP1", + "SelfRegulationSCP2", + "ShortIntervalTask", + "SitStand", + "Sleep", + "SongFamiliarity", + "VisualSpeech", +] + + + +We currently have results for the train/test splits with the following classifiers. + +all_classifiers = [ + "Arsenal", + "CNN", + "CSP-SVM", + "DrCIF", + "HC2", + "IT", + "MRHydra", + "R-KNN", + "R-MDM", + "STC", + "SVM", + "TDE", +] + +See the paper and aeon-neuro for details of these classifier. The overall accuracy +picture is + + + +## UEA archive + +People will still use the UEA archive, so it is worth maintaining a list for sanity +checks. The archive contains 30 datasets, but \ No newline at end of file diff --git a/multiverse/experiments/run_eeg_bakeoff.py b/multiverse/experiments/run_eeg_bakeoff.py new file mode 100644 index 0000000..308fc00 --- /dev/null +++ b/multiverse/experiments/run_eeg_bakeoff.py @@ -0,0 +1,34 @@ +"""Code to reproduce the results in [1]. + +Note this is for guidance only, showing you how to access the classifiers and +datasets. Running it like this on a single machine will take a +very long time. + + +This creates a single result file for each combination of classifier and datasets +which stores all predictions and probability estimates. There is an example file in + +Of course, you dont have to run it like this +[1] +""" +from aeon.datasets import load_classification +from aeon.datasets.tsc_datasets import eeg +from aeon.classification.hybrid import HIVECOTEV2 +from aeon_neuro import classifiers +from tsml_eval import + +classifiers = [] +datasets = eeg + + +def simple_experiment(): + + + + +if __name__ == "__main__": + for d in datasets: + for cls in classifiers: + single_experiment(d, cls) + + diff --git a/multiverse/experiments/run_single_dataset.py b/multiverse/experiments/run_single_dataset.py index 181a426..8940f28 100644 --- a/multiverse/experiments/run_single_dataset.py +++ b/multiverse/experiments/run_single_dataset.py @@ -18,4 +18,4 @@ def single_experiment(): if __name__ == "__main__": - main() + single_experiment() diff --git a/pyproject.toml b/pyproject.toml index 6955e16..62fa239 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,11 +44,13 @@ classifiers = [ requires-python = ">=3.10" license = {file = "LICENSE"} dependencies = [ - "numpy>=1.23", - "pandas>=2.0", - "requests>=2.31", - "tqdm>=4.66", - "aeon>=1.2", + "numpy>=1.23", + "pandas>=2.0", + "requests>=2.31", + "tqdm>=4.66", + "aeon>=1.4", + "tsml_eval>=0.0", + "aeon_neuro>=0.1", ] [tool.setuptools.packages.find] From cf6807cc993dfad47018e40b530c9ae4d14b8d00 Mon Sep 17 00:00:00 2001 From: Tony Bagnall Date: Mon, 23 Mar 2026 20:15:16 +0000 Subject: [PATCH 3/3] eeg_results --- img/all_eval_accuracy_critical_difference.pdf | Bin 0 -> 13851 bytes img/all_eval_logloss_critical_difference.pdf | Bin 0 -> 13856 bytes results/eeg/accuracy_mean.csv | 27 ++++++++++++++++++ results/eeg/balacc_mean.csv | 27 ++++++++++++++++++ 4 files changed, 54 insertions(+) create mode 100644 img/all_eval_accuracy_critical_difference.pdf create mode 100644 img/all_eval_logloss_critical_difference.pdf create mode 100644 results/eeg/accuracy_mean.csv create mode 100644 results/eeg/balacc_mean.csv diff --git a/img/all_eval_accuracy_critical_difference.pdf b/img/all_eval_accuracy_critical_difference.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d9a781dc02c04d3b77f9cbbc28b197089b45fe2f GIT binary patch literal 13851 zcmb_@2{=_<)NmQXHB`pPaAeAO=ZVN1nTJeKrfVoJxwn!gB_tF=5fvFyN}|b-Ss98X zvnC29Dv2h(eJ;KAa{vGJe9w1#c6*<-hqd-v>#Vi*K8u{CzM(2w4Tq4+?}M(EAkYvB z3UJ?n*suXY+U^UaKuA3|x|?5sH-xlw^QO=s3>aVyX=x!SR8M#y>8}Zf0aQAKWeh+% zSh(+?c+er@+@nD-eUlB{jZT4Za{x;>I-L?kh4Anr0%_yp=IKlIh6wY$K>;2%6gspO zpw%}7q)>wC5YpHmkf1wX>&@53kjr1>pciidrW>X{hytjaBOhr^p#|&?@&NmR=?D1D zD4xD@;gPNN4=-24&2X}27jsN%#`Wi9p}eI)b? z&y8Oy|BQ`WZ5%zlYS8uJP5vs3tEf?2g~hHtcL%#%BaGMZ2=^#{u-i>5iGHN1QSs~7 zopZfc&$QHzg{Uj0qs68 z^&WRH?LOztHvX-zH58*M*yws`Au8Ce8{XZG|_e8TQ8+w{UdEt9lsB}wRE z)C-!?XAo<0t5>)5_#ouYA;fa^wu#BFRrI!5K46=eO4(q@8@y#?ds&fEw6*O_4hzgE9|u z-<3)}4KQUoe*zGMSdQKk!Cd3+_DWJxWt}%)cRQe`;KbABZre(d(EI2Xm2O)A@ra&> zT>A~ld5xP}Is7%&gdl`7Rk*$)bz17Js9d?f-(!`OOF5VnA+~m7BaWi=$jVi3iZ8^= zo1Wb?f;h3QK55U2dY}EuysNkP+FSPcm|aUhqSGUr9^um7v*}@mqBD3#_XuC`T5G$$ zhaJ7*RuJ*p&MW0)r z<*cgox}?@7lo+jKvQib3uiTYibnDCN#`47M$gjfR*83li*nH65@^+kxX1g_~X8yVZ zPq(zOJEdVwX4fiEx^D)`2-&tpeAp$Nk~_U#46qzTzAdZQE?2ijwnB2|+alRMevD1o zUu|su!N$r6bjb2i6*)+OQ6ekwpP+jL@d)t6wdxgU^Zsc8Nkg8I9oJC|^ZC2^! z17RoD8K+w80APfw(va&UNj`fE(Z#mVAiP99&8qF0edd+Aq)}}8D(k2{$@{N9w{o~$ z>YbV-x%;`St-6i=2m1}wM-w|bW!umKuU;Of1v&`i1@y={Qm?9?(#fvap`0`Ep#Qzd zIV(lEYrQ9w9)<0q<&~M}jT+ioNL4&PbZFLBS*=+~{C3S}iRkcjrPOOTu1_h9ghigx z?Dh|jRI}cpQTEcWsrck8QE`nvyI>Lff|;Kmrw=KsgiRm=!o#_a+^%sCyLO27>`+M; zf-9H1$Z^tKEbX9}?V5&tCod^y{rWbsVx_!U(<`Z3L6@StHwG?`MNWlOh+1`qO;~&6 z_+3j#HXIoXc~O3_RMgf!o+Ik4Q%78Vu9|(9%WUvb2?2`h+Z7x9?d_rpnq=BWpB{)A z&AuS>9@BnR;K(QKk;~$_YYq=P)My>%J=~eloN#dXMf82%iPs1AEB+wlesnQ$Hb1=l zNv-M+*e%SPvv~)>uzj#SL}M_h-(G`ZG-2RY$mJgx>?#%*PBI!8%l|N%_?S}ON))I5 z{+zq3r%9nxQJI2YXZ(+=iu7rYL{0d)dZtIp6b}gB%n7tHwKl)JmuE*dQ9@%_?$Iw? zLx`H#mzBFFg>Ii?O|X)UkJLZYl;r(p`vEPU0C9IFd->bj&D(Yi7QX}Gl$HBUm&94T_k0Y{_ zsu8{%EkSlzp7U!{Sk7QSIeKZO2x|yin$_;wGl&b*IqvJcsc8iDbi**$J*_pnJaQec zyBodlw=Tcm`T2rUDt z@zk&96eE+eq)DowQc)SUEqn#5OMFvV6bJCaS!r(j7}frSti? z`k0LGu|D#7>BXL%d*c$KG4Pj3Uae)CsF*`X6WWI2kv7D}~tx65%;a;BW7Vn|< zVoUOt+@$NNvFAI>J#?P=p273__Z<0r?bOC<{PRm6#YQD{ISSX)sT3#f6o(Bv(=?hd zH4hy!@7=4p=2}jBJ~61fQ&YXE-Ey1Lk7JL0ihn6|B1_TRq{0QsoQpS$frEhsI!s!E z4x2p!It=l!wVqep!18id%PMhQcSS7k>Wsn*bcJ-+NCb7415dGr__gVFo3jUpCHWI9 zY5n>RSh=2KS=+xip=!=LJiJ{%w7(v7U${>^nh9Zl$3^p+OCsH1%S3hm^)JVk!) zb=*(6-Kqv%iwmR3?QCq^vK+Sw99~=b>{r-}E$c`W=V3w?9KkLi3%>*zVsGN+Srw7Y zgjGDDI0s)oQ-e9w6*0VaOi;tY@dDJOCA=#5SRMXK6q~`6`72Iooxa?sBf8@is>UU+ z_JtK>cC~s7U5Y!yr)>nxy<6mI!7?}}F2uL@QD1EW~Z(U6;gS*8MCS06Ys zllG;JWLf;sRMe-Z=zf>%y|6Pfi3+K0SKTLv3|96_HJo3=Oq-Z^H8gQ4e2X*TuYq{< zrqXmbEt?IUD1JI}wD0iNrWAUZphpu@-=0326#1Y&F?Pzw!{V5#^BcODxsB+l=u$72 zi%o8rrvYWD_bP4O;-8IXeqU3TF_G^TS+E6*K1NL87jsqG8?>r6<79it^$FDK z@d?%!I$EAgY($5Q;_u2g;&*d;FDJ#m~f zOZ;cPo0p|PS!B50Ggpm^?<>8v&0mVC(G>$P^&F!eNUu4ENWB~R!0(n(e)=0z>31l3 zW4~eJhlIyIK?7-hS8Sh~?kul=eJ*HkY2(hG&RvAi; z-e|dW)u+i(sgD?A<|2?(bFOa8vA^Nj&d zM%y_Z&i55I!z||=pEvDGYsi|W%@SBNgj8qqy$GvU$MGAEoXt+rtooZ`G(jF z4@%>lKH6qU-6i-B{BkIgMKIc)K3UdX6&*t@A*U51 zUxecP&86gwS7x%8Zml_qk96Ca(auNym>{to=_$3r+|GWXu$SbBvF>QOTzS*=WqU1c88*N3KFa*5A=lm= zCFROrCnuU^V|vB+*lo_KBJ+(=PLq#%7HG)(RS&G1c&>Rg{w%Q~TJ@;%8FHuR_d+Ry zwIgA#zo%VQEE1CYsh6~&@U__Ng%>}*P5hkUh*^I~X0ddzOnf?FDXtTXvqf39;#v5; zBL3~B{Nw}b%nhZDgqPE%@b_Xc3skL=gBZh#sM&WSmH%%YKX1JqbwH;03bjX&!$j+k9 zzRTzs{tPDcFmd2m)DkSNW`YTo6%Af=t#GwfmEXJ1Ph_nxIs5!@uY%mj3Gr@WgB{5w zrj9m}ak$gR4v%~cEKL8b;n(8j_0lQxOw^mip;jUHvCISA&4wDFpgkw|IDho3Khd~>O5r*6yi4Q076MqLruAcZlX`nT;5U&qZnk7YACb=^mL;$;D~ zL(R3!M%}>O(>y5OH+qZtl%?R)do|@et0w&n^b~3F&)#jXdCa{t*-tol^LKX1Oa1aD ze0JTvm(9noZRl>Dl5*;#OEKX+lPCp)U5d@HASVDL+q5rS3$r5m0KNTdX($jIFAdeH0S33Zl1wKN?ZWS(R#;Uv%lBwN+rAR_<{Ag~M;QJF7C` z7n9HmgIhwhscETkQeT?qJvkitQTDS;prqkdRj=_E*I82)Elm&I#>%#?KC+1iomem+ zckSZ!BNfrQpQx|jhgCyY^{k=CAuEs5M{@}A0_qZJ*o|&PkCT-K=oM)r^c+nNdGe2H z#4*}h%BY=@Le!OYC*NUDDxO!`e<4^{<-=ro{vAZ13?jPkKO? zY@SXlejG*?Dw!IHB5;_+_6(0s3HT}@txF}G(hu$CPwHPYw1)qT_hHc&p0nRLW+I)| zG2tMS01!)F!r@ob+JsuC&DlF7=zOv56l zzbm}q&bI4YH{={p+v=3E?|5l+ten>EzB6Oew_WIJg(QhhpSdyVO8S={U3ro*t(zRq z5oi7}gS+}5d1e3^TGe@j_wFlAjLr?&w~D&;R4dU`ex22BL-9hFvn($-R==92{n13*YjG#NWd$mP`n}!7U5B3BTz*~DRzG?F=!4^V zk1xlR>DOvmn0;TLY^rKkPqh)<{xdPA{Q#X^DG&EK@pJ2zqE>E)sD1oj$v=&zhdTVE zBh;xKpQGMLVe2sbk6lg3t{GJLLNiu^I&Q=YngYeJ7h?^$lH zQAs2a1Tc<|SwHYL@H)A6=B>4gKB}D9!9~xstG&|vvgmEc9-^M==a-ice){fXcg8jR ziOHlx@b+-6vw10d#u9WxUYTjEYnDW0VKh{gZ|$#rM?La=W0UpAUky{!5(O(C{d7)a z!a*kSJ$VU7HiOuz)s+w@+pMh)7Xr~^A%8mZ>ul^Q%mLRSzOiHM+_(1`O0g%&ZxeFh z*zD0~c}lzdRgh{!AN$Hcg~L6qwrR;v_xi>gZ&B>a;@>yR=Pe(1dEXO{^eJwSA2J)5 zP{|}D$1j0OECCB*7*wGx3PZ3$5pXQ-0&wtfk@NQjfcBN3yB8a2uAsu)ZCl2^h7ZF8 zJCpbpw}dbOi@9S7;uhR;{s>_ZL3yo!FCUUu*{O+`s&F*3JV_2jtUF(ov|`ykZ(fDP zXmLyeH#7nJ-{%m6I1PSQa1PmTo*^?W!>ZNdg;69-1$cD%Syw)FfWk@zj<)w4jT$~0 zd0VcZef%1mx5v*B0u|3c$hJGwE^ELi(fZ)MI^QJvCm#Y$nfsc6gcn~(;w90GtJH;_ zf5;pR${Bn7O)qeA`(%ZCKlxnLk%mSI&PVTT`o0k~GD}JtX0jgi<#!%Z9^K;R>uA!% zbtk>lOPj0DhTL;ibE6bO{VS8bG6@Z_m?gx9c&k$XIuy?+o?Qe59g=%A02#&KIebo| zC(r0~PZlAODcAv;OaNaBDC*Ne;z*)&|=@Ofc0g2Sm-lO5byc%#Fwbql?=>sNHR zdv$t`>^%;yAhhGK4WGdY#XFS;L{+-uozmV1IU;7M2U*12!bzh=(fTjjKX#P99ZFfRKq7Y(WTNl$n!h!%er|v7 zLCZy*=hN5?-g{44@}*jGe(7g{BKs>1k7?a%sZ?XVZCtw7WAnE(PsRIe1jN_NHWG~= z-A9}40=j-a7;f~k@{3EFd@{n<*mqIDT{6HkW9!Y^KZq-f4$dy$F1nw|@fXJ=Dkd#O z>;AdNc4a#x{}tQ;RzI)cT_s1otI)6MEWwqcNX6FZtvJu465+Dz$d_akxAP{;lj)BpF8eY`9M|O|mBW;Im%S?HJ<9|&llTiwSi(JsK^&Q%70t;;D?h4W z z@<=ecEg2@0>QLlzHpNz{$of^WZ(Vgox6`wS`muqrKchxg*#5M5=bZS_EUX{_{e6vX znMPQQ@$@!>J}2~t8GTft=vS@)19|piQH6Qk_gh>enx_z*qz4ncrbmhInQW9h{N=E@ zHJs{fctR%bLf*O!7qc3grAY=UvAVrWy-p8WyJc7I`DXmN=sQ?rulM(j3R9~0 zxsZxxcx=3G?New8%52Z$953`XX>BE6KVP2g=tDmzP?mM9rQY1a{JuT*y);>fE6?4@ zzRFiH#ht5{f4hy9O=^hf%#Y7wpEM55ELWi&pfM2(lSpSN*Kk-Y!3Ovs6bpY&PQ<@1 zo99mPD$Pe!WVqv!#ntqhk0-8NiM|hFX@PPfQ5mmiYMmTRsRRKM|nrOYe!*_L6~YRO2& z7<=xb%aVSdw+bX>Nn9Hc?M6LsLfN=yyt={O{??sxbnMLlzM%T}2dmRAHGyNvkM?pt z@tIjG-1yQ%C?RxHZXrLfFRAi)<@s%-x@{8zWWLJd9_gtnJRUysxgjT`DqGJNWxLfs zkaU`ySZOx5I8JB5M(MFn&)?IY#L=7j~ zJx!6}wzo#<--?rZ_=$siDdDwT4OjU;SVtILF0+$r<@ESCIu?1tCYL&?6Cr-F~ADR-zoaA3Dy+fVv% zVk%>w>dh6$%Ttn1ASCRfgb(R_y0|I!;MX;_4$zYi;>Lk7QI_$cdye!N@sRhc?6H#i zf?bam817kX(5G`mF5xV>u7v*O#dK@o#hI=cVuiZW$W|vuk2bBf9}??TFe4!jU+5DZ z55l-{m=zSl)H+wL6^+AOj&VQP7v>F|+}*@92E?ss-dNqh*c zXCfRXaU*6a{z}!#IEYn;v$RF|%Q3Iq7anJnBL`QnOqi$3~-J@Sxpr@zCXwk>4rVTsS!^C{$uWY?~&!K*aQ!n~XQ##uSWNF-?;Q4H-I|af_qciDCIq;=+xj!>^pn zdd#N7A9LL>mJlTPBO7^&9c73kjI^_L)FJ=#V4|eUE;R)aWWM;9Ndl{g zm2duMZKKLx)RmL8o2Op{n+!b`emb!KQN!nITi3dCom6eYv)3=mjt~5ZDHt01_LFtH z>(=9od5C8g1TW=oS@TpCUk&`-<{M&GJF81Exgu0vVH1?hqD_nDExtwO3MW zB3#c6)V0ak?wl5^)0QJXOmXGQ&T5m`6d&SFTH`EPy!Pbl!r||$OB1{fw$|&G$oYis z&$sdI%h@GRD}iT!r}!@IQrW6|7xKKja0&gmq6FGq-irM#Hl4G%SIe5dv6t^vn_;pm zCgCmyy@b06ycL-bg!ZFT;6uD@3^LdvbN!>4ibpX|>?Et4BZ?EHQ$AatzAXL5Mk^#m zEv=-lvv~(@D7jdb7JBz~8S+{}Wt7+PuqxlPcbBJbSgzOkA&_c;Z(748V6bk(x05-~1l~L0+A#d>^{exDu$F23LYXgHCa|lC#q+}E z#G=94fWj|DE9}3|EWfbHHhD3B04x4{3Y*L0V29_#1>k>ic?mQ9#}woana|!U6s8*%#bHgL5uDaL;-CtkRS;D_uT{9G(Z!QM%hE5B9IhcZy!2@ zr22vh1hAqGL;y!wkd3GhAq~JTAR;(*8G+$Y6jQNFxN@o)D4(T7W1&2n}*%kW^5C&=C*&^+LrN-_pqqrw>A2UQFf z>`Koq&}fcPC=;bL!&Cvvf2+Z7as94=$Pm8?VXS~7qTy{b$cOddpY`EU z@SYgT1QZH>{$~}KC=?38=$l`FOdvzQ3#nw|cTPazfeZnyunsUOWMJADdI6jd;ji*rCw|v3Q-ItNj2dqL zRT#3K>xakuQ((P-r3%-75(fW1(&+$a0xStIt$Gk(BBP}bq2UM;zziTpW)^4}LO`+% zm=Of-#~3YRkmL=w5Wrkx<~^Gk1O(3LHHR<^Ujtij%sdWj2s4kv2Jm%moGrxoQVdqH z16nc%a|5PyuH^~zd9Fo)z{VIW&sj*YEyfeW)c_r55W?_2zzQ&W=A9Oh4dc-dygG9& zDo~>N7VM5dSjQLu`!dEY6rkm|qnk$_1mWh9(_nwd7|w8Pz^=_d0bd6$XBj6zwh1VT;JTEEFevMJz@V3bQmU*k;=atxGMBt^a6YG&x%Mrkd6;WIqN|<66hB| z_w#j!_F&b}C^ftaHU9C3-t7Y+}%8Of@Ij= zOMsya1ik@OefSR|*6XWd!1X!`i$un6D<=6}G* zhe9HkFTL8m?#J`0w90)oV!C=i^Fc$}leIX1Os(-`469|nK z!mzN~!oPp$LxBx19*0>(3m6B#Gyj^4Mv-BGE`SkHi^rkK3-O}SWRQXS@3{a6aH0!g zut8b?!=T|;1^)fn2L=mnco)L(@XNel9Bi}~!bq?MSTK%^UxXLv$D+Ad%pzH$Km@gD zJuHE+cs(+45zRO(a0m-%!NIO?A&dZ=(gGMB1sezW_lGY)K^Nh`GvPA{k0XKzV!=2P3idPb@6UcoOn6TM z8(uVy45G*dbII7n@&JB+(OfbUy$5Hszi6fhx%v80f*60qW#bz{0d@l%IsyV*mHaR}L>0jTf> z0u7-cf7e|I4Gjos6CB_HA@#^~vY)>fgtQ=gdC(vX=wJnDX(2o)?r=x7zaB93r_doR zqXW`@tLrWgH#$U|yEF)*8(GuIbPotON3bB%=^j)HgoiH?NNaDhyAQ<+BFx{V`ny?s z(4p;MT75$R#UqFgAx(Az2)gsN-h6EW?f450dhr5a+%S5o2S7ImA8F-5^WRH#1M7kD zgZa%o+I14sB`l-ru4m8lP&ZKUCk=|MAqv&l`reH`hx#IP~z+NJEy76xW@z zI6dOoxX0q|nG1OBrpW*j;Ru_Eu4PodNU9{wc+(+~a`oizVI3vt{YDAb6fx@j4Qn(K zqgV^%+b?j@qpu326o*>jl-`LMof}ig&X7qsmPNRu!Y-)0m6s%Vv?k$tmhM~QWbYJw z`03$en=|F*qxt-0PaiPId2r&D$(opJ2Mh97hfs^`cDmj&v)iCaYE^X8yni(s^fVe9 z*P`h}JUAhf^2_TI+-t-5pqJ_fC2^A2j<4r!wkmvi#dc!Q%g#b!QYQG$40f9hrzVNu zIhCfQ2!ITg&6)}V7zq`R54<^sD3CXCpV(xr^X-OwghF+<(^jdyGA$Qh3N4T=hi4phqn9J4@(_N+W4_WDop-ja5qwBNPt&>kem#e9{1*_CN!Pd5VOocr`G$ZG=mRHDx`qt!x$b@6+=W}vyW9n!_=ngKn zVsz+!DV0LyA@ySQUQ4d`xY4-od#)JKXO9C#%1-T-Y8?p5!G-#@&Ti@x+=ksz>_-h6 z2u1GNSiR;>P5YOZ%9Cznd2hNpUA@fe7ZY!kFI@7m4WT*sfT^M32?!8T&E0i~WU-c<{nuyaX=M}Bb zu!r-{SyC!6_Y~QL)SqhbbXPW&ovQNEplmWnmQ+YaI=As_i62^5=^rKB|KbxAC)G0g zT)Q>B!orFhUt4kW7Dd{Cg2++Il)0MaW82GX_>iyTfi5X(GQ4+B_z_&h)oT`!cawdz z1dfMi$)*%W=nALNQD367&**;JoR6!G95rptFHx|`kD%&|(nSzeW4Q`8MG?Br@64r+ zQoMaFFSY2pNKURYZ#e z-WFZQ8%8{u7AZD2I=x}k*!-S#jasp0i$Z1YaFoN{>U6KXZ|}GUXFY#hrl&u7dZ_Rn zd&>mX`%|o>ME%o)2WJaX`3WZ2!CP`?!t(eDy_-vL)A;LSEf2@)H)+&4-!8blGbttY z?fny+zGc?sx#FW$wFiqT58f&BIpR8FcWMXk>@w>NTmtMH=55-%mtYt`*eIeg82oS7 z!LXb#aXYl*9~$g07FbUb8d%K#u$=g~>Xi*BPW{l_sx{MUVN)?#0$*o#N2>@AXpY8w z_H%JR7bRUjBy=@5z}nQxJbxe0t{kGc`iSi9U%2)V)wN$%?U@v;IK!H3DU%eXf2#AO z*K6nfT0H(@u1wbQx67M1@E9z92^cL+D9g~C)x&}_+Bqfi6KR*;X8xMJ7HNrOWqtHW zW}?whMyomT{RX@J9dB`+AJiRQwRCTO@v1f&k*#-s*OSL`tA9aOF~fBazc zV3yaoiBsUtt-F1kMfM61*^_uB)oVDut$(Aex-p6J^^8K)$!sY#l`zSeOq*`r;x%#y zJ!OJAGR!9~;%dI?+@Gus)iI*Txhqy$7%FBI;acM|zsGya6->;AXH3@C%`DS1v$@WM zXUv=;0r%fL(@bn!qbJ3wKVe%c8y2Rm+3rrAuAlYQKH=mp?lM)N9-e3!g>CYB*?#1H zDK5ZYFwU2=So#a68Ow===En3O9`5CN!$P2=M;YB?kC}& zE+5}?6aVz$hqdDpx*Vk&=oAk}?lgN1-?Qpn7rTZJn)mP16uF%HpomC)+^4y*^MS=q z#~(-Uc$fc@??YCjcS=SGkT@4F76V5F3v^g*2|8?c8|W~^!`73bpuzHdR?9MF{p0%h zm1{Cf&(r14xkMo-d+d42)x|DPKd?T1U_^p1*@8BxZ;zFIax~lddnc;twEeA$Vxrv@ zYMs!4SS%A@e+NhNnoA`7w-B&+;x?2;N%=Cq+{ibGH-@PQg-9oDt>}}T?EIa}%eqXv z!j$#4xTnUHN+o)@k|Ucjgwn8%_4X^Y*ZP_9c6|}M!gB?;3@9p)g{xcoBT0IO|c5mk?ghtt}QB&Jw;=8EbGv z!6Zm5Pd6%P2Ys&L1BG@rV*iF*U!+)^$sIOIKf(;JW7hofy{##m_37xQc z)|)VGI#Xvzv@>P zj@|4jRG%(az<*C>&r|(Ob+gUqR+ZoHe;zT+Iyovn&IBR6gIIu&Nc?Yv`1nQ`A?qwa zs1{-IwcwK0y8G?a)tB6p z_C{D(rBL*;hw0sM@+ChNp11Kj`#L5cgk1TIS~KyP^_h;AI}3Feu&YP|-JuuqMsGcU_~Q#CS>f#X|2+2gLzmoY@ZklTBw@-a8Do zzdSZk>@@UHxv^`isj^;7(r$1>vF%aE(Z^ON-rOTi7N_3*9#9?|5%R()RL(eHZy~fT zr}xei>yTg812W%#v3_Z^6KBFOCN2_-TY|~KTdChWqr*8?Z#}}2;qbIG_-uRjG;Nl^ zqAsXhqRVoRg(EE}M(UE~m1)_nsGag3_SI^{pKq&9bo^kGEm=j_J@m`IOb(H1s2uUQ ztxi(*RnV@V>sT~}eQw5?8uH4p?y2t`?hZMj5cMn!x7%D&)?`%{d-e9F1SB`cIK-+c z`P|QvE9|zynOrnCGa2bT+uS$b9$}qY`Q2pl)7fQ4`KJmFukmSh8!nPvg%R62`Ld}l0K7FMt7s-sSv*g19TM1F&k z)Y%rtn`phQ4sYD^bhvCTpIwG6>Z?PU-#6six9di&($~q)T{3a~Vu7}sa~ny#?>$bC z4!akt%lX|LTK)N{=HaB%#QIp3!%C+}eeT~&B@Nb%9(nowY@tG#pzKe*lNzNj*Up}Q z_T$^`iEf8WaTT+zyt|1C%%OD3R{M<1P*hVE0%_3r%7*#5YSVV=Eh#oRz1aT4@GtM z^sMf{5YSdA9FF~}>FxMTS5&;V!Vc)#BMX6vJkj` zy{VRG^`xJHo&qiD;algXJKVmhenLT;zq3nR9F*I_Yx}rA-+bb-#^atTNyk3ABop2< ziBd4wrPz$Eq-0=ZJA)&%P}@ZMwd?cpDhQ9h9lDX6W}f&#s{KB{cv?`Jv~-DQTfd__ z64y5H(R9;w zRQaWDk1FP_PPZs3yLiFMGN3>!Z=~q_q1Vn%DopsrB(%cdmJn@fTBs-JOYw}6B9I?s zK1l~i7#6B{PCUE9nyz4Bdawd3)3YYrhzFfgJS2Ph!jW7F=A6vRf)yKKsQ*Ppxdo2hkHn^?t(reNMmHnovpYr7X^-!3?N6H)3#fE{;%h zA4Q~9wYcNCgM0Z-4vGwm@SXBHB>K#K_8Z4cl;e6P9ApvzVo6Kb{AyZlp|ogo_74j< zUAXt*On0!=w!xYo{=Rxv1&TE>=vLH$j>PM)x9s5C6VYC|^U8LO-2JNC9n*rNt7GG3 zwJHWqy_c%kL02tR6F2(AjX9^NpMU$(-OOp-)Ci75^ADNaHxH0zhLB;6eI+ZaUT9); zN@U(B=(bWUMbr6o){uvj1oN{k&O6+EX8C?>3d6q3KQ4#cP3#L-IlJy|<7Sq(+ROCE zQ*16LCV0v4*9#7Mxr(_A-@Us0il~i#YUp@dbitkcxElRtt*vI?H>8@X*tSxvMV)`9 z#68$gXICu1eMVO7Ewb*OJ&44Bs7>EhLxBQ$t42 z`yKYKxgvhO>JB|z@?>v-Y5CZ}Ta#O7=>3kj_tsE3v$S8E>3pc>y<%9}+>FsW7q1YK zrChQ$y~YsRNM60BcdF?Y>)Kz`n$@^G-}UA8>(5+s)sH{sN7Jlj>ouKdA@w%#-}IX} z*PD42i&d05{NzUDt}*SvvbxU!73Vi$N3wP^ktvgqc}Z`DnWf1wRum67(D<$`wev(x zo?e>X@N8cPZv=^rt2p^Y>S=rKRp_IV%9_(jQZ(YPbqBK^S386xw?CdeJ-pu9a@s0i z)m3*#^t$fMH6JuHefbcj_JFM?kwy$(_=_^=g8GBeM zrI(AIW!rqI>v`Fm-aw+B%BSb~2R?rHwmsz%ad*q4eUNj6*6D(@!1u|zAur6-*LO)E zvN7r^O4mZ0-%`TAZ|b!A@T+}lTD*AG?VnC(nQ)Lvd{0`!mdzl(aZLln$u?`F!-YUR z*(!G;>g#O$YRrC@Vcz#g*|{r%4JFx6%Iy@i=h*BvU~yc#_61d?eSm#cfc&8+JvL`k z@9*+!C?c5Z%+Yf0CCMcPN`dZY+!DeBET+-|#4Whxe33#Rg7RDe zA3mh6wpA52Rp#hoxtkh*Sf9~&a>cUiUMuAnPm5y`xSBv&|mz zPO&<0y~XDw{i8R5ro?@jU)+;7B;}&$h1DAcpT5g_Ma_MG=bK)@r1NCG>mcb&On7^T zIOpxR)&t)N>RC5#w9jO>4HWeqR2tt#_Ho$K$yIr-+Ebfrz?$@=P;-+cV&hjPYh@A| zVlhjI4e^%MyIW8^<9K#q6m(Gb_7G&8RC(x(`jZ0V`!$l=c4BgzRQI+kI*>o*=RIH;b31D^CS*6Y7?w^U=6JPH}!v3bSm`zd-po zwd#fgpY~dTp8_N4g{k(gEGuIpE z9@Vrz3 zRl?srbNkhbAH-E<2WFQ$i-t1U{^FQK#cE5@x_{2GUDyuFeFbNLH=ow8TrEqfk{{G` z66Z=&pkSNyR%GxfNAB1Y{v{Q~?PO$;aBln}`C;8#w)n|htIrdwQZmMiUPg{BXVG2j z$wW(-1dU5N+9H&L7ZHU+aU>f?YNsZ}=pK_2HeS};_4+W^n=l7(*q5coaYZgl>4?(G zWiQHCo@O#NllTiwSi(7oK_ZEd70t;;t351l>~wglsPkIm_t^rDUZbp07M$1(A+wGU z0dK9*Qp3y}J-#E#o0Ih$g_6ZB^4y};HyE25yCoY}NF0$)w=dgqI?YD0%<4tCPs`2v z$Bqwg>Bk4e|BM-3Ve@nATc?x{W=D#X(ceXEYSfR!nN06A7;r?to6$#=ihkwtH;`jL z8dF;ExUSnJvTF*_r`Gm)rRibf7?XuE3zM)*u(c{yTc|A%Pd^Q@;{tY?=7j{K-8~5MZ7~Po|Ta~O55vs5J3f-*H ze0ZYpqlCn_qOSHbj#lL3eXEqdJ~-+jHPj;dge=%b7JOx2vRCR-XpnVwQ-RE*q1aii z7Mcp1%6`M1Fy~k2ixZxIOBV8(I=%b;m5>rs^-I|$Vt!j`JLQ}my{>PR=;mnudgaj? zt7;a3i0jxpv=x`6iJY)z}3;N(MKT7m8+l6+1k=NJw$Zo$EWun)ep`r zSElW!F%b)sNM|X>a9AwC8u%a-3tw(-0 zE8HC*G;{;@h^N!}S(%_=5;b7K$Aka=*ac!~b~I;yw?dV6&WEjcK3}>NTL)rk{#rp( zl*^C;(&65gx(BlTIZdy&OHRp_>OMXGF1^dDWlE?_pj)NbsC!MN3WCj&U(-GfDY*VY z_m`24`2m~Hhud*h3aqO+VzTVWu4xX594urptle7eim_rnEc1xwvxhT=Di?DoyC&nA z{N?P-FPaW%ozG5e$>|X{Gap&r($QjaVTZ}L3u&h}>nEQYJ5Z|oC9)w|L(2%=9yU&h z*`38MH!P7O+tnX0qcHCw85`Ley^PkKXzdR)R81=LhW$l{z zqJ;gy8&{9R?_Uq$i*H81vplh*Dd2tT?R}hgy=T@5bv$3E9G11qt+1Jkg-#( zW#?yp5^qDa+qraQ9yf2fypV*LhMtVF9CB-$gyZDrRW_shqnHSSNnnHpfx{96L87ps zdB%Pr4nSd){)xxHZ~m-bS@2kZbr0PtMcvURurn&c=Gr*@TlvW+e8gA#JR+L8+6(#K zSw$M>*VsxuoJhkISK__cZdv8re3kubPy69bO*whoZQm+Ni(lUoKQ3V}D4nU_fN5zS zS(|$2eTnDxgQ-3Z$AgZZRq9n~vuC$jH>mdR*i>e)%GDLowP~rx5aPBmLI-s|UNDM3 z@Kwad9=iKZ%p@Q##v&;!F#L%z4=GeR&{FaXb_1G!cx9e({uB&v;j69&(nr7Lqw)o?eXx?X9vfMm9g4%$Cnr4&O^#}HLsHjy_=q4(TAV0M;%n^+im|!+Xgamv%qHt zehm2{@o3L!uC5ckB|M%*tA55Q42-nAK#O`+_IW&5J-(w@Gfk(x$f;SDU)b~^S=x(t zQyNCKoTf>Hh7IjckrOGlYq5L@xCmpZh)ZX(@386c#a%IsCr|}`r!?TbFWN|D&HM^#=JStRyllE5l#=~MJc+qmHuWz{6@>WODTTZZol-5(0Q z-TvvOjZ4d!K8iNs;mc<=(L+Dtiid~4{bY4^*&e-^hj?Z|@KVl}HIFy)Ho@O*z9D9{ zvmYsB?%$fXE9(7hvGkT|?hqEYxQL#fb(hkcB3;f5wcL}n@tqcE(Uv9NN^{}O$-XCU zloaBsCgLPfzAoWq>B#pr)ybX*dRldF$a;r`7FqiY$V$JI7=L@_Z z;gSb&Wy!RvmGzig*Bbu*lYKx&M;XOlW-S!rTQWx_Fu;S0B zu(@PDMur|zmrV1RZ<$$I>f0*md+Z|H?zJXUXt2$r(&&2LWGZ+VX-b}NWAJzc5`ISC zgXTu{384E^A@qDIpY>i>IwMsOo&&_V=H%TV;ZAqq&~wl0&L!EYpivk!q=ExGF*Ffu zmC+b17EZE*r!XVasXmNMJXI75mNEGEPiqGP34^l$34$Q^j*%{iKpId$Djl3Y_$O`g zHwl5%_wn@f0Lg!Fw%>LL2lT@`IKbP30wDuDs6PJg5Kaw3hIml@A*4SA0H^N*ExI?= z1GGIsmLT}wClIu0027kt5$HidAU%A%yy*~<;sYKafDv^d0@%ueltg_9X#iFM5y7s@ z7-T3yAcGLO1%kgongPy0YJjO0kQ%sa1tG0LVk1Hgd_}N@NT3550so(TtEpkNlku;Hb_H7E?Zz|+kY1b$v-{|q(P008HPz8wg_r91#sKn*krzgMY?{d%=b>gqW&<9v2eiw$W%{&*GYANr zan~HeFnkScy)pB1SV5TiIjjL+=la<|j4#Dt6kDJrb2KtArE@KJpwDwH4+t!bG4h;+ z1j}MvF)1JhIa&C%oulqftt>>8154UiPG%+nnp0`@03?79V%m{{tO{ zK{E3B_X1ai{tGX#C;yCy)B`#BaFnwFgd+id{&YVdS11sxibkp8l_76BJ%F|miQN6q zJyn0ImjVI_;u810ZXn11@5ch%Jt0@Jn=eR){XGQe%9y~%pP~={Rm2AUjTmsej>4kR zXaZghrKW^J%cD^83V*>1qI!5DfDytXfESqm0V9tGy5kA`LxW=?_yUTnS+Q2&SoKv00R1jG$M!+7SPmCu-AZpf5rpqvbY}}>aQ@P(1xXM8miMMwLb