From e7fc161ff44d45a886b78240c84ccdb0f00feced Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Wed, 8 Jul 2026 20:27:48 -0600 Subject: [PATCH 01/13] Add initial inputdata generator scaffold and core helpers Create a first working generator entrypoint under tools/inputdata with deterministic output writing and strict-mode handling. Add helper modules for workflow parsing, CIME list extraction, and URL normalization so future iterations can refine behavior without changing CLI structure. --- tools/inputdata/__init__.py | 0 ...enerate_inputdata_manifest.cpython-314.pyc | Bin 0 -> 12008 bytes .../inputdata/generate_inputdata_manifest.py | 248 ++++++++++++++++++ tools/inputdata/lib/__init__.py | 0 .../__pycache__/cime_manifest.cpython-314.pyc | Bin 0 -> 5610 bytes .../__pycache__/url_normalize.cpython-314.pyc | Bin 0 -> 3828 bytes .../workflow_parser.cpython-314.pyc | Bin 0 -> 9340 bytes tools/inputdata/lib/cime_manifest.py | 128 +++++++++ tools/inputdata/lib/url_normalize.py | 70 +++++ tools/inputdata/lib/workflow_parser.py | 154 +++++++++++ 10 files changed, 600 insertions(+) create mode 100644 tools/inputdata/__init__.py create mode 100644 tools/inputdata/__pycache__/generate_inputdata_manifest.cpython-314.pyc create mode 100644 tools/inputdata/generate_inputdata_manifest.py create mode 100644 tools/inputdata/lib/__init__.py create mode 100644 tools/inputdata/lib/__pycache__/cime_manifest.cpython-314.pyc create mode 100644 tools/inputdata/lib/__pycache__/url_normalize.cpython-314.pyc create mode 100644 tools/inputdata/lib/__pycache__/workflow_parser.cpython-314.pyc create mode 100644 tools/inputdata/lib/cime_manifest.py create mode 100644 tools/inputdata/lib/url_normalize.py create mode 100644 tools/inputdata/lib/workflow_parser.py diff --git a/tools/inputdata/__init__.py b/tools/inputdata/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/inputdata/__pycache__/generate_inputdata_manifest.cpython-314.pyc b/tools/inputdata/__pycache__/generate_inputdata_manifest.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a55dc1b825a5ff786ae1a66696f13fb82bc610d8 GIT binary patch literal 12008 zcmb_CTWlNGl|yn!4&M~tlw^srUY0FNj^&s9jAJ{NVp~=inR(G>Xp(Z6P$Y9_Xvb`j zu-gTyZrVm}ij9f|w%THW%Ddf-KlUU0wJA`bSfId?py%Bf(PwVjKGh*~OgRL{lv8j{xda!*8>Zb;9>Fu^6}(eE!Pm=N()BSR zW5{#>dWgx*z_Z!Qge-g25@xSdFoHj11-#A6r0qgy$PRD^g}XuyfSnZX4!Hn!Q+Qj* z1F)CEJs}^!ehT-7IsxvYa3ItT@HPtfUD_V%IrtL8B&-b6&0Om5V=ndeF>?ER>pgs3 z6;k&zBfWpXevPO^Q!JCo3Nay-&G0kuY)Hg}SUer$`4P3q!9!9=EhI$_^FlV01QK>O zCcF+McOi$L#nM3O$;AktjHWWVyb#4IqUjVb@S=_6GtoFnLOoF-$qOT_=$=odg(Qi_ zW0^z>8;IthKGB=vqq+Q*bSf^@LMzFWwCK-d$wDlhDnJd{Xd;z~l4Ke>d@U(@0nzT8 z5she^gVsXHc$OqY=aqaaorv-b6b#D%UB6EomR3Qc=&m(9CKq4=(A@vZvp^!#_SOwB5kWPX06ygMJ$N+O;6z#Jl zdo7uXW#Y+tmo>aVL{9FU$Rh7VE*=9OCsIUo#$R_e>gw4_DTlrYJ{>B!C$_Ou{Q&4*r zUIU^@L;&HAQcl7=#RP$*uH>O_(J1K#Cfq|_iDN=Go94%<3NhcSBzEFyNOxJHAG9=**LsZ5~?=Z&tE~U zMJA%Wn60Y~1pxr46c8_2#pGaAvvt+jOchdxG$Hmt{rld>QE9hCrP~sfeoIu`mZ%I{ zqB3rY%CsdabI201K9OIxN2v78T$f;3$%0(j3(uUXi+ey3_kb+!d;m1eLg4ho3+Ds! z;^qUO2m>dl&z}y!VvXU70jV=3l6;(`sD>?=g2ChjzYrwZtblroBgwb&DUwVStid2? z#9%rbr!SEbRfY6m2?n!y0W`Amj;cUJA|Wabo<4YjL3oo%#L}Qsn(+2uFsEoHO35K5 zNyP=x$O~BlDw*WbCg@^uv;_r2FbHbrN|sN8rsw2EC0b9AtN8^`SF>m)h-i)_5>a?d zaiURD#qi8yRrHC_^%kJ(jqCw7(<}f0|9i-a%(}&0wG4b_8K}8ly01}Nb;V~lbd1(i zWxZwAyKK45mRRo^yRXI?B~;0`VI1KvgF%& zdydw2NWm=|SC6!I!?b#iq9$vmj;g78HW|yEelVM6Q1Cw7@dSnM+nlv8OaWs1hU-UtS zicYH<-NbBpWCM%Uy|lP}iXx@HH|sM+>-_`$KCGIBvNg@Z=vou#Kh%JVZVRc~X}~Ze zJ2^RZSdC-ClmG{_n+dB!>QRe?!5k=5vpDva1auE+(U2B6sGGZ`7WE*OAi%uHCEo@f zMze2 z)J(PyCciVeH241T;;Aq6rW$8nV!vlzWRd>DH!m!mS~*a>@N0J0pHZxUEs{UQCO|9D z5jA9r9DwdVQKTk;h({Skky5K7s!f_NyEbUKGAEU*2?_tGTygL%T@yc{mHg>)1z)zY ztPxyuG|gg5Evb)s86cnP19iy_X-1o9a3wZLt`1k^9KkVz+pJ4v5-2DJK1+n?o5>qI zIRbCUQOu4(HZ!6nPvgT0%uYfE(!^&oqBfCV$bpYe&fp6aBp#Q?(2(SznCcRQv}6We zP$pyDkCz@vNPc+gqvw9`-17C+w>~+w<~s4|sXMNz(hIMk+`RhdlpFA}_}JAXaYLU( zecfn?>=p1GsXp48>>!U6b);5#QAE5;{U|Zs6_ysOE{c3qY;d}$*zBfk39A&9L3dCQ zS>?h6%ZjBJ6j49@-%{ajNt+|7i~zHMnv6V;ooSv0Hk!^}C1F6*es<7yzR+9OIm_bJ z?^`!D3}<_kr!0`seg!CfdnXpv)Hv?sw32$)6S~*o3wEmE8@r!Nl$(VKA)iYpUscrq zXaLtMT7uC4MYe81E?v6x#FdNk$V;yFSBy(Fu!&h|{P+efPhF?ajR68RO=igH0Ed9Z z;QcC$mi?%lY8*8*V`rY_L04-+Ojy&-Gd6=&*rUm@5oOheS$V2*+Q(>ktxN$vvU=7~ z?^9C|){Zgq_I=jW^r8+%6gvXF0#RvA?fWYE=V+A=)b&ZDA7}FnV_QJvG6C*kTfeD5hJ@EDvmlC&Mbcb$Qh>-OVmKx`%a@)$>w&nQBo(eZq<$`4{xNh#Q`Fnrd`=j2P zyX&J@Z@yadchv@t*8JnOzNhY6^cL%XYYYbCripQO-VZQl$I`(2N0vKFrv4J!-&g{& zsN?YH5*VO;ef%14b>j>qc0GD6md@8Trx^nx;QmK(Z_u#31&bV+4aElA zRJKh9vYO4Yp@H(Q!?U4<68vkOb5jq{qsmlKT0fuzYOLvo@|4xHl0p_(~|}&%E03Q6q)-RWA0n=S53P(HC_`4=+@wrDuMd56K*8X@=X*LHe zZX*kQPt@^}ad5ZL;o}z6KaN{b|9+KolYy+%1L_~kQ!mr!#(;pDCOhO*_I*_SgQ^Fg zul}yd*4CB3y7BHs^(gA2dI~WR`&2dsIZaq&XD+gN^%#>^KYJd`21Mq+UIcBpqA0l87A>2cr#hAjf9Mj(|dorTJq3NMvup zn0VYWf%j^6eq_FB#)7-aK~c32m+iw%Xz6X$K2Wv~Y+Cf&tW*TSThoAp{RAtgCoyQ{ z2}D|wL83F5p_&l&;KP$tk|(sEsH&*xk}Q|A%?QagprOm>uK|ET4GtI2tlQj6iSNI& zs9AUTYG(WQrZ?644(olb(P>=NZ|-1xz4x3(pK;Rx(2{1u#&mQopZxK;ADye&J@3dw z&wqIVOHJ^t6V7M!x<56>N?lVax}X6+`H5C&5tzZFVOhSl(Hw@le68@rX9Ic%QAKx|8r2YK}bjk6gqs~(E88JlEB-@bj= z#c@!#EP74#!}!>(OWN7LWtgy`rF{d`yuYc+Itb3iKLU>}O>4;_&u-2d!DtChVXK0* zg>CTFci}60mK0XY{LgiSI}{F4zxQlE^in^Iu3}3HcgStQ&Xk)4ezvA? zN7K&qWb0doiJX>k?Ps$+|HJm#XXWo2C19r-PcjrT(Btm9>(J=$+3!nk&DnpF%G0W| zU>R{w3p2vdW9ioQY~K3JUS|7&X9^!%YqETenjoGVIV+C_R^T}qZj|z7-iIi0$J3G&N?+uX{3;Tap|D%Twq-9SU6t9SSs^aRw6EBp-l#DFJ%c zawq`KJBv}Q!bF`olMT%0)9E1GVo0U~c;v|gtw7)yGMSJLS>^Z;UvLb*92{H-4kqRX zrw$FCKQ#ElE23S#lmO?DLNqVL3%*c3lYs-AK+|=VzC=7C)ZXpVR7f*y{rf1F^dbrXU3AQ9A1lAeRGhKjjK`8&9jHBig|A7(tHlf$K?<4CJ6S*a#JzFI+r- z{zT|y+9tjA!yheJr6)cgk3-=9Xu%>qOT9l@FiDRX`8!(Z2sFj|AU4^GqEsXMq+?+{ zGW#9KLBR(AbF@5D}3Be$3-dC=x*u)T#v47mfTAbm-}{i~9O7MJ=?FC89yTxWf|z z5~VSF9ka`rp@t-{VTQhRQ}h+pglKPK1{HLIc3ITIVd4S|1*9HguA(GEQ4mAX)${?C zEQ+-@?X!`}090@s2_QXPl=i(d*#0%Fo|ESmnMDu$HR)t#q*G+^n4U7cmCt4>6%dX^ znG?0Ne&`o+7iUbgQLxbs(Nbp<1k&Cr?NT0KAvfnr*#Rpw+p21+9h z5IqNe;2!*jv>&%Maey}MZbCG50IyM9fs0OwrY$-FPlh2MV77$WO{`oma|44KG%DRT zlQ#V9LM)Xbe-D(4@aO*zOg2QLP44%P6iDRb?h!Xc2^vGif8YdTvbz7 z+0<1vZTrl$4Gx-0_KDlmw-1-B$A1z3&ajB|HsaM-+70-h;-)O6?4TLES_Eu?D(nqC+45M^l{`?q!Kt(4V)|oPL@u+ zR0+IXJY6z&m)UJ!c5eHd!Qymn@^JCOU6;FfzJ}RdZ-3Rh_l|cj1l&1u$u_)ds_r;g z-f^%rJXx|F`sCEVo%`2wYuxmDN1*2Et$F&Yo}sd52w`7;%|BRk2deIovU>!=><;6i zZo|RYJs-UL!Mls^*3ACp=kJ(%7SFGnJ8F(Sw+~k*PL?N5miCFP zWWNY!0Uy0`^OXl$4a7Y-#^S!qxtBZcaGi^myFCNdp0Tx_vBjBnZ`bln#k*^9YTYxs z`clQSfAP$3{DCDcK3@DddMo-#y5c#zr1`?z`MImRX7_(&xoKIBui3ZP?7da{&a!>y z>cnkz#XeEBA1d1qt-A(leWTUBedWG=wQW19+k)k7!CK$YPka8Zr#AXjb@X(3^fXY^ zMkZlwZVwzGbU5yrn9jcYdZu@La~I>=w=t_?ptGtqy`N6c)4$+FO=LnAu!$5yZm0Ia~v+4)O^9JZ=&p*__c4}cg%~L z#TRQ_SCtzob3-fq>N6E?yvpq_bNg>!|220UPAdKT@9UTjw<5hDv|YU`Y&9@h4op_M z4wYEny4$}zQ0)wsJA)PXSaGIVc#G3tvd$kqyF6KO4X)f+W1m{5kH;&nU8UfOHTIcx z*0yHvFAW^1*bja(w#H5aVoAFMq8ghg9Ec-vaTyhP-<6zf%SI_c0 z)!rlJ-XoQ+qovMcfIUvJ?^U|SOP#wnEIPxU`x;KGgJ7N6LgQ9=`w|sHG&(YX_b=!3 zaPbhY528m;aFb4OFGBtTGPwN(;d^=u7lMi`qR|{lmI!WK2(PD8SLk7ns1t7FzyuQx zKkK76FUOR7XJbG%h8~4--%_$aXsOy0gSKoOa!98oW^7qGG73syrc%p%mtRWg5sx!$pfJEa9H+Zmbr{7LVQA5f?NxgDMq^qmP z$Ug(63jX{uWE&clO7)-2_TMqK-!P`%Fvj09eP1xczh!*CWx9UPT>MI{RjKd6H6`^| zW=0iLm6_Kzb&T4yG_a=fex=r_IyV@|?r9jcZv!76IN%EBgY9sy>46<@r93dgoq`8O yIN^RU21j5I2H=q6!9k2tZd}$dta-7prtyDnblsf$FkCTqm)LHQMYfan{Qm)}QG&$) literal 0 HcmV?d00001 diff --git a/tools/inputdata/generate_inputdata_manifest.py b/tools/inputdata/generate_inputdata_manifest.py new file mode 100644 index 0000000..5838e57 --- /dev/null +++ b/tools/inputdata/generate_inputdata_manifest.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import sys +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +from tools.inputdata.lib.cime_manifest import parse_input_data_lists, run_create_test +from tools.inputdata.lib.url_normalize import ( + filter_candidate_path, + is_public_inputdata_url, + normalize_to_din_relative, + to_public_inputdata_url, +) +from tools.inputdata.lib.workflow_parser import TestRecord, build_state_payload, parse_workflows + + +@dataclass(frozen=True) +class ProvenanceRecord: + workflow: str + test: str + case_dir: str + component_list_file: str + manifest: str + discovery_method: str + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Generate inputdata manifests from E3SM workflows and CIME metadata" + ) + parser.add_argument("--e3sm-root", required=True) + parser.add_argument("--din-loc-root", required=True) + parser.add_argument("--test-root", required=True) + parser.add_argument("--output-root", required=True) + parser.add_argument("--state-out", required=True) + parser.add_argument("--files-out", required=True) + parser.add_argument("--standalone-files-out", required=True) + parser.add_argument("--provenance-out", required=True) + parser.add_argument("--strict", action="store_true") + parser.add_argument("--verbose", action="store_true") + return parser.parse_args() + + +def _write_manifest(urls: list[str], path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + text = "\n".join(urls) + if text: + text += "\n" + path.write_text(text, encoding="utf-8") + + +def _write_json(payload: dict, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _log(message: str, verbose: bool) -> None: + if verbose: + print(message) + + +def _standalone_from_workflow_records(records: list[TestRecord]) -> tuple[list[str], dict[str, list[ProvenanceRecord]]]: + urls: set[str] = set() + provenance: dict[str, list[ProvenanceRecord]] = {} + + for record in records: + # Conservative seed entry for v1 until richer standalone parsing is added. + marker_path = f"atm/scream/standalone/{record.name}.txt" + url = to_public_inputdata_url(marker_path) + urls.add(url) + provenance.setdefault(url, []).append( + ProvenanceRecord( + workflow=record.workflow, + test=record.name, + case_dir="", + component_list_file="workflow-standalone-ref", + manifest="files-standalone.txt", + discovery_method="standalone-ref", + ) + ) + + return sorted(urls), provenance + + +def _normalize_and_collect( + raw_value: str, + din_loc_root: Path, +) -> Optional[str]: + if not filter_candidate_path(raw_value): + return None + + relative = normalize_to_din_relative(raw_value, din_loc_root) + if not relative: + return None + + url = to_public_inputdata_url(relative) + if not is_public_inputdata_url(url): + return None + + return url + + +def _provenance_payload(data: dict[str, list[ProvenanceRecord]]) -> dict[str, list[dict]]: + payload: dict[str, list[dict]] = {} + for url in sorted(data): + entries = sorted( + data[url], + key=lambda rec: ( + rec.workflow, + rec.test, + rec.component_list_file, + rec.case_dir, + rec.manifest, + rec.discovery_method, + ), + ) + payload[url] = [ + { + "workflow": rec.workflow, + "test": rec.test, + "case_dir": rec.case_dir, + "component_list_file": rec.component_list_file, + "manifest": rec.manifest, + "discovery_method": rec.discovery_method, + } + for rec in entries + ] + return payload + + +def main() -> int: + args = parse_args() + + e3sm_root = Path(args.e3sm_root).resolve() + din_loc_root = Path(args.din_loc_root).resolve() + test_root = Path(args.test_root).resolve() + output_root = Path(args.output_root).resolve() + + if not e3sm_root.exists(): + print(f"ERROR: --e3sm-root does not exist: {e3sm_root}", file=sys.stderr) + return 2 + + parse_result = parse_workflows(e3sm_root) + if not parse_result.full_model_tests: + print("ERROR: no full-model tests extracted from workflows", file=sys.stderr) + return 3 + + state = build_state_payload(parse_result) + state["generated_at_utc"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + full_urls: set[str] = set() + full_provenance: dict[str, list[ProvenanceRecord]] = {} + + success_count = 0 + failure_count = 0 + + for test_record in parse_result.full_model_tests: + _log(f"Running create_test for {test_record.name}", args.verbose) + result = run_create_test( + e3sm_root=e3sm_root, + test_name=test_record.name, + test_root=test_root, + output_root=output_root, + din_loc_root=din_loc_root, + workflow=test_record.workflow, + ) + + if not result.success: + failure_count += 1 + print( + f"WARNING: create_test failed for {test_record.name}: {result.error}", + file=sys.stderr, + ) + continue + + success_count += 1 + + for case_dir_value in result.case_dirs: + case_dir = Path(case_dir_value) + entries = parse_input_data_lists(case_dir) + for entry in entries: + url = _normalize_and_collect(entry.raw_value, din_loc_root) + if not url: + continue + full_urls.add(url) + full_provenance.setdefault(url, []).append( + ProvenanceRecord( + workflow=test_record.workflow, + test=test_record.name, + case_dir=entry.case_dir, + component_list_file=entry.component_list_file, + manifest="files.txt", + discovery_method="cime-buildconf", + ) + ) + + if success_count == 0: + print("ERROR: CIME case generation failed for all tests", file=sys.stderr) + return 4 + + if not full_urls: + print("ERROR: resulting full-model manifest would be empty", file=sys.stderr) + return 5 + + standalone_urls, standalone_provenance = _standalone_from_workflow_records( + parse_result.standalone_tests + ) + + combined_provenance = dict(full_provenance) + for url, entries in standalone_provenance.items(): + combined_provenance.setdefault(url, []).extend(entries) + + files_out = Path(args.files_out) + standalone_files_out = Path(args.standalone_files_out) + provenance_out = Path(args.provenance_out) + state_out = Path(args.state_out) + + _write_manifest(sorted(full_urls), files_out) + _write_manifest(sorted(standalone_urls), standalone_files_out) + _write_json(_provenance_payload(combined_provenance), provenance_out) + _write_json(state, state_out) + + if parse_result.warnings: + for warning in parse_result.warnings: + print(f"WARNING: {warning}", file=sys.stderr) + if args.strict: + print("ERROR: strict mode enabled and parser warnings were present", file=sys.stderr) + return 6 + + print( + "SUMMARY " + f"full_tests={len(parse_result.full_model_tests)} " + f"full_success={success_count} " + f"full_failed={failure_count} " + f"full_urls={len(full_urls)} " + f"standalone_refs={len(parse_result.standalone_tests)} " + f"standalone_urls={len(standalone_urls)}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/inputdata/lib/__init__.py b/tools/inputdata/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/inputdata/lib/__pycache__/cime_manifest.cpython-314.pyc b/tools/inputdata/lib/__pycache__/cime_manifest.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..307c53f46d0fdc000a40946711c1ca04dac02f42 GIT binary patch literal 5610 zcmb7IU2GFq7QSPT$77HGlh_IQaUdbZfCFvmQebKM2|w8s6Q-$GRUso|Pr%gJ-aF%_ z$x4-_R;p=N0<^0ol~&tG`$GFtkXG8hcAxsRO4dMTr$I&PQ{SA07PT+zId^Qw32Art zTE6$(-#Pc(^PO|<-Qn}P38W{&=}S>BA>Ux7ksKDmJ^Tj{Wim-*Zj_wlbS{dtZPcdQ zSZUYoP})cNlMda%`uI`dq*He?+A%7gbm^`rIp^3+WI=ZB<*ao(2-L+0kzIRiR?8l% zAi8_Qv2GYEJvCO6J<#rrkX%6b$UdO`jP}X_po5I|$swTI810vB10>e|Eq*J;nNljB zFX$;fTgYn(sJk*LJ(bR-Gz}{JcuKzrrFgoC15&vOs0dTEFstU%b!J&%w<_GjUjR`i z=ZMb91jOUk?Smk;bG&RDf*wZMn-tF|5SlCiOHR`_nlG00SF)OZEU(iU)2`{%6w@h9 zRWey>wxtV~i-o+J*OeTMQKqsv)pXI+wDMLeS5iMAiI`xDijq%VRu#o`E6U|Urj$e4 zqbR>ArE>KizXEfqrsuM-SOL1Bp+yYP(4i#n4^*7*5&Y7;pp=gO&Z$mk(E{95Kfih9W zoYaq-3t$$+3`IcOFpFZ=ju{Ho5<(cM&PIk4w9Sx#%>1Eu69yBU7p3R#tn!^kceH%0 za&U={TC;It8nIQ!7r})=F}a6`t1?NFN!yF>5t4zn9&(1e$jO{+lkGBy4N3w2PWX#Z zcggNuP7svzEh5u&sn2Vd6g%lk{l5J-wjXwTH!F2ITRkh67jK}eEII%NAWk@=+HAk zWv}ep%Ui>W;@PH@l1*NUuB15bd3w!B^5Qh;UFsb*W_> z)95Hz{&B?Cy3{y>aWpjRQlqcvPP!Xk&A_-GA_vCd>+4g}Ew{@-t|^IAoE$1j>(4Xk zN=j=^a_xD0lPJu@$X-`jQOkP*t)=ghJRRm2TDtI-N|mPhHQkn3 zNf$DzDQJ2ItS>8p8Pfr_I$Jc|S}~Vp&Q3G=ONDGc=Am0qO3aQ8AT@~DR?L_ncLHTP z!QFy0%b3nO6HK(GWPL=@VNW0d$Td{+A(r}CvMiY&SXNr3Ko7w%X3Rza&1F&xkhY$= zsk@=U`I)<+-M0?j4IL_v8B%yj>Rpt2m!#-FrRcK1%Lqq}j$Wf3h)$z@_pR<*-i6Rh zk0c@Bt`WiIeiR}uPi3qsb{b;P5ZetgWQdy#v13`3Dx)9z3^9U~HGnm~QWd)lF;MIB z`i1g|M}5TKbv6G%-VpuE9{)AhhY7=tTXJ?SI=kl1 z+;R39k-i)6Tz_XNa&RGX@Mh$*zEAor`t#?O!Xt~}k!tw(Quxeb_)InYYDM~b*Zzur zb>_;C3}wiZ*~G}8&&7?@TU=_5rujBqyAO4_a2yKy!|7SOlKt!*P` z(ctn;+Zf0NS1j3z&F>WXBtHNxQP!GmpmP)tz>hG7JrAixx2Rh8yV9*B#kj~PB(5QArpfmg#*9zu*V zh|(9J$})DGIjEJ%vM+coHkY{@+A{wJz_RLlwtQka{_JP3fBO2Z=`RQG#!pu}#uwwK z=O_M_`bv_0FeL*~B18@we~(-J-ZX%KhECZgKy{P0@n$Tw`c;T^?G5fW;=aib#Al$s z0EpwS01zBW$Io%UQJ>t5x;E@}G@ye~{U*1?l6^m*<1J@MT7`SZ(-0VdSTq;~C z^}?HKUNePy>0l-J#GIPSs~HVBVnC`x$?Q=e7NDuL4!${`$-<@zNH@dHP&%HW7Pz??(1$Qn6cUFVLmjuOQGG1;BMQhp%*I7ul?bFdBA1*u6nO{uU)t& z_N}-`s0XeNo>2MJs@^w@UsLb<*+zc)PSxsQ5a-s-2~j!Q&hC+G@A~KdtH+@*RqJM7 zeN22@+X`m$6E!RrhL5o7D!(v}rRJSxis2MA8E~}rUbD`$Z-7jhYiMYr1T1KP1$)x| zQ^>+w<=*;Q2RMZ**-Cb$c^^V|0BamWJheETCK90UdFT_8+&I1=tV^r+jkCBvmXghw z9C+H`R*@8Bp}|jM7l0h_0EnEi6IKa#EBNiH^9}u_IF(x>7t($`=_%w~%4j#PlV@&yQg z@!3F&*(uW&3@IsYE&Js=rpn@(_xG^=W+tLX$y z_IhEx83`8wqB)hJp1uJSo`!Q6d@DRVt=oV`v-nk<@n^^Y$b&=W6GotO?!eu^Pj_J95W6@}=-qXWun`xp&{^ zgZFx0zIM!L@4k_^o>=Z2THdr!8=^!vCYo4 w)d*y;b6aN(t1F(>UI*6?aWL0k?*;0+0K9B`H_Jg3`kfE__v39~9T+G70h9=FmH+?% literal 0 HcmV?d00001 diff --git a/tools/inputdata/lib/__pycache__/url_normalize.cpython-314.pyc b/tools/inputdata/lib/__pycache__/url_normalize.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c770073d75bbd0683551e979097e96b3e9b9cbe7 GIT binary patch literal 3828 zcmcIn%~Kmk7VptW8X*k=42lwr&DapIz)0Sew}KOgg@C;#4j&m4)mE+7Qv(`6%P^zu z8JQ2)y0n{Ac_|<8VGmSJ9Ldec9P=kgh^*MDq}E%zd+-ex<#LkKzSq)7Hpa0p+f~#3 zy8Cs%*YCal^_%{1V~{}kkALb9-wA~L8#`|CR0!_y77%%Ii^$wKxyIRCC(@pA&o$oW z8O@J-ula0WCy~9jU-m_b?2i(?*%o90=m4Vwas$voMmNYIphZRp<;E@&3qM9-G0q98 zhGE*OoiPmyy54EkUW7IrnnUodYcvbkA3Gndek z)^Z|aWOH^}wbeNe_Qem27d{@khY}EZk|MV}gI^GmhOrLvDYwYUoa~W#*_-fyPQJ%z z0mpY+)pMG|TQ+t4RI_u`aJ;josegs~`sYo$tlAceIO}xVFmBbQC-ls0B1d(_KrTJA zuEn!!j;JWrQ)-Grn_y)Fd{#SDdGd^OaLrG!+HO6-W~Yty4q=wC(;$JRdK%y17a&~a z@jTP+Nh=(zzF4J?o0GYV)fLCidfeG8m*RS>`z$bP$$S?{annE_Ev`J1zro9aOE6ur zFRqXqvYwl$H!B}-pL#zg=}xqMCrR;1PWD1f1X7-6;_h-)B;-ZO*F%JyCbDm$tpQtp zqWR1#*1Foe>+5=e=7Ot5y_Nm1ULpHa+$FG88tzL6k%elnW((N|5ffbhP-X!l&Cr}C ztt@JWq)J%`VQEe^(isRyP0~$s$&&QUk|r&iX-$`Eu^M-{vkupL^#w1Zg_9Vw=YhD) zodrwf$qcy)A@j$U8y>XpD#^p$L%T4GkS_8oKEuVhNeadzG2ZdSFF1jDsxB;RhV2Nt zxuQ`mZQ<5EQl5sPbs8=&Xoj|$r9;lS`HXIBRH^B#;1S}Nb#qSDt)Y00^CT>^-h%3f z$DmXpI8sauZx23L`FrSeDKY$`g`WPrrHz|$xP-&q9P*(ckD)qV!!$IgbGq*Ma)zbZ zKk-IFjrpCxtZHd`24|^efwEYu6o|!)Wt()(@tZjtF1I77bZ#+oTZ?%eAKcAM*731} zIlOADIRVW`TPqm=4=)X%Sf6b!X@=7>3#VAp(u!*ii$-CZsoEn+!xdvhj(bV=R+`756v2=eq z|C@awTsSefCk(>yfme9PU+_krHIqQ-b8C;8uM`})XWg^@Qty_9kzHW~w-Lm9cmI6% z!CYBr-xb<%2S2QQ@a~=v-5T5R?R5;m*uO$ev>DWU0r>nd0N&;fyMg#Q;FEo_KLHmt zCQ!T}P7pn>Fa)G{az{*O3j0atXIS&iG8?QMrur7ag6D8=9!CGKsOL#}jzc(nAud)m$|4WO z^^qd`UcwRykt5}~Swo&WkL>42mQV4=qOjN92@v;qS%S+^L%Nq`)p*`m7B`I^KT=#= zo(8lzTEHsR*F^RrTDc|bJznorjsMkp*LmJp@8fOslB|Uj6@R?W=CU8ILtuSu(tW}i zpbszrx3yJUvP}T_vGGaelc`Zfo|>8&pqdUD%xz7|8JX1q%U;tpwkS?PCZVlZA$C%H z^4G)N&o53)t~Ynzd4A+iiI|_Fg%Aw0x;m#hL57J8-yB{CSPQ6@k~OW&Dx@e33}Zf; zKnWMHVgSg4R!$hNFJ9dWw}p}01GpzuaPOU3o}$>M!g#Akq4?0STiXmiM(BI1EOr#d zj*@t~EOr;g?(G{TF;*56MKSRZfbe4e%09s3#Ky!{Hv>lK3u0M_7KLcx?Q>-zwkyQ; z!>xZCgZzDtb$(jyZ;j+-XEA+Dcw!)>h-U28Q?mN02 zV650u4xKE9PHv5sL(*#P!R{cAuxQs_F5GN~+LR5UMx2~YVZnn>H zm!1%)Fx!U)vwdjl+5s;sHSOMq?yJ0Yv-&C#@WJy4?E0yT&<|lb*cnnvLVt}kM?Zpw zz0y4u6o_U3OxVxV5o?bDBU2tzCa`uWyg9TnwAcQNhoj%NzgG$k7QBN@^T}Aq2`b8b z4*r(#9|wcS5ipm4A0n1DIDUIAn=uwBA`nxAWmgPlm{%zZW2R&~Rx}G-hLky{YeV!J v%%bBg#H0hB;T}^WF14-}zqWJ9Jf6loLn~Oy_U>w}X(+@k38W4dFJv!x1t|t`U(N zB7>a5wIFU7GAIU?^9s*$K@nJPRE&`GL#DwprHr)|hRlN&#lmpoQ2C%$v9^#g!CS9S zRm@w7Xxb0+VNxod*ru$|rt*nx%!M{pV`VU|MJzv*AGwymx79^N+kS&K*M2P{ihawr z70|Zwv9{G>6_nduBx+Y`#A?86814`qfIAuP6l(#mV|cCT0=%B#bz%eH+ZgT=p98#+ z;q_vR*aTnA_!Z@p2GP(?Jnk=WeI8D=24k_f5>z7bm^=*MmT*uBhN3}Ph7Z9TR3;%e z4=EB2jz^`LTRTr2x%RqW_CW6M>GhllMW&=)IYc7~MeYqzDX2&RMUoZ2+_m$B_ZXtb zul9Fe33kuy>Fx{oyL-CEx;(Gz!))DS?-pHV4oxJZ(LgLXCHduDN8oqW-?n4y#8YO` z8q8GUBD#aoXtxxcnw}nk4Vflrd`60e@`uDSUi;uS4gxYu#)!g+1X!|8;X8m;#{|*P z2{jDk3z)z#U{TQoWGz$6M_@8h3dLzysdzXUl>oN|0o^Ee6GTy7NDv$_LS)x6OThs!L8K?t9t_JWX z`Bi20ch9YsSA2H}z?(x9t5}<@N%y-@T|&C1O{C+2ho1`WK^N3VpOxVPo4_MlfjwFa3rK$1zPWR0~k}Q3VWLj%9D~jV~<8; zoq_LTDwg;5rF}7aD`f)lH zRAij5jpEXgR;>!5$d!I2IKRGQ;^Vx~Z*vk2y1v!(XZGOrmU_OsWYC16*x#SGMTGbsn+2#U z%#`(9k0_JLan+(TlRRVG)pKhqI%9$?I2HBqs!5JhMFNq&9*vKKv`dPIr>!uhT2bf~ z2uG-DoQOmvS;kFvy4@Pl>q7BpR0=5py(!2!G@`gM z*=pur$T_xW9NSaZvyPsut@j^0?>6Q7&Sd(|+#AgHy`0_a`^e^-9aytg|FC^t`oXTO zwJ9w$6^~0Ny8sih<8lM?Y(}HlOcfHqON_^88+G_)UkadKg^OfYt7rozkht8oJOg zXgriZ)&vLY;Hk&tCVsQF3_?sK49DI`T1XQ(dhH}aj`B?;!ht2`L4s!t-6v-(RC=wO znK@M$9OyrV4uGIY(~4S_jJ*#8kC~CZ*6Tfp}5@i>X-~BiMmR zLbVJ`hol5LOCEzJ=_aj}jH~a=i*`lex}*ej`$YSpic$G2Bi6d1%-FD2{amhk=P#>w zu2$5{pL(x-A@ZZ1#jd5-(zfHP)wK_dd`f7h(e2ZS>= zs8#jrJmkNT(GI7022E<{|&{B|wCRO8g8c!zZaqPqg!(ry30xzG!AqTKjkR>Um3g|_vMsR)sq!TQ&Zky{huQ*V$$^dqi=_s^hT>O`Sd;#3IR$KprmuF9{+Fdz&TgKkD zV(*v}*6i&$`>u?A*W&)Heb1cnvnuDuHs`9d;r5w#&ZJCP=Z>87MA~^`@xc2>?;Ksa zne93;XZftEo;7N^eg2*ES?Bhgvn%85T0EI`?#Ve1W}F9?61Lyj(N7Y#ZIECDq%%lJPS9M!)l3pWPBju3?%a};*WnXn`B6k%`vcEWz+|6ka*5;R>z6OW47&^Hz{ z4#do`Nz9^oF$1N|#7v+s0lif_@HUN12MTv(ipQW!6g`J+>d*C$h(nhGr(ZsMHZa_O zZa|kOdLF8&7XgYAy?}t-3Pr%4C?%VxlqouE?D*q=JVi=&=Ip&0d+(BvwIA48O1d-7 z?yR%-iCmxVJChc#F4y>%ZT^ia#`iXu^Ns$N7c00AJ9hy5ZSeYKqz|fYG;AigqTOdO zb#k5MOYK^oJ~FITJcZZq!6=$x10z;2ZBUp`1+Fsq^NJVwNmtNd+TiGJg2|-&pJCnv zz0W3-1Ex(RKK>fFTb~kg8`*0}82tr<2g-jduKbWoPv0hBB?bONBW0P^`AXIyopJ1A)ZnNZ)Eyeu6GK#*5gZH}C+Ib&6KuGDUS%QDBWRkh}-c4n$}E;@5nJ-@8#L494l zJr!E)|H<{#<#bgqsQ$|O)b8|wfqSQxD=#dYFKk#yUFTP9A#e1%``aF6M*-%9Q~bJ@ zede5P_VQWa0fP~6BWZ_>-AJ^Xeentw!iwhrAcMAYruzsMly%I~JCCx@$K8Ci%WQe& z082eBU%|^kcfG(sAbO3Y$d*N4Xp-wCaQ`mc&M<;9y#@)i#(V{Db%ZRze-zoY{7C%PESQ?$yqqwkMC#<*J z$9o-mZ87(i>*5AJ5=7C^NAlO|_1|g~jN!1f#VEqpUQ5xhE9}!}ISfLP_YOfXfxl<& zgPHOr8K9xji|n^4iAG-;DvXjH|5tv;$bu~SF-*o%k(c}!HDk%&s$<}hB2#Zl`uXZJ z=y^#$UNk|}yG%dHk{UrYZ(UR5#iGZnk1Sda<+&H)L*+Al!*REMM{-~8KX=y6qBrh| zcrq4t!%w2_fqfUx_2%P9g?Ov{8)H(L=sv8LQ7I7(h9r*xLYc8pJPh$g)g)0Gr!qYQ z`<>Z)GM05WZfmMsaw7kQ-ag(NX-36JPu@*04H)AgMiFp zgk%#H@e@i*?^pR2P>QN)Q`!g42HgXNOkdyORE{!blMjEGk41x105ZRJ@ zjz#04U{pR1h}PuyVU&E-;qHZ>UHR#id)2?F`*~gZf|$E7nt{J#mzX+&N%&W!TffbH z(J5RTT3&5m6rC8F0PqG@XrYo1p8vNaA0D~)>MvgV`D^LXOS#dp4E()#jV;<^q}Ylg zv(rWd$n3NkfTxB9_|)pt5bPU{E2oiP2Jq0eD~}cK+Yrlt4T8{d)f|)qiMSk@X3-=S z;usKzVu1@jD#h4cMDx+m-(rSLH3bt1DHhhE(fv@Y3K$)ysG?P4M3JUskCj;vwNmHb zeBcpRtJ;|3NZI>^tkx9mWFTM0cEKWFs##yFcEz%?uH(`~oJtsg#wQRNwJ*C^;pIxd zUR7;k4~m2h!A~9qvg4hqYjG%BdvJF6Gi$?IWlP$9HCyRVoBhA)+&O!Gtz*aE49uQg z+tvN|+tWhb>bAz&!O!i@bHb{_m2YeOhcb>sSx4WTd99;oQOS1fpR3B4 zTUTr9Zuh*?v$*4~XTB#}b84>a*L4jGyR&sW=B#UVJ&V&Tb^UXvS8eqEP>qYL@Ppy4mBCef_=9Ri5*t&%bp(-Rwv*t2-Z9O*Rua zpGH&Bx+802`nu5}hz*G*j1GCq^3fsf{HXRCg9ae~8UqfV77+#?lF%6h&vH!Oghn!I zcyvr0=$HUVclNDg{=r3-r7!tev9^U$b%ifMpys%Z6l0ukqT))uTRVB#kTmp=q~Wv& zOt=C3U&B@k5PV*P7s<9Mi3TVy#3YNAz5KA@OqXW*5TW~OnwPN9^TQ_aqf&)=(r5o6UQ8fEF z(E^m+;&ELphjLVPo04dS8>`L7_Zr)QrY-PG7$U?|xhc@|h*n2?!>rXoK+syj>ru(j zc;VReZ7BU_w&^T%WgNGq4p+C2CnM1?do7jVB5*4j4~8E%O@9J|$Vie|GXLtrwm<#7 zRB+MslgLu_(!qaf$kZKWTu5Jq1|wj*E%*i$023ucxJm-S2r6j6CvhczfB^TP);_9h z@!1kz7V01fO@kqLL(*O$_42AsmgLkm>W- zU@uN@Vh)Bx@l=bZ3GPqd!vb8U%7enDe~eg});O$I7FU7q8joG399xi+{uvCxWcuF$ znI-Ec;&kU6doqqaOUH8iyqSI86^D1j$h%Dc;qcC#T<1x-bN=FO|2zKF!Nq;=AHH*V z$#?hS&o2G+Qr3L>V~2O$NQ_Q+cXXK^n2D=t)$ID%b`;{{F4OG!2j!&7@%D}H-&kl_ zI`zSoE%nmh4gdA<(jnB9wa0R`r!%#uv$cb>!>}NWV}5YeTr+<* z=k#Qpo>g;8sxjC4LZ5?_uh zH+W*GU&qh;wqmW3t6wJo^0k0Htbu3dhcN}ydf+P literal 0 HcmV?d00001 diff --git a/tools/inputdata/lib/cime_manifest.py b/tools/inputdata/lib/cime_manifest.py new file mode 100644 index 0000000..ba629d0 --- /dev/null +++ b/tools/inputdata/lib/cime_manifest.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import glob +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + + +@dataclass(frozen=True) +class InputListEntry: + case_dir: str + component_list_file: str + raw_value: str + + +@dataclass(frozen=True) +class CaseResult: + test_name: str + workflow: str + success: bool + case_dirs: list[str] + error: Optional[str] = None + + +def run_create_test( + e3sm_root: Path, + test_name: str, + test_root: Path, + output_root: Path, + din_loc_root: Path, + workflow: str, +) -> CaseResult: + create_test = e3sm_root / "cime" / "scripts" / "create_test" + if not create_test.exists(): + return CaseResult( + test_name=test_name, + workflow=workflow, + success=False, + case_dirs=[], + error=f"Missing create_test script: {create_test}", + ) + + command = [ + str(create_test), + test_name, + "--no-build", + "--no-run", + "--namelists-only", + "--test-root", + str(test_root), + "--output-root", + str(output_root), + "--input-dir", + str(din_loc_root), + "--wait", + ] + + process = subprocess.run( + command, + cwd=str(e3sm_root), + text=True, + capture_output=True, + ) + + case_dirs = discover_case_dirs(test_root) + + if process.returncode != 0: + stderr_tail = (process.stderr or process.stdout or "").strip().splitlines() + tail = "\n".join(stderr_tail[-10:]) if stderr_tail else "create_test failed" + return CaseResult( + test_name=test_name, + workflow=workflow, + success=False, + case_dirs=case_dirs, + error=tail, + ) + + return CaseResult( + test_name=test_name, + workflow=workflow, + success=True, + case_dirs=case_dirs, + ) + + +def discover_case_dirs(test_root: Path) -> list[str]: + pattern = str(test_root / "**" / "Buildconf") + buildconf_dirs = [Path(path) for path in glob.glob(pattern, recursive=True)] + case_dirs = sorted({str(path.parent) for path in buildconf_dirs}) + return case_dirs + + +def _extract_candidate_from_line(line: str) -> list[str]: + line = line.split("#", 1)[0].strip() + if not line: + return [] + + if "=" in line: + _, rhs = line.split("=", 1) + else: + rhs = line + + raw_tokens = rhs.replace(",", " ").split() + tokens: list[str] = [] + for token in raw_tokens: + cleaned = token.strip().strip('"').strip("'") + if not cleaned: + continue + tokens.append(cleaned) + return tokens + + +def parse_input_data_lists(case_dir: Path) -> list[InputListEntry]: + entries: list[InputListEntry] = [] + buildconf = case_dir / "Buildconf" + for list_file in sorted(buildconf.glob("*.input_data_list")): + with list_file.open("r", encoding="utf-8", errors="replace") as handle: + for line in handle: + for token in _extract_candidate_from_line(line): + entries.append( + InputListEntry( + case_dir=str(case_dir), + component_list_file=list_file.name, + raw_value=token, + ) + ) + return entries diff --git a/tools/inputdata/lib/url_normalize.py b/tools/inputdata/lib/url_normalize.py new file mode 100644 index 0000000..4c940f3 --- /dev/null +++ b/tools/inputdata/lib/url_normalize.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Optional + +PUBLIC_INPUTDATA_PREFIX = "https://web.lcrc.anl.gov/public/e3sm/inputdata" + + +def filter_candidate_path(value: str) -> bool: + """Return True when a path candidate looks like model inputdata.""" + token = value.strip().strip('"').strip("'") + if not token: + return False + + lowered = token.lower() + if lowered in {"unset", "none", "null", "na", "n/a"}: + return False + + blocked_fragments = ( + "baseline", + "rest", + "restart", + "history", + "output", + "archive", + ) + if any(fragment in lowered for fragment in blocked_fragments): + return False + + if token.endswith("/"): + return False + + # Inputdata files are file-like paths and almost always include separators. + if "/" not in token and "\\" not in token: + return False + + return True + + +def normalize_to_din_relative(path_value: str, din_loc_root: Path) -> Optional[str]: + """Normalize path-like text to a DIN_LOC_ROOT-relative unix-style path.""" + value = path_value.strip().strip('"').strip("'") + value = value.replace("\\", "/") + if value.startswith("$DIN_LOC_ROOT/"): + rel = value[len("$DIN_LOC_ROOT/") :] + return rel.strip("/") + + if value.startswith("${DIN_LOC_ROOT}/"): + rel = value[len("${DIN_LOC_ROOT}/") :] + return rel.strip("/") + + din_root_posix = din_loc_root.as_posix().rstrip("/") + "/" + if value.startswith(din_root_posix): + rel = value[len(din_root_posix) :] + return rel.strip("/") + + # Allow already-relative references such as atm/cam/inic/file.nc. + if not value.startswith("/"): + return value.strip("/") + + return None + + +def to_public_inputdata_url(relative_path: str) -> str: + rel = relative_path.strip().lstrip("/") + return f"{PUBLIC_INPUTDATA_PREFIX}/{rel}" + + +def is_public_inputdata_url(url: str) -> bool: + return url.startswith(PUBLIC_INPUTDATA_PREFIX + "/") diff --git a/tools/inputdata/lib/workflow_parser.py b/tools/inputdata/lib/workflow_parser.py new file mode 100644 index 0000000..b8869e9 --- /dev/null +++ b/tools/inputdata/lib/workflow_parser.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import hashlib +import json +import re +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + + +CREATE_TEST_PATTERN = re.compile( + r"(?:^|\s)(?:\./)?cime/scripts/create_test\s+(?P[A-Za-z0-9_\-.+]+)|" + r"(?:^|\s)create_test\s+(?P[A-Za-z0-9_\-.+]+)" +) +MATRIX_FULL_NAME_PATTERN = re.compile( + r"full_name\s*:\s*[\"']?(?P[A-Za-z0-9_\-.+]+)[\"']?" +) +STANDALONE_PATTERN = re.compile(r"test-all-eamxx") + + +@dataclass(frozen=True) +class TestRecord: + workflow: str + kind: str + name: str + + +@dataclass(frozen=True) +class WorkflowParseResult: + e3sm_sha: str + workflow_hashes: dict[str, str] + full_model_tests: list[TestRecord] + standalone_tests: list[TestRecord] + warnings: list[str] + + + +def collect_workflow_files(e3sm_root: Path) -> list[Path]: + workflows_dir = e3sm_root / ".github" / "workflows" + files = sorted(workflows_dir.glob("*.yml")) + sorted(workflows_dir.glob("*.yaml")) + return sorted(set(files)) + + +def _hash_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _get_e3sm_sha(e3sm_root: Path) -> str: + try: + output = subprocess.check_output( + ["git", "-C", str(e3sm_root), "rev-parse", "HEAD"], + text=True, + ) + return output.strip() + except Exception: + return "unknown" + + +def _parse_create_test_lines(workflow_rel: str, content: str) -> Iterable[TestRecord]: + seen: set[str] = set() + for line in content.splitlines(): + match = CREATE_TEST_PATTERN.search(line) + if not match: + continue + test_name = match.group("test") or match.group("test2") + if not test_name: + continue + test_name = test_name.strip().strip('"').strip("'") + if test_name not in seen: + seen.add(test_name) + yield TestRecord(workflow=workflow_rel, kind="create_test", name=test_name) + + + +def _parse_matrix_full_names(workflow_rel: str, content: str) -> Iterable[TestRecord]: + seen: set[str] = set() + for line in content.splitlines(): + match = MATRIX_FULL_NAME_PATTERN.search(line) + if not match: + continue + test_name = match.group("test").strip() + if test_name not in seen: + seen.add(test_name) + yield TestRecord(workflow=workflow_rel, kind="matrix_full_name", name=test_name) + + + +def _parse_standalone(workflow_rel: str, content: str) -> Iterable[TestRecord]: + lines = content.splitlines() + for idx, line in enumerate(lines): + if not STANDALONE_PATTERN.search(line): + continue + name = f"standalone_ref_line_{idx + 1}" + yield TestRecord(workflow=workflow_rel, kind="standalone_ref", name=name) + + + +def parse_workflows(e3sm_root: Path) -> WorkflowParseResult: + workflow_files = collect_workflow_files(e3sm_root) + if not workflow_files: + raise FileNotFoundError("No workflow YAML files found under E3SM/.github/workflows") + + workflow_hashes: dict[str, str] = {} + full_records: set[TestRecord] = set() + standalone_records: set[TestRecord] = set() + warnings: list[str] = [] + + for path in workflow_files: + relative = path.relative_to(e3sm_root).as_posix() + workflow_hashes[relative] = _hash_file(path) + content = path.read_text(encoding="utf-8", errors="replace") + + full_before = len(full_records) + for record in _parse_create_test_lines(relative, content): + full_records.add(record) + for record in _parse_matrix_full_names(relative, content): + full_records.add(record) + + for record in _parse_standalone(relative, content): + standalone_records.add(record) + + if len(full_records) == full_before and "create_test" in content: + warnings.append(f"Could not confidently parse create_test syntax in {relative}") + + return WorkflowParseResult( + e3sm_sha=_get_e3sm_sha(e3sm_root), + workflow_hashes=dict(sorted(workflow_hashes.items())), + full_model_tests=sorted(full_records, key=lambda r: (r.workflow, r.kind, r.name)), + standalone_tests=sorted(standalone_records, key=lambda r: (r.workflow, r.kind, r.name)), + warnings=sorted(set(warnings)), + ) + + +def build_state_payload(parse_result: WorkflowParseResult, e3sm_ref: str = "master") -> dict: + extracted_tests = [ + {"workflow": rec.workflow, "kind": rec.kind, "name": rec.name} + for rec in parse_result.full_model_tests + parse_result.standalone_tests + ] + extracted_tests = sorted( + extracted_tests, + key=lambda r: (r["workflow"], r["kind"], r["name"]), + ) + + encoded = json.dumps(extracted_tests, sort_keys=True, separators=(",", ":")) + extracted_hash = hashlib.sha256(encoded.encode("utf-8")).hexdigest() + + return { + "e3sm_ref": e3sm_ref, + "e3sm_sha": parse_result.e3sm_sha, + "workflow_files_sha256": parse_result.workflow_hashes, + "extracted_tests_sha256": extracted_hash, + "extracted_tests": extracted_tests, + } From 27514cf15e9058f1f08458f69fe22ee29cdd95db Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Wed, 8 Jul 2026 20:29:47 -0600 Subject: [PATCH 02/13] Remove Python bytecode artifacts and add ignore rules Clean up accidental py_compile outputs from the initial generator commit and prevent recurrence by adding standard Python cache ignore patterns. --- .gitignore | 2 ++ .../generate_inputdata_manifest.cpython-314.pyc | Bin 12008 -> 0 bytes .../__pycache__/cime_manifest.cpython-314.pyc | Bin 5610 -> 0 bytes .../__pycache__/url_normalize.cpython-314.pyc | Bin 3828 -> 0 bytes .../__pycache__/workflow_parser.cpython-314.pyc | Bin 9340 -> 0 bytes 5 files changed, 2 insertions(+) create mode 100644 .gitignore delete mode 100644 tools/inputdata/__pycache__/generate_inputdata_manifest.cpython-314.pyc delete mode 100644 tools/inputdata/lib/__pycache__/cime_manifest.cpython-314.pyc delete mode 100644 tools/inputdata/lib/__pycache__/url_normalize.cpython-314.pyc delete mode 100644 tools/inputdata/lib/__pycache__/workflow_parser.cpython-314.pyc diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7a60b85 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/tools/inputdata/__pycache__/generate_inputdata_manifest.cpython-314.pyc b/tools/inputdata/__pycache__/generate_inputdata_manifest.cpython-314.pyc deleted file mode 100644 index a55dc1b825a5ff786ae1a66696f13fb82bc610d8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12008 zcmb_CTWlNGl|yn!4&M~tlw^srUY0FNj^&s9jAJ{NVp~=inR(G>Xp(Z6P$Y9_Xvb`j zu-gTyZrVm}ij9f|w%THW%Ddf-KlUU0wJA`bSfId?py%Bf(PwVjKGh*~OgRL{lv8j{xda!*8>Zb;9>Fu^6}(eE!Pm=N()BSR zW5{#>dWgx*z_Z!Qge-g25@xSdFoHj11-#A6r0qgy$PRD^g}XuyfSnZX4!Hn!Q+Qj* z1F)CEJs}^!ehT-7IsxvYa3ItT@HPtfUD_V%IrtL8B&-b6&0Om5V=ndeF>?ER>pgs3 z6;k&zBfWpXevPO^Q!JCo3Nay-&G0kuY)Hg}SUer$`4P3q!9!9=EhI$_^FlV01QK>O zCcF+McOi$L#nM3O$;AktjHWWVyb#4IqUjVb@S=_6GtoFnLOoF-$qOT_=$=odg(Qi_ zW0^z>8;IthKGB=vqq+Q*bSf^@LMzFWwCK-d$wDlhDnJd{Xd;z~l4Ke>d@U(@0nzT8 z5she^gVsXHc$OqY=aqaaorv-b6b#D%UB6EomR3Qc=&m(9CKq4=(A@vZvp^!#_SOwB5kWPX06ygMJ$N+O;6z#Jl zdo7uXW#Y+tmo>aVL{9FU$Rh7VE*=9OCsIUo#$R_e>gw4_DTlrYJ{>B!C$_Ou{Q&4*r zUIU^@L;&HAQcl7=#RP$*uH>O_(J1K#Cfq|_iDN=Go94%<3NhcSBzEFyNOxJHAG9=**LsZ5~?=Z&tE~U zMJA%Wn60Y~1pxr46c8_2#pGaAvvt+jOchdxG$Hmt{rld>QE9hCrP~sfeoIu`mZ%I{ zqB3rY%CsdabI201K9OIxN2v78T$f;3$%0(j3(uUXi+ey3_kb+!d;m1eLg4ho3+Ds! z;^qUO2m>dl&z}y!VvXU70jV=3l6;(`sD>?=g2ChjzYrwZtblroBgwb&DUwVStid2? z#9%rbr!SEbRfY6m2?n!y0W`Amj;cUJA|Wabo<4YjL3oo%#L}Qsn(+2uFsEoHO35K5 zNyP=x$O~BlDw*WbCg@^uv;_r2FbHbrN|sN8rsw2EC0b9AtN8^`SF>m)h-i)_5>a?d zaiURD#qi8yRrHC_^%kJ(jqCw7(<}f0|9i-a%(}&0wG4b_8K}8ly01}Nb;V~lbd1(i zWxZwAyKK45mRRo^yRXI?B~;0`VI1KvgF%& zdydw2NWm=|SC6!I!?b#iq9$vmj;g78HW|yEelVM6Q1Cw7@dSnM+nlv8OaWs1hU-UtS zicYH<-NbBpWCM%Uy|lP}iXx@HH|sM+>-_`$KCGIBvNg@Z=vou#Kh%JVZVRc~X}~Ze zJ2^RZSdC-ClmG{_n+dB!>QRe?!5k=5vpDva1auE+(U2B6sGGZ`7WE*OAi%uHCEo@f zMze2 z)J(PyCciVeH241T;;Aq6rW$8nV!vlzWRd>DH!m!mS~*a>@N0J0pHZxUEs{UQCO|9D z5jA9r9DwdVQKTk;h({Skky5K7s!f_NyEbUKGAEU*2?_tGTygL%T@yc{mHg>)1z)zY ztPxyuG|gg5Evb)s86cnP19iy_X-1o9a3wZLt`1k^9KkVz+pJ4v5-2DJK1+n?o5>qI zIRbCUQOu4(HZ!6nPvgT0%uYfE(!^&oqBfCV$bpYe&fp6aBp#Q?(2(SznCcRQv}6We zP$pyDkCz@vNPc+gqvw9`-17C+w>~+w<~s4|sXMNz(hIMk+`RhdlpFA}_}JAXaYLU( zecfn?>=p1GsXp48>>!U6b);5#QAE5;{U|Zs6_ysOE{c3qY;d}$*zBfk39A&9L3dCQ zS>?h6%ZjBJ6j49@-%{ajNt+|7i~zHMnv6V;ooSv0Hk!^}C1F6*es<7yzR+9OIm_bJ z?^`!D3}<_kr!0`seg!CfdnXpv)Hv?sw32$)6S~*o3wEmE8@r!Nl$(VKA)iYpUscrq zXaLtMT7uC4MYe81E?v6x#FdNk$V;yFSBy(Fu!&h|{P+efPhF?ajR68RO=igH0Ed9Z z;QcC$mi?%lY8*8*V`rY_L04-+Ojy&-Gd6=&*rUm@5oOheS$V2*+Q(>ktxN$vvU=7~ z?^9C|){Zgq_I=jW^r8+%6gvXF0#RvA?fWYE=V+A=)b&ZDA7}FnV_QJvG6C*kTfeD5hJ@EDvmlC&Mbcb$Qh>-OVmKx`%a@)$>w&nQBo(eZq<$`4{xNh#Q`Fnrd`=j2P zyX&J@Z@yadchv@t*8JnOzNhY6^cL%XYYYbCripQO-VZQl$I`(2N0vKFrv4J!-&g{& zsN?YH5*VO;ef%14b>j>qc0GD6md@8Trx^nx;QmK(Z_u#31&bV+4aElA zRJKh9vYO4Yp@H(Q!?U4<68vkOb5jq{qsmlKT0fuzYOLvo@|4xHl0p_(~|}&%E03Q6q)-RWA0n=S53P(HC_`4=+@wrDuMd56K*8X@=X*LHe zZX*kQPt@^}ad5ZL;o}z6KaN{b|9+KolYy+%1L_~kQ!mr!#(;pDCOhO*_I*_SgQ^Fg zul}yd*4CB3y7BHs^(gA2dI~WR`&2dsIZaq&XD+gN^%#>^KYJd`21Mq+UIcBpqA0l87A>2cr#hAjf9Mj(|dorTJq3NMvup zn0VYWf%j^6eq_FB#)7-aK~c32m+iw%Xz6X$K2Wv~Y+Cf&tW*TSThoAp{RAtgCoyQ{ z2}D|wL83F5p_&l&;KP$tk|(sEsH&*xk}Q|A%?QagprOm>uK|ET4GtI2tlQj6iSNI& zs9AUTYG(WQrZ?644(olb(P>=NZ|-1xz4x3(pK;Rx(2{1u#&mQopZxK;ADye&J@3dw z&wqIVOHJ^t6V7M!x<56>N?lVax}X6+`H5C&5tzZFVOhSl(Hw@le68@rX9Ic%QAKx|8r2YK}bjk6gqs~(E88JlEB-@bj= z#c@!#EP74#!}!>(OWN7LWtgy`rF{d`yuYc+Itb3iKLU>}O>4;_&u-2d!DtChVXK0* zg>CTFci}60mK0XY{LgiSI}{F4zxQlE^in^Iu3}3HcgStQ&Xk)4ezvA? zN7K&qWb0doiJX>k?Ps$+|HJm#XXWo2C19r-PcjrT(Btm9>(J=$+3!nk&DnpF%G0W| zU>R{w3p2vdW9ioQY~K3JUS|7&X9^!%YqETenjoGVIV+C_R^T}qZj|z7-iIi0$J3G&N?+uX{3;Tap|D%Twq-9SU6t9SSs^aRw6EBp-l#DFJ%c zawq`KJBv}Q!bF`olMT%0)9E1GVo0U~c;v|gtw7)yGMSJLS>^Z;UvLb*92{H-4kqRX zrw$FCKQ#ElE23S#lmO?DLNqVL3%*c3lYs-AK+|=VzC=7C)ZXpVR7f*y{rf1F^dbrXU3AQ9A1lAeRGhKjjK`8&9jHBig|A7(tHlf$K?<4CJ6S*a#JzFI+r- z{zT|y+9tjA!yheJr6)cgk3-=9Xu%>qOT9l@FiDRX`8!(Z2sFj|AU4^GqEsXMq+?+{ zGW#9KLBR(AbF@5D}3Be$3-dC=x*u)T#v47mfTAbm-}{i~9O7MJ=?FC89yTxWf|z z5~VSF9ka`rp@t-{VTQhRQ}h+pglKPK1{HLIc3ITIVd4S|1*9HguA(GEQ4mAX)${?C zEQ+-@?X!`}090@s2_QXPl=i(d*#0%Fo|ESmnMDu$HR)t#q*G+^n4U7cmCt4>6%dX^ znG?0Ne&`o+7iUbgQLxbs(Nbp<1k&Cr?NT0KAvfnr*#Rpw+p21+9h z5IqNe;2!*jv>&%Maey}MZbCG50IyM9fs0OwrY$-FPlh2MV77$WO{`oma|44KG%DRT zlQ#V9LM)Xbe-D(4@aO*zOg2QLP44%P6iDRb?h!Xc2^vGif8YdTvbz7 z+0<1vZTrl$4Gx-0_KDlmw-1-B$A1z3&ajB|HsaM-+70-h;-)O6?4TLES_Eu?D(nqC+45M^l{`?q!Kt(4V)|oPL@u+ zR0+IXJY6z&m)UJ!c5eHd!Qymn@^JCOU6;FfzJ}RdZ-3Rh_l|cj1l&1u$u_)ds_r;g z-f^%rJXx|F`sCEVo%`2wYuxmDN1*2Et$F&Yo}sd52w`7;%|BRk2deIovU>!=><;6i zZo|RYJs-UL!Mls^*3ACp=kJ(%7SFGnJ8F(Sw+~k*PL?N5miCFP zWWNY!0Uy0`^OXl$4a7Y-#^S!qxtBZcaGi^myFCNdp0Tx_vBjBnZ`bln#k*^9YTYxs z`clQSfAP$3{DCDcK3@DddMo-#y5c#zr1`?z`MImRX7_(&xoKIBui3ZP?7da{&a!>y z>cnkz#XeEBA1d1qt-A(leWTUBedWG=wQW19+k)k7!CK$YPka8Zr#AXjb@X(3^fXY^ zMkZlwZVwzGbU5yrn9jcYdZu@La~I>=w=t_?ptGtqy`N6c)4$+FO=LnAu!$5yZm0Ia~v+4)O^9JZ=&p*__c4}cg%~L z#TRQ_SCtzob3-fq>N6E?yvpq_bNg>!|220UPAdKT@9UTjw<5hDv|YU`Y&9@h4op_M z4wYEny4$}zQ0)wsJA)PXSaGIVc#G3tvd$kqyF6KO4X)f+W1m{5kH;&nU8UfOHTIcx z*0yHvFAW^1*bja(w#H5aVoAFMq8ghg9Ec-vaTyhP-<6zf%SI_c0 z)!rlJ-XoQ+qovMcfIUvJ?^U|SOP#wnEIPxU`x;KGgJ7N6LgQ9=`w|sHG&(YX_b=!3 zaPbhY528m;aFb4OFGBtTGPwN(;d^=u7lMi`qR|{lmI!WK2(PD8SLk7ns1t7FzyuQx zKkK76FUOR7XJbG%h8~4--%_$aXsOy0gSKoOa!98oW^7qGG73syrc%p%mtRWg5sx!$pfJEa9H+Zmbr{7LVQA5f?NxgDMq^qmP z$Ug(63jX{uWE&clO7)-2_TMqK-!P`%Fvj09eP1xczh!*CWx9UPT>MI{RjKd6H6`^| zW=0iLm6_Kzb&T4yG_a=fex=r_IyV@|?r9jcZv!76IN%EBgY9sy>46<@r93dgoq`8O yIN^RU21j5I2H=q6!9k2tZd}$dta-7prtyDnblsf$FkCTqm)LHQMYfan{Qm)}QG&$) diff --git a/tools/inputdata/lib/__pycache__/cime_manifest.cpython-314.pyc b/tools/inputdata/lib/__pycache__/cime_manifest.cpython-314.pyc deleted file mode 100644 index 307c53f46d0fdc000a40946711c1ca04dac02f42..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5610 zcmb7IU2GFq7QSPT$77HGlh_IQaUdbZfCFvmQebKM2|w8s6Q-$GRUso|Pr%gJ-aF%_ z$x4-_R;p=N0<^0ol~&tG`$GFtkXG8hcAxsRO4dMTr$I&PQ{SA07PT+zId^Qw32Art zTE6$(-#Pc(^PO|<-Qn}P38W{&=}S>BA>Ux7ksKDmJ^Tj{Wim-*Zj_wlbS{dtZPcdQ zSZUYoP})cNlMda%`uI`dq*He?+A%7gbm^`rIp^3+WI=ZB<*ao(2-L+0kzIRiR?8l% zAi8_Qv2GYEJvCO6J<#rrkX%6b$UdO`jP}X_po5I|$swTI810vB10>e|Eq*J;nNljB zFX$;fTgYn(sJk*LJ(bR-Gz}{JcuKzrrFgoC15&vOs0dTEFstU%b!J&%w<_GjUjR`i z=ZMb91jOUk?Smk;bG&RDf*wZMn-tF|5SlCiOHR`_nlG00SF)OZEU(iU)2`{%6w@h9 zRWey>wxtV~i-o+J*OeTMQKqsv)pXI+wDMLeS5iMAiI`xDijq%VRu#o`E6U|Urj$e4 zqbR>ArE>KizXEfqrsuM-SOL1Bp+yYP(4i#n4^*7*5&Y7;pp=gO&Z$mk(E{95Kfih9W zoYaq-3t$$+3`IcOFpFZ=ju{Ho5<(cM&PIk4w9Sx#%>1Eu69yBU7p3R#tn!^kceH%0 za&U={TC;It8nIQ!7r})=F}a6`t1?NFN!yF>5t4zn9&(1e$jO{+lkGBy4N3w2PWX#Z zcggNuP7svzEh5u&sn2Vd6g%lk{l5J-wjXwTH!F2ITRkh67jK}eEII%NAWk@=+HAk zWv}ep%Ui>W;@PH@l1*NUuB15bd3w!B^5Qh;UFsb*W_> z)95Hz{&B?Cy3{y>aWpjRQlqcvPP!Xk&A_-GA_vCd>+4g}Ew{@-t|^IAoE$1j>(4Xk zN=j=^a_xD0lPJu@$X-`jQOkP*t)=ghJRRm2TDtI-N|mPhHQkn3 zNf$DzDQJ2ItS>8p8Pfr_I$Jc|S}~Vp&Q3G=ONDGc=Am0qO3aQ8AT@~DR?L_ncLHTP z!QFy0%b3nO6HK(GWPL=@VNW0d$Td{+A(r}CvMiY&SXNr3Ko7w%X3Rza&1F&xkhY$= zsk@=U`I)<+-M0?j4IL_v8B%yj>Rpt2m!#-FrRcK1%Lqq}j$Wf3h)$z@_pR<*-i6Rh zk0c@Bt`WiIeiR}uPi3qsb{b;P5ZetgWQdy#v13`3Dx)9z3^9U~HGnm~QWd)lF;MIB z`i1g|M}5TKbv6G%-VpuE9{)AhhY7=tTXJ?SI=kl1 z+;R39k-i)6Tz_XNa&RGX@Mh$*zEAor`t#?O!Xt~}k!tw(Quxeb_)InYYDM~b*Zzur zb>_;C3}wiZ*~G}8&&7?@TU=_5rujBqyAO4_a2yKy!|7SOlKt!*P` z(ctn;+Zf0NS1j3z&F>WXBtHNxQP!GmpmP)tz>hG7JrAixx2Rh8yV9*B#kj~PB(5QArpfmg#*9zu*V zh|(9J$})DGIjEJ%vM+coHkY{@+A{wJz_RLlwtQka{_JP3fBO2Z=`RQG#!pu}#uwwK z=O_M_`bv_0FeL*~B18@we~(-J-ZX%KhECZgKy{P0@n$Tw`c;T^?G5fW;=aib#Al$s z0EpwS01zBW$Io%UQJ>t5x;E@}G@ye~{U*1?l6^m*<1J@MT7`SZ(-0VdSTq;~C z^}?HKUNePy>0l-J#GIPSs~HVBVnC`x$?Q=e7NDuL4!${`$-<@zNH@dHP&%HW7Pz??(1$Qn6cUFVLmjuOQGG1;BMQhp%*I7ul?bFdBA1*u6nO{uU)t& z_N}-`s0XeNo>2MJs@^w@UsLb<*+zc)PSxsQ5a-s-2~j!Q&hC+G@A~KdtH+@*RqJM7 zeN22@+X`m$6E!RrhL5o7D!(v}rRJSxis2MA8E~}rUbD`$Z-7jhYiMYr1T1KP1$)x| zQ^>+w<=*;Q2RMZ**-Cb$c^^V|0BamWJheETCK90UdFT_8+&I1=tV^r+jkCBvmXghw z9C+H`R*@8Bp}|jM7l0h_0EnEi6IKa#EBNiH^9}u_IF(x>7t($`=_%w~%4j#PlV@&yQg z@!3F&*(uW&3@IsYE&Js=rpn@(_xG^=W+tLX$y z_IhEx83`8wqB)hJp1uJSo`!Q6d@DRVt=oV`v-nk<@n^^Y$b&=W6GotO?!eu^Pj_J95W6@}=-qXWun`xp&{^ zgZFx0zIM!L@4k_^o>=Z2THdr!8=^!vCYo4 w)d*y;b6aN(t1F(>UI*6?aWL0k?*;0+0K9B`H_Jg3`kfE__v39~9T+G70h9=FmH+?% diff --git a/tools/inputdata/lib/__pycache__/url_normalize.cpython-314.pyc b/tools/inputdata/lib/__pycache__/url_normalize.cpython-314.pyc deleted file mode 100644 index c770073d75bbd0683551e979097e96b3e9b9cbe7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3828 zcmcIn%~Kmk7VptW8X*k=42lwr&DapIz)0Sew}KOgg@C;#4j&m4)mE+7Qv(`6%P^zu z8JQ2)y0n{Ac_|<8VGmSJ9Ldec9P=kgh^*MDq}E%zd+-ex<#LkKzSq)7Hpa0p+f~#3 zy8Cs%*YCal^_%{1V~{}kkALb9-wA~L8#`|CR0!_y77%%Ii^$wKxyIRCC(@pA&o$oW z8O@J-ula0WCy~9jU-m_b?2i(?*%o90=m4Vwas$voMmNYIphZRp<;E@&3qM9-G0q98 zhGE*OoiPmyy54EkUW7IrnnUodYcvbkA3Gndek z)^Z|aWOH^}wbeNe_Qem27d{@khY}EZk|MV}gI^GmhOrLvDYwYUoa~W#*_-fyPQJ%z z0mpY+)pMG|TQ+t4RI_u`aJ;josegs~`sYo$tlAceIO}xVFmBbQC-ls0B1d(_KrTJA zuEn!!j;JWrQ)-Grn_y)Fd{#SDdGd^OaLrG!+HO6-W~Yty4q=wC(;$JRdK%y17a&~a z@jTP+Nh=(zzF4J?o0GYV)fLCidfeG8m*RS>`z$bP$$S?{annE_Ev`J1zro9aOE6ur zFRqXqvYwl$H!B}-pL#zg=}xqMCrR;1PWD1f1X7-6;_h-)B;-ZO*F%JyCbDm$tpQtp zqWR1#*1Foe>+5=e=7Ot5y_Nm1ULpHa+$FG88tzL6k%elnW((N|5ffbhP-X!l&Cr}C ztt@JWq)J%`VQEe^(isRyP0~$s$&&QUk|r&iX-$`Eu^M-{vkupL^#w1Zg_9Vw=YhD) zodrwf$qcy)A@j$U8y>XpD#^p$L%T4GkS_8oKEuVhNeadzG2ZdSFF1jDsxB;RhV2Nt zxuQ`mZQ<5EQl5sPbs8=&Xoj|$r9;lS`HXIBRH^B#;1S}Nb#qSDt)Y00^CT>^-h%3f z$DmXpI8sauZx23L`FrSeDKY$`g`WPrrHz|$xP-&q9P*(ckD)qV!!$IgbGq*Ma)zbZ zKk-IFjrpCxtZHd`24|^efwEYu6o|!)Wt()(@tZjtF1I77bZ#+oTZ?%eAKcAM*731} zIlOADIRVW`TPqm=4=)X%Sf6b!X@=7>3#VAp(u!*ii$-CZsoEn+!xdvhj(bV=R+`756v2=eq z|C@awTsSefCk(>yfme9PU+_krHIqQ-b8C;8uM`})XWg^@Qty_9kzHW~w-Lm9cmI6% z!CYBr-xb<%2S2QQ@a~=v-5T5R?R5;m*uO$ev>DWU0r>nd0N&;fyMg#Q;FEo_KLHmt zCQ!T}P7pn>Fa)G{az{*O3j0atXIS&iG8?QMrur7ag6D8=9!CGKsOL#}jzc(nAud)m$|4WO z^^qd`UcwRykt5}~Swo&WkL>42mQV4=qOjN92@v;qS%S+^L%Nq`)p*`m7B`I^KT=#= zo(8lzTEHsR*F^RrTDc|bJznorjsMkp*LmJp@8fOslB|Uj6@R?W=CU8ILtuSu(tW}i zpbszrx3yJUvP}T_vGGaelc`Zfo|>8&pqdUD%xz7|8JX1q%U;tpwkS?PCZVlZA$C%H z^4G)N&o53)t~Ynzd4A+iiI|_Fg%Aw0x;m#hL57J8-yB{CSPQ6@k~OW&Dx@e33}Zf; zKnWMHVgSg4R!$hNFJ9dWw}p}01GpzuaPOU3o}$>M!g#Akq4?0STiXmiM(BI1EOr#d zj*@t~EOr;g?(G{TF;*56MKSRZfbe4e%09s3#Ky!{Hv>lK3u0M_7KLcx?Q>-zwkyQ; z!>xZCgZzDtb$(jyZ;j+-XEA+Dcw!)>h-U28Q?mN02 zV650u4xKE9PHv5sL(*#P!R{cAuxQs_F5GN~+LR5UMx2~YVZnn>H zm!1%)Fx!U)vwdjl+5s;sHSOMq?yJ0Yv-&C#@WJy4?E0yT&<|lb*cnnvLVt}kM?Zpw zz0y4u6o_U3OxVxV5o?bDBU2tzCa`uWyg9TnwAcQNhoj%NzgG$k7QBN@^T}Aq2`b8b z4*r(#9|wcS5ipm4A0n1DIDUIAn=uwBA`nxAWmgPlm{%zZW2R&~Rx}G-hLky{YeV!J v%%bBg#H0hB;T}^WF14-}zqWJ9Jf6loLn~Oy_U>w}X(+@k38W4dFJv!x1t|t`U(N zB7>a5wIFU7GAIU?^9s*$K@nJPRE&`GL#DwprHr)|hRlN&#lmpoQ2C%$v9^#g!CS9S zRm@w7Xxb0+VNxod*ru$|rt*nx%!M{pV`VU|MJzv*AGwymx79^N+kS&K*M2P{ihawr z70|Zwv9{G>6_nduBx+Y`#A?86814`qfIAuP6l(#mV|cCT0=%B#bz%eH+ZgT=p98#+ z;q_vR*aTnA_!Z@p2GP(?Jnk=WeI8D=24k_f5>z7bm^=*MmT*uBhN3}Ph7Z9TR3;%e z4=EB2jz^`LTRTr2x%RqW_CW6M>GhllMW&=)IYc7~MeYqzDX2&RMUoZ2+_m$B_ZXtb zul9Fe33kuy>Fx{oyL-CEx;(Gz!))DS?-pHV4oxJZ(LgLXCHduDN8oqW-?n4y#8YO` z8q8GUBD#aoXtxxcnw}nk4Vflrd`60e@`uDSUi;uS4gxYu#)!g+1X!|8;X8m;#{|*P z2{jDk3z)z#U{TQoWGz$6M_@8h3dLzysdzXUl>oN|0o^Ee6GTy7NDv$_LS)x6OThs!L8K?t9t_JWX z`Bi20ch9YsSA2H}z?(x9t5}<@N%y-@T|&C1O{C+2ho1`WK^N3VpOxVPo4_MlfjwFa3rK$1zPWR0~k}Q3VWLj%9D~jV~<8; zoq_LTDwg;5rF}7aD`f)lH zRAij5jpEXgR;>!5$d!I2IKRGQ;^Vx~Z*vk2y1v!(XZGOrmU_OsWYC16*x#SGMTGbsn+2#U z%#`(9k0_JLan+(TlRRVG)pKhqI%9$?I2HBqs!5JhMFNq&9*vKKv`dPIr>!uhT2bf~ z2uG-DoQOmvS;kFvy4@Pl>q7BpR0=5py(!2!G@`gM z*=pur$T_xW9NSaZvyPsut@j^0?>6Q7&Sd(|+#AgHy`0_a`^e^-9aytg|FC^t`oXTO zwJ9w$6^~0Ny8sih<8lM?Y(}HlOcfHqON_^88+G_)UkadKg^OfYt7rozkht8oJOg zXgriZ)&vLY;Hk&tCVsQF3_?sK49DI`T1XQ(dhH}aj`B?;!ht2`L4s!t-6v-(RC=wO znK@M$9OyrV4uGIY(~4S_jJ*#8kC~CZ*6Tfp}5@i>X-~BiMmR zLbVJ`hol5LOCEzJ=_aj}jH~a=i*`lex}*ej`$YSpic$G2Bi6d1%-FD2{amhk=P#>w zu2$5{pL(x-A@ZZ1#jd5-(zfHP)wK_dd`f7h(e2ZS>= zs8#jrJmkNT(GI7022E<{|&{B|wCRO8g8c!zZaqPqg!(ry30xzG!AqTKjkR>Um3g|_vMsR)sq!TQ&Zky{huQ*V$$^dqi=_s^hT>O`Sd;#3IR$KprmuF9{+Fdz&TgKkD zV(*v}*6i&$`>u?A*W&)Heb1cnvnuDuHs`9d;r5w#&ZJCP=Z>87MA~^`@xc2>?;Ksa zne93;XZftEo;7N^eg2*ES?Bhgvn%85T0EI`?#Ve1W}F9?61Lyj(N7Y#ZIECDq%%lJPS9M!)l3pWPBju3?%a};*WnXn`B6k%`vcEWz+|6ka*5;R>z6OW47&^Hz{ z4#do`Nz9^oF$1N|#7v+s0lif_@HUN12MTv(ipQW!6g`J+>d*C$h(nhGr(ZsMHZa_O zZa|kOdLF8&7XgYAy?}t-3Pr%4C?%VxlqouE?D*q=JVi=&=Ip&0d+(BvwIA48O1d-7 z?yR%-iCmxVJChc#F4y>%ZT^ia#`iXu^Ns$N7c00AJ9hy5ZSeYKqz|fYG;AigqTOdO zb#k5MOYK^oJ~FITJcZZq!6=$x10z;2ZBUp`1+Fsq^NJVwNmtNd+TiGJg2|-&pJCnv zz0W3-1Ex(RKK>fFTb~kg8`*0}82tr<2g-jduKbWoPv0hBB?bONBW0P^`AXIyopJ1A)ZnNZ)Eyeu6GK#*5gZH}C+Ib&6KuGDUS%QDBWRkh}-c4n$}E;@5nJ-@8#L494l zJr!E)|H<{#<#bgqsQ$|O)b8|wfqSQxD=#dYFKk#yUFTP9A#e1%``aF6M*-%9Q~bJ@ zede5P_VQWa0fP~6BWZ_>-AJ^Xeentw!iwhrAcMAYruzsMly%I~JCCx@$K8Ci%WQe& z082eBU%|^kcfG(sAbO3Y$d*N4Xp-wCaQ`mc&M<;9y#@)i#(V{Db%ZRze-zoY{7C%PESQ?$yqqwkMC#<*J z$9o-mZ87(i>*5AJ5=7C^NAlO|_1|g~jN!1f#VEqpUQ5xhE9}!}ISfLP_YOfXfxl<& zgPHOr8K9xji|n^4iAG-;DvXjH|5tv;$bu~SF-*o%k(c}!HDk%&s$<}hB2#Zl`uXZJ z=y^#$UNk|}yG%dHk{UrYZ(UR5#iGZnk1Sda<+&H)L*+Al!*REMM{-~8KX=y6qBrh| zcrq4t!%w2_fqfUx_2%P9g?Ov{8)H(L=sv8LQ7I7(h9r*xLYc8pJPh$g)g)0Gr!qYQ z`<>Z)GM05WZfmMsaw7kQ-ag(NX-36JPu@*04H)AgMiFp zgk%#H@e@i*?^pR2P>QN)Q`!g42HgXNOkdyORE{!blMjEGk41x105ZRJ@ zjz#04U{pR1h}PuyVU&E-;qHZ>UHR#id)2?F`*~gZf|$E7nt{J#mzX+&N%&W!TffbH z(J5RTT3&5m6rC8F0PqG@XrYo1p8vNaA0D~)>MvgV`D^LXOS#dp4E()#jV;<^q}Ylg zv(rWd$n3NkfTxB9_|)pt5bPU{E2oiP2Jq0eD~}cK+Yrlt4T8{d)f|)qiMSk@X3-=S z;usKzVu1@jD#h4cMDx+m-(rSLH3bt1DHhhE(fv@Y3K$)ysG?P4M3JUskCj;vwNmHb zeBcpRtJ;|3NZI>^tkx9mWFTM0cEKWFs##yFcEz%?uH(`~oJtsg#wQRNwJ*C^;pIxd zUR7;k4~m2h!A~9qvg4hqYjG%BdvJF6Gi$?IWlP$9HCyRVoBhA)+&O!Gtz*aE49uQg z+tvN|+tWhb>bAz&!O!i@bHb{_m2YeOhcb>sSx4WTd99;oQOS1fpR3B4 zTUTr9Zuh*?v$*4~XTB#}b84>a*L4jGyR&sW=B#UVJ&V&Tb^UXvS8eqEP>qYL@Ppy4mBCef_=9Ri5*t&%bp(-Rwv*t2-Z9O*Rua zpGH&Bx+802`nu5}hz*G*j1GCq^3fsf{HXRCg9ae~8UqfV77+#?lF%6h&vH!Oghn!I zcyvr0=$HUVclNDg{=r3-r7!tev9^U$b%ifMpys%Z6l0ukqT))uTRVB#kTmp=q~Wv& zOt=C3U&B@k5PV*P7s<9Mi3TVy#3YNAz5KA@OqXW*5TW~OnwPN9^TQ_aqf&)=(r5o6UQ8fEF z(E^m+;&ELphjLVPo04dS8>`L7_Zr)QrY-PG7$U?|xhc@|h*n2?!>rXoK+syj>ru(j zc;VReZ7BU_w&^T%WgNGq4p+C2CnM1?do7jVB5*4j4~8E%O@9J|$Vie|GXLtrwm<#7 zRB+MslgLu_(!qaf$kZKWTu5Jq1|wj*E%*i$023ucxJm-S2r6j6CvhczfB^TP);_9h z@!1kz7V01fO@kqLL(*O$_42AsmgLkm>W- zU@uN@Vh)Bx@l=bZ3GPqd!vb8U%7enDe~eg});O$I7FU7q8joG399xi+{uvCxWcuF$ znI-Ec;&kU6doqqaOUH8iyqSI86^D1j$h%Dc;qcC#T<1x-bN=FO|2zKF!Nq;=AHH*V z$#?hS&o2G+Qr3L>V~2O$NQ_Q+cXXK^n2D=t)$ID%b`;{{F4OG!2j!&7@%D}H-&kl_ zI`zSoE%nmh4gdA<(jnB9wa0R`r!%#uv$cb>!>}NWV}5YeTr+<* z=k#Qpo>g;8sxjC4LZ5?_uh zH+W*GU&qh;wqmW3t6wJo^0k0Htbu3dhcN}ydf+P From 7bb517233cb46247bba11ceb239337495b47ae88 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Wed, 8 Jul 2026 20:30:09 -0600 Subject: [PATCH 03/13] Implement static standalone input discovery for generator Replace placeholder standalone manifest entries with static scans of EAMxx scripts/actions for check-input and inputdata-like file paths, then normalize to public URLs and attach source-file provenance. --- .../inputdata/generate_inputdata_manifest.py | 78 +++++++++++++++---- 1 file changed, 63 insertions(+), 15 deletions(-) diff --git a/tools/inputdata/generate_inputdata_manifest.py b/tools/inputdata/generate_inputdata_manifest.py index 5838e57..daebc2a 100644 --- a/tools/inputdata/generate_inputdata_manifest.py +++ b/tools/inputdata/generate_inputdata_manifest.py @@ -3,6 +3,7 @@ import argparse import json +import re import sys from dataclasses import dataclass from datetime import datetime, timezone @@ -64,25 +65,70 @@ def _log(message: str, verbose: bool) -> None: print(message) -def _standalone_from_workflow_records(records: list[TestRecord]) -> tuple[list[str], dict[str, list[ProvenanceRecord]]]: +def _collect_standalone_candidate_paths(e3sm_root: Path) -> list[tuple[str, str]]: + """Return (source_path, candidate_input_path) pairs from static scans.""" + candidates: set[tuple[str, str]] = set() + + static_globs = [ + "components/eamxx/scripts/**/*", + "components/eamxx/**/*.yaml", + "components/eamxx/**/*.yml", + ".github/actions/test-all-eamxx/**/*", + ] + + path_pattern = re.compile(r"(?P(?:\$\{?DIN_LOC_ROOT\}?/)?[A-Za-z0-9_./+-]+\.(?:nc|txt|xml|yaml|yml|json|dat|h5))") + + for pattern in static_globs: + for file_path in sorted(e3sm_root.glob(pattern)): + if not file_path.is_file(): + continue + rel_source = file_path.relative_to(e3sm_root).as_posix() + try: + content = file_path.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + + for line in content.splitlines(): + if "check-input" not in line and "DIN_LOC_ROOT" not in line and "inputdata" not in line: + continue + for match in path_pattern.finditer(line): + raw = match.group("path") + candidates.add((rel_source, raw)) + + return sorted(candidates) + + +def _standalone_from_workflow_records( + e3sm_root: Path, + records: list[TestRecord], + din_loc_root: Path, +) -> tuple[list[str], dict[str, list[ProvenanceRecord]]]: urls: set[str] = set() provenance: dict[str, list[ProvenanceRecord]] = {} + standalone_tests = records or [ + TestRecord(workflow="components/eamxx", kind="standalone_scan", name="standalone-static") + ] + + static_candidates = _collect_standalone_candidate_paths(e3sm_root) + if not static_candidates: + return [], {} - for record in records: - # Conservative seed entry for v1 until richer standalone parsing is added. - marker_path = f"atm/scream/standalone/{record.name}.txt" - url = to_public_inputdata_url(marker_path) + for source_path, raw_path in static_candidates: + url = _normalize_and_collect(raw_path, din_loc_root) + if not url: + continue urls.add(url) - provenance.setdefault(url, []).append( - ProvenanceRecord( - workflow=record.workflow, - test=record.name, - case_dir="", - component_list_file="workflow-standalone-ref", - manifest="files-standalone.txt", - discovery_method="standalone-ref", + for record in standalone_tests: + provenance.setdefault(url, []).append( + ProvenanceRecord( + workflow=record.workflow, + test=record.name, + case_dir="", + component_list_file=source_path, + manifest="files-standalone.txt", + discovery_method="standalone-static", + ) ) - ) return sorted(urls), provenance @@ -208,7 +254,9 @@ def main() -> int: return 5 standalone_urls, standalone_provenance = _standalone_from_workflow_records( - parse_result.standalone_tests + e3sm_root=e3sm_root, + records=parse_result.standalone_tests, + din_loc_root=din_loc_root, ) combined_provenance = dict(full_provenance) From 890e005732045cf63e9fecc7e75da876a077e89e Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Wed, 8 Jul 2026 20:30:38 -0600 Subject: [PATCH 04/13] Add standalone config-mode discovery and artifact scanning Run test-all-eamxx in best-effort config mode, scan generated artifacts for inputdata references, and merge with static standalone discovery while preserving deterministic output and warning/report behavior. --- .../inputdata/generate_inputdata_manifest.py | 115 +++++++++++++++++- 1 file changed, 112 insertions(+), 3 deletions(-) diff --git a/tools/inputdata/generate_inputdata_manifest.py b/tools/inputdata/generate_inputdata_manifest.py index daebc2a..07aae7c 100644 --- a/tools/inputdata/generate_inputdata_manifest.py +++ b/tools/inputdata/generate_inputdata_manifest.py @@ -4,6 +4,7 @@ import argparse import json import re +import subprocess import sys from dataclasses import dataclass from datetime import datetime, timezone @@ -98,10 +99,99 @@ def _collect_standalone_candidate_paths(e3sm_root: Path) -> list[tuple[str, str] return sorted(candidates) +def _run_standalone_config_mode(e3sm_root: Path, output_root: Path) -> tuple[bool, list[str]]: + """Best-effort config-only run for standalone EAMxx discovery.""" + script = e3sm_root / "components" / "eamxx" / "scripts" / "test-all-eamxx" + if not script.exists(): + return False, [f"Missing standalone script: {script}"] + + output_root.mkdir(parents=True, exist_ok=True) + command = [str(script), "--help"] + try: + help_proc = subprocess.run( + command, + cwd=str(e3sm_root), + text=True, + capture_output=True, + ) + except OSError as exc: + return False, [f"Failed to execute standalone script: {exc}"] + + supported_flags = help_proc.stdout + "\n" + help_proc.stderr + run_command = [str(script)] + if "--configure-only" in supported_flags: + run_command.append("--configure-only") + elif "--config-only" in supported_flags: + run_command.append("--config-only") + elif "--dry-run" in supported_flags: + run_command.append("--dry-run") + + if "--work-dir" in supported_flags: + run_command.extend(["--work-dir", str(output_root / "standalone-work")]) + + proc = subprocess.run( + run_command, + cwd=str(e3sm_root), + text=True, + capture_output=True, + ) + + messages: list[str] = [] + if proc.returncode != 0: + tail = (proc.stderr or proc.stdout or "").strip().splitlines() + messages.append("standalone config mode failed") + messages.extend(tail[-10:]) + return False, messages + + return True, messages + + +def _scan_option_b_artifacts(e3sm_root: Path, output_root: Path) -> list[tuple[str, str]]: + candidates: set[tuple[str, str]] = set() + search_roots = [ + output_root, + e3sm_root / "components" / "eamxx" / "ctest-build", + e3sm_root / "components" / "eamxx" / "build", + ] + path_pattern = re.compile(r"(?P(?:\$\{?DIN_LOC_ROOT\}?/)?[A-Za-z0-9_./+-]+\.(?:nc|txt|xml|yaml|yml|json|dat|h5))") + + for root in search_roots: + if not root.exists(): + continue + for file_path in sorted(root.rglob("*")): + if not file_path.is_file(): + continue + if file_path.suffix.lower() not in { + ".txt", + ".cmake", + ".cfg", + ".yaml", + ".yml", + ".log", + ".sh", + ".json", + }: + continue + try: + content = file_path.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + for line in content.splitlines(): + if "DIN_LOC_ROOT" not in line and "inputdata" not in line: + continue + for match in path_pattern.finditer(line): + raw = match.group("path") + candidates.add((file_path.as_posix(), raw)) + + return sorted(candidates) + + def _standalone_from_workflow_records( e3sm_root: Path, records: list[TestRecord], din_loc_root: Path, + output_root: Path, + warnings: list[str], ) -> tuple[list[str], dict[str, list[ProvenanceRecord]]]: urls: set[str] = set() provenance: dict[str, list[ProvenanceRecord]] = {} @@ -110,14 +200,23 @@ def _standalone_from_workflow_records( ] static_candidates = _collect_standalone_candidate_paths(e3sm_root) - if not static_candidates: + option_b_ok, option_b_messages = _run_standalone_config_mode(e3sm_root, output_root) + if not option_b_ok: + warnings.extend(option_b_messages) + + option_b_candidates = _scan_option_b_artifacts(e3sm_root, output_root) if option_b_ok else [] + all_candidates = static_candidates + option_b_candidates + if not all_candidates: return [], {} - for source_path, raw_path in static_candidates: + for source_path, raw_path in all_candidates: url = _normalize_and_collect(raw_path, din_loc_root) if not url: continue urls.add(url) + method = "standalone-static" + if source_path.startswith(output_root.as_posix()) or "/ctest-build/" in source_path: + method = "standalone-config" for record in standalone_tests: provenance.setdefault(url, []).append( ProvenanceRecord( @@ -126,7 +225,7 @@ def _standalone_from_workflow_records( case_dir="", component_list_file=source_path, manifest="files-standalone.txt", - discovery_method="standalone-static", + discovery_method=method, ) ) @@ -253,10 +352,13 @@ def main() -> int: print("ERROR: resulting full-model manifest would be empty", file=sys.stderr) return 5 + standalone_warnings: list[str] = [] standalone_urls, standalone_provenance = _standalone_from_workflow_records( e3sm_root=e3sm_root, records=parse_result.standalone_tests, din_loc_root=din_loc_root, + output_root=output_root, + warnings=standalone_warnings, ) combined_provenance = dict(full_provenance) @@ -280,6 +382,13 @@ def main() -> int: print("ERROR: strict mode enabled and parser warnings were present", file=sys.stderr) return 6 + if standalone_warnings: + for warning in standalone_warnings: + print(f"WARNING: {warning}", file=sys.stderr) + if args.strict: + print("ERROR: strict mode enabled and standalone warnings were present", file=sys.stderr) + return 7 + print( "SUMMARY " f"full_tests={len(parse_result.full_model_tests)} " From 5ed868092617e563e48cfb3fa62d7f24182362bf Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Wed, 8 Jul 2026 20:31:01 -0600 Subject: [PATCH 05/13] Improve create_test case attribution and failure diagnostics Track case directories created per test invocation to reduce cross-test mixing, record start/finish timestamps, and include structured warning fields to make generator failures easier to triage. --- tools/inputdata/generate_inputdata_manifest.py | 6 +++++- tools/inputdata/lib/cime_manifest.py | 18 +++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tools/inputdata/generate_inputdata_manifest.py b/tools/inputdata/generate_inputdata_manifest.py index 07aae7c..0884331 100644 --- a/tools/inputdata/generate_inputdata_manifest.py +++ b/tools/inputdata/generate_inputdata_manifest.py @@ -318,7 +318,11 @@ def main() -> int: if not result.success: failure_count += 1 print( - f"WARNING: create_test failed for {test_record.name}: {result.error}", + "WARNING: create_test failed " + f"test={test_record.name} " + f"started={result.started_at_utc} " + f"finished={result.finished_at_utc} " + f"error={result.error}", file=sys.stderr, ) continue diff --git a/tools/inputdata/lib/cime_manifest.py b/tools/inputdata/lib/cime_manifest.py index ba629d0..e527b79 100644 --- a/tools/inputdata/lib/cime_manifest.py +++ b/tools/inputdata/lib/cime_manifest.py @@ -3,6 +3,7 @@ import glob import subprocess from dataclasses import dataclass +from datetime import datetime, timezone from pathlib import Path from typing import Optional @@ -20,6 +21,8 @@ class CaseResult: workflow: str success: bool case_dirs: list[str] + started_at_utc: str + finished_at_utc: str error: Optional[str] = None @@ -33,14 +36,20 @@ def run_create_test( ) -> CaseResult: create_test = e3sm_root / "cime" / "scripts" / "create_test" if not create_test.exists(): + now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") return CaseResult( test_name=test_name, workflow=workflow, success=False, case_dirs=[], + started_at_utc=now, + finished_at_utc=now, error=f"Missing create_test script: {create_test}", ) + before = set(discover_case_dirs(test_root)) + started_at_utc = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + command = [ str(create_test), test_name, @@ -62,8 +71,11 @@ def run_create_test( text=True, capture_output=True, ) + finished_at_utc = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - case_dirs = discover_case_dirs(test_root) + after = set(discover_case_dirs(test_root)) + new_case_dirs = sorted(after - before) + case_dirs = new_case_dirs if new_case_dirs else sorted(after) if process.returncode != 0: stderr_tail = (process.stderr or process.stdout or "").strip().splitlines() @@ -73,6 +85,8 @@ def run_create_test( workflow=workflow, success=False, case_dirs=case_dirs, + started_at_utc=started_at_utc, + finished_at_utc=finished_at_utc, error=tail, ) @@ -81,6 +95,8 @@ def run_create_test( workflow=workflow, success=True, case_dirs=case_dirs, + started_at_utc=started_at_utc, + finished_at_utc=finished_at_utc, ) From 4e816e77207f7b9b5deb0c81e02f772370655144 Mon Sep 17 00:00:00 2001 From: e3smuser Date: Thu, 9 Jul 2026 03:33:06 +0000 Subject: [PATCH 06/13] Fix inputdata generator CIME compatibility --- tools/inputdata/generate_inputdata_manifest.py | 4 +--- tools/inputdata/lib/cime_manifest.py | 6 ++++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/inputdata/generate_inputdata_manifest.py b/tools/inputdata/generate_inputdata_manifest.py index 0884331..eaa0d13 100644 --- a/tools/inputdata/generate_inputdata_manifest.py +++ b/tools/inputdata/generate_inputdata_manifest.py @@ -296,8 +296,6 @@ def main() -> int: return 3 state = build_state_payload(parse_result) - state["generated_at_utc"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - full_urls: set[str] = set() full_provenance: dict[str, list[ProvenanceRecord]] = {} @@ -341,7 +339,7 @@ def main() -> int: ProvenanceRecord( workflow=test_record.workflow, test=test_record.name, - case_dir=entry.case_dir, + case_dir=test_record.name, component_list_file=entry.component_list_file, manifest="files.txt", discovery_method="cime-buildconf", diff --git a/tools/inputdata/lib/cime_manifest.py b/tools/inputdata/lib/cime_manifest.py index e527b79..5d30574 100644 --- a/tools/inputdata/lib/cime_manifest.py +++ b/tools/inputdata/lib/cime_manifest.py @@ -56,6 +56,12 @@ def run_create_test( "--no-build", "--no-run", "--namelists-only", + "--generate", + "-b", + "local", + "--allow-baseline-overwrite", + "--baseline-root", + str(output_root), "--test-root", str(test_root), "--output-root", From 328b5806547267648d5f87a1c1a3ab8179b6d538 Mon Sep 17 00:00:00 2001 From: e3smuser Date: Thu, 9 Jul 2026 03:51:01 +0000 Subject: [PATCH 07/13] Add some docs --- tools/inputdata/README.md | 50 ++++++++++++++++ .../inputdata/generate_inputdata_manifest.py | 58 +++++++++++++++---- 2 files changed, 97 insertions(+), 11 deletions(-) create mode 100644 tools/inputdata/README.md diff --git a/tools/inputdata/README.md b/tools/inputdata/README.md new file mode 100644 index 0000000..847954f --- /dev/null +++ b/tools/inputdata/README.md @@ -0,0 +1,50 @@ +# Inputdata Generator + +`tools/inputdata` contains the generator that builds the inputdata manifests used by this workspace. + +## What it does + +The main entry point is `python3 -m tools.inputdata.generate_inputdata_manifest`. It scans an E3SM checkout, extracts test names from workflow files, runs CIME `create_test` for full-model cases, and writes manifest and provenance files that describe the input files referenced by those tests. + +It also performs a best-effort standalone EAMxx scan so the generated manifests can include input paths referenced outside of the full-model CIME flow. + +## How it works + +The generator uses a small pipeline: + +1. `tools.inputdata.lib.workflow_parser` reads `.github/workflows/*.yml` and `.yaml` files from the E3SM tree. +2. It collects full-model test names from `create_test` lines and matrix definitions, and records standalone EAMxx references when present. +3. `tools.inputdata.lib.cime_manifest.run_create_test` invokes `cime/scripts/create_test` for each full-model test in a setup-only mode (`--no-build`, `--no-run`, `--namelists-only`). That creates the case metadata and Buildconf files needed to discover input lists, but it does not compile or execute the test case, and it avoids actually pulling model input data during the run. +4. `tools.inputdata.lib.url_normalize` converts discovered local paths into public inputdata URLs when they point at supported data. +5. The generator writes the final outputs in a stable order so repeated runs produce the same files. + +Some `create_test` invocations can still print `FAIL` during setup. In this workflow, that usually means CIME could not finish case setup for that particular test, so the generator records the failure as a warning and moves on to the next test instead of treating it as a fatal error for the whole manifest run. Usually, the needed information is discovered despite this. + +## Inputs + +The command requires: + +- `--e3sm-root`: path to the E3SM checkout +- `--test-root`: directory where CIME test cases are created +- `--output-root`: directory where generated artifacts and temporary outputs are written +- `--state-out`, `--files-out`, `--standalone-files-out`, `--provenance-out`: output files to write + +## Outputs + +The generator writes four files to the paths passed on the command line: + +- `state-out`: workflow and extraction metadata, including the extracted test inventory. This is the snapshot you can inspect to see which workflows and tests were discovered from the E3SM checkout. +- `files-out`: full-model inputdata URLs. This is the primary manifest consumed by the inputdata automation and other tooling that needs the full-model file list. +- `standalone-files-out`: standalone EAMxx inputdata URLs. This is used for the separate standalone manifest so those paths can be tracked independently from the full-model list. +- `provenance-out`: per-URL provenance showing where each input was discovered. This file is useful for tracing a URL back to the workflow, test, and case directory or source scan that produced it. It is supplemental traceability data, not a required input to the generator. For standalone discoveries, `case_dir` is intentionally empty because there is no CIME case directory. + +## Environment notes + +The generator needs an environment with an E3SM checkout and a working CIME `create_test` script available at `cime/scripts/create_test`. It also expects writable paths for `--test-root` and `--output-root`. + +## Related modules + +- [generate_inputdata_manifest.py](generate_inputdata_manifest.py) +- [lib/workflow_parser.py](lib/workflow_parser.py) +- [lib/cime_manifest.py](lib/cime_manifest.py) +- [lib/url_normalize.py](lib/url_normalize.py) \ No newline at end of file diff --git a/tools/inputdata/generate_inputdata_manifest.py b/tools/inputdata/generate_inputdata_manifest.py index eaa0d13..495d8c5 100644 --- a/tools/inputdata/generate_inputdata_manifest.py +++ b/tools/inputdata/generate_inputdata_manifest.py @@ -35,16 +35,51 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Generate inputdata manifests from E3SM workflows and CIME metadata" ) - parser.add_argument("--e3sm-root", required=True) - parser.add_argument("--din-loc-root", required=True) - parser.add_argument("--test-root", required=True) - parser.add_argument("--output-root", required=True) - parser.add_argument("--state-out", required=True) - parser.add_argument("--files-out", required=True) - parser.add_argument("--standalone-files-out", required=True) - parser.add_argument("--provenance-out", required=True) - parser.add_argument("--strict", action="store_true") - parser.add_argument("--verbose", action="store_true") + parser.add_argument( + "--e3sm-root", + required=True, + help="Path to the E3SM checkout that provides workflow files and CIME scripts.", + ) + parser.add_argument( + "--test-root", + required=True, + help="Directory where CIME test cases are created during discovery.", + ) + parser.add_argument( + "--output-root", + required=True, + help="Directory used for generated artifacts and temporary discovery output.", + ) + parser.add_argument( + "--state-out", + required=True, + help="File path for the generated workflow and extraction state JSON.", + ) + parser.add_argument( + "--files-out", + required=True, + help="File path for the generated full-model inputdata manifest.", + ) + parser.add_argument( + "--standalone-files-out", + required=True, + help="File path for the generated standalone EAMxx inputdata manifest.", + ) + parser.add_argument( + "--provenance-out", + required=True, + help="File path for the generated provenance JSON.", + ) + parser.add_argument( + "--strict", + action="store_true", + help="Fail if workflow parsing or standalone discovery emits warnings.", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Print progress messages while discovering tests and input files.", + ) return parser.parse_args() @@ -282,9 +317,10 @@ def main() -> int: args = parse_args() e3sm_root = Path(args.e3sm_root).resolve() - din_loc_root = Path(args.din_loc_root).resolve() test_root = Path(args.test_root).resolve() output_root = Path(args.output_root).resolve() + din_loc_root = (Path.home() / "e3sm-inputdata-empty").resolve() + din_loc_root.mkdir(parents=True, exist_ok=True) if not e3sm_root.exists(): print(f"ERROR: --e3sm-root does not exist: {e3sm_root}", file=sys.stderr) From 47d81b826a766bb0b730bdfb3cc2cc271441d49e Mon Sep 17 00:00:00 2001 From: e3smuser Date: Thu, 9 Jul 2026 04:16:32 +0000 Subject: [PATCH 08/13] Handle files that can't be found on the inputdata servers --- tools/inputdata/README.md | 13 +- .../inputdata/generate_inputdata_manifest.py | 161 +++++++++++++----- tools/inputdata/lib/inputdata_validation.py | 99 +++++++++++ 3 files changed, 229 insertions(+), 44 deletions(-) create mode 100644 tools/inputdata/lib/inputdata_validation.py diff --git a/tools/inputdata/README.md b/tools/inputdata/README.md index 847954f..e4aadb2 100644 --- a/tools/inputdata/README.md +++ b/tools/inputdata/README.md @@ -16,7 +16,8 @@ The generator uses a small pipeline: 2. It collects full-model test names from `create_test` lines and matrix definitions, and records standalone EAMxx references when present. 3. `tools.inputdata.lib.cime_manifest.run_create_test` invokes `cime/scripts/create_test` for each full-model test in a setup-only mode (`--no-build`, `--no-run`, `--namelists-only`). That creates the case metadata and Buildconf files needed to discover input lists, but it does not compile or execute the test case, and it avoids actually pulling model input data during the run. 4. `tools.inputdata.lib.url_normalize` converts discovered local paths into public inputdata URLs when they point at supported data. -5. The generator writes the final outputs in a stable order so repeated runs produce the same files. +5. `tools.inputdata.lib.inputdata_validation.validate_inputdata_reference` checks the candidate URL with a lightweight remote existence probe without downloading the file. It tries the E3SM public inputdata server first, then the CESM mirror. When both endpoints definitively return not-found responses, the entry is recorded as missing; when the servers cannot be verified cleanly, the entry is kept but marked unverified in provenance. +6. The generator writes the final outputs in a stable order so repeated runs produce the same files. Some `create_test` invocations can still print `FAIL` during setup. In this workflow, that usually means CIME could not finish case setup for that particular test, so the generator records the failure as a warning and moves on to the next test instead of treating it as a fatal error for the whole manifest run. Usually, the needed information is discovered despite this. @@ -27,15 +28,16 @@ The command requires: - `--e3sm-root`: path to the E3SM checkout - `--test-root`: directory where CIME test cases are created - `--output-root`: directory where generated artifacts and temporary outputs are written -- `--state-out`, `--files-out`, `--standalone-files-out`, `--provenance-out`: output files to write +- `--state-out`, `--files-out`, `--standalone-files-out`, `--missing-files-out`, `--provenance-out`: output files to write ## Outputs -The generator writes four files to the paths passed on the command line: +The generator writes five files to the paths passed on the command line: - `state-out`: workflow and extraction metadata, including the extracted test inventory. This is the snapshot you can inspect to see which workflows and tests were discovered from the E3SM checkout. -- `files-out`: full-model inputdata URLs. This is the primary manifest consumed by the inputdata automation and other tooling that needs the full-model file list. +- `files-out`: full-model inputdata URLs confirmed present on the E3SM or CESM server. This is the primary manifest consumed by the inputdata automation and other tooling that needs the full-model file list. - `standalone-files-out`: standalone EAMxx inputdata URLs. This is used for the separate standalone manifest so those paths can be tracked independently from the full-model list. +- `missing-files-out`: relative paths of input files that could not be confirmed on either the E3SM or CESM server. A `WARNING: missing inputdata` line is also printed to stderr for each entry. Pass `--strict` to fail the run when any missing entries are found. - `provenance-out`: per-URL provenance showing where each input was discovered. This file is useful for tracing a URL back to the workflow, test, and case directory or source scan that produced it. It is supplemental traceability data, not a required input to the generator. For standalone discoveries, `case_dir` is intentionally empty because there is no CIME case directory. ## Environment notes @@ -47,4 +49,5 @@ The generator needs an environment with an E3SM checkout and a working CIME `cre - [generate_inputdata_manifest.py](generate_inputdata_manifest.py) - [lib/workflow_parser.py](lib/workflow_parser.py) - [lib/cime_manifest.py](lib/cime_manifest.py) -- [lib/url_normalize.py](lib/url_normalize.py) \ No newline at end of file +- [lib/url_normalize.py](lib/url_normalize.py) +- [lib/inputdata_validation.py](lib/inputdata_validation.py) \ No newline at end of file diff --git a/tools/inputdata/generate_inputdata_manifest.py b/tools/inputdata/generate_inputdata_manifest.py index 495d8c5..3335b4e 100644 --- a/tools/inputdata/generate_inputdata_manifest.py +++ b/tools/inputdata/generate_inputdata_manifest.py @@ -12,12 +12,8 @@ from typing import Optional from tools.inputdata.lib.cime_manifest import parse_input_data_lists, run_create_test -from tools.inputdata.lib.url_normalize import ( - filter_candidate_path, - is_public_inputdata_url, - normalize_to_din_relative, - to_public_inputdata_url, -) +from tools.inputdata.lib.inputdata_validation import InputDataValidationResult, validate_inputdata_reference +from tools.inputdata.lib.url_normalize import filter_candidate_path, is_public_inputdata_url from tools.inputdata.lib.workflow_parser import TestRecord, build_state_payload, parse_workflows @@ -29,6 +25,10 @@ class ProvenanceRecord: component_list_file: str manifest: str discovery_method: str + relative_path: str + validation_status: str + checked_urls: tuple[str, str] + validation_message: str = "" def parse_args() -> argparse.Namespace: @@ -70,6 +70,11 @@ def parse_args() -> argparse.Namespace: required=True, help="File path for the generated provenance JSON.", ) + parser.add_argument( + "--missing-files-out", + required=True, + help="File path for a list of input files that could not be found on either server.", + ) parser.add_argument( "--strict", action="store_true", @@ -227,6 +232,7 @@ def _standalone_from_workflow_records( din_loc_root: Path, output_root: Path, warnings: list[str], + missing_provenance: list[dict[str, object]], ) -> tuple[list[str], dict[str, list[ProvenanceRecord]]]: urls: set[str] = set() provenance: dict[str, list[ProvenanceRecord]] = {} @@ -245,44 +251,68 @@ def _standalone_from_workflow_records( return [], {} for source_path, raw_path in all_candidates: - url = _normalize_and_collect(raw_path, din_loc_root) - if not url: + validation = _normalize_and_validate(raw_path, din_loc_root) + if not validation: continue - urls.add(url) method = "standalone-static" if source_path.startswith(output_root.as_posix()) or "/ctest-build/" in source_path: method = "standalone-config" + for record in standalone_tests: - provenance.setdefault(url, []).append( - ProvenanceRecord( - workflow=record.workflow, - test=record.name, - case_dir="", - component_list_file=source_path, - manifest="files-standalone.txt", - discovery_method=method, - ) + provenance_record = ProvenanceRecord( + workflow=record.workflow, + test=record.name, + case_dir="", + component_list_file=source_path, + manifest="files-standalone.txt", + discovery_method=method, + relative_path=validation.relative_path, + validation_status=validation.status, + checked_urls=validation.checked_urls, + validation_message=validation.message, ) + if validation.status == "missing": + missing_provenance.append( + { + "workflow": provenance_record.workflow, + "test": provenance_record.test, + "case_dir": provenance_record.case_dir, + "component_list_file": provenance_record.component_list_file, + "manifest": provenance_record.manifest, + "discovery_method": provenance_record.discovery_method, + "relative_path": provenance_record.relative_path, + "checked_urls": list(provenance_record.checked_urls), + "validation_status": provenance_record.validation_status, + "validation_message": provenance_record.validation_message, + } + ) + continue + + url = validation.resolved_url or validation.checked_urls[0] + urls.add(url) + provenance.setdefault(url, []).append(provenance_record) return sorted(urls), provenance -def _normalize_and_collect( +def _normalize_and_validate( raw_value: str, din_loc_root: Path, -) -> Optional[str]: +) -> Optional[InputDataValidationResult]: if not filter_candidate_path(raw_value): return None - relative = normalize_to_din_relative(raw_value, din_loc_root) - if not relative: + validation_result = validate_inputdata_reference(raw_value, din_loc_root) + if not validation_result: return None - url = to_public_inputdata_url(relative) - if not is_public_inputdata_url(url): + if not validation_result.resolved_url: + return validation_result + + if not is_public_inputdata_url(validation_result.resolved_url): return None - return url + return validation_result def _provenance_payload(data: dict[str, list[ProvenanceRecord]]) -> dict[str, list[dict]]: @@ -297,6 +327,8 @@ def _provenance_payload(data: dict[str, list[ProvenanceRecord]]) -> dict[str, li rec.case_dir, rec.manifest, rec.discovery_method, + rec.validation_status, + rec.relative_path, ), ) payload[url] = [ @@ -307,6 +339,10 @@ def _provenance_payload(data: dict[str, list[ProvenanceRecord]]) -> dict[str, li "component_list_file": rec.component_list_file, "manifest": rec.manifest, "discovery_method": rec.discovery_method, + "relative_path": rec.relative_path, + "validation_status": rec.validation_status, + "checked_urls": list(rec.checked_urls), + "validation_message": rec.validation_message, } for rec in entries ] @@ -334,6 +370,7 @@ def main() -> int: state = build_state_payload(parse_result) full_urls: set[str] = set() full_provenance: dict[str, list[ProvenanceRecord]] = {} + missing_provenance: list[dict[str, object]] = [] success_count = 0 failure_count = 0 @@ -367,21 +404,44 @@ def main() -> int: case_dir = Path(case_dir_value) entries = parse_input_data_lists(case_dir) for entry in entries: - url = _normalize_and_collect(entry.raw_value, din_loc_root) - if not url: + validation = _normalize_and_validate(entry.raw_value, din_loc_root) + if not validation: continue - full_urls.add(url) - full_provenance.setdefault(url, []).append( - ProvenanceRecord( - workflow=test_record.workflow, - test=test_record.name, - case_dir=test_record.name, - component_list_file=entry.component_list_file, - manifest="files.txt", - discovery_method="cime-buildconf", - ) + + provenance_record = ProvenanceRecord( + workflow=test_record.workflow, + test=test_record.name, + case_dir=test_record.name, + component_list_file=entry.component_list_file, + manifest="files.txt", + discovery_method="cime-buildconf", + relative_path=validation.relative_path, + validation_status=validation.status, + checked_urls=validation.checked_urls, + validation_message=validation.message, ) + if validation.status == "missing": + missing_provenance.append( + { + "workflow": provenance_record.workflow, + "test": provenance_record.test, + "case_dir": provenance_record.case_dir, + "component_list_file": provenance_record.component_list_file, + "manifest": provenance_record.manifest, + "discovery_method": provenance_record.discovery_method, + "relative_path": provenance_record.relative_path, + "checked_urls": list(provenance_record.checked_urls), + "validation_status": provenance_record.validation_status, + "validation_message": provenance_record.validation_message, + } + ) + continue + + url = validation.resolved_url or validation.checked_urls[0] + full_urls.add(url) + full_provenance.setdefault(url, []).append(provenance_record) + if success_count == 0: print("ERROR: CIME case generation failed for all tests", file=sys.stderr) return 4 @@ -397,6 +457,7 @@ def main() -> int: din_loc_root=din_loc_root, output_root=output_root, warnings=standalone_warnings, + missing_provenance=missing_provenance, ) combined_provenance = dict(full_provenance) @@ -406,13 +467,34 @@ def main() -> int: files_out = Path(args.files_out) standalone_files_out = Path(args.standalone_files_out) provenance_out = Path(args.provenance_out) + missing_files_out = Path(args.missing_files_out) state_out = Path(args.state_out) + missing_relative_paths = sorted( + {entry["relative_path"] for entry in missing_provenance} + ) + + provenance_payload = _provenance_payload(combined_provenance) + if missing_provenance: + provenance_payload["missing"] = missing_provenance + _write_manifest(sorted(full_urls), files_out) _write_manifest(sorted(standalone_urls), standalone_files_out) - _write_json(_provenance_payload(combined_provenance), provenance_out) + _write_manifest(missing_relative_paths, missing_files_out) + _write_json(provenance_payload, provenance_out) _write_json(state, state_out) + for entry in sorted(missing_provenance, key=lambda e: e["relative_path"]): + print( + f"WARNING: missing inputdata " + f"path={entry['relative_path']} " + f"checked={entry['checked_urls']}", + file=sys.stderr, + ) + if missing_provenance and args.strict: + print("ERROR: strict mode enabled and missing inputdata entries were found", file=sys.stderr) + return 8 + if parse_result.warnings: for warning in parse_result.warnings: print(f"WARNING: {warning}", file=sys.stderr) @@ -434,7 +516,8 @@ def main() -> int: f"full_failed={failure_count} " f"full_urls={len(full_urls)} " f"standalone_refs={len(parse_result.standalone_tests)} " - f"standalone_urls={len(standalone_urls)}" + f"standalone_urls={len(standalone_urls)} " + f"missing={len(missing_relative_paths)}" ) return 0 diff --git a/tools/inputdata/lib/inputdata_validation.py b/tools/inputdata/lib/inputdata_validation.py new file mode 100644 index 0000000..c342292 --- /dev/null +++ b/tools/inputdata/lib/inputdata_validation.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import ssl +from dataclasses import dataclass +from typing import Optional +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +from tools.inputdata.lib.url_normalize import normalize_to_din_relative, to_public_inputdata_url + +CESM_INPUTDATA_PREFIX = "https://svn-ccsm-inputdata.cgd.ucar.edu/trunk/inputdata" + + +@dataclass(frozen=True) +class InputDataValidationResult: + relative_path: str + checked_urls: tuple[str, str] + resolved_url: Optional[str] + status: str + message: str = "" + + +@dataclass(frozen=True) +class UrlProbeResult: + exists: bool + verified: bool + message: str = "" + + +def _probe_url(url: str) -> UrlProbeResult: + request = Request( + url, + method="HEAD", + headers={"User-Agent": "Mozilla/5.0", "Accept-Encoding": "identity"}, + ) + context = ssl._create_unverified_context() + try: + with urlopen(request, timeout=20, context=context) as response: + status = getattr(response, "status", 200) + if 200 <= status < 400: + return UrlProbeResult(exists=True, verified=True) + return UrlProbeResult(exists=False, verified=False, message=f"unexpected status {status}") + except HTTPError as exc: + if exc.code in {404, 410}: + return UrlProbeResult(exists=False, verified=True, message=f"http {exc.code}") + return UrlProbeResult(exists=False, verified=False, message=f"http {exc.code}") + except (URLError, OSError, TimeoutError) as exc: + return UrlProbeResult(exists=False, verified=False, message=str(exc)) + + +def validate_inputdata_reference(path_value: str, din_loc_root) -> Optional[InputDataValidationResult]: + relative_path = normalize_to_din_relative(path_value, din_loc_root) + if not relative_path: + return None + + e3sm_url = to_public_inputdata_url(relative_path) + cesm_url = f"{CESM_INPUTDATA_PREFIX}/{relative_path.strip().lstrip('/')}" + + e3sm_probe = _probe_url(e3sm_url) + if e3sm_probe.exists: + return InputDataValidationResult( + relative_path=relative_path, + checked_urls=(e3sm_url, cesm_url), + resolved_url=e3sm_url, + status="e3sm", + ) + + cesm_probe = _probe_url(cesm_url) + if cesm_probe.exists: + return InputDataValidationResult( + relative_path=relative_path, + checked_urls=(e3sm_url, cesm_url), + resolved_url=cesm_url, + status="cesm", + ) + + if e3sm_probe.verified and cesm_probe.verified: + return InputDataValidationResult( + relative_path=relative_path, + checked_urls=(e3sm_url, cesm_url), + resolved_url=None, + status="missing", + message="not found on E3SM or CESM", + ) + + message_parts = [] + if e3sm_probe.message: + message_parts.append(f"e3sm: {e3sm_probe.message}") + if cesm_probe.message: + message_parts.append(f"cesm: {cesm_probe.message}") + message = "; ".join(message_parts) if message_parts else "validation unavailable" + + return InputDataValidationResult( + relative_path=relative_path, + checked_urls=(e3sm_url, cesm_url), + resolved_url=e3sm_url, + status="unverified", + message=message, + ) From 7ca5a2bd175fae63b3e5c8e6dc66b105dff1d16a Mon Sep 17 00:00:00 2001 From: e3smuser Date: Thu, 9 Jul 2026 14:50:53 +0000 Subject: [PATCH 09/13] Add empty files-missing.txt file so diffs will be picked up --- inputdata/files-missing.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 inputdata/files-missing.txt diff --git a/inputdata/files-missing.txt b/inputdata/files-missing.txt new file mode 100644 index 0000000..e69de29 From a66f25ebf029faa4315677ab76a1ca440e4caa6a Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Thu, 9 Jul 2026 08:53:52 -0600 Subject: [PATCH 10/13] small readme update --- tools/inputdata/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/inputdata/README.md b/tools/inputdata/README.md index e4aadb2..1f370e9 100644 --- a/tools/inputdata/README.md +++ b/tools/inputdata/README.md @@ -1,6 +1,8 @@ # Inputdata Generator -`tools/inputdata` contains the generator that builds the inputdata manifests used by this workspace. +`tools/inputdata` contains the generator that builds the list of inputdata files required for Github +Continuous Integration tests. The purpose is to produce the files that should be included in the CI +Docker container, avoiding the need for them to be downloaded during CI. ## What it does From a06ce80c547170947e399e661f6eebcda9206927 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Thu, 9 Jul 2026 09:20:24 -0600 Subject: [PATCH 11/13] Add automated inputdata manifest update workflow Introduce a new GitHub Actions workflow that regenerates inputdata manifests on a weekly schedule and via manual workflow_dispatch, then opens a pull request only when tracked manifest artifacts change. This commit adds: - .github/workflows/update-inputdata-manifest.yml - weekly cron + manual trigger - regenerate step using tools.inputdata.generate_inputdata_manifest - managed-file diff gate for conditional PR creation - concurrency guard and job timeout hardening - date-stamped branch naming for bot-created PRs - tools/inputdata/build_pr_metadata.py - reusable helper that builds PR body markdown from git diff stats and e3sm-workflow-state metadata - tools/inputdata/README.md - related-modules entry for the new metadata helper The existing inputdata image build workflow remains separate and unchanged; this new workflow handles manifest regeneration and PR creation only. --- .../workflows/update-inputdata-manifest.yml | 113 ++++++++++++++++++ tools/inputdata/README.md | 1 + tools/inputdata/build_pr_metadata.py | 92 ++++++++++++++ 3 files changed, 206 insertions(+) create mode 100644 .github/workflows/update-inputdata-manifest.yml create mode 100644 tools/inputdata/build_pr_metadata.py diff --git a/.github/workflows/update-inputdata-manifest.yml b/.github/workflows/update-inputdata-manifest.yml new file mode 100644 index 0000000..9019b9e --- /dev/null +++ b/.github/workflows/update-inputdata-manifest.yml @@ -0,0 +1,113 @@ +name: update-inputdata-manifest + +on: + schedule: + - cron: "0 12 * * 1" + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +concurrency: + group: update-inputdata-manifest + cancel-in-progress: false + +jobs: + regenerate: + runs-on: ubuntu-22.04 + timeout-minutes: 240 + container: + image: ghcr.io/e3sm-project/containers-ghci:main + steps: + - name: Checkout containers + uses: actions/checkout@v4 + with: + submodules: recursive + show-progress: false + persist-credentials: false + + - name: Checkout E3SM + uses: actions/checkout@v4 + with: + repository: E3SM-Project/E3SM + path: E3SM + ref: master + submodules: recursive + show-progress: false + persist-credentials: false + + - name: Regenerate inputdata manifests + env: + NETCDF_C_ROOT: /usr + NETCDF_FORTRAN_ROOT: /usr + PARALLEL_NETCDF_ROOT: /usr + run: | + set -euo pipefail + mkdir -p /tmp/e3sm-inputdata-cases /tmp/e3sm-inputdata-output + python3 -m tools.inputdata.generate_inputdata_manifest \ + --e3sm-root E3SM \ + --test-root /tmp/e3sm-inputdata-cases \ + --output-root /tmp/e3sm-inputdata-output \ + --state-out inputdata/e3sm-workflow-state.json \ + --files-out inputdata/files.txt \ + --standalone-files-out inputdata/files-standalone.txt \ + --missing-files-out inputdata/files-missing.txt \ + --provenance-out inputdata/provenance.json \ + --verbose + + - name: Detect manifest changes + id: detect + run: | + set -euo pipefail + files=( + inputdata/files.txt + inputdata/files-standalone.txt + inputdata/files-missing.txt + inputdata/e3sm-workflow-state.json + inputdata/provenance.json + ) + + if git diff --quiet -- "${files[@]}"; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Build PR metadata + if: steps.detect.outputs.changed == 'true' + id: metadata + run: | + set -euo pipefail + + stamp="$(date -u +%Y%m%d)" + branch="bot/update-inputdata-manifest-${stamp}-${GITHUB_RUN_ID}" + echo "branch=${branch}" >> "$GITHUB_OUTPUT" + + summary_path="/tmp/inputdata-pr-summary.md" + + python3 -m tools.inputdata.build_pr_metadata \ + --output-path "${summary_path}" \ + --state-path inputdata/e3sm-workflow-state.json + + echo "body_path=${summary_path}" >> "$GITHUB_OUTPUT" + + - name: Create pull request + if: steps.detect.outputs.changed == 'true' + uses: peter-evans/create-pull-request@v6 + with: + branch: ${{ steps.metadata.outputs.branch }} + delete-branch: true + base: main + title: "Update E3SM inputdata manifest" + commit-message: "Update inputdata manifest from E3SM workflow tests" + body-path: ${{ steps.metadata.outputs.body_path }} + add-paths: | + inputdata/files.txt + inputdata/files-standalone.txt + inputdata/files-missing.txt + inputdata/e3sm-workflow-state.json + inputdata/provenance.json + labels: | + automated + inputdata \ No newline at end of file diff --git a/tools/inputdata/README.md b/tools/inputdata/README.md index 1f370e9..13d4d48 100644 --- a/tools/inputdata/README.md +++ b/tools/inputdata/README.md @@ -49,6 +49,7 @@ The generator needs an environment with an E3SM checkout and a working CIME `cre ## Related modules - [generate_inputdata_manifest.py](generate_inputdata_manifest.py) +- [build_pr_metadata.py](build_pr_metadata.py) - [lib/workflow_parser.py](lib/workflow_parser.py) - [lib/cime_manifest.py](lib/cime_manifest.py) - [lib/url_normalize.py](lib/url_normalize.py) diff --git a/tools/inputdata/build_pr_metadata.py b/tools/inputdata/build_pr_metadata.py new file mode 100644 index 0000000..bc8a14b --- /dev/null +++ b/tools/inputdata/build_pr_metadata.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import subprocess +from pathlib import Path + +MANAGED_FILES = [ + "inputdata/files.txt", + "inputdata/files-standalone.txt", + "inputdata/files-missing.txt", + "inputdata/e3sm-workflow-state.json", + "inputdata/provenance.json", +] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Build PR body content for inputdata manifest updates" + ) + parser.add_argument( + "--output-path", + required=True, + help="Path where the markdown PR body should be written.", + ) + parser.add_argument( + "--state-path", + required=True, + help="Path to inputdata/e3sm-workflow-state.json.", + ) + return parser.parse_args() + + +def _git_numstat(managed_files: list[str]) -> str: + output = subprocess.check_output( + ["git", "diff", "--numstat", "--", *managed_files], + text=True, + ) + return output.strip() + + +def _render_body(state: dict, numstat: str) -> str: + tests = state.get("extracted_tests", []) + e3sm_sha = state.get("e3sm_sha", "unknown") + + lines: list[str] = [] + lines.append("Automated inputdata manifest refresh from E3SM workflows.") + lines.append("") + lines.append("Source snapshot") + lines.append(f"- E3SM SHA: {e3sm_sha}") + lines.append(f"- Extracted tests: {len(tests)}") + lines.append("") + lines.append("Changed files") + if numstat: + for row in numstat.splitlines(): + added, removed, path = row.split("\t", maxsplit=2) + lines.append(f"- {path}: +{added} / -{removed} lines") + else: + lines.append("- No managed file changes detected") + lines.append("") + lines.append("Files in scope") + for path in MANAGED_FILES: + lines.append(f"- {path}") + lines.append("") + lines.append("Review guidance") + lines.append("- Please review the provenance and missing-file updates before merge.") + lines.append( + "- Merging this PR will trigger normal inputdata image build/publish workflows." + ) + + return "\n".join(lines) + "\n" + + +def main() -> int: + args = parse_args() + output_path = Path(args.output_path) + state_path = Path(args.state_path) + + with state_path.open("r", encoding="utf-8") as file: + state = json.load(file) + + numstat = _git_numstat(MANAGED_FILES) + body = _render_body(state, numstat) + + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(body, encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From f11f0ae62c0de4c5f84115c9fa2f51bf769272c7 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Thu, 9 Jul 2026 09:27:09 -0600 Subject: [PATCH 12/13] Add detailed root README.md --- README.md | 125 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ readme | 1 - 2 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 README.md delete mode 100644 readme diff --git a/README.md b/README.md new file mode 100644 index 0000000..7d576f1 --- /dev/null +++ b/README.md @@ -0,0 +1,125 @@ +# E3SM Containers Repository + +This repository contains Docker build contexts and automation for container +images used by E3SM CI and related workflows. + +## Repository Layout + +- `ghci/`: Base GHCI CI container build context. +- `standalone-ghci/`: Standalone GHCI container build context. +- `inputdata/`: Inputdata container build context and manifest files. +- `e3sm-diags-test-data/`: E3SM diags test-data image build context. +- `tools/inputdata/`: Inputdata manifest generation and support utilities. +- `.github/workflows/`: GitHub Actions workflows for build/publish and + manifest automation. + +## Workflow Summary + +All workflows live in `.github/workflows`. + +### `ghci.yaml` + +- Purpose: Build the `ghcr.io//-ghci` image. +- Triggers: + - `pull_request` to `main` when `ghci/**` or workflow file changes. + - `push` to `main` on same paths. + - tags matching `ghci-*`. +- Behavior: + - Always builds on PRs and pushes. + - Pushes image only when event is not `pull_request`. + +### `standalone-ghci.yaml` + +- Purpose: Build the `ghcr.io//-standalone-ghci` image. +- Triggers: + - `pull_request` to `main` when `standalone-ghci/**` or workflow file + changes. + - `push` to `main` on same paths. + - tags matching `standalone-ghci-*`. +- Behavior: + - Uses `ubuntu-22.04-arm` runner matrix. + - Pushes image only when event is not `pull_request`. + +### `e3sm-diags-test-data.yaml` + +- Purpose: Build the `ghcr.io//-e3sm-diags-test-data` image. +- Triggers: + - `merge_group` to `main`. + - `pull_request` to `main` when `e3sm-diags-test-data/**` or workflow file + changes. + - `push` to `main` on same paths. + - tags matching `e3sm-diags-test-data-*`. +- Behavior: + - Pushes image only when event is not `pull_request`. + +### `inputdata.yaml` + +- Purpose: Build inputdata images from manifest files in `inputdata/`. +- Triggers: + - `pull_request` to `main` when `inputdata/**` or workflow file changes. + - `push` to `main` on same paths. + - tags matching `inputdata-*`. +- Behavior: + - Matrix builds two variants: + - `files` from `inputdata/files.txt`. + - `files-standalone` from `inputdata/files-standalone.txt`. + - Pushes images only when event is not `pull_request`. + +### `update-inputdata-manifest.yml` + +- Purpose: Regenerate inputdata manifests from E3SM workflows and open a PR + when generated outputs change. +- Triggers: + - Weekly schedule (`cron: 0 12 * * 1`). + - Manual `workflow_dispatch`. +- Behavior: + - Checks out this repository and `E3SM-Project/E3SM` at `master`. + - Runs `python3 -m tools.inputdata.generate_inputdata_manifest`. + - Compares managed files: + - `inputdata/files.txt` + - `inputdata/files-standalone.txt` + - `inputdata/files-missing.txt` + - `inputdata/e3sm-workflow-state.json` + - `inputdata/provenance.json` + - If no changes: exits without opening a PR. + - If changes exist: uses `peter-evans/create-pull-request` to open a real + PR with generated metadata. + +## How Workflows Fit Together + +1. `update-inputdata-manifest.yml` is the source-of-truth updater for + inputdata manifest content. +2. When it opens a PR that modifies files under `inputdata/`, + `inputdata.yaml` runs automatically on that PR and validates container builds + without pushing images. +3. After the PR is reviewed and merged to `main`, `inputdata.yaml` runs again + on the `push` event and publishes updated inputdata images to GHCR. +4. `ghci.yaml`, `standalone-ghci.yaml`, and `e3sm-diags-test-data.yaml` are + independent image pipelines and do not depend on inputdata-manifest updates. + +In short: one workflow updates inputdata manifests, a different workflow builds +and publishes inputdata images from those manifests. + +## Common Trigger Pattern + +Most image workflows follow the same model: + +- Build on PRs for validation. +- Publish on `push` to `main` or on release-style tags. +- Scope execution with `paths` filters to avoid unrelated builds. + +## For New Contributors + +If you are updating manifest generation logic: + +1. Change code under `tools/inputdata/`. +2. Run the generator locally if needed. +3. Use `workflow_dispatch` on `update-inputdata-manifest.yml` to test the + automation path. + +If you are updating container images directly: + +1. Modify the corresponding directory (`ghci/`, `standalone-ghci/`, + `inputdata/`, or `e3sm-diags-test-data/`). +2. Open a PR and confirm the matching workflow build succeeds. +3. Merge to `main` to publish new images. \ No newline at end of file diff --git a/readme b/readme deleted file mode 100644 index bc2e652..0000000 --- a/readme +++ /dev/null @@ -1 +0,0 @@ -Container recipes for the E3SM project From b2d8fba86bd96a8cf0a52698720743e6ef7ee773 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Thu, 9 Jul 2026 09:41:51 -0600 Subject: [PATCH 13/13] Only trigger PR when file list changes, not file metadata --- .../workflows/update-inputdata-manifest.yml | 2 -- README.md | 3 ++- tools/inputdata/build_pr_metadata.py | 21 +++++++++++++------ 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/update-inputdata-manifest.yml b/.github/workflows/update-inputdata-manifest.yml index 9019b9e..46a147c 100644 --- a/.github/workflows/update-inputdata-manifest.yml +++ b/.github/workflows/update-inputdata-manifest.yml @@ -64,8 +64,6 @@ jobs: inputdata/files.txt inputdata/files-standalone.txt inputdata/files-missing.txt - inputdata/e3sm-workflow-state.json - inputdata/provenance.json ) if git diff --quiet -- "${files[@]}"; then diff --git a/README.md b/README.md index 7d576f1..d657dcc 100644 --- a/README.md +++ b/README.md @@ -75,10 +75,11 @@ All workflows live in `.github/workflows`. - Behavior: - Checks out this repository and `E3SM-Project/E3SM` at `master`. - Runs `python3 -m tools.inputdata.generate_inputdata_manifest`. - - Compares managed files: + - Opens a PR only when these payload-defining files change: - `inputdata/files.txt` - `inputdata/files-standalone.txt` - `inputdata/files-missing.txt` + - If a PR is opened, it also includes related metadata files: - `inputdata/e3sm-workflow-state.json` - `inputdata/provenance.json` - If no changes: exits without opening a PR. diff --git a/tools/inputdata/build_pr_metadata.py b/tools/inputdata/build_pr_metadata.py index bc8a14b..010d9d5 100644 --- a/tools/inputdata/build_pr_metadata.py +++ b/tools/inputdata/build_pr_metadata.py @@ -6,14 +6,19 @@ import subprocess from pathlib import Path -MANAGED_FILES = [ +TRIGGER_FILES = [ "inputdata/files.txt", "inputdata/files-standalone.txt", "inputdata/files-missing.txt", +] + +ADDITIONAL_PR_FILES = [ "inputdata/e3sm-workflow-state.json", "inputdata/provenance.json", ] +PR_FILES = [*TRIGGER_FILES, *ADDITIONAL_PR_FILES] + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( @@ -32,9 +37,9 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def _git_numstat(managed_files: list[str]) -> str: +def _git_numstat(paths: list[str]) -> str: output = subprocess.check_output( - ["git", "diff", "--numstat", "--", *managed_files], + ["git", "diff", "--numstat", "--", *paths], text=True, ) return output.strip() @@ -59,8 +64,12 @@ def _render_body(state: dict, numstat: str) -> str: else: lines.append("- No managed file changes detected") lines.append("") - lines.append("Files in scope") - for path in MANAGED_FILES: + lines.append("PR trigger files") + for path in TRIGGER_FILES: + lines.append(f"- {path}") + lines.append("") + lines.append("Additional files included when a PR is opened") + for path in ADDITIONAL_PR_FILES: lines.append(f"- {path}") lines.append("") lines.append("Review guidance") @@ -80,7 +89,7 @@ def main() -> int: with state_path.open("r", encoding="utf-8") as file: state = json.load(file) - numstat = _git_numstat(MANAGED_FILES) + numstat = _git_numstat(PR_FILES) body = _render_body(state, numstat) output_path.parent.mkdir(parents=True, exist_ok=True)