From 6a79d1a4b7a5268b9dd9a12fb1fc463eeaa333de Mon Sep 17 00:00:00 2001 From: crhysc Date: Thu, 2 Oct 2025 16:24:41 -0400 Subject: [PATCH 1/6] add invalid structures error handling --- atomgpt/inverse_models/inverse_models.py | 117 +++++++++++++++++------ 1 file changed, 86 insertions(+), 31 deletions(-) diff --git a/atomgpt/inverse_models/inverse_models.py b/atomgpt/inverse_models/inverse_models.py index 1b7ff27..d2ba9fd 100644 --- a/atomgpt/inverse_models/inverse_models.py +++ b/atomgpt/inverse_models/inverse_models.py @@ -27,6 +27,7 @@ from jarvis.io.vasp.inputs import Poscar import csv import os +import numpy as np from pydantic_settings import BaseSettings import sys import json @@ -205,42 +206,96 @@ def load_model(path="", config=None): FastLanguageModel.for_inference(model) return model, tokenizer, config +def _validate_atoms(atoms): + if atoms is None: + return False, "atoms_is_none" + try: + lat = np.asarray(getattr(atoms, "lattice_mat", None), dtype=float) + if lat.shape != (3, 3): + return False, f"bad_lattice_shape:{getattr(atoms,'lattice_mat',None)}" + if not np.isfinite(lat).all(): + return False, "nonfinite_lattice" + n = getattr(atoms, "num_atoms", None) + if n is None or n <= 0: + return False, f"num_atoms_invalid:{n}" + _ = Poscar(atoms).to_string() + return True, "" + except Exception as e: + return False, f"poscar_fail:{type(e).__name__}:{e}" + +def _poscar_one_line(at): + return Poscar(at).to_string().replace("\n", "\\n") + +def _misses_path(csv_out, config): + fname = getattr(config, "miss_csv", None) + if fname is None or not str(fname).strip(): + root, ext = os.path.splitext(csv_out) + fname = root + ".misses.csv" + os.makedirs(os.path.dirname(os.path.abspath(fname)), exist_ok=True) + return fname def evaluate( - test_set=[], model="", tokenizer="", csv_out="out.csv", config="" + test_set=[], + model="", + tokenizer="", + csv_out="out.csv", + config="", ): print("Testing\n", len(test_set)) - f = open(csv_out, "w") - f.write("id,target,prediction\n") + os.makedirs(os.path.dirname(os.path.abspath(csv_out)), exist_ok=True) + miss_csv_out = _misses_path(csv_out, config) + + with open(csv_out, "w", newline="") as f_ok, open(miss_csv_out, "w", newline="") as f_miss: + ok_writer = csv.writer(f_ok) + miss_writer = csv.writer(f_miss) + ok_writer.writerow(["id", "target", "prediction"]) + miss_writer.writerow(["id", "stage", "error", "detail", "raw_text_preview"]) + + for i in tqdm(test_set, total=len(test_set)): + sample_id = i.get("id", "") + target_mat = None + target_err = None + try: + target_mat = text2atoms("\n" + i["output"]) + ok, detail = _validate_atoms(target_mat) + if not ok: + target_err = detail + except Exception as e: + target_err = f"text2atoms:{type(e).__name__}:{e}" + + if target_err: + miss_writer.writerow([sample_id, "target", "invalid_target", target_err, (i.get("output","")[:240])]) + continue + + gen_mat = None + gen_err = None + try: + gen_mat = gen_atoms( + prompt=i["input"], + tokenizer=tokenizer, + model=model, + alpaca_prompt=config.alpaca_prompt, + instruction=config.instruction, + ) + ok, detail = _validate_atoms(gen_mat) + if not ok: + gen_err = detail + except Exception as e: + gen_err = f"gen_atoms:{type(e).__name__}:{e}" + + if gen_err: + miss_writer.writerow([sample_id, "prediction", "invalid_prediction", gen_err, ""]) + continue + + try: + ok_writer.writerow([ + sample_id, + _poscar_one_line(target_mat), + _poscar_one_line(gen_mat), + ]) + except Exception as e: + miss_writer.writerow([sample_id, "write", "write_failed", f"{type(e).__name__}:{e}", ""]) - for i in tqdm(test_set, total=len(test_set)): - # try: - # prompt = i["input"] - # print("prompt", prompt) - gen_mat = gen_atoms( - prompt=i["input"], - tokenizer=tokenizer, - model=model, - alpaca_prompt=config.alpaca_prompt, - instruction=config.instruction, - ) - target_mat = text2atoms("\n" + i["output"]) - print("target_mat", target_mat) - print("genmat", gen_mat) - line = ( - i["id"] - + "," - + Poscar(target_mat).to_string().replace("\n", "\\n") - + "," - + Poscar(gen_mat).to_string().replace("\n", "\\n") - + "\n" - ) - f.write(line) - # print() - # except Exception as exp: - # print("Error", exp) - # pass - f.close() def batch_evaluate( From 92f2596d08aac049103bfc2577b574423a44ad81 Mon Sep 17 00:00:00 2001 From: "C. Rhys Campbell" <149001340+crhysc@users.noreply.github.com> Date: Tue, 18 Nov 2025 10:40:57 -0500 Subject: [PATCH 2/6] add error handling for inverse_predict.py --- atomgpt/inverse_models/inverse_predict.py | 111 ++++++++++++++-------- 1 file changed, 71 insertions(+), 40 deletions(-) diff --git a/atomgpt/inverse_models/inverse_predict.py b/atomgpt/inverse_models/inverse_predict.py index 378ca35..003cc08 100644 --- a/atomgpt/inverse_models/inverse_predict.py +++ b/atomgpt/inverse_models/inverse_predict.py @@ -111,12 +111,9 @@ def predict( prop_val=None, dtype=None, max_seq_length=1058, - load_in_4bit=None, # temp_config["load_in_4bit"] - verbose=True, # temp_config["load_in_4bit"] + load_in_4bit=None, + verbose=True, ): - # if not os.path.exists("config_name"): - - # config_name=os.path.join(output_dir,"config.json") print("config_path", config_path) if output_dir is not None: config_name = os.path.join(output_dir, "config.json") @@ -125,7 +122,7 @@ def predict( config_name = os.path.join(parent, "config.json") adapter = os.path.join(output_dir, "adapter_config.json") if os.path.exists(adapter): - model_name = output_dir # temp_config["model_name"] + model_name = output_dir if config_path is not None: config_name = config_path if verbose: @@ -142,7 +139,6 @@ def predict( pprint.pprint(temp_config) if model_name is None: model_name = temp_config["model_name"] - # output_dir = temp_config["output_dir"] if load_in_4bit is None: load_in_4bit = temp_config["load_in_4bit"] @@ -150,6 +146,7 @@ def predict( print("Model used:", model_name) print("config used:", config_path) print("formula:", formula) + model = None tokenizer = None try: @@ -161,29 +158,28 @@ def predict( device_map="auto", ) FastLanguageModel.for_inference(model) - except: + except Exception: tokenizer = AutoTokenizer.from_pretrained( model_name, gguf_file=filename ) model = AutoModelForCausalLM.from_pretrained( model_name, gguf_file=filename ) - pass + atoms_arr = [] lines = [] if formula is None: - # if dat_path is None: - f = open(pred_csv, "r") - lines = f.read().splitlines() - f.close() + with open(pred_csv, "r") as f: + lines = f.read().splitlines() else: if dat_path is not None: lines = [dat_path] - lines = [formula] + else: + lines = [formula] mem = [] - for i in lines: + for idx, i in enumerate(lines): prompt = i if ".dat" in i or dat_path is not None: if dat_path is None: @@ -198,21 +194,18 @@ def predict( formula=formula, background_subs=background_subs, ) - # y[y < 0.1] = 0 - y_new_str = y # "\n".join(["{0:.2f}".format(x) for x in y]) + y_new_str = y try: if ".dat" in i: formula = str(_formula.split("/")[-1].split(".dat")[0]) except Exception: pass - # gen_mat = main_spectra(spectra=[[y_new_str,y]],formulas=[formula],model=model,tokenizer=tokenizer,device='cuda')[0] prompt = ( "The chemical formula is " + formula + " The " + temp_config["prop"] + " is " - # + " The XRD is " + y_new_str + ". Generate atomic structure description with lattice lengths, angles, coordinates and atom types." ) @@ -224,34 +217,72 @@ def predict( + " The " + temp_config["prop"] + " is " - # + " The XRD is " + str(prop_val) + ". Generate atomic structure description with lattice lengths, angles, coordinates and atom types." ) if verbose: - print("prompt here", prompt.replace("\n", ",")) - gen_mat = gen_atoms( - prompt=prompt, - model=model, - tokenizer=tokenizer, - alpaca_prompt=temp_config["alpaca_prompt"], - instruction=temp_config["instruction"], - device=device, - ) - if verbose: - print("gen atoms", gen_mat) - print("gen atoms spacegroup", gen_mat.spacegroup()) - print("intvl", intvl) - if relax: - gen_mat = relax_atoms(atoms=gen_mat) + print(f"[{idx}] prompt:", prompt.replace("\n", ",")) + + info = {"prompt": prompt} + gen_mat = None + + # --- NEW: robust error handling around generation / structure use --- + try: + gen_mat = gen_atoms( + prompt=prompt, + model=model, + tokenizer=tokenizer, + alpaca_prompt=temp_config["alpaca_prompt"], + instruction=temp_config["instruction"], + device=device, + ) + if verbose: - print("gen atoms relax", gen_mat, gen_mat.spacegroup()) - atoms_arr.append(gen_mat.to_dict()) - info = {} - info["prompt"] = prompt - info["atoms"] = gen_mat.to_dict() + print(f"[{idx}] gen atoms:", gen_mat) + # spacegroup() can fail for broken structures, so guard it + try: + print(f"[{idx}] gen atoms spacegroup:", gen_mat.spacegroup()) + except Exception as e_sg: + print( + f"[WARN] Failed to compute spacegroup for sample {idx}: {e_sg}" + ) + + if relax: + try: + gen_mat = relax_atoms(atoms=gen_mat) + if verbose: + print( + f"[{idx}] gen atoms relax:", + gen_mat, + gen_mat.spacegroup(), + ) + except Exception as e_relax: + print( + f"[WARN] Relaxation failed for sample {idx}, " + "continuing with unrelaxed structure." + ) + print(traceback.format_exc()) + + # this is another common crash point if gen_mat is invalid + atoms_dict = gen_mat.to_dict() + atoms_arr.append(atoms_dict) + info["atoms"] = atoms_dict + + except Exception as e: + print( + f"[ERROR] Failed to generate a valid structure for sample {idx} " + f"(input: {i}): {e}" + ) + # optional: print full traceback for debugging + print(traceback.format_exc()) + info["error"] = str(e) + # do NOT re-raise; just skip this structure and move on + mem.append(info) + continue + mem.append(info) + dumpjson(data=mem, filename=fname) return model, tokenizer, temp_config From 7fe1e1ea803e7917aeb8d174c66c924a260533fc Mon Sep 17 00:00:00 2001 From: Charles Campbell Date: Wed, 5 Nov 2025 12:50:42 -0500 Subject: [PATCH 3/6] num_proc --- atomgpt/inverse_models/inverse_models.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/atomgpt/inverse_models/inverse_models.py b/atomgpt/inverse_models/inverse_models.py index 9d8fb78..8388265 100644 --- a/atomgpt/inverse_models/inverse_models.py +++ b/atomgpt/inverse_models/inverse_models.py @@ -569,10 +569,12 @@ def tokenize_function(example): train_dataset = train_dataset.map( formatting_prompts_func_with_prompt, batched=True, + num_proc=config.dataset_num_proc ) eval_dataset = eval_dataset.map( formatting_prompts_func_with_prompt, batched=True, + num_proc=config.dataset_num_proc ) # Compute the actual max sequence length in raw text lengths = [ @@ -582,8 +584,8 @@ def tokenize_function(example): max_seq_length = max(lengths) print(f"🧠 Suggested max_seq_length based on dataset: {max_seq_length}") - tokenized_train = train_dataset.map(tokenize_function, batched=True) - tokenized_eval = eval_dataset.map(tokenize_function, batched=True) + tokenized_train = train_dataset.map(tokenize_function, batched=True, num_proc=config.dataset_num_proc) + tokenized_eval = eval_dataset.map(tokenize_function, batched=True, num_proc=config.dataset_num_proc) tokenized_train.set_format( type="torch", columns=["input_ids", "attention_mask", "output"] ) From 030e794bdfde6f33306e460ac8198b9871c83ab0 Mon Sep 17 00:00:00 2001 From: "C. Rhys Campbell" <149001340+crhysc@users.noreply.github.com> Date: Tue, 18 Nov 2025 12:36:10 -0500 Subject: [PATCH 4/6] if gen_mat = None: ... --- atomgpt/inverse_models/inverse_predict.py | 108 ++++++++-------------- 1 file changed, 39 insertions(+), 69 deletions(-) diff --git a/atomgpt/inverse_models/inverse_predict.py b/atomgpt/inverse_models/inverse_predict.py index 003cc08..9e36a57 100644 --- a/atomgpt/inverse_models/inverse_predict.py +++ b/atomgpt/inverse_models/inverse_predict.py @@ -79,13 +79,10 @@ def relax_atoms( calculator = AlignnAtomwiseCalculator(path=default_path(), device="cpu") t1 = time.time() - # if calculator is None: - # return atoms ase_atoms = atoms.ase_converter() ase_atoms.calc = calculator ase_atoms = ExpCellFilter(ase_atoms, constant_volume=constant_volume) - # TODO: Make it work with any other optimizer dyn = FIRE(ase_atoms) dyn.run(fmax=fmax, steps=nsteps) en = ase_atoms.atoms.get_potential_energy() @@ -111,8 +108,8 @@ def predict( prop_val=None, dtype=None, max_seq_length=1058, - load_in_4bit=None, - verbose=True, + load_in_4bit=None, # temp_config["load_in_4bit"] + verbose=True, # temp_config["load_in_4bit"] ): print("config_path", config_path) if output_dir is not None: @@ -139,6 +136,7 @@ def predict( pprint.pprint(temp_config) if model_name is None: model_name = temp_config["model_name"] + # output_dir = temp_config["output_dir"] if load_in_4bit is None: load_in_4bit = temp_config["load_in_4bit"] @@ -146,7 +144,6 @@ def predict( print("Model used:", model_name) print("config used:", config_path) print("formula:", formula) - model = None tokenizer = None try: @@ -158,28 +155,29 @@ def predict( device_map="auto", ) FastLanguageModel.for_inference(model) - except Exception: + except: tokenizer = AutoTokenizer.from_pretrained( model_name, gguf_file=filename ) model = AutoModelForCausalLM.from_pretrained( model_name, gguf_file=filename ) - + pass atoms_arr = [] lines = [] if formula is None: - with open(pred_csv, "r") as f: - lines = f.read().splitlines() + # if dat_path is None: + f = open(pred_csv, "r") + lines = f.read().splitlines() + f.close() else: if dat_path is not None: lines = [dat_path] - else: - lines = [formula] + lines = [formula] mem = [] - for idx, i in enumerate(lines): + for i in lines: prompt = i if ".dat" in i or dat_path is not None: if dat_path is None: @@ -222,74 +220,47 @@ def predict( ) if verbose: - print(f"[{idx}] prompt:", prompt.replace("\n", ",")) - - info = {"prompt": prompt} - gen_mat = None + print("prompt here", prompt.replace("\n", ",")) - # --- NEW: robust error handling around generation / structure use --- - try: - gen_mat = gen_atoms( - prompt=prompt, - model=model, - tokenizer=tokenizer, - alpaca_prompt=temp_config["alpaca_prompt"], - instruction=temp_config["instruction"], - device=device, - ) - - if verbose: - print(f"[{idx}] gen atoms:", gen_mat) - # spacegroup() can fail for broken structures, so guard it - try: - print(f"[{idx}] gen atoms spacegroup:", gen_mat.spacegroup()) - except Exception as e_sg: - print( - f"[WARN] Failed to compute spacegroup for sample {idx}: {e_sg}" - ) - - if relax: - try: - gen_mat = relax_atoms(atoms=gen_mat) - if verbose: - print( - f"[{idx}] gen atoms relax:", - gen_mat, - gen_mat.spacegroup(), - ) - except Exception as e_relax: - print( - f"[WARN] Relaxation failed for sample {idx}, " - "continuing with unrelaxed structure." - ) - print(traceback.format_exc()) - - # this is another common crash point if gen_mat is invalid - atoms_dict = gen_mat.to_dict() - atoms_arr.append(atoms_dict) - info["atoms"] = atoms_dict + gen_mat = gen_atoms( + prompt=prompt, + model=model, + tokenizer=tokenizer, + alpaca_prompt=temp_config["alpaca_prompt"], + instruction=temp_config["instruction"], + device=device, + ) - except Exception as e: + if gen_mat is None: print( - f"[ERROR] Failed to generate a valid structure for sample {idx} " - f"(input: {i}): {e}" + "The returned structure is invalid. Here is the output:", + gen_mat, ) - # optional: print full traceback for debugging - print(traceback.format_exc()) - info["error"] = str(e) - # do NOT re-raise; just skip this structure and move on + info = {} + info["prompt"] = prompt + info["error"] = "Invalid structure returned by AtomGPT (None)." mem.append(info) + # skip the rest of the loop for this entry continue + if verbose: + print("gen atoms", gen_mat) + print("gen atoms spacegroup", gen_mat.spacegroup()) + print("intvl", intvl) + if relax: + gen_mat = relax_atoms(atoms=gen_mat) + if verbose: + print("gen atoms relax", gen_mat, gen_mat.spacegroup()) + atoms_arr.append(gen_mat.to_dict()) + info = {} + info["prompt"] = prompt + info["atoms"] = gen_mat.to_dict() mem.append(info) - dumpjson(data=mem, filename=fname) return model, tokenizer, temp_config if __name__ == "__main__": - # output_dir = make_id_prop() - # output_dir="." args = parser.parse_args(sys.argv[1:]) print("args.config_path", args.config_path) predict( @@ -302,5 +273,4 @@ def predict( config_path=args.config_path, prop_val=args.prop_val, background_subs=args.background_subs, - # config_name=args.config_name, ) From b53bb3856e83f0147777f2ae81083c50e0f91dae Mon Sep 17 00:00:00 2001 From: "C. Rhys Campbell" <149001340+crhysc@users.noreply.github.com> Date: Tue, 18 Nov 2025 12:42:01 -0500 Subject: [PATCH 5/6] rm "Here is the output" --- atomgpt/inverse_models/inverse_predict.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/atomgpt/inverse_models/inverse_predict.py b/atomgpt/inverse_models/inverse_predict.py index 9e36a57..109e3f9 100644 --- a/atomgpt/inverse_models/inverse_predict.py +++ b/atomgpt/inverse_models/inverse_predict.py @@ -233,8 +233,7 @@ def predict( if gen_mat is None: print( - "The returned structure is invalid. Here is the output:", - gen_mat, + "The structure returned by gen_mat() is not a valid crystal structure. ) info = {} info["prompt"] = prompt From ac7e0019678c56a94e0bec8e0c3bddce7ef4b3c3 Mon Sep 17 00:00:00 2001 From: "C. Rhys Campbell" <149001340+crhysc@users.noreply.github.com> Date: Tue, 18 Nov 2025 12:43:10 -0500 Subject: [PATCH 6/6] terminate string literal --- atomgpt/inverse_models/inverse_predict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atomgpt/inverse_models/inverse_predict.py b/atomgpt/inverse_models/inverse_predict.py index 109e3f9..63054cf 100644 --- a/atomgpt/inverse_models/inverse_predict.py +++ b/atomgpt/inverse_models/inverse_predict.py @@ -233,7 +233,7 @@ def predict( if gen_mat is None: print( - "The structure returned by gen_mat() is not a valid crystal structure. + "The structure returned by gen_mat() is not a valid crystal structure." ) info = {} info["prompt"] = prompt